From 37d8d94004e6861607502814e71db07127e90b20 Mon Sep 17 00:00:00 2001 From: Kris <2893855659@qq.com> Date: Tue, 15 Jul 2025 11:21:08 +0800 Subject: [PATCH] =?UTF-8?q?=E3=80=90feat=E3=80=91=E5=AF=B9=E8=B1=A1?= =?UTF-8?q?=E5=A4=9A=E6=80=81=E5=AD=97=E6=AE=B5=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/celnet/datadump/entity/DataBatch.java | 4 +- .../celnet/datadump/entity/DataObject.java | 7 + .../celnet/datadump/entity/LinkConfig.java | 75 +++++++ .../celnet/datadump/job/DataDumpNewJob.java | 41 ++++ .../celnet/datadump/mapper/CustomMapper.java | 6 + .../datadump/mapper/LinkConfigMapper.java | 16 ++ .../datadump/service/CommonService.java | 17 ++ .../datadump/service/LinkConfigService.java | 10 + .../service/impl/CommonServiceImpl.java | 192 +++++++++--------- .../impl/DataImportBatchServiceImpl.java | 71 +++++-- .../impl/DataImportNewServiceImpl.java | 46 +++-- .../service/impl/DataImportServiceImpl.java | 191 +++++++++-------- .../service/impl/LinkConfigServiceImpl.java | 16 ++ .../com/celnet/datadump/util/DataUtil.java | 3 +- src/main/resources/mapper/CustomMapper.xml | 4 + 15 files changed, 479 insertions(+), 220 deletions(-) create mode 100644 src/main/java/com/celnet/datadump/entity/LinkConfig.java create mode 100644 src/main/java/com/celnet/datadump/mapper/LinkConfigMapper.java create mode 100644 src/main/java/com/celnet/datadump/service/LinkConfigService.java create mode 100644 src/main/java/com/celnet/datadump/service/impl/LinkConfigServiceImpl.java diff --git a/src/main/java/com/celnet/datadump/entity/DataBatch.java b/src/main/java/com/celnet/datadump/entity/DataBatch.java index d3b7764..5851234 100644 --- a/src/main/java/com/celnet/datadump/entity/DataBatch.java +++ b/src/main/java/com/celnet/datadump/entity/DataBatch.java @@ -127,7 +127,7 @@ public class DataBatch implements Serializable, Cloneable { @TableField(value = "created_date", fill = FieldFill.INSERT) @ExcelIgnore @ApiModelProperty(value = "创建时间") - private LocalDateTime createdDate; + private Date createdDate; /** * 最后更新时间 @@ -135,7 +135,7 @@ public class DataBatch implements Serializable, Cloneable { @TableField(value = "last_modified_date", fill = FieldFill.INSERT_UPDATE) @ExcelIgnore @ApiModelProperty(value = "最后更新时间") - private LocalDateTime lastModifiedDate; + private Date lastModifiedDate; /** * sf新增数据量 diff --git a/src/main/java/com/celnet/datadump/entity/DataObject.java b/src/main/java/com/celnet/datadump/entity/DataObject.java index b806059..e028c47 100644 --- a/src/main/java/com/celnet/datadump/entity/DataObject.java +++ b/src/main/java/com/celnet/datadump/entity/DataObject.java @@ -42,6 +42,13 @@ public class DataObject implements Serializable { @ApiModelProperty(value = "对象名称") private String label; + /** + * 对象名称 + */ + @TableField("keyPrefix") + @ApiModelProperty(value = "对象ID前缀") + private String keyPrefix; + /** * 是否启用 */ diff --git a/src/main/java/com/celnet/datadump/entity/LinkConfig.java b/src/main/java/com/celnet/datadump/entity/LinkConfig.java new file mode 100644 index 0000000..2b6e4c6 --- /dev/null +++ b/src/main/java/com/celnet/datadump/entity/LinkConfig.java @@ -0,0 +1,75 @@ +package com.celnet.datadump.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Getter; +import lombok.Setter; + +import java.io.Serializable; + +/** + * @author Kris + * @date 2025/07/10 + */ +@Getter +@Setter +@TableName("link_config") +@ApiModel(value = "映射信息表") +public class LinkConfig implements Serializable { + + private static final long serialVersionUID = 1L; + + /** + * id + */ + @TableId(value = "id", type = IdType.AUTO) + @ApiModelProperty(value = "id") + private Integer id; + + /** + * 对象api + */ + @TableField("api") + @ApiModelProperty(value = "对象api") + private String api; + + /** + * 对象标签 + */ + @TableField("label") + @ApiModelProperty(value = "对象标签") + private String label; + + /** + * 多态ID字段 + */ + @TableField("field") + @ApiModelProperty(value = "多态ID字段") + private String field; + + /** + * 映射字段 + */ + @TableField("link_field") + @ApiModelProperty(value = "映射字段") + private String linkField; + + /** + * 是否创建 + */ + @TableField("is_create") + @ApiModelProperty(value = "是否创建") + private Boolean isCreate; + + /** + * 是否映射 + */ + @TableField("is_link") + @ApiModelProperty(value = "是否映射") + private Boolean isLink; + +} diff --git a/src/main/java/com/celnet/datadump/job/DataDumpNewJob.java b/src/main/java/com/celnet/datadump/job/DataDumpNewJob.java index 6dc7724..3cb245b 100644 --- a/src/main/java/com/celnet/datadump/job/DataDumpNewJob.java +++ b/src/main/java/com/celnet/datadump/job/DataDumpNewJob.java @@ -147,4 +147,45 @@ public class DataDumpNewJob { } + /** + * 创建关联字段 + * @return result + */ + @XxlJob("createLinkTypeFieldJob") + public ReturnT createLinkTypeJob(String paramStr) throws Exception{ + log.info("createLinkTypeFieldJob execute start .................."); + + SalesforceParam param = new SalesforceParam(); + try { + if (StringUtils.isNotBlank(paramStr)) { + param = JSON.parseObject(paramStr, SalesforceParam.class); + } + } catch (Throwable throwable) { + return new ReturnT<>(500, "参数解析失败！"); + } + + return commonService.createLinkTypeField(param); + } + + /** + * 更新关联类型 + * @return result + */ + @XxlJob("updateLinkTypeJob") + public ReturnT updateLinkTypeJob(String paramStr) throws Exception{ + log.info("updateLinkTypeJob execute start .................."); + + SalesforceParam param = new SalesforceParam(); + try { + if (StringUtils.isNotBlank(paramStr)) { + param = JSON.parseObject(paramStr, SalesforceParam.class); + } + } catch (Throwable throwable) { + return new ReturnT<>(500, "参数解析失败！"); + } + + return commonService.updateLinkType(param); + } + + } diff --git a/src/main/java/com/celnet/datadump/mapper/CustomMapper.java b/src/main/java/com/celnet/datadump/mapper/CustomMapper.java index de93003..1a0f2a8 100644 --- a/src/main/java/com/celnet/datadump/mapper/CustomMapper.java +++ b/src/main/java/com/celnet/datadump/mapper/CustomMapper.java @@ -57,6 +57,12 @@ public interface CustomMapper { */ public void createTable(@Param("tableName") String tableName, @Param("tableComment") String tableComment, @Param("maps") List> maps, @Param("index") List> index); + /** + * 创建字段 + * @param tableName + */ + public int createField(@Param("tableName") String tableName, @Param("fieldName") String fieldName); + /** * 更新方法 * diff --git a/src/main/java/com/celnet/datadump/mapper/LinkConfigMapper.java b/src/main/java/com/celnet/datadump/mapper/LinkConfigMapper.java new file mode 100644 index 0000000..95de751 --- /dev/null +++ b/src/main/java/com/celnet/datadump/mapper/LinkConfigMapper.java @@ -0,0 +1,16 @@ +package com.celnet.datadump.mapper; + + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.celnet.datadump.entity.LinkConfig; +import org.apache.ibatis.annotations.Mapper; + +/** + *

+ * Mapper 接口 + *

+ */ +@Mapper +public interface LinkConfigMapper extends BaseMapper { + +} diff --git a/src/main/java/com/celnet/datadump/service/CommonService.java b/src/main/java/com/celnet/datadump/service/CommonService.java index da6c3f2..54706e4 100644 --- a/src/main/java/com/celnet/datadump/service/CommonService.java +++ b/src/main/java/com/celnet/datadump/service/CommonService.java @@ -57,4 +57,21 @@ public interface CommonService { String getDocumentId(PartnerConnection partnerConnection, String contentVersionId) throws Exception; ReturnT getChatter(SalesforceParam param) throws Exception; + + /** + * 创建关联字段 + * @param param 参数 + * @return ReturnT + * @throws Exception exception + */ + ReturnT createLinkTypeField(SalesforceParam param) throws Exception; + + /** + * 更新关联类型 + * @param param 参数 + * @return ReturnT + * @throws Exception exception + */ + ReturnT updateLinkType(SalesforceParam param) throws Exception; + } diff --git a/src/main/java/com/celnet/datadump/service/LinkConfigService.java b/src/main/java/com/celnet/datadump/service/LinkConfigService.java new file mode 100644 index 0000000..bb07137 --- /dev/null +++ b/src/main/java/com/celnet/datadump/service/LinkConfigService.java @@ -0,0 +1,10 @@ +package com.celnet.datadump.service; + +import com.baomidou.mybatisplus.extension.service.IService; +import com.celnet.datadump.entity.DataLog; +import com.celnet.datadump.entity.LinkConfig; + + +public interface LinkConfigService extends IService { + +} diff --git a/src/main/java/com/celnet/datadump/service/impl/CommonServiceImpl.java b/src/main/java/com/celnet/datadump/service/impl/CommonServiceImpl.java index 7cdd160..1e31f05 100644 --- a/src/main/java/com/celnet/datadump/service/impl/CommonServiceImpl.java +++ b/src/main/java/com/celnet/datadump/service/impl/CommonServiceImpl.java @@ -83,6 +83,8 @@ public class CommonServiceImpl implements CommonService { private SalesforceTargetConnect salesforceTargetConnect; @Autowired private MetaclassConfigService metaclassConfigService; + @Autowired + private LinkConfigService linkConfigService; @Override public ReturnT increment(SalesforceParam param) throws Exception { @@ -766,33 +768,7 @@ public class CommonServiceImpl implements CommonService { maps.add(paramMap); maps.add(paramMap2); } - // Task和Event - if ("Task".equals(api) || "Event".equals(api)){ - Map paramwhoMap = Maps.newHashMap(); - paramwhoMap.put("key", "WhoId_Type__c"); - paramwhoMap.put("value", jsonObject.get("Who_Type")); - maps.add(paramwhoMap); - Map paramwhatMap = Maps.newHashMap(); - paramwhatMap.put("key", "WhatId_Type__c"); - paramwhatMap.put("value", jsonObject.get("What_Type")); - maps.add(paramwhatMap); - } - if ("TaskRelation".equals(api) || "EventRelation".equals(api)){ - Map paramMap = Maps.newHashMap(); - paramMap.put("key", "Relation_Type__c"); - paramMap.put("value", jsonObject.get("Relation_Type")); - maps.add(paramMap); - } - - //附件关联表 插入更新时给关联对象赋值 -// if ("ContentDocumentLink".equals(api)) { -// String linkedEntity_Type = records[i].getChild("LinkedEntity").getChild("Type").getValue().toString(); -// Map paramMap = Maps.newHashMap(); -// paramMap.put("key", "linkedEntity_Type"); -// paramMap.put("value", linkedEntity_Type); -// maps.add(paramMap); -// } if (existsIds.contains(id)) { customMapper.updateById(api, maps, id); } else { @@ -880,7 +856,17 @@ public class CommonServiceImpl implements CommonService { if (field.getReferenceTo().length <= 3) { join = StringUtils.join(field.getReferenceTo(), ","); } else { - log.warn("referenceTo too long set null, api:{}, field:{}, reference to:{}", apiName, field.getName(), StringUtils.join(field.getReferenceTo(), ",")); + log.info("referenceTo too long set null, api:{}, field:{}, reference to:{}", apiName, field.getName(), StringUtils.join(field.getReferenceTo(), ",")); + } + if (field.getReferenceTo().length > 1){ + if (field.getName().contains("Id") && !"OwnerId".equals(field.getName())){ + LinkConfig linkConfig = new LinkConfig(); + linkConfig.setApi(apiName); + linkConfig.setLabel(label); + linkConfig.setField(field.getName()); + linkConfig.setLinkField(StringUtils.replace(field.getName(), "Id", "_Type")); + linkConfigService.save(linkConfig); + } } // picklist保存到picklist表 if ("picklist".equalsIgnoreCase(sfType)) { @@ -898,9 +884,6 @@ public class CommonServiceImpl implements CommonService { } } dataField.setReferenceTo(join); - if ("WhoId".equals(field.getName()) || "WhatId".equals(field.getName())){ - dataField.setReferenceTo(null); - } fieldList.add(dataField); fields.add(field.getName()); } @@ -958,70 +941,6 @@ public class CommonServiceImpl implements CommonService { map.put("name", "new_id"); list.add(map); - if ("Task".equals(apiName) || "Event".equals(apiName)){ - Map LinkedMap = Maps.newHashMap(); - LinkedMap.put("type", "varchar(18)"); - LinkedMap.put("comment", "whatId关联对象"); - LinkedMap.put("name", "WhatId_Type__c"); - list.add(LinkedMap); - Map LinkedMap1 = Maps.newHashMap(); - LinkedMap1.put("type", "varchar(18)"); - LinkedMap1.put("comment", "whoId关联对象"); - LinkedMap1.put("name", "WhoId_Type__c"); - list.add(LinkedMap1); - -// DataField dataField = new DataField(); -// dataField.setApi(apiName); -// dataField.setField("WhoId_Type__c"); -// dataField.setName("whoId关联对象"); -// fieldList.add(dataField); -// -// DataField dataField1 = new DataField(); -// dataField1.setApi(apiName); -// dataField1.setField("WhatId_Type__c"); -// dataField1.setName("whatId关联对象"); -// fieldList.add(dataField1); - } - - if ("TaskRelation".equals(apiName) || "EventRelation".equals(apiName)){ - Map LinkedMap = Maps.newHashMap(); - LinkedMap.put("type", "varchar(18)"); - LinkedMap.put("comment", "relationId关联对象"); - LinkedMap.put("name", "Relation_Type__c"); - list.add(LinkedMap); - -// DataField dataField = new DataField(); -// dataField.setApi(apiName); -// dataField.setField("Relation_Type__c"); -// dataField.setName("relationId关联对象"); -// fieldList.add(dataField); - } - - if ("ContentDocumentLink".equals(apiName)){ - //文档关联表新增关联对象字段 - Map LinkedMap = Maps.newHashMap(); - LinkedMap.put("type", "varchar(255)"); - LinkedMap.put("comment", "关联对象"); - LinkedMap.put("name", "LinkedEntity_Type"); - list.add(LinkedMap); - } - - if ("Attachment".equals(apiName) || "FeedComment".equals(apiName) - || "FeedItem".equals(apiName) || "Note".equals(apiName)){ - //文档关联表新增关联对象字段 - Map LinkedMap = Maps.newHashMap(); - LinkedMap.put("type", "varchar(255)"); - LinkedMap.put("comment", "关联对象"); - LinkedMap.put("name", "Parent_Type"); - list.add(LinkedMap); - -// DataField dataField = new DataField(); -// dataField.setApi(apiName); -// dataField.setField("Parent.Type"); -// dataField.setName("关联对象"); -// fieldList.add(dataField); - } - customMapper.createTable(apiName, label, list, index); // 生成字段映射 QueryWrapper delfieldQw = new QueryWrapper<>(); @@ -1074,6 +993,7 @@ public class CommonServiceImpl implements CommonService { if (count>0){ update.setIsEditable(false); } + update.setKeyPrefix(dsr.getKeyPrefix()); dataObjectService.saveOrUpdate(update); } } finally { @@ -1324,6 +1244,90 @@ public class CommonServiceImpl implements CommonService { return null; } + @Override + public ReturnT createLinkTypeField(SalesforceParam param) throws Exception { + try { + List apis; + if (StringUtils.isBlank(param.getApi())) { + apis = dataObjectService.list().stream().map(DataObject::getName).collect(Collectors.toList()); + } else { + apis = DataUtil.toIdList(param.getApi()); + } + QueryWrapper wrapper = new QueryWrapper<>(); + wrapper.eq("is_link",1).eq("is_create",0); + if (!apis.isEmpty()){ + wrapper.in("api", apis); + } + + List list = linkConfigService.list(wrapper); + for (LinkConfig config : list) { + if (StringUtils.isBlank(customMapper.checkTable(config.getApi()))){ + log.info("表api：{} 不存在！", config.getApi()); + continue; + } + log.info("表api：{} 未创建字段：{}，现在创建！", config.getApi(),config.getLinkField()); + //创建字段 + customMapper.createField(config.getApi(), config.getLinkField()); + config.setIsCreate(true); + linkConfigService.updateById(config); + } + }catch (Exception e){ + log.error(e.getMessage()); + } + return ReturnT.SUCCESS; + } + + @Override + public ReturnT updateLinkType(SalesforceParam param) throws Exception { + try { + List apis; + if (StringUtils.isBlank(param.getApi())) { + apis = dataObjectService.list().stream().map(DataObject::getName).collect(Collectors.toList()); + } else { + apis = DataUtil.toIdList(param.getApi()); + } + QueryWrapper wrapper = new QueryWrapper<>(); + wrapper.eq("is_link","1").eq("is_create",1); + if (!apis.isEmpty()){ + wrapper.in("api", apis); + } + Map resultMap = dataObjectService.list().stream() + .collect(Collectors.toMap( + DataObject::getKeyPrefix, + DataObject::getName, + (existing, replacement) -> replacement)); + + List list = linkConfigService.list(wrapper); + for (LinkConfig config : list) { + + //若是该对象已经存在数据，则遍历写入值 + Integer dbNum = customMapper.countBySQL(config.getApi(),null); + log.info("表api：{} 存在" +dbNum+ "条数据！赋值当前表多态类型字段：{}", config.getApi(),config.getLinkField()); + + if (dbNum >0 ) { + int page = dbNum % 2000 == 0 ? dbNum / 2000 : (dbNum / 2000) + 1; + for (int i = 0; i < page; i++) { + List> mapList = customMapper.list("*", config.getApi(), "1=1 order by Id limit " + i * 2000 + ",2000"); + List> updateMapList = new ArrayList<>(); + for (int j = 1; j <= mapList.size(); j++) { + Map map = mapList.get(j - 1); + if (map.get(config.getField()) != null){ + String type = resultMap.get(map.get(config.getField()).toString().substring(0, 3)); + Map paramMap = Maps.newHashMap(); + paramMap.put("key", config.getLinkField()); + paramMap.put("value", type); + updateMapList.add(paramMap); + customMapper.updateById(config.getApi(), updateMapList, String.valueOf(mapList.get(j - 1).get("Id") != null?mapList.get(j - 1).get("Id") : mapList.get(j - 1).get("id"))); + } + } + } + } + } + }catch (Exception e){ + log.error(e.getMessage()); + } + return ReturnT.SUCCESS; + } public Date getLastDay(String batchType,Date endDate,Date startDate){ switch (batchType) { diff --git a/src/main/java/com/celnet/datadump/service/impl/DataImportBatchServiceImpl.java b/src/main/java/com/celnet/datadump/service/impl/DataImportBatchServiceImpl.java index b6ff05c..bbe7c4d 100644 --- a/src/main/java/com/celnet/datadump/service/impl/DataImportBatchServiceImpl.java +++ b/src/main/java/com/celnet/datadump/service/impl/DataImportBatchServiceImpl.java @@ -75,6 +75,8 @@ public class DataImportBatchServiceImpl implements DataImportBatchService { @Autowired private DataBatchHistoryService dataBatchHistoryService; + @Autowired + private LinkConfigService linkConfigService; /** * Insert入口 @@ -217,6 +219,20 @@ public class DataImportBatchServiceImpl implements DataImportBatchService { log.info("执行api:{}, 执行page:{}, 执行size:{}", api, i+1, size); List insertList = new ArrayList<>(); + //查询当前对象多态字段映射 + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("api",api).eq("is_create",1).eq("is_link",true); + List configs = linkConfigService.list(queryWrapper); + Map fieldMap = new HashMap<>(); + if (!configs.isEmpty()) { + fieldMap = configs.stream() + .collect(Collectors.toMap( + LinkConfig::getField, // Key提取器 + LinkConfig::getLinkField, // Value提取器 + (oldVal, newVal) -> newVal // 解决重复Key冲突（保留新值） + )); + } + //判断引用对象是否存在new_id DataObject update = new DataObject(); update.setName(api); @@ -252,21 +268,19 @@ public class DataImportBatchServiceImpl implements DataImportBatchService { } if (dataField.getIsCreateable() !=null && dataField.getIsCreateable() && !dataField.getIsNillable() && !dataField.getIsDefaultedOnCreate()) { if ("reference".equals(dataField.getSfType())){ + //引用类型 String reference = dataField.getReferenceTo(); - if (StringUtils.isEmpty(reference)){ - if ("ParentId".equals(dataField.getField())){ - reference = data.get(j-1).get("Parent_Type")!=null?data.get(j-1).get("Parent_Type").toString():null; - }else if ("RelationId".equals(dataField.getField())){ - reference = data.get(j-1).get("Relation_Type__c")!=null?data.get(j-1).get("Relation_Type__c").toString():null; - }else if ("WhatId".equals(dataField.getField())){ - reference = data.get(j-1).get("WhatId_Type__c")!=null?data.get(j-1).get("WhatId_Type__c").toString():null; - }else if ("WhoId".equals(dataField.getField())){ - reference = data.get(j-1).get("WhoId_Type__c")!=null?data.get(j-1).get("WhoId_Type__c").toString():null; - } + //引用类型字段 + String linkfield = fieldMap.get(dataField.getApi()); + + if (StringUtils.isNotBlank(linkfield)){ + reference = data.get(j-1).get(linkfield)!=null?data.get(j-1).get(linkfield).toString():null; } + if (reference == null){ continue; } + log.info("----------" + dataField.getField() + "的引用类型 ------------" + reference); List> referenceMap = customMapper.list("new_id", reference, "new_id is not null limit 1"); if (referenceMap.isEmpty()){ QueryWrapper maxIndex = new QueryWrapper<>(); @@ -516,9 +530,20 @@ public class DataImportBatchServiceImpl implements DataImportBatchService { return; } - //判断引用对象是否存在new_id - DataObject update = new DataObject(); - update.setName(api); + //查询当前对象多态字段映射 + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("api",api).eq("is_create",1).eq("is_link",true); + List configs = linkConfigService.list(queryWrapper); + Map fieldMap = new HashMap<>(); + if (!configs.isEmpty()) { + fieldMap = configs.stream() + .collect(Collectors.toMap( + LinkConfig::getField, // Key提取器 + LinkConfig::getLinkField, // Value提取器 + (oldVal, newVal) -> newVal // 解决重复Key冲突（保留新值） + )); + } + // 总更新数 int sfNum = 0; @@ -536,6 +561,7 @@ public class DataImportBatchServiceImpl implements DataImportBatchService { for (DataField dataField : list) { String field = dataField.getField(); String reference_to = dataField.getReferenceTo(); + String value = String.valueOf(map.get(field)); //根据旧sfid查找引用对象新sfid if (field.equals("Id")) { @@ -545,17 +571,26 @@ public class DataImportBatchServiceImpl implements DataImportBatchService { } else if ( "WhoId".equals(field) ||"WhatId".equals(field)){ continue; } else if (StringUtils.isNotBlank(reference_to) && !"data_picklist".equals(reference_to)) { - if ( StringUtils.isNotEmpty(String.valueOf(map.get(field))) && !"OwnerId".equals(field) - && !"Owner_Type".equals(field)) { + + if (!"null".equals(value) && StringUtils.isNotEmpty(value)) { + //引用类型字段 + String linkfield = fieldMap.get(dataField.getApi()); + if (StringUtils.isNotBlank(linkfield)){ + reference_to = map.get(linkfield)!=null?map.get(linkfield).toString():null; + } + if (reference_to == null){ + continue; + } + log.info("----------" + dataField.getField() + "的引用类型 ------------" + reference_to); //判断reference_to内是否包含User字符串 - if (reference_to.contains("User")) { + if (reference_to.contains(",User") || reference_to.contains("User,")) { reference_to = "User"; } - Map m = customMapper.getById("new_id", reference_to, String.valueOf(map.get(field))); + Map m = customMapper.getById("new_id", reference_to, value); if (m != null && !m.isEmpty()) { account.put(field, m.get("new_id")); }else { - String message = "对象类型:" + api + "的数据:"+ map.get("Id") +"的引用对象:" + dataField.getReferenceTo() + "的数据:"+ map.get(field) +"不存在！"; + String message = "对象类型:" + api + "的数据:"+ map.get("Id") +"的引用对象:" + reference_to + "的数据:"+ map.get(field) +"不存在！"; EmailUtil.send("DataDump ERROR", message); log.info(message); return; diff --git a/src/main/java/com/celnet/datadump/service/impl/DataImportNewServiceImpl.java b/src/main/java/com/celnet/datadump/service/impl/DataImportNewServiceImpl.java index d5eb58d..495b7a9 100644 --- a/src/main/java/com/celnet/datadump/service/impl/DataImportNewServiceImpl.java +++ b/src/main/java/com/celnet/datadump/service/impl/DataImportNewServiceImpl.java @@ -8,10 +8,7 @@ import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper; import com.celnet.datadump.config.SalesforceConnect; import com.celnet.datadump.config.SalesforceExecutor; import com.celnet.datadump.config.SalesforceTargetConnect; -import com.celnet.datadump.entity.DataBatch; -import com.celnet.datadump.entity.DataBatchHistory; -import com.celnet.datadump.entity.DataField; -import com.celnet.datadump.entity.DataObject; +import com.celnet.datadump.entity.*; import com.celnet.datadump.global.Const; import com.celnet.datadump.global.SystemConfigCode; import com.celnet.datadump.mapper.CustomMapper; @@ -72,6 +69,8 @@ public class DataImportNewServiceImpl implements DataImportNewService { @Autowired private CommonService commonService; + @Autowired + private LinkConfigService linkConfigService; /** * Get返写个人客户联系人入口 @@ -415,7 +414,7 @@ public class DataImportNewServiceImpl implements DataImportNewService { } } } - return null; + return ReturnT.SUCCESS; } /** @@ -489,7 +488,7 @@ public class DataImportNewServiceImpl implements DataImportNewService { } } } - return null; + return ReturnT.SUCCESS; } /** @@ -727,9 +726,20 @@ public class DataImportNewServiceImpl implements DataImportNewService { return; } - //判断引用对象是否存在new_id - DataObject update = new DataObject(); - update.setName(api); + //查询当前对象多态字段映射 + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("api",api).eq("is_create",1).eq("is_link",true); + List configs = linkConfigService.list(queryWrapper); + Map fieldMap = new HashMap<>(); + if (!configs.isEmpty()) { + fieldMap = configs.stream() + .collect(Collectors.toMap( + LinkConfig::getField, // Key提取器 + LinkConfig::getLinkField, // Value提取器 + (oldVal, newVal) -> newVal // 解决重复Key冲突（保留新值） + )); + } + int targetCount = 0; //批量插入200一次 @@ -752,12 +762,18 @@ public class DataImportNewServiceImpl implements DataImportNewService { account.setId(String.valueOf(map.get("new_id"))); } else if (!DataUtil.isUpdate(field) || (dataField.getIsCreateable() != null && !dataField.getIsCreateable())) { continue; - } else if ( "WhoId".equals(field) ||"WhatId".equals(field)){ - continue; } else if (StringUtils.isNotBlank(reference_to) && !"data_picklist".equals(reference_to)) { - if (!"null".equals(value) && StringUtils.isNotEmpty(value) && !"OwnerId".equals(field) - && !"Owner_Type".equals(field) ) { + if (!"null".equals(value) && StringUtils.isNotEmpty(value)) { + //引用类型字段 + String linkfield = fieldMap.get(dataField.getApi()); + if (StringUtils.isNotBlank(linkfield)){ + reference_to = map.get(linkfield)!=null?map.get(linkfield).toString():null; + } + if (reference_to == null){ + continue; + } + log.info("----------" + dataField.getField() + "的引用类型 ------------" + reference_to); //判断reference_to内是否包含User字符串 if (reference_to.contains(",User") || reference_to.contains("User,")) { reference_to = "User"; @@ -766,7 +782,7 @@ public class DataImportNewServiceImpl implements DataImportNewService { if (m != null && !m.isEmpty()) { account.setField(field, m.get("new_id")); }else { - String message = "对象类型:" + api + "的数据:"+ map.get("Id") +"的引用对象:" + dataField.getReferenceTo() + "的数据:"+ map.get(field) +"不存在！"; + String message = "对象类型:" + api + "的数据:"+ map.get("Id") +"的引用对象:" + reference_to + "的数据:"+ map.get(field) +"不存在！"; EmailUtil.send("DataDump ERROR", message); log.info(message); return; @@ -852,7 +868,7 @@ public class DataImportNewServiceImpl implements DataImportNewService { HashMap map = new HashMap<>(); for (int i = 0; i < accounts.length; i++) { SObject account = accounts[i]; - if (errorId.equals(account.getId()) || errorId.equals(account.getField("Id"))){ + if (errorId != null && (errorId.equals(account.getId()) || errorId.equals(account.getField("Id")))){ for (DataField dataField : list) { try { Object value = account.getField(dataField.getField()); diff --git a/src/main/java/com/celnet/datadump/service/impl/DataImportServiceImpl.java b/src/main/java/com/celnet/datadump/service/impl/DataImportServiceImpl.java index c4dc122..1f2206d 100644 --- a/src/main/java/com/celnet/datadump/service/impl/DataImportServiceImpl.java +++ b/src/main/java/com/celnet/datadump/service/impl/DataImportServiceImpl.java @@ -75,7 +75,7 @@ public class DataImportServiceImpl implements DataImportService { private CommonService commonService; @Autowired - private SystemConfigService systemConfigService; + private LinkConfigService linkConfigService; @Override @@ -274,106 +274,117 @@ public class DataImportServiceImpl implements DataImportService { return; } + //查询当前对象多态字段映射 + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("api",api).eq("is_create",1).eq("is_link",true); + List configs = linkConfigService.list(queryWrapper); + Map fieldMap = new HashMap<>(); + if (!configs.isEmpty()) { + fieldMap = configs.stream() + .collect(Collectors.toMap( + LinkConfig::getField, // Key提取器 + LinkConfig::getLinkField, // Value提取器 + (oldVal, newVal) -> newVal // 解决重复Key冲突（保留新值） + )); + } + //批量插入200一次 int page = count%200 == 0 ? count/200 : (count/200) + 1; SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); DataObject update = new DataObject(); update.setName(api); for (int i = 0; i < page; i++) { - - List> data = customMapper.list("*", api, "new_id is null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "' limit 200"); - int size = data.size(); - log.info("执行api:{}, 执行page:{}, 执行size:{}", api, i+1, size); - SObject[] accounts = new SObject[size]; - String[] ids = new String[size]; - for (int j = 1; j <= size; j++) { - SObject account = new SObject(); - account.setType(api); - //找出sf对象必填字段，并且给默认值 - for (DataField dataField : list) { - if ("OwnerId".equals(dataField.getField()) || "Owner_Type".equals(dataField.getField()) - || "Id".equals(dataField.getField())){ - continue; - } - if ("CreatedDate".equals(dataField.getField()) && dataField.getIsCreateable()){ - //object类型转Date类型 - Date date; - try { - date = sdf.parse(String.valueOf(data.get(j - 1).get("CreatedDate"))); - }catch (ParseException e){ - //解决当时间秒为0时，转换秒精度丢失问题 - date = sdf.parse(data.get(j - 1).get("CreatedDate")+":00"); - } - Calendar calendar = Calendar.getInstance(); - calendar.setTime(date); - account.setField("CreatedDate", calendar); - continue; - } - if ("CreatedById".equals(dataField.getField()) && dataField.getIsCreateable()){ - Map CreatedByIdMap = customMapper.getById("new_id", "User", data.get(j-1).get("CreatedById").toString()); - if(CreatedByIdMap.get("new_id") != null && StringUtils.isNotEmpty(CreatedByIdMap.get("new_id").toString())){ - account.setField("CreatedById", CreatedByIdMap.get("new_id")); - } - continue; - } - if (dataField.getIsCreateable() != null && dataField.getIsCreateable() && !dataField.getIsNillable() && !dataField.getIsDefaultedOnCreate()) { - if ("reference".equals(dataField.getSfType())){ - log.info("----------" + dataField.getField() + " ------------" + dataField.getSfType()); - String reference = dataField.getReferenceTo(); - if (StringUtils.isEmpty(reference)){ - if ("ParentId".equals(dataField.getField())){ - reference = data.get(j-1).get("Parent_Type")!=null?data.get(j-1).get("Parent_Type").toString():null; - }else if ("RelationId".equals(dataField.getField())){ - reference = data.get(j-1).get("Relation_Type__c")!=null?data.get(j-1).get("Relation_Type__c").toString():null; - }else if ("WhatId".equals(dataField.getField())){ - reference = data.get(j-1).get("WhatId_Type__c")!=null?data.get(j-1).get("WhatId_Type__c").toString():null; - }else if ("WhoId".equals(dataField.getField())){ - reference = data.get(j-1).get("WhoId_Type__c")!=null?data.get(j-1).get("WhoId_Type__c").toString():null; - } - } - if (reference == null){ - continue; - } - List> referenceMap = customMapper.list("new_id", reference, "new_id is not null limit 1"); - if (referenceMap.isEmpty()){ - QueryWrapper maxIndex = new QueryWrapper<>(); - maxIndex.select("IFNULL(max(data_index),0) as data_index"); - maxIndex.ne("name", api); - Map map = dataObjectService.getMap(maxIndex); - //如果必填lookup字段没有值，跳过 - update.setDataIndex(Integer.parseInt(map.get("data_index").toString()) + 1); - dataObjectService.updateById(update); - return; - }else{ - account.setField(dataField.getField(), referenceMap.get(0).get("new_id")); - continue; - } - } - if ("picklist".equals(dataField.getSfType())){ - List> pickList = customMapper.list("value", "data_picklist", "api = '"+api+"' and field = '"+dataField.getField()+"' limit 1"); - account.setField(dataField.getField(), pickList.get(0).get("value")); + try { + List> data = customMapper.list("*", api, "new_id is null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "' limit 200"); + int size = data.size(); + log.info("执行api:{}, 执行page:{}, 执行size:{}", api, i+1, size); + SObject[] accounts = new SObject[size]; + String[] ids = new String[size]; + for (int j = 1; j <= size; j++) { + SObject account = new SObject(); + account.setType(api); + //找出sf对象必填字段，并且给默认值 + for (DataField dataField : list) { + if ("OwnerId".equals(dataField.getField()) || "Owner_Type".equals(dataField.getField()) + || "Id".equals(dataField.getField())){ continue; } - account.setField(dataField.getField(), DataUtil.fieldTypeToSf(dataField)); + if ("CreatedDate".equals(dataField.getField()) && dataField.getIsCreateable()){ + //object类型转Date类型 + Date date; + try { + date = sdf.parse(String.valueOf(data.get(j - 1).get("CreatedDate"))); + }catch (ParseException e){ + //解决当时间秒为0时，转换秒精度丢失问题 + date = sdf.parse(data.get(j - 1).get("CreatedDate")+":00"); + } + Calendar calendar = Calendar.getInstance(); + calendar.setTime(date); + account.setField("CreatedDate", calendar); + continue; + } + if ("CreatedById".equals(dataField.getField()) && dataField.getIsCreateable()){ + Map CreatedByIdMap = customMapper.getById("new_id", "User", data.get(j-1).get("CreatedById").toString()); + if(CreatedByIdMap.get("new_id") != null && StringUtils.isNotEmpty(CreatedByIdMap.get("new_id").toString())){ + account.setField("CreatedById", CreatedByIdMap.get("new_id")); + } + continue; + } + if (dataField.getIsCreateable() != null && dataField.getIsCreateable() && !dataField.getIsNillable() && !dataField.getIsDefaultedOnCreate()) { + if ("reference".equals(dataField.getSfType())){ + //引用类型 + String reference = dataField.getReferenceTo(); + //引用类型字段 + String linkfield = fieldMap.get(dataField.getApi()); + + if (StringUtils.isNotBlank(linkfield)){ + reference = data.get(j-1).get(linkfield)!=null?data.get(j-1).get(linkfield).toString():null; + } + + if (reference == null){ + continue; + } + log.info("----------" + dataField.getField() + "的引用类型 ------------" + reference); + + List> referenceMap = customMapper.list("new_id", reference, "new_id is not null limit 1"); + if (referenceMap.isEmpty()){ + QueryWrapper maxIndex = new QueryWrapper<>(); + maxIndex.select("IFNULL(max(data_index),0) as data_index"); + maxIndex.ne("name", api); + Map map = dataObjectService.getMap(maxIndex); + //如果必填lookup字段没有值，跳过 + update.setDataIndex(Integer.parseInt(map.get("data_index").toString()) + 1); + dataObjectService.updateById(update); + return; + }else{ + account.setField(dataField.getField(), referenceMap.get(0).get("new_id")); + continue; + } + } + if ("picklist".equals(dataField.getSfType())){ + List> pickList = customMapper.list("value", "data_picklist", "api = '"+api+"' and field = '"+dataField.getField()+"' limit 1"); + account.setField(dataField.getField(), pickList.get(0).get("value")); + continue; + } + account.setField(dataField.getField(), DataUtil.fieldTypeToSf(dataField)); + } + } + + if (api.equals("vlink__Wechat_User__c")){ + List> maps = customMapper.list("new_id", "vlink__Wechat_Account__c", "new_id is not null and id = '" + data.get(j - 1).get("vlink__Wechat_Account__c") + "' limit 1"); + if (!maps.isEmpty()){ + Map referenceMap = maps.get(0); + account.setField("vlink__Wechat_Account__c", referenceMap.get("new_id")); + } + } + + ids[j-1] = data.get(j-1).get("Id").toString(); + accounts[j-1] = account; + if (i*200+j == count){ + break; } } - if (api.equals("vlink__Wechat_User__c")){ - List> maps = customMapper.list("new_id", "vlink__Wechat_Account__c", "new_id is not null and id = '" + data.get(j - 1).get("vlink__Wechat_Account__c") + "' limit 1"); - if (!maps.isEmpty()){ - Map referenceMap = maps.get(0); - account.setField("vlink__Wechat_Account__c", referenceMap.get("new_id")); - } - } - - ids[j-1] = data.get(j-1).get("Id").toString(); - accounts[j-1] = account; - if (i*200+j == count){ - break; - } - } - - try { if (infoFlag != null && "1".equals(infoFlag.get("value"))){ printAccountsDetails(accounts, list); } diff --git a/src/main/java/com/celnet/datadump/service/impl/LinkConfigServiceImpl.java b/src/main/java/com/celnet/datadump/service/impl/LinkConfigServiceImpl.java new file mode 100644 index 0000000..f4aaa23 --- /dev/null +++ b/src/main/java/com/celnet/datadump/service/impl/LinkConfigServiceImpl.java @@ -0,0 +1,16 @@ +package com.celnet.datadump.service.impl; + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.celnet.datadump.entity.DataLog; +import com.celnet.datadump.entity.LinkConfig; +import com.celnet.datadump.mapper.DataLogMapper; +import com.celnet.datadump.mapper.LinkConfigMapper; +import com.celnet.datadump.service.DataLogService; +import com.celnet.datadump.service.LinkConfigService; +import org.springframework.stereotype.Service; + + +@Service +public class LinkConfigServiceImpl extends ServiceImpl implements LinkConfigService { + +} diff --git a/src/main/java/com/celnet/datadump/util/DataUtil.java b/src/main/java/com/celnet/datadump/util/DataUtil.java index 92b55df..f9fa9fb 100644 --- a/src/main/java/com/celnet/datadump/util/DataUtil.java +++ b/src/main/java/com/celnet/datadump/util/DataUtil.java @@ -1,5 +1,6 @@ package com.celnet.datadump.util; +import cn.hutool.core.lang.UUID; import com.alibaba.fastjson2.JSONArray; import com.alibaba.fastjson2.JSONObject; import com.celnet.datadump.entity.DataField; @@ -388,7 +389,7 @@ public class DataUtil { case "datetime": return date; default: - return "初始"+System.currentTimeMillis(); + return "初始"+ UUID.randomUUID(); } } diff --git a/src/main/resources/mapper/CustomMapper.xml b/src/main/resources/mapper/CustomMapper.xml index 9190f1a..70468de 100644 --- a/src/main/resources/mapper/CustomMapper.xml +++ b/src/main/resources/mapper/CustomMapper.xml @@ -28,6 +28,10 @@ ) COMMENT = '${tableComment}'; + + ALTER TABLE `${tableName}` ADD COLUMN `${fieldName}` varchar(255) + +