kris/data-dump
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

【feat】对象多态字段处理

Browse Source
This commit is contained in:
Kris 2025-07-15 11:21:08 +08:00
parent b353084aca
commit 37d8d94004
15 changed files with 479 additions and 220 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/java/com/celnet/datadump/entity/DataBatch.java
View File

@ -127,7 +127,7 @@ public class DataBatch implements Serializable, Cloneable {
@TableField(value = "created_date", fill = FieldFill.INSERT)
@ExcelIgnore
@ApiModelProperty(value = "创建时间")
private LocalDateTime createdDate;
private Date createdDate;
/**
* 最后更新时间
@ -135,7 +135,7 @@ public class DataBatch implements Serializable, Cloneable {
@TableField(value = "last_modified_date", fill = FieldFill.INSERT_UPDATE)
@ExcelIgnore
@ApiModelProperty(value = "最后更新时间")
private LocalDateTime lastModifiedDate;
private Date lastModifiedDate;
/**
* sf新增数据量

7
src/main/java/com/celnet/datadump/entity/DataObject.java
View File

@ -42,6 +42,13 @@ public class DataObject implements Serializable {
@ApiModelProperty(value = "对象名称")
private String label;
/**
* 对象名称
*/
@TableField("keyPrefix")
@ApiModelProperty(value = "对象ID前缀")
private String keyPrefix;
/**
* 是否启用
*/

75
src/main/java/com/celnet/datadump/entity/LinkConfig.java Normal file
View File

@ -0,0 +1,75 @@
package com.celnet.datadump.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Getter;
import lombok.Setter;
import java.io.Serializable;
/**
* @author Kris
* @date 2025/07/10
*/
@Getter
@Setter
@TableName("link_config")
@ApiModel(value = "映射信息表")
public class LinkConfig implements Serializable {
private static final long serialVersionUID = 1L;
/**
* id
*/
@TableId(value = "id", type = IdType.AUTO)
@ApiModelProperty(value = "id")
private Integer id;
/**
* 对象api
*/
@TableField("api")
@ApiModelProperty(value = "对象api")
private String api;
/**
* 对象标签
*/
@TableField("label")
@ApiModelProperty(value = "对象标签")
private String label;
/**
* 多态ID字段
*/
@TableField("field")
@ApiModelProperty(value = "多态ID字段")
private String field;
/**
* 映射字段
*/
@TableField("link_field")
@ApiModelProperty(value = "映射字段")
private String linkField;
/**
* 是否创建
*/
@TableField("is_create")
@ApiModelProperty(value = "是否创建")
private Boolean isCreate;
/**
* 是否映射
*/
@TableField("is_link")
@ApiModelProperty(value = "是否映射")
private Boolean isLink;
}

41
src/main/java/com/celnet/datadump/job/DataDumpNewJob.java
View File

@ -147,4 +147,45 @@ public class DataDumpNewJob {
}
/**
* 创建关联字段
* @return result
*/
@XxlJob("createLinkTypeFieldJob")
public ReturnT<String> createLinkTypeJob(String paramStr) throws Exception{
log.info("createLinkTypeFieldJob execute start ..................");
SalesforceParam param = new SalesforceParam();
try {
if (StringUtils.isNotBlank(paramStr)) {
param = JSON.parseObject(paramStr, SalesforceParam.class);
}
} catch (Throwable throwable) {
return new ReturnT<>(500, "参数解析失败！");
}
return commonService.createLinkTypeField(param);
}
/**
* 更新关联类型
* @return result
*/
@XxlJob("updateLinkTypeJob")
public ReturnT<String> updateLinkTypeJob(String paramStr) throws Exception{
log.info("updateLinkTypeJob execute start ..................");
SalesforceParam param = new SalesforceParam();
try {
if (StringUtils.isNotBlank(paramStr)) {
param = JSON.parseObject(paramStr, SalesforceParam.class);
}
} catch (Throwable throwable) {
return new ReturnT<>(500, "参数解析失败！");
}
return commonService.updateLinkType(param);
}
}

6
src/main/java/com/celnet/datadump/mapper/CustomMapper.java
View File

@ -57,6 +57,12 @@ public interface CustomMapper {
*/
public void createTable(@Param("tableName") String tableName, @Param("tableComment") String tableComment, @Param("maps") List<Map<String, Object>> maps, @Param("index") List<Map<String, Object>> index);
/**
* 创建字段
* @param tableName
*/
public int createField(@Param("tableName") String tableName, @Param("fieldName") String fieldName);
/**
* 更新方法
*

16
src/main/java/com/celnet/datadump/mapper/LinkConfigMapper.java Normal file
View File

@ -0,0 +1,16 @@
package com.celnet.datadump.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.celnet.datadump.entity.LinkConfig;
import org.apache.ibatis.annotations.Mapper;
/**
* <p>
* Mapper 接口
* </p>
*/
@Mapper
public interface LinkConfigMapper extends BaseMapper<LinkConfig> {
}

17
src/main/java/com/celnet/datadump/service/CommonService.java
View File

@ -57,4 +57,21 @@ public interface CommonService {
String getDocumentId(PartnerConnection partnerConnection, String contentVersionId) throws Exception;
ReturnT<String> getChatter(SalesforceParam param) throws Exception;
/**
* 创建关联字段
* @param param 参数
* @return ReturnT
* @throws Exception exception
*/
ReturnT<String> createLinkTypeField(SalesforceParam param) throws Exception;
/**
* 更新关联类型
* @param param 参数
* @return ReturnT
* @throws Exception exception
*/
ReturnT<String> updateLinkType(SalesforceParam param) throws Exception;
}

10
src/main/java/com/celnet/datadump/service/LinkConfigService.java Normal file
View File

@ -0,0 +1,10 @@
package com.celnet.datadump.service;
import com.baomidou.mybatisplus.extension.service.IService;
import com.celnet.datadump.entity.DataLog;
import com.celnet.datadump.entity.LinkConfig;
public interface LinkConfigService extends IService<LinkConfig> {
}

192
src/main/java/com/celnet/datadump/service/impl/CommonServiceImpl.java
View File

@ -83,6 +83,8 @@ public class CommonServiceImpl implements CommonService {
private SalesforceTargetConnect salesforceTargetConnect;
@Autowired
private MetaclassConfigService metaclassConfigService;
@Autowired
private LinkConfigService linkConfigService;
@Override
public ReturnT<String> increment(SalesforceParam param) throws Exception {
@ -766,33 +768,7 @@ public class CommonServiceImpl implements CommonService {
maps.add(paramMap);
maps.add(paramMap2);
}
// Task和Event
if ("Task".equals(api) || "Event".equals(api)){
Map<String, Object> paramwhoMap = Maps.newHashMap();
paramwhoMap.put("key", "WhoId_Type__c");
paramwhoMap.put("value", jsonObject.get("Who_Type"));
maps.add(paramwhoMap);
Map<String, Object> paramwhatMap = Maps.newHashMap();
paramwhatMap.put("key", "WhatId_Type__c");
paramwhatMap.put("value", jsonObject.get("What_Type"));
maps.add(paramwhatMap);
}
if ("TaskRelation".equals(api) || "EventRelation".equals(api)){
Map<String, Object> paramMap = Maps.newHashMap();
paramMap.put("key", "Relation_Type__c");
paramMap.put("value", jsonObject.get("Relation_Type"));
maps.add(paramMap);
}
//附件关联表 插入更新时给关联对象赋值
// if ("ContentDocumentLink".equals(api)) {
// String linkedEntity_Type = records[i].getChild("LinkedEntity").getChild("Type").getValue().toString();
// Map<String, Object> paramMap = Maps.newHashMap();
// paramMap.put("key", "linkedEntity_Type");
// paramMap.put("value", linkedEntity_Type);
// maps.add(paramMap);
// }
if (existsIds.contains(id)) {
customMapper.updateById(api, maps, id);
} else {
@ -880,7 +856,17 @@ public class CommonServiceImpl implements CommonService {
if (field.getReferenceTo().length <= 3) {
join = StringUtils.join(field.getReferenceTo(), ",");
} else {
log.warn("referenceTo too long set null, api:{}, field:{}, reference to:{}", apiName, field.getName(), StringUtils.join(field.getReferenceTo(), ","));
log.info("referenceTo too long set null, api:{}, field:{}, reference to:{}", apiName, field.getName(), StringUtils.join(field.getReferenceTo(), ","));
}
if (field.getReferenceTo().length > 1){
if (field.getName().contains("Id") && !"OwnerId".equals(field.getName())){
LinkConfig linkConfig = new LinkConfig();
linkConfig.setApi(apiName);
linkConfig.setLabel(label);
linkConfig.setField(field.getName());
linkConfig.setLinkField(StringUtils.replace(field.getName(), "Id", "_Type"));
linkConfigService.save(linkConfig);
}
}
// picklist保存到picklist表
if ("picklist".equalsIgnoreCase(sfType)) {
@ -898,9 +884,6 @@ public class CommonServiceImpl implements CommonService {
}
}
dataField.setReferenceTo(join);
if ("WhoId".equals(field.getName()) || "WhatId".equals(field.getName())){
dataField.setReferenceTo(null);
}
fieldList.add(dataField);
fields.add(field.getName());
}
@ -958,70 +941,6 @@ public class CommonServiceImpl implements CommonService {
map.put("name", "new_id");
list.add(map);
if ("Task".equals(apiName) || "Event".equals(apiName)){
Map<String, Object> LinkedMap = Maps.newHashMap();
LinkedMap.put("type", "varchar(18)");
LinkedMap.put("comment", "whatId关联对象");
LinkedMap.put("name", "WhatId_Type__c");
list.add(LinkedMap);
Map<String, Object> LinkedMap1 = Maps.newHashMap();
LinkedMap1.put("type", "varchar(18)");
LinkedMap1.put("comment", "whoId关联对象");
LinkedMap1.put("name", "WhoId_Type__c");
list.add(LinkedMap1);
// DataField dataField = new DataField();
// dataField.setApi(apiName);
// dataField.setField("WhoId_Type__c");
// dataField.setName("whoId关联对象");
// fieldList.add(dataField);
//
// DataField dataField1 = new DataField();
// dataField1.setApi(apiName);
// dataField1.setField("WhatId_Type__c");
// dataField1.setName("whatId关联对象");
// fieldList.add(dataField1);
}
if ("TaskRelation".equals(apiName) || "EventRelation".equals(apiName)){
Map<String, Object> LinkedMap = Maps.newHashMap();
LinkedMap.put("type", "varchar(18)");
LinkedMap.put("comment", "relationId关联对象");
LinkedMap.put("name", "Relation_Type__c");
list.add(LinkedMap);
// DataField dataField = new DataField();
// dataField.setApi(apiName);
// dataField.setField("Relation_Type__c");
// dataField.setName("relationId关联对象");
// fieldList.add(dataField);
}
if ("ContentDocumentLink".equals(apiName)){
//文档关联表新增关联对象字段
Map<String, Object> LinkedMap = Maps.newHashMap();
LinkedMap.put("type", "varchar(255)");
LinkedMap.put("comment", "关联对象");
LinkedMap.put("name", "LinkedEntity_Type");
list.add(LinkedMap);
}
if ("Attachment".equals(apiName) || "FeedComment".equals(apiName)
|| "FeedItem".equals(apiName) || "Note".equals(apiName)){
//文档关联表新增关联对象字段
Map<String, Object> LinkedMap = Maps.newHashMap();
LinkedMap.put("type", "varchar(255)");
LinkedMap.put("comment", "关联对象");
LinkedMap.put("name", "Parent_Type");
list.add(LinkedMap);
// DataField dataField = new DataField();
// dataField.setApi(apiName);
// dataField.setField("Parent.Type");
// dataField.setName("关联对象");
// fieldList.add(dataField);
}
customMapper.createTable(apiName, label, list, index);
// 生成字段映射
QueryWrapper<DataField> delfieldQw = new QueryWrapper<>();
@ -1074,6 +993,7 @@ public class CommonServiceImpl implements CommonService {
if (count>0){
update.setIsEditable(false);
}
update.setKeyPrefix(dsr.getKeyPrefix());
dataObjectService.saveOrUpdate(update);
}
} finally {
@ -1324,6 +1244,90 @@ public class CommonServiceImpl implements CommonService {
return null;
}
@Override
public ReturnT<String> createLinkTypeField(SalesforceParam param) throws Exception {
try {
List<String> apis;
if (StringUtils.isBlank(param.getApi())) {
apis = dataObjectService.list().stream().map(DataObject::getName).collect(Collectors.toList());
} else {
apis = DataUtil.toIdList(param.getApi());
}
QueryWrapper<LinkConfig> wrapper = new QueryWrapper<>();
wrapper.eq("is_link",1).eq("is_create",0);
if (!apis.isEmpty()){
wrapper.in("api", apis);
}
List<LinkConfig> list = linkConfigService.list(wrapper);
for (LinkConfig config : list) {
if (StringUtils.isBlank(customMapper.checkTable(config.getApi()))){
log.info("表api：{} 不存在！", config.getApi());
continue;
}
log.info("表api：{} 未创建字段：{}，现在创建！", config.getApi(),config.getLinkField());
//创建字段
customMapper.createField(config.getApi(), config.getLinkField());
config.setIsCreate(true);
linkConfigService.updateById(config);
}
}catch (Exception e){
log.error(e.getMessage());
}
return ReturnT.SUCCESS;
}
@Override
public ReturnT<String> updateLinkType(SalesforceParam param) throws Exception {
try {
List<String> apis;
if (StringUtils.isBlank(param.getApi())) {
apis = dataObjectService.list().stream().map(DataObject::getName).collect(Collectors.toList());
} else {
apis = DataUtil.toIdList(param.getApi());
}
QueryWrapper<LinkConfig> wrapper = new QueryWrapper<>();
wrapper.eq("is_link","1").eq("is_create",1);
if (!apis.isEmpty()){
wrapper.in("api", apis);
}
Map<String, String> resultMap = dataObjectService.list().stream()
.collect(Collectors.toMap(
DataObject::getKeyPrefix,
DataObject::getName,
(existing, replacement) -> replacement));
List<LinkConfig> list = linkConfigService.list(wrapper);
for (LinkConfig config : list) {
//若是该对象已经存在数据，则遍历写入值
Integer dbNum = customMapper.countBySQL(config.getApi(),null);
log.info("表api：{} 存在" +dbNum+ "条数据！赋值当前表多态类型字段：{}", config.getApi(),config.getLinkField());
if (dbNum >0 ) {
int page = dbNum % 2000 == 0 ? dbNum / 2000 : (dbNum / 2000) + 1;
for (int i = 0; i < page; i++) {
List<Map<String, Object>> mapList = customMapper.list("*", config.getApi(), "1=1 order by Id limit " + i * 2000 + ",2000");
List<Map<String, Object>> updateMapList = new ArrayList<>();
for (int j = 1; j <= mapList.size(); j++) {
Map<String, Object> map = mapList.get(j - 1);
if (map.get(config.getField()) != null){
String type = resultMap.get(map.get(config.getField()).toString().substring(0, 3));
Map<String, Object> paramMap = Maps.newHashMap();
paramMap.put("key", config.getLinkField());
paramMap.put("value", type);
updateMapList.add(paramMap);
customMapper.updateById(config.getApi(), updateMapList, String.valueOf(mapList.get(j - 1).get("Id") != null?mapList.get(j - 1).get("Id") : mapList.get(j - 1).get("id")));
}
}
}
}
}
}catch (Exception e){
log.error(e.getMessage());
}
return ReturnT.SUCCESS;
}
public Date getLastDay(String batchType,Date endDate,Date startDate){
switch (batchType) {

71
src/main/java/com/celnet/datadump/service/impl/DataImportBatchServiceImpl.java
View File

@ -75,6 +75,8 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
@Autowired
private DataBatchHistoryService dataBatchHistoryService;
@Autowired
private LinkConfigService linkConfigService;
/**
* Insert入口
@ -217,6 +219,20 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
log.info("执行api:{}, 执行page:{}, 执行size:{}", api, i+1, size);
List<JSONObject> insertList = new ArrayList<>();
//查询当前对象多态字段映射
QueryWrapper<LinkConfig> queryWrapper = new QueryWrapper<>();
queryWrapper.eq("api",api).eq("is_create",1).eq("is_link",true);
List<LinkConfig> configs = linkConfigService.list(queryWrapper);
Map<String, String> fieldMap = new HashMap<>();
if (!configs.isEmpty()) {
fieldMap = configs.stream()
.collect(Collectors.toMap(
LinkConfig::getField, // Key提取器
LinkConfig::getLinkField, // Value提取器
(oldVal, newVal) -> newVal // 解决重复Key冲突（保留新值）
));
}
//判断引用对象是否存在new_id
DataObject update = new DataObject();
update.setName(api);
@ -252,21 +268,19 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
}
if (dataField.getIsCreateable() !=null && dataField.getIsCreateable() && !dataField.getIsNillable() && !dataField.getIsDefaultedOnCreate()) {
if ("reference".equals(dataField.getSfType())){
//引用类型
String reference = dataField.getReferenceTo();
if (StringUtils.isEmpty(reference)){
if ("ParentId".equals(dataField.getField())){
reference = data.get(j-1).get("Parent_Type")!=null?data.get(j-1).get("Parent_Type").toString():null;
}else if ("RelationId".equals(dataField.getField())){
reference = data.get(j-1).get("Relation_Type__c")!=null?data.get(j-1).get("Relation_Type__c").toString():null;
}else if ("WhatId".equals(dataField.getField())){
reference = data.get(j-1).get("WhatId_Type__c")!=null?data.get(j-1).get("WhatId_Type__c").toString():null;
}else if ("WhoId".equals(dataField.getField())){
reference = data.get(j-1).get("WhoId_Type__c")!=null?data.get(j-1).get("WhoId_Type__c").toString():null;
}
//引用类型字段
String linkfield = fieldMap.get(dataField.getApi());
if (StringUtils.isNotBlank(linkfield)){
reference = data.get(j-1).get(linkfield)!=null?data.get(j-1).get(linkfield).toString():null;
}
if (reference == null){
continue;
}
log.info("----------" + dataField.getField() + "的引用类型 ------------" + reference);
List<Map<String, Object>> referenceMap = customMapper.list("new_id", reference, "new_id is not null limit 1");
if (referenceMap.isEmpty()){
QueryWrapper<DataObject> maxIndex = new QueryWrapper<>();
@ -516,9 +530,20 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
return;
}
//判断引用对象是否存在new_id
DataObject update = new DataObject();
update.setName(api);
//查询当前对象多态字段映射
QueryWrapper<LinkConfig> queryWrapper = new QueryWrapper<>();
queryWrapper.eq("api",api).eq("is_create",1).eq("is_link",true);
List<LinkConfig> configs = linkConfigService.list(queryWrapper);
Map<String, String> fieldMap = new HashMap<>();
if (!configs.isEmpty()) {
fieldMap = configs.stream()
.collect(Collectors.toMap(
LinkConfig::getField, // Key提取器
LinkConfig::getLinkField, // Value提取器
(oldVal, newVal) -> newVal // 解决重复Key冲突（保留新值）
));
}
// 总更新数
int sfNum = 0;
@ -536,6 +561,7 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
for (DataField dataField : list) {
String field = dataField.getField();
String reference_to = dataField.getReferenceTo();
String value = String.valueOf(map.get(field));
//根据旧sfid查找引用对象新sfid
if (field.equals("Id")) {
@ -545,17 +571,26 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
} else if ( "WhoId".equals(field) ||"WhatId".equals(field)){
continue;
} else if (StringUtils.isNotBlank(reference_to) && !"data_picklist".equals(reference_to)) {
if ( StringUtils.isNotEmpty(String.valueOf(map.get(field))) && !"OwnerId".equals(field)
&& !"Owner_Type".equals(field)) {
if (!"null".equals(value) && StringUtils.isNotEmpty(value)) {
//引用类型字段
String linkfield = fieldMap.get(dataField.getApi());
if (StringUtils.isNotBlank(linkfield)){
reference_to = map.get(linkfield)!=null?map.get(linkfield).toString():null;
}
if (reference_to == null){
continue;
}
log.info("----------" + dataField.getField() + "的引用类型 ------------" + reference_to);
//判断reference_to内是否包含User字符串
if (reference_to.contains("User")) {
if (reference_to.contains(",User") || reference_to.contains("User,")) {
reference_to = "User";
}
Map<String, Object> m = customMapper.getById("new_id", reference_to, String.valueOf(map.get(field)));
Map<String, Object> m = customMapper.getById("new_id", reference_to, value);
if (m != null && !m.isEmpty()) {
account.put(field, m.get("new_id"));
}else {
String message = "对象类型:" + api + "的数据:"+ map.get("Id") +"的引用对象:" + dataField.getReferenceTo() + "的数据:"+ map.get(field) +"不存在！";
String message = "对象类型:" + api + "的数据:"+ map.get("Id") +"的引用对象:" + reference_to + "的数据:"+ map.get(field) +"不存在！";
EmailUtil.send("DataDump ERROR", message);
log.info(message);
return;

46
src/main/java/com/celnet/datadump/service/impl/DataImportNewServiceImpl.java
View File

@ -8,10 +8,7 @@ import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper;
import com.celnet.datadump.config.SalesforceConnect;
import com.celnet.datadump.config.SalesforceExecutor;
import com.celnet.datadump.config.SalesforceTargetConnect;
import com.celnet.datadump.entity.DataBatch;
import com.celnet.datadump.entity.DataBatchHistory;
import com.celnet.datadump.entity.DataField;
import com.celnet.datadump.entity.DataObject;
import com.celnet.datadump.entity.*;
import com.celnet.datadump.global.Const;
import com.celnet.datadump.global.SystemConfigCode;
import com.celnet.datadump.mapper.CustomMapper;
@ -72,6 +69,8 @@ public class DataImportNewServiceImpl implements DataImportNewService {
@Autowired
private CommonService commonService;
@Autowired
private LinkConfigService linkConfigService;
/**
* Get返写个人客户联系人入口
@ -415,7 +414,7 @@ public class DataImportNewServiceImpl implements DataImportNewService {
}
}
}
return null;
return ReturnT.SUCCESS;
}
/**
@ -489,7 +488,7 @@ public class DataImportNewServiceImpl implements DataImportNewService {
}
}
}
return null;
return ReturnT.SUCCESS;
}
/**
@ -727,9 +726,20 @@ public class DataImportNewServiceImpl implements DataImportNewService {
return;
}
//判断引用对象是否存在new_id
DataObject update = new DataObject();
update.setName(api);
//查询当前对象多态字段映射
QueryWrapper<LinkConfig> queryWrapper = new QueryWrapper<>();
queryWrapper.eq("api",api).eq("is_create",1).eq("is_link",true);
List<LinkConfig> configs = linkConfigService.list(queryWrapper);
Map<String, String> fieldMap = new HashMap<>();
if (!configs.isEmpty()) {
fieldMap = configs.stream()
.collect(Collectors.toMap(
LinkConfig::getField, // Key提取器
LinkConfig::getLinkField, // Value提取器
(oldVal, newVal) -> newVal // 解决重复Key冲突（保留新值）
));
}
int targetCount = 0;
//批量插入200一次
@ -752,12 +762,18 @@ public class DataImportNewServiceImpl implements DataImportNewService {
account.setId(String.valueOf(map.get("new_id")));
} else if (!DataUtil.isUpdate(field) || (dataField.getIsCreateable() != null && !dataField.getIsCreateable())) {
continue;
} else if ( "WhoId".equals(field) ||"WhatId".equals(field)){
continue;
} else if (StringUtils.isNotBlank(reference_to) && !"data_picklist".equals(reference_to)) {
if (!"null".equals(value) && StringUtils.isNotEmpty(value) && !"OwnerId".equals(field)
&& !"Owner_Type".equals(field) ) {
if (!"null".equals(value) && StringUtils.isNotEmpty(value)) {
//引用类型字段
String linkfield = fieldMap.get(dataField.getApi());
if (StringUtils.isNotBlank(linkfield)){
reference_to = map.get(linkfield)!=null?map.get(linkfield).toString():null;
}
if (reference_to == null){
continue;
}
log.info("----------" + dataField.getField() + "的引用类型 ------------" + reference_to);
//判断reference_to内是否包含User字符串
if (reference_to.contains(",User") || reference_to.contains("User,")) {
reference_to = "User";
@ -766,7 +782,7 @@ public class DataImportNewServiceImpl implements DataImportNewService {
if (m != null && !m.isEmpty()) {
account.setField(field, m.get("new_id"));
}else {
String message = "对象类型:" + api + "的数据:"+ map.get("Id") +"的引用对象:" + dataField.getReferenceTo() + "的数据:"+ map.get(field) +"不存在！";
String message = "对象类型:" + api + "的数据:"+ map.get("Id") +"的引用对象:" + reference_to + "的数据:"+ map.get(field) +"不存在！";
EmailUtil.send("DataDump ERROR", message);
log.info(message);
return;
@ -852,7 +868,7 @@ public class DataImportNewServiceImpl implements DataImportNewService {
HashMap<String, String> map = new HashMap<>();
for (int i = 0; i < accounts.length; i++) {
SObject account = accounts[i];
if (errorId.equals(account.getId()) || errorId.equals(account.getField("Id"))){
if (errorId != null && (errorId.equals(account.getId()) || errorId.equals(account.getField("Id")))){
for (DataField dataField : list) {
try {
Object value = account.getField(dataField.getField());

191
src/main/java/com/celnet/datadump/service/impl/DataImportServiceImpl.java
View File

@ -75,7 +75,7 @@ public class DataImportServiceImpl implements DataImportService {
private CommonService commonService;
@Autowired
private SystemConfigService systemConfigService;
private LinkConfigService linkConfigService;
@Override
@ -274,106 +274,117 @@ public class DataImportServiceImpl implements DataImportService {
return;
}
//查询当前对象多态字段映射
QueryWrapper<LinkConfig> queryWrapper = new QueryWrapper<>();
queryWrapper.eq("api",api).eq("is_create",1).eq("is_link",true);
List<LinkConfig> configs = linkConfigService.list(queryWrapper);
Map<String, String> fieldMap = new HashMap<>();
if (!configs.isEmpty()) {
fieldMap = configs.stream()
.collect(Collectors.toMap(
LinkConfig::getField, // Key提取器
LinkConfig::getLinkField, // Value提取器
(oldVal, newVal) -> newVal // 解决重复Key冲突（保留新值）
));
}
//批量插入200一次
int page = count%200 == 0 ? count/200 : (count/200) + 1;
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
DataObject update = new DataObject();
update.setName(api);
for (int i = 0; i < page; i++) {
List<Map<String, Object>> data = customMapper.list("*", api, "new_id is null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "' limit 200");
int size = data.size();
log.info("执行api:{}, 执行page:{}, 执行size:{}", api, i+1, size);
SObject[] accounts = new SObject[size];
String[] ids = new String[size];
for (int j = 1; j <= size; j++) {
SObject account = new SObject();
account.setType(api);
//找出sf对象必填字段，并且给默认值
for (DataField dataField : list) {
if ("OwnerId".equals(dataField.getField()) || "Owner_Type".equals(dataField.getField())
|| "Id".equals(dataField.getField())){
continue;
}
if ("CreatedDate".equals(dataField.getField()) && dataField.getIsCreateable()){
//object类型转Date类型
Date date;
try {
date = sdf.parse(String.valueOf(data.get(j - 1).get("CreatedDate")));
}catch (ParseException e){
//解决当时间秒为0时，转换秒精度丢失问题
date = sdf.parse(data.get(j - 1).get("CreatedDate")+":00");
}
Calendar calendar = Calendar.getInstance();
calendar.setTime(date);
account.setField("CreatedDate", calendar);
continue;
}
if ("CreatedById".equals(dataField.getField()) && dataField.getIsCreateable()){
Map<String, Object> CreatedByIdMap = customMapper.getById("new_id", "User", data.get(j-1).get("CreatedById").toString());
if(CreatedByIdMap.get("new_id") != null && StringUtils.isNotEmpty(CreatedByIdMap.get("new_id").toString())){
account.setField("CreatedById", CreatedByIdMap.get("new_id"));
}
continue;
}
if (dataField.getIsCreateable() != null && dataField.getIsCreateable() && !dataField.getIsNillable() && !dataField.getIsDefaultedOnCreate()) {
if ("reference".equals(dataField.getSfType())){
log.info("----------" + dataField.getField() + " ------------" + dataField.getSfType());
String reference = dataField.getReferenceTo();
if (StringUtils.isEmpty(reference)){
if ("ParentId".equals(dataField.getField())){
reference = data.get(j-1).get("Parent_Type")!=null?data.get(j-1).get("Parent_Type").toString():null;
}else if ("RelationId".equals(dataField.getField())){
reference = data.get(j-1).get("Relation_Type__c")!=null?data.get(j-1).get("Relation_Type__c").toString():null;
}else if ("WhatId".equals(dataField.getField())){
reference = data.get(j-1).get("WhatId_Type__c")!=null?data.get(j-1).get("WhatId_Type__c").toString():null;
}else if ("WhoId".equals(dataField.getField())){
reference = data.get(j-1).get("WhoId_Type__c")!=null?data.get(j-1).get("WhoId_Type__c").toString():null;
}
}
if (reference == null){
continue;
}
List<Map<String, Object>> referenceMap = customMapper.list("new_id", reference, "new_id is not null limit 1");
if (referenceMap.isEmpty()){
QueryWrapper<DataObject> maxIndex = new QueryWrapper<>();
maxIndex.select("IFNULL(max(data_index),0) as data_index");
maxIndex.ne("name", api);
Map<String, Object> map = dataObjectService.getMap(maxIndex);
//如果必填lookup字段没有值，跳过
update.setDataIndex(Integer.parseInt(map.get("data_index").toString()) + 1);
dataObjectService.updateById(update);
return;
}else{
account.setField(dataField.getField(), referenceMap.get(0).get("new_id"));
continue;
}
}
if ("picklist".equals(dataField.getSfType())){
List<Map<String, Object>> pickList = customMapper.list("value", "data_picklist", "api = '"+api+"' and field = '"+dataField.getField()+"' limit 1");
account.setField(dataField.getField(), pickList.get(0).get("value"));
try {
List<Map<String, Object>> data = customMapper.list("*", api, "new_id is null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "' limit 200");
int size = data.size();
log.info("执行api:{}, 执行page:{}, 执行size:{}", api, i+1, size);
SObject[] accounts = new SObject[size];
String[] ids = new String[size];
for (int j = 1; j <= size; j++) {
SObject account = new SObject();
account.setType(api);
//找出sf对象必填字段，并且给默认值
for (DataField dataField : list) {
if ("OwnerId".equals(dataField.getField()) || "Owner_Type".equals(dataField.getField())
|| "Id".equals(dataField.getField())){
continue;
}
account.setField(dataField.getField(), DataUtil.fieldTypeToSf(dataField));
if ("CreatedDate".equals(dataField.getField()) && dataField.getIsCreateable()){
//object类型转Date类型
Date date;
try {
date = sdf.parse(String.valueOf(data.get(j - 1).get("CreatedDate")));
}catch (ParseException e){
//解决当时间秒为0时，转换秒精度丢失问题
date = sdf.parse(data.get(j - 1).get("CreatedDate")+":00");
}
Calendar calendar = Calendar.getInstance();
calendar.setTime(date);
account.setField("CreatedDate", calendar);
continue;
}
if ("CreatedById".equals(dataField.getField()) && dataField.getIsCreateable()){
Map<String, Object> CreatedByIdMap = customMapper.getById("new_id", "User", data.get(j-1).get("CreatedById").toString());
if(CreatedByIdMap.get("new_id") != null && StringUtils.isNotEmpty(CreatedByIdMap.get("new_id").toString())){
account.setField("CreatedById", CreatedByIdMap.get("new_id"));
}
continue;
}
if (dataField.getIsCreateable() != null && dataField.getIsCreateable() && !dataField.getIsNillable() && !dataField.getIsDefaultedOnCreate()) {
if ("reference".equals(dataField.getSfType())){
//引用类型
String reference = dataField.getReferenceTo();
//引用类型字段
String linkfield = fieldMap.get(dataField.getApi());
if (StringUtils.isNotBlank(linkfield)){
reference = data.get(j-1).get(linkfield)!=null?data.get(j-1).get(linkfield).toString():null;
}
if (reference == null){
continue;
}
log.info("----------" + dataField.getField() + "的引用类型 ------------" + reference);
List<Map<String, Object>> referenceMap = customMapper.list("new_id", reference, "new_id is not null limit 1");
if (referenceMap.isEmpty()){
QueryWrapper<DataObject> maxIndex = new QueryWrapper<>();
maxIndex.select("IFNULL(max(data_index),0) as data_index");
maxIndex.ne("name", api);
Map<String, Object> map = dataObjectService.getMap(maxIndex);
//如果必填lookup字段没有值，跳过
update.setDataIndex(Integer.parseInt(map.get("data_index").toString()) + 1);
dataObjectService.updateById(update);
return;
}else{
account.setField(dataField.getField(), referenceMap.get(0).get("new_id"));
continue;
}
}
if ("picklist".equals(dataField.getSfType())){
List<Map<String, Object>> pickList = customMapper.list("value", "data_picklist", "api = '"+api+"' and field = '"+dataField.getField()+"' limit 1");
account.setField(dataField.getField(), pickList.get(0).get("value"));
continue;
}
account.setField(dataField.getField(), DataUtil.fieldTypeToSf(dataField));
}
}
if (api.equals("vlink__Wechat_User__c")){
List<Map<String, Object>> maps = customMapper.list("new_id", "vlink__Wechat_Account__c", "new_id is not null and id = '" + data.get(j - 1).get("vlink__Wechat_Account__c") + "' limit 1");
if (!maps.isEmpty()){
Map<String, Object> referenceMap = maps.get(0);
account.setField("vlink__Wechat_Account__c", referenceMap.get("new_id"));
}
}
ids[j-1] = data.get(j-1).get("Id").toString();
accounts[j-1] = account;
if (i*200+j == count){
break;
}
}
if (api.equals("vlink__Wechat_User__c")){
List<Map<String, Object>> maps = customMapper.list("new_id", "vlink__Wechat_Account__c", "new_id is not null and id = '" + data.get(j - 1).get("vlink__Wechat_Account__c") + "' limit 1");
if (!maps.isEmpty()){
Map<String, Object> referenceMap = maps.get(0);
account.setField("vlink__Wechat_Account__c", referenceMap.get("new_id"));
}
}
ids[j-1] = data.get(j-1).get("Id").toString();
accounts[j-1] = account;
if (i*200+j == count){
break;
}
}
try {
if (infoFlag != null && "1".equals(infoFlag.get("value"))){
printAccountsDetails(accounts, list);
}

16
src/main/java/com/celnet/datadump/service/impl/LinkConfigServiceImpl.java Normal file
View File

@ -0,0 +1,16 @@
package com.celnet.datadump.service.impl;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.celnet.datadump.entity.DataLog;
import com.celnet.datadump.entity.LinkConfig;
import com.celnet.datadump.mapper.DataLogMapper;
import com.celnet.datadump.mapper.LinkConfigMapper;
import com.celnet.datadump.service.DataLogService;
import com.celnet.datadump.service.LinkConfigService;
import org.springframework.stereotype.Service;
@Service
public class LinkConfigServiceImpl extends ServiceImpl<LinkConfigMapper, LinkConfig> implements LinkConfigService {
}

3
src/main/java/com/celnet/datadump/util/DataUtil.java
View File

@ -1,5 +1,6 @@
package com.celnet.datadump.util;
import cn.hutool.core.lang.UUID;
import com.alibaba.fastjson2.JSONArray;
import com.alibaba.fastjson2.JSONObject;
import com.celnet.datadump.entity.DataField;
@ -388,7 +389,7 @@ public class DataUtil {
case "datetime":
return date;
default:
return "初始"+System.currentTimeMillis();
return "初始"+ UUID.randomUUID();
}
}

4
src/main/resources/mapper/CustomMapper.xml
View File

@ -28,6 +28,10 @@
) COMMENT = '${tableComment}';
</update>
<update id="createField">
ALTER TABLE `${tableName}` ADD COLUMN `${fieldName}` varchar(255)
</update>
<select id="getIds" resultType="String">
SELECT
id