kris/data-dump
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

【feat】增加org配置校验任务

Browse Source
This commit is contained in:
Kris 2025-07-09 16:04:55 +08:00
parent 2253747562
commit 6680beb43a
7 changed files with 49 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
src/main/java/com/celnet/datadump/controller/OrgConfigController.java
View File

@ -1,15 +1,21 @@
package com.celnet.datadump.controller; package com.celnet.datadump.controller;
import com.alibaba.fastjson.JSON;
import com.baomidou.mybatisplus.core.toolkit.Wrappers; import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.celnet.datadump.annotation.LogServiceAnnotation;
import com.celnet.datadump.constant.OperateTypeConstant;
import com.celnet.datadump.entity.OrgConfig; import com.celnet.datadump.entity.OrgConfig;
import com.celnet.datadump.entity.SystemConfig; import com.celnet.datadump.entity.SystemConfig;
import com.celnet.datadump.global.Result; import com.celnet.datadump.global.Result;
import com.celnet.datadump.param.SalesforceParam;
import com.celnet.datadump.service.OrgConfigService; import com.celnet.datadump.service.OrgConfigService;
import com.celnet.datadump.service.SystemConfigService; import com.celnet.datadump.service.SystemConfigService;
import com.xxl.job.core.biz.model.ReturnT;
import io.swagger.annotations.Api; import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiOperation;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*; import org.springframework.web.bind.annotation.*;
@ -53,5 +59,14 @@ public class OrgConfigController {
public Result delete(@RequestBody OrgConfig orgConfig) { public Result delete(@RequestBody OrgConfig orgConfig) {
return Result.success(orgConfigService.removeById(orgConfig)); return Result.success(orgConfigService.removeById(orgConfig));
} }
@PostMapping("/getOrgConfigJob")
@ApiOperation("校验org配置")
@LogServiceAnnotation(operateType = OperateTypeConstant.TYPE_SELECT, remark = "数据更新同步（新）")
public ReturnT<String> getOrgConfigJob(String paramStr) throws Exception {
orgConfigService.verifyOrgConfig();
return ReturnT.SUCCESS;
}
} }

2
src/main/java/com/celnet/datadump/job/OrgConfigJob.java
View File

@ -21,6 +21,6 @@ public class OrgConfigJob {
orgConfigService.verifyOrgConfig(); orgConfigService.verifyOrgConfig();
return null; return ReturnT.SUCCESS;
} }
} }

8
src/main/java/com/celnet/datadump/service/impl/CommonServiceImpl.java
View File

@ -581,7 +581,8 @@ public class CommonServiceImpl implements CommonService {
} }
fields.add(field.getName()); fields.add(field.getName());
} }
if ("Attachment".equals(api) || "FeedItem".equals(api)) { if ("Attachment".equals(api) || "FeedItem".equals(api)
|| "FeedComment".equals(api) || "Note".equals(api)) {
fields.add("Parent.type"); fields.add("Parent.type");
} }
if ("Task".equals(api) || "Event".equals(api)) { if ("Task".equals(api) || "Event".equals(api)) {
@ -897,6 +898,9 @@ public class CommonServiceImpl implements CommonService {
} }
} }
dataField.setReferenceTo(join); dataField.setReferenceTo(join);
if ("WhoId".equals(field.getName()) || "WhatId".equals(field.getName())){
dataField.setReferenceTo(null);
}
fieldList.add(dataField); fieldList.add(dataField);
fields.add(field.getName()); fields.add(field.getName());
} }
@ -1003,7 +1007,7 @@ public class CommonServiceImpl implements CommonService {
} }
if ("Attachment".equals(apiName) || "FeedComment".equals(apiName) if ("Attachment".equals(apiName) || "FeedComment".equals(apiName)
|| "FeedItem".equals(apiName)){ || "FeedItem".equals(apiName) || "Note".equals(apiName)){
//文档关联表新增关联对象字段 //文档关联表新增关联对象字段
Map<String, Object> LinkedMap = Maps.newHashMap(); Map<String, Object> LinkedMap = Maps.newHashMap();
LinkedMap.put("type", "varchar(255)"); LinkedMap.put("type", "varchar(255)");

13
src/main/java/com/celnet/datadump/service/impl/DataImportBatchServiceImpl.java
View File

@ -255,15 +255,18 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
String reference = dataField.getReferenceTo(); String reference = dataField.getReferenceTo();
if (StringUtils.isEmpty(reference)){ if (StringUtils.isEmpty(reference)){
if ("ParentId".equals(dataField.getField())){ if ("ParentId".equals(dataField.getField())){
reference = data.get(j-1).get("Parent_Type").toString(); reference = data.get(j-1).get("Parent_Type")!=null?data.get(j-1).get("Parent_Type").toString():null;
}else if ("RelationId".equals(dataField.getField())){ }else if ("RelationId".equals(dataField.getField())){
reference = data.get(j-1).get("Relation_Type__c").toString(); reference = data.get(j-1).get("Relation_Type__c")!=null?data.get(j-1).get("Relation_Type__c").toString():null;
}else if ("WhatId".equals(dataField.getField())){ }else if ("WhatId".equals(dataField.getField())){
reference = data.get(j-1).get("WhatId_Type__c").toString(); reference = data.get(j-1).get("WhatId_Type__c")!=null?data.get(j-1).get("WhatId_Type__c").toString():null;
}else if ("WhoId".equals(dataField.getField())){ }else if ("WhoId".equals(dataField.getField())){
reference = data.get(j-1).get("WhoId_Type__c").toString(); reference = data.get(j-1).get("WhoId_Type__c")!=null?data.get(j-1).get("WhoId_Type__c").toString():null;
} }
} }
if (reference == null){
continue;
}
List<Map<String, Object>> referenceMap = customMapper.list("new_id", reference, "new_id is not null limit 1"); List<Map<String, Object>> referenceMap = customMapper.list("new_id", reference, "new_id is not null limit 1");
if (referenceMap.isEmpty()){ if (referenceMap.isEmpty()){
QueryWrapper<DataObject> maxIndex = new QueryWrapper<>(); QueryWrapper<DataObject> maxIndex = new QueryWrapper<>();
@ -539,6 +542,8 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
account.put("Id",String.valueOf(map.get("new_id"))); account.put("Id",String.valueOf(map.get("new_id")));
} else if (!DataUtil.isUpdate(field) || (dataField.getIsCreateable() != null && !dataField.getIsCreateable())) { } else if (!DataUtil.isUpdate(field) || (dataField.getIsCreateable() != null && !dataField.getIsCreateable())) {
continue; continue;
} else if ( "WhoId".equals(field) ||"WhatId".equals(field)){
continue;
} else if (StringUtils.isNotBlank(reference_to) && !"data_picklist".equals(reference_to)) { } else if (StringUtils.isNotBlank(reference_to) && !"data_picklist".equals(reference_to)) {
if ( StringUtils.isNotEmpty(String.valueOf(map.get(field))) && !"OwnerId".equals(field) if ( StringUtils.isNotEmpty(String.valueOf(map.get(field))) && !"OwnerId".equals(field)
&& !"Owner_Type".equals(field)) { && !"Owner_Type".equals(field)) {

10
src/main/java/com/celnet/datadump/service/impl/DataImportNewServiceImpl.java
View File

@ -191,7 +191,7 @@ public class DataImportNewServiceImpl implements DataImportNewService {
//表内数据总量 //表内数据总量
Integer count = customMapper.countBySQL(api, "where new_id is null and IsPersonAccount = 1 and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "'"); Integer count = customMapper.countBySQL(api, "where new_id is null and IsPersonAccount = 1 and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "'");
log.error("总Insert数据 count:{}；-开始时间:{}；-结束时间:{}；-api:{}；", count, beginDateStr, endDateStr, api); log.info("总Insert数据 count:{}；-开始时间:{}；-结束时间:{}；-api:{}；", count, beginDateStr, endDateStr, api);
if (count == 0) { if (count == 0) {
return; return;
} }
@ -717,7 +717,7 @@ public class DataImportNewServiceImpl implements DataImportNewService {
} }
//表内数据总量 //表内数据总量
Integer count = customMapper.countBySQL(api, sql); Integer count = customMapper.countBySQL(api, sql);
log.error("总Update数据 count:{}；-开始时间:{}；-结束时间:{}；-api:{}；", count, beginDateStr, endDateStr, api); log.info("总Update数据 count:{}；-开始时间:{}；-结束时间:{}；-api:{}；", count, beginDateStr, endDateStr, api);
if(count == 0){ if(count == 0){
return; return;
@ -748,10 +748,12 @@ public class DataImportNewServiceImpl implements DataImportNewService {
account.setId(String.valueOf(map.get("new_id"))); account.setId(String.valueOf(map.get("new_id")));
} else if (!DataUtil.isUpdate(field) || (dataField.getIsCreateable() != null && !dataField.getIsCreateable())) { } else if (!DataUtil.isUpdate(field) || (dataField.getIsCreateable() != null && !dataField.getIsCreateable())) {
continue; continue;
} else if ( "WhoId".equals(field) ||"WhatId".equals(field)){
continue;
} else if (StringUtils.isNotBlank(reference_to) && !"data_picklist".equals(reference_to)) { } else if (StringUtils.isNotBlank(reference_to) && !"data_picklist".equals(reference_to)) {
if (!"null".equals(value) && StringUtils.isNotEmpty(value) && (!"OwnerId".equals(field) if (!"null".equals(value) && StringUtils.isNotEmpty(value) && !"OwnerId".equals(field)
&& !"Owner_Type".equals(field))) { && !"Owner_Type".equals(field) ) {
//判断reference_to内是否包含User字符串 //判断reference_to内是否包含User字符串
if (reference_to.contains("User")) { if (reference_to.contains("User")) {
reference_to = "User"; reference_to = "User";

18
src/main/java/com/celnet/datadump/service/impl/DataImportServiceImpl.java
View File

@ -200,6 +200,7 @@ public class DataImportServiceImpl implements DataImportService {
} }
for (DataObject dataObject : dataObjects) { for (DataObject dataObject : dataObjects) {
DataObject update = new DataObject(); DataObject update = new DataObject();
log.info("autoImmigration api: {}", dataObject.getName());
TimeUnit.MILLISECONDS.sleep(1); TimeUnit.MILLISECONDS.sleep(1);
try { try {
String api = dataObject.getName(); String api = dataObject.getName();
@ -243,10 +244,8 @@ public class DataImportServiceImpl implements DataImportService {
} catch (Throwable e) { } catch (Throwable e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} finally { } finally {
if (dataObject != null) { update.setDataLock(0);
update.setDataLock(0); dataObjectService.updateById(update);
dataObjectService.updateById(update);
}
} }
} }
// 等待当前所有线程执行完成 // 等待当前所有线程执行完成
@ -323,15 +322,18 @@ public class DataImportServiceImpl implements DataImportService {
String reference = dataField.getReferenceTo(); String reference = dataField.getReferenceTo();
if (StringUtils.isEmpty(reference)){ if (StringUtils.isEmpty(reference)){
if ("ParentId".equals(dataField.getField())){ if ("ParentId".equals(dataField.getField())){
reference = data.get(j-1).get("Parent_Type").toString(); reference = data.get(j-1).get("Parent_Type")!=null?data.get(j-1).get("Parent_Type").toString():null;
}else if ("RelationId".equals(dataField.getField())){ }else if ("RelationId".equals(dataField.getField())){
reference = data.get(j-1).get("Relation_Type__c").toString(); reference = data.get(j-1).get("Relation_Type__c")!=null?data.get(j-1).get("Relation_Type__c").toString():null;
}else if ("WhatId".equals(dataField.getField())){ }else if ("WhatId".equals(dataField.getField())){
reference = data.get(j-1).get("WhatId_Type__c").toString(); reference = data.get(j-1).get("WhatId_Type__c")!=null?data.get(j-1).get("WhatId_Type__c").toString():null;
}else if ("WhoId".equals(dataField.getField())){ }else if ("WhoId".equals(dataField.getField())){
reference = data.get(j-1).get("WhoId_Type__c").toString(); reference = data.get(j-1).get("WhoId_Type__c")!=null?data.get(j-1).get("WhoId_Type__c").toString():null;
} }
} }
if (reference == null){
continue;
}
List<Map<String, Object>> referenceMap = customMapper.list("new_id", reference, "new_id is not null limit 1"); List<Map<String, Object>> referenceMap = customMapper.list("new_id", reference, "new_id is not null limit 1");
if (referenceMap.isEmpty()){ if (referenceMap.isEmpty()){
QueryWrapper<DataObject> maxIndex = new QueryWrapper<>(); QueryWrapper<DataObject> maxIndex = new QueryWrapper<>();

16
src/main/java/com/celnet/datadump/service/impl/OrgConfigServiceImpl.java
View File

@ -67,14 +67,8 @@ public class OrgConfigServiceImpl extends ServiceImpl<OrgConfigMapper, OrgConfig
config.setReadTimeout(60 * 60 * 1000); config.setReadTimeout(60 * 60 * 1000);
PartnerConnection connection = new PartnerConnection(config); PartnerConnection connection = new PartnerConnection(config);
String orgId = connection.getUserInfo().getOrganizationId(); String orgId = connection.getUserInfo().getOrganizationId();
if (StringUtils.isEmpty(orgId)){
String message = "源ORG连接配置错误！";
String format = String.format("ORG连接异常！, \ncause:\n%s", message);
EmailUtil.send("DataDump ERROR", format);
flag = false;
}
} catch (ConnectionException e) { } catch (ConnectionException e) {
String message = "源ORG连接配置错误！"; String message = "源ORG连接配置错误！,\n地址：" + sourceOrgUrl;
String format = String.format("ORG连接异常！, \ncause:\n%s", message); String format = String.format("ORG连接异常！, \ncause:\n%s", message);
EmailUtil.send("DataDump ERROR", format); EmailUtil.send("DataDump ERROR", format);
log.error("exception message", e); log.error("exception message", e);
@ -109,14 +103,8 @@ public class OrgConfigServiceImpl extends ServiceImpl<OrgConfigMapper, OrgConfig
config.setReadTimeout(60 * 60 * 1000); config.setReadTimeout(60 * 60 * 1000);
PartnerConnection connection = new PartnerConnection(config); PartnerConnection connection = new PartnerConnection(config);
String orgId = connection.getUserInfo().getOrganizationId(); String orgId = connection.getUserInfo().getOrganizationId();
if (StringUtils.isEmpty(orgId)){
String message = "目标ORG连接配置错误！";
String format = String.format("ORG连接异常！, \ncause:\n%s", message);
EmailUtil.send("DataDump ERROR", format);
flag = false;
}
} catch (ConnectionException e) { } catch (ConnectionException e) {
String message = "目标ORG连接配置错误！"; String message = "目标ORG连接配置错误！,\n地址：" + targetOrgUrl;
String format = String.format("ORG连接异常！, \ncause:\n%s", message); String format = String.format("ORG连接异常！, \ncause:\n%s", message);
EmailUtil.send("DataDump ERROR", format); EmailUtil.send("DataDump ERROR", format);
log.error("exception message", e); log.error("exception message", e);