kris/data-dump
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

【feat】 优化RecorceType反写，优化数据质量校验增加忽略开始i时间

Browse Source 
(cherry picked from commit 46bb968e11)
This commit is contained in:
Kris 2025-10-14 10:15:04 +08:00
parent aca16ebe7b
commit 51dd28b1fc
4 changed files with 33 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/com/celnet/datadump/job/DataDumpNewJob.java
View File

@ -204,7 +204,7 @@ public class DataDumpNewJob {
dataImportNewService.getNewIdByField(api,"DeveloperName,Type");
break;
case "RecordType":
dataImportNewService.getNewIdByField(api,"DeveloperName,SObjectType");
dataImportNewService.getNewIdByField(api,"DeveloperName,SobjectType");
break;
case "AccountContactRelation":
dataImportNewService.getNewIdByField(api,"AccountId,ContactId");

5
src/main/java/com/celnet/datadump/param/DataVerifyParam.java
View File

@ -56,4 +56,9 @@ public class DataVerifyParam {
@ApiModelProperty(value = "结束时间", hidden = true)
private Date endDate;
@ApiModelProperty(value = "忽略开始时间", hidden = true)
private Date ignoreBeginDate;
}

2
src/main/java/com/celnet/datadump/service/impl/DataImportServiceImpl.java
View File

@ -459,7 +459,7 @@ public class DataImportServiceImpl implements DataImportService {
} catch (InterruptedException interruptedException){
return;
} catch (Exception e) {
log.info(JSON.toJSONString(e.getMessage()));
log.info(JSON.toJSONString(e.getMessage()),e);
}
}

35
src/main/java/com/celnet/datadump/service/impl/DataVerifyServiceImpl.java
View File

@ -421,7 +421,7 @@ public class DataVerifyServiceImpl implements DataVerifyService {
log.info("开始时间: {}", new Date(startTime));
try {
performQualityCheckAndWriteToExcel(dataObjects, sourceConnect, targetConnect, filePath, param.getSendEmail(),param.getType());
performQualityCheckAndWriteToExcel(dataObjects, sourceConnect, targetConnect, filePath, param.getSendEmail(),param.getType(),param.getIgnoreBeginDate());
long endTime = System.currentTimeMillis();
log.info("数据质量校验执行完成，耗时: {} ms", endTime - startTime);
} catch (Exception e) {
@ -500,7 +500,7 @@ public class DataVerifyServiceImpl implements DataVerifyService {
private void performQualityCheckAndWriteToExcel(List<DataObject> dataObjects,
PartnerConnection sourceConn,
PartnerConnection targetConn,
String filePath, boolean sendEmail ,Integer type) throws Exception {
String filePath, boolean sendEmail ,Integer type, Date ignoreBeginDate) throws Exception {
try (ExcelWriter excelWriter = EasyExcel.write(filePath).build()) {
// 1. 创建汇总 Sheet
WriteSheet summarySheet = createSummarySheet(excelWriter);
@ -510,7 +510,7 @@ public class DataVerifyServiceImpl implements DataVerifyService {
List<Future<?>> futures = Lists.newArrayList();
dataObjects.forEach(dataObject -> {
Future<?> future = salesforceExecutor.execute(() -> {
processDataObject(dataObject, excelWriter, summarySheet, fieldTypeSheet, sourceConn, targetConn, sendEmail, type);
processDataObject(dataObject, excelWriter, summarySheet, fieldTypeSheet, sourceConn, targetConn, sendEmail, type, ignoreBeginDate);
}, 1, 1);
futures.add(future);
});
@ -560,7 +560,7 @@ public class DataVerifyServiceImpl implements DataVerifyService {
*/
private void processDataObject(DataObject dataObject, ExcelWriter excelWriter, WriteSheet summarySheet,
WriteSheet fieldTypeSheet, PartnerConnection sourceConn,
PartnerConnection targetConn, boolean sendEmail, Integer type) {
PartnerConnection targetConn, boolean sendEmail, Integer type, Date ignoreBeginDate) {
// 1. 准备对象相关信息
String objectApi = dataObject.getName();
List<DataField> fields = getVerifiableFields(objectApi);
@ -574,7 +574,6 @@ public class DataVerifyServiceImpl implements DataVerifyService {
String qualityCheckFilePath = generateObjectQualityCheckFilePath(objectApi);
// 3. 处理每个批次
List<DataBatch> batches = getDataBatches(objectApi);
log.info("对象: {} 待同步的批次数量: {}", objectApi, batches.size());
@ -587,7 +586,7 @@ public class DataVerifyServiceImpl implements DataVerifyService {
String batchQualityCheckFilePath = generateObjectQualityCheckFilePath(objectApi + "_" + batch.getId());
try (ExcelWriter objectWriter = EasyExcel.write(batchQualityCheckFilePath).build()) {
WriteSheet detailSheet = createDetailSheet(excelWriter, dataObject);
batchHasError = processDataBatch(batch, objectApi, fields, excelWriter, objectWriter, summarySheet, detailSheet, sourceConn, targetConn, fieldMap) || batchHasError;
batchHasError = processDataBatch(batch, objectApi, fields, excelWriter, objectWriter, summarySheet, detailSheet, sourceConn, targetConn, fieldMap, ignoreBeginDate) || batchHasError;
}catch (Exception e){
log.error("API={},批次ID={}处理异常：{}",objectApi,batch.getId(), e.getMessage(),e);
}
@ -609,7 +608,7 @@ public class DataVerifyServiceImpl implements DataVerifyService {
WriteSheet detailSheet = createDetailSheet(excelWriter, dataObject);
for (DataBatch batch : batches) {
hasError = processDataBatch(batch, objectApi, fields, excelWriter, objectWriter, summarySheet, detailSheet, sourceConn, targetConn, fieldMap) || hasError;
hasError = processDataBatch(batch, objectApi, fields, excelWriter, objectWriter, summarySheet, detailSheet, sourceConn, targetConn, fieldMap, ignoreBeginDate) || hasError;
}
}catch (Exception e){
log.error("API={},处理异常：{}",objectApi, e.getMessage(),e);
@ -722,10 +721,11 @@ public class DataVerifyServiceImpl implements DataVerifyService {
*/
private boolean processDataBatch(DataBatch batch, String objectApi, List<DataField> fields,
ExcelWriter excelWriter,ExcelWriter objectWriter, WriteSheet summarySheet, WriteSheet detailSheet,
PartnerConnection sourceConn, PartnerConnection targetConn,Map<String, Map<String, Field>> fieldMap) {
PartnerConnection sourceConn, PartnerConnection targetConn,Map<String, Map<String, Field>> fieldMap, Date ignoreBeginDate) {
String startDate = DateFormatUtils.format(batch.getSyncStartDate(), "yyyy-MM-dd HH:mm:ss");
String endDate = DateFormatUtils.format(batch.getSyncEndDate(), "yyyy-MM-dd HH:mm:ss");
log.info("开始处理数据批次: 批次ID={} 批次总量(大于十万只校验十万)={} 批次开始时间={} 批次结束时间={} 对象API={}", batch.getId(),batch.getFirstSfNum(),startDate,endDate, objectApi);
log.info("开始处理数据批次: 批次ID={} 批次总量(大于十万只校验十万)={} 批次开始时间={} 批次结束时间={} 对象API={} 忽略开始时间={}", batch.getId(),batch.getFirstSfNum(),startDate,endDate, objectApi, ignoreBeginDate);
// 分页处理记录
int page = 0;
@ -832,6 +832,23 @@ public class DataVerifyServiceImpl implements DataVerifyService {
continue;
}
if (ignoreBeginDate != null) {
Object lastModifiedObj = sourceRecord.getField("LastModifiedDate");
if (lastModifiedObj != null) {
try {
// 解析Salesforce返回的日期格式 2025-10-12T03:27:54.000Z
String lastModifiedStr = lastModifiedObj.toString();
Date lastModifiedDate = org.apache.commons.lang3.time.DateUtils.parseDate(lastModifiedStr, "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
if (lastModifiedDate.before(ignoreBeginDate)) {
continue;
}
} catch (Exception e) {
log.warn("解析LastModifiedDate失败: {}", lastModifiedObj, e);
}
}
}
// 比对字段值
for (DataField field : fields) {
String fieldName = field.getField();