From 46bb968e1108d20b0f2954f9abe58f25938a26f7 Mon Sep 17 00:00:00 2001 From: Kris <2893855659@qq.com> Date: Tue, 14 Oct 2025 10:15:04 +0800 Subject: [PATCH] =?UTF-8?q?=E3=80=90feat=E3=80=91=20=E4=BC=98=E5=8C=96Reco?= =?UTF-8?q?rceType=E5=8F=8D=E5=86=99=EF=BC=8C=E4=BC=98=E5=8C=96=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E8=B4=A8=E9=87=8F=E6=A0=A1=E9=AA=8C=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E5=BF=BD=E7=95=A5=E5=BC=80=E5=A7=8Bi=E6=97=B6=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../celnet/datadump/job/DataDumpNewJob.java | 2 +- .../datadump/param/DataVerifyParam.java | 5 +++ .../service/impl/DataImportServiceImpl.java | 2 +- .../service/impl/DataVerifyServiceImpl.java | 35 ++++++++++++++----- 4 files changed, 33 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/celnet/datadump/job/DataDumpNewJob.java b/src/main/java/com/celnet/datadump/job/DataDumpNewJob.java index 7c3b106..7ee9b08 100644 --- a/src/main/java/com/celnet/datadump/job/DataDumpNewJob.java +++ b/src/main/java/com/celnet/datadump/job/DataDumpNewJob.java @@ -204,7 +204,7 @@ public class DataDumpNewJob { dataImportNewService.getNewIdByField(api,"DeveloperName,Type"); break; case "RecordType": - dataImportNewService.getNewIdByField(api,"DeveloperName,SObjectType"); + dataImportNewService.getNewIdByField(api,"DeveloperName,SobjectType"); break; case "AccountContactRelation": dataImportNewService.getNewIdByField(api,"AccountId,ContactId"); diff --git a/src/main/java/com/celnet/datadump/param/DataVerifyParam.java b/src/main/java/com/celnet/datadump/param/DataVerifyParam.java index 258890c..42d8594 100644 --- a/src/main/java/com/celnet/datadump/param/DataVerifyParam.java +++ b/src/main/java/com/celnet/datadump/param/DataVerifyParam.java @@ -56,4 +56,9 @@ public class DataVerifyParam { @ApiModelProperty(value = "结束时间", hidden = true) private Date endDate; + @ApiModelProperty(value = "忽略开始时间", hidden = true) + private Date ignoreBeginDate; + + + } diff --git a/src/main/java/com/celnet/datadump/service/impl/DataImportServiceImpl.java b/src/main/java/com/celnet/datadump/service/impl/DataImportServiceImpl.java index f381c0d..8c71013 100644 --- a/src/main/java/com/celnet/datadump/service/impl/DataImportServiceImpl.java +++ b/src/main/java/com/celnet/datadump/service/impl/DataImportServiceImpl.java @@ -459,7 +459,7 @@ public class DataImportServiceImpl implements DataImportService { } catch (InterruptedException interruptedException){ return; } catch (Exception e) { - log.info(JSON.toJSONString(e.getMessage())); + log.info(JSON.toJSONString(e.getMessage()),e); } } diff --git a/src/main/java/com/celnet/datadump/service/impl/DataVerifyServiceImpl.java b/src/main/java/com/celnet/datadump/service/impl/DataVerifyServiceImpl.java index 41e9cf5..9fd54b3 100644 --- a/src/main/java/com/celnet/datadump/service/impl/DataVerifyServiceImpl.java +++ b/src/main/java/com/celnet/datadump/service/impl/DataVerifyServiceImpl.java @@ -421,7 +421,7 @@ public class DataVerifyServiceImpl implements DataVerifyService { log.info("开始时间: {}", new Date(startTime)); try { - performQualityCheckAndWriteToExcel(dataObjects, sourceConnect, targetConnect, filePath, param.getSendEmail(),param.getType()); + performQualityCheckAndWriteToExcel(dataObjects, sourceConnect, targetConnect, filePath, param.getSendEmail(),param.getType(),param.getIgnoreBeginDate()); long endTime = System.currentTimeMillis(); log.info("数据质量校验执行完成，耗时: {} ms", endTime - startTime); } catch (Exception e) { @@ -500,7 +500,7 @@ public class DataVerifyServiceImpl implements DataVerifyService { private void performQualityCheckAndWriteToExcel(List dataObjects, PartnerConnection sourceConn, PartnerConnection targetConn, - String filePath, boolean sendEmail ,Integer type) throws Exception { + String filePath, boolean sendEmail ,Integer type, Date ignoreBeginDate) throws Exception { try (ExcelWriter excelWriter = EasyExcel.write(filePath).build()) { // 1. 创建汇总 Sheet WriteSheet summarySheet = createSummarySheet(excelWriter); @@ -510,7 +510,7 @@ public class DataVerifyServiceImpl implements DataVerifyService { List> futures = Lists.newArrayList(); dataObjects.forEach(dataObject -> { Future future = salesforceExecutor.execute(() -> { - processDataObject(dataObject, excelWriter, summarySheet, fieldTypeSheet, sourceConn, targetConn, sendEmail, type); + processDataObject(dataObject, excelWriter, summarySheet, fieldTypeSheet, sourceConn, targetConn, sendEmail, type, ignoreBeginDate); }, 1, 1); futures.add(future); }); @@ -560,7 +560,7 @@ public class DataVerifyServiceImpl implements DataVerifyService { */ private void processDataObject(DataObject dataObject, ExcelWriter excelWriter, WriteSheet summarySheet, WriteSheet fieldTypeSheet, PartnerConnection sourceConn, - PartnerConnection targetConn, boolean sendEmail, Integer type) { + PartnerConnection targetConn, boolean sendEmail, Integer type, Date ignoreBeginDate) { // 1. 准备对象相关信息 String objectApi = dataObject.getName(); List fields = getVerifiableFields(objectApi); @@ -574,7 +574,6 @@ public class DataVerifyServiceImpl implements DataVerifyService { String qualityCheckFilePath = generateObjectQualityCheckFilePath(objectApi); - // 3. 处理每个批次 List batches = getDataBatches(objectApi); log.info("对象: {} 待同步的批次数量: {}", objectApi, batches.size()); @@ -587,7 +586,7 @@ public class DataVerifyServiceImpl implements DataVerifyService { String batchQualityCheckFilePath = generateObjectQualityCheckFilePath(objectApi + "_" + batch.getId()); try (ExcelWriter objectWriter = EasyExcel.write(batchQualityCheckFilePath).build()) { WriteSheet detailSheet = createDetailSheet(excelWriter, dataObject); - batchHasError = processDataBatch(batch, objectApi, fields, excelWriter, objectWriter, summarySheet, detailSheet, sourceConn, targetConn, fieldMap) || batchHasError; + batchHasError = processDataBatch(batch, objectApi, fields, excelWriter, objectWriter, summarySheet, detailSheet, sourceConn, targetConn, fieldMap, ignoreBeginDate) || batchHasError; }catch (Exception e){ log.error("API={},批次ID={}处理异常：{}",objectApi,batch.getId(), e.getMessage(),e); } @@ -609,7 +608,7 @@ public class DataVerifyServiceImpl implements DataVerifyService { WriteSheet detailSheet = createDetailSheet(excelWriter, dataObject); for (DataBatch batch : batches) { - hasError = processDataBatch(batch, objectApi, fields, excelWriter, objectWriter, summarySheet, detailSheet, sourceConn, targetConn, fieldMap) || hasError; + hasError = processDataBatch(batch, objectApi, fields, excelWriter, objectWriter, summarySheet, detailSheet, sourceConn, targetConn, fieldMap, ignoreBeginDate) || hasError; } }catch (Exception e){ log.error("API={},处理异常：{}",objectApi, e.getMessage(),e); @@ -722,10 +721,11 @@ public class DataVerifyServiceImpl implements DataVerifyService { */ private boolean processDataBatch(DataBatch batch, String objectApi, List fields, ExcelWriter excelWriter,ExcelWriter objectWriter, WriteSheet summarySheet, WriteSheet detailSheet, - PartnerConnection sourceConn, PartnerConnection targetConn,Map> fieldMap) { + PartnerConnection sourceConn, PartnerConnection targetConn,Map> fieldMap, Date ignoreBeginDate) { String startDate = DateFormatUtils.format(batch.getSyncStartDate(), "yyyy-MM-dd HH:mm:ss"); String endDate = DateFormatUtils.format(batch.getSyncEndDate(), "yyyy-MM-dd HH:mm:ss"); - log.info("开始处理数据批次: 批次ID={} 批次总量(大于十万只校验十万)={} 批次开始时间={} 批次结束时间={} 对象API={}", batch.getId(),batch.getFirstSfNum(),startDate,endDate, objectApi); + + log.info("开始处理数据批次: 批次ID={} 批次总量(大于十万只校验十万)={} 批次开始时间={} 批次结束时间={} 对象API={} 忽略开始时间={}", batch.getId(),batch.getFirstSfNum(),startDate,endDate, objectApi, ignoreBeginDate); // 分页处理记录 int page = 0; @@ -832,6 +832,23 @@ public class DataVerifyServiceImpl implements DataVerifyService { continue; } + if (ignoreBeginDate != null) { + Object lastModifiedObj = sourceRecord.getField("LastModifiedDate"); + if (lastModifiedObj != null) { + try { + // 解析Salesforce返回的日期格式 2025-10-12T03:27:54.000Z + String lastModifiedStr = lastModifiedObj.toString(); + Date lastModifiedDate = org.apache.commons.lang3.time.DateUtils.parseDate(lastModifiedStr, "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + + if (lastModifiedDate.before(ignoreBeginDate)) { + continue; + } + } catch (Exception e) { + log.warn("解析LastModifiedDate失败: {}", lastModifiedObj, e); + } + } + } + // 比对字段值 for (DataField field : fields) { String fieldName = field.getField();