Compare commits

..

No commits in common. "72b4043e21197dfe7abcd1177cdca4c4b7c552f1" and "aca16ebe7b99b9c10088548fa439dcbbbfbf69e1" have entirely different histories.

5 changed files with 12 additions and 34 deletions

View File

@ -204,7 +204,7 @@ public class DataDumpNewJob {
dataImportNewService.getNewIdByField(api,"DeveloperName,Type"); dataImportNewService.getNewIdByField(api,"DeveloperName,Type");
break; break;
case "RecordType": case "RecordType":
dataImportNewService.getNewIdByField(api,"DeveloperName,SobjectType"); dataImportNewService.getNewIdByField(api,"DeveloperName,SObjectType");
break; break;
case "AccountContactRelation": case "AccountContactRelation":
dataImportNewService.getNewIdByField(api,"AccountId,ContactId"); dataImportNewService.getNewIdByField(api,"AccountId,ContactId");

View File

@ -56,9 +56,4 @@ public class DataVerifyParam {
@ApiModelProperty(value = "结束时间", hidden = true) @ApiModelProperty(value = "结束时间", hidden = true)
private Date endDate; private Date endDate;
@ApiModelProperty(value = "忽略开始时间", hidden = true)
private Date ignoreBeginDate;
} }

View File

@ -3660,7 +3660,7 @@ public class DataImportNewServiceImpl implements DataImportNewService {
}catch (InterruptedException e){ }catch (InterruptedException e){
return; return;
} catch (Exception e) { } catch (Exception e) {
log.error("insertSingle error api:{}", api,e); log.error("insertSingle error api:{},错误信息:{}", api, JSON.toJSONString(e));
} }
} }
if (z*10000+size >= count){ if (z*10000+size >= count){

View File

@ -459,7 +459,7 @@ public class DataImportServiceImpl implements DataImportService {
} catch (InterruptedException interruptedException){ } catch (InterruptedException interruptedException){
return; return;
} catch (Exception e) { } catch (Exception e) {
log.info(JSON.toJSONString(e.getMessage()),e); log.info(JSON.toJSONString(e.getMessage()));
} }
} }

View File

@ -421,7 +421,7 @@ public class DataVerifyServiceImpl implements DataVerifyService {
log.info("开始时间: {}", new Date(startTime)); log.info("开始时间: {}", new Date(startTime));
try { try {
performQualityCheckAndWriteToExcel(dataObjects, sourceConnect, targetConnect, filePath, param.getSendEmail(),param.getType(),param.getIgnoreBeginDate()); performQualityCheckAndWriteToExcel(dataObjects, sourceConnect, targetConnect, filePath, param.getSendEmail(),param.getType());
long endTime = System.currentTimeMillis(); long endTime = System.currentTimeMillis();
log.info("数据质量校验执行完成,耗时: {} ms", endTime - startTime); log.info("数据质量校验执行完成,耗时: {} ms", endTime - startTime);
} catch (Exception e) { } catch (Exception e) {
@ -500,7 +500,7 @@ public class DataVerifyServiceImpl implements DataVerifyService {
private void performQualityCheckAndWriteToExcel(List<DataObject> dataObjects, private void performQualityCheckAndWriteToExcel(List<DataObject> dataObjects,
PartnerConnection sourceConn, PartnerConnection sourceConn,
PartnerConnection targetConn, PartnerConnection targetConn,
String filePath, boolean sendEmail ,Integer type, Date ignoreBeginDate) throws Exception { String filePath, boolean sendEmail ,Integer type) throws Exception {
try (ExcelWriter excelWriter = EasyExcel.write(filePath).build()) { try (ExcelWriter excelWriter = EasyExcel.write(filePath).build()) {
// 1. 创建汇总 Sheet // 1. 创建汇总 Sheet
WriteSheet summarySheet = createSummarySheet(excelWriter); WriteSheet summarySheet = createSummarySheet(excelWriter);
@ -510,7 +510,7 @@ public class DataVerifyServiceImpl implements DataVerifyService {
List<Future<?>> futures = Lists.newArrayList(); List<Future<?>> futures = Lists.newArrayList();
dataObjects.forEach(dataObject -> { dataObjects.forEach(dataObject -> {
Future<?> future = salesforceExecutor.execute(() -> { Future<?> future = salesforceExecutor.execute(() -> {
processDataObject(dataObject, excelWriter, summarySheet, fieldTypeSheet, sourceConn, targetConn, sendEmail, type, ignoreBeginDate); processDataObject(dataObject, excelWriter, summarySheet, fieldTypeSheet, sourceConn, targetConn, sendEmail, type);
}, 1, 1); }, 1, 1);
futures.add(future); futures.add(future);
}); });
@ -560,7 +560,7 @@ public class DataVerifyServiceImpl implements DataVerifyService {
*/ */
private void processDataObject(DataObject dataObject, ExcelWriter excelWriter, WriteSheet summarySheet, private void processDataObject(DataObject dataObject, ExcelWriter excelWriter, WriteSheet summarySheet,
WriteSheet fieldTypeSheet, PartnerConnection sourceConn, WriteSheet fieldTypeSheet, PartnerConnection sourceConn,
PartnerConnection targetConn, boolean sendEmail, Integer type, Date ignoreBeginDate) { PartnerConnection targetConn, boolean sendEmail, Integer type) {
// 1. 准备对象相关信息 // 1. 准备对象相关信息
String objectApi = dataObject.getName(); String objectApi = dataObject.getName();
List<DataField> fields = getVerifiableFields(objectApi); List<DataField> fields = getVerifiableFields(objectApi);
@ -574,6 +574,7 @@ public class DataVerifyServiceImpl implements DataVerifyService {
String qualityCheckFilePath = generateObjectQualityCheckFilePath(objectApi); String qualityCheckFilePath = generateObjectQualityCheckFilePath(objectApi);
// 3. 处理每个批次 // 3. 处理每个批次
List<DataBatch> batches = getDataBatches(objectApi); List<DataBatch> batches = getDataBatches(objectApi);
log.info("对象: {} 待同步的批次数量: {}", objectApi, batches.size()); log.info("对象: {} 待同步的批次数量: {}", objectApi, batches.size());
@ -586,7 +587,7 @@ public class DataVerifyServiceImpl implements DataVerifyService {
String batchQualityCheckFilePath = generateObjectQualityCheckFilePath(objectApi + "_" + batch.getId()); String batchQualityCheckFilePath = generateObjectQualityCheckFilePath(objectApi + "_" + batch.getId());
try (ExcelWriter objectWriter = EasyExcel.write(batchQualityCheckFilePath).build()) { try (ExcelWriter objectWriter = EasyExcel.write(batchQualityCheckFilePath).build()) {
WriteSheet detailSheet = createDetailSheet(excelWriter, dataObject); WriteSheet detailSheet = createDetailSheet(excelWriter, dataObject);
batchHasError = processDataBatch(batch, objectApi, fields, excelWriter, objectWriter, summarySheet, detailSheet, sourceConn, targetConn, fieldMap, ignoreBeginDate) || batchHasError; batchHasError = processDataBatch(batch, objectApi, fields, excelWriter, objectWriter, summarySheet, detailSheet, sourceConn, targetConn, fieldMap) || batchHasError;
}catch (Exception e){ }catch (Exception e){
log.error("API={},批次ID={}处理异常:{}",objectApi,batch.getId(), e.getMessage(),e); log.error("API={},批次ID={}处理异常:{}",objectApi,batch.getId(), e.getMessage(),e);
} }
@ -608,7 +609,7 @@ public class DataVerifyServiceImpl implements DataVerifyService {
WriteSheet detailSheet = createDetailSheet(excelWriter, dataObject); WriteSheet detailSheet = createDetailSheet(excelWriter, dataObject);
for (DataBatch batch : batches) { for (DataBatch batch : batches) {
hasError = processDataBatch(batch, objectApi, fields, excelWriter, objectWriter, summarySheet, detailSheet, sourceConn, targetConn, fieldMap, ignoreBeginDate) || hasError; hasError = processDataBatch(batch, objectApi, fields, excelWriter, objectWriter, summarySheet, detailSheet, sourceConn, targetConn, fieldMap) || hasError;
} }
}catch (Exception e){ }catch (Exception e){
log.error("API={},处理异常:{}",objectApi, e.getMessage(),e); log.error("API={},处理异常:{}",objectApi, e.getMessage(),e);
@ -721,11 +722,10 @@ public class DataVerifyServiceImpl implements DataVerifyService {
*/ */
private boolean processDataBatch(DataBatch batch, String objectApi, List<DataField> fields, private boolean processDataBatch(DataBatch batch, String objectApi, List<DataField> fields,
ExcelWriter excelWriter,ExcelWriter objectWriter, WriteSheet summarySheet, WriteSheet detailSheet, ExcelWriter excelWriter,ExcelWriter objectWriter, WriteSheet summarySheet, WriteSheet detailSheet,
PartnerConnection sourceConn, PartnerConnection targetConn,Map<String, Map<String, Field>> fieldMap, Date ignoreBeginDate) { PartnerConnection sourceConn, PartnerConnection targetConn,Map<String, Map<String, Field>> fieldMap) {
String startDate = DateFormatUtils.format(batch.getSyncStartDate(), "yyyy-MM-dd HH:mm:ss"); String startDate = DateFormatUtils.format(batch.getSyncStartDate(), "yyyy-MM-dd HH:mm:ss");
String endDate = DateFormatUtils.format(batch.getSyncEndDate(), "yyyy-MM-dd HH:mm:ss"); String endDate = DateFormatUtils.format(batch.getSyncEndDate(), "yyyy-MM-dd HH:mm:ss");
log.info("开始处理数据批次: 批次ID={} 批次总量(大于十万只校验十万)={} 批次开始时间={} 批次结束时间={} 对象API={}", batch.getId(),batch.getFirstSfNum(),startDate,endDate, objectApi);
log.info("开始处理数据批次: 批次ID={} 批次总量(大于十万只校验十万)={} 批次开始时间={} 批次结束时间={} 对象API={} 忽略开始时间={}", batch.getId(),batch.getFirstSfNum(),startDate,endDate, objectApi, ignoreBeginDate);
// 分页处理记录 // 分页处理记录
int page = 0; int page = 0;
@ -832,23 +832,6 @@ public class DataVerifyServiceImpl implements DataVerifyService {
continue; continue;
} }
if (ignoreBeginDate != null) {
Object lastModifiedObj = sourceRecord.getField("LastModifiedDate");
if (lastModifiedObj != null) {
try {
// 解析Salesforce返回的日期格式 2025-10-12T03:27:54.000Z
String lastModifiedStr = lastModifiedObj.toString();
Date lastModifiedDate = org.apache.commons.lang3.time.DateUtils.parseDate(lastModifiedStr, "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
if (lastModifiedDate.before(ignoreBeginDate)) {
continue;
}
} catch (Exception e) {
log.warn("解析LastModifiedDate失败: {}", lastModifiedObj, e);
}
}
}
// 比对字段值 // 比对字段值
for (DataField field : fields) { for (DataField field : fields) {
String fieldName = field.getField(); String fieldName = field.getField();