diff --git a/src/main/java/com/celnet/datadump/job/DataDumpNewJob.java b/src/main/java/com/celnet/datadump/job/DataDumpNewJob.java index 7ee9b08..a23834b 100644 --- a/src/main/java/com/celnet/datadump/job/DataDumpNewJob.java +++ b/src/main/java/com/celnet/datadump/job/DataDumpNewJob.java @@ -198,7 +198,7 @@ public class DataDumpNewJob { case "BrandTemplate": case "EmailTemplate": case "Folder": - dataImportNewService.getNewIdByField(api,"DeveloperName"); + dataImportNewService.getNewIdByField(api,"DeveloperName,Type,NamespacePrefix"); break; case "Group": dataImportNewService.getNewIdByField(api,"DeveloperName,Type"); diff --git a/src/main/java/com/celnet/datadump/service/impl/DataVerifyServiceImpl.java b/src/main/java/com/celnet/datadump/service/impl/DataVerifyServiceImpl.java index 9fd54b3..d83029d 100644 --- a/src/main/java/com/celnet/datadump/service/impl/DataVerifyServiceImpl.java +++ b/src/main/java/com/celnet/datadump/service/impl/DataVerifyServiceImpl.java @@ -4,6 +4,9 @@ import com.alibaba.excel.EasyExcel; import com.alibaba.excel.ExcelWriter; import com.alibaba.excel.util.FileUtils; import com.alibaba.excel.write.metadata.WriteSheet; +import com.alibaba.fastjson2.JSON; +import com.alibaba.fastjson2.JSONArray; +import com.alibaba.fastjson2.JSONObject; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.celnet.datadump.config.SalesforceConnect; @@ -21,6 +24,7 @@ import com.celnet.datadump.util.EmailUtil; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.sforce.soap.partner.DescribeSObjectResult; import com.sforce.soap.partner.Field; import com.sforce.soap.partner.PartnerConnection; import com.sforce.soap.partner.QueryResult; @@ -29,6 +33,7 @@ import com.xxl.job.core.biz.model.ReturnT; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.DateFormatUtils; import org.apache.commons.lang3.time.DateUtils; @@ -563,7 +568,12 @@ public class DataVerifyServiceImpl implements DataVerifyService { PartnerConnection targetConn, boolean sendEmail, Integer type, Date ignoreBeginDate) { // 1. 准备对象相关信息 String objectApi = dataObject.getName(); + List fields = getVerifiableFields(objectApi); + log.info("对象: {} 待校验字段列表: {}", objectApi, fields.stream() + .map(DataField::getField) + .collect(Collectors.joining(", "))); + if (fields.isEmpty()) { log.info("对象: {} 没有需要校验的字段！！！", objectApi); return; @@ -832,6 +842,7 @@ public class DataVerifyServiceImpl implements DataVerifyService { continue; } + boolean isSame = true; if (ignoreBeginDate != null) { Object lastModifiedObj = sourceRecord.getField("LastModifiedDate"); if (lastModifiedObj != null) { @@ -841,7 +852,7 @@ public class DataVerifyServiceImpl implements DataVerifyService { Date lastModifiedDate = org.apache.commons.lang3.time.DateUtils.parseDate(lastModifiedStr, "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); if (lastModifiedDate.before(ignoreBeginDate)) { - continue; + isSame = false; } } catch (Exception e) { log.warn("解析LastModifiedDate失败: {}", lastModifiedObj, e); @@ -849,40 +860,63 @@ public class DataVerifyServiceImpl implements DataVerifyService { } } - // 比对字段值 - for (DataField field : fields) { - String fieldName = field.getField(); + if (isSame){ + // 比对字段值 + for (DataField field : fields) { + String fieldName = field.getField(); - if (Const.EXCLUDE_FIELDS.contains(fieldName)) - continue; // 跳过字段比较 + if (Const.EXCLUDE_FIELDS.contains(fieldName)) + continue; // 跳过字段比较 - Object sourceValue = sourceRecord.getField(fieldName); - Object targetValue = targetRecord.getField(fieldName); + Object sourceValue = sourceRecord.getField(fieldName); + Object targetValue = targetRecord.getField(fieldName); - Field sorceField = sourceFieldMap.get(fieldName); - Field targetField = targetFieldMap.get(fieldName); + Field sorceField = sourceFieldMap.get(fieldName); + Field targetField = targetFieldMap.get(fieldName); - if (sourceValue == null && targetValue == null) { - continue; - } - - //判断reference_to内是否包含User字符串 - String referenceTo = field.getReferenceTo(); - if (field.getReferenceTo().contains(",User") || field.getReferenceTo().contains("User,")) { - referenceTo = "User"; - } - - if (field.getSfType().equals("reference") && !field.getReferenceTo().equals("data_picklist")) { - - if (field.getReferenceTo().length() > 1 && !"User".equals(referenceTo)){ + if (sourceValue == null && targetValue == null) { continue; } - if (StringUtils.isBlank(referenceTo)){ - continue; + + //判断reference_to内是否包含User字符串 + String referenceTo = field.getReferenceTo(); + if (referenceTo !=null && (referenceTo.contains(",User") || referenceTo.contains("User,"))) { + referenceTo = "User"; } - - Map objectMap = customMapper.getById("new_id", referenceTo, String.valueOf(sourceValue)); - if (objectMap == null || !String.valueOf(objectMap.get("new_id")).equals(String.valueOf(targetValue))) { + + if (referenceTo != null &&field.getSfType().equals("reference") && !referenceTo.equals("data_picklist")) { + + if (field.getReferenceTo().length() > 1 && !"User".equals(referenceTo)){ + continue; + } + if (StringUtils.isBlank(referenceTo)){ + continue; + } + + Map objectMap = customMapper.getById("new_id", referenceTo, String.valueOf(sourceValue)); + if (objectMap == null || !String.valueOf(objectMap.get("new_id")).equals(String.valueOf(targetValue))) { + List row = Lists.newArrayList(); + row.add(String.valueOf(batch.getId())); + row.add(id); + row.add(newId); + row.add(objectApi); + row.add(field.getField()); + row.add(field.getName()); + row.add(String.valueOf(sourceValue)); + row.add(String.valueOf(targetValue)); + row.add("字段值不一致"); + row.add(String.valueOf(sorceField.getType())); + row.add(String.valueOf(targetField.getType())); + row.add(String.valueOf(sorceField.isCreateable())); + row.add(String.valueOf(targetField.isCreateable())); + row.add(String.valueOf(sourceRecord.getField("CreatedDate"))); + row.add(String.valueOf(sourceRecord.getField("LastModifiedDate"))); + row.add(String.valueOf(targetRecord.getField("LastModifiedDate"))); + // 写入到Excel的detailSheet，而不是添加到某个detailData列表 + objectWriter.write(Collections.singletonList(row), detailSheet); + isError = true; + } + } else if (!Objects.equals(sourceValue, targetValue)){ List row = Lists.newArrayList(); row.add(String.valueOf(batch.getId())); row.add(id); @@ -904,27 +938,6 @@ public class DataVerifyServiceImpl implements DataVerifyService { objectWriter.write(Collections.singletonList(row), detailSheet); isError = true; } - } else if (!Objects.equals(sourceValue, targetValue)){ - List row = Lists.newArrayList(); - row.add(String.valueOf(batch.getId())); - row.add(id); - row.add(newId); - row.add(objectApi); - row.add(field.getField()); - row.add(field.getName()); - row.add(String.valueOf(sourceValue)); - row.add(String.valueOf(targetValue)); - row.add("字段值不一致"); - row.add(String.valueOf(sorceField.getType())); - row.add(String.valueOf(targetField.getType())); - row.add(String.valueOf(sorceField.isCreateable())); - row.add(String.valueOf(targetField.isCreateable())); - row.add(String.valueOf(sourceRecord.getField("CreatedDate"))); - row.add(String.valueOf(sourceRecord.getField("LastModifiedDate"))); - row.add(String.valueOf(targetRecord.getField("LastModifiedDate"))); - // 写入到Excel的detailSheet，而不是添加到某个detailData列表 - objectWriter.write(Collections.singletonList(row), detailSheet); - isError = true; } } } @@ -1190,7 +1203,7 @@ private void sendIncrementalQualityCheckEmail(DataVerifyParam param, String file String endDate = DateFormatUtils.format(endwDate, "yyyy-MM-dd HH:mm:ss"); // 分页处理记录 - int page = 0; + String maxId = null; int totalRecordNum = 0; int targetRecordNum = 0; int sourceRecordNum = 0; @@ -1200,20 +1213,44 @@ private void sendIncrementalQualityCheckEmail(DataVerifyParam param, String file Map sourceFieldMap = fieldMap.get("source"); Map targetFieldMap = fieldMap.get("target"); + try { + DescribeSObjectResult dsr = sourceConn.describeSObject(objectApi); + Field[] dsrFields = dsr.getFields(); + while (true) { - while (true) { - log.info("开始处理增量数据: 对象API={} 开始时间={} 结束时间={} 批次={}", objectApi, startDate, endDate, page); - List> recordIds = queryRecordIds(objectApi, startDate, endDate, page); - if (recordIds.isEmpty()) break; - int i = 0; - while (true){ - int startIndex = MAX_BATCH_RECORDS * i; - int endIndex = Math.min(startIndex + MAX_BATCH_RECORDS, recordIds.size()); - if (startIndex >= endIndex) { + log.info("开始处理增量数据: 对象API={} 开始时间={} 结束时间={} 最大Id={}", objectApi, startDate, endDate, maxId); + String sql = "SELECT ID FROM " + objectApi + " WHERE LastModifiedDate >= '" + startDate + "'" + + " AND LastModifiedDate < '" + endDate + "'" + + " AND Id > '" + maxId + "'" + + " ORDER BY Id ASC LIMIT 200"; + + QueryResult queryResult = sourceConn.queryAll(sql); + if (ObjectUtils.isEmpty(queryResult) || ObjectUtils.isEmpty(queryResult.getRecords())) { break; } - List> idList = recordIds.subList(startIndex, endIndex); + SObject[] records = queryResult.getRecords(); + JSONArray objects = DataUtil.toJsonArray(records, dsrFields); + // 获取最大修改时间和等于该修改时间的数据id + { + Date maxDate = objects.getJSONObject(objects.size() - 1).getDate("LastModifiedDate"); + maxId = objects.stream() + .map(t -> (JSONObject) t) + .filter(t -> maxDate.equals(t.getDate("LastModifiedDate"))) + .map(t -> t.getString(Const.ID)) + .max(String::compareTo).get(); + } + // 遍历出ID集合 + List idsList = objects.stream() + .map(t -> (JSONObject) t) + .map(t -> t.getString(Const.ID)) + .collect(Collectors.toList()); + + // 使用ID集合作为条件查询本地数据库 + String idCondition = String.join("','", idsList); + String idSql = " Id IN ('" + idCondition + "')"; + List> idList = customMapper.list("Id,new_id", objectApi, idSql); + totalRecordNum += idList.size(); List ids = Lists.newArrayList(); @@ -1314,11 +1351,11 @@ private void sendIncrementalQualityCheckEmail(DataVerifyParam param, String file //判断reference_to内是否包含User字符串 String referenceTo = field.getReferenceTo(); - if (field.getReferenceTo().contains(",User") || field.getReferenceTo().contains("User,")) { + if (referenceTo !=null && (referenceTo.contains(",User") || referenceTo.contains("User,"))) { referenceTo = "User"; } - if (field.getSfType().equals("reference") && !field.getReferenceTo().equals("data_picklist")) { + if (referenceTo != null &&field.getSfType().equals("reference") && !referenceTo.equals("data_picklist")) { if (field.getReferenceTo().length() > 1 && !"User".equals(referenceTo)){ continue; @@ -1374,15 +1411,12 @@ private void sendIncrementalQualityCheckEmail(DataVerifyParam param, String file } } } - i++; - } - - page++; - - if (page * MAX_PAGE_RECORDS >= MAX_TOTAL_RECORDS) { - log.info("达到最大处理记录数限制，停止处理"); - break; // 防止无限循环 } + TimeUnit.MILLISECONDS.sleep(1); + } catch (InterruptedException e) { + return isError; + } catch (Exception throwable) { + log.error("verify error", throwable); } // 写入批次汇总结果 @@ -1419,7 +1453,7 @@ private void sendIncrementalQualityCheckEmail(DataVerifyParam param, String file } } } catch (Exception e) { - log.info("查询ORG数据时发生异常: {}", e.getMessage()); + log.info("查询ORG数据时发生异常，SQL: {}，错误信息: {}", idSql, e.getMessage(), e); } return queryMap; diff --git a/src/main/resources/mapper/CustomMapper.xml b/src/main/resources/mapper/CustomMapper.xml index 104f13b..48a1e38 100644 --- a/src/main/resources/mapper/CustomMapper.xml +++ b/src/main/resources/mapper/CustomMapper.xml @@ -143,7 +143,14 @@ SET new_id = #{newId} - ${map.key} = #{map.value} + + + ${map.key} IS NULL + + + ${map.key} = #{map.value} + +