kris/data-dump
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

【feat】 提交修复

Browse Source 
(cherry picked from commit e312510717)
This commit is contained in:
Kris 2025-10-20 18:05:11 +08:00
parent e00593e229
commit 3e60887577
3 changed files with 114 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/com/celnet/datadump/job/DataDumpNewJob.java
View File

@ -198,7 +198,7 @@ public class DataDumpNewJob {
case "BrandTemplate":
case "EmailTemplate":
case "Folder":
dataImportNewService.getNewIdByField(api,"DeveloperName");
dataImportNewService.getNewIdByField(api,"DeveloperName,Type,NamespacePrefix");
break;
case "Group":
dataImportNewService.getNewIdByField(api,"DeveloperName,Type");

176
src/main/java/com/celnet/datadump/service/impl/DataVerifyServiceImpl.java
View File

@ -4,6 +4,9 @@ import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelWriter;
import com.alibaba.excel.util.FileUtils;
import com.alibaba.excel.write.metadata.WriteSheet;
import com.alibaba.fastjson2.JSON;
import com.alibaba.fastjson2.JSONArray;
import com.alibaba.fastjson2.JSONObject;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.celnet.datadump.config.SalesforceConnect;
@ -21,6 +24,7 @@ import com.celnet.datadump.util.EmailUtil;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.sforce.soap.partner.DescribeSObjectResult;
import com.sforce.soap.partner.Field;
import com.sforce.soap.partner.PartnerConnection;
import com.sforce.soap.partner.QueryResult;
@ -29,6 +33,7 @@ import com.xxl.job.core.biz.model.ReturnT;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.apache.commons.lang3.time.DateUtils;
@ -563,7 +568,12 @@ public class DataVerifyServiceImpl implements DataVerifyService {
PartnerConnection targetConn, boolean sendEmail, Integer type, Date ignoreBeginDate) {
// 1. 准备对象相关信息
String objectApi = dataObject.getName();
List<DataField> fields = getVerifiableFields(objectApi);
log.info("对象: {} 待校验字段列表: {}", objectApi, fields.stream()
.map(DataField::getField)
.collect(Collectors.joining(", ")));
if (fields.isEmpty()) {
log.info("对象: {} 没有需要校验的字段！！！", objectApi);
return;
@ -832,6 +842,7 @@ public class DataVerifyServiceImpl implements DataVerifyService {
continue;
}
boolean isSame = true;
if (ignoreBeginDate != null) {
Object lastModifiedObj = sourceRecord.getField("LastModifiedDate");
if (lastModifiedObj != null) {
@ -841,7 +852,7 @@ public class DataVerifyServiceImpl implements DataVerifyService {
Date lastModifiedDate = org.apache.commons.lang3.time.DateUtils.parseDate(lastModifiedStr, "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
if (lastModifiedDate.before(ignoreBeginDate)) {
continue;
isSame = false;
}
} catch (Exception e) {
log.warn("解析LastModifiedDate失败: {}", lastModifiedObj, e);
@ -849,40 +860,63 @@ public class DataVerifyServiceImpl implements DataVerifyService {
}
}
// 比对字段值
for (DataField field : fields) {
String fieldName = field.getField();
if (isSame){
// 比对字段值
for (DataField field : fields) {
String fieldName = field.getField();
if (Const.EXCLUDE_FIELDS.contains(fieldName))
continue; // 跳过字段比较
if (Const.EXCLUDE_FIELDS.contains(fieldName))
continue; // 跳过字段比较
Object sourceValue = sourceRecord.getField(fieldName);
Object targetValue = targetRecord.getField(fieldName);
Object sourceValue = sourceRecord.getField(fieldName);
Object targetValue = targetRecord.getField(fieldName);
Field sorceField = sourceFieldMap.get(fieldName);
Field targetField = targetFieldMap.get(fieldName);
Field sorceField = sourceFieldMap.get(fieldName);
Field targetField = targetFieldMap.get(fieldName);
if (sourceValue == null && targetValue == null) {
continue;
}
//判断reference_to内是否包含User字符串
String referenceTo = field.getReferenceTo();
if (field.getReferenceTo().contains(",User") || field.getReferenceTo().contains("User,")) {
referenceTo = "User";
}
if (field.getSfType().equals("reference") && !field.getReferenceTo().equals("data_picklist")) {
if (field.getReferenceTo().length() > 1 && !"User".equals(referenceTo)){
if (sourceValue == null && targetValue == null) {
continue;
}
if (StringUtils.isBlank(referenceTo)){
continue;
//判断reference_to内是否包含User字符串
String referenceTo = field.getReferenceTo();
if (referenceTo !=null && (referenceTo.contains(",User") || referenceTo.contains("User,"))) {
referenceTo = "User";
}
Map<String, Object> objectMap = customMapper.getById("new_id", referenceTo, String.valueOf(sourceValue));
if (objectMap == null || !String.valueOf(objectMap.get("new_id")).equals(String.valueOf(targetValue))) {
if (referenceTo != null &&field.getSfType().equals("reference") && !referenceTo.equals("data_picklist")) {
if (field.getReferenceTo().length() > 1 && !"User".equals(referenceTo)){
continue;
}
if (StringUtils.isBlank(referenceTo)){
continue;
}
Map<String, Object> objectMap = customMapper.getById("new_id", referenceTo, String.valueOf(sourceValue));
if (objectMap == null || !String.valueOf(objectMap.get("new_id")).equals(String.valueOf(targetValue))) {
List<String> row = Lists.newArrayList();
row.add(String.valueOf(batch.getId()));
row.add(id);
row.add(newId);
row.add(objectApi);
row.add(field.getField());
row.add(field.getName());
row.add(String.valueOf(sourceValue));
row.add(String.valueOf(targetValue));
row.add("字段值不一致");
row.add(String.valueOf(sorceField.getType()));
row.add(String.valueOf(targetField.getType()));
row.add(String.valueOf(sorceField.isCreateable()));
row.add(String.valueOf(targetField.isCreateable()));
row.add(String.valueOf(sourceRecord.getField("CreatedDate")));
row.add(String.valueOf(sourceRecord.getField("LastModifiedDate")));
row.add(String.valueOf(targetRecord.getField("LastModifiedDate")));
// 写入到Excel的detailSheet，而不是添加到某个detailData列表
objectWriter.write(Collections.singletonList(row), detailSheet);
isError = true;
}
} else if (!Objects.equals(sourceValue, targetValue)){
List<String> row = Lists.newArrayList();
row.add(String.valueOf(batch.getId()));
row.add(id);
@ -904,27 +938,6 @@ public class DataVerifyServiceImpl implements DataVerifyService {
objectWriter.write(Collections.singletonList(row), detailSheet);
isError = true;
}
} else if (!Objects.equals(sourceValue, targetValue)){
List<String> row = Lists.newArrayList();
row.add(String.valueOf(batch.getId()));
row.add(id);
row.add(newId);
row.add(objectApi);
row.add(field.getField());
row.add(field.getName());
row.add(String.valueOf(sourceValue));
row.add(String.valueOf(targetValue));
row.add("字段值不一致");
row.add(String.valueOf(sorceField.getType()));
row.add(String.valueOf(targetField.getType()));
row.add(String.valueOf(sorceField.isCreateable()));
row.add(String.valueOf(targetField.isCreateable()));
row.add(String.valueOf(sourceRecord.getField("CreatedDate")));
row.add(String.valueOf(sourceRecord.getField("LastModifiedDate")));
row.add(String.valueOf(targetRecord.getField("LastModifiedDate")));
// 写入到Excel的detailSheet，而不是添加到某个detailData列表
objectWriter.write(Collections.singletonList(row), detailSheet);
isError = true;
}
}
}
@ -1190,7 +1203,7 @@ private void sendIncrementalQualityCheckEmail(DataVerifyParam param, String file
String endDate = DateFormatUtils.format(endwDate, "yyyy-MM-dd HH:mm:ss");
// 分页处理记录
int page = 0;
String maxId = null;
int totalRecordNum = 0;
int targetRecordNum = 0;
int sourceRecordNum = 0;
@ -1200,20 +1213,44 @@ private void sendIncrementalQualityCheckEmail(DataVerifyParam param, String file
Map<String, Field> sourceFieldMap = fieldMap.get("source");
Map<String, Field> targetFieldMap = fieldMap.get("target");
try {
DescribeSObjectResult dsr = sourceConn.describeSObject(objectApi);
Field[] dsrFields = dsr.getFields();
while (true) {
while (true) {
log.info("开始处理增量数据: 对象API={} 开始时间={} 结束时间={} 批次={}", objectApi, startDate, endDate, page);
List<Map<String, Object>> recordIds = queryRecordIds(objectApi, startDate, endDate, page);
if (recordIds.isEmpty()) break;
int i = 0;
while (true){
int startIndex = MAX_BATCH_RECORDS * i;
int endIndex = Math.min(startIndex + MAX_BATCH_RECORDS, recordIds.size());
if (startIndex >= endIndex) {
log.info("开始处理增量数据: 对象API={} 开始时间={} 结束时间={} 最大Id={}", objectApi, startDate, endDate, maxId);
String sql = "SELECT ID FROM " + objectApi + " WHERE LastModifiedDate >= '" + startDate + "'" +
" AND LastModifiedDate < '" + endDate + "'" +
" AND Id > '" + maxId + "'" +
" ORDER BY Id ASC LIMIT 200";
QueryResult queryResult = sourceConn.queryAll(sql);
if (ObjectUtils.isEmpty(queryResult) || ObjectUtils.isEmpty(queryResult.getRecords())) {
break;
}
List<Map<String, Object>> idList = recordIds.subList(startIndex, endIndex);
SObject[] records = queryResult.getRecords();
JSONArray objects = DataUtil.toJsonArray(records, dsrFields);
// 获取最大修改时间和等于该修改时间的数据id
{
Date maxDate = objects.getJSONObject(objects.size() - 1).getDate("LastModifiedDate");
maxId = objects.stream()
.map(t -> (JSONObject) t)
.filter(t -> maxDate.equals(t.getDate("LastModifiedDate")))
.map(t -> t.getString(Const.ID))
.max(String::compareTo).get();
}
// 遍历出ID集合
List<String> idsList = objects.stream()
.map(t -> (JSONObject) t)
.map(t -> t.getString(Const.ID))
.collect(Collectors.toList());
// 使用ID集合作为条件查询本地数据库
String idCondition = String.join("','", idsList);
String idSql = " Id IN ('" + idCondition + "')";
List<Map<String, Object>> idList = customMapper.list("Id,new_id", objectApi, idSql);
totalRecordNum += idList.size();
List<String> ids = Lists.newArrayList();
@ -1314,11 +1351,11 @@ private void sendIncrementalQualityCheckEmail(DataVerifyParam param, String file
//判断reference_to内是否包含User字符串
String referenceTo = field.getReferenceTo();
if (field.getReferenceTo().contains(",User") || field.getReferenceTo().contains("User,")) {
if (referenceTo !=null && (referenceTo.contains(",User") || referenceTo.contains("User,"))) {
referenceTo = "User";
}
if (field.getSfType().equals("reference") && !field.getReferenceTo().equals("data_picklist")) {
if (referenceTo != null &&field.getSfType().equals("reference") && !referenceTo.equals("data_picklist")) {
if (field.getReferenceTo().length() > 1 && !"User".equals(referenceTo)){
continue;
@ -1374,15 +1411,12 @@ private void sendIncrementalQualityCheckEmail(DataVerifyParam param, String file
}
}
}
i++;
}
page++;
if (page * MAX_PAGE_RECORDS >= MAX_TOTAL_RECORDS) {
log.info("达到最大处理记录数限制，停止处理");
break; // 防止无限循环
}
TimeUnit.MILLISECONDS.sleep(1);
} catch (InterruptedException e) {
return isError;
} catch (Exception throwable) {
log.error("verify error", throwable);
}
// 写入批次汇总结果
@ -1419,7 +1453,7 @@ private void sendIncrementalQualityCheckEmail(DataVerifyParam param, String file
}
}
} catch (Exception e) {
log.info("查询ORG数据时发生异常: {}", e.getMessage());
log.info("查询ORG数据时发生异常，SQL: {}，错误信息: {}", idSql, e.getMessage(), e);
}
return queryMap;

9
src/main/resources/mapper/CustomMapper.xml
View File

@ -143,7 +143,14 @@
SET new_id = #{newId}
<where>
<foreach item="map" collection="maps" open="" separator=" AND " close="">
${map.key} = #{map.value}
<choose>
<when test="map.value == null">
${map.key} IS NULL
</when>
<otherwise>
${map.key} = #{map.value}
</otherwise>
</choose>
</foreach>
</where>
</update>