diff --git a/src/main/java/com/celnet/datadump/job/DataDumpNewJob.java b/src/main/java/com/celnet/datadump/job/DataDumpNewJob.java index a13939b..7c3b106 100644 --- a/src/main/java/com/celnet/datadump/job/DataDumpNewJob.java +++ b/src/main/java/com/celnet/datadump/job/DataDumpNewJob.java @@ -207,7 +207,7 @@ public class DataDumpNewJob { dataImportNewService.getNewIdByField(api,"DeveloperName,SObjectType"); break; case "AccountContactRelation": - dataImportNewService.getNewIdByField(api,"AccountId,ContactId,IsPrimary"); + dataImportNewService.getNewIdByField(api,"AccountId,ContactId"); break; case "TaskRelation": dataImportNewService.getNewIdByField(api,"TaskId,RelationId,RelationType"); diff --git a/src/main/java/com/celnet/datadump/service/impl/DataImportNewServiceImpl.java b/src/main/java/com/celnet/datadump/service/impl/DataImportNewServiceImpl.java index 5a1fc8d..7dc8990 100644 --- a/src/main/java/com/celnet/datadump/service/impl/DataImportNewServiceImpl.java +++ b/src/main/java/com/celnet/datadump/service/impl/DataImportNewServiceImpl.java @@ -2959,8 +2959,10 @@ public class DataImportNewServiceImpl implements DataImportNewService { }else { sql = "select " + joined + " from "+ api +" LIMIT 200"; } + log.info("执行SQL查询: {}", sql); QueryResult queryResult = connect.queryAll(sql); if (ObjectUtils.isEmpty(queryResult) || ObjectUtils.isEmpty(queryResult.getRecords())) { + log.info("查询结果为空,结束查询"); break; } @@ -2968,12 +2970,14 @@ public class DataImportNewServiceImpl implements DataImportNewService { objects = DataUtil.toJsonArray(records, dsrFields); maxId = ((JSONObject) objects.get(objects.size() - 1)).getString(Const.ID); + log.info("处理批次数据,记录数: {}", objects.size()); // 批量处理记录 processRecords(api, objects, stringList); int totalRecords = records.length; count += totalRecords; + log.info("已处理记录总数: {}", count); if (totalRecords < 200) { log.info("无更多数据,反写NewId任务结束!!!"); @@ -3017,16 +3021,18 @@ public class DataImportNewServiceImpl implements DataImportNewService { List> maps = buildUpdateMaps(api, jsonObject, stringList); if (maps!=null && !maps.isEmpty()) { + log.info("准备更新记录,API: {}, ID: {}, 匹配字段信息: {}", api, id, maps); customMapper.updateNewIdByFields(api, maps, id); + }else if (maps == null){ + log.info("{}无需更新记录,数据ID: {}", api, id); } } } private List> buildUpdateMaps(String api, JSONObject jsonObject, List stringList) throws Exception { List> maps = Lists.newArrayList(); - Set keys = jsonObject.keySet(); - for (String key : keys) { + for (String key : stringList) { Map paramMap = null; switch (api) { @@ -3040,31 +3046,33 @@ public class DataImportNewServiceImpl implements DataImportNewService { case "EventRelation": paramMap = handleTaskEventRelation(api, key, jsonObject); if (paramMap == null){ + log.info("{}处理失败,字段: {}, 数据ID: {}", api, key, jsonObject.getString("Id")); return null; } break; case "CollaborationGroupMember": paramMap = handleCollaborationGroupMember(key, jsonObject); if (paramMap == null){ + log.info("CollaborationGroupMember处理失败,字段: {}, 数据ID: {}", key, jsonObject.getString("Id")); return null; } break; case "FeedAttachment": paramMap = handleFeedAttachment(key, jsonObject); if (paramMap == null){ + log.info("FeedAttachment处理失败,字段: {}, 数据ID: {}", key, jsonObject.getString("Id")); return null; } break; default: - if (stringList.contains(key)) { - paramMap = Maps.newHashMap(); - paramMap.put("key", key); - paramMap.put("value", jsonObject.getString(key)); - } + paramMap = Maps.newHashMap(); + paramMap.put("key", key); + paramMap.put("value", jsonObject.getString(key)); + log.info("普通字段处理,API: {}, 字段: {}, 值: {}", api, key, jsonObject.getString(key)); break; } - if (paramMap != null) { + if (!paramMap.isEmpty()) { maps.add(paramMap); } } @@ -3075,9 +3083,19 @@ public class DataImportNewServiceImpl implements DataImportNewService { private Map handleAccountContactRelation(String key, JSONObject jsonObject) throws Exception { switch (key) { case "ContactId": - return getNewIdMap(key, "Contact", jsonObject.getString(key)); + Map contactMap = getNewIdMap(key, "Contact", jsonObject.getString(key)); + if (contactMap == null){ + log.info("AccountContactRelation,字段: {},关联对象段: {}, ContactId: {} 未找到对应的new_id", key,"Contact", jsonObject.getString(key)); + return null; + } + return contactMap; case "AccountId": - return getNewIdMap(key, "Account", jsonObject.getString(key)); + Map accountMap = getNewIdMap(key, "Account", jsonObject.getString(key)); + if (accountMap == null){ + log.info("AccountContactRelation,字段: {},关联对象段: {}, AccountId: {} 未找到对应的new_id", key,"Account", jsonObject.getString(key)); + return null; + } + return accountMap; default: return null; } @@ -3086,11 +3104,26 @@ public class DataImportNewServiceImpl implements DataImportNewService { private Map handleTaskEventRelation(String api, String key, JSONObject jsonObject) throws Exception { switch (key) { case "TaskId": - return getNewIdMap(key, "Task", jsonObject.getString(key)); + Map taskObjectMap = getNewIdMap(key, "Task", jsonObject.getString(key)); + if (taskObjectMap == null){ + log.info("TaskEventRelation,字段: {},关联对象段: {}, TaskId: {} 未找到对应的new_id", key,"Task", jsonObject.getString(key)); + return null; + } + return taskObjectMap; case "EventId": - return getNewIdMap(key, "Event", jsonObject.getString(key)); + Map eventObjectMap = getNewIdMap(key, "Event", jsonObject.getString(key)); + if (eventObjectMap == null){ + log.info("TaskEventRelation,字段: {},关联对象段: {}, EventId: {} 未找到对应的new_id", key,"Event", jsonObject.getString(key)); + return null; + } + return eventObjectMap; case "RelationId": - return getNewIdMap(key, jsonObject.getString("Relation_Type"), jsonObject.getString(key)); + Map relationObjectMap = getNewIdMap(key, jsonObject.getString("Relation_Type"), jsonObject.getString(key)); + if (relationObjectMap == null){ + log.info("TaskEventRelation,字段: {},关联对象段: {}, RelationId: {} 未找到对应的new_id", key,jsonObject.getString("Relation_Type"), jsonObject.getString(key)); + return null; + } + return relationObjectMap; default: return null; } @@ -3099,9 +3132,19 @@ public class DataImportNewServiceImpl implements DataImportNewService { private Map handleCollaborationGroupMember(String key, JSONObject jsonObject) throws Exception { switch (key) { case "CollaborationGroupId": - return getNewIdMap(key, "CollaborationGroup", jsonObject.getString(key)); + Map groupObjectMap = getNewIdMap(key, "CollaborationGroup", jsonObject.getString(key)); + if (groupObjectMap == null){ + log.info("CollaborationGroupMember,字段: {},关联对象段: {}, CollaborationGroupId: {} 未找到对应的new_id", key,"CollaborationGroup", jsonObject.getString(key)); + return null; + } + return groupObjectMap; case "MemberId": - return getNewIdMap(key, jsonObject.getString("Member_Type"), jsonObject.getString(key)); + Map memberObjectMap = getNewIdMap(key, jsonObject.getString("Member_Type"), jsonObject.getString(key)); + if (memberObjectMap == null){ + log.info("CollaborationGroupMember,字段: {},关联对象段: {}, MemberId: {} 未找到对应的new_id", key,jsonObject.getString("Member_Type"), jsonObject.getString(key)); + return null; + } + return memberObjectMap; default: return null; } @@ -3110,9 +3153,19 @@ public class DataImportNewServiceImpl implements DataImportNewService { private Map handleFeedAttachment(String key, JSONObject jsonObject) throws Exception { switch (key) { case "FeedEntityId": - return getNewIdMap(key, jsonObject.getString("FeedEntity_Type"), jsonObject.getString(key)); + Map feedEntityObjectMap = getNewIdMap(key, jsonObject.getString("FeedEntity_Type"), jsonObject.getString(key)); + if (feedEntityObjectMap == null){ + log.info("FeedAttachment,字段: {},关联对象段: {}, FeedEntityId: {} 未找到对应的new_id", key,jsonObject.getString("FeedEntity_Type"), jsonObject.getString(key)); + return null; + } + return feedEntityObjectMap; case "RecordId": - return getNewIdMap(key, jsonObject.getString("RecordId_Type"), jsonObject.getString(key)); + Map recordObjectMap = getNewIdMap(key, jsonObject.getString("RecordId_Type"), jsonObject.getString(key)); + if (recordObjectMap == null){ + log.info("FeedAttachment,字段: {},关联对象段: {}, RecordId: {} 未找到对应的new_id", key,jsonObject.getString("RecordId_Type"), jsonObject.getString(key)); + return null; + } + return recordObjectMap; default: return null; } @@ -3121,8 +3174,7 @@ public class DataImportNewServiceImpl implements DataImportNewService { private Map getNewIdMap(String key, String tableName, String id) throws Exception { Map m = customMapper.getByNewId("Id", tableName, id); if (m == null || m.get("Id") == null) { - log.info("关联对象未找到,跳过处理。字段: {}, 表名: {}, ID: {}", key, tableName, id); - return null; // 返回null表示跳过该字段 + return null; } Map paramMap = Maps.newHashMap(); paramMap.put("key", key); diff --git a/src/main/java/com/celnet/datadump/service/impl/DataVerifyServiceImpl.java b/src/main/java/com/celnet/datadump/service/impl/DataVerifyServiceImpl.java index 3fdce02..03878d2 100644 --- a/src/main/java/com/celnet/datadump/service/impl/DataVerifyServiceImpl.java +++ b/src/main/java/com/celnet/datadump/service/impl/DataVerifyServiceImpl.java @@ -993,6 +993,7 @@ public class DataVerifyServiceImpl implements DataVerifyService { Date endDate = param.getEndDate(); if (endDate == null) { endDate = new Date(); + param.setEndDate(endDate); } try { @@ -1092,10 +1093,11 @@ public class DataVerifyServiceImpl implements DataVerifyService { * @param param 参数 * @param filePath 文件路径 */ - private void sendIncrementalQualityCheckEmail(DataVerifyParam param, String filePath) { +private void sendIncrementalQualityCheckEmail(DataVerifyParam param, String filePath) { String head = "增量数据质量校验报告"; String text = head + "生成时间:" + DateFormatUtils.format(new Date(), "yyyy-MM-dd HH:mm:ss") + - ",开始时间:" + DateFormatUtils.format(param.getBeginDate(), "yyyy-MM-dd HH:mm:ss"); + ",开始时间:" + DateFormatUtils.format(param.getBeginDate(), "yyyy-MM-dd HH:mm:ss") + + (param.getEndDate() != null ? ",结束时间:" + DateFormatUtils.format(param.getEndDate(), "yyyy-MM-dd HH:mm:ss") : ""); // 发送邮件(如果需要) if (param.getSendEmail()) {