From f6a97dbf7fcd268740c6d0df4e1c164ffd5a8c11 Mon Sep 17 00:00:00 2001 From: "DESKTOP-368IV5S\\Kris" <13243687953@163.com> Date: Thu, 15 Jan 2026 12:21:01 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E9=87=8D=E5=86=99insertBatchDataToTarg?= =?UTF-8?q?et=E6=96=B9=E6=B3=95=EF=BC=8C=E5=8F=82=E8=80=83Salesforce?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E6=8F=92=E5=85=A5=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 重写DataiIntegrationBatchServiceImpl中的insertBatchDataToTarget方法 - 参考CommonServiceImpl.java中的Salesforce数据插入逻辑 - 优化字段映射结构，使用List>存储字段映射 - 增加字段过滤，排除is_dump和is_upload等标记字段 - 添加线程休眠机制，避免API限流 - 改进错误处理，增加完整异常堆栈信息 - 添加工作流提示词文档（03-01, 03-02, 03-03） - 添加Salesforce参考代码（CommonServiceImpl.java, DataImportNewServiceImpl.java） --- .../DataiIntegrationBatchServiceImpl.java | 639 ++- ...工作流提示词.md => 03-01创建工作流提示词.md} | 0 .../docs/prompts/03-02创建工作流提示词.md | 194 + .../docs/prompts/03-03创建工作流提示词.md | 156 + .../data-dump/CommonServiceImpl.java | 1981 ++++++++ .../data-dump/DataImportNewServiceImpl.java | 4002 +++++++++++++++++ 6 files changed, 6648 insertions(+), 324 deletions(-) rename datai-scenes/datai-scene-salesforce/docs/prompts/{03创建工作流提示词.md => 03-01创建工作流提示词.md} (100%) create mode 100644 datai-scenes/datai-scene-salesforce/docs/prompts/03-02创建工作流提示词.md create mode 100644 datai-scenes/datai-scene-salesforce/docs/prompts/03-03创建工作流提示词.md create mode 100644 datai-scenes/datai-scene-salesforce/docs/reference-code/salesforce-pubsub-realtime-sync/data-dump/CommonServiceImpl.java create mode 100644 datai-scenes/datai-scene-salesforce/docs/reference-code/salesforce-pubsub-realtime-sync/data-dump/DataImportNewServiceImpl.java diff --git a/datai-scenes/datai-scene-salesforce/datai-salesforce-integration/src/main/java/com/datai/integration/service/impl/DataiIntegrationBatchServiceImpl.java b/datai-scenes/datai-scene-salesforce/datai-salesforce-integration/src/main/java/com/datai/integration/service/impl/DataiIntegrationBatchServiceImpl.java index 9e78247f..b0219c9f 100644 --- a/datai-scenes/datai-scene-salesforce/datai-salesforce-integration/src/main/java/com/datai/integration/service/impl/DataiIntegrationBatchServiceImpl.java +++ b/datai-scenes/datai-scene-salesforce/datai-salesforce-integration/src/main/java/com/datai/integration/service/impl/DataiIntegrationBatchServiceImpl.java @@ -13,7 +13,7 @@ import com.datai.common.utils.SecurityUtils; import com.datai.integration.model.param.DataiSyncParam; import com.datai.salesforce.common.utils.SoqlBuilder; import com.datai.integration.util.ConvertUtil; -import com.sforce.soap.partner.PartnerConnection; +import com.sforce.soap.partner.*; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -27,10 +27,7 @@ import com.datai.integration.service.IDataiIntegrationFieldService; import com.datai.integration.service.IDataiIntegrationSyncLogService; import com.datai.integration.mapper.CustomMapper; import com.datai.integration.factory.impl.SOAPConnectionFactory; -import com.sforce.soap.partner.DescribeSObjectResult; -import com.sforce.soap.partner.Field; import com.datai.integration.core.IPartnerV1Connection; -import com.sforce.soap.partner.QueryResult; import com.sforce.soap.partner.sobject.SObject; import com.sforce.ws.ConnectionException; import lombok.extern.slf4j.Slf4j; @@ -998,197 +995,202 @@ public class DataiIntegrationBatchServiceImpl implements IDataiIntegrationBatchS public Map insertBatchDataToTarget(Integer id, String targetOrgType) { Map result = new HashMap<>(); long startTime = System.currentTimeMillis(); - + log.info("开始插入批次数据到目标系统，批次ID: {}, 目标ORG类型: {}", id, targetOrgType); + + if (id == null) { + log.error("批次ID不能为空"); + result.put("success", false); + result.put("message", "批次ID不能为空"); + return result; + } + + if (targetOrgType == null || targetOrgType.trim().isEmpty()) { + log.error("目标ORG类型不能为空"); + result.put("success", false); + result.put("message", "目标ORG类型不能为空"); + return result; + } + + DataiIntegrationBatch batch = selectDataiIntegrationBatchById(id); + if (batch == null) { + log.error("批次不存在，批次ID: {}", id); + result.put("success", false); + result.put("message", "批次不存在"); + return result; + } + + String objectApi = batch.getApi(); + if (objectApi == null || objectApi.trim().isEmpty()) { + log.error("批次对象API为空，批次ID: {}", id); + result.put("success", false); + result.put("message", "批次对象API不能为空"); + return result; + } + + String tableName = objectApi.trim(); + if (customMapper.checkTable(tableName) == null) { + log.error("本地表不存在，表名: {}", tableName); + result.put("success", false); + result.put("message", "本地表不存在: " + tableName); + return result; + } + + log.info("准备插入批次 {} 的对象 {} 数据到目标系统", id, objectApi); + + IPartnerV1Connection targetConnection = null; try { - log.info("开始插入批次数据到目标系统，批次ID: {}, 目标ORG类型: {}", id, targetOrgType); - - if (id == null) { - log.error("批次ID不能为空"); + targetConnection = soapConnectionFactory.getConnection(targetOrgType); + if (targetConnection == null) { + log.error("无法获取目标系统连接，ORG类型: {}", targetOrgType); result.put("success", false); - result.put("message", "批次ID不能为空"); + result.put("message", "无法获取目标系统连接"); return result; } - if (targetOrgType == null || targetOrgType.trim().isEmpty()) { - log.error("目标ORG类型不能为空"); + DescribeSObjectResult objDetail = targetConnection.describeSObject(objectApi); + if (objDetail == null) { + log.error("目标系统不存在该对象，对象API: {}", objectApi); result.put("success", false); - result.put("message", "目标ORG类型不能为空"); + result.put("message", "目标系统不存在该对象: " + objectApi); return result; } - DataiIntegrationBatch batch = selectDataiIntegrationBatchById(id); - if (batch == null) { - log.error("批次不存在，批次ID: {}", id); + String batchField = batch.getBatchField(); + String whereClause = ""; + if (batchField != null && !batchField.trim().isEmpty()) { + whereClause = "WHERE " + batchField.trim(); + } + + log.info("开始查询本地批次数据，表名: {}, 条件: {}", tableName, whereClause); + List> dataList = customMapper.list("*", tableName, whereClause); + + if (dataList == null || dataList.isEmpty()) { + log.warn("本地批次没有数据，表名: {}, 条件: {}", tableName, whereClause); result.put("success", false); - result.put("message", "批次不存在"); + result.put("message", "本地批次没有数据"); return result; } - String objectApi = batch.getApi(); - if (objectApi == null || objectApi.trim().isEmpty()) { - log.error("批次对象API为空，批次ID: {}", id); - result.put("success", false); - result.put("message", "批次对象API不能为空"); - return result; - } + log.info("查询到 {} 条本地批次数据，准备插入到目标系统", dataList.size()); - String tableName = objectApi.trim(); - if (customMapper.checkTable(tableName) == null) { - log.error("本地表不存在，表名: {}", tableName); - result.put("success", false); - result.put("message", "本地表不存在: " + tableName); - return result; - } + int successCount = 0; + int failureCount = 0; + List> failedRecords = new ArrayList<>(); - log.info("准备插入批次 {} 的对象 {} 数据到目标系统", id, objectApi); - - IPartnerV1Connection targetConnection = null; - try { - targetConnection = soapConnectionFactory.getConnection(targetOrgType); - if (targetConnection == null) { - log.error("无法获取目标系统连接，ORG类型: {}", targetOrgType); - result.put("success", false); - result.put("message", "无法获取目标系统连接"); - return result; - } - - DescribeSObjectResult objDetail = targetConnection.describeSObject(objectApi); - if (objDetail == null) { - log.error("目标系统不存在该对象，对象API: {}", objectApi); - result.put("success", false); - result.put("message", "目标系统不存在该对象: " + objectApi); - return result; - } - - String batchField = batch.getBatchField(); - String whereClause = ""; - if (batchField != null && !batchField.trim().isEmpty()) { - whereClause = "WHERE " + batchField.trim() + " = '" + batch.getBatchValue() + "'"; - } - - log.info("开始查询本地批次数据，表名: {}, 条件: {}", tableName, whereClause); - List> dataList = customMapper.list("*", tableName, whereClause); - - if (dataList == null || dataList.isEmpty()) { - log.warn("本地批次没有数据，表名: {}, 条件: {}", tableName, whereClause); - result.put("success", false); - result.put("message", "本地批次没有数据"); - return result; - } - - log.info("查询到 {} 条本地批次数据，准备插入到目标系统", dataList.size()); - - int successCount = 0; - int failureCount = 0; - List> failedRecords = new ArrayList<>(); - - for (int i = 0; i < dataList.size(); i++) { - Map dataMap = dataList.get(i); - try { - String localId = (String) dataMap.get("id"); - if (localId == null || localId.isEmpty()) { - log.warn("记录ID为空，跳过该记录，索引: {}", i); - failureCount++; - Map failedRecord = new HashMap<>(); - failedRecord.put("index", i); - failedRecord.put("reason", "记录ID为空"); - failedRecords.add(failedRecord); - continue; - } - - Map fieldsToInsert = new HashMap<>(); - for (String key : dataMap.keySet()) { - if (!"id".equalsIgnoreCase(key) && !"new_id".equalsIgnoreCase(key) && - !"is_insert".equalsIgnoreCase(key) && !"is_update".equalsIgnoreCase(key) && - dataMap.get(key) != null) { - fieldsToInsert.put(key, dataMap.get(key)); - } - } - - if (fieldsToInsert.isEmpty()) { - log.warn("记录没有需要插入的字段，跳过该记录，ID: {}", localId); - continue; - } - - SObject sObject = new SObject(); - sObject.setType(objectApi); - for (Map.Entry entry : fieldsToInsert.entrySet()) { - sObject.setField(entry.getKey(), entry.getValue()); - } - - SaveResult[] saveResults = targetConnection.create(new SObject[]{sObject}); - if (saveResults != null && saveResults.length > 0 && saveResults[0].isSuccess()) { - String newId = saveResults[0].getId(); - - List> updateMaps = new ArrayList<>(); - Map newIdMap = new HashMap<>(); - newIdMap.put("key", "new_id"); - newIdMap.put("value", newId); - updateMaps.add(newIdMap); - - Map isInsertMap = new HashMap<>(); - isInsertMap.put("key", "is_insert"); - isInsertMap.put("value", 1); - updateMaps.add(isInsertMap); - - customMapper.updateById(tableName, updateMaps, localId); - - successCount++; - log.debug("记录插入成功，本地ID: {}, 新ID: {}", localId, newId); - } else { - String errorMsg = saveResults != null && saveResults.length > 0 ? - saveResults[0].getErrors()[0].getMessage() : "未知错误"; - log.error("记录插入失败，本地ID: {}, 错误: {}", localId, errorMsg); - failureCount++; - Map failedRecord = new HashMap<>(); - failedRecord.put("index", i); - failedRecord.put("localId", localId); - failedRecord.put("reason", errorMsg); - failedRecords.add(failedRecord); - } - - if ((i + 1) % 100 == 0) { - log.info("已插入 {}/{} 条记录", i + 1, dataList.size()); - } - - } catch (Exception e) { - log.error("插入记录失败，索引: {}, 错误: {}", i, e.getMessage()); + for (int i = 0; i < dataList.size(); i++) { + Map dataMap = dataList.get(i); + try { + String localId = (String) dataMap.get("id"); + if (localId == null || localId.isEmpty()) { + log.warn("记录ID为空，跳过该记录，索引: {}", i); failureCount++; Map failedRecord = new HashMap<>(); failedRecord.put("index", i); - failedRecord.put("reason", e.getMessage()); + failedRecord.put("reason", "记录ID为空"); + failedRecords.add(failedRecord); + continue; + } + + List> maps = new ArrayList<>(); + for (String key : dataMap.keySet()) { + Object value = dataMap.get(key); + if (value != null && !"id".equalsIgnoreCase(key) && + !"new_id".equalsIgnoreCase(key) && + !"is_insert".equalsIgnoreCase(key) && + !"is_update".equalsIgnoreCase(key) && + !"is_dump".equalsIgnoreCase(key) && + !"is_upload".equalsIgnoreCase(key)) { + + Map paramMap = new HashMap<>(); + paramMap.put("key", key); + paramMap.put("value", value); + maps.add(paramMap); + } + } + + if (maps.isEmpty()) { + log.warn("记录没有需要插入的字段，跳过该记录，ID: {}", localId); + continue; + } + + SObject sObject = new SObject(); + sObject.setType(objectApi); + for (Map paramMap : maps) { + String key = (String) paramMap.get("key"); + Object value = paramMap.get("value"); + sObject.setField(key, value); + } + + SaveResult[] saveResults = targetConnection.create(new SObject[]{sObject}); + if (saveResults != null && saveResults.length > 0 && saveResults[0].isSuccess()) { + String newId = saveResults[0].getId(); + + List> updateMaps = new ArrayList<>(); + Map newIdMap = new HashMap<>(); + newIdMap.put("key", "new_id"); + newIdMap.put("value", newId); + updateMaps.add(newIdMap); + + Map isInsertMap = new HashMap<>(); + isInsertMap.put("key", "is_insert"); + isInsertMap.put("value", 1); + updateMaps.add(isInsertMap); + + customMapper.updateById(tableName, updateMaps, localId); + + successCount++; + log.debug("记录插入成功，本地ID: {}, 新ID: {}", localId, newId); + } else { + String errorMsg = saveResults != null && saveResults.length > 0 ? + saveResults[0].getErrors()[0].getMessage() : "未知错误"; + log.error("记录插入失败，本地ID: {}, 错误: {}", localId, errorMsg); + failureCount++; + Map failedRecord = new HashMap<>(); + failedRecord.put("index", i); + failedRecord.put("localId", localId); + failedRecord.put("reason", errorMsg); failedRecords.add(failedRecord); } - } - long endTime = System.currentTimeMillis(); - long duration = endTime - startTime; - - result.put("success", true); - result.put("message", "批次数据插入完成"); - result.put("batchId", id); - result.put("api", objectApi); - result.put("label", batch.getLabel()); - result.put("targetOrgType", targetOrgType); - result.put("totalCount", dataList.size()); - result.put("successCount", successCount); - result.put("failureCount", failureCount); - result.put("duration", duration); - result.put("failedRecords", failedRecords); - - log.info("批次数据插入完成，批次ID: {}, 对象API: {}, 总数: {}, 成功: {}, 失败: {}, 耗时: {}ms", - id, objectApi, dataList.size(), successCount, failureCount, duration); - - } finally { - if (targetConnection != null) { - try { - targetConnection.logout(); - } catch (Exception e) { - log.error("关闭目标系统连接失败", e); + if ((i + 1) % 100 == 0) { + log.info("已插入 {}/{} 条记录", i + 1, dataList.size()); } + + try { + TimeUnit.MILLISECONDS.sleep(1); + } catch (InterruptedException e) { + log.warn("线程休眠被中断", e); + Thread.currentThread().interrupt(); + } + + } catch (Exception e) { + log.error("插入记录失败，索引: {}, 错误: {}", i, e.getMessage(), e); + failureCount++; + Map failedRecord = new HashMap<>(); + failedRecord.put("index", i); + failedRecord.put("reason", e.getMessage()); + failedRecords.add(failedRecord); } } + long endTime = System.currentTimeMillis(); + long duration = endTime - startTime; + + result.put("success", true); + result.put("message", "批次数据插入完成"); + result.put("batchId", id); + result.put("api", objectApi); + result.put("label", batch.getLabel()); + result.put("targetOrgType", targetOrgType); + result.put("totalCount", dataList.size()); + result.put("successCount", successCount); + result.put("failureCount", failureCount); + result.put("duration", duration); + result.put("failedRecords", failedRecords); + + log.info("批次数据插入完成，批次ID: {}, 对象API: {}, 总数: {}, 成功: {}, 失败: {}, 耗时: {}ms", + id, objectApi, dataList.size(), successCount, failureCount, duration); + } catch (Exception e) { log.error("插入批次数据到目标系统失败，批次ID: {}", id, e); result.put("success", false); @@ -1201,186 +1203,175 @@ public class DataiIntegrationBatchServiceImpl implements IDataiIntegrationBatchS public Map updateBatchDataToTarget(Integer id, String targetOrgType) { Map result = new HashMap<>(); long startTime = System.currentTimeMillis(); - + + log.info("开始更新批次数据到目标系统，批次ID: {}, 目标ORG类型: {}", id, targetOrgType); + + if (id == null) { + log.error("批次ID不能为空"); + result.put("success", false); + result.put("message", "批次ID不能为空"); + return result; + } + + if (targetOrgType == null || targetOrgType.trim().isEmpty()) { + log.error("目标ORG类型不能为空"); + result.put("success", false); + result.put("message", "目标ORG类型不能为空"); + return result; + } + + DataiIntegrationBatch batch = selectDataiIntegrationBatchById(id); + if (batch == null) { + log.error("批次不存在，批次ID: {}", id); + result.put("success", false); + result.put("message", "批次不存在"); + return result; + } + + String objectApi = batch.getApi(); + if (objectApi == null || objectApi.trim().isEmpty()) { + log.error("批次对象API为空，批次ID: {}", id); + result.put("success", false); + result.put("message", "批次对象API不能为空"); + return result; + } + + String tableName = objectApi.trim(); + if (customMapper.checkTable(tableName) == null) { + log.error("本地表不存在，表名: {}", tableName); + result.put("success", false); + result.put("message", "本地表不存在: " + tableName); + return result; + } + + log.info("准备更新批次 {} 的对象 {} 数据到目标系统", id, objectApi); + + IPartnerV1Connection targetConnection = null; try { - log.info("开始更新批次数据到目标系统，批次ID: {}, 目标ORG类型: {}", id, targetOrgType); - - if (id == null) { - log.error("批次ID不能为空"); + targetConnection = soapConnectionFactory.getConnection(targetOrgType); + if (targetConnection == null) { + log.error("无法获取目标系统连接，ORG类型: {}", targetOrgType); result.put("success", false); - result.put("message", "批次ID不能为空"); + result.put("message", "无法获取目标系统连接"); return result; } - if (targetOrgType == null || targetOrgType.trim().isEmpty()) { - log.error("目标ORG类型不能为空"); + DescribeSObjectResult objDetail = targetConnection.describeSObject(objectApi); + if (objDetail == null) { + log.error("目标系统不存在该对象，对象API: {}", objectApi); result.put("success", false); - result.put("message", "目标ORG类型不能为空"); + result.put("message", "目标系统不存在该对象: " + objectApi); return result; } - DataiIntegrationBatch batch = selectDataiIntegrationBatchById(id); - if (batch == null) { - log.error("批次不存在，批次ID: {}", id); + String batchField = batch.getBatchField(); + String whereClause = ""; + if (batchField != null && !batchField.trim().isEmpty()) { + whereClause = "WHERE " + batchField.trim(); + } + + log.info("开始查询本地批次数据，表名: {}, 条件: {}", tableName, whereClause); + List> dataList = customMapper.list("*", tableName, whereClause); + + if (dataList == null || dataList.isEmpty()) { + log.warn("本地批次没有数据，表名: {}, 条件: {}", tableName, whereClause); result.put("success", false); - result.put("message", "批次不存在"); + result.put("message", "本地批次没有数据"); return result; } - String objectApi = batch.getApi(); - if (objectApi == null || objectApi.trim().isEmpty()) { - log.error("批次对象API为空，批次ID: {}", id); - result.put("success", false); - result.put("message", "批次对象API不能为空"); - return result; - } + log.info("查询到 {} 条本地批次数据，准备更新到目标系统", dataList.size()); - String tableName = objectApi.trim(); - if (customMapper.checkTable(tableName) == null) { - log.error("本地表不存在，表名: {}", tableName); - result.put("success", false); - result.put("message", "本地表不存在: " + tableName); - return result; - } + int successCount = 0; + int failureCount = 0; + List> failedRecords = new ArrayList<>(); - log.info("准备更新批次 {} 的对象 {} 数据到目标系统", id, objectApi); + for (int i = 0; i < dataList.size(); i++) { + Map dataMap = dataList.get(i); + try { + String localId = (String) dataMap.get("id"); + String newId = (String) dataMap.get("new_id"); - IPartnerV1Connection targetConnection = null; - try { - targetConnection = soapConnectionFactory.getConnection(targetOrgType); - if (targetConnection == null) { - log.error("无法获取目标系统连接，ORG类型: {}", targetOrgType); - result.put("success", false); - result.put("message", "无法获取目标系统连接"); - return result; - } - - DescribeSObjectResult objDetail = targetConnection.describeSObject(objectApi); - if (objDetail == null) { - log.error("目标系统不存在该对象，对象API: {}", objectApi); - result.put("success", false); - result.put("message", "目标系统不存在该对象: " + objectApi); - return result; - } - - String batchField = batch.getBatchField(); - String whereClause = ""; - if (batchField != null && !batchField.trim().isEmpty()) { - whereClause = "WHERE " + batchField.trim() + " = '" + batch.getBatchValue() + "'"; - } - - log.info("开始查询本地批次数据，表名: {}, 条件: {}", tableName, whereClause); - List> dataList = customMapper.list("*", tableName, whereClause); - - if (dataList == null || dataList.isEmpty()) { - log.warn("本地批次没有数据，表名: {}, 条件: {}", tableName, whereClause); - result.put("success", false); - result.put("message", "本地批次没有数据"); - return result; - } - - log.info("查询到 {} 条本地批次数据，准备更新到目标系统", dataList.size()); - - int successCount = 0; - int failureCount = 0; - List> failedRecords = new ArrayList<>(); - - for (int i = 0; i < dataList.size(); i++) { - Map dataMap = dataList.get(i); - try { - String localId = (String) dataMap.get("id"); - String newId = (String) dataMap.get("new_id"); - - if (localId == null || localId.isEmpty()) { - log.warn("记录ID为空，跳过该记录，索引: {}", i); - failureCount++; - Map failedRecord = new HashMap<>(); - failedRecord.put("index", i); - failedRecord.put("reason", "记录ID为空"); - failedRecords.add(failedRecord); - continue; - } - - if (newId == null || newId.isEmpty()) { - log.warn("记录new_id为空，跳过该记录，本地ID: {}", localId); - failureCount++; - Map failedRecord = new HashMap<>(); - failedRecord.put("index", i); - failedRecord.put("localId", localId); - failedRecord.put("reason", "new_id为空，请先执行插入操作"); - failedRecords.add(failedRecord); - continue; - } - - Map fieldsToUpdate = new HashMap<>(); - for (String key : dataMap.keySet()) { - if (!"id".equalsIgnoreCase(key) && !"new_id".equalsIgnoreCase(key) && - !"is_insert".equalsIgnoreCase(key) && !"is_update".equalsIgnoreCase(key) && - dataMap.get(key) != null) { - fieldsToUpdate.put(key, dataMap.get(key)); - } - } - - if (fieldsToUpdate.isEmpty()) { - log.warn("记录没有需要更新的字段，跳过该记录，本地ID: {}", localId); - continue; - } - - targetConnection.update(objectApi, newId, fieldsToUpdate); - - List> updateMaps = new ArrayList<>(); - Map isUpdateMap = new HashMap<>(); - isUpdateMap.put("key", "is_update"); - isUpdateMap.put("value", 1); - updateMaps.add(isUpdateMap); - - customMapper.updateById(tableName, updateMaps, localId); - - successCount++; - log.debug("记录更新成功，本地ID: {}, 目标ID: {}", localId, newId); - - if ((i + 1) % 100 == 0) { - log.info("已更新 {}/{} 条记录", i + 1, dataList.size()); - } - - } catch (Exception e) { - log.error("更新记录失败，索引: {}, 错误: {}", i, e.getMessage()); + if (localId == null || localId.isEmpty()) { + log.warn("记录ID为空，跳过该记录，索引: {}", i); failureCount++; Map failedRecord = new HashMap<>(); failedRecord.put("index", i); - failedRecord.put("reason", e.getMessage()); + failedRecord.put("reason", "记录ID为空"); failedRecords.add(failedRecord); + continue; } - } - long endTime = System.currentTimeMillis(); - long duration = endTime - startTime; - - result.put("success", true); - result.put("message", "批次数据更新完成"); - result.put("batchId", id); - result.put("api", objectApi); - result.put("label", batch.getLabel()); - result.put("targetOrgType", targetOrgType); - result.put("totalCount", dataList.size()); - result.put("successCount", successCount); - result.put("failureCount", failureCount); - result.put("duration", duration); - result.put("failedRecords", failedRecords); - - log.info("批次数据更新完成，批次ID: {}, 对象API: {}, 总数: {}, 成功: {}, 失败: {}, 耗时: {}ms", - id, objectApi, dataList.size(), successCount, failureCount, duration); - - } finally { - if (targetConnection != null) { - try { - targetConnection.logout(); - } catch (Exception e) { - log.error("关闭目标系统连接失败", e); + if (newId == null || newId.isEmpty()) { + log.warn("记录new_id为空，跳过该记录，本地ID: {}", localId); + failureCount++; + Map failedRecord = new HashMap<>(); + failedRecord.put("index", i); + failedRecord.put("localId", localId); + failedRecord.put("reason", "new_id为空，请先执行插入操作"); + failedRecords.add(failedRecord); + continue; } + + Map fieldsToUpdate = new HashMap<>(); + for (String key : dataMap.keySet()) { + if (!"id".equalsIgnoreCase(key) && !"new_id".equalsIgnoreCase(key) && + !"is_insert".equalsIgnoreCase(key) && !"is_update".equalsIgnoreCase(key) && + dataMap.get(key) != null) { + fieldsToUpdate.put(key, dataMap.get(key)); + } + } + + if (fieldsToUpdate.isEmpty()) { + log.warn("记录没有需要更新的字段，跳过该记录，本地ID: {}", localId); + continue; + } + + + + List> updateMaps = new ArrayList<>(); + Map isUpdateMap = new HashMap<>(); + isUpdateMap.put("key", "is_update"); + isUpdateMap.put("value", 1); + updateMaps.add(isUpdateMap); + + customMapper.updateById(tableName, updateMaps, localId); + + successCount++; + log.debug("记录更新成功，本地ID: {}, 目标ID: {}", localId, newId); + + if ((i + 1) % 100 == 0) { + log.info("已更新 {}/{} 条记录", i + 1, dataList.size()); + } + + } catch (Exception e) { + log.error("更新记录失败，索引: {}, 错误: {}", i, e.getMessage()); + failureCount++; + Map failedRecord = new HashMap<>(); + failedRecord.put("index", i); + failedRecord.put("reason", e.getMessage()); + failedRecords.add(failedRecord); } } + long endTime = System.currentTimeMillis(); + long duration = endTime - startTime; + + result.put("success", true); + result.put("message", "批次数据更新完成"); + result.put("batchId", id); + result.put("api", objectApi); + result.put("label", batch.getLabel()); + result.put("targetOrgType", targetOrgType); + result.put("totalCount", dataList.size()); + result.put("successCount", successCount); + result.put("failureCount", failureCount); + result.put("duration", duration); + result.put("failedRecords", failedRecords); + + log.info("批次数据更新完成，批次ID: {}, 对象API: {}, 总数: {}, 成功: {}, 失败: {}, 耗时: {}ms", + id, objectApi, dataList.size(), successCount, failureCount, duration); + } catch (Exception e) { log.error("更新批次数据到目标系统失败，批次ID: {}", id, e); result.put("success", false); diff --git a/datai-scenes/datai-scene-salesforce/docs/prompts/03创建工作流提示词.md b/datai-scenes/datai-scene-salesforce/docs/prompts/03-01创建工作流提示词.md similarity index 100% rename from datai-scenes/datai-scene-salesforce/docs/prompts/03创建工作流提示词.md rename to datai-scenes/datai-scene-salesforce/docs/prompts/03-01创建工作流提示词.md diff --git a/datai-scenes/datai-scene-salesforce/docs/prompts/03-02创建工作流提示词.md b/datai-scenes/datai-scene-salesforce/docs/prompts/03-02创建工作流提示词.md new file mode 100644 index 00000000..01a8a874 --- /dev/null +++ b/datai-scenes/datai-scene-salesforce/docs/prompts/03-02创建工作流提示词.md @@ -0,0 +1,194 @@ +#### 角色设定 (Role Definition) + +> 你是项目的 **"SSOT 架构师" (Single Source of Truth Architect)**。你的核心职责是维护项目文档的唯一性、准确性和可追溯性。你拒绝在没有文档依据的情况下编写代码。你严格遵守以下七步工作流。 + +### 核心原则：无文档不开发 + +> **规则：** AI 严禁在未更新 SSOT 文档的情况下生成业务代码。所有操作必须遵循“需求 -> 决策 -> 提示词 -> 执行 -> 复盘”的单向链条。 + +#### 优化后的七步工作流 (Refined Workflow) + +**1. 初始化协作基线 (Bootstrap)** + +- **操作**：在根目录创建 `README.md` (项目蓝图)、`CONTRIBUTING.md` (AI与人类的协作协议)、`CHANGELOG.md`。 + +- **约束**：`README.md` 必须包含指向 `docs/index.md` 的显著链接。 + + +**2. 建立真源中心 (SSOT Hub)** + +- **操作**：创建 `docs/index.md`。 + +- **规则**：这是项目的**唯一**入口。它不仅是列表，更是导航图。它必须包含对 Requirements, Design, ADRs, Prompts 的动态索引。**严禁出现“孤儿文件” (即未被 index 索引的文件)。** + + +**3. 落地信息架构 (Information Architecture)** + +- **操作**：创建标准目录结构： + + - `docs/requirements/` (需求) + + - `docs/design/` (系统设计) + + - `docs/decisions/adr/` (架构决策记录) + + - `docs/prompts/` (提示词资产 - **Treat Prompts as Code**) + + - `docs/sessions/` (AI 会话快照) + + - `docs/retros/` (迭代复盘) + + +**4. 注入智能模板：ADR (架构决策)** + +- **文件**：`docs/decisions/adr/0000-template.md` + +- **优化结构**： + + - `Title`: 决策标题 + + - `Status`: [Draft | Accepted | Deprecated] + + - `Context`: 为什么要做这个决策？ + + - `Decision`: 我们决定做什么？ + + - `Consequences`: 好的后果与坏的后果 (Trade-offs) + + - `Compliance`: 如何验证决策已被执行？ + + +**5. 注入智能模板：Prompt First (提示词资产)** + +- **文件**：`docs/prompts/0000-template.md` + +- **优化结构**： + + - `Goal`: 该提示词解决什么具体问题？ + + - `Context Links`: 引用了哪些 docs (必须是相对链接)？ + + - `Inputs`: 需要用户提供哪些变量？ + + - `Prompt Content`: 实际的提示词内容 (Markdown Code Block)。 + + - `Output Definition`: 期望的输出格式 (JSON/Markdown/Code)。 + + - `Version`: 提示词的版本号。 + + +**6. 注入智能模板：Session Logs (会话上下文)** + +- **文件**：`docs/sessions/YYYYMMDD-TaskName.md` + +- **关键补充**：记录不仅仅是“聊天记录”，而是“状态恢复点”。 + + - `Trigger`: 触发本次会话的需求或 Bug ID。 + + - `Used Prompts`: 链接到使用了 `docs/prompts/` 下的哪个文件。 + + - `Outcome`: 最终产生了哪些代码变更 (Commit Hash)。 + + +**7. 注入智能模板：Retro & Loop (闭环复盘)** + +- **文件**：`docs/retros/YYYYMMDD-Review.md` + +- **强制闭环规则**： + + - 如果发现 Prompt 效果不好 -> **必须**修改 `docs/prompts/` 下的对应文件。 + + - 如果发现流程问题 -> **必须**更新 `CONTRIBUTING.md`。 + + - 不要只记录问题，要产生 Action Item。 + +### 执行阶段与具体逻辑 + +#### 阶段 1：需求定义与入库 (Requirements) + +- **动作**：在 `docs/requirements/` 创建新需求文档（如 `REQ-001.md`）。 + +- **逻辑**： + + 1. 解析用户原始需求，将其拆解为：用户故事、验收标准 (AC)。 + + 2. 同步更新 `docs/index.md`，将该需求挂载到任务清单中。 + + +#### 阶段 2：方案决策 (Architecture Decision) + +- **动作**：基于 `docs/decisions/adr/0000-template.md` 创建决策记录。 + +- **逻辑**： + + 1. 针对该需求，分析至少两种技术方案。 + + 2. 记录选定方案的理由、潜在风险及回滚策略。 + + 3. **强制校验**：检查方案是否冲突于已有的 ADR 记录。 + + +#### 阶段 3：提示词资产化 (Prompt Engineering) + +- **动作**：基于 `docs/prompts/0000-template.md` 编写执行提示词。 + +- **逻辑**： + + 1. **引用真源**：提示词开头必须引用 `docs/requirements/` 和 `docs/design/` 的文件链接。 + + 2. 定义输出格式（如：必须包含单元测试，必须符合某设计模式）。 + + 3. 将此 Prompt 存入版本库，作为可重复调用的“代码”。 + + +#### 阶段 4：执行会话与代码生成 (Session Execution) + +- **动作**：基于 `docs/sessions/YYYYMMDD-template.md` 记录执行过程。 + +- **逻辑**： + + 1. **首句申明**：明确当前“现状”与“目标”。 + + 2. 加载阶段 3 准备的 Prompt，观察输出。 + + 3. 记录 AI 的质疑、替代方案以及最终的复现步骤。 + + +#### 阶段 5：变更记录与归档 (Changelog) + +- **动作**：更新 `CHANGELOG.md` 和 `docs/changelog/`。 + +- **逻辑**： + + 1. 简述本次变更对项目的影响。 + + 2. 在 `docs/index.md` 中标注该需求已完成。 + + +#### 阶段 6：闭环复盘 (Retrospective) + +- **动作**：基于 `docs/retros/YYYYMMDD-template.md` 进行总结。 + +- **逻辑**： + + 1. 对比目标与实际产出。 + + 2. **提取模式**：记录 3 条有效的 Prompt 技巧和 3 个避免的坑。 + + 3. **模板迭代**：如果发现模板不适用，立即执行一次“模板更新”。 + + +--- + +### 自动化约束检查（插入到提示词末尾） + +> **执行顺序自检清单：** +> +> 1. 我是否已经为这个任务创建了需求文档？ +> +> 2. 我是否已经根据 ADR 模板做出了架构决策？ +> +> 3. 我是否已经编写并保存了专门的执行 Prompt？ +> +> 4. **只有上述三点全为“是”，我才开始编写/生成业务代码。** +> \ No newline at end of file diff --git a/datai-scenes/datai-scene-salesforce/docs/prompts/03-03创建工作流提示词.md b/datai-scenes/datai-scene-salesforce/docs/prompts/03-03创建工作流提示词.md new file mode 100644 index 00000000..aa281547 --- /dev/null +++ b/datai-scenes/datai-scene-salesforce/docs/prompts/03-03创建工作流提示词.md @@ -0,0 +1,156 @@ +## 1. 角色设定 (Role Definition) + +你是本项目的 **"SSOT 架构师" (Single Source of Truth Architect)**。 你的核心原则是：**代码是文档的下游产物**。你拒绝在没有文档支撑的情况下直接编写或修改业务代码。你严格遵守以下“八步工作流”来保证上下文的完整性和可追溯性。 + +--- + +### 核心原则：无文档不开发 + +> **规则：** AI 严禁在未更新 SSOT 文档的情况下生成业务代码。所有操作必须遵循“需求 -> 决策 -> 提示词 -> 执行 -> 复盘”的单向链条。 + + + +## 2. 核心工作流 (The 8-Step Workflow) + +在接到任何开发任务时，你必须按顺序执行以下步骤。严禁跳步。 + +### **Phase 1: 需求定义 (Requirements)** + +- **动作**：在 `docs/requirements/` 创建或更新需求文件。 + +- **执行逻辑**： + + 1. 将模糊的需求转化为清晰的用户故事和验收标准 (AC)。 + + 2. 同步更新 `docs/index.md` 的需求索引。 + + +### **Phase 2: 架构决策 (Decisions)** + +- **动作**：在 `docs/decisions/adr/` 创建 ADR (Architecture Decision Record)。 + +- **执行逻辑**： + + 1. 如果涉及新的技术选型或复杂逻辑，必须使用 `0000-template.md` 记录决策。 + + 2. 明确记录：背景、决策内容、备选方案、后果 (Trade-offs)。 + + +### **Phase 3: 提示词资产化 (Prompt Engineering)** + +- **动作**：在 `docs/prompts/` 编写或调用特定的 Task Prompt。 + +- **执行逻辑**： + + 1. **Treat Prompts as Code**: 提示词必须有输入变量、输出定义和版本号。 + + 2. **制定搜索策略**: 在 Prompt 中明确规定接下来需要查找哪些文件作为参考。 + + +### **Phase 4: 上下文锚定 (Context Anchoring) [CRITICAL]** + +- **动作**：执行只读操作（搜索与读取），不产生代码。 + +- **执行逻辑**： + + 1. **Stop & Look**: 暂停生成。根据 Phase 3 的策略，查找项目中现有的类似代码、接口定义或全局配置。 + + 2. **Mount**: 读取相关文件内容进入上下文。 + + 3. **Align**: 分析现有代码风格（命名、错误处理、注释），声明：“我将参考 `xxx.ts` 的模式进行开发”。 + + +### **Phase 5: 执行与记录 (Execution & Logging)** + +- **动作**：编写代码，并实时记录在 `docs/sessions/`。 + +- **执行逻辑**： + + 1. **Session Log**: 记录 `现状` + `目标` + `引用的上下文文件`。 + + 2. **Coding**: 基于 Phase 4 的锚定生成代码，确保风格一致。 + + 3. 记录关键的质疑、替代方案以及最终的复现步骤。 + + +### **Phase 6: 闭环复盘 (Retrospective)** + +- **动作**：在 `docs/retros/` 创建复盘记录。 + +- **执行逻辑**： + + 1. **指标对比**: 目标 vs 实际结果。 + + 2. **模式提取**: 记录 3 条有效的 Prompt 模式，3 条踩坑记录。 + + 3. **反哺**: 如果发现流程或模板问题，**立即更新** `docs/prompts/` 或 `CONTRIBUTING.md`。 + + +### **Phase 7: 测试与验证 (Verification)** + +- **动作**：运行测试。 + +- **执行逻辑**： + + 1. 确保代码通过所有单元测试。 + + 2. 确保满足 Phase 1 定义的验收标准 (AC)。 + + +### **Phase 8: 版本提交 (Git Commit)** + +- **动作**：执行 Git 提交。 + +- **执行逻辑**： + + 1. **原子性**: 确保本次提交仅包含当前任务的变更。 + + 2. **文档同源**: 提交必须包含代码变更 **以及** 上述步骤产生的所有文档变更 (Requirements, ADR, Prompts, Session Logs, Retros)。 + + 3. **Commit Message 规范**: + + Plaintext + + ``` + feat(scope): 简短描述 [Req-ID] + + - Implements: [需求链接/ID] + - Decision: [ADR 链接/ID] + - Context: Based on session [Session Log ID] + ``` + + +--- + +## 3. 目录与模板规范 (Directory & Templates) + +你维护以下目录结构，并确保所有新文件都基于模板创建： + +Plaintext + +``` +/ +├── README.md # 项目入口 (包含指向 docs/index.md 的链接) +├── CONTRIBUTING.md # 协作规则 (包含本 prompt 定义的工作流) +├── CHANGELOG.md # 变更日志 +└── docs/ + ├── index.md # 单一真源入口 (动态索引所有子目录内容) + ├── requirements/ # 需求文档 + ├── design/ # 系统设计 + ├── decisions/adr/ # 架构决策 (模板包含: Status, Context, Decision, Consequences) + ├── prompts/ # 提示词资产 (模板包含: Goal, Reference Strategy, Inputs, Output Format) + ├── sessions/ # 会话记录 (模板包含: Context Mounting, Execution Process) + └── retros/ # 迭代复盘 (模板包含: Insights, Action Items for Prompt Improvement) +``` + +--- + +## 4. 启动指令 (Initialization Trigger) + +当用户输入 **"Start Workflow: [任务名称]"** 时，你将： + +1. 检查 `docs/index.md` 是否存在，若不存在则初始化整个目录结构。 + +2. 询问用户该任务的具体需求，并自动开始 **Phase 1**。 + +3. 在整个过程中，每完成一个 Phase，向用户简要汇报并请求许可进入下一个 Phase。 \ No newline at end of file diff --git a/datai-scenes/datai-scene-salesforce/docs/reference-code/salesforce-pubsub-realtime-sync/data-dump/CommonServiceImpl.java b/datai-scenes/datai-scene-salesforce/docs/reference-code/salesforce-pubsub-realtime-sync/data-dump/CommonServiceImpl.java new file mode 100644 index 00000000..f9fbb6ce --- /dev/null +++ b/datai-scenes/datai-scene-salesforce/docs/reference-code/salesforce-pubsub-realtime-sync/data-dump/CommonServiceImpl.java @@ -0,0 +1,1981 @@ +package com.celnet.datadump.service.impl; + +import com.alibaba.fastjson2.JSON; +import com.alibaba.fastjson2.JSONArray; +import com.alibaba.fastjson2.JSONObject; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.celnet.datadump.config.SalesforceConnect; +import com.celnet.datadump.config.SalesforceExecutor; +import com.celnet.datadump.config.SalesforceTargetConnect; +import com.celnet.datadump.entity.*; +import com.celnet.datadump.global.Const; +import com.celnet.datadump.global.TypeCode; +import com.celnet.datadump.mapper.CustomMapper; +import com.celnet.datadump.param.DataDumpParam; +import com.celnet.datadump.param.DataDumpSpecialParam; +import com.celnet.datadump.param.SalesforceParam; +import com.celnet.datadump.service.*; +import com.celnet.datadump.util.DataUtil; +import com.celnet.datadump.util.EmailUtil; +import com.celnet.datadump.util.SqlUtil; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.sforce.soap.partner.*; +import com.sforce.soap.partner.sobject.SObject; +import com.xxl.job.core.biz.model.ReturnT; +import com.xxl.job.core.log.XxlJobLogger; +import com.xxl.job.core.util.DateUtil; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.DateFormatUtils; +import org.apache.commons.lang3.time.DateUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.*; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; + +import static com.celnet.datadump.global.SystemConfigCode.*; + +/** + * @author Red + * @description + * @date 2022/09/23 + */ +@Service +@Slf4j +public class CommonServiceImpl implements CommonService { + + @Autowired + private SalesforceConnect salesforceConnect; + @Autowired + private CustomMapper customMapper; + @Autowired + private SalesforceExecutor salesforceExecutor; + @Autowired + private DataBatchHistoryService dataBatchHistoryService; + @Autowired + private DataBatchService dataBatchService; + @Autowired + private DataObjectService dataObjectService; + @Autowired + private DataFieldService dataFieldService; + @Autowired + private FileService fileService; + @Autowired + private DataPicklistService dataPicklistService; + @Autowired + private DataDumpSpecialService dataDumpSpecialService; + @Autowired + private DataReportService dataReportService; + @Autowired + private DataReportDetailService dataReportDetailService; + @Autowired + private SalesforceTargetConnect salesforceTargetConnect; + @Autowired + private MetaclassConfigService metaclassConfigService; + @Autowired + private LinkConfigService linkConfigService; + @Autowired + private DataLogService dataLogService; + @Autowired + private CommonService commonService; + + @Override + public ReturnT increment(SalesforceParam param) throws Exception { + QueryWrapper qw = new QueryWrapper<>(); + if (StringUtils.isNotBlank(param.getApi())) { + List apis = DataUtil.toIdList(param.getApi()); + qw.in("name", apis); + } + qw.eq("need_update", true) + .isNotNull("last_update_date"); + List list = dataObjectService.list(qw); + if (CollectionUtils.isEmpty(list)) { + return new ReturnT<>(500, ("表" + param.getApi() + "不存在或未开启更新")); + } + List> futures = Lists.newArrayList(); + try { + DataReport dataReport = new DataReport(); + dataReport.setType(TypeCode.INCREMENT); + dataReport.setApis(list.stream().map(DataObject::getName).collect(Collectors.joining(","))); + dataReportService.save(dataReport); + for (DataObject dataObject : list) { + Future future = salesforceExecutor.execute(() -> { + try { + Date updateTime = new Date(); + SalesforceParam salesforceParam = new SalesforceParam(); + salesforceParam.setApi(dataObject.getName()); + salesforceParam.setBeginModifyDate(dataObject.getLastUpdateDate()); + salesforceParam.setType(2); + // 更新字段值不为空 按更新字段里的字段校验 + if (StringUtils.isNotBlank(dataObject.getUpdateField())) { + salesforceParam.setUpdateField(dataObject.getUpdateField()); + } + PartnerConnection connect = dumpData(salesforceParam, dataReport); + dataObject.setLastUpdateDate(updateTime); + dataObjectService.updateById(dataObject); + // 增量batch更新 + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("name", dataObject.getName()).orderByDesc("sync_start_date").last("limit 1"); + DataBatch dataBatch = dataBatchService.getOne(queryWrapper); + if (dataBatch != null) { + if (dataBatch.getSyncEndDate().before(updateTime)) { + DataBatch update = dataBatch.clone(); + update.setSyncEndDate(updateTime); + int failCount = 0; + while (true) { + try { + SalesforceParam countParam = new SalesforceParam(); + countParam.setApi(dataBatch.getName()); + countParam.setBeginCreateDate(dataBatch.getSyncStartDate()); + countParam.setEndCreateDate(dataBatch.getSyncEndDate()); + // 存在isDeleted 只查询IsDeleted为false的 + if (dataFieldService.hasDeleted(countParam.getApi())) { + countParam.setIsDeleted(false); + } else { + // 不存在 过滤 + countParam.setIsDeleted(null); + } + // sf count + Integer sfNum = countSfNum(connect, countParam); + update.setSfNum(sfNum); + // db count + Integer dbNum = customMapper.count(countParam); + update.setDbNum(dbNum); + update.setSyncStatus(dbNum.equals(sfNum) ? 1 : 0); + update.setFirstDbNum(dbNum); + update.setFirstSyncDate(updateTime); + update.setFirstSfNum(sfNum); + QueryWrapper updateQw = new QueryWrapper<>(); + updateQw.eq("name", dataBatch.getName()) + .eq("sync_start_date", dataBatch.getSyncStartDate()) + .eq("sync_end_date", dataBatch.getSyncEndDate()); + dataBatchService.update(update, updateQw); + failCount = 0; + break; + } catch (InterruptedException e) { + throw e; + } catch (Throwable throwable) { + failCount++; + log.error("verify error", throwable); + if (failCount > Const.MAX_FAIL_COUNT) { + throw throwable; + } + TimeUnit.MINUTES.sleep(1); + } + } + } + } + } catch (Throwable throwable) { + log.error("salesforceExecutor error", throwable); + throw new RuntimeException(throwable); + } + }, 0, 0); + futures.add(future); + } + // 等待当前所有线程执行完成 + salesforceExecutor.waitForFutures(futures.toArray(new Future[]{})); + // 存在附件的开始dump + list.stream().filter(t -> StringUtils.isNotBlank(t.getBlobField())).forEach(t -> { + try { + fileService.dumpFile(t.getName(), t.getBlobField(), true); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + return ReturnT.SUCCESS; + } catch (Throwable throwable) { + salesforceExecutor.remove(futures.toArray(new Future[]{})); + throw throwable; + } + } + + @Override + public ReturnT incrementNew(SalesforceParam param) throws Exception { + QueryWrapper qw = new QueryWrapper<>(); + if (StringUtils.isNotBlank(param.getApi())) { + List apis = DataUtil.toIdList(param.getApi()); + qw.in("name", apis); + } + qw.eq("need_update", true) + .isNotNull("last_update_date"); + List list = dataObjectService.list(qw); + if (CollectionUtils.isEmpty(list)) { + return new ReturnT<>(500, ("表" + param.getApi() + "不存在或未开启更新")); + } + List> futures = Lists.newArrayList(); + try { + DataReport dataReport = new DataReport(); + dataReport.setType(TypeCode.INCREMENT); + dataReport.setApis(list.stream().map(DataObject::getName).collect(Collectors.joining(","))); + dataReportService.save(dataReport); + for (DataObject dataObject : list) { + //无创建时间对象 + if (!hasCreatedDate(dataObject.getName())) { + DataDumpSpecialParam dataDumpSpecialParam = new DataDumpSpecialParam(); + dataDumpSpecialParam.setApi(dataObject.getName()); + Future future = dataDumpSpecialService.getDataNew(dataDumpSpecialParam, salesforceConnect.createConnect(),dataObject); + // 等待当前所有线程执行完成 + salesforceExecutor.waitForFutures(future); + continue; + } + Future future = salesforceExecutor.execute(() -> { + try { + Date updateTime = new Date(); + SalesforceParam salesforceParam = new SalesforceParam(); + salesforceParam.setApi(dataObject.getName()); + salesforceParam.setBeginModifyDate(dataObject.getLastUpdateDate()); + salesforceParam.setType(2); + // 更新字段值不为空 按更新字段里的字段校验 + if (StringUtils.isNotBlank(dataObject.getUpdateField())) { + salesforceParam.setUpdateField(dataObject.getUpdateField()); + } + dumpDataNew(salesforceParam, dataReport,dataObject); + dataObject.setLastUpdateDate(updateTime); + dataObject.setNeedUpdate( false); + dataObjectService.updateById(dataObject); + + } catch (Throwable throwable) { + log.error("salesforceExecutor error", throwable); + throw new RuntimeException(throwable); + } + }, 0, 0); + futures.add(future); + } + // 等待当前所有线程执行完成 + salesforceExecutor.waitForFutures(futures.toArray(new Future[]{})); + return ReturnT.SUCCESS; + } catch (Throwable throwable) { + salesforceExecutor.remove(futures.toArray(new Future[]{})); + throw throwable; + } + } + + @Override + public ReturnT dump(SalesforceParam param) throws Exception { + List> futures = Lists.newArrayList(); + try { + if (StringUtils.isNotBlank(param.getApi())) { + // 手动任务 + ReturnT result = manualDump(param, futures); + if (result != null) { + return result; + } + } else { + // 自动任务 + autoDump(param, futures); + } + return ReturnT.SUCCESS; + } catch (Throwable throwable) { + salesforceExecutor.remove(futures.toArray(new Future[]{})); + log.error("dump error", throwable); + throw throwable; + } + } + + /** + * 自动dump + * @param param 参数 + * @param futures futures + */ + private void autoDump(SalesforceParam param, List> futures) throws InterruptedException { + QueryWrapper qw = new QueryWrapper<>(); + qw.eq("data_work", 1) + .eq("data_lock", 0) + .orderByAsc("data_index") + .last(" limit 10"); + while (true) { + List dataObjects = dataObjectService.list(qw); + if (CollectionUtils.isEmpty(dataObjects)) { + break; + } + for (DataObject dataObject : dataObjects) { + DataObject update = new DataObject(); + TimeUnit.MILLISECONDS.sleep(1); + String api = dataObject.getName(); + try { + log.info("dump apis: {}", api); + XxlJobLogger.log("dump apis: {}", api); + // 检测表是否存在 不存在创建 + checkApi(api, true); + // 检测是否有创建时间 没有创建时间 按特殊表处理 直接根据id来获取 + if (!hasCreatedDate(api)) { + try { + DataDumpSpecialParam dataDumpSpecialParam = new DataDumpSpecialParam(); + dataDumpSpecialParam.setApi(api); + update.setName(dataObject.getName()); + update.setDataLock(1); + dataObjectService.updateById(update); + param.setApi(api); + salesforceExecutor.waitForFutures(dataDumpSpecialService.getData(dataDumpSpecialParam, salesforceConnect.createConnect())); + update.setDataWork(0); + } catch (Throwable e) { + throw new RuntimeException(e); + } finally { + if (StringUtils.isNotBlank(update.getName())) { + update.setDataLock(0); + dataObjectService.updateById(update); + } + } + continue; + } + update.setName(dataObject.getName()); + update.setDataLock(1); + dataObjectService.updateById(update); + param.setApi(api); + List salesforceParams; + QueryWrapper dbQw = new QueryWrapper<>(); + dbQw.eq("name", api); + List list = dataBatchService.list(dbQw); + AtomicInteger batch = new AtomicInteger(1); + if (CollectionUtils.isEmpty(list)) { + // 没有批次 + salesforceParams = DataUtil.splitTask(param); + } else { + // 有批次 先过滤首次执行过的 + salesforceParams = list.stream().filter(t -> t.getFirstSyncDate() == null) + .map(t -> { + SalesforceParam salesforceParam = param.clone(); + salesforceParam.setApi(t.getName()); + salesforceParam.setBeginCreateDate(t.getSyncStartDate()); + salesforceParam.setEndCreateDate(t.getSyncEndDate()); + salesforceParam.setBatch(batch.getAndIncrement()); + return salesforceParam; + }).collect(Collectors.toList()); + } + // 批次都执行过了 重新同步 + if (CollectionUtils.isEmpty(salesforceParams)) { + salesforceParams = DataUtil.splitTask(param); + } + for (SalesforceParam salesforceParam : salesforceParams) { + Future future = salesforceExecutor.execute(() -> { + try { + dumpData(salesforceParam); + } catch (Throwable throwable) { + log.error("salesforceExecutor error", throwable); + throw new RuntimeException(throwable); + } + }, salesforceParam.getBatch(), 0); + futures.add(future); + } + // 等待当前所有线程执行完成 + salesforceExecutor.waitForFutures(futures.toArray(new Future[]{})); + + } catch (Throwable e) { + String message = e.getMessage(); + String format = String.format("获取表数据 error, api name: %s, \nparam: %s, \ncause:\n%s", api, com.alibaba.fastjson2.JSON.toJSONString(param, DataDumpParam.getFilter()), message); + EmailUtil.send("DataDump ERROR", format); + throw new RuntimeException(e); + } finally { + update.setDataWork(0); + update.setDataLock(0); + dataObjectService.updateById(update); + } + } + // 等待当前所有线程执行完成 + salesforceExecutor.waitForFutures(futures.toArray(new Future[]{})); + futures.clear(); + } + + } + + private ReturnT manualDump(SalesforceParam param, List> futures) throws InterruptedException { + List apis; + apis = DataUtil.toIdList(param.getApi()); + String join = StringUtils.join(apis, ","); + log.info("dump apis: {}", join); + XxlJobLogger.log("dump apis: {}", join); + // 全量的时候 检测是否有自动任务锁住的表 + boolean isFull = param.getBeginDate() == null && param.getEndDate() == null && CollectionUtils.isEmpty(param.getIds()); + if (isFull) { + QueryWrapper qw = new QueryWrapper<>(); + qw.eq("data_lock", 1).in("name", apis); + List list = dataObjectService.list(qw); + if (CollectionUtils.isNotEmpty(list)) { + String apiNames = list.stream().map(DataObject::getName).collect(Collectors.joining()); + return new ReturnT<>(500, "api:" + apiNames + " is locked"); + } + } + // 根据参数获取sql + for (String api : apis) { + DataObject update = new DataObject(); + TimeUnit.MILLISECONDS.sleep(1); + try { + checkApi(api, true); + if (!hasCreatedDate(api)) { + DataDumpSpecialParam dataDumpSpecialParam = new DataDumpSpecialParam(); + dataDumpSpecialParam.setApi(api); + Future future = dataDumpSpecialService.getData(dataDumpSpecialParam, salesforceConnect.createConnect()); + // 等待当前所有线程执行完成 + salesforceExecutor.waitForFutures(future); + continue; + } + param.setApi(api); + List salesforceParams = null; + if (isFull) { + update.setName(api); + update.setDataLock(1); + dataObjectService.updateById(update); + QueryWrapper dbQw = new QueryWrapper<>(); + dbQw.eq("name", api) + .isNull("first_sync_date"); + if (param.getBeginDate() != null && param.getEndDate() != null) { + dbQw.eq("sync_start_date", param.getBeginDate()); + dbQw.eq("sync_end_date", param.getEndDate()); + } + List list = dataBatchService.list(dbQw); + AtomicInteger batch = new AtomicInteger(1); + if (CollectionUtils.isNotEmpty(list)) { + salesforceParams = list.stream().map(t -> { + SalesforceParam salesforceParam = param.clone(); + salesforceParam.setApi(t.getName()); + salesforceParam.setBeginCreateDate(t.getSyncStartDate()); + salesforceParam.setEndCreateDate(t.getSyncEndDate()); + salesforceParam.setBatch(batch.getAndIncrement()); + return salesforceParam; + }).collect(Collectors.toList()); + } else { + salesforceParams = DataUtil.splitTask(param); + } + } else { + salesforceParams = DataUtil.splitTask(param); + } + // 手动任务优先执行 + for (SalesforceParam salesforceParam : salesforceParams) { + Future future = salesforceExecutor.execute(() -> { + try { + dumpData(salesforceParam); + } catch (Throwable throwable) { + log.error("salesforceExecutor error", throwable); + throw new RuntimeException(throwable); + } + }, salesforceParam.getBatch(), 1); + futures.add(future); + } + + // 等待当前所有线程执行完成 + salesforceExecutor.waitForFutures(futures.toArray(new Future[]{})); + + } catch (Throwable e) { + log.error("manualDump error", e); + throw new RuntimeException(e); + } finally { + if (isFull) { + update.setDataWork(0); + update.setName(api); + update.setDataLock(0); + dataObjectService.updateById(update); + } + } + } + return null; + } + + + + /** + * 数据传输主体 + * + * @param param 参数 + */ + private PartnerConnection dumpData(SalesforceParam param) throws Throwable { + return dumpData(param, null); + } + + /** + * 数据传输主体 + * + * @param param 参数 + */ + private PartnerConnection dumpData(SalesforceParam param, DataReport dataReport) throws Throwable { + String api = param.getApi(); + PartnerConnection connect = null; + try { + DataBatchHistory dataBatchHistory = new DataBatchHistory(); + dataBatchHistory.setName(api); + dataBatchHistory.setStartDate(new Date()); + dataBatchHistory.setSyncStartDate(param.getBeginCreateDate()); + if (param.getEndCreateDate() != null) { + dataBatchHistory.setSyncEndDate(DateUtils.addSeconds(param.getEndCreateDate(), -1)); + } + dataBatchHistory.setBatch(param.getBatch()); + if (param.getBeginCreateDate() != null && param.getEndCreateDate() != null) { + log.info("NO.{} dump {}, date: {} ~ {} start", param.getBatch(), + param.getApi(), + DateFormatUtils.format(param.getBeginCreateDate(), "yyyy-MM-dd HH:mm:ss"), + DateFormatUtils.format(param.getEndCreateDate(), "yyyy-MM-dd HH:mm:ss")); + } + connect = salesforceConnect.createConnect(); + // 存在isDeleted 只查询IsDeleted为false的 + if (dataFieldService.hasDeleted(param.getApi())) { + param.setIsDeleted(false); + } else { + // 不存在 过滤 + param.setIsDeleted(null); + } + // 若count数量过多 可能导致超时出不来结果 对该任务做进一步拆分 + int failCount = 0; + boolean isSuccess = false; + while (failCount <= Const.MAX_FAIL_COUNT) { + try { + // 不能 count Id + if (!"FeedItem".equals(api) && !"ContentFolderMember ".equals(api)){ + dataBatchHistory.setSfNum(countSfNum(connect, param)); + } + isSuccess = true; + break; + } catch (Throwable throwable) { + failCount++; + } + } + // 不成功 做任务拆分 + if (!isSuccess) { + if (splitTask(param)) { + return connect; + } + } + getAllSfData(param, connect, dataReport); + updateDataBatch(param, dataBatchHistory); + } catch (Throwable throwable) { + log.error("dataDumpJob error api:{}", api, throwable); + String type = param.getType() == 1 ? "存量" : "增量"; + String format = String.format("%s数据迁移 error, api name: %s, \nparam: %s, \ncause:\n%s", type, api, JSON.toJSONString(param, DataDumpParam.getFilter()), throwable); + EmailUtil.send("DataDump ERROR", format); + throw throwable; + } + return connect; + } + + /** + * 数据传输主体（新） + * + * @param param 参数 + */ + private void dumpDataNew(SalesforceParam param, DataReport dataReport , DataObject dataObject) throws Throwable { + String api = param.getApi(); + PartnerConnection connect ; + try { + DataBatchHistory dataBatchHistory = new DataBatchHistory(); + dataBatchHistory.setName(api); + dataBatchHistory.setStartDate(new Date()); + dataBatchHistory.setSyncStartDate(param.getBeginCreateDate()); + if (param.getEndCreateDate() != null) { + dataBatchHistory.setSyncEndDate(DateUtils.addSeconds(param.getEndCreateDate(), -1)); + } + dataBatchHistory.setBatch(param.getBatch()); + + connect = salesforceConnect.createConnect(); + // 存在isDeleted 只查询IsDeleted为false的 + if (dataFieldService.hasDeleted(param.getApi())) { + param.setIsDeleted(false); + } else { + // 不存在 过滤 + param.setIsDeleted(null); + } + + if (!"FeedItem".equals(api) && !"ContentFolderMember ".equals(api)) { + dataBatchHistory.setSfNum(countSfNum(connect, param)); + } + + getAllSfData(param, connect, dataReport); + + insertDataBatch(param, dataBatchHistory, dataObject); + + } catch (Throwable throwable) { + log.error("dataDumpJob error api:{}", api, throwable); + String type = param.getType() == 1 ? "存量" : "增量"; + String format = String.format("%s数据迁移 error, api name: %s, \nparam: %s, \ncause:\n%s", type, api, JSON.toJSONString(param, DataDumpParam.getFilter()), throwable); + EmailUtil.send("DataDump ERROR", format); + throw throwable; + } + } + + /** + * 任务拆分 + * + * @param param 任务参数 + * @return true成功 false失败 + */ + private boolean splitTask(SalesforceParam param) { + List> futures = Lists.newArrayList(); + try { + QueryWrapper qw = new QueryWrapper<>(); + qw.eq("name", param.getApi()) + .eq("sync_start_date", param.getBeginCreateDate()) + .eq("sync_end_date", param.getEndCreateDate()); + DataBatch dataBatch = dataBatchService.getOne(qw); + // 如果已经是同一天了 则抛出错误 不继续执行 + if (DateUtils.isSameDay(dataBatch.getSyncEndDate(), dataBatch.getSyncStartDate())) { + throw new Exception("can't count sf num"); + } + DataBatch dataBatch1 = dataBatch.clone(); + DataBatch dataBatch2 = dataBatch.clone(); + Date midDate = DateUtils.addMilliseconds(dataBatch.getSyncStartDate(), + (int) (dataBatch.getSyncEndDate().getTime() - dataBatch.getSyncStartDate().getTime())); + Date now = new Date(); + dataBatch1.setSyncEndDate(midDate); + dataBatch2.setSyncStartDate(midDate); + dataBatchService.remove(qw); + dataBatchService.save(dataBatch1); + dataBatchService.save(dataBatch2); + SalesforceParam param1 = param.clone(); + SalesforceParam param2 = param.clone(); + param1.setEndCreateDate(midDate); + Future future1 = salesforceExecutor.execute(() -> { + try { + dumpData(param1); + } catch (Throwable throwable) { + log.error("salesforceExecutor error", throwable); + throw new RuntimeException(throwable); + } + }, param1.getBatch(), 0); + param2.setBeginCreateDate(midDate); + Future future2 = salesforceExecutor.execute(() -> { + try { + dumpData(param2); + } catch (Throwable throwable) { + log.error("salesforceExecutor error", throwable); + throw new RuntimeException(throwable); + } + }, param2.getBatch(), 0); + futures = Lists.newArrayList(future1, future2); + // 等待当前所有线程执行完成 + salesforceExecutor.waitForFutures(futures.toArray(new Future[]{})); + return true; + + } catch (Throwable throwable) { + salesforceExecutor.remove(futures.toArray(new Future[]{})); + } + return false; + } + + /** + * 遍历获取所有sf数据并处理 + * + * @param param 参数 + * @param connect sf connect + */ + private void getAllSfData(SalesforceParam param, PartnerConnection connect, DataReport dataReport) throws Throwable { + JSONArray objects = null; + String api = param.getApi(); + Map map = Maps.newHashMap(); + String dateName = param.getType() == 1 ? Const.CREATED_DATE : param.getUpdateField(); + int count = 0; + Date lastCreatedDate = null; + String maxId = null; + Field[] dsrFields = null; + // 获取sf字段 + { + List fields = Lists.newArrayList(); + DescribeSObjectResult dsr = connect.describeSObject(api); + dsrFields = dsr.getFields(); + for (Field field : dsrFields) { + // 不查询文件 + if ("base64".equalsIgnoreCase(field.getType().toString())) { + continue; + } + fields.add(field.getName()); + } + + DataObject dataObject = dataObjectService.getById(api); + if (dataObject != null && StringUtils.isNotBlank(dataObject.getExtraField())) { + fields.addAll(Arrays.asList(StringUtils.split(dataObject.getExtraField().replaceAll(StringUtils.SPACE, StringUtils.EMPTY), ","))); + } + param.setSelect(StringUtils.join(fields, ",")); + } + // 获取数据库字段进行比对 + List fields = customMapper.getFields(api).stream().map(String::toUpperCase).collect(Collectors.toList()); + int failCount = 0,batch = 0; + while (true) { + try { + // 获取创建时间 + param.setTimestamp(lastCreatedDate); + // 判断是否存在要排除的id + param.setMaxId(maxId); + map.put("param", param); + String sql; + if (param.getType() == 1) { + // type 1 按创建时间纬度 + sql = SqlUtil.showSql("com.celnet.datadump.mapper.SalesforceMapper.list", map); + } else { + // type 2 按修改时间纬度 + sql = SqlUtil.showSql("com.celnet.datadump.mapper.SalesforceMapper.listByModifyTime", map); + } + log.info("query sql: {}", sql); + XxlJobLogger.log("query sql: {}", sql); + DataLog dataLog = new DataLog(api, null, new Date(), null, "数据拉取，拉取第" + batch + "批数据", "QuerySF"); + QueryResult queryResult = connect.queryAll(sql); + if (ObjectUtils.isEmpty(queryResult) || ObjectUtils.isEmpty(queryResult.getRecords())) { + break; + } + dataLog.setEndTime(new Date()); + dataLogService.save(dataLog); + + SObject[] records = queryResult.getRecords(); + objects = DataUtil.toJsonArray(records, dsrFields); + // 获取最大修改时间和等于该修改时间的数据id + { + // 获取当前批次最后一条记录的时间戳，用于下一批次的查询条件 + Date maxDate = objects.getJSONObject(objects.size() - 1).getDate(dateName); + // 在当前批次中找出所有具有相同时间戳的记录，并获取其中ID最大的那条记录的ID + // 这个ID将作为下一批次查询的起始位置，避免重复处理相同时间戳的数据 + maxId = objects.stream() + .map(t -> (JSONObject) t) + .filter(t -> maxDate.equals(t.getDate(dateName))) + .map(t -> t.getString(Const.ID)) + .max(String::compareTo).get(); + lastCreatedDate = maxDate; + } + DataLog dataLog1 = new DataLog(api, null, new Date(), null, "数据拉取，Upsert第" + batch + "批数据", "UpsertDB"); + // 存储更新 + saveOrUpdate(api, fields, records, objects, true); + dataLog1.setEndTime(new Date()); + dataLogService.save(dataLog1); + + count += records.length; + TimeUnit.MILLISECONDS.sleep(1); + String format = DateFormatUtils.format(lastCreatedDate, "yyyy-MM-dd HH:mm:ss"); + log.info("dump success count: {}, timestamp: {}", count, format); + XxlJobLogger.log("dump success count: {}, timestamp: {}", count, format); + failCount = 0; + objects = null; + batch ++; + } catch (InterruptedException e) { + throw e; + } catch (Throwable throwable) { + failCount++; + log.error("dataDumpJob error api:{}, data:{}", api, JSON.toJSONString(objects), throwable); + if (failCount > Const.MAX_FAIL_COUNT) { + throwable.addSuppressed(new Exception("dataDump error data:" + JSON.toJSONString(objects))); + throw throwable; + } + TimeUnit.MINUTES.sleep(1); + } + } + if (ObjectUtils.isNotEmpty(dataReport)) { + DataReportDetail dataReportDetail = new DataReportDetail(); + dataReportDetail.setApi(param.getApi()); + dataReportDetail.setReportId(dataReport.getId()); + dataReportDetail.setNum(count); + dataReportDetailService.save(dataReportDetail); + } + } + + /** + * 批次存在 且首次时间为0或者为空 赋值、保存执行记录 + * + * @param param 参数 + * @param dataBatchHistory 执行记录 + */ + private void updateDataBatch(SalesforceParam param, DataBatchHistory dataBatchHistory) { + if (dataBatchHistory == null) { + return; + } + dataBatchHistory.setEndDate(new Date()); + Integer num = customMapper.count(param); + dataBatchHistory.setDbNum(num); + if (dataBatchHistory.getSfNum() != null) { + dataBatchHistory.setSyncStatus(dataBatchHistory.getSfNum().equals(dataBatchHistory.getDbNum()) ? 1 : 0); + } + dataBatchHistory.setCost(DataUtil.calTime(dataBatchHistory.getEndDate(), dataBatchHistory.getStartDate())); + QueryWrapper qw = new QueryWrapper<>(); + qw.eq("name", param.getApi()) + .eq("sync_start_date", param.getBeginCreateDate()) + .eq("sync_end_date", param.getEndCreateDate()) + .isNull("first_sync_date"); + DataBatch dataBatch = dataBatchService.getOne(qw); + if (dataBatch == null) { + return; + } + dataBatch.setFirstSfNum(dataBatchHistory.getSfNum()); + dataBatch.setFirstDbNum(dataBatchHistory.getDbNum()); + dataBatch.setFirstSyncDate(dataBatchHistory.getStartDate()); + dataBatch.setSyncStatus(dataBatchHistory.getSyncStatus()); + dataBatchService.update(dataBatch, qw); + + log.info("count db num: {}", num); + XxlJobLogger.log("count db num: {}", num); + dataBatchHistoryService.save(dataBatchHistory); + log.info("dataDumpJob done api:{}, count:{}", dataBatchHistory.getName(), num); + } + + /** + * 执行增量任务，新增一个批次 + * @param param 参数 + * @param dataBatchHistory 执行记录 + */ + private void insertDataBatch(SalesforceParam param, DataBatchHistory dataBatchHistory, DataObject dataObject) { + + dataBatchHistory.setEndDate(new Date()); + Integer num = customMapper.count(param); + dataBatchHistory.setDbNum(num); + if (dataBatchHistory.getSfNum() != null) { + dataBatchHistory.setSyncStatus(dataBatchHistory.getSfNum().equals(dataBatchHistory.getDbNum()) ? 1 : 0); + } + dataBatchHistory.setCost(DataUtil.calTime(dataBatchHistory.getEndDate(), dataBatchHistory.getStartDate())); + DataBatch dataBatch = new DataBatch(); + dataBatch.setName(dataObject.getName()); + dataBatch.setLabel(dataObject.getLabel()); + dataBatch.setFirstSfNum(dataBatchHistory.getSfNum()); + dataBatch.setFirstDbNum(dataBatchHistory.getDbNum()); + dataBatch.setFirstSyncDate(dataBatchHistory.getStartDate()); + dataBatch.setSyncStatus(dataBatchHistory.getSyncStatus()); + dataBatch.setSyncStartDate(dataObject.getLastUpdateDate()); + dataBatch.setSyncEndDate(new Date()); + dataBatchService.save(dataBatch); + + log.info("count db num: {}", num); + XxlJobLogger.log("count db num: {}", num); + dataBatchHistoryService.save(dataBatchHistory); + log.info("dataDumpJob done api:{}, count:{}", dataBatchHistory.getName(), num); + } + + /** + * 获取需同步sf数据数量 + * + * @param connect connect + * @param param 参数 + * @return sf统计数量 + */ + @Override + public Integer countSfNum(PartnerConnection connect, SalesforceParam param) throws Exception { + Map map = Maps.newHashMap(); + map.put("param", param); + String sql = SqlUtil.showSql("com.celnet.datadump.mapper.SalesforceMapper.count", map); + log.info("count sql: {}", sql); + XxlJobLogger.log("count sql: {}", sql); + QueryResult queryResult = connect.queryAll(sql); + SObject record = queryResult.getRecords()[0]; + Integer num = (Integer) record.getField("num"); + log.info("count sf num: {}", num); + XxlJobLogger.log("count sf num: {}", num); + return num; + } + + /** + * 插入或更新数据 + * + * @param api 表名 + * @param fields 字段 + * @param records 数据list + * @param objects 数据 json格式 + * @param insert 不存在是否插入 + */ + @Override + public Integer saveOrUpdate(String api, List fields, SObject[] records, JSONArray objects, boolean insert) throws Throwable { + // 根据id查询数据库 取出已存在数据的id + List ids = Arrays.stream(records).map(SObject::getId).collect(Collectors.toList()); + DataObject one = dataObjectService.getById(api); + QueryWrapper dbQw = new QueryWrapper<>(); + dbQw.eq("api", api); + List list = dataFieldService.list(dbQw).stream().map(DataField::getField).collect(Collectors.toList()); + List existsIds = customMapper.getIds(api, ids); + for (int i = 0; i < objects.size(); i++) { + JSONObject jsonObject = objects.getJSONObject(i); + try { + // update + String id = jsonObject.getString(Const.ID); + List> maps = Lists.newArrayList(); + for (String key : list) { + if (fields.stream().anyMatch(key::equalsIgnoreCase)) { + // ContentVersion表，插入更新时把title中的/替换为- + if ("ContentVersion".equals(one.getName()) && "Title".equals(key)){ + String title = StringUtils.replace(String.valueOf(jsonObject.get("Title")), "/", "-"); + Map paramMap = Maps.newHashMap(); + paramMap.put("key", "Title"); + paramMap.put("value", title); + maps.add(paramMap); + continue; + } + + // Document,Attachment，插入更新时把Name中的/替换为- + if (("Document".equals(one.getName()) || "Attachment".equals(one.getName())) && "Name".equals(key)){ + String name = StringUtils.replace(String.valueOf(jsonObject.get("Name")), "/", "-"); + Map paramMap = Maps.newHashMap(); + paramMap.put("key", "Name"); + paramMap.put("value", name); + maps.add(paramMap); + continue; + } + if ("is_dump".equals( key) || "is_upload".equals( key)){ + Map paramMap = Maps.newHashMap(); + paramMap.put("key", key); + paramMap.put("value", false); + maps.add(paramMap); + continue; + } + Map paramMap = Maps.newHashMap(); + paramMap.put("key", key); + paramMap.put("value", jsonObject.get(key)); + maps.add(paramMap); + } + } + + + if (existsIds.contains(id)) { + Map paramMap = Maps.newHashMap(); + paramMap.put("key", "is_update"); + paramMap.put("value",0); + maps.add(paramMap); + customMapper.updateById(api, maps, id); + } else { + if (insert) { + customMapper.save(api, maps); + } + } + TimeUnit.MILLISECONDS.sleep(1); + } catch (InterruptedException e) { + throw e; + } catch (Exception e) { + log.info("存量任务异常！" + "对象API:" + api + "存量数据下载异常！" ,e); + EmailUtil.send("存量任务异常！", "对象API:" + api + "存量数据下载异常！"); + throw new RuntimeException(e); + } + } + return existsIds.size(); + } + + /** + * 检测表是否存在 不存在则创建 + * + * @param apiName 表名 + * @param createBatch sf 连接 + */ + @Override + public void checkApi(String apiName, Boolean createBatch) throws Exception { + // 加个锁 避免重复执行创建api + log.info("check api:{}", apiName); + + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("name",apiName); + long count = metaclassConfigService.count(queryWrapper); + + try { + boolean hasCreatedDate = false; + + PartnerConnection connection = salesforceConnect.createConnect(); + DataObject dataObject = dataObjectService.getById(apiName); + Date now = new Date(); + if (StringUtils.isBlank(customMapper.checkTable(apiName))) { + log.info("api:{} doesn't exist create", apiName); + // 构建字段 + List> list = Lists.newArrayList(); + DescribeSObjectResult dsr = connection.describeSObject(apiName); + String label = dsr.getLabel(); + List fieldList = Lists.newArrayList(); + List fields = Lists.newArrayList(); + String blobField = null; + List dataPicklists = Lists.newArrayList(); + for (Field field : dsr.getFields()) { + + // 过滤字段 + if (Const.TABLE_FILTERS.contains(field.getName())) { + continue; + } + if (Const.CREATED_DATE.equalsIgnoreCase(field.getName())) { + hasCreatedDate = true; + } + Map map = Maps.newHashMap(); + String sfType = field.getType().toString(); + if ("base64".equalsIgnoreCase(sfType)) { + blobField = field.getName(); + } + map.put("type", DataUtil.fieldTypeToMysql(field)); + // 英文' 转换为 \' + map.put("comment", field.getLabel().replaceAll("'", "\\\\'")); + map.put("name", field.getName()); + list.add(map); + DataField dataField = new DataField(); + dataField.setApi(apiName); + dataField.setSfType(sfType); + dataField.setField(field.getName()); + dataField.setName(field.getLabel()); + dataField.setIsCreateable(field.getCreateable()); + dataField.setIsUpdateable(field.getUpdateable()); + dataField.setIsNillable(field.getNillable()); + dataField.setCalculatedFormula(field.getCalculatedFormula()); + dataField.setIsDefaultedOnCreate(field.getDefaultedOnCreate()); + String join = null; + // 会有非常多映射关系的字段在 这里置空不管 + if (field.getReferenceTo().length <= 3) { + join = StringUtils.join(field.getReferenceTo(), ","); + } else { + log.info("referenceTo too long set null, api:{}, field:{}, reference to:{}", apiName, field.getName(), StringUtils.join(field.getReferenceTo(), ",")); + } + if ("textarea".equalsIgnoreCase(field.getType().toString()) && field.isHtmlFormatted()) { + log.info("检测到富文本字段: {}，API: {}", field.getName(), apiName); + dataField.setIsHtmlFormatted(true); + Map htmlMap = Maps.newHashMap(); + htmlMap.put("type", "text"); + htmlMap.put("comment", field.getName()+ "_back"); + htmlMap.put("name", field.getName()+ "_back"); + list.add(htmlMap); + } + if (field.getReferenceTo().length > 1){ + if (field.getName().contains("Id") && !"OwnerId".equals(field.getName()) && !"UserOrGroupId".equals(field.getName())){ + LinkConfig linkConfig = new LinkConfig(); + linkConfig.setApi(apiName); + linkConfig.setLabel(label); + linkConfig.setField(field.getName()); + linkConfig.setLinkField(StringUtils.replace(field.getName(), "Id", "_Type")); + linkConfigService.save(linkConfig); + } + } + // picklist保存到picklist表 + if ("picklist".equalsIgnoreCase(sfType) || "multipicklist".equalsIgnoreCase(sfType)) { + join = "data_picklist"; + PicklistEntry[] picklistValues = field.getPicklistValues(); + if (ArrayUtils.isNotEmpty(picklistValues)) { + for (PicklistEntry picklistValue : picklistValues) { + DataPicklist dataPicklist = new DataPicklist(); + dataPicklist.setApi(apiName); + dataPicklist.setField(field.getName()); + dataPicklist.setLabel(picklistValue.getLabel()); + dataPicklist.setValue(picklistValue.getValue()); + dataPicklists.add(dataPicklist); + } + } + } + dataField.setReferenceTo(join); + fieldList.add(dataField); + fields.add(field.getName()); + } + // 存在ownerId字段 + String extraField = ""; + if (dataObject != null && StringUtils.isNotBlank(dataObject.getExtraField())) { + extraField = dataObject.getExtraField(); + } + if (fields.stream().anyMatch(Const.OWNER_ID::equalsIgnoreCase)) { + if (StringUtils.isNotBlank(extraField)) { + extraField += ",Owner.Type"; + } else { + extraField = "Owner.Type"; + } + } + // 是否存在额外字段 配置上表 + if (StringUtils.isNotBlank(extraField)) { + Set extras = Arrays.stream(StringUtils.split(extraField.replaceAll(StringUtils.SPACE, StringUtils.EMPTY), ",")).collect(Collectors.toSet()); + for (String extra : extras) { + String fieldName = extra.replaceAll("\\.", "_"); + DataField dataField = new DataField(); + dataField.setApi(apiName); + dataField.setField(fieldName); + dataField.setName(extra); + fieldList.add(dataField); + fields.add(fieldName); + Map map = Maps.newHashMap(); + map.put("type", "varchar(255)"); + map.put("comment", extra); + map.put("name", fieldName); + list.add(map); + } + } + // 存在附件的表 作特殊处理 构建两个字段 + if (StringUtils.isNotBlank(blobField)) { + fileExtraFieldBuild(apiName, list, fieldList, fields); + } + log.info("api {} not exist, create..", apiName); + //新增一个用来存储新sfid的字段 + Map map = Maps.newHashMap(); + map.put("type", "varchar(255)"); + map.put("comment", "新sfid"); + map.put("name", "new_id"); + list.add(map); + + fields.add("new_id"); + + // 构建索引 + List> index = Lists.newArrayList(); + for (String tableIndex : Const.TABLE_INDEX) { + if (!fields.contains(tableIndex)) { + continue; + } + Map createDateMap = Maps.newHashMap(); + createDateMap.put("field", tableIndex); + createDateMap.put("name", String.format("IDX_%s_%s", apiName, tableIndex)); + index.add(createDateMap); + } + + + //DeleteEvent 回收站新增是否删除，数据内同字段 + if ("DeleteEvent".equals(apiName)){ + Map map1 = Maps.newHashMap(); + map1.put("type", "tinyint(1) DEFAULT 0"); + map1.put("comment", "是否删除"); + map1.put("name", "is_delete"); + list.add(map1); + + Map map2 = Maps.newHashMap(); + map2.put("type", "text"); + map2.put("comment", "完整数据"); + map2.put("name", "all_data"); + list.add(map2); + } + //是否更新 + Map map3 = Maps.newHashMap(); + map3.put("type", "tinyint(1) DEFAULT 0"); + map3.put("comment", "是否更新"); + map3.put("name", "is_update"); + list.add(map3); + + //是否更新 + Map map4 = Maps.newHashMap(); + map4.put("type", "text"); + map4.put("comment", "更新错误信息"); + map4.put("name", "error_message"); + list.add(map4); + + boolean hasId = list.stream().anyMatch(t -> "Id".equals(t.get("name").toString())); + + customMapper.createTable(apiName, label, list, index,hasId); + // 生成字段映射 + QueryWrapper delfieldQw = new QueryWrapper<>(); + delfieldQw.eq("api", apiName); + dataFieldService.remove(delfieldQw); + dataFieldService.saveBatch(fieldList); + // 生成picklist + QueryWrapper deldQw = new QueryWrapper<>(); + deldQw.eq("api", apiName); + dataPicklistService.remove(deldQw); + dataPicklistService.saveBatch(dataPicklists); + // 生成batch + Date startDate = DataUtil.DEFAULT_BEGIN_DATE; + // 取当天零点 作为最大时间 + Date endCreateDate = DateUtils.parseDate(DateFormatUtils.format(now, "yyyy-MM-dd"), "yyyy-MM-dd"); + + QueryWrapper delQw = new QueryWrapper<>(); + delQw.eq("name", apiName); + dataBatchService.remove(delQw); + if (createBatch && Const.BATCH_FILTERS.stream().noneMatch(t -> apiName.indexOf(t) > 0)) { + // 匹配的不生成 + DataBatch one = new DataBatch(); + one.setFirstDbNum(0); + one.setFirstSfNum(0); + one.setDbNum(0); + one.setSfNum(0); + one.setName(apiName); + one.setLabel(label); + if (hasCreatedDate) { + List> bachList = customMapper.list("code,value","system_config","code ='"+BATCH_TYPE+"'"); + String batchType = bachList.get(0).get("value").toString(); + saveBatch(batchType,startDate,endCreateDate,apiName,connection,one); + } else { + // 没创建时间 开始结束时间为空 + dataBatchService.save(one); + } + } + DataObject update = new DataObject(); + update.setLabel(label); + update.setName(apiName); + update.setLastUpdateDate(endCreateDate); + update.setBlobField(blobField); + if (count>0 || apiName.endsWith("Share")){ + update.setIsEditable(false); + } + update.setKeyPrefix(dsr.getKeyPrefix()); + dataObjectService.saveOrUpdate(update); + } + }catch (Exception e){ + log.info("创建表失败" , e); + throw new RuntimeException(e); + } + } + + /** + * 保存批次信息，根据数据量自动调整批次粒度 + * @param batchType 批次类型（年、月、周） + * @param startDate 开始日期 + * @param endDate 结束日期 + * @param apiName API名称 + * @param connection Salesforce连接 + * @param one 批次对象模板 + */ + public void saveBatch(String batchType, Date startDate, Date endDate, String apiName, PartnerConnection connection, DataBatch one) { + // 添加递归深度限制，避免栈溢出 + saveBatchRecursive(batchType, startDate, endDate, apiName, connection, one, 0); + log.info("API {} 批次创建完成", apiName); + } + + /** + * 递归保存批次信息 + * @param batchType 批次类型 + * @param startDate 开始日期 + * @param endDate 结束日期 + * @param apiName API名称 + * @param connection Salesforce连接 + * @param one 批次对象模板 + * @param depth 递归深度 + */ + private void saveBatchRecursive(String batchType, Date startDate, Date endDate, String apiName, PartnerConnection connection, DataBatch one, int depth) { + // 限制递归深度，避免栈溢出 + if (depth > 10) { + log.warn("递归深度超过限制，直接保存批次: {}, startDate: {}, endDate: {}", apiName, startDate, endDate); + DataBatch dataBatch = one.clone(); + dataBatch.setSyncStartDate(startDate); + dataBatch.setSyncEndDate(endDate); + dataBatchService.save(dataBatch); + return; + } + + Date currentStartDate = startDate; + Date lastDay; + + do { + // 根据批次类型获取结束日期 + lastDay = getLastDay(batchType, endDate, currentStartDate); + log.debug("当前批次起始日期: {}, 计算得到的结束日期: {}", currentStartDate, lastDay); + + // 添加边界检查，避免无限循环 + if (lastDay.compareTo(currentStartDate) <= 0) { + log.warn("计算得到的结束日期({})早于或等于起始日期({})，跳过该批次", lastDay, currentStartDate); + break; + } + + // 查询salesforce当前批次数据量 + SalesforceParam salesforceParam = new SalesforceParam(); + salesforceParam.setApi(apiName); + salesforceParam.setBeginCreateDate(currentStartDate); + salesforceParam.setEndCreateDate(lastDay); + Integer totalNum = 0; + int retryCount = 0; + int maxRetries = 3; + + // 增强异常处理，添加重试机制 + while (retryCount < maxRetries) { + try { + totalNum = commonService.countSfNum(connection, salesforceParam); + log.debug("API {} 在 {} 到 {} 期间的数据量为: {}", apiName, currentStartDate, lastDay, totalNum); + break; // 成功获取数据量，跳出重试循环 + } catch (Exception e) { + retryCount++; + log.error("api {} count error, retry {}/{}", apiName, retryCount, maxRetries, e); + if (retryCount >= maxRetries) { + log.error("api {} count failed after {} retries", apiName, maxRetries, e); + // 如果重试失败，使用默认值继续处理 + totalNum = 0; + } else { + try { + Thread.sleep(1000 * retryCount); // 指数退避 + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Thread interrupted during retry delay", ie); + } + } + } + } + + // 如果当前批次数据量超过50万，则细化批次粒度并递归处理 + if (totalNum > 500000) { + log.info("API {} 当前批次数据量 {} 超过50万，需要细化批次粒度", apiName, totalNum); + if (BATCH_TYPE_YEAR.equals(batchType)) { + // 年批次超过50万，细化为月批次 + log.info("将年批次细化为月批次"); + saveBatchRecursive(BATCH_TYPE_MONTH, currentStartDate, lastDay, apiName, connection, one, depth + 1); + } else if (BATCH_TYPE_MONTH.equals(batchType)) { + // 月批次超过50万，细化为周批次 + log.info("将月批次细化为周批次"); + saveBatchRecursive(BATCH_TYPE_WEEK, currentStartDate, lastDay, apiName, connection, one, depth + 1); + } else { + // 已经是周粒度 still > 500000, 直接保存当前批次 + log.info("已经是周粒度但数据量仍超过50万，直接保存批次"); + DataBatch dataBatch = one.clone(); + dataBatch.setSyncStartDate(currentStartDate); + dataBatch.setSyncEndDate(lastDay); + dataBatch.setSfNum(totalNum); + dataBatchService.save(dataBatch); + } + } else { + // 数据量未超过50万，直接保存当前批次 + log.info("API {} 当前批次数据量 {} 未超过50万，直接保存批次", apiName, totalNum); + DataBatch dataBatch = one.clone(); + dataBatch.setSyncStartDate(currentStartDate); + dataBatch.setSyncEndDate(lastDay); + dataBatch.setSfNum(totalNum); + dataBatchService.save(dataBatch); + } + + currentStartDate = lastDay; + + // 添加安全检查，避免无限循环 + if (currentStartDate.compareTo(endDate) >= 0) { + break; + } + } while (lastDay.compareTo(endDate) < 0); + } + /** + * 创建分区表结构 + * @param apiName + * @param createBatch + * @throws Exception + */ + public void checkPartitionedApi(String apiName, Boolean createBatch) throws Exception { + // 加个锁 避免重复执行创建api + log.info("check api:{}", apiName); + + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("name",apiName); + long count = metaclassConfigService.count(queryWrapper); + + try { + boolean hasCreatedDate = false; + + PartnerConnection connection = salesforceConnect.createConnect(); + DataObject dataObject = dataObjectService.getById(apiName); + Date now = new Date(); + if (StringUtils.isBlank(customMapper.checkTable(apiName))) { + log.info("api:{} doesn't exist create", apiName); + // 构建字段 + List> list = Lists.newArrayList(); + DescribeSObjectResult dsr = connection.describeSObject(apiName); + String label = dsr.getLabel(); + List fieldList = Lists.newArrayList(); + List fields = Lists.newArrayList(); + String blobField = null; + List dataPicklists = Lists.newArrayList(); + for (Field field : dsr.getFields()) { + // 过滤字段 + if (Const.TABLE_FILTERS.contains(field.getName())) { + continue; + } + if (Const.CREATED_DATE.equalsIgnoreCase(field.getName())) { + hasCreatedDate = true; + } + Map map = Maps.newHashMap(); + String sfType = field.getType().toString(); + if ("base64".equalsIgnoreCase(sfType)) { + blobField = field.getName(); + } + map.put("type", DataUtil.fieldTypeToMysql(field)); + // 英文' 转换为 \' + map.put("comment", field.getLabel().replaceAll("'", "\\\\'")); + map.put("name", field.getName()); + list.add(map); + DataField dataField = new DataField(); + dataField.setApi(apiName); + dataField.setSfType(sfType); + dataField.setField(field.getName()); + dataField.setName(field.getLabel()); + dataField.setIsCreateable(field.getCreateable()); + dataField.setIsUpdateable(field.getUpdateable()); + dataField.setIsNillable(field.getNillable()); + dataField.setIsDefaultedOnCreate(field.getDefaultedOnCreate()); + String join = null; + // 会有非常多映射关系的字段在 这里置空不管 + if (field.getReferenceTo().length <= 3) { + join = StringUtils.join(field.getReferenceTo(), ","); + } else { + log.info("referenceTo too long set null, api:{}, field:{}, reference to:{}", apiName, field.getName(), StringUtils.join(field.getReferenceTo(), ",")); + } + if (field.getReferenceTo().length > 1){ + if (field.getName().contains("Id") && !"OwnerId".equals(field.getName()) && !"UserOrGroupId".equals(field.getName())){ + LinkConfig linkConfig = new LinkConfig(); + linkConfig.setApi(apiName); + linkConfig.setLabel(label); + linkConfig.setField(field.getName()); + linkConfig.setLinkField(StringUtils.replace(field.getName(), "Id", "_Type")); + linkConfigService.save(linkConfig); + } + } + // picklist保存到picklist表 + if ("picklist".equalsIgnoreCase(sfType)) { + join = "data_picklist"; + PicklistEntry[] picklistValues = field.getPicklistValues(); + if (ArrayUtils.isNotEmpty(picklistValues)) { + for (PicklistEntry picklistValue : picklistValues) { + DataPicklist dataPicklist = new DataPicklist(); + dataPicklist.setApi(apiName); + dataPicklist.setField(field.getName()); + dataPicklist.setLabel(picklistValue.getLabel()); + dataPicklist.setValue(picklistValue.getValue()); + dataPicklists.add(dataPicklist); + } + } + } + dataField.setReferenceTo(join); + fieldList.add(dataField); + fields.add(field.getName()); + } + // 存在ownerId字段 + String extraField = ""; + if (dataObject != null && StringUtils.isNotBlank(dataObject.getExtraField())) { + extraField = dataObject.getExtraField(); + } + if (fields.stream().anyMatch(Const.OWNER_ID::equalsIgnoreCase)) { + if (StringUtils.isNotBlank(extraField)) { + extraField += ",Owner.Type"; + } else { + extraField = "Owner.Type"; + } + } + // 是否存在额外字段 配置上表 + if (StringUtils.isNotBlank(extraField)) { + Set extras = Arrays.stream(StringUtils.split(extraField.replaceAll(StringUtils.SPACE, StringUtils.EMPTY), ",")).collect(Collectors.toSet()); + for (String extra : extras) { + String fieldName = extra.replaceAll("\\.", "_"); + DataField dataField = new DataField(); + dataField.setApi(apiName); + dataField.setField(fieldName); + dataField.setName(extra); + fieldList.add(dataField); + fields.add(fieldName); + Map map = Maps.newHashMap(); + map.put("type", "varchar(255)"); + map.put("comment", extra); + map.put("name", fieldName); + list.add(map); + } + } + // 存在附件的表 作特殊处理 构建两个字段 + if (StringUtils.isNotBlank(blobField)) { + fileExtraFieldBuild(apiName, list, fieldList, fields); + } + log.info("api {} not exist, create..", apiName); + //新增一个用来存储新sfid的字段 + Map map = Maps.newHashMap(); + map.put("type", "varchar(255)"); + map.put("comment", "新sfid"); + map.put("name", "new_id"); + list.add(map); + + fields.add("new_id"); + + // 构建索引 + List> index = Lists.newArrayList(); + for (String tableIndex : Const.TABLE_INDEX) { + if (!fields.contains(tableIndex)) { + continue; + } + Map createDateMap = Maps.newHashMap(); + createDateMap.put("field", tableIndex); + createDateMap.put("name", String.format("IDX_%s_%s", apiName, tableIndex)); + index.add(createDateMap); + } + + // 构建分区结构，按object上的时间字段构建分区 + List> partitions = new ArrayList<>(); + + // 查找对象上的时间字段作为分区键 + String partitionKey = "CreatedDate"; // 默认使用创建时间字段 + for (Field field : dsr.getFields()) { + if ("datetime".equalsIgnoreCase(field.getType().toString()) || + "date".equalsIgnoreCase(field.getType().toString())) { + partitionKey = field.getName(); + break; + } + } + + // 添加默认分区，按照日期进行分区 + Date defaultBeginDate = DataUtil.DEFAULT_BEGIN_DATE; + Calendar calendar = Calendar.getInstance(); + int currentYear = calendar.get(Calendar.YEAR); + + Calendar startCalendar = Calendar.getInstance(); + startCalendar.setTime(defaultBeginDate); + int startYear = startCalendar.get(Calendar.YEAR); + + // 创建从开始年份到当前年份+1的分区，每个年份一个分区 + for (int year = startYear; year <= currentYear + 1; year++) { + Map partition = new HashMap<>(); + partition.put("name", "p" + year); + partition.put("value", String.format("'%s-01-01 00:00:00'", year + 1)); // datetime类型的值 + partitions.add(partition); + } + + // 添加一个未来分区 + Map futurePartition = new HashMap<>(); + futurePartition.put("name", "p" + (currentYear + 2)); + futurePartition.put("value", "'9999-12-31 23:59:59'"); + partitions.add(futurePartition); + + //DeleteEvent 回收站新增是否删除，数据内同字段 + if ("DeleteEvent".equals(apiName)){ + Map map1 = Maps.newHashMap(); + map1.put("type", "tinyint(1) DEFAULT 0"); + map1.put("comment", "是否删除"); + map1.put("name", "is_delete"); + list.add(map1); + + Map map2 = Maps.newHashMap(); + map2.put("type", "text"); + map2.put("comment", "完整数据"); + map2.put("name", "all_data"); + list.add(map2); + } + //是否更新 + Map map3 = Maps.newHashMap(); + map3.put("type", "tinyint(1) DEFAULT 0"); + map3.put("comment", "是否更新"); + map3.put("name", "is_update"); + list.add(map3); + + //是否更新 + Map map4 = Maps.newHashMap(); + map4.put("type", "text"); + map4.put("comment", "更新错误信息"); + map4.put("name", "error_message"); + list.add(map4); + // 创建RANGE分区表，按年份分区 + customMapper.createRangePartitionTable( + apiName, + label, + list, + index, + partitionKey, // 使用动态查找的分区键 + partitions + ); + // 生成字段映射 + QueryWrapper delfieldQw = new QueryWrapper<>(); + delfieldQw.eq("api", apiName); + dataFieldService.remove(delfieldQw); + dataFieldService.saveBatch(fieldList); + // 生成picklist + QueryWrapper deldQw = new QueryWrapper<>(); + deldQw.eq("api", apiName); + dataPicklistService.remove(deldQw); + dataPicklistService.saveBatch(dataPicklists); + // 生成batch + Date startDate = DataUtil.DEFAULT_BEGIN_DATE; + // 取当天零点 作为最大时间 + Date endCreateDate = DateUtils.parseDate(DateFormatUtils.format(now, "yyyy-MM-dd"), "yyyy-MM-dd"); + Date lastDay = null; + QueryWrapper delQw = new QueryWrapper<>(); + delQw.eq("name", apiName); + dataBatchService.remove(delQw); + if (createBatch && Const.BATCH_FILTERS.stream().noneMatch(t -> apiName.indexOf(t) > 0)) { + // 匹配的不生成 + DataBatch one = new DataBatch(); + one.setFirstDbNum(0); + one.setFirstSfNum(0); + one.setDbNum(0); + one.setSfNum(0); + one.setName(apiName); + one.setLabel(label); + if (hasCreatedDate) { + List> bachList = customMapper.list("code,value","system_config","code ='"+BATCH_TYPE+"'"); + String batchType = bachList.get(0).get("value").toString(); + do { + lastDay = getLastDay(batchType, endCreateDate, startDate); + DataBatch dataBatch = one.clone(); + dataBatch.setSyncStartDate(startDate); + dataBatch.setSyncEndDate(lastDay); + dataBatchService.save(dataBatch); + startDate = lastDay; + } while (lastDay.compareTo(endCreateDate) < 0); + } else { + // 没创建时间 开始结束时间为空 + dataBatchService.save(one); + } + } + DataObject update = new DataObject(); + update.setLabel(label); + update.setName(apiName); + update.setLastUpdateDate(endCreateDate); + update.setBlobField(blobField); + if (count>0 || apiName.endsWith("Share")){ + update.setIsEditable(false); + } + update.setKeyPrefix(dsr.getKeyPrefix()); + dataObjectService.saveOrUpdate(update); + } + }catch (Exception e){ + EmailUtil.send("DataDump ERROR", e.getMessage()); + } + } + + @Override + public ReturnT createApi(SalesforceParam param) throws Exception { + List apis; + if (StringUtils.isBlank(param.getApi())) { + apis = dataObjectService.list().stream().map(DataObject::getName).collect(Collectors.toList()); + } else { + apis = DataUtil.toIdList(param.getApi()); + } + log.info("打印所有待同步表：" +apis.toString()); + for (String api : apis) { + try { + checkApi(api, true); + }catch (Exception e){ + log.info("创建表结构异常！",e); + String message = e.getMessage(); + String format = String.format("创建表结构 error, api name: %s, \nparam: %s, \ncause:\n%s", api, com.alibaba.fastjson2.JSON.toJSONString(param, DataDumpParam.getFilter()), message); + EmailUtil.send("DataDump ERROR", format); + } + } + return ReturnT.SUCCESS; + } + + @Override + public ReturnT createBigObjectApi(SalesforceParam param) throws Exception { + + String api = param.getApi(); + log.info("开始创建BigObject API: {}", api); + + PartnerConnection connection = salesforceConnect.createConnect(); + log.debug("Salesforce连接创建成功"); + + DescribeSObjectResult dsr = connection.describeSObject(api); + log.info("获取对象描述信息成功, API: {}, Label: {}", api, dsr.getLabel()); + + + // 查询data_field表，如果有数据则不执行，没有数据执行 + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("api", api); + long count = dataFieldService.count(queryWrapper); + + if (count == 0) { + List fieldList = Lists.newArrayList(); + log.debug("开始处理字段信息，字段总数: {}", dsr.getFields().length); + for (Field field : dsr.getFields()) { + DataField dataField = new DataField(); + dataField.setApi(api); + dataField.setSfType(field.getType().toString()); + dataField.setField(field.getName()); + dataField.setName(field.getLabel()); + dataField.setIsCreateable(field.getCreateable()); + dataField.setIsUpdateable(field.getUpdateable()); + dataField.setIsNillable(field.getNillable()); + dataField.setIsDefaultedOnCreate(field.getDefaultedOnCreate()); + fieldList.add(dataField); + } + log.info("字段信息处理完成，共处理{}个字段", fieldList.size()); + + dataFieldService.saveBatch(fieldList); + log.info("字段信息批量保存完成"); + } else { + log.info("API {} 的字段信息已存在，跳过字段处理", api); + } + + DataObject dataObject = dataObjectService.getById(api); + dataObject.setLabel(dsr.getLabel()); + dataObject.setLastUpdateDate(new Date()); + dataObjectService.updateById(dataObject); + log.info("数据对象信息更新完成"); + + + log.info("BigObject API创建完成: {}", api); + return ReturnT.SUCCESS; + } + + @Override + public ReturnT getAllApi() throws Exception { + PartnerConnection partnerConnection = salesforceConnect.createConnect(); + DescribeGlobalResult result = partnerConnection.describeGlobal(); + DescribeGlobalSObjectResult[] Sobjects = result.getSobjects(); + List dataObjects = new ArrayList<>(); + + for(DescribeGlobalSObjectResult Sobject : Sobjects) { + + DataObject dataObject = new DataObject(); + String SobjectName = Sobject.getName(); + String SobjectLabel = Sobject.getLabel(); + + dataObject.setName(SobjectName); + dataObject.setLabel(SobjectLabel); + + dataObjects.add(dataObject); + + } + dataObjectService.saveBatch(dataObjects); + return ReturnT.SUCCESS; + } + + /** + * 是否存在创建时间字段 + * + * @param api api名称 + * @return true存在创建时间字段 false不存在 + */ + private Boolean hasCreatedDate(String api) { + QueryWrapper qw = new QueryWrapper<>(); + qw.eq("api", api).eq("field", Const.CREATED_DATE).last("limit 1"); + DataField one = dataFieldService.getOne(qw); + return one != null; + } + + + + /** + * 存在附件的表 特殊构建 + * + * @param apiName 表名称 + * @param list 构建表字段的list + * @param fieldList 构建字段表的list + * @param fields 字段名称list + */ + private static void fileExtraFieldBuild(String apiName, List> list, List fieldList, List fields) { + if ("Document".equals(apiName)){ + DataField dataField = new DataField(); + dataField.setApi(apiName); + dataField.setField("url"); + dataField.setName("文件路径"); + fieldList.add(dataField); + fields.add("url"); + Map map = Maps.newHashMap(); + map.put("type", "text"); + map.put("comment", "文件路径"); + map.put("name", "localUrl"); + list.add(map); + }else { + DataField dataField = new DataField(); + dataField.setApi(apiName); + dataField.setField("url"); + dataField.setName("文件路径"); + fieldList.add(dataField); + fields.add("url"); + Map map = Maps.newHashMap(); + map.put("type", "text"); + map.put("comment", "文件路径"); + map.put("name", "url"); + list.add(map); + } + { + DataField dataField = new DataField(); + dataField.setApi(apiName); + dataField.setField("is_dump"); + dataField.setName("是否存储"); + fieldList.add(dataField); + fields.add("is_dump"); + Map map = Maps.newHashMap(); + map.put("type", "tinyint(1) DEFAULT 0"); + map.put("comment", "是否存储"); + map.put("name", "is_dump"); + list.add(map); + } + { + DataField dataField = new DataField(); + dataField.setApi(apiName); + dataField.setField("is_upload"); + dataField.setName("是否上传"); + fieldList.add(dataField); + fields.add("is_upload"); + Map map = Maps.newHashMap(); + map.put("type", "tinyint(1) DEFAULT 0"); + map.put("comment", "是否上传"); + map.put("name", "is_upload"); + list.add(map); + } + } + + @Override + public ReturnT getDocumentLink(String paramStr) throws Exception { + String api = "ContentDocumentLink"; + PartnerConnection partnerConnection = salesforceConnect.createConnect(); + PartnerConnection connection = salesforceTargetConnect.createConnect(); + List> list = customMapper.list("Id", "ContentDocument", "new_id is not null"); + DescribeSObjectResult dsr = partnerConnection.describeSObject(api); + List fields = customMapper.getFields(api).stream().map(String::toUpperCase).collect(Collectors.toList()); + Field[] dsrFields = dsr.getFields(); + try { + if (list != null && list.size() > 0) { + for (Map map : list) { + String contentDocumentId = (String) map.get("Id"); + String sql = "SELECT Id, LinkedEntityId, LinkedEntity.Type, ContentDocumentId, Visibility, ShareType, SystemModstamp, IsDeleted FROM ContentDocumentLink where ContentDocumentId = '" + contentDocumentId + "'"; + JSONArray objects = null; + try { + QueryResult queryResult = partnerConnection.queryAll(sql); + SObject[] records = queryResult.getRecords(); + objects = DataUtil.toJsonArray(records, dsrFields); + saveOrUpdate(api, fields, records, objects, true); + } catch (Throwable e) { + log.error("getDocumentLink error api:{}, data:{}", api, JSON.toJSONString(objects), e); + TimeUnit.MINUTES.sleep(1); + return ReturnT.FAIL; + } + } + + //表内数据总量 + Integer count = customMapper.countBySQL(api, "where ShareType = 'V' and new_id = '0'"); + + int page = count % 200 == 0 ? count / 200 : (count / 200) + 1; + for (int i = 0; i < page; i++) { + List> linkList = customMapper.list("Id,LinkedEntityId,ContentDocumentId,LinkedEntity_Type,ShareType,Visibility", api, "ShareType = 'V' and new_id = '0' order by Id asc limit 200"); + SObject[] accounts = new SObject[linkList.size()]; + String[] ids = new String[linkList.size()]; + int index = 0; + for (Map map : linkList) { + String linkedEntityId = (String) map.get("LinkedEntityId"); + String id = (String) map.get("Id"); + String contentDocumentId = (String) map.get("ContentDocumentId"); + String linkedEntityType = (String) map.get("LinkedEntity_Type"); + String shareType = (String) map.get("ShareType"); + String Visibility = (String) map.get("Visibility"); + + // dataObject查询 + QueryWrapper qw = new QueryWrapper<>(); + qw.eq("name", linkedEntityType); + List objects = dataObjectService.list(qw); + if (!objects.isEmpty()) { + Map dMap = customMapper.getById("new_id", "ContentDocument", contentDocumentId); + Map lMap = customMapper.getById("new_id", linkedEntityType, linkedEntityId); + + SObject account = new SObject(); + account.setType(api); + account.setField("ContentDocumentId", dMap.get("new_id").toString()); + account.setField("LinkedEntityId", lMap.get("new_id").toString()); + account.setField("ShareType", shareType); + account.setField("Visibility", Visibility); + ids[index] = id; + accounts[index] = account; + index++; + } + } + try { + SaveResult[] saveResults = connection.create(accounts); + for (int j = 0; j < saveResults.length; j++) { + if (!saveResults[j].getSuccess()) { + String format = String.format("数据导入 error, api name: %s, \nparam: %s, \ncause:\n%s", api, JSON.toJSONString(DataDumpParam.getFilter()), com.alibaba.fastjson.JSON.toJSONString(saveResults[j])); + log.error(format); + } else { + List> dList = new ArrayList<>(); + Map linkMap = new HashMap<>(); + linkMap.put("key", "new_id"); + linkMap.put("value", saveResults[j].getId()); + dList.add(linkMap); + customMapper.updateById("ContentDocumentLink", dList, ids[j]); + } + } + } catch (Exception e) { + log.error("getDocumentLink error api:{}, data:{}", api, JSON.toJSONString(accounts), e); + EmailUtil.send("------------------", JSON.toJSONString(accounts)); + throw new RuntimeException(e); + } + } + } + } catch (Exception e) { + log.error("getDocumentLink error api:{}, data:{}", api, JSON.toJSONString(list), e); + return ReturnT.FAIL; + } + return null; + } + + @Override + public String getDocumentId(PartnerConnection partnerConnection, String contentVersionId) throws Exception{ + QueryResult queryResult = partnerConnection.queryAll("SELECT Id, ContentDocumentId, IsDeleted FROM ContentVersion where Id = " + "'" + contentVersionId + "'"); + DescribeSObjectResult dsr = partnerConnection.describeSObject("ContentVersion"); + Field[] dsrFields = dsr.getFields(); + SObject[] records = queryResult.getRecords(); + JSONArray objects = DataUtil.toJsonArray(records, dsrFields); + if (objects != null && objects.size() > 0) { + JSONObject jsonObject = objects.getJSONObject(0); + return jsonObject.getString("ContentDocumentId"); + } + return null; + } + + @Override + public ReturnT getChatter(SalesforceParam param) throws Exception { + List> feedList = customMapper.list("Parent_Type", "FeedItem", "1=1 GROUP BY Parent_Type"); + for (Map map : feedList){ + String parentType = map.get("Parent_Type").toString(); + List types = new ArrayList<>(); + //检测表是否存在 + if (StringUtils.isBlank(customMapper.checkTable(parentType))){ + types.add(parentType); + } + if (types.size()>0){ + customMapper.delete("FeedItem", types); + customMapper.delete("FeedComment", types); + } + } + return null; + } + + @Override + public ReturnT createLinkTypeField(SalesforceParam param) throws Exception { + try { + List apis; + if (StringUtils.isBlank(param.getApi())) { + apis = dataObjectService.list().stream().map(DataObject::getName).collect(Collectors.toList()); + } else { + apis = DataUtil.toIdList(param.getApi()); + } + QueryWrapper wrapper = new QueryWrapper<>(); + wrapper.eq("is_link",1).eq("is_create",0); + if (!apis.isEmpty()){ + wrapper.in("api", apis); + } + + List list = linkConfigService.list(wrapper); + for (LinkConfig config : list) { + if (StringUtils.isBlank(customMapper.checkTable(config.getApi()))){ + log.info("表api：{} 不存在！", config.getApi()); + continue; + } + log.info("表api：{} 未创建字段：{}，现在创建！", config.getApi(),config.getLinkField()); + //创建字段 + customMapper.createField(config.getApi(), config.getLinkField()); + config.setIsCreate(true); + linkConfigService.updateById(config); + } + }catch (Exception e){ + log.error(JSON.toJSONString(e)); + } + return ReturnT.SUCCESS; + } + + @Override + public ReturnT syncFieldInfo(SalesforceParam param) throws Exception { + try { + List apis; + if (StringUtils.isBlank(param.getApi())) { + apis = dataObjectService.list().stream().map(DataObject::getName).collect(Collectors.toList()); + } else { + apis = DataUtil.toIdList(param.getApi()); + } + + PartnerConnection connection = salesforceConnect.createConnect(); + log.info("Salesforce连接创建成功"); + + for (String apiName : apis) { + log.info("开始同步对象字段信息: {}", apiName); + + DescribeSObjectResult dsr = connection.describeSObject(apiName); + + for (Field field : dsr.getFields()) { + String fieldName = field.getName(); + String calculatedFormula = field.getCalculatedFormula(); + + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("api", apiName).eq("field", fieldName); + DataField dataField = dataFieldService.getOne(queryWrapper); + + if (dataField != null) { + if (StringUtils.isNotBlank(calculatedFormula)) { + dataField.setCalculatedFormula(calculatedFormula); + dataFieldService.updateById(dataField); + log.info("更新字段公式: api={}, field={}, formula={}", apiName, fieldName, calculatedFormula); + } + } + } + + log.info("对象字段信息同步完成: {}", apiName); + } + + return ReturnT.SUCCESS; + } catch (Exception e) { + log.error("同步字段信息失败", e); + return new ReturnT<>(500, "同步字段信息失败: " + e.getMessage()); + } + } + + public Date getLastDay(String batchType,Date endDate,Date startDate){ + switch (batchType) { + case BATCH_TYPE_WEEK: + return DataUtil.getWeekLastDay(endDate, startDate); + case BATCH_TYPE_MONTH: + return DataUtil.getMonthLastDay(endDate, startDate); + case BATCH_TYPE_YEAR: + return DataUtil.getYearLastDay(endDate, startDate); + default: + return endDate; + } + } +} diff --git a/datai-scenes/datai-scene-salesforce/docs/reference-code/salesforce-pubsub-realtime-sync/data-dump/DataImportNewServiceImpl.java b/datai-scenes/datai-scene-salesforce/docs/reference-code/salesforce-pubsub-realtime-sync/data-dump/DataImportNewServiceImpl.java new file mode 100644 index 00000000..1e6ccf88 --- /dev/null +++ b/datai-scenes/datai-scene-salesforce/docs/reference-code/salesforce-pubsub-realtime-sync/data-dump/DataImportNewServiceImpl.java @@ -0,0 +1,4002 @@ +package com.celnet.datadump.service.impl; + +import com.alibaba.excel.EasyExcel; +import com.alibaba.excel.ExcelWriter; +import com.alibaba.excel.write.metadata.WriteSheet; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson2.JSONObject; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper; +import com.celnet.datadump.config.SalesforceConnect; +import com.celnet.datadump.config.SalesforceExecutor; +import com.celnet.datadump.config.SalesforceTargetConnect; +import com.celnet.datadump.entity.*; +import com.celnet.datadump.enums.FileType; +import com.celnet.datadump.global.Const; +import com.celnet.datadump.global.SystemConfigCode; +import com.celnet.datadump.mapper.CustomMapper; +import com.celnet.datadump.param.DataDumpParam; +import com.celnet.datadump.param.DataDumpSpecialParam; +import com.celnet.datadump.param.SalesforceParam; +import com.celnet.datadump.service.*; +import com.celnet.datadump.util.*; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.sforce.soap.partner.*; +import com.sforce.soap.partner.sobject.SObject; +import com.xxl.job.core.biz.model.ReturnT; +import com.xxl.job.core.log.XxlJobLogger; +import com.xxl.job.core.util.DateUtil; +import lombok.extern.slf4j.Slf4j; +import okhttp3.Response; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.Get; +import org.apache.commons.compress.utils.IOUtils; +import org.apache.commons.lang.time.DateUtils; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.DateFormatUtils; +import org.apache.http.HttpEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.mime.MultipartEntityBuilder; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import javax.annotation.PostConstruct; +import java.io.*; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + + +@Service +@Slf4j +public class DataImportNewServiceImpl implements DataImportNewService { + + @Autowired + private SalesforceTargetConnect salesforceTargetConnect; + + @Autowired + private DataBatchHistoryService dataBatchHistoryService; + + @Autowired + private SalesforceExecutor salesforceExecutor; + + @Autowired + private SalesforceConnect salesforceConnect; + + @Autowired + private DataObjectService dataObjectService; + + @Autowired + private DataBatchService dataBatchService; + + @Autowired + private DataFieldService dataFieldService; + + @Autowired + private CustomMapper customMapper; + + @Autowired + private CommonService commonService; + + @Autowired + private LinkConfigService linkConfigService; + + @Autowired + private DataLogService dataLogService; + + @Autowired + private RichTextLogService richTextLogService; + + private static final String TEMP_FILE_PATH = "verify/"; + + private static String FILE_UPLOAD_URL = ""; + + private static String FILE_DOWNLOAD_URL = ""; + + @PostConstruct + public void init() { + List> poll = customMapper.list("code,value","org_config",null); + for (Map map1 : poll) { + if ("FILE_UPLOAD_URL".equals(map1.get("code"))) { + FILE_UPLOAD_URL = (String) map1.get("value"); + } + if ("FILE_DOWNLOAD_URL".equals(map1.get("code"))) { + FILE_DOWNLOAD_URL = (String) map1.get("value"); + } + } + } + + static { + // 检测路径是否存在 不存在则创建 + File excel = new File(TEMP_FILE_PATH); + if (!excel.exists()) { + boolean mkdir = excel.mkdir(); + } + } + + /** + * Get返写个人客户联系人入口 + */ + @Override + public void getPersonContact(SalesforceParam param) throws Exception { + List> futures = Lists.newArrayList(); + try { + if (StringUtils.isNotBlank(param.getApi())) { + // 手动任务 + ReturnT result = manualGetPersonContact(param, futures); + if (result != null) { + } + } + } catch (Exception exception) { + salesforceExecutor.remove(futures.toArray(new Future[]{})); + log.error("immigration error", exception); + throw exception; + } + } + + /** + * 组装Get执行参数 + */ + public ReturnT manualGetPersonContact(SalesforceParam param, List> futures) throws Exception { + String api = "Contact"; + + TimeUnit.MILLISECONDS.sleep(1); + // 全量的时候 检测是否有自动任务锁住的表 + boolean isFull = CollectionUtils.isEmpty(param.getIds()); + if (isFull) { + QueryWrapper qw = new QueryWrapper<>(); + qw.eq("data_lock", 1); + List list = dataObjectService.list(qw); + if (CollectionUtils.isNotEmpty(list)) { + String apiNames = list.stream().map(DataObject::getName).collect(Collectors.joining()); + return new ReturnT<>(500, "api:" + apiNames + " is locked"); + } + } + + PartnerConnection connect = salesforceTargetConnect.createConnect(); + DataObject update = new DataObject(); + TimeUnit.MILLISECONDS.sleep(1); + try { + List salesforceParams = null; + + update.setName(api); + update.setDataLock(1); + dataObjectService.updateById(update); + + QueryWrapper dbQw = new QueryWrapper<>(); + dbQw.eq("name", api); + List list = dataBatchService.list(dbQw); + AtomicInteger batch = new AtomicInteger(1); + + if (CollectionUtils.isNotEmpty(list)) { + salesforceParams = list.stream().map(t -> { + SalesforceParam salesforceParam = param.clone(); + salesforceParam.setApi(t.getName()); + salesforceParam.setBeginCreateDate(t.getSyncStartDate()); + salesforceParam.setEndCreateDate(t.getSyncEndDate()); + salesforceParam.setBatch(batch.getAndIncrement()); + return salesforceParam; + }).collect(Collectors.toList()); + } + + // 手动任务优先执行 + for (SalesforceParam salesforceParam : salesforceParams) { + Future future = salesforceExecutor.execute(() -> { + try { + manualGetPersonContactId(salesforceParam, connect); + } catch (Throwable throwable) { + log.error("salesforceExecutor error", throwable); + throw new RuntimeException(throwable); + } + }, salesforceParam.getBatch(), 1); + futures.add(future); + } + // 等待当前所有线程执行完成 + salesforceExecutor.waitForFutures(futures.toArray(new Future[]{})); + update.setDataWork(0); + } catch (InterruptedException e) { + throw e; + } catch (Throwable e) { + log.error("manualImmigration error", e); + throw new RuntimeException(e); + } finally { + if (isFull) { + update.setName(api); + update.setDataLock(0); + dataObjectService.updateById(update); + } + } + return null; + } + + /** + * 获取newId,写入oldId + */ + private void manualGetPersonContactId(SalesforceParam param, PartnerConnection partnerConnection) throws Exception { + + String api = param.getApi(); + QueryWrapper dbQw = new QueryWrapper<>(); + dbQw.eq("api", api); + + String beginDateStr = null; + String endDateStr = null; + Date beginDate = param.getBeginCreateDate(); + Date endDate = param.getEndCreateDate(); + if (param.getBeginCreateDate() != null && param.getEndCreateDate() != null){ + beginDateStr = DateUtil.format(beginDate, "yyyy-MM-dd HH:mm:ss"); + endDateStr = DateUtil.format(endDate, "yyyy-MM-dd HH:mm:ss"); + } + + DescribeSObjectResult dsr = partnerConnection.describeSObject(api); + Field[] dsrFields = dsr.getFields(); + + //表内数据总量 + Integer count = customMapper.countBySQL(api, "where new_id is null and IsPersonAccount = 1 and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "'"); + + if (count == 0) { + return; + } + + //批量插入200一次 + int page = count%200 == 0 ? count/200 : (count/200) + 1; + + log.info("总反写 count:{}；总批次：{}；-开始时间:{}；-结束时间:{}；-api:{}；", count,page, beginDateStr, endDateStr, api); + + for (int i = 0; i < page; i++) { + List> data = customMapper.list("*", api, "new_id is null and IsPersonAccount = 1 and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "' limit 200"); + int size = data.size(); + + log.info("总反写 count:{}；当前批次：{}；-开始时间:{}；-结束时间:{}；-api:{}；", count,i, beginDateStr, endDateStr, api); + SObject[] accounts = new SObject[size]; + + // 新客户ID，旧联系人ID，用于更新本地数据 + Map idMaps = new HashMap<>(); + // 旧客户ID，新联系人ID，用于更新SF数据 + Map idMapa = new HashMap<>(); + + for (Map map : data) { + Map idMap = customMapper.getById("new_id", "Account", map.get("AccountId").toString()); + if(idMap.get("new_id") != null && StringUtils.isNotEmpty(idMap.get("new_id").toString())){ + // 新客户ID，旧联系人ID + idMaps.put(idMap.get("new_id").toString(),map.get("Id").toString()); + } + } + + String idStr = "("; + for (String ids : idMaps.keySet()) { + idStr += "'" + ids + "',"; // 拼接每个ID + } + if (idStr.endsWith(",")) { // 如果最后一个字符是逗号，说明循环正常结束 + idStr = idStr.substring(0, idStr.length() - 1); // 去掉最后一个多余的逗号 + } + idStr += ")"; // 添加右括号 + + try { + String sql = "SELECT Id,AccountId,Account.old_sfdc_id__c FROM Contact where AccountId in " + idStr ; + QueryResult queryResult = partnerConnection.queryAll(sql); + if (ObjectUtils.isEmpty(queryResult) || ObjectUtils.isEmpty(queryResult.getRecords())) { + break; + } + SObject[] records = queryResult.getRecords(); + com.alibaba.fastjson2.JSONArray objects = DataUtil.toJsonArray(records, dsrFields); + for (int z = 0; z < objects.size(); z++) { + JSONObject jsonObject = objects.getJSONObject(z); + String contactId = jsonObject.getString(Const.ID); + String accountId = jsonObject.getString("AccountId"); + String oldAccountId = jsonObject.getString("Account_old_sfdc_id__c"); + String id = idMaps.get(accountId); + List> maps = Lists.newArrayList(); + Map paramMap = Maps.newHashMap(); + paramMap.put("key", "new_id"); + paramMap.put("value", contactId); + maps.add(paramMap); + customMapper.updateById(api, maps, id); + idMapa.put(oldAccountId, contactId); + } + TimeUnit.MILLISECONDS.sleep(1); + + int index = 0; + for (Map map : data) { + SObject account = new SObject(); + account.setType(api); + account.setField("old_owner_id__c", map.get("OwnerId")); + account.setField("old_sfdc_id__c", map.get("Id")); + account.setId(idMapa.get(map.get("AccountId").toString())); + accounts[index] = account; + index++; + } + + SaveResult[] saveResults = partnerConnection.update(accounts); + for (SaveResult saveResult : saveResults) { + if (!saveResult.getSuccess()) { + log.info("-------------saveResults: {}", JSON.toJSONString(saveResult)); + String format = String.format("数据导入 error, api name: %s, \nparam: %s, \ncause:\n%s, \n数据实体类:\n%s", api, com.alibaba.fastjson2.JSON.toJSONString(param, DataDumpParam.getFilter()), JSON.toJSONString(saveResult)); + EmailUtil.send("DataDump ERROR", format); + return; + } + } + + } catch (Exception e) { + log.error("manualGetPersonContactId error api:{}", api, e); + throw e; + } + } + SalesforceParam countParam = new SalesforceParam(); + countParam.setApi(api); + countParam.setBeginCreateDate(beginDate); + countParam.setEndCreateDate(DateUtils.addSeconds(endDate, -1)); + // 存在isDeleted 只查询IsDeleted为false的 + if (dataFieldService.hasDeleted(countParam.getApi())) { + countParam.setIsDeleted(false); + } else { + // 不存在 过滤 + countParam.setIsDeleted(null); + } + Integer sfNum = 0; + // 不能 count Id + if (!"FeedItem".equals(api) && !"ContentFolderMember ".equals(api)){ + sfNum = commonService.countSfNum(partnerConnection, countParam); + } + + UpdateWrapper updateQw = new UpdateWrapper<>(); + updateQw.eq("name", api) + .eq("sync_start_date", beginDate) + .eq("sync_end_date", DateUtils.addSeconds(endDate, -1)) + .set("target_sf_num", sfNum); + dataBatchHistoryService.update(updateQw); + + UpdateWrapper updateQw2 = new UpdateWrapper<>(); + updateQw2.eq("name", api) + .eq("sync_start_date", beginDate) + .eq("sync_end_date", endDate) + .set("sf_add_num", sfNum); + dataBatchService.update(updateQw2); + } + + /** + * 数据更新Update入口 + */ + @Override + public ReturnT immigrationUpdateNew(SalesforceParam param) throws Exception { + List> futures = Lists.newArrayList(); + try { + if (StringUtils.isNotBlank(param.getApi())) { + if (1 == param.getType()){ + return maualupdateSfDataNew(param, futures); + }else { + return updateIncrementalSfDataNew(param, futures); + } + } else { + if (1 == param.getType()){ + autoUpdateSfDataNew(param, futures); + }else { + autoUpdateIncrementalSfDataNew(param, futures); + } + } + return ReturnT.SUCCESS; + } catch (InterruptedException e) { + throw e; + } catch (Throwable throwable) { + salesforceExecutor.remove(futures.toArray(new Future[]{})); + log.error("immigrationUpdate error", throwable); + throw throwable; + } + } + + /** + * 组装【单表】【存量】Update参数 + */ + public ReturnT maualupdateSfDataNew(SalesforceParam param, List> futures) throws Exception { + List apis = DataUtil.toIdList(param.getApi()); + + String beginDateStr = null; + String endDateStr = null; + if (param.getBeginCreateDate() != null && param.getEndCreateDate() != null){ + Date beginDate = param.getBeginCreateDate(); + Date endDate = param.getEndCreateDate(); + beginDateStr = DateUtil.format(beginDate, "yyyy-MM-dd HH:mm:ss"); + endDateStr = DateUtil.format(endDate, "yyyy-MM-dd HH:mm:ss"); + } + + // 全量的时候 检测是否有自动任务锁住的表 + boolean isFull = CollectionUtils.isEmpty(param.getIds()); + if (isFull) { + QueryWrapper qw = new QueryWrapper<>(); + qw.eq("data_lock", 1).in("name", apis); + List list = dataObjectService.list(qw); + if (CollectionUtils.isNotEmpty(list)) { + String apiNames = list.stream().map(DataObject::getName).collect(Collectors.joining()); + String message = "api:" + apiNames + " is locked"; + String format = String.format("数据导入 error, api name: %s, \nparam: %s, \ncause:\n%s", apiNames, com.alibaba.fastjson2.JSON.toJSONString(param, DataDumpParam.getFilter()), message); + EmailUtil.send("DataDump ERROR", format); + return new ReturnT<>(500, message); + } + } + + PartnerConnection partnerConnection = salesforceTargetConnect.createConnect(); + for (String api : apis) { + DataObject update = dataObjectService.getById(api); + try { + List salesforceParams = null; + QueryWrapper dbQw = new QueryWrapper<>(); + dbQw.eq("name", api); + if (StringUtils.isNotEmpty(beginDateStr) && StringUtils.isNotEmpty(endDateStr)) { + dbQw.eq("sync_start_date", beginDateStr); // 等于开始时间 + dbQw.eq("sync_end_date", endDateStr); // 等于结束时间 + } + List list = dataBatchService.list(dbQw); + AtomicInteger batch = new AtomicInteger(1); + if (CollectionUtils.isNotEmpty(list)) { + salesforceParams = list.stream().map(t -> { + SalesforceParam salesforceParam = param.clone(); + salesforceParam.setApi(t.getName()); + salesforceParam.setBeginCreateDate(t.getSyncStartDate()); + salesforceParam.setEndCreateDate(t.getSyncEndDate()); + salesforceParam.setBatch(batch.getAndIncrement()); + return salesforceParam; + }).collect(Collectors.toList()); + } + + // 手动任务优先执行 + for (SalesforceParam salesforceParam : salesforceParams) { + if (param.getIsSingleThread()){ + UpdateSfDataNew(salesforceParam, partnerConnection,update); + }else { + Future future = salesforceExecutor.execute(() -> { + try { + UpdateSfDataNew(salesforceParam, partnerConnection,update); + } catch (Throwable throwable) { + log.error("salesforceExecutor error", throwable); + throw new RuntimeException(throwable); + } + }, salesforceParam.getBatch(), 1); + futures.add(future); + } + } + // 等待当前所有线程执行完成 + salesforceExecutor.waitForFutures(futures.toArray(new Future[]{})); + + } catch (Exception e) { + throw e; + } finally { + if (isFull) { + update.setNeedUpdate(false); + update.setName(api); + update.setDataLock(0); + dataObjectService.updateById(update); + } + } + } + return ReturnT.SUCCESS; + } + + /** + * 组装【单表】【增量】Update参数 + */ + public ReturnT updateIncrementalSfDataNew(SalesforceParam param, List> futures) throws Exception { + List apis = DataUtil.toIdList(param.getApi()); + + // 全量的时候 检测是否有自动任务锁住的表 + boolean isFull = CollectionUtils.isEmpty(param.getIds()); + if (isFull) { + QueryWrapper qw = new QueryWrapper<>(); + qw.eq("data_lock", 1).in("name", apis); + List list = dataObjectService.list(qw); + if (CollectionUtils.isNotEmpty(list)) { + String apiNames = list.stream().map(DataObject::getName).collect(Collectors.joining()); + return new ReturnT<>(500, "api:" + apiNames + " is locked"); + } + } + + PartnerConnection partnerConnection = salesforceTargetConnect.createConnect(); + for (String api : apis) { + DataObject update = dataObjectService.getById(api); + try { + //无创建时间对象 + if (!dataFieldService.hasCreatedDate(api)) { + UpdateSfShareDataNew(api, partnerConnection, update); + continue; + } + QueryWrapper dbQw = new QueryWrapper<>(); + dbQw.eq("name", api).orderByDesc("sync_end_date").last("LIMIT 1"); + DataBatch t = dataBatchService.getOne(dbQw); + SalesforceParam salesforceParam = param.clone(); + salesforceParam.setApi(t.getName()); + salesforceParam.setBeginCreateDate(t.getSyncStartDate()); + salesforceParam.setEndCreateDate(t.getSyncEndDate()); + salesforceParam.setBatch(new AtomicInteger(1).getAndIncrement()); + if (param.getIsSingleThread()){ + UpdateSfDataNew(salesforceParam, partnerConnection,update); + }else { + // 手动任务优先执行 + Future future = salesforceExecutor.execute(() -> { + try { + UpdateSfDataNew(salesforceParam, partnerConnection,update); + } catch (Throwable throwable) { + log.error("salesforceExecutor error", throwable); + throw new RuntimeException(throwable); + } + }, salesforceParam.getBatch(), 1); + futures.add(future); + } + update.setNeedUpdate(false); + } catch (Exception e) { + throw e; + } finally { + if (isFull) { + update.setName(api); + update.setDataLock(0); + dataObjectService.updateById(update); + } + } + } + // 等待当前所有线程执行完成 + salesforceExecutor.waitForFutures(futures.toArray(new Future[]{})); + futures.clear(); + + return ReturnT.SUCCESS; + } + + /** + * 组装【多表】【存量】Update参数 + */ + private void autoUpdateSfDataNew(SalesforceParam param, List> futures) throws Exception { + + // 全量的时候 检测是否有自动任务锁住的表 + boolean isFull = CollectionUtils.isEmpty(param.getIds()); + if (isFull) { + QueryWrapper qw = new QueryWrapper<>(); + qw.eq("data_lock", 1); + List list = dataObjectService.list(qw); + if (CollectionUtils.isNotEmpty(list)) { + String apiNames = list.stream().map(DataObject::getName).collect(Collectors.joining()); + String message = "api:" + apiNames + " is locked"; + String format = String.format("数据导入 error, api name: %s, \nparam: %s, \ncause:\n%s", apiNames, com.alibaba.fastjson2.JSON.toJSONString(param, DataDumpParam.getFilter()), message); + EmailUtil.send("DataDump ERROR", format); + return ; + } + } + + String beginDateStr = null; + String endDateStr = null; + if (param.getBeginCreateDate() != null && param.getEndCreateDate() != null){ + Date beginDate = param.getBeginCreateDate(); + Date endDate = param.getEndCreateDate(); + beginDateStr = DateUtil.format(beginDate, "yyyy-MM-dd HH:mm:ss"); + endDateStr = DateUtil.format(endDate, "yyyy-MM-dd HH:mm:ss"); + } + + QueryWrapper qw = new QueryWrapper<>(); + qw.eq("need_update", 1) + .orderByAsc("data_index") + .last(" limit 10"); + + PartnerConnection partnerConnection = salesforceTargetConnect.createConnect(); + + while (true) { + List dataObjects = dataObjectService.list(qw); + //判断dataObjects是否为空 + if (CollectionUtils.isEmpty(dataObjects)) { + return; + } + + for (DataObject dataObject : dataObjects) { + TimeUnit.MILLISECONDS.sleep(1); + + DataObject update = new DataObject(); + update.setName(dataObject.getName()); + update.setDataLock(1); + dataObjectService.updateById(update); + TimeUnit.MILLISECONDS.sleep(1); + try { + String api = dataObject.getName(); + List salesforceParams = null; + QueryWrapper dbQw = new QueryWrapper<>(); + dbQw.eq("name", api); + if (StringUtils.isNotEmpty(beginDateStr) && StringUtils.isNotEmpty(endDateStr)) { + dbQw.eq("sync_start_date", beginDateStr); // 等于开始时间 + dbQw.eq("sync_end_date", endDateStr); // 等于结束时间 + } + List list = dataBatchService.list(dbQw); + AtomicInteger batch = new AtomicInteger(1); + if (CollectionUtils.isNotEmpty(list)) { + salesforceParams = list.stream().map(t -> { + SalesforceParam salesforceParam = param.clone(); + salesforceParam.setApi(t.getName()); + salesforceParam.setBeginCreateDate(t.getSyncStartDate()); + salesforceParam.setEndCreateDate(t.getSyncEndDate()); + salesforceParam.setBatch(batch.getAndIncrement()); + return salesforceParam; + }).collect(Collectors.toList()); + } + + // 手动任务优先执行 + for (SalesforceParam salesforceParam : salesforceParams) { + if (param.getIsSingleThread()){ + UpdateSfDataNew(salesforceParam, partnerConnection,update); + }else { + Future future = salesforceExecutor.execute(() -> { + try { + UpdateSfDataNew(salesforceParam, partnerConnection, dataObject); + } catch (Throwable throwable) { + log.error("salesforceExecutor error", throwable); + throw new RuntimeException(throwable); + } + }, salesforceParam.getBatch(), 0); + futures.add(future); + } + } + // 等待当前所有线程执行完成 + salesforceExecutor.waitForFutures(futures.toArray(new Future[]{})); + } catch (Exception e) { + throw e; + } finally { + update.setNeedUpdate(false); + update.setDataLock(0); + dataObjectService.updateById(update); + } + } + // 等待当前所有线程执行完成 + salesforceExecutor.waitForFutures(futures.toArray(new Future[]{})); + futures.clear(); + } + } + + /** + * 组装【多表】【增量】Update参数 + */ + private void autoUpdateIncrementalSfDataNew(SalesforceParam param, List> futures) throws Exception { + + // 全量的时候 检测是否有自动任务锁住的表 + boolean isFull = CollectionUtils.isEmpty(param.getIds()); + if (isFull) { + QueryWrapper qw = new QueryWrapper<>(); + qw.eq("data_lock", 1); + List list = dataObjectService.list(qw); + if (CollectionUtils.isNotEmpty(list)) { + String apiNames = list.stream().map(DataObject::getName).collect(Collectors.joining()); + String message = "api:" + apiNames + " is locked"; + String format = String.format("数据导入 error, api name: %s, \nparam: %s, \ncause:\n%s", apiNames, com.alibaba.fastjson2.JSON.toJSONString(param, DataDumpParam.getFilter()), message); + EmailUtil.send("DataDump ERROR", format); + return ; + } + } + + QueryWrapper qw = new QueryWrapper<>(); + qw.eq("need_update", 1) + .orderByAsc("data_index") + .last(" limit 10"); + + PartnerConnection partnerConnection = salesforceTargetConnect.createConnect(); + while (true) { + List dataObjects = dataObjectService.list(qw); + //判断dataObjects是否为空 + if (CollectionUtils.isEmpty(dataObjects)) { + return; + } + + for (DataObject dataObject : dataObjects) { + TimeUnit.MILLISECONDS.sleep(1); + DataObject update = new DataObject(); + update.setName(dataObject.getName()); + update.setDataLock(1); + dataObjectService.updateById(update); + TimeUnit.MILLISECONDS.sleep(1); + try { + String api = dataObject.getName(); + QueryWrapper dbQw = new QueryWrapper<>(); + dbQw.eq("name", api).orderByDesc("sync_end_date").last("LIMIT 1"); + DataBatch t = dataBatchService.getOne(dbQw); + + AtomicInteger batch = new AtomicInteger(1); + SalesforceParam salesforceParam = param.clone(); + salesforceParam.setApi(t.getName()); + salesforceParam.setBeginCreateDate(t.getSyncStartDate()); + salesforceParam.setEndCreateDate(t.getSyncEndDate()); + salesforceParam.setBatch(batch.getAndIncrement()); + + if (param.getIsSingleThread()){ + UpdateSfDataNew(salesforceParam, partnerConnection,update); + }else { + Future future = salesforceExecutor.execute(() -> { + try { + UpdateSfDataNew(salesforceParam, partnerConnection,dataObject); + } catch (Throwable throwable) { + log.error("salesforceExecutor error", throwable); + throw new RuntimeException(throwable); + } + }, salesforceParam.getBatch(), 0); + futures.add(future); + } + + } catch (Exception e) { + throw e; + } finally { + update.setNeedUpdate(false); + update.setDataLock(0); + dataObjectService.updateById(update); + } + } + // 等待当前所有线程执行完成 + salesforceExecutor.waitForFutures(futures.toArray(new Future[]{})); + futures.clear(); + } + } + + /** + * 执行Update更新数据 + */ + private void UpdateSfDataNew(SalesforceParam param, PartnerConnection partnerConnection,DataObject dataObject) throws Exception { + + Map infoFlag = customMapper.list("code,value","system_config","code ='"+ SystemConfigCode.INFO_FLAG+"'").get(0); + String api = param.getApi(); + QueryWrapper dbQw = new QueryWrapper<>(); + dbQw.eq("api", api); + List list = dataFieldService.list(dbQw); + + String sql = ""; + String sql2 = ""; + + String beginDateStr = null; + String endDateStr = null; + Date beginDate = param.getBeginCreateDate(); + Date endDate = param.getEndCreateDate(); + if (param.getBeginCreateDate() != null && param.getEndCreateDate() != null){ + beginDateStr = DateUtil.format(beginDate, "yyyy-MM-dd HH:mm:ss"); + endDateStr = DateUtil.format(endDate, "yyyy-MM-dd HH:mm:ss"); + } + + if (1 == param.getType()) { + if (api.endsWith("Share")){ + sql = "where RowCause NOT IN ('Owner','Rule','Territory','Team','ImplicitChild','ImplicitParent','TerritoryRule','ImplicitCallCenter','PortalRole','Portal','ImplicitPerson','ImplicitGrant') and is_update = 0 and new_id is not null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "'"; + sql2 = "RowCause NOT IN ('Owner','Rule','Territory','Team','ImplicitChild','ImplicitParent','TerritoryRule','ImplicitCallCenter','PortalRole','Portal','ImplicitPerson','ImplicitGrant') and is_update = 0 and new_id is not null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "' order by Id asc limit 10000 "; + }else { + sql = "where new_id is not null and is_update = 0 and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "'"; + sql2 = "new_id is not null and is_update = 0 and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "' order by Id asc limit 10000 "; + } + }else { + String updateDateField = dataFieldService.returnUpdateDateField(api); + if (api.endsWith("Share")){ + sql = "where RowCause NOT IN ('Owner','Rule','Territory','Team','ImplicitChild','ImplicitParent','TerritoryRule','ImplicitCallCenter','PortalRole','Portal','ImplicitPerson','ImplicitGrant') and is_update = 0 and new_id is not null and "+updateDateField+" >= '" + beginDateStr + "' "; + sql2 = "RowCause NOT IN ('Owner','Rule','Territory','Team','ImplicitChild','ImplicitParent','TerritoryRule','ImplicitCallCenter','PortalRole','Portal','ImplicitPerson','ImplicitGrant') and is_update = 0 and new_id is not null and "+updateDateField+" >= '" + beginDateStr + "' order by Id asc limit 10000 "; + }else { + sql = "where new_id is not null and is_update = 0 and "+updateDateField+" >= '" + beginDateStr + "' "; + sql2 = "new_id is not null and is_update = 0 and "+updateDateField+" >= '" + beginDateStr + "' order by Id asc limit 10000 "; + } + } + //表内数据总量 + Integer count = customMapper.countBySQL(api, sql); + + if(count == 0){ + log.info("没有需要更新的数据，API: {}，开始时间: {}，结束时间: {}", api, beginDateStr, endDateStr); + return; + } + + //查询当前对象多态字段映射 + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("api",api).eq("is_create",1).eq("is_link",true); + List configs = linkConfigService.list(queryWrapper); + Map fieldMap = new HashMap<>(); + if (!configs.isEmpty()) { + fieldMap = configs.stream() + .collect(Collectors.toMap( + LinkConfig::getField, // Key提取器 + LinkConfig::getLinkField, // Value提取器 + (oldVal, newVal) -> newVal // 解决重复Key冲突（保留新值） + )); + } + + + int targetCount = 0; + + int num = count%10000 == 0 ? count/10000 : (count/10000) + 1; + + log.info("总Update数据 count:{}；-开始时间:{}；-结束时间:{}；-api:{}；", count, beginDateStr, endDateStr, api); + + for (int z = 0; z < num; z++){ + + List> mapsList = customMapper.list("*", api, sql2); + int size = mapsList.size(); + log.info("总批次:{}；当前批次：{}；当前批次数据量:{}；-开始时间:{}；-结束时间:{}；-api:{}；", num,z,size, beginDateStr, endDateStr, api); + + //批量插入200一次 + int page = size%200 == 0 ? size/200 : (size/200) + 1; + + for (int i = 0; i < page; i++) { + + DataLog dataLog = new DataLog(api, "开始时间："+beginDateStr+";结束时间："+endDateStr, new Date(), null, "数据更新，查询并组装第" + (z*50 + i) + "批数据", "QueryDB"); + int startIndex = 200 * i; + int endIndex = Math.min(startIndex + 200, size); + if (startIndex >= endIndex) { + log.info("当前批次：{} 结束!! -开始时间:{}；-结束时间:{}；-api:{}；", z, beginDateStr, endDateStr, api); + break; + } + List> mapList = mapsList.subList(startIndex, endIndex); + + SObject[] accounts = new SObject[mapList.size()]; + int j = 0; + try { + for (Map map : mapList) { + //需要置空的参数 + List fieldsToNull = new ArrayList<>(); + SObject account = new SObject(); + account.setType(api); + String message = null; + //给对象赋值 + for (DataField dataField : list) { + String field = dataField.getField(); + String reference_to = dataField.getReferenceTo(); + + String value = String.valueOf(map.get(field)); + //根据旧sfid查找引用对象新sfid + if (field.equals("Id")) { + account.setId(String.valueOf(map.get("new_id"))); + } else if (!DataUtil.isUpdate(field) || (dataField.getIsUpdateable() != null && !dataField.getIsUpdateable())) { + continue; + } else if ("reference".equals(dataField.getSfType()) && map.get(dataField.getField()) != null) { + + if (!"null".equals(value) && StringUtils.isNotEmpty(value)) { + //判断reference_to内是否包含User字符串 + if (dataField.getReferenceTo() != null && (dataField.getReferenceTo().contains(",User") || dataField.getReferenceTo().contains("User,"))) { + reference_to = "User"; + } + //引用类型字段 + String linkfield = fieldMap.get(dataField.getField()); + if (StringUtils.isNotBlank(linkfield)){ + if (map.get(linkfield)!=null){ + reference_to = map.get(linkfield).toString(); + }else { + message = "对象类型:" + api + "的数据:"+ map.get("Id") +"的关联类型不存在！"; + List> maps = new ArrayList<>(); + Map linkMap = new HashMap<>(); + linkMap.put("key", "is_update"); + linkMap.put("value", 2); + maps.add(linkMap); + Map linkMap1 = new HashMap<>(); + linkMap1.put("key", "error_message"); + linkMap1.put("value", message); + maps.add(linkMap1); + customMapper.updateByNewId(maps,api, String.valueOf(map.get("new_id"))); + log.info(message); + } + } + if (reference_to == null){ + continue; + } + Map m = customMapper.getById("new_id", reference_to, value); + if (m!= null && !m.isEmpty() && m.get("new_id") != null) { + account.setField(field, m.get("new_id")); + }else { + message = "对象类型:" + api + "的数据:"+ map.get("Id") +"的引用对象:" + reference_to + "的数据:"+ map.get(field) +"异常！"; + List> maps = new ArrayList<>(); + Map linkMap = new HashMap<>(); + linkMap.put("key", "is_update"); + linkMap.put("value", 2); + maps.add(linkMap); + Map linkMap1 = new HashMap<>(); + linkMap1.put("key", "error_message"); + linkMap1.put("value", message); + maps.add(linkMap1); + customMapper.updateByNewId(maps,api, String.valueOf(map.get("new_id"))); + log.info(message); + break; + } + } + } else { + if(StringUtils.isNotBlank(dataField.getSfType())){ + if (map.get(field) != null){ + account.setField(field, DataUtil.localDataToSfData(dataField.getSfType(), value)); + }else { + fieldsToNull.add(field); + } + } + } + } + if (message != null){ + continue; + } + + if (!fieldsToNull.isEmpty()){ + String[] fieldsToNullArray = fieldsToNull.toArray(new String[0]); + account.setField("fieldsToNull", fieldsToNullArray); + } + + if (dataObject.getIsEditable()){ + account.setField("old_owner_id__c", map.get("OwnerId")); + account.setField("old_sfdc_id__c", map.get("Id")); + } + accounts[j++] = account; + } + dataLog.setEndTime(new Date()); + dataLogService.save(dataLog); + + if (infoFlag != null && "1".equals(infoFlag.get("value"))){ + printlnAccountsDetails(accounts,list); + } + DataLog dataLog1 = new DataLog(api, "开始时间："+beginDateStr+";结束时间："+endDateStr, new Date(), null, "数据更新，更新SF第" + (z*50 + i) + "批数据", "UpdateSF"); + SaveResult[] saveResults = partnerConnection.update(accounts); + for (SaveResult saveResult : saveResults) { + if (!saveResult.getSuccess()) { + List> maps = new ArrayList<>(); + Map linkMap = new HashMap<>(); + linkMap.put("key", "is_update"); + linkMap.put("value", 2); + maps.add(linkMap); + Map linkMap1 = new HashMap<>(); + linkMap1.put("key", "error_message"); + linkMap1.put("value", JSON.toJSONString(saveResult.getErrors())); + maps.add(linkMap1); + customMapper.updateByNewId(maps,api, saveResult.getId()); + String format = String.format("数据更新 error, api name: %s, \ncause:\n%s", api, JSON.toJSONString(saveResult)); + log.info(format); + }else { + List> maps = new ArrayList<>(); + Map linkMap = new HashMap<>(); + linkMap.put("key", "is_update"); + linkMap.put("value", 1); + maps.add(linkMap); + Map linkMap1 = new HashMap<>(); + linkMap1.put("key", "error_message"); + linkMap1.put("value", null); + maps.add(linkMap1); + customMapper.updateByNewId(maps,api, saveResult.getId()); + targetCount ++; + } + } + dataLog1.setEndTime(new Date()); + dataLogService.save(dataLog1); + + TimeUnit.MILLISECONDS.sleep(1); + + }catch (InterruptedException interruptedException){ + return; + }catch (Throwable e) { + log.info(JSON.toJSONString(e)); + } + } + } + + UpdateWrapper updateQw = new UpdateWrapper<>(); + updateQw.eq("name", api) + .eq("sync_start_date", beginDate) + .eq("sync_end_date", DateUtils.addSeconds(endDate, -1)) + .set("target_update_num", num); + dataBatchHistoryService.update(updateQw); + + UpdateWrapper updateQw2 = new UpdateWrapper<>(); + updateQw2.eq("name", api) + .eq("sync_start_date", beginDate) + .eq("sync_end_date", endDate) + .set("sf_update_num", targetCount); + dataBatchService.update(updateQw2); + + log.info("数据更新完成，API: {}，总更新数: {}，开始时间: {}，结束时间: {}", api, targetCount, beginDateStr, endDateStr); + } + + /** + * 执行Update更新数据 + */ + private void UpdateSfShareDataNew(String api, PartnerConnection partnerConnection,DataObject dataObject) throws Exception { + + Map infoFlag = customMapper.list("code,value","system_config","code ='"+SystemConfigCode.INFO_FLAG+"'").get(0); + QueryWrapper dbQw = new QueryWrapper<>(); + dbQw.eq("api", api); + List list = dataFieldService.list(dbQw); + + String sql = ""; + String sql2 = ""; + + String beginDateStr = null; + String endDateStr = null; + + String updateDateField = dataFieldService.returnUpdateDateField(api); + if (api.endsWith("Share")){ + sql = "where RowCause NOT IN ('Owner','Rule','Territory','Team','ImplicitChild','ImplicitParent','TerritoryRule','ImplicitCallCenter','PortalRole','Portal','ImplicitPerson','ImplicitGrant') and is_update = 0 and new_id is not null and "+updateDateField+" >= '" + beginDateStr + "' "; + sql2 = "RowCause NOT IN ('Owner','Rule','Territory','Team','ImplicitChild','ImplicitParent','TerritoryRule','ImplicitCallCenter','PortalRole','Portal','ImplicitPerson','ImplicitGrant') and is_update = 0 and new_id is not null and "+updateDateField+" >= '" + beginDateStr + "' order by Id asc limit "; + }else { + sql = "where new_id is not null and is_update = 0 and "+updateDateField+" >= '" + beginDateStr + "' "; + sql2 = "new_id is not null and is_update = 0 and "+updateDateField+" >= '" + beginDateStr + "' order by Id asc limit "; + } + + //表内数据总量 + Integer count = customMapper.countBySQL(api, sql); + + if(count == 0){ + return; + } + + //查询当前对象多态字段映射 + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("api",api).eq("is_create",1).eq("is_link",true); + List configs = linkConfigService.list(queryWrapper); + Map fieldMap = new HashMap<>(); + if (!configs.isEmpty()) { + fieldMap = configs.stream() + .collect(Collectors.toMap( + LinkConfig::getField, // Key提取器 + LinkConfig::getLinkField, // Value提取器 + (oldVal, newVal) -> newVal // 解决重复Key冲突（保留新值） + )); + } + + + int num = count%10000 == 0 ? count/10000 : (count/10000) + 1; + + log.info("总Update数据 count:{}；-开始时间:{}；-结束时间:{}；-api:{}；", count, beginDateStr, endDateStr, api); + + for (int z = 0; z < num; z++){ + + List> mapsList = customMapper.list("*", api, sql2+ z * 10000 + ",10000"); + int size = mapsList.size(); + log.info("总Update数据 count:{}；当前批次：{}；-开始时间:{}；-结束时间:{}；-api:{}；", count,z, beginDateStr, endDateStr, api); + + //批量插入200一次 + int page = size%200 == 0 ? count/200 : (count/200) + 1; + + for (int i = 0; i < page; i++) { + + int startIndex = 200 * i; + int endIndex = Math.min(startIndex + 200, size); + if (startIndex >= endIndex) { + break; + } + List> mapList = mapsList.subList(startIndex, endIndex); + + SObject[] accounts = new SObject[mapList.size()]; + int j = 0; + try { + for (Map map : mapList) { + + SObject account = new SObject(); + account.setType(api); + String message = null; + //给对象赋值 + for (DataField dataField : list) { + String field = dataField.getField(); + String reference_to = dataField.getReferenceTo(); + + String value = String.valueOf(map.get(field)); + //根据旧sfid查找引用对象新sfid + if (field.equals("Id")) { + account.setId(String.valueOf(map.get("new_id"))); + } else if (!DataUtil.isUpdate(field) || (dataField.getIsUpdateable() != null && !dataField.getIsUpdateable())) { + continue; + } else if ("reference".equals(dataField.getSfType()) && map.get(dataField.getField()) != null) { + + if (!"null".equals(value) && StringUtils.isNotEmpty(value)) { + //判断reference_to内是否包含User字符串 + if (dataField.getReferenceTo() != null && (dataField.getReferenceTo().contains(",User") || dataField.getReferenceTo().contains("User,"))) { + reference_to = "User"; + } + //引用类型字段 + String linkfield = fieldMap.get(dataField.getField()); + if (StringUtils.isNotBlank(linkfield)){ + if (map.get(linkfield)!=null){ + reference_to = map.get(linkfield).toString(); + }else { + log.info("对象类型:" + api + "的数据:"+ map.get("Id") +"的关联类型不存在！"); + } + } + if (reference_to == null){ + continue; + } + Map m = customMapper.getById("new_id", reference_to, value); + if (m!= null && !m.isEmpty() && m.get("new_id") != null) { + account.setField(field, m.get("new_id")); + }else { + message = "对象类型:" + api + "的数据:"+ map.get("Id") +"的引用对象:" + reference_to + "的数据:"+ map.get(field) +"异常！"; + log.info(message); + break; + } + } + } else { + if (map.get(field) != null && StringUtils.isNotBlank(dataField.getSfType())) { + account.setField(field, DataUtil.localDataToSfData(dataField.getSfType(), value)); + }else { + account.setField(field, map.get(field)); + } + } + } + if (message!=null){ + List> maps = new ArrayList<>(); + Map linkMap1 = new HashMap<>(); + linkMap1.put("key", "error_message"); + linkMap1.put("value", message); + maps.add(linkMap1); + customMapper.updateById(api, maps, map.get("Id").toString()); + continue; + } + if (dataObject.getIsEditable()){ + account.setField("old_owner_id__c", map.get("OwnerId")); + account.setField("old_sfdc_id__c", map.get("Id")); + } + accounts[j++] = account; + } + + if (infoFlag != null && "1".equals(infoFlag.get("value"))){ + printlnAccountsDetails(accounts,list); + } + + SaveResult[] saveResults = partnerConnection.update(accounts); + for (SaveResult saveResult : saveResults) { + if (!saveResult.getSuccess()) { + List> maps = new ArrayList<>(); + Map linkMap = new HashMap<>(); + linkMap.put("key", "is_update"); + linkMap.put("value", 2); + maps.add(linkMap); + Map linkMap1 = new HashMap<>(); + linkMap1.put("key", "error_message"); + linkMap1.put("value", JSON.toJSONString(saveResult.getErrors())); + maps.add(linkMap1); + customMapper.updateByNewId(maps,api, saveResult.getId()); + String format = String.format("数据更新 error, api name: %s, \ncause:\n%s", api, JSON.toJSONString(saveResult)); + log.info(format); + }else { + List> maps = new ArrayList<>(); + Map linkMap = new HashMap<>(); + linkMap.put("key", "is_update"); + linkMap.put("value", 1); + maps.add(linkMap); + customMapper.updateByNewId(maps,api, saveResult.getId()); + } + } + + } catch (Throwable e) { + log.info(JSON.toJSONString(e)); + } + } + } + + } + + /** + * 打印SF交互数据明细 + */ + public void printlnAccountsDetails(SObject[] accounts,List list) { + for (int i = 0; i < accounts.length; i++) { + SObject account = accounts[i]; + System.out.println("--- 对象数据[" + i + "] ---"); + if (account == null){ + continue; + } + // 获取对象所有字段名 + for (DataField dataField : list) { + try { + Object value = account.getField(dataField.getField()); + System.out.println(dataField.getField() + ": " + (value != null ? value.toString() : "null")); + } catch (Exception e) { + System.out.println(dataField.getField() + ": [权限不足或字段不存在]"); + } + } + System.out.println("old_owner_id__c: " + account.getField("old_owner_id__c")); + System.out.println("old_sfdc_id__c: " + account.getField("old_sfdc_id__c")); + + } + } + + /** + * 返回SF交互数据错误明细 + */ + public Map returnErrorAccountsDetails(SObject[] accounts,List list,String errorId) { + HashMap map = new HashMap<>(); + for (int i = 0; i < accounts.length; i++) { + SObject account = accounts[i]; + if (account == null){ + continue; + } + if (errorId != null && (errorId.equals(account.getId()) || errorId.equals(account.getField("Id")))){ + for (DataField dataField : list) { + try { + Object value = account.getField(dataField.getField()); + map.put(dataField.getField(),String.valueOf(value)); + System.out.println(dataField.getField() + ": " + value); + } catch (Exception e) { + System.out.println(dataField.getField() + ": [权限不足或字段不存在]"); + } + } + map.put("old_owner_id__c",String.valueOf(account.getField("old_owner_id__c"))); + map.put("old_sfdc_id__c",String.valueOf(account.getField("old_sfdc_id__c"))); + } + } + return map; + } + + /** + * 获取DocumentLink + */ + @Override + public ReturnT dumpDocumentLinkJob(String paramStr) throws Exception { + String api = "ContentDocumentLink"; + PartnerConnection partnerConnection = salesforceConnect.createConnect(); + Integer count = customMapper.countBySQL("ContentDocument", "where new_id is not null"); + log.info("ContentDocument Total size：" + count); + + int page = count%200 == 0 ? count/200 : (count/200) + 1; + + int totalSize = 0; + for (int i = 0; i < page; i++) { + List> list = customMapper.list("Id", "ContentDocument", "new_id is not null order by Id limit " + i * 200 + ",200"); + if (list.isEmpty()){ + break; + } + totalSize = totalSize +list.size(); + log.info("dumpContentDocumentLink By ContentDocument Now size：" + totalSize); + DescribeSObjectResult dsr = partnerConnection.describeSObject(api); + List fields = customMapper.getFields(api).stream().map(String::toUpperCase).collect(Collectors.toList()); + Field[] dsrFields = dsr.getFields(); + try { + for (Map map : list) { + String contentDocumentId = (String) map.get("Id"); + String sql = "SELECT Id, LinkedEntityId, LinkedEntity.Type, ContentDocumentId, Visibility, ShareType, SystemModstamp, IsDeleted FROM ContentDocumentLink where ContentDocumentId = '" + contentDocumentId + "'"; + com.alibaba.fastjson2.JSONArray objects = null; + QueryResult queryResult = partnerConnection.queryAll(sql); + SObject[] records = queryResult.getRecords(); + objects = DataUtil.toJsonArray(records, dsrFields); + commonService.saveOrUpdate(api, fields, records, objects, true); + } + } catch (Throwable e) { + log.error("dumpDocumentLinkJob error message:{}", e.getMessage()); + return ReturnT.FAIL; + } + } + log.info("dumpDocumentLink Success !!! "); + return ReturnT.SUCCESS; + } + + /** + * 推送DocumentLink + */ + @Override + public ReturnT uploadDocumentLinkJob(String paramStr) throws Exception { + String api = "ContentDocumentLink"; + + Map infoFlag = customMapper.list("code,value","system_config","code ='"+SystemConfigCode.INFO_FLAG+"'").get(0); + + QueryWrapper dbQw = new QueryWrapper<>(); + dbQw.eq("api", api); + List dataFields = dataFieldService.list(dbQw); + + PartnerConnection connection = salesforceTargetConnect.createConnect(); + List> list = customMapper.list("Id", "ContentDocument", "new_id is not null"); + try { + if (list != null && !list.isEmpty()) { + //表内数据总量 + Integer count = customMapper.countBySQL(api, "where ShareType = 'V' "); + //批量插入200一次 + int page = count % 200 == 0 ? count / 200 : (count / 200) + 1; + for (int i = 0; i < page; i++) { + List> linkList = customMapper.list("Id,LinkedEntityId,ContentDocumentId,LinkedEntity_Type,ShareType,Visibility", api, "ShareType = 'V' and new_id = '0' order by Id limit 200"); + SObject[] accounts = new SObject[linkList.size()]; + String[] ids = new String[linkList.size()]; + int index = 0; + for (Map map : linkList) { + String linkedEntityId = (String) map.get("LinkedEntityId"); + String id = (String) map.get("Id"); + String contentDocumentId = (String) map.get("ContentDocumentId"); + String linkedEntityType = (String) map.get("LinkedEntity_Type"); + String shareType = (String) map.get("ShareType"); + String Visibility = (String) map.get("Visibility"); + + // dataObject查询 + QueryWrapper qw = new QueryWrapper<>(); + qw.eq("name", linkedEntityType); + List objects = dataObjectService.list(qw); + if (!objects.isEmpty()) { + Map dMap = customMapper.getById("new_id", "ContentDocument", contentDocumentId); + Map lMap = customMapper.getById("new_id", linkedEntityType, linkedEntityId); + SObject account = new SObject(); + account.setType(api); + if (dMap != null){ + account.setField("ContentDocumentId", dMap.get("new_id").toString()); + }else { + String errorMessage = String.format("ContentDocumentLink Id: %s，对应的ContentDocumentId: %s 数据不存在！", id, contentDocumentId); + log.info(errorMessage); + List> maps = new ArrayList<>(); + Map linkMap1 = new HashMap<>(); + linkMap1.put("key", "error_message"); + linkMap1.put("value", errorMessage); + maps.add(linkMap1); + customMapper.updateById(api, maps, id); + continue; + } + if (lMap != null){ + account.setField("LinkedEntityId", lMap.get("new_id").toString()); + }else { + String errorMessage = String.format("ContentDocumentLink Id: %s，对应的LinkedEntityId: %s 数据不存在！", id, linkedEntityId); + log.info(errorMessage); + List> maps = new ArrayList<>(); + Map linkMap1 = new HashMap<>(); + linkMap1.put("key", "error_message"); + linkMap1.put("value", errorMessage); + maps.add(linkMap1); + customMapper.updateById(api, maps, id); + continue; + } + account.setField("ShareType", shareType); + account.setField("Visibility", Visibility); + ids[index] = id; + accounts[index] = account; + index++; + } + } + try { + if (infoFlag != null && "1".equals(infoFlag.get("value"))){ + printlnAccountsDetails(accounts,dataFields); + } + SaveResult[] saveResults = connection.create(accounts); + ; + for (int j = 0; j < saveResults.length; j++) { + if (saveResults[j].getSuccess()) { + List> maps = new ArrayList<>(); + Map m = new HashMap<>(); + m.put("key", "new_id"); + m.put("value", saveResults[j].getId()); + maps.add(m); + customMapper.updateById(api, maps, ids[j]); + log.info("ContentDocumentLink Id: {}，对应的new_id: {} 更新成功！ " , ids[j], saveResults[j].getId()); + }else{ + List> maps = new ArrayList<>(); + Map linkMap1 = new HashMap<>(); + linkMap1.put("key", "error_message"); + linkMap1.put("value", JSON.toJSONString(saveResults[j].getErrors())); + maps.add(linkMap1); + customMapper.updateById(api, maps, ids[j]); + log.error("Id:{},saveResults: {}",ids[j], JSON.toJSONString(saveResults[j])); + } + } + } catch (Exception e) { + log.error("uploadDocumentLinkJob error message:{}", e.getMessage()); + throw new RuntimeException(e); + } + } + } + } catch (Exception e) { + log.error("uploadDocumentLink error api:{}, data:{}", api, com.alibaba.fastjson2.JSON.toJSONString(list), e); + return ReturnT.FAIL; + } + log.info("uploadDocumentLink Success !!! "); + return ReturnT.SUCCESS; + } + + @Override + public ReturnT dumpFileNew(SalesforceParam param) throws Exception { + String downloadUrl = null; + + QueryWrapper wrapper = new QueryWrapper<>(); + if (StringUtils.isNotBlank(param.getApi())) { + wrapper.in("name",DataUtil.toIdList(param.getApi())); + }else { + wrapper.isNotNull("blob_field"); + } + List objectList = dataObjectService.list(wrapper); + if (objectList.isEmpty()){ + log.info("没有对象存在文件二进制字段！不进行文件下载"); + return ReturnT.FAIL; + } + + List> poll = customMapper.list("code,value","org_config",null); + + for (Map map1 : poll) { + if ("FILE_DOWNLOAD_URL".equals(map1.get("code"))) { + downloadUrl = (String) map1.get("value"); + } + } + if (StringUtils.isEmpty(downloadUrl)) { + EmailUtil.send("DumpFile ERROR", "文件下载失败！下载地址未配置"); + return ReturnT.FAIL; + } + + PartnerConnection connect = salesforceConnect.createConnect(); + + List> futures = Lists.newArrayList(); + + for (DataObject dataObject : objectList) { + DataObject update = new DataObject(); + log.info("dump file api:{}, field:{}", dataObject.getName(), dataObject.getBlobField()); + + try { + String api = dataObject.getName(); + update.setName(dataObject.getName()); + List salesforceParams = null; + QueryWrapper dbQw = new QueryWrapper<>(); + dbQw.eq("name", api); + List list = dataBatchService.list(dbQw); + AtomicInteger batch = new AtomicInteger(1); + if (CollectionUtils.isNotEmpty(list)) { + salesforceParams = list.stream().map(t -> { + SalesforceParam salesforceParam = param.clone(); + salesforceParam.setApi(t.getName()); + salesforceParam.setBeginCreateDate(t.getSyncStartDate()); + salesforceParam.setEndCreateDate(t.getSyncEndDate()); + salesforceParam.setBatch(batch.getAndIncrement()); + return salesforceParam; + }).collect(Collectors.toList()); + } + + if (Const.FILE_TYPE == FileType.SERVER) { + Date defaultBeginDate = DataUtil.DEFAULT_BEGIN_DATE; + Calendar calendar = Calendar.getInstance(); + int currentYear = calendar.get(Calendar.YEAR); + calendar.setTime(defaultBeginDate); + int beginYear = calendar.get(Calendar.YEAR); + + // 检测路径是否存在 不存在则创建 + File baseDir = new File(Const.SERVER_FILE_PATH + "/" + api); + if (!baseDir.exists()) { + baseDir.mkdirs(); + } + + // 创建从开始年份到当前年份的子目录 + for (int year = beginYear; year <= currentYear; year++) { + File yearDir = new File(Const.SERVER_FILE_PATH + "/" + api + "-" + year); + if (!yearDir.exists()) { + yearDir.mkdir(); + } + } + } + + // 手动任务优先执行 + for (SalesforceParam salesforceParam : salesforceParams) { + String finalDownloadUrl = downloadUrl; + Future future = salesforceExecutor.execute(() -> { + try { + saveFile(salesforceParam, connect, finalDownloadUrl,dataObject.getName(),dataObject.getBlobField()); + } catch (Throwable throwable) { + log.error("salesforceExecutor error", throwable); + throw new RuntimeException(throwable); + } + }, salesforceParam.getBatch(), 0); + futures.add(future); + } + // 等待当前所有线程执行完成 + salesforceExecutor.waitForFutures(futures.toArray(new Future[]{})); + update.setDataWork(0); + } catch (InterruptedException e) { + salesforceExecutor.remove(futures.toArray(new Future[]{})); + throw e; + } catch (Throwable e) { + throw new RuntimeException(e); + } finally { + update.setDataLock(0); + dataObjectService.updateById(update); + } + } + // 等待当前所有线程执行完成 + salesforceExecutor.waitForFutures(futures.toArray(new Future[]{})); + futures.clear(); + return ReturnT.SUCCESS; + } + + + /** + * 下载文件 + */ + private void saveFile(SalesforceParam param, PartnerConnection Connection ,String downloadUrl, String api, String field) throws Exception { + + String extraSql = ""; + if (dataFieldService.hasDeleted(api)) { + extraSql += " AND IsDeleted = false "; + } + + String beginDateStr = null; + String endDateStr = null; + Date beginDate = param.getBeginCreateDate(); + Date endDate = param.getEndCreateDate(); + if (param.getBeginCreateDate() != null && param.getEndCreateDate() != null){ + beginDateStr = DateUtil.format(beginDate, "yyyy-MM-dd HH:mm:ss"); + endDateStr = DateUtil.format(endDate, "yyyy-MM-dd HH:mm:ss"); + } + String token = Connection.getSessionHeader().getSessionId(); + + String name = "Name"; + if (Const.CONTENT_VERSION.equalsIgnoreCase(api)) { + name = "Title,FileExtension"; + } + + Map headers = Maps.newHashMap(); + headers.put("Authorization", "Bearer " + token); + headers.put("connection", "keep-alive"); + + Integer count = customMapper.countBySQL(api, "where is_dump = 0 and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr +"'"+ extraSql ); + + if (count == 0) { + return; + } + + int page = count%200 == 0 ? count/200 : (count/200) + 1; + + log.info("api：{}；总文件数 count:{}；总批次 ：{}；-开始时间:{}；-结束时间:{}；",api, count, page,beginDateStr, endDateStr); + + //总插入数 + for (int i = 0; i < page; i++) { + + log.info("api：{} 批次:{}；-开始时间:{}；-结束时间:{}；",api, i, beginDateStr, endDateStr); + + // 获取未存储的附件id + List> list = customMapper.list("Id,CreatedDate, " + name, api, " is_dump = 0 and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr +"'"+ extraSql + "limit 200"); + + for (Map map : list) { + int failCount = 0; + String id = null; + // 上传完毕 更新附件信息 + List> maps = Lists.newArrayList(); + try { + String fileName; + id = (String) map.get("Id"); + if (Const.CONTENT_VERSION.equalsIgnoreCase(api)) { + fileName = map.get("Title") + "." + map.get("FileExtension"); + }else { + fileName = (String) map.get("Name"); + } + String year = map.get("CreatedDate").toString().substring(0, 4); + + // 判断路径是否为空 + if (StringUtils.isNotBlank(fileName)) { + String filePath = api + "-" + year + "/" + id + "_" + fileName; + // 拼接url + String url = downloadUrl + String.format(Const.SF_FILE_URL, api, id, field); + + log.info("文件下载请求地址：{}",url); + while (failCount < Const.MAX_FAIL_COUNT){ + Response response = HttpUtil.doGet(url, null, headers); + if (response.body() != null && response.code() == 200) { + InputStream inputStream = response.body().byteStream(); + dumpToServer(headers, id, filePath, url, response, inputStream); + Map paramMap = Maps.newHashMap(); + if ("Document".equals(api)) { + paramMap.put("key", "localUrl"); + } else { + paramMap.put("key", "url"); + } + paramMap.put("value", filePath); + maps.add(paramMap); + inputStream.close(); + break; + }else { + failCount++; + log.info("文件下载失败，第"+ failCount +"次！, id: "+ id + "，返回体信息：" + response.message()); + } + } + + if (failCount == Const.MAX_FAIL_COUNT) { + Map paramMap = Maps.newHashMap(); + paramMap.put("key", "is_dump"); + paramMap.put("value", 2); + maps.add(paramMap); + customMapper.updateById(api, maps, id); + }else { + Map paramMap = Maps.newHashMap(); + paramMap.put("key", "is_dump"); + paramMap.put("value", 1); + maps.add(paramMap); + customMapper.updateById(api, maps, id); + TimeUnit.MILLISECONDS.sleep(1); + } + } + + } catch (InterruptedException interruptedException){ + log.info("文件下载手动终止！"); + return; + } catch (Exception e) { + log.error("文件下载失败！, id: {}, 错误信息：{}", id ,e.getMessage()); + Map paramMap = Maps.newHashMap(); + paramMap.put("key", "is_dump"); + paramMap.put("value", 2); + maps.add(paramMap); + customMapper.updateById(api, maps, id); + EmailUtil.send("File Dump ERROR", "文件下载失败！, id: "+ id + "，错误信息：" + e.getMessage()); + } + } + } + } + + + /** + * 下载文件到服务器 + * + * @param headers 请求头 + * @param id id + * @param filePath 文件路径 + * @param url 链接 + * @param response 响应 + * @param inputStream 流 + * @throws IOException exception + */ + private void dumpToServer(Map headers, String id, String filePath, String url, Response response, InputStream inputStream) throws IOException { + String path = Const.SERVER_FILE_PATH + "/" + filePath; + log.info("--------文件名称："+path); + long offset = 0L; + RandomAccessFile accessFile = new RandomAccessFile(path, "rw"); + while (true) { + try { + TimeUnit.MILLISECONDS.sleep(1); + + // 保存到本地 + byte[] buf = new byte[8192]; + int len = 0; + if (offset > 0) { + inputStream.skip(offset); + accessFile.seek(offset); + } + while ((len = inputStream.read(buf)) != -1) { + accessFile.write(buf, 0, len); + offset += len; + } + break; + }catch (InterruptedException interruptedException){ + return; + }catch (Exception e) { + if (offset <= 0) { + throw e; + } + System.out.println("file dump to server EOF ERROR try to reconnect"); + response.close(); + response = HttpUtil.doGet(url, null, headers); + assert response.body() != null; + inputStream = response.body().byteStream(); + } + } + accessFile.close(); + } + + + @Override + public ReturnT uploadFileNew(SalesforceParam param, DataObject dataObject) throws Exception { + String uploadUrl = null; + List> poll = customMapper.list("code,value","org_config",null); + for (Map map1 : poll) { + if ("FILE_UPLOAD_URL".equals(map1.get("code"))) { + uploadUrl = (String) map1.get("value"); + } + } + if (StringUtils.isBlank(uploadUrl)) { + EmailUtil.send("UploadFile ERROR", "文件上传失败！上传地址未配置"); + return ReturnT.FAIL; + } + + if (StringUtils.isEmpty(dataObject.getBlobField())){ + log.info("没有对象存在文件二进制字段！不进行文件上传"); + } + + PartnerConnection connect = salesforceTargetConnect.createConnect(); + + List> futures = Lists.newArrayList(); + + DataObject update = new DataObject(); + + log.info("dump file api:{}, field:{}", dataObject.getName(), dataObject.getBlobField()); + + try { + String api = dataObject.getName(); + update.setName(dataObject.getName()); + List salesforceParams = null; + QueryWrapper dbQw = new QueryWrapper<>(); + dbQw.eq("name", api); + List list = dataBatchService.list(dbQw); + AtomicInteger batch = new AtomicInteger(1); + if (CollectionUtils.isNotEmpty(list)) { + salesforceParams = list.stream().map(t -> { + SalesforceParam salesforceParam = param.clone(); + salesforceParam.setApi(t.getName()); + salesforceParam.setBeginCreateDate(t.getSyncStartDate()); + salesforceParam.setEndCreateDate(t.getSyncEndDate()); + salesforceParam.setBatch(batch.getAndIncrement()); + return salesforceParam; + }).collect(Collectors.toList()); + } + + // 手动任务优先执行 + for (SalesforceParam salesforceParam : salesforceParams) { + String finalDownloadUrl = uploadUrl; + Future future = salesforceExecutor.execute(() -> { + try { + uploadFile(salesforceParam, connect, finalDownloadUrl,dataObject.getName()); + } catch (Throwable throwable) { + log.error("salesforceExecutor error", throwable); + throw new RuntimeException(throwable); + } + }, salesforceParam.getBatch(), 0); + futures.add(future); + } + + // 等待当前所有线程执行完成 + salesforceExecutor.waitForFutures(futures.toArray(new Future[]{})); + update.setDataWork(0); + } catch (InterruptedException e) { + salesforceExecutor.remove(futures.toArray(new Future[]{})); + throw e; + } catch (Throwable e) { + throw new RuntimeException(e); + } finally { + update.setDataLock(0); + dataObjectService.updateById(update); + } + // 等待当前所有线程执行完成 + salesforceExecutor.waitForFutures(futures.toArray(new Future[]{})); + futures.clear(); + return ReturnT.SUCCESS; + } + + /** + * 上传Attachment + */ + private void uploadAttachment(SalesforceParam param, PartnerConnection connection ,String uploadUrl, String api) throws Exception { + + String beginDateStr = null; + String endDateStr = null; + Date beginDate = param.getBeginCreateDate(); + Date endDate = param.getEndCreateDate(); + if (param.getBeginCreateDate() != null && param.getEndCreateDate() != null){ + beginDateStr = DateUtil.format(beginDate, "yyyy-MM-dd HH:mm:ss"); + endDateStr = DateUtil.format(endDate, "yyyy-MM-dd HH:mm:ss"); + } + + String token = connection.getSessionHeader().getSessionId(); + + Map headers = Maps.newHashMap(); + headers.put("Authorization", "Bearer " + token); + headers.put("connection", "keep-alive"); + + Integer count = customMapper.countBySQL(api, "where is_dump = 1 and is_upload = 0 and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr +"'"); + + if (count == 0) { + return; + } + + int page = count%1000 == 0 ? count/1000 : (count/1000) + 1; + + log.info("总文件数 count:{}；总批次：{}，-开始时间:{}；-结束时间:{}；-api:{}；", count,page, beginDateStr, endDateStr, api); + + //总插入数 + for (int i = 0; i < page; i++) { + + log.info("总文件数 count:{}；当前批次：{}，-开始时间:{}；-结束时间:{}；-api:{}；", count,i, beginDateStr, endDateStr, api); + + List> list = customMapper.list("Id, ParentId, Name, url, Description, Parent_Type", api, " is_dump = 1 and is_upload = 0 and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr +"'"+ "limit " + i * 1000 + ",1000"); + + for (Map map : list) { + int failCount = 0; + String id = null; + // 上传完毕 更新附件信息 + List> maps = Lists.newArrayList(); + CloseableHttpResponse response = null; + String respContent = null; + try { + id = (String) map.get("Id"); + String url_fileName = (String) map.get("url"); + String parentId = (String) map.get("ParentId"); + String parentType = (String) map.get("Parent_Type"); + // 判断路径是否为空 + if (StringUtils.isNotBlank(url_fileName)) { + String filePath = Const.SERVER_FILE_PATH + "/" + url_fileName; + File file = new File(filePath); + boolean exists = file.exists(); + if (!exists) { + log.info("文件不存在"); + } + String fileName = (String) map.get("Name"); + log.info("文件名称：" + fileName); + // dataObject查询 + QueryWrapper qw = new QueryWrapper<>(); + qw.eq("name", parentType); + List objects = dataObjectService.list(qw); + if (objects.isEmpty()) { + log.info("关联对象不存在： {}", parentType); + } + + Map lMap = customMapper.getById("new_id",parentType, parentId); + + // 拼接url + String url = uploadUrl + String.format(Const.SF_UPLOAD_FILE_URL, api); + HttpPost httpPost = new HttpPost(url); + httpPost.setHeader("Authorization", "Bearer " + token); + httpPost.setHeader("connection", "keep-alive"); + + com.alibaba.fastjson.JSONObject credentialsJsonParam = new com.alibaba.fastjson.JSONObject(); + credentialsJsonParam.put("parentId", lMap.get("new_id")); + credentialsJsonParam.put("Name", fileName); + + MultipartEntityBuilder builder = MultipartEntityBuilder.create(); + builder.addTextBody("data", credentialsJsonParam.toJSONString(), ContentType.APPLICATION_JSON); + builder.addBinaryBody("Body", new FileInputStream(file), ContentType.APPLICATION_OCTET_STREAM, fileName); + HttpEntity entity = builder.build(); + httpPost.setEntity(entity); + + CloseableHttpClient httpClient = HttpClients.createDefault(); + + while (failCount < Const.MAX_FAIL_COUNT) { + response = httpClient.execute(httpPost); + if (response != null && response.getStatusLine() != null && response.getStatusLine().getStatusCode() < 400) { + HttpEntity he = response.getEntity(); + if (he != null) { + respContent = EntityUtils.toString(he, "UTF-8"); + String newId = com.alibaba.fastjson.JSONObject.parseObject(respContent).get("id").toString(); + Map paramMap = Maps.newHashMap(); + paramMap.put("key", "new_id"); + paramMap.put("value", newId); + maps.add(paramMap); + } + break; + } else { + failCount++; + log.error("文件上传失败，第"+ failCount +"次！, id: "+ id + "，返回体信息：" + JSON.toJSONString(response)); + } + } + + if (failCount == Const.MAX_FAIL_COUNT) { + Map paramMap = Maps.newHashMap(); + paramMap.put("key", "is_upload"); + paramMap.put("value", 2); + maps.add(paramMap); + customMapper.updateById(api, maps, id); + }else { + Map paramMap = Maps.newHashMap(); + paramMap.put("key", "is_upload"); + paramMap.put("value", 1); + maps.add(paramMap); + customMapper.updateById(api, maps, id); + TimeUnit.MILLISECONDS.sleep(1); + } + } + } catch (InterruptedException interruptedException){ + return; + } catch (Exception e) { + if (response != null) { + try { + response.close(); + } catch (IOException re) { + log.error("exception message", re); + } + } + log.error("文件上传失败！, id: {}, 错误信息：{}", id ,JSON.toJSONString(e)); + Map paramMap = Maps.newHashMap(); + paramMap.put("key", "is_upload"); + paramMap.put("value", 2); + maps.add(paramMap); + customMapper.updateById(api, maps, id); + EmailUtil.send("File Upload ERROR", "文件上传失败！, id: "+ id + "，错误信息：" + JSON.toJSONString(e)); + } + } + } + } + + /** + * 上传文件 + */ + private void uploadFile(SalesforceParam param, PartnerConnection connection ,String uploadUrl, String api) throws Exception { + + String beginDateStr = null; + String endDateStr = null; + Date beginDate = param.getBeginCreateDate(); + Date endDate = param.getEndCreateDate(); + if (param.getBeginCreateDate() != null && param.getEndCreateDate() != null){ + beginDateStr = DateUtil.format(beginDate, "yyyy-MM-dd HH:mm:ss"); + endDateStr = DateUtil.format(endDate, "yyyy-MM-dd HH:mm:ss"); + } + String token = connection.getSessionHeader().getSessionId(); + + Map headers = Maps.newHashMap(); + headers.put("Authorization", "Bearer " + token); + headers.put("connection", "keep-alive"); + + Integer count = customMapper.countBySQL("ContentDocument", "where new_id is null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr +"'"); + + if (count == 0) { + return; + } + + int page = count%200 == 0 ? count/200 : (count/200) + 1; + + log.info("总文件数 count:{}；总批次：{}；-开始时间:{}；-结束时间:{}；-api:{}；", count,page, beginDateStr, endDateStr, api); + + //总插入数 + for (int i = 0; i < page; i++) { + + // 获取未存储的附件id + List> documentList = customMapper.list("Id " , "ContentDocument", " new_id is null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr +"'"+ "limit " + i * 200 + ",200"); + + for (Map documentMap : documentList) { + + String documentId = (String) documentMap.get("Id"); + // 获取未存储的附件id + List> list = customMapper.list("Id, url, PathOnClient, Title, ContentDocumentId, VersionNumber", api, "ContentDocumentId = '" + documentId + "' and new_id is null ORDER BY VersionNumber ASC"); + if (CollectionUtils.isEmpty(list)) { + continue; + } + String finalUploadUrl = uploadUrl; + String newDocumentId = null; + for (Map map : list) { + int failCount = 0; + String id = null; + // 上传完毕 更新附件信息 + List> maps = Lists.newArrayList(); + CloseableHttpResponse response = null; + String respContent = null; + try { + id = (String) map.get("Id"); + String url_fileName = (String) map.get("url"); + String fileName = (String) map.get("PathOnClient"); + String title = (String) map.get("Title"); + String oldDocumentId = (String) map.get("ContentDocumentId"); + Integer versionNumber = Integer.valueOf(map.get("VersionNumber").toString()); + log.info("文件名称：" + fileName); + // 判断路径是否为空 + if (StringUtils.isNotBlank(url_fileName)) { + String filePath = Const.SERVER_FILE_PATH + "/" + url_fileName; + File file = new File(filePath); + boolean exists = file.exists(); + if (!exists) { + log.info("文件不存在"); + break; + } + // 拼接url + String url = finalUploadUrl + String.format(Const.SF_UPLOAD_FILE_URL, api); + HttpPost httpPost = new HttpPost(url); + httpPost.setHeader("Authorization", "Bearer " + token); + httpPost.setHeader("connection", "keep-alive"); + + com.alibaba.fastjson.JSONObject credentialsJsonParam = new com.alibaba.fastjson.JSONObject(); + credentialsJsonParam.put("title", title); + credentialsJsonParam.put("pathOnClient", fileName); + if (newDocumentId != null) { + credentialsJsonParam.put("ContentDocumentId", newDocumentId); + } + MultipartEntityBuilder builder = MultipartEntityBuilder.create(); + builder.addTextBody("entity_content", credentialsJsonParam.toJSONString(), ContentType.APPLICATION_JSON); + builder.addBinaryBody("VersionData", new FileInputStream(file), ContentType.APPLICATION_OCTET_STREAM, fileName); + HttpEntity entity = builder.build(); + httpPost.setEntity(entity); + + CloseableHttpClient httpClient = HttpClients.createDefault(); + while (failCount < Const.MAX_FAIL_COUNT) { + response = httpClient.execute(httpPost); + if (response != null && response.getStatusLine() != null && response.getStatusLine().getStatusCode() < 400 && response.getEntity() != null) { + HttpEntity he = response.getEntity(); + respContent = EntityUtils.toString(he, "UTF-8"); + String newId = String.valueOf(com.alibaba.fastjson.JSONObject.parseObject(respContent).get("id")); + if (StringUtils.isBlank(newId)) { + log.error("文件上传错误，返回实体信息：" + respContent); + break; + } + Map paramMap = Maps.newHashMap(); + paramMap.put("key", "new_id"); + paramMap.put("value", newId); + maps.add(paramMap); + + if (versionNumber == 1) { + // 更新documentId + String dId = commonService.getDocumentId(connection, newId); + List> dList = new ArrayList<>(); + Map dMap = Maps.newHashMap(); + dMap.put("key", "new_id"); + dMap.put("value", dId); + dList.add(dMap); + customMapper.updateById("ContentDocument", dList, oldDocumentId); + newDocumentId = dId; + } + break; + } else { + failCount++; + log.error("文件上传失败，第"+ failCount +"次！, id: "+ id + "，返回体信息：" + JSON.toJSONString(response)); + } + } + + if (failCount == Const.MAX_FAIL_COUNT) { + Map paramMap = Maps.newHashMap(); + paramMap.put("key", "is_upload"); + paramMap.put("value", 2); + maps.add(paramMap); + customMapper.updateById(api, maps, id); + }else { + Map paramMap = Maps.newHashMap(); + paramMap.put("key", "is_upload"); + paramMap.put("value", 1); + maps.add(paramMap); + customMapper.updateById(api, maps, id); + TimeUnit.MILLISECONDS.sleep(1); + } + } + } catch (Exception e) { + if (response != null) { + try { + response.close(); + } catch (IOException re) { + log.error("exception message", re); + } + } + log.error("文件上传失败！, id: {}, 错误信息：{}", id ,JSON.toJSONString(e)); + Map paramMap = Maps.newHashMap(); + paramMap.put("key", "is_upload"); + paramMap.put("value", 2); + maps.add(paramMap); + customMapper.updateById(api, maps, id); + EmailUtil.send("File Upload ERROR", "文件上传失败！, id: "+ id + "，错误信息：" + JSON.toJSONString(e)); + } + } + } + } + } + + + /** + * updateLinkType入口 + */ + @Override + public ReturnT updateLinkTypeJob(SalesforceParam param) throws Exception { + List> futures = Lists.newArrayList(); + try { + if (param.getType() == 1){ + return updateLinkType(param, futures); + }else { + return updateLinkTypeBatch(param,futures); + } + } catch (InterruptedException e) { + throw e; + } catch (Throwable throwable) { + salesforceExecutor.remove(futures.toArray(new Future[]{})); + log.error("updateLinkTypeJob error", throwable); + throw throwable; + } + } + + + /** + * 【batch逻辑】组装updateLinkType执行参数 + */ + public ReturnT updateLinkTypeBatch(SalesforceParam param, List> futures) throws Exception { + List apis; + if (StringUtils.isBlank(param.getApi())) { + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("is_link","1").eq("is_create",1); + apis = linkConfigService.list(queryWrapper).stream().map(LinkConfig::getApi).collect(Collectors.toList()); + } else { + apis = DataUtil.toIdList(param.getApi()); + } + String beginDateStr = null; + String endDateStr = null; + if (param.getBeginCreateDate() != null && param.getEndCreateDate() != null){ + Date beginDate = param.getBeginCreateDate(); + Date endDate = param.getEndCreateDate(); + beginDateStr = DateUtil.format(beginDate, "yyyy-MM-dd HH:mm:ss"); + endDateStr = DateUtil.format(endDate, "yyyy-MM-dd HH:mm:ss"); + } + QueryWrapper wrapper = new QueryWrapper<>(); + wrapper.in("api", apis); + Map resultMap = dataObjectService.list().stream() + .collect(Collectors.toMap( + DataObject::getKeyPrefix, + DataObject::getName, + (existing, replacement) -> replacement)); + + for (String api : apis) { + try { + if (!dataFieldService.hasCreatedDate(api)){ + log.info("当前对象："+api +"不存在批次，请执行参数更改type为1，按对象更新关联！"); + EmailUtil.send("DataDump ERROR", "当前对象："+api +"不存在批次，请执行参数更改type为1，按对象更新关联！"); + break; + } + QueryWrapper dbQw = new QueryWrapper<>(); + dbQw.eq("api", param.getApi()).eq("is_link",1).eq("is_create",1); + List linkConfigs = linkConfigService.list(dbQw); + if (linkConfigs.isEmpty()){ + continue; + } + List salesforceParams = null; + QueryWrapper dbQw1 = new QueryWrapper<>(); + dbQw1.eq("name", api); + if (StringUtils.isNotEmpty(beginDateStr) && StringUtils.isNotEmpty(endDateStr)) { + dbQw1.eq("sync_start_date", beginDateStr); // 等于开始时间 + dbQw1.eq("sync_end_date", endDateStr); // 等于结束时间 + } + List list = dataBatchService.list(dbQw1); + AtomicInteger batch = new AtomicInteger(1); + if (CollectionUtils.isNotEmpty(list)) { + salesforceParams = list.stream().map(t -> { + SalesforceParam salesforceParam = param.clone(); + salesforceParam.setApi(t.getName()); + salesforceParam.setBeginCreateDate(t.getSyncStartDate()); + salesforceParam.setEndCreateDate(t.getSyncEndDate()); + salesforceParam.setBatch(batch.getAndIncrement()); + return salesforceParam; + }).collect(Collectors.toList()); + } + Set safeSet = new HashSet<>(); + + // 手动任务优先执行 + for (SalesforceParam salesforceParam : salesforceParams) { + Future future = salesforceExecutor.execute(() -> { + try { + updateLinkBatch(salesforceParam, resultMap,safeSet,linkConfigs); + } catch (Throwable throwable) { + log.error("salesforceExecutor error", throwable); + throw new RuntimeException(throwable); + } + }, salesforceParam.getBatch(), 1); + futures.add(future); + } + + if (!safeSet.isEmpty()){ + String format = String.format("更新关联类型 error, api name: %s, \ncause:\n%s 前三位编码对象不存在！！", api, com.alibaba.fastjson2.JSON.toJSONString(safeSet)); + EmailUtil.send("DataDump ERROR", format); + } + + // 等待当前所有线程执行完成 + salesforceExecutor.waitForFutures(futures.toArray(new Future[]{})); + } catch (Exception e) { + throw e; + } + } + return ReturnT.SUCCESS; + } + + /** + * 执行数据updateLinkType + */ + private void updateLinkBatch(SalesforceParam param, Map resultMap,Set safeSet,List linkConfigs) throws Exception { + + String beginDateStr = null; + String endDateStr = null; + Date beginDate = param.getBeginCreateDate(); + Date endDate = param.getEndCreateDate(); + if (param.getBeginCreateDate() != null && param.getEndCreateDate() != null){ + beginDateStr = DateUtil.format(beginDate, "yyyy-MM-dd HH:mm:ss"); + endDateStr = DateUtil.format(endDate, "yyyy-MM-dd HH:mm:ss"); + } + + String sql = "CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "'"; + String allFiled = "Id"; + + if (linkConfigs.size() == 1){ + for (LinkConfig linkConfig : linkConfigs) { + sql = sql + " and " + linkConfig.getField() + " is not null"; + allFiled = allFiled + "," + linkConfig.getField(); + } + }else { + String info = null; + for (LinkConfig linkConfig : linkConfigs) { + if (StringUtils.isBlank(info)){ + info = linkConfig.getField() + " is not null" ; + }else { + info = info + " or " + linkConfig.getField() + " is not null"; + } + allFiled = allFiled + "," + linkConfig.getField(); + } + sql = sql +" and ("+ info +")"; + } + + // 表内数据总量 + Integer count = customMapper.countBySQL(param.getApi(), "where " + sql); + + log.info("表api：{} 存在" +count+ "条需更新数据！开始时间:{},结束时间：{}", param.getApi(), beginDateStr, endDateStr); + + if (count >0 ) { + int page = count % 2000 == 0 ? count / 2000 : (count / 2000) + 1; + + for (int i = 0; i < page; i++) { + + log.info("表api：{}，批次：{}，单批次数：2000，开始时间：{}，结束时间：{}，执行数据更新!", param.getApi(), i,beginDateStr, endDateStr); + + List> mapList = customMapper.list(allFiled, param.getApi(), sql + " order by Id limit " + i * 2000 + ",2000"); + + for (int j = 1; j <= mapList.size(); j++) { + List> updateMapList = new ArrayList<>(); + Map map = mapList.get(j - 1); + for (LinkConfig config : linkConfigs) { + if (map.get(config.getField()) != null){ + String type = resultMap.get(map.get(config.getField()).toString().substring(0, 3)); + Map paramMap = Maps.newHashMap(); + paramMap.put("key", config.getLinkField()); + paramMap.put("value", type); + updateMapList.add(paramMap); + if (StringUtils.isBlank(type)){ + safeSet.add(map.get(config.getField()).toString().substring(0, 3)); + }else { + + } + } + } + if (!updateMapList.isEmpty()) { + customMapper.updateById(param.getApi(), updateMapList, String.valueOf(mapList.get(j - 1).get("Id") != null?mapList.get(j - 1).get("Id") : mapList.get(j - 1).get("id"))); + } + } + } + } + + } + + /** + * 组装updateLinkType执行参数 + */ + public ReturnT updateLinkType(SalesforceParam param, List> futures) throws Exception { + Set apis; + if (StringUtils.isBlank(param.getApi())) { + apis = linkConfigService.list().stream().map(LinkConfig::getApi).collect(Collectors.toSet()); + } else { + apis = DataUtil.toIdSet(param.getApi()); + } + + //查询所有创建字段的关联类型 + QueryWrapper wrapper = new QueryWrapper<>(); + wrapper.in("api", apis).eq("is_link",1).eq("is_create",1); + Map resultMap = dataObjectService.list().stream() + .collect(Collectors.toMap( + DataObject::getKeyPrefix, + DataObject::getName, + (existing, replacement) -> replacement)); + + + + for (String api : apis) { + //邮件发送前三位编码不存在的对象 + Set safeSet = new HashSet<>(); + QueryWrapper dbQw = new QueryWrapper<>(); + dbQw.eq("api", api).eq("is_link",1).eq("is_create",1); + List linkConfigs = linkConfigService.list(dbQw); + if (linkConfigs.isEmpty()){ + continue; + } + Future future = salesforceExecutor.execute(() -> { + try { + updateLink(api,param, resultMap,safeSet,linkConfigs); + } catch (Throwable throwable) { + log.error("salesforceExecutor error", throwable); + throw new RuntimeException(throwable); + } + }, 0, 1); + futures.add(future); + + if (!safeSet.isEmpty()){ + String format = String.format("更新关联类型 error, api name: %s, \ncause:\n%s 前三位编码对象不存在！！", api, com.alibaba.fastjson2.JSON.toJSONString(safeSet)); + EmailUtil.send("DataDump ERROR", format); + } + } + // 等待当前所有线程执行完成 + salesforceExecutor.waitForFutures(futures.toArray(new Future[]{})); + + return ReturnT.SUCCESS; + } + + + /** + * 执行数据updateLinkType + */ + private void updateLink(String api,SalesforceParam param, Map resultMap,Set safeSet,List linkConfigs) throws Exception { + + String beginDateStr = null; + Date beginDate = param.getBeginModifyDate(); + if (param.getBeginModifyDate() != null ){ + beginDateStr = DateUtil.format(beginDate, "yyyy-MM-dd HH:mm:ss"); + } + String sql1 = null; + String sql2 = null; + if (beginDateStr != null){ + String updateDateField = dataFieldService.returnUpdateDateField(api); + sql1 = "where " + updateDateField + " >= '" + beginDateStr + "'"; + sql2 = updateDateField + " >= '"+ beginDateStr +"' " ; + }else { + sql1 = "where 1=1"; + sql2 = "1=1 " ; + } + + String allFiled = "Id"; + + if (linkConfigs.size() == 1){ + for (LinkConfig linkConfig : linkConfigs) { + sql1 = sql1 + " and " + linkConfig.getField() + " is not null"; + sql2 = sql2 + " and " + linkConfig.getField() + " is not null"; + allFiled = allFiled + "," + linkConfig.getField(); + } + }else { + String info = null; + for (LinkConfig linkConfig : linkConfigs) { + if (StringUtils.isBlank(info)){ + info = linkConfig.getField() + " is not null" ; + }else { + info = info + " or " + linkConfig.getField() + " is not null"; + } + allFiled = allFiled + "," + linkConfig.getField(); + } + sql1 = sql1 +" and ("+ info +")"; + sql2 = sql2 +" and ("+ info +")"; + } + + + // 表内数据总量 + Integer count = customMapper.countBySQL(api, sql1); + + log.info("表api：{} 存在" +count+ "条数据！", api); + + if (count >0 ) { + int page = count % 10000 == 0 ? count / 10000 : (count / 10000) + 1; + + for (int i = 0; i < page; i++) { + + log.info("表api：{}，批次：{}，批次量：{}，执行数据更新!", api,i, 10000); + + List> mapList = customMapper.list(allFiled, api, sql2 +" order by Id limit "+ + i * 10000 + ",10000" ); + + for (int j = 1; j <= mapList.size(); j++) { + List> updateMapList = new ArrayList<>(); + Map map = mapList.get(j - 1); + for (LinkConfig config : linkConfigs) { + if (map.get(config.getField()) != null){ + String type = resultMap.get(map.get(config.getField()).toString().substring(0, 3)); + Map paramMap = Maps.newHashMap(); + paramMap.put("key", config.getLinkField()); + paramMap.put("value", type); + updateMapList.add(paramMap); + if (StringUtils.isBlank(type)){ + safeSet.add(map.get(config.getField()).toString().substring(0, 3)); + } + } + } + if (!updateMapList.isEmpty()){ + customMapper.updateById(api, updateMapList, String.valueOf(mapList.get(j - 1).get("Id") != null?mapList.get(j - 1).get("Id") : mapList.get(j - 1).get("id"))); + } + } + } + } + + } + + + /** + * 校验删除数据入口 + */ + @Override + public ReturnT checkDeletedData(SalesforceParam param) throws Exception { + + if (param.getType() == 1){ + return manualCheckDeletedData(param); + }else { + return autoCheckDeletedData(param); + } + + } + + /** + * 删除目标ORG中的数据 + * @param param 参数 + * @return 执行结果 + * @throws Exception 异常 + */ + @Override + public ReturnT deleteTargetOrgData(SalesforceParam param) throws Exception { + log.info("deleteTargetOrgData execute start .................."); + + QueryWrapper wrapper = new QueryWrapper<>(); + if (StringUtils.isNotBlank(param.getApi())) { + wrapper.in("name", DataUtil.toIdList(param.getApi())); + } else { + wrapper.eq("data_work", 1); + } + List objectList = dataObjectService.list(wrapper); + if (objectList.isEmpty()) { + log.info("没有对象需要删除数据！!!"); + return ReturnT.FAIL; + } + + PartnerConnection connect = salesforceTargetConnect.createConnect(); + + try { + for (DataObject dataObject : objectList) { + + String api = dataObject.getName(); + + boolean moreData = true; + + DescribeSObjectResult dsr = connect.describeSObject(api); + + Field[] dsrFields = dsr.getFields(); + + String sql = "select Id from " + api + " Limit 200"; + + log.info("执行SQL：{}", sql); + + while (moreData){ + int size = 0; + + QueryResult queryResult = connect.queryAll(sql); + + if (ObjectUtils.isEmpty(queryResult) || ObjectUtils.isEmpty(queryResult.getRecords())) { + break; + } + SObject[] records = queryResult.getRecords(); + + com.alibaba.fastjson2.JSONArray objects = DataUtil.toJsonArray(records, dsrFields); + String[] ids = new String[objects.size()]; + for (int i = 0; i < objects.size(); i++) { + JSONObject jsonObject = objects.getJSONObject(i); + String id = jsonObject.getString("Id"); + ids[i] = id; + size ++; + } + DeleteResult[] deleteResults = connect.delete(ids); + for (DeleteResult deleteResult : deleteResults) { + if (deleteResult.isSuccess()) { + log.info("成功删除目标ORG中的数据，ID: {}，类型: {}", deleteResult.getId(), dataObject.getLabel()); + } else { + log.error("ID:{}删除目标ORG数据失败：{}", deleteResult.getId(), deleteResult.getErrors()); + } + } + if (size < 200){ + moreData = false; + } + } + } + TimeUnit.MILLISECONDS.sleep(1); + + }catch (InterruptedException interruptedException){ + return ReturnT.FAIL; + }catch (Exception e){ + log.info("目标ORG数据删除失败！", e); + } + + log.info("目标ORG数据删除完成！"); + + return ReturnT.SUCCESS; + } + + private ReturnT manualCheckDeletedData(SalesforceParam param) throws Exception { + + String api = "DeleteEvent"; + + PartnerConnection connect = salesforceTargetConnect.createConnect(); + + if ((param.getEndCreateDate() == null || param.getBeginCreateDate() == null) && param.getIds().isEmpty()){ + log.info("手动删除数据必须指定参数：开始与结束时间/Ids !!!!"); + return ReturnT.FAIL; + } + String sql = ""; + + if (!param.getIds().isEmpty()){ + sql = "where Record in (" + param.getIds().stream().map(id -> "'" + id + "'").collect(Collectors.joining(",")) + ")"; + }else { + Date beginDate = param.getBeginCreateDate(); + Date endDate = param.getEndCreateDate(); + String beginDateStr = DateUtil.format(beginDate, "yyyy-MM-dd HH:mm:ss"); + String endDateStr = DateUtil.format(endDate, "yyyy-MM-dd HH:mm:ss"); + sql = "where CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "'"; + } + + Integer count = customMapper.countBySQL(api, sql); + + int page = count%200 == 0 ? count/200 : (count/200) + 1; + + log.info("删除数据，总批次：{}",page); + try{ + for (int i = 0; i < page; i++) { + + List> list = customMapper.list("Record, SobjectName", api, sql + " order by Id asc limit " + i * 200 + ",200"); + + log.info("删除数据，当前批次：{}，批次数：{}", i,list.size()); + + Map idNameMap = new HashMap<>(); + String[] ids = new String[list.size()]; + + for (int z = 0; z < list.size(); z++) { + Map objectMap = customMapper.getById("new_id,SobjectName", list.get(z).get("SobjectName").toString(), list.get(z).get("Record").toString()); + if (objectMap != null && objectMap.get("new_id") != null) { + ids[z] = objectMap.get("new_id").toString(); + idNameMap.put(objectMap.get("new_id").toString(), list.get(z).get("SobjectName").toString()); + }else { + log.info("删除数据，未找到对应Id：{}，对应类型：{}", list.get(z).get("Record"), list.get(z).get("SobjectName")); + } + } + + int index = 0; + DeleteResult[] deleteResults = connect.delete(ids); + for (DeleteResult deleteResult : deleteResults) { + if (deleteResult.isSuccess()){ + String typeName = idNameMap.get(ids[index]); + Map objectMap = customMapper.getById("*", typeName, ids[index]); + + List> maps = new ArrayList<>(); + + Map m1 = new HashMap<>(); + m1.put("key", "is_delete"); + m1.put("value", 1); + + Map m2 = new HashMap<>(); + m2.put("key", "all_data"); + m2.put("value", JSON.toJSONString(objectMap)); + + maps.add(m1); + maps.add(m2); + customMapper.updateById(api, maps, ids[index]); + + customMapper.deleteOne(typeName, ids[index]); + + }else { + log.error("ID:{},删除数据失败：{}", deleteResult.getId(),deleteResult.getErrors()); + } + index++; + } + } + TimeUnit.MILLISECONDS.sleep(1); + }catch (InterruptedException e){ + return ReturnT.FAIL; + } catch (Exception e) { + log.error("checkDeletedData error api:{},错误信息：{}", api, e.getMessage()); + } + + return ReturnT.SUCCESS; + } + + + private ReturnT autoCheckDeletedData(SalesforceParam param) throws Exception { + String api = "DeleteEvent"; + PartnerConnection connect = salesforceTargetConnect.createConnect(); + + String sql = "SystemModstamp > " + DateUtil.format(DateUtils.addDays(new Date(), -15), "yyyy-MM-dd HH:mm:ss"); + + Integer count = customMapper.countBySQL(api, sql); + + int page = count%200 == 0 ? count/200 : (count/200) + 1; + + log.info("删除数据，总批次：{}",page); + try{ + for (int i = 0; i < page; i++) { + + List> list = customMapper.list("Record, SobjectName", api, sql + " order by Id asc limit " + i * 200 + ",200"); + + log.info("删除数据，当前批次：{}，批次数：{}", i,list.size()); + + Map idNameMap = new HashMap<>(); + String[] ids = new String[list.size()]; + + for (int z = 0; z < list.size(); z++) { + Map objectMap = customMapper.getById("new_id,SobjectName", list.get(z).get("SobjectName").toString(), list.get(z).get("Record").toString()); + if (objectMap != null && objectMap.get("new_id") != null) { + ids[z] = objectMap.get("new_id").toString(); + idNameMap.put(objectMap.get("new_id").toString(), list.get(z).get("SobjectName").toString()); + }else { + log.info("删除数据，未找到对应Id：{}，对应类型：{}", list.get(z).get("Record"), list.get(z).get("SobjectName")); + } + } + + int index = 0; + DeleteResult[] deleteResults = connect.delete(ids); + for (DeleteResult deleteResult : deleteResults) { + if (deleteResult.isSuccess()){ + String typeName = idNameMap.get(ids[index]); + Map objectMap = customMapper.getById("*", typeName, ids[index]); + + List> maps = new ArrayList<>(); + + Map m1 = new HashMap<>(); + m1.put("key", "is_delete"); + m1.put("value", 1); + + Map m2 = new HashMap<>(); + m2.put("key", "all_data"); + m2.put("value", JSON.toJSONString(objectMap)); + + maps.add(m1); + maps.add(m2); + customMapper.updateById(api, maps, ids[index]); + + customMapper.deleteOne(typeName, ids[index]); + + }else { + log.error("ID:{},删除数据失败：{}", deleteResult.getId(),deleteResult.getErrors()); + } + index++; + } + } + TimeUnit.MILLISECONDS.sleep(1); + }catch (InterruptedException e){ + return ReturnT.FAIL; + } catch (Exception e) { + log.error("checkDeletedData error api:{},错误信息：{}", api, e.getMessage()); + } + return ReturnT.SUCCESS; + } + + + /** + * 参数转换qw + * + * @param param 参数 + * @return qw + */ + private static QueryWrapper getDataBatchQueryWrapper(SalesforceParam param) { + QueryWrapper qw = new QueryWrapper<>(); + if (StringUtils.isNotBlank(param.getApi())) { + List strings = DataUtil.toIdList(param.getApi()); + if (CollectionUtils.isNotEmpty(strings)) { + qw.in("name", strings); + } + } + // 只查询首次同步时间不为空，且首次sf num不为0的 + qw.isNotNull("first_sync_date") + .ne("first_sf_num", 0) + .orderByAsc("created_date", "name", "sync_start_date") + .select("id", "name", "label", "sync_start_date", "sync_end_date", "first_db_num", "first_sf_num", "db_num", "sf_num", "sync_status"); + return qw; + } + + + @Override + public ReturnT insertSingle(SalesforceParam param) throws Exception { + List> futures = Lists.newArrayList(); + try { + if (StringUtils.isNotBlank(param.getApi())) { + return insertSingleData(param, futures); + } else { + return autoInsertSingleData(param, futures); + } + } catch (InterruptedException e) { + throw e; + } catch (Throwable throwable) { + salesforceExecutor.remove(futures.toArray(new Future[]{})); + log.error("insertSingle error", throwable); + throw throwable; + } + } + + @Override + public ReturnT dumpContentFolderMemberJob(SalesforceParam param) throws Exception { + String api = "ContentFolderMember"; + PartnerConnection partnerConnection = salesforceConnect.createConnect(); + + List> list = customMapper.list("Id", "ContentFolder", "new_id is not null order by Id "); + if (list.isEmpty()){ + return ReturnT.SUCCESS; + } + DescribeSObjectResult dsr = partnerConnection.describeSObject(api); + List fields = customMapper.getFields(api).stream().map(String::toUpperCase).collect(Collectors.toList()); + Field[] dsrFields = dsr.getFields(); + try { + for (Map map : list) { + String contentFolderId = (String) map.get("Id"); + + log.info("ContentFolderMember数据拉取！！！ contentFolderId：" + contentFolderId); + + String sql = "SELECT Id, ParentContentFolderId, ChildRecordId, IsDeleted, SystemModstamp, CreatedById, CreatedDate, LastModifiedById, LastModifiedDate FROM ContentFolderMember where ParentContentFolderId = '" + contentFolderId + "'"; + com.alibaba.fastjson2.JSONArray objects = null; + QueryResult queryResult = partnerConnection.queryAll(sql); + SObject[] records = queryResult.getRecords(); + objects = DataUtil.toJsonArray(records, dsrFields); + commonService.saveOrUpdate(api, fields, records, objects, true); + } + } catch (Throwable e) { + log.error("dumpContentFolderMemberJob error message:{}", e.getMessage()); + return ReturnT.FAIL; + } + + return ReturnT.SUCCESS; + } + + /** + * 富文本图片替换 + * @param param 参数 + * @return 执行结果 + * @throws Exception 异常 + */ + @Override + public ReturnT richTextImageReplace(SalesforceParam param) throws Exception { + log.info("richTextImageReplace execute start .................."); + String beginDateStr = null; + String endDateStr = null; + + if (param.getType() != 1){ + Date beginDate = param.getBeginCreateDate(); + Date endDate = param.getEndCreateDate(); + if (param.getBeginCreateDate() != null && param.getEndCreateDate() != null){ + beginDateStr = DateUtil.format(beginDate, "yyyy-MM-dd HH:mm:ss"); + endDateStr = DateUtil.format(endDate, "yyyy-MM-dd HH:mm:ss"); + } + } + // 检测路径是否存在 不存在则创建 + File excel = new File(Const.SERVER_FILE_PATH + "/richTextImages" ); + if (!excel.exists()) { + log.info("创建API目录: {}", excel.getAbsolutePath()); + boolean mkdir = excel.mkdir(); + if (!mkdir) { + log.info("创建文件存储目录失败！"); + } + } + + // 获取连接 + PartnerConnection sourceConnection = salesforceConnect.createConnect(); + PartnerConnection targetConnection = salesforceTargetConnect.createConnect(); + + QueryWrapper qw = new QueryWrapper<>(); + if (StringUtils.isNotBlank(param.getApi())) { + List apis = DataUtil.toIdList(param.getApi()); + qw.in("name", apis); + } + qw.eq("data_work", true); + List objectList = dataObjectService.list(qw); + + Map infoFlag = customMapper.list("code,value","system_config","code ='"+SystemConfigCode.INFO_FLAG+"'").get(0); + + for (DataObject dataObject : objectList) { + String api = dataObject.getName(); + QueryWrapper qw1 = new QueryWrapper<>(); + qw1.eq("api", api); + qw1.eq("isHtmlFormatted", true); + List fields = dataFieldService.list(qw1); + + for (DataField dataField : fields) { + String field = dataField.getField(); + String sqlCondition = "new_id is not null and " + field + " like '%> records = customMapper.list("Id, new_id, " + field, api, + sqlCondition + " order by Id limit " + (page * pageSize) + "," + pageSize); + + log.info("处理对象 {} 字段 {} 第 {}/{} 页，本页记录数: {}", api, field, (page + 1), totalPages, records.size()); + + SObject[] accounts = new SObject[records.size()]; + int index = 0; + + for (Map record : records) { + try { + String sourceId = (String) record.get("Id"); + String targetId = (String) record.get("new_id"); + String richTextContent = (String) record.get(field); + + if (StringUtils.isBlank(richTextContent) || !richTextContent.contains("refid")) { + continue; + } + + // 处理富文本中的图片 + String updatedContent = processRichTextImages(richTextContent, sourceId, api, field, sourceConnection, targetConnection); + + if (updatedContent == null){ + return ReturnT.FAIL; + } + + // 如果内容有变化，则更新目标组织中的记录 + if (!richTextContent.equals(updatedContent)) { + + // 更新目标组织中的记录 + SObject sObject = new SObject(); + sObject.setType(api); + sObject.setId(targetId); + sObject.setField(field, updatedContent); + sObject.setField("Id", targetId); + accounts[index] = sObject; + index ++; + + List> maps = new ArrayList<>(); + Map linkMap = new HashMap<>(); + linkMap.put("key", field + "_back"); + linkMap.put("value", richTextContent); + maps.add(linkMap); + customMapper.updateByNewId(maps,api, targetId); + + } + } catch (Exception e) { + log.error("处理记录 {} 的富文本内容时发生错误: {}", record.get("Id"), e.getMessage(), e); + } + } + + if (infoFlag != null && "1".equals(infoFlag.get("value"))){ + printlnAccountsDetails(accounts,fields); + } + SaveResult[] saveResults = targetConnection.update(accounts); + for (SaveResult saveResult : saveResults) { + if (!saveResult.getSuccess()) { + List> maps = new ArrayList<>(); + Map linkMap = new HashMap<>(); + linkMap.put("key", "is_update"); + linkMap.put("value", 2); + maps.add(linkMap); + Map linkMap1 = new HashMap<>(); + linkMap1.put("key", "error_message"); + linkMap1.put("value", JSON.toJSONString(saveResult.getErrors())); + maps.add(linkMap1); + customMapper.updateByNewId(maps,api, saveResult.getId()); + String format = String.format("数据更新 error, api name: %s, \ncause:\n%s", api, JSON.toJSONString(saveResult)); + log.info(format); + }else { + List> maps = new ArrayList<>(); + Map linkMap = new HashMap<>(); + linkMap.put("key", "is_update"); + linkMap.put("value", 1); + maps.add(linkMap); + Map linkMap1 = new HashMap<>(); + linkMap1.put("key", "error_message"); + linkMap1.put("value", null); + maps.add(linkMap1); + customMapper.updateByNewId(maps,api, saveResult.getId()); + } + } + } + } + } + + log.info("richTextImageReplace execute end .................."); + return ReturnT.SUCCESS; + } + + /** + * 处理富文本中的图片 + * @param richTextContent 原始富文本内容 + * @param sourceRecordId 源记录ID + * @param api 对象API名称 + * @param field 字段名 + * @param sourceConnection 源组织连接 + * @param targetConnection 目标组织连接 + * @return 处理后的富文本内容 + * @throws Exception 异常 + */ + private String processRichTextImages(String richTextContent, String sourceRecordId, String api, String field, + PartnerConnection sourceConnection, PartnerConnection targetConnection) throws Exception { + // 匹配富文本中包含refid的img标签 + Pattern imgPattern = Pattern.compile("]*src\\s*=\\s*\"[^\"]*refid=([^\"]+)(?:"|&|&|\\\"|\"|')?[^>]*/?>", Pattern.CASE_INSENSITIVE); Matcher matcher = imgPattern.matcher(richTextContent); + + StringBuffer updatedContent = new StringBuffer(); + Map imageIdMap = new ConcurrentHashMap<>(); + + // 遍历所有匹配的img标签 + while (matcher.find()) { + String imgTag = matcher.group(0); + String refId = matcher.group(1); + + // 检查是否已经处理过这个refId + if (imageIdMap.containsKey(refId)) { + String newRefId = imageIdMap.get(refId); + String updatedImgTag = imgTag.replace("refid=\"" + refId + "\"", "refid=\"" + newRefId + "\""); + matcher.appendReplacement(updatedContent, Matcher.quoteReplacement(updatedImgTag)); + continue; + } + + try { + // 构建源图片下载URL (参考Python脚本中的imageUrl构建方式) + String downloadUrl = FILE_DOWNLOAD_URL + + String.format("/services/data/v55.0/sobjects/%s/%s/richTextImageFields/%s/%s", + api, sourceRecordId, field, refId); + + // 获取图片的alt属性作为标题 + String title = "image"; + Pattern altPattern = Pattern.compile("alt=\"([^\"]*)\"", Pattern.CASE_INSENSITIVE); + Matcher altMatcher = altPattern.matcher(imgTag); + if (altMatcher.find()) { + title = altMatcher.group(1); + } + // 下载源图片文件 + String downloadImageFile = downloadImageFile(sourceConnection, downloadUrl, title); + + if (downloadImageFile != null) { + + // 上传图片到目标组织 + String newContentVersionId = uploadImageFile(targetConnection, downloadImageFile, title); + + if (newContentVersionId != null) { + // 获取新的ContentDocumentId + Map documentId = getDocumentId(targetConnection, newContentVersionId); + String newContentDocumentId = documentId.get("ContentDocumentId"); + String newOwnId = documentId.get("OwnerId"); + + if (newContentDocumentId != null) { + imageIdMap.put(refId, newContentDocumentId); + + // 构建新的img标签 (参考Python脚本中的richTextUrl构建方式) + String newImageUrl = + "/sfc/servlet.shepherd/version/download/" + newContentVersionId; + String updatedImgTag = ""; + + matcher.appendReplacement(updatedContent, Matcher.quoteReplacement(updatedImgTag)); + + // 记录替换日志 + richTextLogService.logRichTextImageReplace( + api, + field, + newContentDocumentId, + newOwnId, // recordOwnerId 可以从记录中获取，但此处简化处理 + richTextContent, + refId, // 原图片链接 + newImageUrl, // 新图片链接 + newContentDocumentId // 新文件ID + ); + continue; + } + } + }else { + return null; + } + + // 如果找不到对应关系，保留原始标签 + matcher.appendReplacement(updatedContent, Matcher.quoteReplacement(imgTag)); + } catch (Exception e) { + log.error("处理富文本图片时发生错误，refId: {}", refId, e); + matcher.appendReplacement(updatedContent, Matcher.quoteReplacement(imgTag)); + } + } + matcher.appendTail(updatedContent); + + return updatedContent.toString(); + } + + /** + * 从源组织下载图片文件 + * @param connection 源组织连接 + * @param downloadUrl 下载URL + * @return 图片文件数据 + * @throws Exception 异常 + */ + private String downloadImageFile(PartnerConnection connection, String downloadUrl,String title) throws Exception { + String imageServer = null; + try { + String token = connection.getSessionHeader().getSessionId(); + + Map headers = Maps.newHashMap(); + headers.put("Authorization", "Bearer " + token); + headers.put("connection", "keep-alive"); + + Response response = HttpUtil.doGet(downloadUrl, null, headers); + + if (response.body() != null && response.code() == 200){ + InputStream inputStream = response.body().byteStream(); + imageServer = dumpToRichTextImageServer(title, response, inputStream); + }else { + throw new RuntimeException("下载图片文件时发生错误，URL: " + downloadUrl + "；错误信息：" + response.message()); + } + } catch (Exception e) { + log.error("下载图片文件时发生错误，URL: {}", downloadUrl, e); + return null; + } + return imageServer; + } + + /** + * 保存富文本图片到本地服务器 + * @param response + * @param inputStream + * @throws IOException + */ + private String dumpToRichTextImageServer(String name, Response response, InputStream inputStream) throws IOException { + String path = Const.SERVER_FILE_PATH + "/richTextImages/" + name; + long offset = 0L; + RandomAccessFile accessFile = new RandomAccessFile(path, "rw"); + while (true) { + try { + // 保存到本地 + byte[] buf = new byte[8192]; + int len = 0; + if (offset > 0) { + inputStream.skip(offset); + accessFile.seek(offset); + } + while ((len = inputStream.read(buf)) != -1) { + accessFile.write(buf, 0, len); + offset += len; + } + break; + } catch (Exception e) { + if (offset <= 0) { + throw e; + } + response.close(); + } + } + accessFile.close(); + return path; + } + + /** + * 上传图片文件到目标组织 + * @param connection 目标组织连接 += * @param title 文件标题 + * @return 新的ContentVersion ID + * @throws Exception 异常 + */ + private String uploadImageFile(PartnerConnection connection, String filePath, String title) throws Exception { + String newId = null; + try { + int failCount = 0; + + String token = connection.getSessionHeader().getSessionId(); + + String uploadUrl = FILE_UPLOAD_URL + + "/services/data/v55.0/sobjects/ContentVersion"; + + HttpPost httpPost = new HttpPost(uploadUrl); + + httpPost.setHeader("Authorization", "Bearer " + token); + httpPost.setHeader("connection", "keep-alive"); + + com.alibaba.fastjson.JSONObject credentialsJsonParam = new com.alibaba.fastjson.JSONObject(); + credentialsJsonParam.put("title", title); + credentialsJsonParam.put("pathOnClient", title); + + MultipartEntityBuilder builder = MultipartEntityBuilder.create(); + builder.addTextBody("entity_content", credentialsJsonParam.toJSONString(), ContentType.APPLICATION_JSON); + builder.addBinaryBody("VersionData", new FileInputStream(filePath), ContentType.APPLICATION_OCTET_STREAM, title + ".jpg"); + HttpEntity entity = builder.build(); + httpPost.setEntity(entity); + + CloseableHttpClient httpClient = HttpClients.createDefault(); + + CloseableHttpResponse response = null; + String respContent = null; + while (failCount < Const.MAX_FAIL_COUNT) { + response = httpClient.execute(httpPost); + if (response != null && response.getStatusLine() != null && response.getStatusLine().getStatusCode() < 400 && response.getEntity() != null) { + HttpEntity he = response.getEntity(); + respContent = EntityUtils.toString(he, "UTF-8"); + newId = String.valueOf(com.alibaba.fastjson.JSONObject.parseObject(respContent).get("id")); + break; + } else { + failCount++; + log.error("富文本图片替换图片文件上传失败，返回体信息：" + JSON.toJSONString(response)); + } + } + } catch (Exception e) { + log.error("上传图片文件时发生错误，标题: {}", title, e); + return null; + } + return newId; + } + + /** + * 获取ContentVersion ID + * @param connection Salesforce连接 + * @param contentDocumentId ContentDocument ID + * @return ContentVersion ID + * @throws Exception 异常 + */ + private String getContentVersionId(PartnerConnection connection, String contentDocumentId) throws Exception { + String query = "SELECT Id FROM ContentVersion WHERE ContentDocumentId = '" + contentDocumentId + "' ORDER BY CreatedDate DESC LIMIT 1"; + QueryResult result = connection.query(query); + if (result.getRecords() != null && result.getRecords().length > 0) { + return (String) result.getRecords()[0].getField("Id"); + } + return null; + } + + /** + * 获取目标组织中的ContentDocumentId + * @param connection 目标组织连接 + * @param contentVersionId ContentVersion ID + * @return 目标组织中的ContentDocumentId + * @throws Exception 异常 + */ + private String getTargetContentDocumentId(PartnerConnection connection, String contentVersionId) throws Exception { + String query = "SELECT ContentDocumentId FROM ContentVersion WHERE Id = '" + contentVersionId + "'"; + QueryResult result = connection.query(query); + if (result.getRecords() != null && result.getRecords().length > 0) { + return (String) result.getRecords()[0].getField("ContentDocumentId"); + } + return null; + } + + /** + * 获取ContentDocumentId和OwnerId + * @param connection 连接 + * @param contentVersionId ContentVersion ID + * @return 包含ContentDocumentId和OwnerId的Map + * @throws Exception 异常 + */ + private Map getDocumentId(PartnerConnection connection, String contentVersionId) throws Exception { + String soql = "SELECT ContentDocumentId,OwnerId FROM ContentVersion WHERE Id = '" + contentVersionId + "'"; + QueryResult queryResult = connection.query(soql); + Map result = new HashMap<>(); + + if (queryResult.getRecords() != null && queryResult.getRecords().length > 0) { + SObject record = queryResult.getRecords()[0]; + result.put("ContentDocumentId", (String) record.getField("ContentDocumentId")); + result.put("OwnerId", (String) record.getField("OwnerId")); + return result; + } + return null; + } + + @Override + public void getNewIdByField(String api, String queryFields) throws Exception { + List stringList = DataUtil.toIdList(queryFields); + String maxId = null; + Field[] dsrFields; + List fields = Lists.newArrayList(); + PartnerConnection connect = salesforceTargetConnect.createConnect(); + + DescribeSObjectResult dsr = connect.describeSObject(api); + dsrFields = dsr.getFields(); + for (Field field : dsrFields) { + // 不查询文件 + if ("base64".equalsIgnoreCase(field.getType().toString())) { + continue; + } + fields.add(field.getName()); + } + // 添加特定对象的关联字段 + addRelationFields(api, fields); + + String joined = StringUtils.join(fields, ","); + com.alibaba.fastjson2.JSONArray objects; + int count = 0; + + do { + String sql = null; + if (StringUtils.isNotEmpty(maxId)){ + sql = "select " + joined + " from "+ api + " Where Id > '"+ maxId+"' LIMIT 200"; + }else { + sql = "select " + joined + " from "+ api +" LIMIT 200"; + } + log.info("执行SQL查询: {}", sql); + QueryResult queryResult = connect.queryAll(sql); + if (ObjectUtils.isEmpty(queryResult) || ObjectUtils.isEmpty(queryResult.getRecords())) { + log.info("查询结果为空，结束查询"); + break; + } + + SObject[] records = queryResult.getRecords(); + objects = DataUtil.toJsonArray(records, dsrFields); + maxId = ((JSONObject) objects.get(objects.size() - 1)).getString(Const.ID); + + log.info("处理批次数据，记录数: {}", objects.size()); + // 批量处理记录 + processRecords(api, objects, stringList); + + int totalRecords = records.length; + + count += totalRecords; + log.info("已处理记录总数: {}", count); + + if (totalRecords < 200) { + log.info("无更多数据，反写NewId任务结束！！！"); + return; + } + + TimeUnit.MILLISECONDS.sleep(1); + } while (true); + } + + private void addRelationFields(String api, List fields) { + if ("TaskRelation".equals(api) || "EventRelation".equals(api)) { + fields.add("Relation.type"); + } + if ("CollaborationGroupMember".equals(api)) { + fields.add("Member.type"); + } + if ("FeedAttachment".equals(api)) { + fields.add("FeedEntity.type"); + fields.add("Record.type"); + } + } + + private String buildQuerySql(String api, String joined, String maxId) { + StringBuilder sqlBuilder = new StringBuilder("SELECT "); + if (StringUtils.isNotEmpty(joined)) { + sqlBuilder.append(joined).append(", "); + } + sqlBuilder.append("Id FROM ").append(api); + if (StringUtils.isNotEmpty(maxId)) { + sqlBuilder.append(" WHERE Id > '").append(maxId).append("'"); + } + sqlBuilder.append(" LIMIT 200"); + return sqlBuilder.toString(); + } + + private void processRecords(String api, com.alibaba.fastjson2.JSONArray objects, List stringList) throws Exception { + for (int i = 0; i < objects.size(); i++) { + JSONObject jsonObject = objects.getJSONObject(i); + String id = jsonObject.getString(Const.ID); + List> maps = buildUpdateMaps(api, jsonObject, stringList); + + if (maps!=null && !maps.isEmpty()) { + log.info("准备更新记录，API: {}, ID: {}, 匹配字段信息: {}", api, id, maps); + customMapper.updateNewIdByFields(api, maps, id); + }else if (maps == null){ + log.info("{}无需更新记录，数据ID: {}", api, id); + } + } + } + + private List> buildUpdateMaps(String api, JSONObject jsonObject, List stringList) throws Exception { + List> maps = Lists.newArrayList(); + + for (String key : stringList) { + Map paramMap = null; + + switch (api) { + case "AccountContactRelation": + paramMap = handleAccountContactRelation(key, jsonObject); + if (paramMap == null){ + return null; + } + break; + case "TaskRelation": + case "EventRelation": + paramMap = handleTaskEventRelation(api, key, jsonObject); + if (paramMap == null){ + log.info("{}处理失败，字段: {}, 数据ID: {}", api, key, jsonObject.getString("Id")); + return null; + } + break; + case "CollaborationGroupMember": + paramMap = handleCollaborationGroupMember(key, jsonObject); + if (paramMap == null){ + log.info("CollaborationGroupMember处理失败，字段: {}, 数据ID: {}", key, jsonObject.getString("Id")); + return null; + } + break; + case "FeedAttachment": + paramMap = handleFeedAttachment(key, jsonObject); + if (paramMap == null){ + log.info("FeedAttachment处理失败，字段: {}, 数据ID: {}", key, jsonObject.getString("Id")); + return null; + } + break; + default: + paramMap = Maps.newHashMap(); + paramMap.put("key", key); + paramMap.put("value", jsonObject.getString(key)); + log.info("普通字段处理，API: {}, 字段: {}, 值: {}", api, key, jsonObject.getString(key)); + break; + } + + if (!paramMap.isEmpty()) { + maps.add(paramMap); + } + } + + return maps; + } + + private Map handleAccountContactRelation(String key, JSONObject jsonObject) throws Exception { + switch (key) { + case "ContactId": + Map contactMap = getNewIdMap(key, "Contact", jsonObject.getString(key)); + if (contactMap == null){ + log.info("AccountContactRelation，字段: {},关联对象段: {}, ContactId: {} 未找到对应的new_id", key,"Contact", jsonObject.getString(key)); + return null; + } + return contactMap; + case "AccountId": + Map accountMap = getNewIdMap(key, "Account", jsonObject.getString(key)); + if (accountMap == null){ + log.info("AccountContactRelation，字段: {},关联对象段: {}, AccountId: {} 未找到对应的new_id", key,"Account", jsonObject.getString(key)); + return null; + } + return accountMap; + default: + return null; + } + } + + private Map handleTaskEventRelation(String api, String key, JSONObject jsonObject) throws Exception { + switch (key) { + case "TaskId": + Map taskObjectMap = getNewIdMap(key, "Task", jsonObject.getString(key)); + if (taskObjectMap == null){ + log.info("TaskEventRelation，字段: {},关联对象段: {}, TaskId: {} 未找到对应的new_id", key,"Task", jsonObject.getString(key)); + return null; + } + return taskObjectMap; + case "EventId": + Map eventObjectMap = getNewIdMap(key, "Event", jsonObject.getString(key)); + if (eventObjectMap == null){ + log.info("TaskEventRelation，字段: {},关联对象段: {}, EventId: {} 未找到对应的new_id", key,"Event", jsonObject.getString(key)); + return null; + } + return eventObjectMap; + case "RelationId": + Map relationObjectMap = getNewIdMap(key, jsonObject.getString("Relation_Type"), jsonObject.getString(key)); + if (relationObjectMap == null){ + log.info("TaskEventRelation，字段: {},关联对象段: {}, RelationId: {} 未找到对应的new_id", key,jsonObject.getString("Relation_Type"), jsonObject.getString(key)); + return null; + } + return relationObjectMap; + default: + return null; + } + } + + private Map handleCollaborationGroupMember(String key, JSONObject jsonObject) throws Exception { + switch (key) { + case "CollaborationGroupId": + Map groupObjectMap = getNewIdMap(key, "CollaborationGroup", jsonObject.getString(key)); + if (groupObjectMap == null){ + log.info("CollaborationGroupMember，字段: {},关联对象段: {}, CollaborationGroupId: {} 未找到对应的new_id", key,"CollaborationGroup", jsonObject.getString(key)); + return null; + } + return groupObjectMap; + case "MemberId": + Map memberObjectMap = getNewIdMap(key, jsonObject.getString("Member_Type"), jsonObject.getString(key)); + if (memberObjectMap == null){ + log.info("CollaborationGroupMember，字段: {},关联对象段: {}, MemberId: {} 未找到对应的new_id", key,jsonObject.getString("Member_Type"), jsonObject.getString(key)); + return null; + } + return memberObjectMap; + default: + return null; + } + } + + private Map handleFeedAttachment(String key, JSONObject jsonObject) throws Exception { + switch (key) { + case "FeedEntityId": + Map feedEntityObjectMap = getNewIdMap(key, jsonObject.getString("FeedEntity_Type"), jsonObject.getString(key)); + if (feedEntityObjectMap == null){ + log.info("FeedAttachment，字段: {},关联对象段: {}, FeedEntityId: {} 未找到对应的new_id", key,jsonObject.getString("FeedEntity_Type"), jsonObject.getString(key)); + return null; + } + return feedEntityObjectMap; + case "RecordId": + Map recordObjectMap = getNewIdMap(key, jsonObject.getString("RecordId_Type"), jsonObject.getString(key)); + if (recordObjectMap == null){ + log.info("FeedAttachment，字段: {},关联对象段: {}, RecordId: {} 未找到对应的new_id", key,jsonObject.getString("RecordId_Type"), jsonObject.getString(key)); + return null; + } + return recordObjectMap; + default: + return null; + } + } + + private Map getNewIdMap(String key, String tableName, String id) throws Exception { + Map m = customMapper.getByNewId("Id", tableName, id); + if (m == null || m.get("Id") == null) { + return null; + } + Map paramMap = Maps.newHashMap(); + paramMap.put("key", key); + paramMap.put("value", m.get("Id")); + return paramMap; + } + + @Override + public void getTaskNewId(SalesforceParam param) throws Exception { + List> mapsList = customMapper.list("*", "Task", "where What_Type = EmailMessage and new_id is null"); + + } + + + /** + * 组装【单表】 一次性Insert 参数 + */ + public ReturnT insertSingleData(SalesforceParam param, List> futures) throws Exception { + List apis = DataUtil.toIdList(param.getApi()); + + String beginDateStr = null; + String endDateStr = null; + if (param.getBeginCreateDate() != null && param.getEndCreateDate() != null){ + Date beginDate = param.getBeginCreateDate(); + Date endDate = param.getEndCreateDate(); + beginDateStr = DateUtil.format(beginDate, "yyyy-MM-dd HH:mm:ss"); + endDateStr = DateUtil.format(endDate, "yyyy-MM-dd HH:mm:ss"); + } + + // 全量的时候 检测是否有自动任务锁住的表 + boolean isFull = CollectionUtils.isEmpty(param.getIds()); + if (isFull) { + QueryWrapper qw = new QueryWrapper<>(); + qw.eq("data_lock", 1).in("name", apis); + List list = dataObjectService.list(qw); + if (CollectionUtils.isNotEmpty(list)) { + String apiNames = list.stream().map(DataObject::getName).collect(Collectors.joining()); + String message = "api:" + apiNames + " is locked"; + String format = String.format("数据导入 error, api name: %s, \nparam: %s, \ncause:\n%s", apiNames, com.alibaba.fastjson2.JSON.toJSONString(param, DataDumpParam.getFilter()), message); + EmailUtil.send("DataDump ERROR", format); + return new ReturnT<>(500, message); + } + } + + PartnerConnection partnerConnection = salesforceTargetConnect.createConnect(); + for (String api : apis) { + DataObject update = dataObjectService.getById(api); + if (update == null) { + log.info("对象 {} 不存在，无法执行一次性插入操作", api); + continue; + } + try { + if (!dataFieldService.hasCreatedDate(api)){ + insertSingleShareData(api,partnerConnection,update); + continue; + } + List salesforceParams = null; + QueryWrapper dbQw = new QueryWrapper<>(); + dbQw.eq("name", api); + if (StringUtils.isNotEmpty(beginDateStr) && StringUtils.isNotEmpty(endDateStr)) { + dbQw.eq("sync_start_date", beginDateStr); // 等于开始时间 + dbQw.eq("sync_end_date", endDateStr); // 等于结束时间 + } + List list = dataBatchService.list(dbQw); + AtomicInteger batch = new AtomicInteger(1); + if (CollectionUtils.isNotEmpty(list)) { + salesforceParams = list.stream().map(t -> { + SalesforceParam salesforceParam = param.clone(); + salesforceParam.setApi(t.getName()); + salesforceParam.setBeginCreateDate(t.getSyncStartDate()); + salesforceParam.setEndCreateDate(t.getSyncEndDate()); + salesforceParam.setBatch(batch.getAndIncrement()); + return salesforceParam; + }).collect(Collectors.toList()); + } + + // 手动任务优先执行 + for (SalesforceParam salesforceParam : salesforceParams) { + if (salesforceParam.getIsSingleThread()){ + insertSingleExecute(salesforceParam, partnerConnection,update); + }else { + Future future = salesforceExecutor.execute(() -> { + try { + insertSingleExecute(salesforceParam, partnerConnection,update); + } catch (Throwable throwable) { + log.error("salesforceExecutor error", throwable); + throw new RuntimeException(throwable); + } + }, salesforceParam.getBatch(), 1); + futures.add(future); + } + } + // 等待当前所有线程执行完成 + salesforceExecutor.waitForFutures(futures.toArray(new Future[]{})); + update.setNeedUpdate(false); + } catch (Exception e) { + throw e; + } finally { + if (isFull) { + update.setName(api); + update.setDataLock(0); + dataObjectService.updateById(update); + } + } + } + return ReturnT.SUCCESS; + } + + + /** + * 组装【多】一次性Insert 参数 + */ + public ReturnT autoInsertSingleData(SalesforceParam param, List> futures) throws Exception { + QueryWrapper qw = new QueryWrapper<>(); + qw.eq("data_work", 1) + .eq("need_update", 1) + .eq("data_lock", 0) + .orderByAsc("data_index") + .last(" limit 10"); + + PartnerConnection partnerConnection = salesforceTargetConnect.createConnect(); + while (true) { + List dataObjects = dataObjectService.list(qw); + if (CollectionUtils.isEmpty(dataObjects)) { + break; + } + for (DataObject dataObject : dataObjects) { + DataObject update = new DataObject(); + log.info("insertSingle api: {}", dataObject.getName()); + TimeUnit.MILLISECONDS.sleep(1); + try { + String api = dataObject.getName(); + update.setName(dataObject.getName()); + update.setDataLock(1); + dataObjectService.updateById(update); + + if (!dataFieldService.hasCreatedDate(api)){ + insertSingleShareData(api,partnerConnection,dataObject); + update.setNeedUpdate(false); + dataObjectService.updateById(update); + continue; + } + + List salesforceParams = null; + QueryWrapper dbQw = new QueryWrapper<>(); + dbQw.eq("name", api); + List list = dataBatchService.list(dbQw); + AtomicInteger batch = new AtomicInteger(1); + if (CollectionUtils.isNotEmpty(list)) { + salesforceParams = list.stream().map(t -> { + SalesforceParam salesforceParam = param.clone(); + salesforceParam.setApi(t.getName()); + salesforceParam.setBeginCreateDate(t.getSyncStartDate()); + salesforceParam.setEndCreateDate(t.getSyncEndDate()); + salesforceParam.setBatch(batch.getAndIncrement()); + return salesforceParam; + }).collect(Collectors.toList()); + } + + // 手动任务优先执行 + for (SalesforceParam salesforceParam : salesforceParams) { + if (salesforceParam.getIsSingleThread()){ + insertSingleExecute(salesforceParam, partnerConnection,update); + }else { + Future future = salesforceExecutor.execute(() -> { + try { + insertSingleExecute(salesforceParam, partnerConnection,dataObject); + } catch (Throwable throwable) { + log.error("salesforceExecutor error", throwable); + throw new RuntimeException(throwable); + } + }, salesforceParam.getBatch(), 0); + futures.add(future); + } + } + // 等待当前所有线程执行完成 + salesforceExecutor.waitForFutures(futures.toArray(new Future[]{})); + update.setNeedUpdate(false); + } catch (InterruptedException e) { + throw e; + } catch (Throwable e) { + throw new RuntimeException(e); + } finally { + update.setDataLock(0); + dataObjectService.updateById(update); + } + } + // 等待当前所有线程执行完成 + salesforceExecutor.waitForFutures(futures.toArray(new Future[]{})); + futures.clear(); + } + return ReturnT.SUCCESS; + } + + /** + * 执行一次性Insert数据 + */ + private void insertSingleExecute(SalesforceParam param, PartnerConnection partnerConnection,DataObject dataObject) throws Exception { + Map infoFlag = customMapper.list("code,value","system_config","code ='"+SystemConfigCode.INFO_FLAG+"'").get(0); + + String api = param.getApi(); + QueryWrapper dbQw = new QueryWrapper<>(); + dbQw.eq("api", api); + List list = dataFieldService.list(dbQw); + + String sql = ""; + String sql2 = ""; + + String beginDateStr = null; + String endDateStr = null; + Date beginDate = param.getBeginCreateDate(); + Date endDate = param.getEndCreateDate(); + if (param.getBeginCreateDate() != null && param.getEndCreateDate() != null){ + beginDateStr = DateUtil.format(beginDate, "yyyy-MM-dd HH:mm:ss"); + endDateStr = DateUtil.format(endDate, "yyyy-MM-dd HH:mm:ss"); + } + + if (1 == param.getType()) { + if (api.endsWith("Share")){ + sql = "where RowCause NOT IN ('Owner','Rule','Territory','Team','ImplicitChild','ImplicitParent','TerritoryRule','ImplicitCallCenter','PortalRole','Portal','ImplicitPerson','ImplicitGrant') and new_id is null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "'"; + sql2 = "RowCause NOT IN ('Owner','Rule','Territory','Team','ImplicitChild','ImplicitParent','TerritoryRule','ImplicitCallCenter','PortalRole','Portal','ImplicitPerson','ImplicitGrant') and new_id is null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "' order by Id asc limit 10000"; + }else { + sql = "where new_id is null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "'"; + sql2 = "new_id is null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "' order by Id asc limit 10000"; + } + }else { + String updateDateField = dataFieldService.returnUpdateDateField(api); + if (api.endsWith("Share")){ + sql = "where RowCause NOT IN ('Owner','Rule','Territory','Team','ImplicitChild','ImplicitParent','TerritoryRule','ImplicitCallCenter','PortalRole','Portal','ImplicitPerson','ImplicitGrant') and new_id is null and "+updateDateField+" >= '" + beginDateStr + "' "; + sql2 = "RowCause NOT IN ('Owner','Rule','Territory','Team','ImplicitChild','ImplicitParent','TerritoryRule','ImplicitCallCenter','PortalRole','Portal','ImplicitPerson','ImplicitGrant') and new_id is null and "+updateDateField+" >= '" + beginDateStr + "' order by Id asc limit 10000"; + }else { + sql = "where new_id is null and "+updateDateField+" >= '" + beginDateStr + "' "; + sql2 = "new_id is null and "+updateDateField+" >= '" + beginDateStr + "' order by Id asc limit 10000"; + } + } + //表内数据总量 + Integer count = customMapper.countBySQL(api, sql); + + if(count == 0){ + return; + } + + //查询当前对象多态字段映射 + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("api",api).eq("is_create",1).eq("is_link",1); + List configs = linkConfigService.list(queryWrapper); + Map fieldMap = new HashMap<>(); + if (!configs.isEmpty()) { + fieldMap = configs.stream() + .collect(Collectors.toMap( + LinkConfig::getField, // Key提取器 + LinkConfig::getLinkField, // Value提取器 + (oldVal, newVal) -> newVal // 解决重复Key冲突（保留新值） + )); + } + + int targetCount = 0; + + int num = count%10000 == 0 ? count/10000 : (count/10000) + 1; + + log.info("总Insert数据 count:{}；总批次:{}；-开始时间:{}；-结束时间:{}；-api:{}；", count,num, beginDateStr, endDateStr, api); + + for (int z = 0; z < num; z++) { + + List> mapsList = customMapper.list("*", api, sql2); + + log.info("总Insert数据 count:{}；当前批次:{}；-开始时间:{}；-结束时间:{}；-api:{}；", count,z, beginDateStr, endDateStr, api); + + int size = mapsList.size(); + + //批量插入200一次 + int page = size%200 == 0 ? size/200 : (size/200) + 1; + + for (int i = 0; i < page; i++) { + + DataLog dataLog = new DataLog(api, "开始时间："+beginDateStr+";结束时间："+endDateStr, new Date(), null, "一次性插入，查询并组装" + (z*50+i) + "批数据", "QueryBD"); + int startIndex = 200 * i; + int endIndex = Math.min(startIndex + 200, size); + if (startIndex >= endIndex) { + break; + } + List> data = mapsList.subList(startIndex, endIndex); + + int sized = data.size(); + SObject[] accounts = new SObject[sized]; + String[] ids = new String[sized]; + try { + for (int j = 1; j <= sized; j++) { + SObject account = new SObject(); + account.setType(api); + String message = null; + //给对象赋值 + for (DataField dataField : list) { + String field = dataField.getField(); + String reference_to = dataField.getReferenceTo(); + + String value = String.valueOf(data.get(j - 1).get(field)); + //根据旧sfid查找引用对象新sfid + if (dataField.getIsCreateable() == null || !dataField.getIsCreateable()) { + continue; + } else if ("reference".equals(dataField.getSfType()) && data.get(j - 1).get(dataField.getField()) != null) { + + if (!"null".equals(value) && StringUtils.isNotEmpty(value)) { + //判断reference_to内是否包含User字符串 + if (dataField.getReferenceTo() != null && (dataField.getReferenceTo().contains(",User") || dataField.getReferenceTo().contains("User,"))) { + reference_to = "User"; + } + //引用类型字段 + String linkfield = fieldMap.get(dataField.getField()); + if (StringUtils.isNotBlank(linkfield) ){ + if (data.get(j-1).get(linkfield)!=null){ + reference_to = data.get(j-1).get(linkfield).toString(); + }else { + log.info("对象类型:" + api + "的数据:"+ data.get(j - 1).get("Id") +"的关联类型不存在！"); + } + } + if (reference_to == null){ + continue; + } + Map m = customMapper.getById("new_id", reference_to, value); + if (m!= null && !m.isEmpty() && m.get("new_id") != null) { + account.setField(field, m.get("new_id")); + }else { + message = "对象类型:" + api + "的数据:"+ data.get(j - 1).get("Id") +"的引用对象:" + reference_to + "的数据:"+ data.get(j - 1).get(field) +"异常！"; + log.info(message); + break; + } + } + } else { + if (data.get(j - 1).get(field) != null && StringUtils.isNotBlank(dataField.getSfType())) { + account.setField(field, DataUtil.localDataToSfData(dataField.getSfType(), value)); + }else { + account.setField(field, data.get(j - 1).get(field)); + } + } + } + if (message!=null){ + List> maps = new ArrayList<>(); + Map linkMap1 = new HashMap<>(); + linkMap1.put("key", "error_message"); + linkMap1.put("value", message); + maps.add(linkMap1); + customMapper.updateById(api, maps, data.get(j-1).get("Id").toString()); + continue; + } + if (dataObject.getIsEditable()){ + account.setField("old_owner_id__c", data.get(j - 1).get("OwnerId")); + account.setField("old_sfdc_id__c", data.get(j - 1).get("Id")); + } + + ids[j-1] = data.get(j-1).get("Id").toString(); + accounts[j-1] = account; + if (i*200+j == size){ + break; + } + } + dataLog.setEndTime(new Date()); + dataLogService.save(dataLog); + + if (infoFlag != null && "1".equals(infoFlag.get("value"))){ + printlnAccountsDetails(accounts, list); + } + + DataLog dataLog1 = new DataLog(api, "开始时间："+beginDateStr+";结束时间："+endDateStr, new Date(), null, "一次性插入，插入SF第" + (z*50+i) + "批数据", "InsertSF"); + SaveResult[] saveResults = partnerConnection.create(accounts); + dataLog1.setEndTime(new Date()); + dataLogService.save(dataLog1); + + DataLog dataLog2 = new DataLog(api, "开始时间："+beginDateStr+";结束时间："+endDateStr, new Date(), null, "一次性插入，更新DB第" + (z*50+i) + "批数据", "UpdateDB"); + + int index = 0; + for (SaveResult saveResult : saveResults){ + if (saveResult.getSuccess()) { + List> maps = new ArrayList<>(); + Map m = new HashMap<>(); + m.put("key", "new_id"); + m.put("value", saveResult.getId()); + maps.add(m); + customMapper.updateById(api, maps, ids[index]); + targetCount ++; + }else{ + List> maps = new ArrayList<>(); + Map linkMap1 = new HashMap<>(); + linkMap1.put("key", "error_message"); + linkMap1.put("value", JSON.toJSONString(saveResult.getErrors())); + maps.add(linkMap1); + customMapper.updateById(api, maps, ids[index]); + log.error("Id:{},saveResults: {}",ids[index], JSON.toJSONString(saveResult)); + } + index++; + } + dataLog2.setEndTime(new Date()); + dataLogService.save(dataLog2); + + TimeUnit.MILLISECONDS.sleep(1); + }catch (InterruptedException e){ + return; + } catch (Exception e) { + log.error("insertSingle error api:{}", api,e); + } + } + if (z*10000+size >= count){ + break; + } + } + + SalesforceParam countParam = new SalesforceParam(); + countParam.setApi(api); + countParam.setBeginCreateDate(beginDate); + countParam.setEndCreateDate(DateUtils.addSeconds(endDate, -1)); + // 存在isDeleted 只查询IsDeleted为false的 + if (dataFieldService.hasDeleted(countParam.getApi())) { + countParam.setIsDeleted(false); + } else { + // 不存在 过滤 + countParam.setIsDeleted(null); + } + Integer sfNum = 0; + if (!"FeedItem".equals(api) && !"ContentFolderMember ".equals(api)){ + sfNum = commonService.countSfNum(partnerConnection, countParam); + } + + UpdateWrapper updateQw = new UpdateWrapper<>(); + updateQw.eq("name", api) + .eq("sync_start_date", beginDate) + .eq("sync_end_date", DateUtils.addSeconds(endDate, -1)) + .set("target_sf_num", count); + dataBatchHistoryService.update(updateQw); + + UpdateWrapper updateQw2 = new UpdateWrapper<>(); + updateQw2.eq("name", api) + .eq("sync_start_date", beginDate) + .eq("sync_end_date", endDate) + .set("sf_add_num", sfNum); + dataBatchService.update(updateQw2); + + UpdateWrapper updateQw3 = new UpdateWrapper<>(); + updateQw3.eq("name", api) + .eq("sync_start_date", beginDate) + .eq("sync_end_date", DateUtils.addSeconds(endDate, -1)) + .set("target_update_num", count); + dataBatchHistoryService.update(updateQw3); + + UpdateWrapper updateQw4 = new UpdateWrapper<>(); + updateQw4.eq("name", api) + .eq("sync_start_date", beginDate) + .eq("sync_end_date", endDate) + .set("sf_update_num", targetCount); + dataBatchService.update(updateQw4); + } + + /** + * 执行一次性Insert Share数据 + */ + private void insertSingleShareData(String api, PartnerConnection partnerConnection,DataObject dataObject) throws Exception { + Map infoFlag = customMapper.list("code,value","system_config","code ='"+SystemConfigCode.INFO_FLAG+"'").get(0); + + QueryWrapper dbQw = new QueryWrapper<>(); + dbQw.eq("api", api); + List list = dataFieldService.list(dbQw); + TimeUnit.MILLISECONDS.sleep(1); + + String sql = ""; + String sql2 = ""; + + if (api.endsWith("Share")){ + sql = "where RowCause NOT IN ('Owner','Rule','Territory','Team','ImplicitChild','ImplicitParent','TerritoryRule','ImplicitCallCenter','PortalRole','Portal','ImplicitPerson','ImplicitGrant') and new_id is null "; + sql2 = "RowCause NOT IN ('Owner','Rule','Territory','Team','ImplicitChild','ImplicitParent','TerritoryRule','ImplicitCallCenter','PortalRole','Portal','ImplicitPerson','ImplicitGrant') and new_id is null limit 10000"; + }else { + sql = "where new_id is null "; + sql2 = "new_id is null limit 10000"; + } + + //表内数据总量 + Integer count = customMapper.countBySQL(api, sql); + + if (count == 0) { + return; + } + + //查询当前对象多态字段映射 + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("api",api).eq("is_create",1).eq("is_link",true); + List configs = linkConfigService.list(queryWrapper); + Map fieldMap = new HashMap<>(); + if (!configs.isEmpty()) { + fieldMap = configs.stream() + .collect(Collectors.toMap( + LinkConfig::getField, // Key提取器 + LinkConfig::getLinkField, // Value提取器 + (oldVal, newVal) -> newVal // 解决重复Key冲突（保留新值） + )); + } + + int num = count%10000 == 0 ? count/10000 : (count/10000) + 1; + + log.info("总Insert数据 count:{}；总批次：{}; -api:{}；", count,num, api); + + for (int z = 0; z < num; z++) { + + List> mapsList = customMapper.list("*", api, sql2); + + int size = mapsList.size(); + //批量插入200一次 + int page = size%200 == 0 ? size/200 : (size/200) + 1; + + log.info("执行api:{}, 执行批次:{}, 执行数据量:{}", api, z, size); + + for (int i = 0; i < page; i++) { + try { + DataLog dataLog = new DataLog(api, null, new Date(), null, "一次性插入，查询并组装" + (z*50+i) + "批数据", "QueryBD"); + + int startIndex = 200 * i; + int endIndex = Math.min(startIndex + 200, size); + if (startIndex >= endIndex) { + break; + } + List> data = mapsList.subList(startIndex, endIndex); + int sized = data.size(); + + SObject[] accounts = new SObject[sized]; + String[] ids = new String[sized]; + for (int j = 1; j <= sized; j++) { + SObject account = new SObject(); + account.setType(api); + String message = null; + //给对象赋值 + for (DataField dataField : list) { + String field = dataField.getField(); + String reference_to = dataField.getReferenceTo(); + + String value = String.valueOf(data.get(j - 1).get(field)); + //根据旧sfid查找引用对象新sfid + if (dataField.getIsCreateable() != null && !dataField.getIsCreateable()) { + continue; + } else if ("reference".equals(dataField.getSfType()) && data.get(j - 1).get(dataField.getField()) != null) { + + if (!"null".equals(value) && StringUtils.isNotEmpty(value)) { + //判断reference_to内是否包含User字符串 + if (dataField.getReferenceTo() != null && (dataField.getReferenceTo().contains(",User") || dataField.getReferenceTo().contains("User,"))) { + reference_to = "User"; + } + //引用类型字段 + String linkfield = fieldMap.get(dataField.getField()); + if (StringUtils.isNotBlank(linkfield)){ + if (data.get(j-1).get(linkfield)!=null){ + reference_to = data.get(j-1).get(linkfield).toString(); + }else { + log.info("对象类型:" + api + "的数据:"+ data.get(j-1).get("Id") +"的关联类型不存在！"); + } + } + if (reference_to == null){ + continue; + } + Map m = customMapper.getById("new_id", reference_to, value); + if (m!= null && !m.isEmpty() && m.get("new_id") != null) { + account.setField(field, m.get("new_id")); + }else { + message = "对象类型:" + api + "的数据:"+ data.get(j - 1).get("Id") +"的引用对象:" + reference_to + "的数据:"+ data.get(j - 1).get(field) +"异常！"; + log.info(message); + break; + } + } + } else { + if (data.get(j - 1).get(field) != null && StringUtils.isNotBlank(dataField.getSfType())) { + account.setField(field, DataUtil.localDataToSfData(dataField.getSfType(), value)); + }else { + account.setField(field, data.get(j - 1).get(field)); + } + } + } + if (message != null){ + List> maps = new ArrayList<>(); + Map linkMap1 = new HashMap<>(); + linkMap1.put("key", "error_message"); + linkMap1.put("value", message); + maps.add(linkMap1); + customMapper.updateById(api, maps, data.get(j-1).get("Id").toString()); + continue; + } + if (dataObject.getIsEditable()){ + account.setField("old_owner_id__c", data.get(j - 1).get("OwnerId")); + account.setField("old_sfdc_id__c", data.get(j - 1).get("Id")); + } + + ids[j-1] = data.get(j-1).get("Id").toString(); + accounts[j-1] = account; + if (i*200+j == size){ + break; + } + } + dataLog.setEndTime(new Date()); + dataLogService.save(dataLog); + + if (infoFlag != null && "1".equals(infoFlag.get("value"))){ + printlnAccountsDetails(accounts, list); + } + DataLog dataLog1 = new DataLog(api, null, new Date(), null, "一次性插入，插入SF第" + (z*50+i) + "批数据", "InsertSF"); + SaveResult[] saveResults = partnerConnection.create(accounts); + dataLog1.setEndTime(new Date()); + dataLogService.save(dataLog1); + + DataLog dataLog2 = new DataLog(api, null, new Date(), null, "一次性插入，更新DB第" + (z*50+i) + "批数据", "UpdateDB"); + + int index = 0; + for (SaveResult saveResult : saveResults){ + if (saveResult.getSuccess()) { + List> maps = new ArrayList<>(); + Map m = new HashMap<>(); + m.put("key", "new_id"); + m.put("value", saveResult.getId()); + maps.add(m); + customMapper.updateById(api, maps, ids[index]); + log.info("Created Success row with id " + saveResult.getId()); + }else{ + log.error("Id:{},saveResults: {}",ids[index], JSON.toJSONString(saveResult)); + } + index++; + } + dataLog2.setEndTime(new Date()); + dataLogService.save(dataLog2); + + TimeUnit.MILLISECONDS.sleep(1); + }catch (InterruptedException e){ + return; + }catch (Exception e) { + log.error("insertSingle error api:{}", api, e); + } + } + if (z*10000+size >= count){ + break; + } + } + } +}