【feat】 提交修复
This commit is contained in:
parent
713e545420
commit
656ca02ceb
@ -10,6 +10,7 @@ import com.sforce.soap.partner.PartnerConnection;
|
|||||||
import com.sforce.ws.ConnectionException;
|
import com.sforce.ws.ConnectionException;
|
||||||
import com.sforce.ws.ConnectorConfig;
|
import com.sforce.ws.ConnectorConfig;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
@ -54,16 +55,19 @@ public class SalesforceConnect {
|
|||||||
if ("SOURCE_ORG_PASSWORD".equals(map1.get("code"))) {
|
if ("SOURCE_ORG_PASSWORD".equals(map1.get("code"))) {
|
||||||
map.put("password", String.valueOf(map1.get("value")));
|
map.put("password", String.valueOf(map1.get("value")));
|
||||||
}
|
}
|
||||||
if ("SOURCE_ORG_SEESION".equals(map1.get("code"))) {
|
if ("SOURCE_ORG_TOKEN".equals(map1.get("code"))) {
|
||||||
map.put("seesionId", (String) map1.get("value"));
|
map.put("token", map1.get("value") == null ? "" :(String) map1.get("value"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
String username = map.get("username");
|
String username = map.get("username");
|
||||||
ConnectorConfig config = new ConnectorConfig();
|
ConnectorConfig config = new ConnectorConfig();
|
||||||
config.setUsername(username);
|
config.setUsername(username);
|
||||||
|
if (StringUtils.isNotEmpty(map.get("token"))){
|
||||||
|
config.setPassword(map.get("password") + map.get("token"));
|
||||||
|
}else {
|
||||||
config.setPassword(map.get("password"));
|
config.setPassword(map.get("password"));
|
||||||
config.setSessionId(map.get("seesionId"));
|
}
|
||||||
String url = map.get("url");
|
String url = map.get("url");
|
||||||
config.setAuthEndpoint(url);
|
config.setAuthEndpoint(url);
|
||||||
config.setServiceEndpoint(url);
|
config.setServiceEndpoint(url);
|
||||||
@ -110,11 +114,15 @@ public class SalesforceConnect {
|
|||||||
if ("SOURCE_ORG_PASSWORD".equals(map1.get("code"))) {
|
if ("SOURCE_ORG_PASSWORD".equals(map1.get("code"))) {
|
||||||
map.put("password", String.valueOf(map1.get("value")));
|
map.put("password", String.valueOf(map1.get("value")));
|
||||||
}
|
}
|
||||||
if ("SOURCE_ORG_SEESION".equals(map1.get("code"))) {
|
if ("SOURCE_ORG_TOKEN".equals(map1.get("code"))) {
|
||||||
map.put("seesionId", (String) map1.get("value"));
|
map.put("token", map1.get("value") == null ? "" :(String) map1.get("value"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if (StringUtils.isNotEmpty(map.get("token"))){
|
||||||
|
return BulkUtil.getBulkConnection(map.get("username"),map.get("password") + map.get("token"),map.get("url"));
|
||||||
|
}else {
|
||||||
return BulkUtil.getBulkConnection(map.get("username"),map.get("password"),map.get("url"));
|
return BulkUtil.getBulkConnection(map.get("username"),map.get("password"),map.get("url"));
|
||||||
|
}
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
failCount ++;
|
failCount ++;
|
||||||
log.error("源ORG连接异常!休眠一分钟再次发起重试~~", e);
|
log.error("源ORG连接异常!休眠一分钟再次发起重试~~", e);
|
||||||
|
|||||||
@ -10,6 +10,7 @@ import com.sforce.soap.partner.PartnerConnection;
|
|||||||
import com.sforce.ws.ConnectionException;
|
import com.sforce.ws.ConnectionException;
|
||||||
import com.sforce.ws.ConnectorConfig;
|
import com.sforce.ws.ConnectorConfig;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
@ -49,16 +50,19 @@ public class SalesforceTargetConnect {
|
|||||||
if ("TARGET_ORG_PASSWORD".equals(map1.get("code"))) {
|
if ("TARGET_ORG_PASSWORD".equals(map1.get("code"))) {
|
||||||
map.put("password", (String) map1.get("value"));
|
map.put("password", (String) map1.get("value"));
|
||||||
}
|
}
|
||||||
if ("TARGET_ORG_SEESION".equals(map1.get("code"))) {
|
if ("TARGET_ORG_TOKEN".equals(map1.get("code"))) {
|
||||||
map.put("seesionId", (String) map1.get("value"));
|
map.put("token", map1.get("value") == null ? "" :(String) map1.get("value"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
String username = map.get("username");
|
String username = map.get("username");
|
||||||
ConnectorConfig config = new ConnectorConfig();
|
ConnectorConfig config = new ConnectorConfig();
|
||||||
config.setUsername(username);
|
config.setUsername(username);
|
||||||
|
if (StringUtils.isNotEmpty(map.get("token"))){
|
||||||
|
config.setPassword(map.get("password") + map.get("token"));
|
||||||
|
}else {
|
||||||
config.setPassword(map.get("password"));
|
config.setPassword(map.get("password"));
|
||||||
config.setSessionId(map.get("seesionId"));
|
}
|
||||||
String url = map.get("url");
|
String url = map.get("url");
|
||||||
config.setAuthEndpoint(url);
|
config.setAuthEndpoint(url);
|
||||||
config.setServiceEndpoint(url);
|
config.setServiceEndpoint(url);
|
||||||
@ -109,11 +113,11 @@ public class SalesforceTargetConnect {
|
|||||||
if ("TARGET_ORG_PASSWORD".equals(map1.get("code"))) {
|
if ("TARGET_ORG_PASSWORD".equals(map1.get("code"))) {
|
||||||
map.put("password", (String) map1.get("value"));
|
map.put("password", (String) map1.get("value"));
|
||||||
}
|
}
|
||||||
if ("TARGET_ORG_SEESION".equals(map1.get("code"))) {
|
if ("TARGET_ORG_TOKEN".equals(map1.get("code"))) {
|
||||||
map.put("seesionId", (String) map1.get("value"));
|
map.put("token", map1.get("value") == null ? "" :(String) map1.get("value"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return BulkUtil.getBulkConnection(map.get("username"),map.get("password"),map.get("url"));
|
return BulkUtil.getBulkConnection(map.get("username"),map.get("password") + map.get("token"),map.get("url"));
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
failCount++;
|
failCount++;
|
||||||
log.error("目标ORG连接异常!休眠一分钟再次发起重试~~", e);
|
log.error("目标ORG连接异常!休眠一分钟再次发起重试~~", e);
|
||||||
|
|||||||
@ -199,7 +199,7 @@ public class CommonBatchServiceImpl implements CommonBatchService {
|
|||||||
.filter(field -> !"SystemModstamp".equals(field) && !"LastModifiedDate".equals(field))
|
.filter(field -> !"SystemModstamp".equals(field) && !"LastModifiedDate".equals(field))
|
||||||
.collect(Collectors.joining(", "));
|
.collect(Collectors.joining(", "));
|
||||||
|
|
||||||
String sql = "select " + fieldStr + " from " + param.getApi() ;
|
String sql = "select Id__c,OppId__c from " + param.getApi();
|
||||||
log.info("构建查询SQL: {}", sql);
|
log.info("构建查询SQL: {}", sql);
|
||||||
|
|
||||||
job = BulkUtil.createJob(bulkConnect, api, OperationEnum.queryAll);
|
job = BulkUtil.createJob(bulkConnect, api, OperationEnum.queryAll);
|
||||||
@ -234,7 +234,7 @@ public class CommonBatchServiceImpl implements CommonBatchService {
|
|||||||
|
|
||||||
// 使用确定性命名策略,避免使用随机时间戳
|
// 使用确定性命名策略,避免使用随机时间戳
|
||||||
String fileName = Const.SERVER_FILE_PATH + "/" + api+ "/" + DateFormatUtils.format(new Date(), "yyyyMMddHHmmss");
|
String fileName = Const.SERVER_FILE_PATH + "/" + api+ "/" + DateFormatUtils.format(new Date(), "yyyyMMddHHmmss");
|
||||||
final long maxFileSize = 50 * 1024 * 1024; // 50MB
|
final long maxFileSize = 20 * 1024 * 1024; // 50MB
|
||||||
|
|
||||||
int fileIndex = 0;
|
int fileIndex = 0;
|
||||||
Path currentFilePath = Paths.get(fileName + "_" + fileIndex + ".csv");
|
Path currentFilePath = Paths.get(fileName + "_" + fileIndex + ".csv");
|
||||||
@ -502,15 +502,9 @@ public class CommonBatchServiceImpl implements CommonBatchService {
|
|||||||
|
|
||||||
// 使用Bulk V2 API导入所有文件
|
// 使用Bulk V2 API导入所有文件
|
||||||
for (File csvFile : files) {
|
for (File csvFile : files) {
|
||||||
// 添加重试机制
|
|
||||||
int maxRetries = 3;
|
|
||||||
int retryCount = 0;
|
|
||||||
boolean success = false;
|
|
||||||
|
|
||||||
while (retryCount < maxRetries && !success) {
|
|
||||||
try {
|
|
||||||
String filePath = csvFile.getAbsolutePath();
|
String filePath = csvFile.getAbsolutePath();
|
||||||
log.info("开始处理CSV文件: {}, 重试次数: {}", filePath, retryCount);
|
try {
|
||||||
|
log.info("开始处理CSV文件: {}", filePath);
|
||||||
|
|
||||||
// 检查CSV文件是否为空或只有一行标题
|
// 检查CSV文件是否为空或只有一行标题
|
||||||
try (BufferedReader reader = new BufferedReader(new FileReader(csvFile))) {
|
try (BufferedReader reader = new BufferedReader(new FileReader(csvFile))) {
|
||||||
@ -546,7 +540,7 @@ public class CommonBatchServiceImpl implements CommonBatchService {
|
|||||||
closeBulkV2Job(url, jobId, targetBulkConnect);
|
closeBulkV2Job(url, jobId, targetBulkConnect);
|
||||||
log.info("作业关闭请求已发送,作业ID: {}", jobId);
|
log.info("作业关闭请求已发送,作业ID: {}", jobId);
|
||||||
|
|
||||||
// 等待作业完成
|
// <EFBFBD>ab等待作业完成
|
||||||
log.info("等待作业完成,作业ID: {}", jobId);
|
log.info("等待作业完成,作业ID: {}", jobId);
|
||||||
BulkUtil.waitForBulkV2JobCompletion(url, jobId, targetBulkConnect);
|
BulkUtil.waitForBulkV2JobCompletion(url, jobId, targetBulkConnect);
|
||||||
|
|
||||||
@ -564,29 +558,14 @@ public class CommonBatchServiceImpl implements CommonBatchService {
|
|||||||
downloadJobResultFiles(url, jobId, targetBulkConnect, api, csvFile.getName());
|
downloadJobResultFiles(url, jobId, targetBulkConnect, api, csvFile.getName());
|
||||||
log.info("作业结果文件下载完成,作业ID: {}", jobId);
|
log.info("作业结果文件下载完成,作业ID: {}", jobId);
|
||||||
|
|
||||||
success = true;
|
|
||||||
TimeUnit.SECONDS.sleep(30);
|
TimeUnit.SECONDS.sleep(30);
|
||||||
|
|
||||||
} catch (InterruptedException interruptedExc) {
|
} catch (InterruptedException interruptedExc) {
|
||||||
return ReturnT.FAIL;
|
return ReturnT.FAIL;
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
retryCount++;
|
EmailUtil.send("DataDump Import Failed", "BigObject数据导入失败,API: " + api +",CSV文件:" + filePath + "错误信息: " + e.getMessage());
|
||||||
log.error("处理文件 {} 时发生错误,重试次数: {}/{},错误: {}",
|
throw new RuntimeException( e);
|
||||||
csvFile.getName(), retryCount, maxRetries, e.getMessage(), e);
|
|
||||||
|
|
||||||
// 对于特定可重试的错误,如并发修改异常,进行重试
|
|
||||||
if (shouldRetry(e) && retryCount < maxRetries) {
|
|
||||||
// 指数退避策略
|
|
||||||
long backoffTime = (long) Math.pow(2, retryCount) * 1000;
|
|
||||||
log.info("等待 {} ms 后进行重试...", backoffTime);
|
|
||||||
Thread.sleep(backoffTime);
|
|
||||||
} else if (retryCount >= maxRetries) {
|
|
||||||
log.error("处理文件 {} 达到最大重试次数,放弃处理", csvFile.getName());
|
|
||||||
throw e;
|
|
||||||
} else {
|
|
||||||
throw e;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -609,6 +609,7 @@ public class DataVerifyServiceImpl implements DataVerifyService {
|
|||||||
// 2. 处理字段相关信息
|
// 2. 处理字段相关信息
|
||||||
Map<String, Map<String, Field>> fieldMap = handlerFieldInfo(fields, fieldTypeSheet, targetConn, sourceConn, dataObject, excelWriter);
|
Map<String, Map<String, Field>> fieldMap = handlerFieldInfo(fields, fieldTypeSheet, targetConn, sourceConn, dataObject, excelWriter);
|
||||||
|
|
||||||
|
HashSet<String> objectHashSet = new HashSet<>();
|
||||||
|
|
||||||
// 3. 处理每个批次
|
// 3. 处理每个批次
|
||||||
List<DataBatch> batches = getDataBatches(objectApi);
|
List<DataBatch> batches = getDataBatches(objectApi);
|
||||||
@ -621,7 +622,7 @@ public class DataVerifyServiceImpl implements DataVerifyService {
|
|||||||
// 为每个批次单独创建校验文件
|
// 为每个批次单独创建校验文件
|
||||||
String batchQualityCheckFilePath = generateObjectQualityCheckFilePath(objectApi + "_" + batch.getId());
|
String batchQualityCheckFilePath = generateObjectQualityCheckFilePath(objectApi + "_" + batch.getId());
|
||||||
try (ExcelWriter objectWriter = EasyExcel.write(batchQualityCheckFilePath).build()) {
|
try (ExcelWriter objectWriter = EasyExcel.write(batchQualityCheckFilePath).build()) {
|
||||||
batchHasError = processDataBatch(batch, objectApi, fields, excelWriter, objectWriter, summarySheet, dataObject, sourceConn, targetConn, fieldMap, ignoreBeginDate) || batchHasError;
|
batchHasError = processDataBatch(batch, objectApi, fields, excelWriter, objectWriter, summarySheet, dataObject, sourceConn, targetConn, fieldMap, ignoreBeginDate,objectHashSet) || batchHasError;
|
||||||
}catch (Exception e){
|
}catch (Exception e){
|
||||||
log.error("API={},批次ID={}处理异常:{}",objectApi,batch.getId(), e.getMessage(),e);
|
log.error("API={},批次ID={}处理异常:{}",objectApi,batch.getId(), e.getMessage(),e);
|
||||||
}
|
}
|
||||||
@ -649,6 +650,7 @@ public class DataVerifyServiceImpl implements DataVerifyService {
|
|||||||
arrayList.add(currentFilePath);
|
arrayList.add(currentFilePath);
|
||||||
objectWriter = EasyExcel.write(currentFilePath).build();
|
objectWriter = EasyExcel.write(currentFilePath).build();
|
||||||
|
|
||||||
|
try {
|
||||||
for (DataBatch batch : batches) {
|
for (DataBatch batch : batches) {
|
||||||
String startDate = DateFormatUtils.format(batch.getSyncStartDate(), "yyyy-MM-dd HH:mm:ss");
|
String startDate = DateFormatUtils.format(batch.getSyncStartDate(), "yyyy-MM-dd HH:mm:ss");
|
||||||
String endDate = DateFormatUtils.format(batch.getSyncEndDate(), "yyyy-MM-dd HH:mm:ss");
|
String endDate = DateFormatUtils.format(batch.getSyncEndDate(), "yyyy-MM-dd HH:mm:ss");
|
||||||
@ -841,6 +843,10 @@ public class DataVerifyServiceImpl implements DataVerifyService {
|
|||||||
if (sourceValue == null && targetValue == null) {
|
if (sourceValue == null && targetValue == null) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
if (targetField == null){
|
||||||
|
objectHashSet.add(fieldName);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
//判断reference_to内是否包含User字符串
|
//判断reference_to内是否包含User字符串
|
||||||
String referenceTo = field.getReferenceTo();
|
String referenceTo = field.getReferenceTo();
|
||||||
@ -948,7 +954,12 @@ public class DataVerifyServiceImpl implements DataVerifyService {
|
|||||||
log.error("API={},处理异常:{}",objectApi, e.getMessage(),e);
|
log.error("API={},处理异常:{}",objectApi, e.getMessage(),e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
} finally {
|
||||||
|
// 确保ExcelWriter被正确关闭,这样数据才会被写入文件
|
||||||
|
if (objectWriter != null) {
|
||||||
|
objectWriter.finish();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (sendEmail && hasError){
|
if (sendEmail && hasError){
|
||||||
sendObjectQualityCheckEmailList(objectApi, arrayList);
|
sendObjectQualityCheckEmailList(objectApi, arrayList);
|
||||||
@ -957,6 +968,12 @@ public class DataVerifyServiceImpl implements DataVerifyService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!objectHashSet.isEmpty()){
|
||||||
|
String missingFields = "对象 " +objectApi + "在目标ORG中不存在以下字段:" + String.join(", ", objectHashSet);
|
||||||
|
log.info(missingFields);
|
||||||
|
EmailUtil.send("数据质量校验 ERROR", missingFields);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1052,7 +1069,7 @@ public class DataVerifyServiceImpl implements DataVerifyService {
|
|||||||
*/
|
*/
|
||||||
private boolean processDataBatch(DataBatch batch, String objectApi, List<DataField> fields,
|
private boolean processDataBatch(DataBatch batch, String objectApi, List<DataField> fields,
|
||||||
ExcelWriter excelWriter,ExcelWriter objectWriter, WriteSheet summarySheet, DataObject dataObject,
|
ExcelWriter excelWriter,ExcelWriter objectWriter, WriteSheet summarySheet, DataObject dataObject,
|
||||||
PartnerConnection sourceConn, PartnerConnection targetConn,Map<String, Map<String, Field>> fieldMap, Date ignoreBeginDate) {
|
PartnerConnection sourceConn, PartnerConnection targetConn,Map<String, Map<String, Field>> fieldMap, Date ignoreBeginDate,Set< String> objectHashSet) {
|
||||||
|
|
||||||
WriteSheet detailSheet = createDetailSheet(excelWriter, dataObject);
|
WriteSheet detailSheet = createDetailSheet(excelWriter, dataObject);
|
||||||
|
|
||||||
@ -1073,6 +1090,7 @@ public class DataVerifyServiceImpl implements DataVerifyService {
|
|||||||
Map<String, Field> sourceFieldMap = fieldMap.get("source");
|
Map<String, Field> sourceFieldMap = fieldMap.get("source");
|
||||||
Map<String, Field> targetFieldMap = fieldMap.get("target");
|
Map<String, Field> targetFieldMap = fieldMap.get("target");
|
||||||
|
|
||||||
|
|
||||||
while (true) {
|
while (true) {
|
||||||
List<Map<String, Object>> recordIds = queryRecordIds(objectApi, startDate, endDate, page);
|
List<Map<String, Object>> recordIds = queryRecordIds(objectApi, startDate, endDate, page);
|
||||||
if (recordIds.isEmpty()) break;
|
if (recordIds.isEmpty()) break;
|
||||||
@ -1203,6 +1221,11 @@ public class DataVerifyServiceImpl implements DataVerifyService {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (targetField == null){
|
||||||
|
objectHashSet.add(fieldName);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
//判断reference_to内是否包含User字符串
|
//判断reference_to内是否包含User字符串
|
||||||
String referenceTo = field.getReferenceTo();
|
String referenceTo = field.getReferenceTo();
|
||||||
if (referenceTo !=null && (referenceTo.contains(",User") || referenceTo.contains("User,"))) {
|
if (referenceTo !=null && (referenceTo.contains(",User") || referenceTo.contains("User,"))) {
|
||||||
@ -1491,17 +1514,25 @@ private void sendIncrementalQualityCheckEmail(DataVerifyParam param, String file
|
|||||||
|
|
||||||
String qualityCheckFilePath = this.generateObjectIncrementalQualityCheckFilePath(objectApi, beginDate);
|
String qualityCheckFilePath = this.generateObjectIncrementalQualityCheckFilePath(objectApi, beginDate);
|
||||||
|
|
||||||
|
HashSet<String> objectHashSet = new HashSet<>();
|
||||||
|
|
||||||
boolean hasError = false;
|
boolean hasError = false;
|
||||||
try (ExcelWriter objectWriter = EasyExcel.write(qualityCheckFilePath).build()) {
|
try (ExcelWriter objectWriter = EasyExcel.write(qualityCheckFilePath).build()) {
|
||||||
WriteSheet detailSheet = this.createDetailSheet(objectWriter, dataObject); // 修复:使用正确的ExcelWriter
|
WriteSheet detailSheet = this.createDetailSheet(objectWriter, dataObject); // 修复:使用正确的ExcelWriter
|
||||||
|
|
||||||
// 3. 处理增量数据(不通过batch)
|
// 3. 处理增量数据(不通过batch)
|
||||||
hasError = this.processIncrementalData(objectApi, fields, excelWriter, objectWriter, summarySheet, detailSheet,
|
hasError = this.processIncrementalData(objectApi, fields, excelWriter, objectWriter, summarySheet, detailSheet,
|
||||||
sourceConn, targetConn, fieldMap, beginDate, endDate);
|
sourceConn, targetConn, fieldMap, beginDate, endDate, objectHashSet);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.info("处理对象 {} 增量数据时发生异常", objectApi, e);
|
log.info("处理对象 {} 增量数据时发生异常", objectApi, e);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!objectHashSet.isEmpty()){
|
||||||
|
String missingFields = "对象 " +objectApi + "在目标ORG中不存在以下字段:" + String.join(", ", objectHashSet);
|
||||||
|
log.info(missingFields);
|
||||||
|
EmailUtil.send("数据质量校验 ERROR", missingFields);
|
||||||
|
}
|
||||||
|
|
||||||
if (sendEmail && hasError) {
|
if (sendEmail && hasError) {
|
||||||
this.sendObjectQualityCheckEmail(objectApi + "_增量", qualityCheckFilePath);
|
this.sendObjectQualityCheckEmail(objectApi + "_增量", qualityCheckFilePath);
|
||||||
}
|
}
|
||||||
@ -1525,7 +1556,7 @@ private void sendIncrementalQualityCheckEmail(DataVerifyParam param, String file
|
|||||||
private boolean processIncrementalData(String objectApi, List<DataField> fields,
|
private boolean processIncrementalData(String objectApi, List<DataField> fields,
|
||||||
ExcelWriter excelWriter, ExcelWriter objectWriter, WriteSheet summarySheet, WriteSheet detailSheet,
|
ExcelWriter excelWriter, ExcelWriter objectWriter, WriteSheet summarySheet, WriteSheet detailSheet,
|
||||||
PartnerConnection sourceConn, PartnerConnection targetConn,
|
PartnerConnection sourceConn, PartnerConnection targetConn,
|
||||||
Map<String, Map<String, Field>> fieldMap, Date beginDate, Date endwDate) {
|
Map<String, Map<String, Field>> fieldMap, Date beginDate, Date endwDate ,Set< String> objectHashSet) {
|
||||||
String startDate = DateFormatUtils.format(beginDate, "yyyy-MM-dd HH:mm:ss");
|
String startDate = DateFormatUtils.format(beginDate, "yyyy-MM-dd HH:mm:ss");
|
||||||
String endDate = DateFormatUtils.format(endwDate, "yyyy-MM-dd HH:mm:ss");
|
String endDate = DateFormatUtils.format(endwDate, "yyyy-MM-dd HH:mm:ss");
|
||||||
|
|
||||||
@ -1540,6 +1571,8 @@ private void sendIncrementalQualityCheckEmail(DataVerifyParam param, String file
|
|||||||
|
|
||||||
Map<String, Field> sourceFieldMap = fieldMap.get("source");
|
Map<String, Field> sourceFieldMap = fieldMap.get("source");
|
||||||
Map<String, Field> targetFieldMap = fieldMap.get("target");
|
Map<String, Field> targetFieldMap = fieldMap.get("target");
|
||||||
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
DescribeSObjectResult dsr = sourceConn.describeSObject(objectApi);
|
DescribeSObjectResult dsr = sourceConn.describeSObject(objectApi);
|
||||||
Field[] dsrFields = dsr.getFields();
|
Field[] dsrFields = dsr.getFields();
|
||||||
@ -1676,6 +1709,11 @@ private void sendIncrementalQualityCheckEmail(DataVerifyParam param, String file
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (targetField == null){
|
||||||
|
objectHashSet.add(fieldName);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
//判断reference_to内是否包含User字符串
|
//判断reference_to内是否包含User字符串
|
||||||
String referenceTo = field.getReferenceTo();
|
String referenceTo = field.getReferenceTo();
|
||||||
if (referenceTo !=null && (referenceTo.contains(",User") || referenceTo.contains("User,"))) {
|
if (referenceTo !=null && (referenceTo.contains(",User") || referenceTo.contains("User,"))) {
|
||||||
@ -1747,6 +1785,7 @@ private void sendIncrementalQualityCheckEmail(DataVerifyParam param, String file
|
|||||||
log.error("verify error", throwable);
|
log.error("verify error", throwable);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// 写入批次汇总结果
|
// 写入批次汇总结果
|
||||||
List<String> row = Lists.newArrayList();
|
List<String> row = Lists.newArrayList();
|
||||||
row.add(" ");
|
row.add(" ");
|
||||||
|
|||||||
@ -561,6 +561,8 @@ public class FileServiceImpl implements FileService {
|
|||||||
log.info("文件上传错误,返回:" + JSON.toJSONString(response));
|
log.info("文件上传错误,返回:" + JSON.toJSONString(response));
|
||||||
throw new RuntimeException();
|
throw new RuntimeException();
|
||||||
}
|
}
|
||||||
|
}else {
|
||||||
|
log.info("文件路径URL为空,数据库执行SQL!!!");
|
||||||
}
|
}
|
||||||
Map<String, Object> paramMap = Maps.newHashMap();
|
Map<String, Object> paramMap = Maps.newHashMap();
|
||||||
paramMap.put("key", "is_upload");
|
paramMap.put("key", "is_upload");
|
||||||
|
|||||||
@ -14,6 +14,7 @@ import com.celnet.datadump.util.EmailUtil;
|
|||||||
import com.sforce.soap.partner.PartnerConnection;
|
import com.sforce.soap.partner.PartnerConnection;
|
||||||
import com.sforce.ws.ConnectionException;
|
import com.sforce.ws.ConnectionException;
|
||||||
import com.sforce.ws.ConnectorConfig;
|
import com.sforce.ws.ConnectorConfig;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
@ -23,11 +24,10 @@ import java.util.HashMap;
|
|||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
import static com.celnet.datadump.global.SystemConfigCode.EXECUTOR_SIZE;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* org配置表 服务实现类
|
* org配置表 服务实现类
|
||||||
*/
|
*/
|
||||||
|
@Slf4j
|
||||||
@Service
|
@Service
|
||||||
public class OrgConfigServiceImpl extends ServiceImpl<OrgConfigMapper, OrgConfig> implements OrgConfigService {
|
public class OrgConfigServiceImpl extends ServiceImpl<OrgConfigMapper, OrgConfig> implements OrgConfigService {
|
||||||
|
|
||||||
@ -43,31 +43,40 @@ public class OrgConfigServiceImpl extends ServiceImpl<OrgConfigMapper, OrgConfig
|
|||||||
String uploadUrl = null;
|
String uploadUrl = null;
|
||||||
String dumpUrl = null;
|
String dumpUrl = null;
|
||||||
|
|
||||||
|
Map<String, String> sourceOrgMap = new HashMap<>();
|
||||||
try {
|
try {
|
||||||
List<Map<String, Object>> poll = customerMapper.list("code,value","org_config",null);
|
List<Map<String, Object>> poll = customerMapper.list("code,value","org_config",null);
|
||||||
//遍历poll,找出code值为SOURCE_ORG_URL,SOURCE_ORG_USERNAME,SOURCE_ORG_PASSWORD的value值
|
//遍历poll,找出code值为SOURCE_ORG_URL,SOURCE_ORG_USERNAME,SOURCE_ORG_PASSWORD的value值
|
||||||
Map<String, String> map = new HashMap<>();
|
|
||||||
for (Map<String, Object> map1 : poll) {
|
for (Map<String, Object> map1 : poll) {
|
||||||
if ("SOURCE_ORG_URL".equals(map1.get("code"))) {
|
if ("SOURCE_ORG_URL".equals(map1.get("code"))) {
|
||||||
map.put("url", (String) map1.get("value"));
|
sourceOrgMap.put("url", (String) map1.get("value"));
|
||||||
sourceOrgUrl = map1.get("value").toString();
|
sourceOrgUrl = map1.get("value").toString();
|
||||||
}
|
}
|
||||||
if ("SOURCE_ORG_USERNAME".equals(map1.get("code"))) {
|
if ("SOURCE_ORG_USERNAME".equals(map1.get("code"))) {
|
||||||
map.put("username", (String) map1.get("value"));
|
sourceOrgMap.put("username", (String) map1.get("value"));
|
||||||
}
|
}
|
||||||
if ("SOURCE_ORG_PASSWORD".equals(map1.get("code"))) {
|
if ("SOURCE_ORG_PASSWORD".equals(map1.get("code"))) {
|
||||||
map.put("password", String.valueOf(map1.get("value")));
|
sourceOrgMap.put("password", String.valueOf(map1.get("value")));
|
||||||
}
|
}
|
||||||
if ("FILE_DOWNLOAD_URL".equals(map1.get("code"))) {
|
if ("FILE_DOWNLOAD_URL".equals(map1.get("code"))) {
|
||||||
dumpUrl = (String) map1.get("value");
|
dumpUrl = (String) map1.get("value");
|
||||||
}
|
}
|
||||||
|
if ("SOURCE_ORG_TOKEN".equals(map1.get("code"))) {
|
||||||
|
sourceOrgMap.put("token", map1.get("value") == null ? "" :(String) map1.get("value"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
String username = map.get("username");
|
String username = sourceOrgMap.get("username");
|
||||||
ConnectorConfig config = new ConnectorConfig();
|
ConnectorConfig config = new ConnectorConfig();
|
||||||
config.setUsername(username);
|
config.setUsername(username);
|
||||||
config.setPassword(map.get("password"));
|
if (StringUtils.isNotEmpty(sourceOrgMap.get("token"))){
|
||||||
String url = map.get("url");
|
config.setPassword(sourceOrgMap.get("password") + sourceOrgMap.get("token"));
|
||||||
|
}else {
|
||||||
|
config.setPassword(sourceOrgMap.get("password"));
|
||||||
|
}
|
||||||
|
log.info("username:{}", username);
|
||||||
|
log.info("password:{}", sourceOrgMap.get("password") + sourceOrgMap.get("token"));
|
||||||
|
String url = sourceOrgMap.get("url");
|
||||||
config.setAuthEndpoint(url);
|
config.setAuthEndpoint(url);
|
||||||
config.setServiceEndpoint(url);
|
config.setServiceEndpoint(url);
|
||||||
config.setConnectionTimeout(60 * 60 * 1000);
|
config.setConnectionTimeout(60 * 60 * 1000);
|
||||||
@ -75,38 +84,52 @@ public class OrgConfigServiceImpl extends ServiceImpl<OrgConfigMapper, OrgConfig
|
|||||||
PartnerConnection connection = new PartnerConnection(config);
|
PartnerConnection connection = new PartnerConnection(config);
|
||||||
String orgId = connection.getUserInfo().getOrganizationId();
|
String orgId = connection.getUserInfo().getOrganizationId();
|
||||||
} catch (ConnectionException e) {
|
} catch (ConnectionException e) {
|
||||||
String message = "源ORG连接配置错误!,\n地址:" + sourceOrgUrl + "\n错误信息:\n" + e.getMessage() ;;
|
log.info("exception message", e);
|
||||||
String format = String.format("ORG连接异常!, \ncause:\n%s", message);
|
String message = String.format("源ORG连接配置错误!%n地址:%s%n用户名:%s%nToken:%s%n错误信息:%s",
|
||||||
|
sourceOrgUrl,
|
||||||
|
sourceOrgMap.get("username"),
|
||||||
|
sourceOrgMap.get("token"),
|
||||||
|
e.getMessage());
|
||||||
|
String format = String.format("ORG连接异常!%n详细信息:%n%s", message);
|
||||||
EmailUtil.send("DataDump ERROR", format);
|
EmailUtil.send("DataDump ERROR", format);
|
||||||
log.error("exception message", e);
|
|
||||||
flag = false;
|
flag = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
Map<String, String> targetOrgMap = new HashMap<>();
|
||||||
try {
|
try {
|
||||||
List<Map<String, Object>> poll = customerMapper.list("code,value","org_config",null);
|
List<Map<String, Object>> poll = customerMapper.list("code,value","org_config",null);
|
||||||
//遍历poll,找出code值为TARGET_ORG_URL,TARGET_ORG_USERNAME,TARGET_ORG_PASSWORD的value值
|
//遍历poll,找出code值为TARGET_ORG_URL,TARGET_ORG_USERNAME,TARGET_ORG_PASSWORD的value值
|
||||||
Map<String, String> map = new HashMap<>();
|
|
||||||
for (Map<String, Object> map1 : poll) {
|
for (Map<String, Object> map1 : poll) {
|
||||||
if ("TARGET_ORG_URL".equals(map1.get("code"))) {
|
if ("TARGET_ORG_URL".equals(map1.get("code"))) {
|
||||||
map.put("url", (String) map1.get("value"));
|
targetOrgMap.put("url", (String) map1.get("value"));
|
||||||
targetOrgUrl = map1.get("value").toString();
|
targetOrgUrl = map1.get("value").toString();
|
||||||
}
|
}
|
||||||
if ("TARGET_ORG_USERNAME".equals(map1.get("code"))) {
|
if ("TARGET_ORG_USERNAME".equals(map1.get("code"))) {
|
||||||
map.put("username", (String) map1.get("value"));
|
targetOrgMap.put("username", (String) map1.get("value"));
|
||||||
}
|
}
|
||||||
if ("TARGET_ORG_PASSWORD".equals(map1.get("code"))) {
|
if ("TARGET_ORG_PASSWORD".equals(map1.get("code"))) {
|
||||||
map.put("password", (String) map1.get("value"));
|
targetOrgMap.put("password", (String) map1.get("value"));
|
||||||
}
|
}
|
||||||
if ("FILE_UPLOAD_URL".equals(map1.get("code"))) {
|
if ("FILE_UPLOAD_URL".equals(map1.get("code"))) {
|
||||||
uploadUrl = (String) map1.get("value");
|
uploadUrl = (String) map1.get("value");
|
||||||
}
|
}
|
||||||
|
if ("TARGET_ORG_TOKEN".equals(map1.get("code"))) {
|
||||||
|
targetOrgMap.put("token", map1.get("value") == null ? "" :(String) map1.get("value"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
String username = map.get("username");
|
String username = targetOrgMap.get("username");
|
||||||
ConnectorConfig config = new ConnectorConfig();
|
ConnectorConfig config = new ConnectorConfig();
|
||||||
config.setUsername(username);
|
config.setUsername(username);
|
||||||
config.setPassword(map.get("password"));
|
if (StringUtils.isNotEmpty(targetOrgMap.get("token"))){
|
||||||
String url = map.get("url");
|
config.setPassword(targetOrgMap.get("password") + targetOrgMap.get("token"));
|
||||||
|
}else {
|
||||||
|
config.setPassword(targetOrgMap.get("password"));
|
||||||
|
}
|
||||||
|
log.info("username:{}", username);
|
||||||
|
log.info("password:{}", targetOrgMap.get("password") + targetOrgMap.get("token"));
|
||||||
|
String url = targetOrgMap.get("url");
|
||||||
config.setAuthEndpoint(url);
|
config.setAuthEndpoint(url);
|
||||||
config.setServiceEndpoint(url);
|
config.setServiceEndpoint(url);
|
||||||
config.setConnectionTimeout(60 * 60 * 1000);
|
config.setConnectionTimeout(60 * 60 * 1000);
|
||||||
@ -114,10 +137,14 @@ public class OrgConfigServiceImpl extends ServiceImpl<OrgConfigMapper, OrgConfig
|
|||||||
PartnerConnection connection = new PartnerConnection(config);
|
PartnerConnection connection = new PartnerConnection(config);
|
||||||
String orgId = connection.getUserInfo().getOrganizationId();
|
String orgId = connection.getUserInfo().getOrganizationId();
|
||||||
} catch (ConnectionException e) {
|
} catch (ConnectionException e) {
|
||||||
String message = "目标ORG连接配置错误!,\n地址:" + targetOrgUrl + "\n错误信息:\n" + e.getMessage() ;
|
log.info("exception message", e);
|
||||||
String format = String.format("ORG连接异常!, \ncause:\n%s", message);
|
String message = String.format("目标ORG连接配置错误!%n地址:%s%n用户名:%s%nToken:%s%n错误信息:%s",
|
||||||
|
targetOrgUrl,
|
||||||
|
targetOrgMap.get("username"),
|
||||||
|
targetOrgMap.get("token"),
|
||||||
|
e.getMessage());
|
||||||
|
String format = String.format("ORG连接异常!%n详细信息:%n%s", message);
|
||||||
EmailUtil.send("DataDump ERROR", format);
|
EmailUtil.send("DataDump ERROR", format);
|
||||||
log.error("exception message", e);
|
|
||||||
flag = false;
|
flag = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user