[[feat] 20250910

This commit is contained in:
Kris 2025-09-10 22:10:36 +08:00
parent dd0a291e87
commit bc14185993
9 changed files with 279 additions and 30 deletions

View File

@ -55,8 +55,11 @@ public interface CustomMapper {
* @param tableName tableName
* @param maps maps
*/
public void createTable(@Param("tableName") String tableName, @Param("tableComment") String tableComment, @Param("maps") List<Map<String, Object>> maps, @Param("index") List<Map<String, Object>> index);
void createTable(@Param("tableName") String tableName,
@Param("tableComment") String tableComment,
@Param("maps") List<Map<String, Object>> maps,
@Param("index") List<Map<String, Object>> index,
@Param("hasId") boolean hasId);
/**
* 创建RANGE分区表
*

View File

@ -2,7 +2,10 @@ package com.celnet.datadump.service.impl;
import com.alibaba.fastjson2.JSON;
import com.alibaba.fastjson2.JSONArray;
import com.alibaba.fastjson2.JSONObject;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper;
import com.celnet.datadump.config.SalesforceConnect;
import com.celnet.datadump.config.SalesforceExecutor;
import com.celnet.datadump.entity.*;
@ -117,7 +120,7 @@ public class CommonBatchServiceImpl implements CommonBatchService {
if (!dataFieldService.hasCreatedDate(api)) {
DataDumpSpecialParam dataDumpSpecialParam = new DataDumpSpecialParam();
dataDumpSpecialParam.setApi(api);
Future<?> future = dataDumpSpecialService.getData(dataDumpSpecialParam, salesforceConnect.createConnect());
Future<?> future = getData(dataDumpSpecialParam, salesforceConnect.createBulkConnect());
// 等待当前所有线程执行完成
salesforceExecutor.waitForFutures(future);
continue;
@ -222,6 +225,119 @@ public class CommonBatchServiceImpl implements CommonBatchService {
return bulkConnect;
}
public Future<?> getData(DataDumpSpecialParam param, BulkConnection bulkConnect) {
String api = param.getApi();
return salesforceExecutor.execute(() -> {
try {
// 检测表 不存在就生成 但不生成批次
commonService.checkApi(api, true);
int count = 0;
boolean hasMore = true;
QueryWrapper<DataField> wrapper = new QueryWrapper<>();
wrapper.eq("api", api);
List<DataField> list = dataFieldService.list(wrapper);
Map<String, Object> map = Maps.newHashMap();
SalesforceParam salesforceParam = new SalesforceParam();
salesforceParam.setApi(api);
salesforceParam.setIdField(param.getField());
String maxId = null;
List<String> fields = Lists.newArrayList();
for (DataField field : list) {
if (!"base64".equalsIgnoreCase(field.getSfType()) && field.getSfType() != null){
fields.add(field.getField());
}
}
salesforceParam.setSelect(StringUtils.join(fields, ","));
while (hasMore) {
int sfData = 0;
salesforceParam.setMaxId(maxId);
salesforceParam.setLimit(10000);
map.put("param", salesforceParam);
JobInfo job = null;
try {
String sql = SqlUtil.showSql("com.celnet.datadump.mapper.SalesforceMapper.listOrderByIdNew", map);
log.info("query sql: {}", sql);
XxlJobLogger.log("query sql: {}", sql);
job = BulkUtil.createJob(bulkConnect, api, OperationEnum.queryAll);
log.info("创建Bulk作业成功, 作业ID: {}", job.getId());
BatchInfo batchInfo = bulkConnect.createBatchFromStream(job, new ByteArrayInputStream(sql.getBytes(StandardCharsets.UTF_8)));
log.info("创建批次成功, 批次ID: {}", batchInfo.getId());
int completionOne = BulkUtil.awaitCompletionOne(bulkConnect, job, batchInfo);
log.info("批次处理完成, 处理记录数: {}", completionOne);
if (completionOne == 0){
log.info("无更多数据, 结束处理");
BulkUtil.closeJob(bulkConnect, job.getId());
break;
}
List<Map<String, Object>> batchRecords = new ArrayList<>();
QueryResultList queryResultList = bulkConnect.getQueryResultList(job.getId(), batchInfo.getId());
log.info("获取查询结果列表, 结果数量: {}", queryResultList.getResult().length);
for (final String resultId : queryResultList.getResult()) {
InputStream resultStream = bulkConnect.getQueryResultStream(job.getId(), batchInfo.getId(), resultId);
CSVReader rdr = new CSVReader(resultStream);
List<String> headers = rdr.nextRecord();
log.debug("处理结果ID: {}, 表头数量: {}", resultId, headers.size());
// 批量处理记录
List<String> record;
while ((record = rdr.nextRecord()) != null) {
Map<String, Object> recordMap = new HashMap<>();
for (int i = 0; i < headers.size() && i < record.size(); i++) {
recordMap.put(headers.get(i), record.get(i));
}
batchRecords.add(recordMap);
}
}
if (!batchRecords.isEmpty()){
log.info("开始保存或更新数据, 记录数: {}", batchRecords.size());
sfData = saveOrUpdate(api, batchRecords, true);
count += sfData;
maxId = batchRecords.get(batchRecords.size() - 1).get("Id").toString();
log.info("处理结果: api:{}, sf数量:{}, 当前批次最大Id:{}", api, sfData, maxId);
} else {
log.info("批次无数据记录");
}
BulkUtil.closeJob(bulkConnect, job.getId());
log.info("关闭作业成功, 作业ID: {}", job.getId());
}catch (InterruptedException interruptedException){
return ;
} catch (Exception e) {
log.error("处理查询结果时异常: ", e);
throw new AsyncApiException("处理查询结果时异常: " + e.getMessage(), AsyncExceptionCode.Unknown);
}finally {
if (sfData != 10000){
hasMore = false;
log.info("当前批次数据不足10000条, 结束循环");
}
}
}
UpdateWrapper<DataBatch> updateWrapper = new UpdateWrapper<>();
updateWrapper.eq("name", param.getApi());
updateWrapper.set("first_db_num", count);
updateWrapper.set("first_sf_num", count);
dataBatchService.update(updateWrapper);
} catch (Throwable throwable) {
String format = String.format("数据特殊表迁移 error, api name: %s, \nparam: %s, \ncause:\n%s", api, JSON.toJSONString(param), throwable);
EmailUtil.send("DataDump ERROR", format);
log.error("dataDumpSpecial error ", throwable);
throw new RuntimeException(throwable);
}
}, 0, 0);
}
private int getAllBulkV1SfData(SalesforceParam param, BulkConnection bulkConnect) throws Throwable {

View File

@ -85,6 +85,8 @@ public class CommonServiceImpl implements CommonService {
private LinkConfigService linkConfigService;
@Autowired
private DataLogService dataLogService;
@Autowired
private CommonService commonService;
@Override
public ReturnT<String> increment(SalesforceParam param) throws Exception {
@ -1019,7 +1021,7 @@ public class CommonServiceImpl implements CommonService {
}
}
// picklist保存到picklist表
if ("picklist".equalsIgnoreCase(sfType)) {
if ("picklist".equalsIgnoreCase(sfType) || "multipicklist".equalsIgnoreCase(sfType)) {
join = "data_picklist";
PicklistEntry[] picklistValues = field.getPicklistValues();
if (ArrayUtils.isNotEmpty(picklistValues)) {
@ -1122,7 +1124,9 @@ public class CommonServiceImpl implements CommonService {
map4.put("name", "error_message");
list.add(map4);
customMapper.createTable(apiName, label, list, index);
boolean hasId = list.stream().anyMatch(t -> "Id".equals(t.get("name").toString()));
customMapper.createTable(apiName, label, list, index,hasId);
// 生成字段映射
QueryWrapper<DataField> delfieldQw = new QueryWrapper<>();
delfieldQw.eq("api", apiName);
@ -1137,7 +1141,7 @@ public class CommonServiceImpl implements CommonService {
Date startDate = DataUtil.DEFAULT_BEGIN_DATE;
// 取当天零点 作为最大时间
Date endCreateDate = DateUtils.parseDate(DateFormatUtils.format(now, "yyyy-MM-dd"), "yyyy-MM-dd");
Date lastDay = null;
QueryWrapper<DataBatch> delQw = new QueryWrapper<>();
delQw.eq("name", apiName);
dataBatchService.remove(delQw);
@ -1153,14 +1157,7 @@ public class CommonServiceImpl implements CommonService {
if (hasCreatedDate) {
List<Map<String, Object>> bachList = customMapper.list("code,value","system_config","code ='"+BATCH_TYPE+"'");
String batchType = bachList.get(0).get("value").toString();
do {
lastDay = getLastDay(batchType, endCreateDate, startDate);
DataBatch dataBatch = one.clone();
dataBatch.setSyncStartDate(startDate);
dataBatch.setSyncEndDate(lastDay);
dataBatchService.save(dataBatch);
startDate = lastDay;
} while (lastDay.compareTo(endCreateDate) < 0);
saveBatch(batchType,startDate,endCreateDate,apiName,connection,one);
} else {
// 没创建时间 开始结束时间为空
dataBatchService.save(one);
@ -1182,6 +1179,127 @@ public class CommonServiceImpl implements CommonService {
}
}
/**
* 保存批次信息根据数据量自动调整批次粒度
* @param batchType 批次类型
* @param startDate 开始日期
* @param endDate 结束日期
* @param apiName API名称
* @param connection Salesforce连接
* @param one 批次对象模板
*/
public void saveBatch(String batchType, Date startDate, Date endDate, String apiName, PartnerConnection connection, DataBatch one) {
// 添加递归深度限制避免栈溢出
saveBatchRecursive(batchType, startDate, endDate, apiName, connection, one, 0);
log.info("API {} 批次创建完成", apiName);
}
/**
* 递归保存批次信息
* @param batchType 批次类型
* @param startDate 开始日期
* @param endDate 结束日期
* @param apiName API名称
* @param connection Salesforce连接
* @param one 批次对象模板
* @param depth 递归深度
*/
private void saveBatchRecursive(String batchType, Date startDate, Date endDate, String apiName, PartnerConnection connection, DataBatch one, int depth) {
// 限制递归深度避免栈溢出
if (depth > 10) {
log.warn("递归深度超过限制,直接保存批次: {}, startDate: {}, endDate: {}", apiName, startDate, endDate);
DataBatch dataBatch = one.clone();
dataBatch.setSyncStartDate(startDate);
dataBatch.setSyncEndDate(endDate);
dataBatchService.save(dataBatch);
return;
}
Date currentStartDate = startDate;
Date lastDay;
do {
// 根据批次类型获取结束日期
lastDay = getLastDay(batchType, endDate, currentStartDate);
log.debug("当前批次起始日期: {}, 计算得到的结束日期: {}", currentStartDate, lastDay);
// 添加边界检查避免无限循环
if (lastDay.compareTo(currentStartDate) <= 0) {
log.warn("计算得到的结束日期({})早于或等于起始日期({}),跳过该批次", lastDay, currentStartDate);
break;
}
// 查询salesforce当前批次数据量
SalesforceParam salesforceParam = new SalesforceParam();
salesforceParam.setApi(apiName);
salesforceParam.setBeginCreateDate(currentStartDate);
salesforceParam.setEndCreateDate(lastDay);
Integer totalNum = 0;
int retryCount = 0;
int maxRetries = 3;
// 增强异常处理添加重试机制
while (retryCount < maxRetries) {
try {
totalNum = commonService.countSfNum(connection, salesforceParam);
log.debug("API {} 在 {} 到 {} 期间的数据量为: {}", apiName, currentStartDate, lastDay, totalNum);
break; // 成功获取数据量跳出重试循环
} catch (Exception e) {
retryCount++;
log.error("api {} count error, retry {}/{}", apiName, retryCount, maxRetries, e);
if (retryCount >= maxRetries) {
log.error("api {} count failed after {} retries", apiName, maxRetries, e);
// 如果重试失败使用默认值继续处理
totalNum = 0;
} else {
try {
Thread.sleep(1000 * retryCount); // 指数退避
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
throw new RuntimeException("Thread interrupted during retry delay", ie);
}
}
}
}
// 如果当前批次数据量超过50万则细化批次粒度并递归处理
if (totalNum > 500000) {
log.info("API {} 当前批次数据量 {} 超过50万需要细化批次粒度", apiName, totalNum);
if (BATCH_TYPE_YEAR.equals(batchType)) {
// 年批次超过50万细化为月批次
log.info("将年批次细化为月批次");
saveBatchRecursive(BATCH_TYPE_MONTH, currentStartDate, lastDay, apiName, connection, one, depth + 1);
} else if (BATCH_TYPE_MONTH.equals(batchType)) {
// 月批次超过50万细化为周批次
log.info("将月批次细化为周批次");
saveBatchRecursive(BATCH_TYPE_WEEK, currentStartDate, lastDay, apiName, connection, one, depth + 1);
} else {
// 已经是周粒度 still > 500000, 直接保存当前批次
log.info("已经是周粒度但数据量仍超过50万直接保存批次");
DataBatch dataBatch = one.clone();
dataBatch.setSyncStartDate(currentStartDate);
dataBatch.setSyncEndDate(lastDay);
dataBatch.setSfNum(totalNum);
dataBatchService.save(dataBatch);
}
} else {
// 数据量未超过50万直接保存当前批次
log.info("API {} 当前批次数据量 {} 未超过50万直接保存批次", apiName, totalNum);
DataBatch dataBatch = one.clone();
dataBatch.setSyncStartDate(currentStartDate);
dataBatch.setSyncEndDate(lastDay);
dataBatch.setSfNum(totalNum);
dataBatchService.save(dataBatch);
}
currentStartDate = lastDay;
// 添加安全检查避免无限循环
if (currentStartDate.compareTo(endDate) >= 0) {
break;
}
} while (lastDay.compareTo(endDate) < 0);
}
/**
* 创建分区表结构
* @param apiName

View File

@ -400,7 +400,7 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
continue;
}
}
if ("picklist".equals(dataField.getSfType())){
if ("picklist".equals(dataField.getSfType()) || "multipicklist".equals(dataField.getSfType())){
List<Map<String, Object>> pickList = customMapper.list("value", "data_picklist", "api = '"+api+"' and field = '"+dataField.getField()+"' limit 1");
account.put(dataField.getField(), pickList.get(0).get("value"));
continue;

View File

@ -739,25 +739,26 @@ public class DataImportNewServiceImpl implements DataImportNewService {
if (1 == param.getType()) {
if (api.endsWith("Share")){
sql = "where RowCause = 'Manual' and is_update = 0 and new_id is not null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "'";
sql2 = "RowCause = 'Manual' and is_update = 0 and new_id is not null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "' order by Id asc limit ";
sql2 = "RowCause = 'Manual' and is_update = 0 and new_id is not null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "' order by Id asc limit 10000 ";
}else {
sql = "where new_id is not null and is_update = 0 and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "'";
sql2 = "new_id is not null and is_update = 0 and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "' order by Id asc limit ";
sql2 = "new_id is not null and is_update = 0 and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "' order by Id asc limit 10000 ";
}
}else {
String updateDateField = dataFieldService.returnUpdateDateField(api);
if (api.endsWith("Share")){
sql = "where RowCause = 'Manual' and is_update = 0 and new_id is not null and "+updateDateField+" >= '" + beginDateStr + "' ";
sql2 = "RowCause = 'Manual' and is_update = 0 and new_id is not null and "+updateDateField+" >= '" + beginDateStr + "' order by Id asc limit ";
sql2 = "RowCause = 'Manual' and is_update = 0 and new_id is not null and "+updateDateField+" >= '" + beginDateStr + "' order by Id asc limit 10000 ";
}else {
sql = "where new_id is not null and and is_update = 0 "+updateDateField+" >= '" + beginDateStr + "' ";
sql2 = "new_id is not null and is_update = 0 and "+updateDateField+" >= '" + beginDateStr + "' order by Id asc limit ";
sql = "where new_id is not null and is_update = 0 and "+updateDateField+" >= '" + beginDateStr + "' ";
sql2 = "new_id is not null and is_update = 0 and "+updateDateField+" >= '" + beginDateStr + "' order by Id asc limit 10000 ";
}
}
//表内数据总量
Integer count = customMapper.countBySQL(api, sql);
if(count == 0){
log.info("没有需要更新的数据API: {},开始时间: {},结束时间: {}", api, beginDateStr, endDateStr);
return;
}
@ -784,19 +785,20 @@ public class DataImportNewServiceImpl implements DataImportNewService {
for (int z = 0; z < num; z++){
List<Map<String, Object>> mapsList = customMapper.list("*", api, sql2+ z * 10000 + ",10000");
List<Map<String, Object>> mapsList = customMapper.list("*", api, sql2);
int size = mapsList.size();
log.info("Update数据 count:{};当前批次:{}-开始时间:{}-结束时间:{}-api:{}", count,z, beginDateStr, endDateStr, api);
log.info("批次:{};当前批次:{};当前批次数据量:{}-开始时间:{}-结束时间:{}-api:{}", num,z,size, beginDateStr, endDateStr, api);
//批量插入200一次
int page = size%200 == 0 ? count/200 : (count/200) + 1;
int page = size%200 == 0 ? size/200 : (size/200) + 1;
for (int i = 0; i < page; i++) {
DataLog dataLog = new DataLog(api, "开始时间:"+beginDateStr+";结束时间:"+endDateStr, new Date(), null, "数据更新,查询并组装第" + z*50 + i + "批数据", "QueryDB");
DataLog dataLog = new DataLog(api, "开始时间:"+beginDateStr+";结束时间:"+endDateStr, new Date(), null, "数据更新,查询并组装第" + (z*50 + i) + "批数据", "QueryDB");
int startIndex = 200 * i;
int endIndex = Math.min(startIndex + 200, size);
if (startIndex >= endIndex) {
log.info("当前批次:{} 结束!! -开始时间:{}-结束时间:{}-api:{}", z, beginDateStr, endDateStr, api);
break;
}
List<Map<String, Object>> mapList = mapsList.subList(startIndex, endIndex);
@ -880,7 +882,7 @@ public class DataImportNewServiceImpl implements DataImportNewService {
if (infoFlag != null && "1".equals(infoFlag.get("value"))){
printlnAccountsDetails(accounts,list);
}
DataLog dataLog1 = new DataLog(api, "开始时间:"+beginDateStr+";结束时间:"+endDateStr, new Date(), null, "数据更新更新SF第" + z*50 + i + "批数据", "UpdateSF");
DataLog dataLog1 = new DataLog(api, "开始时间:"+beginDateStr+";结束时间:"+endDateStr, new Date(), null, "数据更新更新SF第" + (z*50 + i) + "批数据", "UpdateSF");
SaveResult[] saveResults = partnerConnection.update(accounts);
for (SaveResult saveResult : saveResults) {
if (!saveResult.getSuccess()) {
@ -932,6 +934,8 @@ public class DataImportNewServiceImpl implements DataImportNewService {
.eq("sync_end_date", endDate)
.set("sf_update_num", targetCount);
dataBatchService.update(updateQw2);
log.info("数据更新完成API: {},总更新数: {},开始时间: {},结束时间: {}", api, targetCount, beginDateStr, endDateStr);
}
/**

View File

@ -316,7 +316,7 @@ public class DataImportServiceImpl implements DataImportService {
if (startIndex >= endIndex) {
break;
}
DataLog dataLog = new DataLog(api, "开始时间:"+beginDateStr+";结束时间:"+endDateStr, new Date(), null, "获取new_id查询并组装第" + z*50 + i + "批数据", "QueryDB");
DataLog dataLog = new DataLog(api, "开始时间:"+beginDateStr+";结束时间:"+endDateStr, new Date(), null, "获取new_id查询并组装第" + (z*50 + i) + "批数据", "QueryDB");
List<Map<String, Object>> mapList = mapsList.subList(startIndex, endIndex);
int sized = mapList.size();
@ -392,7 +392,7 @@ public class DataImportServiceImpl implements DataImportService {
continue;
}
}
if ("picklist".equals(dataField.getSfType())){
if ("picklist".equals(dataField.getSfType()) || "multipicklist".equals(dataField.getSfType())){
List<Map<String, Object>> pickList = customMapper.list("value", "data_picklist", "api = '"+api+"' and field = '"+dataField.getField()+"' limit 1");
account.setField(dataField.getField(), pickList.get(0).get("value"));
continue;
@ -420,7 +420,7 @@ public class DataImportServiceImpl implements DataImportService {
dataLog.setEndTime(new Date());
dataLogService.save(dataLog);
DataLog dataLog1 = new DataLog(api, "开始时间:"+beginDateStr+";结束时间:"+endDateStr, new Date(), null, "获取new_id插入SF第" + z*50 + i + "批数据", "InserSF");
DataLog dataLog1 = new DataLog(api, "开始时间:"+beginDateStr+";结束时间:"+endDateStr, new Date(), null, "获取new_id插入SF第" + (z*50 + i) + "批数据", "InserSF");
if (infoFlag != null && "1".equals(infoFlag.get("value"))){
printAccountsDetails(accounts, list);
}
@ -429,7 +429,7 @@ public class DataImportServiceImpl implements DataImportService {
dataLog1.setEndTime(new Date());
dataLogService.save(dataLog1);
DataLog dataLog2 = new DataLog(api, "开始时间:"+beginDateStr+";结束时间:"+endDateStr, new Date(), null, "获取new_id更新DB第" + z*50 + i + "批数据", "UpdateDB");
DataLog dataLog2 = new DataLog(api, "开始时间:"+beginDateStr+";结束时间:"+endDateStr, new Date(), null, "获取new_id更新DB第" + (z*50 + i) + "批数据", "UpdateDB");
int index = 0;
for (SaveResult saveResult : saveResults){

View File

@ -475,7 +475,7 @@ public class FileServiceImpl implements FileService {
boolean index = false;
String documentId = (String) documentMap.get("Id");
// 获取未存储的附件id
List<Map<String, Object>> list = customMapper.list("Id, url, PathOnClient, Title, ContentDocumentId, VersionNumber", api, "ContentDocumentId = '" + documentId + "' and new_id is null ORDER BY VersionNumber ASC");
List<Map<String, Object>> list = customMapper.list("Id, url, PathOnClient, Title, ContentDocumentId, VersionNumber,Description", api, "ContentDocumentId = '" + documentId + "' and new_id is null ORDER BY VersionNumber ASC");
if (CollectionUtils.isEmpty(list)) {
continue;
}
@ -497,6 +497,7 @@ public class FileServiceImpl implements FileService {
String fileName = (String) map.get("PathOnClient");
String title = (String) map.get("Title");
String oldDocumentId = (String) map.get("ContentDocumentId");
String description = (String) map.get("Description");
int versionNumber = Integer.parseInt(map.get("VersionNumber").toString());
log.info("文件名称:" + fileName);
@ -521,6 +522,7 @@ public class FileServiceImpl implements FileService {
if (newDocumentId != null) {
credentialsJsonParam.put("ContentDocumentId", newDocumentId);
}
credentialsJsonParam.put("description", description);
MultipartEntityBuilder builder = MultipartEntityBuilder.create();
builder.addTextBody("entity_content", credentialsJsonParam.toJSONString(), ContentType.APPLICATION_JSON);
builder.addBinaryBody("VersionData", new FileInputStream(file), ContentType.APPLICATION_OCTET_STREAM, fileName);

View File

@ -21,6 +21,9 @@
<foreach item="map" collection="maps">
`${map.name}` ${map.type} comment '${map.comment}',
</foreach>
<if test="!hasId">
`id` INT NOT NULL AUTO_INCREMENT,
</if>
<foreach item="map" collection="index">
<choose>
<when test="map.field == 'new_id'">

View File

@ -202,6 +202,9 @@
order by ${param.idField} asc
</if>
</if>
<if test="param != null and param.limit != null">
limit #{param.limit}
</if>
</select>
</mapper>