kris/data-dump
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

【feat】 优化创建表结构以及拉取存量数据任务

Browse Source
This commit is contained in:
Kris 2025-06-19 11:35:20 +08:00
parent face908fab
commit 996e977aa3
1 changed files with 20 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
src/main/java/com/celnet/datadump/service/impl/CommonServiceImpl.java
View File

@ -234,8 +234,8 @@ public class CommonServiceImpl implements CommonService {
for (DataObject dataObject : dataObjects) {
DataObject update = new DataObject();
TimeUnit.MILLISECONDS.sleep(1);
String api = dataObject.getName();
try {
String api = dataObject.getName();
log.info("dump apis: {}", api);
XxlJobLogger.log("dump apis: {}", api);
// 检测表是否存在 不存在创建
@ -310,14 +310,16 @@ public class CommonServiceImpl implements CommonService {
if (StringUtils.isNotBlank(dataObject.getBlobField())) {
fileService.dumpFile(dataObject.getName(), dataObject.getBlobField(), true);
}
update.setDataWork(0);
} catch (Throwable e) {
String message = e.getMessage();
String format = String.format("获取表数据 error, api name: %s, \nparam: %s, \ncause:\n%s", api, com.alibaba.fastjson2.JSON.toJSONString(param, DataDumpParam.getFilter()), message);
EmailUtil.send("DataDump ERROR", format);
//异常错误，移除所有线程
salesforceExecutor.remove(futures.toArray(new Future<?>[]{}));
throw new RuntimeException(e);
} finally {
if (dataObject != null) {
update.setDataLock(0);
dataObjectService.updateById(update);
}
update.setDataLock(0);
dataObjectService.updateById(update);
}
}
// 等待当前所有线程执行完成
@ -809,6 +811,8 @@ public class CommonServiceImpl implements CommonService {
List<Map<String, Object>> list = Lists.newArrayList();
DescribeSObjectResult dsr = connection.describeSObject(apiName);
String label = dsr.getLabel();
boolean isCustomObject = dsr.isCustom(); // 自定义对象才支持新增字段
boolean isUpdateable = dsr.isUpdateable(); // 对象本身是否可修改
List<DataField> fieldList = Lists.newArrayList();
List<String> fields = Lists.newArrayList();
String blobField = null;
@ -1006,6 +1010,9 @@ public class CommonServiceImpl implements CommonService {
update.setName(apiName);
update.setLastUpdateDate(endCreateDate);
update.setBlobField(blobField);
if(!isCustomObject && !isUpdateable){
update.setIsEditable(false);
}
dataObjectService.saveOrUpdate(update);
}
} finally {
@ -1023,7 +1030,13 @@ public class CommonServiceImpl implements CommonService {
}
log.info("打印所有待同步表：" +apis.toString());
for (String api : apis) {
checkApi(api, true);
try {
checkApi(api, true);
}catch (Exception e){
String message = e.getMessage();
String format = String.format("创建表结构 error, api name: %s, \nparam: %s, \ncause:\n%s", api, com.alibaba.fastjson2.JSON.toJSONString(param, DataDumpParam.getFilter()), message);
EmailUtil.send("DataDump ERROR", format);
}
}
return ReturnT.SUCCESS;
}