kris/data-dump
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

【feat】 优化创建表结构以及拉取存量数据任务

Browse Source
This commit is contained in:
Kris 2025-06-19 11:35:20 +08:00
parent face908fab
commit 996e977aa3
1 changed files with 20 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
src/main/java/com/celnet/datadump/service/impl/CommonServiceImpl.java
View File

@ -234,8 +234,8 @@ public class CommonServiceImpl implements CommonService {
for (DataObject dataObject : dataObjects) { for (DataObject dataObject : dataObjects) {
DataObject update = new DataObject(); DataObject update = new DataObject();
TimeUnit.MILLISECONDS.sleep(1); TimeUnit.MILLISECONDS.sleep(1);
try {
String api = dataObject.getName(); String api = dataObject.getName();
try {
log.info("dump apis: {}", api); log.info("dump apis: {}", api);
XxlJobLogger.log("dump apis: {}", api); XxlJobLogger.log("dump apis: {}", api);
// 检测表是否存在 不存在创建 // 检测表是否存在 不存在创建
@ -310,16 +310,18 @@ public class CommonServiceImpl implements CommonService {
if (StringUtils.isNotBlank(dataObject.getBlobField())) { if (StringUtils.isNotBlank(dataObject.getBlobField())) {
fileService.dumpFile(dataObject.getName(), dataObject.getBlobField(), true); fileService.dumpFile(dataObject.getName(), dataObject.getBlobField(), true);
} }
update.setDataWork(0);
} catch (Throwable e) { } catch (Throwable e) {
String message = e.getMessage();
String format = String.format("获取表数据 error, api name: %s, \nparam: %s, \ncause:\n%s", api, com.alibaba.fastjson2.JSON.toJSONString(param, DataDumpParam.getFilter()), message);
EmailUtil.send("DataDump ERROR", format);
//异常错误，移除所有线程
salesforceExecutor.remove(futures.toArray(new Future<?>[]{}));
throw new RuntimeException(e); throw new RuntimeException(e);
} finally { } finally {
if (dataObject != null) {
update.setDataLock(0); update.setDataLock(0);
dataObjectService.updateById(update); dataObjectService.updateById(update);
} }
} }
}
// 等待当前所有线程执行完成 // 等待当前所有线程执行完成
salesforceExecutor.waitForFutures(futures.toArray(new Future<?>[]{})); salesforceExecutor.waitForFutures(futures.toArray(new Future<?>[]{}));
futures.clear(); futures.clear();
@ -809,6 +811,8 @@ public class CommonServiceImpl implements CommonService {
List<Map<String, Object>> list = Lists.newArrayList(); List<Map<String, Object>> list = Lists.newArrayList();
DescribeSObjectResult dsr = connection.describeSObject(apiName); DescribeSObjectResult dsr = connection.describeSObject(apiName);
String label = dsr.getLabel(); String label = dsr.getLabel();
boolean isCustomObject = dsr.isCustom(); // 自定义对象才支持新增字段
boolean isUpdateable = dsr.isUpdateable(); // 对象本身是否可修改
List<DataField> fieldList = Lists.newArrayList(); List<DataField> fieldList = Lists.newArrayList();
List<String> fields = Lists.newArrayList(); List<String> fields = Lists.newArrayList();
String blobField = null; String blobField = null;
@ -1006,6 +1010,9 @@ public class CommonServiceImpl implements CommonService {
update.setName(apiName); update.setName(apiName);
update.setLastUpdateDate(endCreateDate); update.setLastUpdateDate(endCreateDate);
update.setBlobField(blobField); update.setBlobField(blobField);
if(!isCustomObject && !isUpdateable){
update.setIsEditable(false);
}
dataObjectService.saveOrUpdate(update); dataObjectService.saveOrUpdate(update);
} }
} finally { } finally {
@ -1023,7 +1030,13 @@ public class CommonServiceImpl implements CommonService {
} }
log.info("打印所有待同步表：" +apis.toString()); log.info("打印所有待同步表：" +apis.toString());
for (String api : apis) { for (String api : apis) {
try {
checkApi(api, true); checkApi(api, true);
}catch (Exception e){
String message = e.getMessage();
String format = String.format("创建表结构 error, api name: %s, \nparam: %s, \ncause:\n%s", api, com.alibaba.fastjson2.JSON.toJSONString(param, DataDumpParam.getFilter()), message);
EmailUtil.send("DataDump ERROR", format);
}
} }
return ReturnT.SUCCESS; return ReturnT.SUCCESS;
} }