kris/data-dump
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

【feat】 获取NewId增加oldId传值

Browse Source
This commit is contained in:
Kris 2025-08-27 18:42:56 +08:00
parent 912efda685
commit 186100bb96
6 changed files with 41 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
.run/data-dump（亚数) .run.xml Normal file
View File

@ -0,0 +1,14 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="data-dump（亚数) " type="docker-deploy" factoryName="dockerfile" server-name="亚数Docker">
<deployment type="dockerfile">
<settings>
<option name="imageTag" value="data-dump" />
<option name="buildCliOptions" value="--platform=linux/amd64" />
<option name="containerName" value="data-dump" />
<option name="commandLineOptions" value="-p 90:80 -v /usr/local/data-dump/file:/data-dump/file -v /usr/local/data-dump/logs:/data-dump/logs --restart=always --network=host" />
<option name="sourceFilePath" value="Dockerfile" />
</settings>
</deployment>
<method v="2" />
</configuration>
</component>

15
src/main/java/com/celnet/datadump/service/impl/CommonServiceImpl.java
View File

@ -238,6 +238,7 @@ public class CommonServiceImpl implements CommonService {
} }
dumpDataNew(salesforceParam, dataReport,dataObject); dumpDataNew(salesforceParam, dataReport,dataObject);
dataObject.setLastUpdateDate(updateTime); dataObject.setLastUpdateDate(updateTime);
dataObject.setNeedUpdate( false);
dataObjectService.updateById(dataObject); dataObjectService.updateById(dataObject);
} catch (Throwable throwable) { } catch (Throwable throwable) {
@ -867,7 +868,7 @@ public class CommonServiceImpl implements CommonService {
DataObject one = dataObjectService.getById(api); DataObject one = dataObjectService.getById(api);
QueryWrapper<DataField> dbQw = new QueryWrapper<>(); QueryWrapper<DataField> dbQw = new QueryWrapper<>();
dbQw.eq("api", api); dbQw.eq("api", api);
List<String> list = dataFieldService.list(dbQw).stream().map(DataField::getField).collect(Collectors.toList()); List<DataField> listed = dataFieldService.list(dbQw);
List<String> existsIds = customMapper.getIds(api, ids); List<String> existsIds = customMapper.getIds(api, ids);
for (int i = 0; i < objects.size(); i++) { for (int i = 0; i < objects.size(); i++) {
JSONObject jsonObject = objects.getJSONObject(i); JSONObject jsonObject = objects.getJSONObject(i);
@ -875,10 +876,10 @@ public class CommonServiceImpl implements CommonService {
// update // update
String id = jsonObject.getString(Const.ID); String id = jsonObject.getString(Const.ID);
List<Map<String, Object>> maps = Lists.newArrayList(); List<Map<String, Object>> maps = Lists.newArrayList();
for (String key : list) { for (DataField field : listed) {
if (fields.stream().anyMatch(key::equalsIgnoreCase)) { if (StringUtils.isNotEmpty(field.getSfType())) {
// ContentVersion表，插入更新时把title中的/替换为- // ContentVersion表，插入更新时把title中的/替换为-
if ("ContentVersion".equals(one.getName()) && "Title".equals(key)){ if ("ContentVersion".equals(one.getName()) && "Title".equals(field.getField())){
String title = StringUtils.replace(String.valueOf(jsonObject.get("Title")), "/", "-"); String title = StringUtils.replace(String.valueOf(jsonObject.get("Title")), "/", "-");
Map<String, Object> paramMap = Maps.newHashMap(); Map<String, Object> paramMap = Maps.newHashMap();
paramMap.put("key", "Title"); paramMap.put("key", "Title");
@ -888,7 +889,7 @@ public class CommonServiceImpl implements CommonService {
} }
// Document,Attachment，插入更新时把Name中的/替换为- // Document,Attachment，插入更新时把Name中的/替换为-
if (("Document".equals(one.getName()) || "Attachment".equals(one.getName())) && "Name".equals(key)){ if (("Document".equals(one.getName()) || "Attachment".equals(one.getName())) && "Name".equals(field.getField())){
String name = StringUtils.replace(String.valueOf(jsonObject.get("Name")), "/", "-"); String name = StringUtils.replace(String.valueOf(jsonObject.get("Name")), "/", "-");
Map<String, Object> paramMap = Maps.newHashMap(); Map<String, Object> paramMap = Maps.newHashMap();
paramMap.put("key", "Name"); paramMap.put("key", "Name");
@ -898,8 +899,8 @@ public class CommonServiceImpl implements CommonService {
} }
Map<String, Object> paramMap = Maps.newHashMap(); Map<String, Object> paramMap = Maps.newHashMap();
paramMap.put("key", key); paramMap.put("key", field.getField());
paramMap.put("value", jsonObject.get(key)); paramMap.put("value", jsonObject.get(field.getField()));
maps.add(paramMap); maps.add(paramMap);
} }
} }

1
src/main/java/com/celnet/datadump/service/impl/DataDumpSpecialServiceImpl.java
View File

@ -295,6 +295,7 @@ public class DataDumpSpecialServiceImpl implements DataDumpSpecialService {
} }
updateDataBatch(connect, api); updateDataBatch(connect, api);
dataObject.setLastUpdateDate(new Date()); dataObject.setLastUpdateDate(new Date());
dataObject.setNeedUpdate( false);
dataObjectService.updateById(dataObject); dataObjectService.updateById(dataObject);
} catch (Throwable throwable) { } catch (Throwable throwable) {
String format = String.format("数据特殊表迁移 error, api name: %s, \nparam: %s, \ncause:\n%s", api, JSON.toJSONString(param), throwable); String format = String.format("数据特殊表迁移 error, api name: %s, \nparam: %s, \ncause:\n%s", api, JSON.toJSONString(param), throwable);

5
src/main/java/com/celnet/datadump/service/impl/DataImportNewServiceImpl.java
View File

@ -2245,14 +2245,14 @@ public class DataImportNewServiceImpl implements DataImportNewService {
public ReturnT<String> checkDeletedData(SalesforceParam param) throws Exception { public ReturnT<String> checkDeletedData(SalesforceParam param) throws Exception {
if (param.getType() == 1){ if (param.getType() == 1){
return munalCheckDeletedData(param); return manualCheckDeletedData(param);
}else { }else {
return autoCheckDeletedData(param); return autoCheckDeletedData(param);
} }
} }
private ReturnT<String> munalCheckDeletedData(SalesforceParam param) throws Exception { private ReturnT<String> manualCheckDeletedData(SalesforceParam param) throws Exception {
String api = "DeleteEvent"; String api = "DeleteEvent";
@ -2954,7 +2954,6 @@ public class DataImportNewServiceImpl implements DataImportNewService {
List<Map<String, Object>> data = mapsList.subList(startIndex, endIndex); List<Map<String, Object>> data = mapsList.subList(startIndex, endIndex);
int sized = data.size(); int sized = data.size();
log.info("执行api:{}, 执行批次:{}, 执行数据量:{}，开始时间：{}，结束时间：{}", api, i+1, size,beginDateStr,endDateStr);
SObject[] accounts = new SObject[sized]; SObject[] accounts = new SObject[sized];
String[] ids = new String[sized]; String[] ids = new String[sized];
try { try {

14
src/main/java/com/celnet/datadump/service/impl/DataImportServiceImpl.java
View File

@ -262,6 +262,9 @@ public class DataImportServiceImpl implements DataImportService {
dbQw.eq("api", api); dbQw.eq("api", api);
List<DataField> list = dataFieldService.list(dbQw); List<DataField> list = dataFieldService.list(dbQw);
TimeUnit.MILLISECONDS.sleep(1); TimeUnit.MILLISECONDS.sleep(1);
QueryWrapper<DataObject> dbQw2 = new QueryWrapper<>();
dbQw2.eq("name", api);
DataObject dataObject = dataObjectService.getOne(dbQw2);
String beginDateStr = null; String beginDateStr = null;
String endDateStr = null; String endDateStr = null;
@ -292,8 +295,6 @@ public class DataImportServiceImpl implements DataImportService {
} }
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
DataObject update = new DataObject();
update.setName(api);
int num = count%10000 == 0 ? count/10000 : (count/10000) + 1; int num = count%10000 == 0 ? count/10000 : (count/10000) + 1;
@ -378,8 +379,8 @@ public class DataImportServiceImpl implements DataImportService {
maxIndex.ne("name", api); maxIndex.ne("name", api);
Map<String, Object> map = dataObjectService.getMap(maxIndex); Map<String, Object> map = dataObjectService.getMap(maxIndex);
//如果必填lookup字段没有值，跳过 //如果必填lookup字段没有值，跳过
update.setDataIndex(Integer.parseInt(map.get("data_index").toString()) + 1); dataObject.setDataIndex(Integer.parseInt(map.get("data_index").toString()) + 1);
dataObjectService.updateById(update); dataObjectService.updateById(dataObject);
return; return;
}else{ }else{
account.setField(dataField.getField(), referenceMap.get(0).get("new_id")); account.setField(dataField.getField(), referenceMap.get(0).get("new_id"));
@ -395,6 +396,11 @@ public class DataImportServiceImpl implements DataImportService {
} }
} }
if (dataObject.getIsEditable()){
account.setField("old_owner_id__c", mapList.get(j - 1).get("OwnerId"));
account.setField("old_sfdc_id__c", mapList.get(j - 1).get("Id"));
}
ids[j-1] = mapList.get(j-1).get("Id").toString(); ids[j-1] = mapList.get(j-1).get("Id").toString();
accounts[j-1] = account; accounts[j-1] = account;
if (i*200+j == size){ if (i*200+j == size){

9
src/main/java/com/celnet/datadump/service/impl/DataVerifyServiceImpl.java
View File

@ -157,19 +157,21 @@ public class DataVerifyServiceImpl implements DataVerifyService {
for (DataObject dataObject : list) { for (DataObject dataObject : list) {
String api = dataObject.getName(); String api = dataObject.getName();
String extraSql = ""; String extraSql = "";
String andexSql = "";
SalesforceParam salesforceParam = new SalesforceParam(); SalesforceParam salesforceParam = new SalesforceParam();
salesforceParam.setApi(api); salesforceParam.setApi(api);
Boolean hasDeleted = dataFieldService.hasDeleted(api); Boolean hasDeleted = dataFieldService.hasDeleted(api);
if (hasDeleted) { if (hasDeleted) {
salesforceParam.setIsDeleted(false); salesforceParam.setIsDeleted(false);
extraSql = " WHERE IsDeleted = false "; extraSql = " WHERE IsDeleted = false ";
andexSql = " AND IsDeleted = false ";
} }
String sql = "SELECT Count(Id) num FROM " + api + extraSql; String sql = "SELECT Count(Id) num FROM " + api + extraSql;
String oldIdSql = sql.contains("WHERE") ? sql + " AND old_sfdc_id__c != null":sql + " WHERE old_sfdc_id__c != null"; String oldIdSql = sql.contains("WHERE") ? sql + " AND old_sfdc_id__c != null":sql + " WHERE old_sfdc_id__c != null" + andexSql;
String duplicateIdSql = "SELECT old_sfdc_id__c FROM " + api + " WHERE old_sfdc_id__c != NULL GROUP BY old_sfdc_id__c HAVING COUNT(old_sfdc_id__c) > 1"; String duplicateIdSql = "SELECT old_sfdc_id__c FROM " + api + " WHERE old_sfdc_id__c != NULL "+ andexSql +" GROUP BY old_sfdc_id__c HAVING COUNT(old_sfdc_id__c) > 1" ;
if (param.getBeginDate() != null){ if (param.getBeginDate() != null){
String dateField = dataFieldService.returnDateField(api); String dateField = dataFieldService.returnDateField(api);
@ -297,7 +299,7 @@ public class DataVerifyServiceImpl implements DataVerifyService {
} }
idStr += ")"; idStr += ")";
String duplicateOldIdSql = "SELECT Id,old_sfdc_id__c FROM " + api + " WHERE old_sfdc_id__c IN " + idStr; String duplicateOldIdSql = "SELECT Id,old_sfdc_id__c FROM " + api + " WHERE old_sfdc_id__c IN " + idStr + andexSql;
QueryResult queryResult1 = targetConnect.queryAll(duplicateOldIdSql); QueryResult queryResult1 = targetConnect.queryAll(duplicateOldIdSql);
if (queryResult1 != null && queryResult1.getRecords() != null && queryResult1.getRecords().length > 0) { if (queryResult1 != null && queryResult1.getRecords() != null && queryResult1.getRecords().length > 0) {
for (SObject record : queryResult1.getRecords()) { for (SObject record : queryResult1.getRecords()) {
@ -368,6 +370,7 @@ public class DataVerifyServiceImpl implements DataVerifyService {
return null; return null;
} }