diff --git a/.run/data-dump（亚数) .run.xml b/.run/data-dump（亚数) .run.xml new file mode 100644 index 0000000..7eeb246 --- /dev/null +++ b/.run/data-dump（亚数) .run.xml @@ -0,0 +1,14 @@ + + + + + + + + + \ No newline at end of file diff --git a/src/main/java/com/celnet/datadump/service/impl/CommonServiceImpl.java b/src/main/java/com/celnet/datadump/service/impl/CommonServiceImpl.java index e37a3c0..d33f53e 100644 --- a/src/main/java/com/celnet/datadump/service/impl/CommonServiceImpl.java +++ b/src/main/java/com/celnet/datadump/service/impl/CommonServiceImpl.java @@ -238,6 +238,7 @@ public class CommonServiceImpl implements CommonService { } dumpDataNew(salesforceParam, dataReport,dataObject); dataObject.setLastUpdateDate(updateTime); + dataObject.setNeedUpdate( false); dataObjectService.updateById(dataObject); } catch (Throwable throwable) { @@ -867,7 +868,7 @@ public class CommonServiceImpl implements CommonService { DataObject one = dataObjectService.getById(api); QueryWrapper dbQw = new QueryWrapper<>(); dbQw.eq("api", api); - List list = dataFieldService.list(dbQw).stream().map(DataField::getField).collect(Collectors.toList()); + List listed = dataFieldService.list(dbQw); List existsIds = customMapper.getIds(api, ids); for (int i = 0; i < objects.size(); i++) { JSONObject jsonObject = objects.getJSONObject(i); @@ -875,10 +876,10 @@ public class CommonServiceImpl implements CommonService { // update String id = jsonObject.getString(Const.ID); List> maps = Lists.newArrayList(); - for (String key : list) { - if (fields.stream().anyMatch(key::equalsIgnoreCase)) { + for (DataField field : listed) { + if (StringUtils.isNotEmpty(field.getSfType())) { // ContentVersion表，插入更新时把title中的/替换为- - if ("ContentVersion".equals(one.getName()) && "Title".equals(key)){ + if ("ContentVersion".equals(one.getName()) && "Title".equals(field.getField())){ String title = StringUtils.replace(String.valueOf(jsonObject.get("Title")), "/", "-"); Map paramMap = Maps.newHashMap(); paramMap.put("key", "Title"); @@ -888,7 +889,7 @@ public class CommonServiceImpl implements CommonService { } // Document,Attachment，插入更新时把Name中的/替换为- - if (("Document".equals(one.getName()) || "Attachment".equals(one.getName())) && "Name".equals(key)){ + if (("Document".equals(one.getName()) || "Attachment".equals(one.getName())) && "Name".equals(field.getField())){ String name = StringUtils.replace(String.valueOf(jsonObject.get("Name")), "/", "-"); Map paramMap = Maps.newHashMap(); paramMap.put("key", "Name"); @@ -898,8 +899,8 @@ public class CommonServiceImpl implements CommonService { } Map paramMap = Maps.newHashMap(); - paramMap.put("key", key); - paramMap.put("value", jsonObject.get(key)); + paramMap.put("key", field.getField()); + paramMap.put("value", jsonObject.get(field.getField())); maps.add(paramMap); } } diff --git a/src/main/java/com/celnet/datadump/service/impl/DataDumpSpecialServiceImpl.java b/src/main/java/com/celnet/datadump/service/impl/DataDumpSpecialServiceImpl.java index a614d9d..2d0cb96 100644 --- a/src/main/java/com/celnet/datadump/service/impl/DataDumpSpecialServiceImpl.java +++ b/src/main/java/com/celnet/datadump/service/impl/DataDumpSpecialServiceImpl.java @@ -295,6 +295,7 @@ public class DataDumpSpecialServiceImpl implements DataDumpSpecialService { } updateDataBatch(connect, api); dataObject.setLastUpdateDate(new Date()); + dataObject.setNeedUpdate( false); dataObjectService.updateById(dataObject); } catch (Throwable throwable) { String format = String.format("数据特殊表迁移 error, api name: %s, \nparam: %s, \ncause:\n%s", api, JSON.toJSONString(param), throwable); diff --git a/src/main/java/com/celnet/datadump/service/impl/DataImportNewServiceImpl.java b/src/main/java/com/celnet/datadump/service/impl/DataImportNewServiceImpl.java index 4bd564f..5f7c9ba 100644 --- a/src/main/java/com/celnet/datadump/service/impl/DataImportNewServiceImpl.java +++ b/src/main/java/com/celnet/datadump/service/impl/DataImportNewServiceImpl.java @@ -2245,14 +2245,14 @@ public class DataImportNewServiceImpl implements DataImportNewService { public ReturnT checkDeletedData(SalesforceParam param) throws Exception { if (param.getType() == 1){ - return munalCheckDeletedData(param); + return manualCheckDeletedData(param); }else { return autoCheckDeletedData(param); } } - private ReturnT munalCheckDeletedData(SalesforceParam param) throws Exception { + private ReturnT manualCheckDeletedData(SalesforceParam param) throws Exception { String api = "DeleteEvent"; @@ -2954,7 +2954,6 @@ public class DataImportNewServiceImpl implements DataImportNewService { List> data = mapsList.subList(startIndex, endIndex); int sized = data.size(); - log.info("执行api:{}, 执行批次:{}, 执行数据量:{}，开始时间：{}，结束时间：{}", api, i+1, size,beginDateStr,endDateStr); SObject[] accounts = new SObject[sized]; String[] ids = new String[sized]; try { diff --git a/src/main/java/com/celnet/datadump/service/impl/DataImportServiceImpl.java b/src/main/java/com/celnet/datadump/service/impl/DataImportServiceImpl.java index 22f4882..0cee5a7 100644 --- a/src/main/java/com/celnet/datadump/service/impl/DataImportServiceImpl.java +++ b/src/main/java/com/celnet/datadump/service/impl/DataImportServiceImpl.java @@ -262,6 +262,9 @@ public class DataImportServiceImpl implements DataImportService { dbQw.eq("api", api); List list = dataFieldService.list(dbQw); TimeUnit.MILLISECONDS.sleep(1); + QueryWrapper dbQw2 = new QueryWrapper<>(); + dbQw2.eq("name", api); + DataObject dataObject = dataObjectService.getOne(dbQw2); String beginDateStr = null; String endDateStr = null; @@ -292,8 +295,6 @@ public class DataImportServiceImpl implements DataImportService { } SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); - DataObject update = new DataObject(); - update.setName(api); int num = count%10000 == 0 ? count/10000 : (count/10000) + 1; @@ -378,8 +379,8 @@ public class DataImportServiceImpl implements DataImportService { maxIndex.ne("name", api); Map map = dataObjectService.getMap(maxIndex); //如果必填lookup字段没有值，跳过 - update.setDataIndex(Integer.parseInt(map.get("data_index").toString()) + 1); - dataObjectService.updateById(update); + dataObject.setDataIndex(Integer.parseInt(map.get("data_index").toString()) + 1); + dataObjectService.updateById(dataObject); return; }else{ account.setField(dataField.getField(), referenceMap.get(0).get("new_id")); @@ -395,6 +396,11 @@ public class DataImportServiceImpl implements DataImportService { } } + if (dataObject.getIsEditable()){ + account.setField("old_owner_id__c", mapList.get(j - 1).get("OwnerId")); + account.setField("old_sfdc_id__c", mapList.get(j - 1).get("Id")); + } + ids[j-1] = mapList.get(j-1).get("Id").toString(); accounts[j-1] = account; if (i*200+j == size){ diff --git a/src/main/java/com/celnet/datadump/service/impl/DataVerifyServiceImpl.java b/src/main/java/com/celnet/datadump/service/impl/DataVerifyServiceImpl.java index 626ef1b..f3fc82b 100644 --- a/src/main/java/com/celnet/datadump/service/impl/DataVerifyServiceImpl.java +++ b/src/main/java/com/celnet/datadump/service/impl/DataVerifyServiceImpl.java @@ -157,19 +157,21 @@ public class DataVerifyServiceImpl implements DataVerifyService { for (DataObject dataObject : list) { String api = dataObject.getName(); String extraSql = ""; + String andexSql = ""; SalesforceParam salesforceParam = new SalesforceParam(); salesforceParam.setApi(api); Boolean hasDeleted = dataFieldService.hasDeleted(api); if (hasDeleted) { salesforceParam.setIsDeleted(false); extraSql = " WHERE IsDeleted = false "; + andexSql = " AND IsDeleted = false "; } String sql = "SELECT Count(Id) num FROM " + api + extraSql; - String oldIdSql = sql.contains("WHERE") ? sql + " AND old_sfdc_id__c != null":sql + " WHERE old_sfdc_id__c != null"; + String oldIdSql = sql.contains("WHERE") ? sql + " AND old_sfdc_id__c != null":sql + " WHERE old_sfdc_id__c != null" + andexSql; - String duplicateIdSql = "SELECT old_sfdc_id__c FROM " + api + " WHERE old_sfdc_id__c != NULL GROUP BY old_sfdc_id__c HAVING COUNT(old_sfdc_id__c) > 1"; + String duplicateIdSql = "SELECT old_sfdc_id__c FROM " + api + " WHERE old_sfdc_id__c != NULL "+ andexSql +" GROUP BY old_sfdc_id__c HAVING COUNT(old_sfdc_id__c) > 1" ; if (param.getBeginDate() != null){ String dateField = dataFieldService.returnDateField(api); @@ -297,7 +299,7 @@ public class DataVerifyServiceImpl implements DataVerifyService { } idStr += ")"; - String duplicateOldIdSql = "SELECT Id,old_sfdc_id__c FROM " + api + " WHERE old_sfdc_id__c IN " + idStr; + String duplicateOldIdSql = "SELECT Id,old_sfdc_id__c FROM " + api + " WHERE old_sfdc_id__c IN " + idStr + andexSql; QueryResult queryResult1 = targetConnect.queryAll(duplicateOldIdSql); if (queryResult1 != null && queryResult1.getRecords() != null && queryResult1.getRecords().length > 0) { for (SObject record : queryResult1.getRecords()) { @@ -368,6 +370,7 @@ public class DataVerifyServiceImpl implements DataVerifyService { + return null; }