【feat】优化更新数据时的逻辑

(cherry picked from commit da2ae51a90)

src/main/java/com/celnet/datadump/service/impl/DataImportBatchServiceImpl.java

@@ -247,7 +247,7 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
                                 maxIndex.ne("name", api);
                                 Map<String, Object> map = dataObjectService.getMap(maxIndex);
                                 //如果必填lookup字段没有值，跳过
-                                update.setDataIndex(Integer.parseInt(map.get("data_index").toString()+1));
+                                update.setDataIndex(Integer.parseInt(map.get("data_index").toString()) + 1);
                                 dataObjectService.updateById(update);
                                 String message = "api:" + api + "的引用对象:" + reference + "不存在数据！";
                                 String format = String.format("数据导入 error, api name: %s, \nparam: %s, \ncause:\n%s",  api, com.alibaba.fastjson2.JSON.toJSONString(param, DataDumpParam.getFilter()), message);
@@ -298,13 +298,12 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
 
                 List<BatchInfo> batchInfos = BulkUtil.createBatchesFromCSVFile(bulkConnection, salesforceInsertJob, fullPath);
 
-                BulkUtil.closeJob(bulkConnection, salesforceInsertJob.getId());
-
                 BulkUtil.awaitCompletion(bulkConnection, salesforceInsertJob, batchInfos);
 
                 sfNum = sfNum + checkInsertResults(bulkConnection, salesforceInsertJob, batchInfos, api, ids);
 
-                new File(fullPath).delete();
+                BulkUtil.closeJob(bulkConnection, salesforceInsertJob.getId());
+
             } catch (Exception e) {
                 log.error("manualCreatedNewId error api:{}", api, e);
                 throw e;
@@ -528,8 +527,8 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
                     } else if (!DataUtil.isUpdate(field) || (dataField.getIsCreateable() != null && !dataField.getIsCreateable())) {
                         continue;
                     } else if (StringUtils.isNotBlank(reference_to) && !"data_picklist".equals(reference_to)) {
-                        if (!"null".equals(map.get(field)) && null != map.get(field) && !"OwnerId".equals(field)
+                        if ( StringUtils.isNotEmpty(String.valueOf(map.get(field))) && !"OwnerId".equals(field)
-                                || !"Owner_Type".equals(field)) {
+                                && !"Owner_Type".equals(field)) {
                             //判断reference_to内是否包含User字符串
                             if (reference_to.contains("User")) {
                                 reference_to = "User";
@@ -538,22 +537,15 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
                             if (m != null && !m.isEmpty()) {
                                 account.put(field, m.get("new_id"));
                             }else {
-                                QueryWrapper<DataObject> maxIndex = new QueryWrapper<>();
+                                String message = "对象类型:" + api + "的数据:"+ map.get("Id") +"的引用对象:" + dataField.getReferenceTo() + "的数据:"+ map.get(field) +"不存在！";
-                                maxIndex.select("IFNULL(max(data_index),0) as data_index");
+                                EmailUtil.send("DataDump ERROR", message);
-                                maxIndex.ne("name", api);
-                                Map<String, Object> mapTo = dataObjectService.getMap(maxIndex);
-                                //如果必填lookup字段没有值，跳过
-                                update.setDataIndex(Integer.parseInt(mapTo.get("data_index").toString()+1));
-                                dataObjectService.updateById(update);
-                                String message = "对象类型:" + api + "的数据:"+ m.get("Id") +"的引用对象:" + dataField.getReferenceTo() + "的数据:"+ map.get(field) +"不存在！";                                String format = String.format("数据导入 error, api name: %s, \nparam: %s, \ncause:\n%s",  api, com.alibaba.fastjson2.JSON.toJSONString(param, DataDumpParam.getFilter()), message);
-                                EmailUtil.send("DataDump ERROR", format);
                                 log.info(message);
                                 return;
                             }
                         }
                     } else {
                         if (map.get(field) != null && StringUtils.isNotBlank(dataField.getSfType())) {
-                            account.put(field, DataUtil.localDataToSfData(dataField.getSfType(), String.valueOf(map.get(field))));
+                            account.put(field, DataUtil.localBulkDataToSfData(dataField.getSfType(), String.valueOf(map.get(field))));
                         }else {
                             account.put(field, map.get(field));
                         }
@@ -573,12 +565,12 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
 
                 List<BatchInfo> batchInfos = BulkUtil.createBatchesFromCSVFile(bulkConnection, salesforceInsertJob, fullPath);
 
-                BulkUtil.closeJob(bulkConnection, salesforceInsertJob.getId());
-
                 BulkUtil.awaitCompletion(bulkConnection, salesforceInsertJob, batchInfos);
 
                 sfNum = sfNum + checkUpdateResults(bulkConnection, salesforceInsertJob, batchInfos,api);
 
+                BulkUtil.closeJob(bulkConnection, salesforceInsertJob.getId());
+
                 new File(fullPath).delete();
             } catch (Throwable e) {
                 log.info(e.getMessage());

src/main/java/com/celnet/datadump/service/impl/DataImportNewServiceImpl.java

@@ -742,51 +742,38 @@ public class DataImportNewServiceImpl implements DataImportNewService {
                     String field = dataField.getField();
                     String reference_to = dataField.getReferenceTo();
 
+                    String value = String.valueOf(map.get(field));
                     //根据旧sfid查找引用对象新sfid
                     if (field.equals("Id")) {
                         account.setId(String.valueOf(map.get("new_id")));
                     } else if (!DataUtil.isUpdate(field) || (dataField.getIsCreateable() != null && !dataField.getIsCreateable())) {
                         continue;
                     } else if (StringUtils.isNotBlank(reference_to) && !"data_picklist".equals(reference_to)) {
-                        if (!"null".equals(map.get(field)) && null != map.get(field) && !"OwnerId".equals(field)
+
-                                || !"Owner_Type".equals(field)) {
+                        if (!"null".equals(value) && StringUtils.isNotEmpty(value) && (!"OwnerId".equals(field)
+                                && !"Owner_Type".equals(field))) {
                             //判断reference_to内是否包含User字符串
                             if (reference_to.contains("User")) {
                                 reference_to = "User";
                             }
-                            Map<String, Object> m = customMapper.getById("new_id", reference_to, String.valueOf(map.get(field)));
+                            Map<String, Object> m = customMapper.getById("new_id", reference_to, value);
                             if (m != null && !m.isEmpty()) {
                                 account.setField(field, m.get("new_id"));
                             }else {
-                                QueryWrapper<DataObject> maxIndex = new QueryWrapper<>();
+                                String message = "对象类型:" + api + "的数据:"+ map.get("Id") +"的引用对象:" + dataField.getReferenceTo() + "的数据:"+ map.get(field) +"不存在！";
-                                maxIndex.select("IFNULL(max(data_index),0) as data_index");
+                                EmailUtil.send("DataDump ERROR", message);
-                                maxIndex.ne("name", api);
-                                Map<String, Object> mapTo = dataObjectService.getMap(maxIndex);
-                                //如果必填lookup字段没有值，跳过
-                                update.setDataIndex(Integer.parseInt(mapTo.get("data_index").toString()) +1);
-                                dataObjectService.updateById(update);
-                                String message = "对象类型:" + api + "的数据:"+ m.get("Id") +"的引用对象:" + dataField.getReferenceTo() + "的数据:"+ map.get(field) +"不存在！";                                String format = String.format("数据导入 error, api name: %s, \nparam: %s, \ncause:\n%s",  api, com.alibaba.fastjson2.JSON.toJSONString(param, DataDumpParam.getFilter()), message);
-                                EmailUtil.send("DataDump ERROR", format);
                                 log.info(message);
                                 return;
                             }
                         }
                     } else {
                         if (map.get(field) != null && StringUtils.isNotBlank(dataField.getSfType())) {
-                            account.setField(field, DataUtil.localDataToSfData(dataField.getSfType(), String.valueOf(map.get(field))));
+                            account.setField(field, DataUtil.localDataToSfData(dataField.getSfType(), value));
                         }else {
-                            if (api.equals("Account")){
-                                if ("1".equals(map.get("IsPersonAccount")) && field.equals("Name")){
-                                    continue;
-                                }else if("0".equals(map.get("IsPersonAccount")) && field.equals("LastName")){
-                                    continue;
-                                }
-                            } else {
                             account.setField(field, map.get(field));
                         }
                     }
                 }
-                }
                 account.setField("old_owner_id__c", map.get("OwnerId"));
                 account.setField("old_sfdc_id__c", map.get("Id"));
                 accounts[j++] = account;

src/main/java/com/celnet/datadump/service/impl/DataImportServiceImpl.java

@@ -309,9 +309,8 @@ public class DataImportServiceImpl implements DataImportService {
                                 maxIndex.select("IFNULL(max(data_index),0) as data_index");
                                 maxIndex.ne("name", api);
                                 Map<String, Object> map = dataObjectService.getMap(maxIndex);
-
                                 //如果必填lookup字段没有值，跳过
-                                update.setDataIndex(Integer.parseInt(map.get("data_index").toString()+1));
+                                update.setDataIndex(Integer.parseInt(map.get("data_index").toString()) + 1);
                                 dataObjectService.updateById(update);
                                 return;
                             }else{

src/main/java/com/celnet/datadump/service/impl/FileManagerServiceImpl.java

@@ -169,11 +169,12 @@ public class FileManagerServiceImpl implements FileManagerService {
                     for (Map<String, Object> map1 : poll) {
                         if ("FILE_DOWNLOAD_URL".equals(map1.get("code"))) {
                             downloadUrl = (String) map1.get("value");
-                        }else {
+                        }
+                    }
+                    if (StringUtils.isNotBlank(downloadUrl)) {
                         EmailUtil.send("DumpFile ERROR", "文件下载失败！下载地址未配置");
                         return;
                     }
-                    }
                     String url = downloadUrl + String.format(Const.SF_RICH_TEXT_FILE_URL, objectApi, recordId, fieldApi, fieldId);
 
                     try {
@@ -307,11 +308,12 @@ public class FileManagerServiceImpl implements FileManagerService {
                     for (Map<String, Object> map1 : poll) {
                         if ("FILE_UPLOAD_URL".equals(map1.get("code"))) {
                             uploadUrl = (String) map1.get("value");
-                        }else {
+                        }
+                    }
+                    if (StringUtils.isBlank(uploadUrl)) {
                         EmailUtil.send("UploadFile ERROR", "文件上传失败！上传地址未配置");
                         return;
                     }
-                    }
 
                     // 拼接url
                     String url = uploadUrl + String.format(Const.SF_UPLOAD_RICH_TEXT_FILE_URL, objectApi, recordId);

src/main/java/com/celnet/datadump/service/impl/FileServiceImpl.java

@@ -139,11 +139,12 @@ public class FileServiceImpl implements FileService {
         for (Map<String, Object> map1 : poll) {
             if ("FILE_DOWNLOAD_URL".equals(map1.get("code"))) {
                 downloadUrl = (String) map1.get("value");
-            }else {
+            }
+        }
+        if (StringUtils.isNotBlank(downloadUrl)) {
             EmailUtil.send("DumpFile ERROR", "文件下载失败！下载地址未配置");
             return;
         }
-        }
         log.info("dump file api:{}, field:{}", api, field);
         PartnerConnection connect = salesforceConnect.createConnect();
         String token = connect.getSessionHeader().getSessionId();
@@ -431,17 +432,17 @@ public class FileServiceImpl implements FileService {
 
     @Override
     public void uploadFile(String api, String field, Boolean singleThread) {
-
         String uploadUrl = null;
         List<Map<String, Object>> poll = customMapper.list("code,value","org_config",null);
         for (Map<String, Object> map1 : poll) {
             if ("FILE_UPLOAD_URL".equals(map1.get("code"))) {
                 uploadUrl = (String) map1.get("value");
-            }else {
+            }
+        }
+        if (StringUtils.isBlank(uploadUrl)) {
             EmailUtil.send("UploadFile ERROR", "文件上传失败！上传地址未配置");
             return;
         }
-        }
 
         log.info("upload file api:{}, field:{}", api, field);
         PartnerConnection connect = salesforceTargetConnect.createConnect();
@@ -618,12 +619,12 @@ public class FileServiceImpl implements FileService {
         for (Map<String, Object> map1 : poll) {
             if ("FILE_UPLOAD_URL".equals(map1.get("code"))) {
                 uploadUrl = (String) map1.get("value");
-            }else {
+            }
+        }
+        if (StringUtils.isBlank(uploadUrl)) {
             EmailUtil.send("UploadFile ERROR", "文件上传失败！上传地址未配置");
             return;
         }
-        }
-
         log.info("upload file api:{}, field:{}", api, field);
         PartnerConnection connect = salesforceTargetConnect.createConnect();
         String token = connect.getSessionHeader().getSessionId();

src/main/java/com/celnet/datadump/util/DataUtil.java

@@ -392,6 +392,39 @@ public class DataUtil {
         }
     }
 
+    public static Object localBulkDataToSfData(String fieldType, String data) throws ParseException {
+        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
+        SimpleDateFormat sd = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
+        Date date;
+        //date转Calendar类型
+        Calendar calendar = Calendar.getInstance();
+        switch (fieldType) {
+            case "int":
+                return Integer.parseInt(data);
+            case "double":
+            case "currency":
+            case "percent":
+                return new BigDecimal(data);
+            case "boolean":
+                return Boolean.valueOf(data);
+            case "date":
+                return data+"T08:00:00Z";
+            case "datetime":
+                try {
+                    date = sd.parse(data);
+                }catch (ParseException e){
+                    //解决当时间秒为0时，转换秒精度丢失问题
+                    date = sd.parse(data+":00");
+                }
+                calendar.setTime(date);
+                return calendar;
+            case "time":
+                return adjustHour(data);
+            default:
+                return data;
+        }
+    }
+
     public static Object localDataToSfData(String fieldType, String data) throws ParseException {
         SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
         SimpleDateFormat sd = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");