【feat】 问题优化

src/main/java/com/celnet/datadump/controller/JobController.java

@@ -299,4 +299,56 @@ public class JobController {
         param.setEndCreateDate(param.getEndDate());
         return dataImportNewService.immigrationUpdateNew(param);
     }
+
+    /**
+     * 数据更新同步（新）
+     * @param paramStr
+     * @author kris
+     * @return
+     * @throws Exception
+     */
+    @PostMapping("/dataImportJob")
+    @ApiOperation("获取new_id")
+    @LogServiceAnnotation(operateType = OperateTypeConstant.TYPE_UPDATE, remark = "获取new_id")
+    public ReturnT<String> dataImportJob(String paramStr) throws Exception {
+        log.info("dataImportJob execute start ..................");
+        SalesforceParam param = new SalesforceParam();
+        try {
+            if (StringUtils.isNotBlank(paramStr)) {
+                param = JSON.parseObject(paramStr, SalesforceParam.class);
+            }
+        } catch (Throwable throwable) {
+            return new ReturnT<>(500, "参数解析失败！");
+        }
+        // 参数转换
+        param.setBeginCreateDate(param.getBeginDate());
+        param.setEndCreateDate(param.getEndDate());
+        return dataImportService.immigration(param);
+    }
+
+    /**
+     * 数据更新同步（新）
+     * @param paramStr
+     * @author kris
+     * @return
+     * @throws Exception
+     */
+    @PostMapping("/dataUpdateJob")
+    @ApiOperation("更新目标org数据")
+    @LogServiceAnnotation(operateType = OperateTypeConstant.TYPE_UPDATE, remark = "更新目标org数据")
+    public ReturnT<String> dataUpdateJob(String paramStr) throws Exception {
+        log.info("dataUpdateJob execute start ..................");
+        SalesforceParam param = new SalesforceParam();
+        try {
+            if (StringUtils.isNotBlank(paramStr)) {
+                param = JSON.parseObject(paramStr, SalesforceParam.class);
+            }
+        } catch (Throwable throwable) {
+            return new ReturnT<>(500, "参数解析失败！");
+        }
+        // 参数转换
+        param.setBeginCreateDate(param.getBeginDate());
+        param.setEndCreateDate(param.getEndDate());
+        return dataImportNewService.immigrationUpdateNew(param);
+    }
 }

src/main/java/com/celnet/datadump/service/impl/CommonServiceImpl.java

@@ -366,10 +366,7 @@ public class CommonServiceImpl implements CommonService {
                     }
                     // 等待当前所有线程执行完成
                     salesforceExecutor.waitForFutures(futures.toArray(new Future<?>[]{}));
-                    // 附件表 跑一遍dumpFile
-                    if (StringUtils.isNotBlank(dataObject.getBlobField())) {
-                        fileService.dumpFile(dataObject.getName(), dataObject.getBlobField(), true);
-                    }
+
                     update.setDataWork(0);
                 } catch (Throwable e) {
                     String message = e.getMessage();
@@ -377,12 +374,10 @@ public class CommonServiceImpl implements CommonService {
                     EmailUtil.send("DataDump ERROR", format);
                     throw new RuntimeException(e);
                 } finally {
-                    if (dataObject != null) {
                     update.setDataLock(0);
                     dataObjectService.updateById(update);
                 }
             }
-            }
             // 等待当前所有线程执行完成
             salesforceExecutor.waitForFutures(futures.toArray(new Future<?>[]{}));
             futures.clear();
@@ -462,11 +457,7 @@ public class CommonServiceImpl implements CommonService {
 
                 // 等待当前所有线程执行完成
                 salesforceExecutor.waitForFutures(futures.toArray(new Future<?>[]{}));
-                // 附件表 跑一遍dumpFile
-                DataObject one = dataObjectService.getById(api);
-                if (StringUtils.isNotBlank(one.getBlobField())) {
-                    fileService.dumpFile(one.getName(), one.getBlobField(), true);
-                }
+
                 update.setDataWork(0);
             } catch (Throwable e) {
                 log.error("manualDump error", e);

src/main/java/com/celnet/datadump/service/impl/DataImportBatchServiceImpl.java

@@ -361,7 +361,7 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
                             //引用类型
                             String reference = dataField.getReferenceTo();
                             //引用类型字段
-                            String linkfield = fieldMap.get(dataField.getApi());
+                            String linkfield = fieldMap.get(dataField.getField());
 
                             if (StringUtils.isNotBlank(linkfield)){
                                 reference = data.get(j-1).get(linkfield)!=null?data.get(j-1).get(linkfield).toString():null;
@@ -395,7 +395,7 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
                             account.put(dataField.getField(), pickList.get(0).get("value"));
                             continue;
                         }
-                        account.put(dataField.getField(), DataUtil.fieldTypeToSf(dataField));
+                        account.put(dataField.getField(), DataUtil.fieldTypeBulkToSf(dataField));
                     }
 
                 }
@@ -748,7 +748,7 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
 
                         if (!"null".equals(value) && StringUtils.isNotEmpty(value)) {
                             //引用类型字段
-                            String linkfield = fieldMap.get(dataField.getApi());
+                            String linkfield = fieldMap.get(dataField.getField());
                             if (StringUtils.isNotBlank(linkfield)){
                                 reference_to = map.get(linkfield)!=null?map.get(linkfield).toString():null;
                             }
@@ -1122,7 +1122,7 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
                             //引用类型
                             String reference_to = dataField.getReferenceTo();
                             //引用类型字段
-                            String linkfield = fieldMap.get(dataField.getApi());
+                            String linkfield = fieldMap.get(dataField.getField());
 
                             if (StringUtils.isNotBlank(linkfield)){
                                 reference_to = data.get(j-1).get(linkfield)!=null?data.get(j-1).get(linkfield).toString():null;

src/main/java/com/celnet/datadump/service/impl/DataImportNewServiceImpl.java

@@ -758,6 +758,7 @@ public class DataImportNewServiceImpl implements DataImportNewService {
             List<Map<String, Object>> mapList = customMapper.list("*", api,  sql2+ i * 200 + ",200");
             SObject[] accounts = new SObject[mapList.size()];
             int j = 0;
+            try {
                 for (Map<String, Object> map : mapList) {
                 SObject account = new SObject();
                 account.setType(api);
@@ -776,7 +777,7 @@ public class DataImportNewServiceImpl implements DataImportNewService {
 
                         if (!"null".equals(value) && StringUtils.isNotEmpty(value)) {
                             //引用类型字段
-                            String linkfield = fieldMap.get(dataField.getApi());
+                            String linkfield = fieldMap.get(dataField.getField());
                             if (StringUtils.isNotBlank(linkfield)){
                                 reference_to = map.get(linkfield)!=null?map.get(linkfield).toString():null;
                             }
@@ -813,7 +814,6 @@ public class DataImportNewServiceImpl implements DataImportNewService {
                 accounts[j++] = account;
             }
 
-            try {
                 if (infoFlag != null && "1".equals(infoFlag.get("value"))){
                     printlnAccountsDetails(accounts,list);
                 }
@@ -856,6 +856,9 @@ public class DataImportNewServiceImpl implements DataImportNewService {
         for (int i = 0; i < accounts.length; i++) {
             SObject account = accounts[i];
             System.out.println("---  对象数据[" + i + "] ---");
+            if (account == null){
+                continue;
+            }
             // 获取对象所有字段名
             for (DataField dataField : list) {
                 try {
@@ -865,8 +868,8 @@ public class DataImportNewServiceImpl implements DataImportNewService {
                     System.out.println(dataField.getField() + ": [权限不足或字段不存在]");
                 }
             }
-            System.out.println("old_owner_id__c: " + (account.getField("old_owner_id__c") != null ? account.getField("old_owner_id__c").toString() : "null"));
-            System.out.println("old_sfdc_id__c: " + (account.getField("old_sfdc_id__c") != null ? account.getField("old_sfdc_id__c").toString() : "null"));
+            System.out.println("old_owner_id__c: " + account.getField("old_owner_id__c"));
+            System.out.println("old_sfdc_id__c: " + account.getField("old_sfdc_id__c"));
 
         }
     }
@@ -878,12 +881,15 @@ public class DataImportNewServiceImpl implements DataImportNewService {
         HashMap<String, String> map = new HashMap<>();
         for (int i = 0; i < accounts.length;  i++) {
             SObject account = accounts[i];
+            if (account == null){
+                continue;
+            }
             if (errorId != null && (errorId.equals(account.getId()) || errorId.equals(account.getField("Id")))){
                 for (DataField dataField : list) {
                     try {
                         Object value = account.getField(dataField.getField());
                         map.put(dataField.getField(),String.valueOf(value));
-                        System.out.println(dataField.getField()  + ": " + (value != null ? value.toString()  : "null"));
+                        System.out.println(dataField.getField()  + ": " + value);
                     } catch (Exception e) {
                         System.out.println(dataField.getField() + ": [权限不足或字段不存在]");
                     }

src/main/java/com/celnet/datadump/service/impl/DataImportServiceImpl.java

@@ -335,7 +335,7 @@ public class DataImportServiceImpl implements DataImportService {
                                 //引用类型
                                 String reference = dataField.getReferenceTo();
                                 //引用类型字段
-                                String linkfield = fieldMap.get(dataField.getApi());
+                                String linkfield = fieldMap.get(dataField.getField());
 
                                 if (StringUtils.isNotBlank(linkfield)){
                                     reference = data.get(j-1).get(linkfield)!=null?data.get(j-1).get(linkfield).toString():null;
@@ -370,14 +370,6 @@ public class DataImportServiceImpl implements DataImportService {
                         }
                     }
 
-                    if (api.equals("vlink__Wechat_User__c")){
-                        List<Map<String, Object>> maps = customMapper.list("new_id", "vlink__Wechat_Account__c", "new_id is not null and id = '" + data.get(j - 1).get("vlink__Wechat_Account__c") + "' limit 1");
-                        if (!maps.isEmpty()){
-                            Map<String, Object> referenceMap = maps.get(0);
-                            account.setField("vlink__Wechat_Account__c",  referenceMap.get("new_id"));
-                        }
-                    }
-
                     ids[j-1] = data.get(j-1).get("Id").toString();
                     accounts[j-1] = account;
                     if (i*200+j == count){
@@ -406,8 +398,7 @@ public class DataImportServiceImpl implements DataImportService {
                 }
                 TimeUnit.MILLISECONDS.sleep(1);
             } catch (Exception e) {
-                log.error("manualCreatedNewId error api:{}", api, e);
-                throw e;
+                log.error("manualCreatedNewId error api:{},错误信息：{}", api, e.getMessage());
             }
         }
         SalesforceParam countParam = new SalesforceParam();
@@ -740,18 +731,19 @@ public class DataImportServiceImpl implements DataImportService {
         for (int i = 0; i < accounts.length;  i++) {
             SObject account = accounts[i];
             System.out.println("---  Account[" + i + "] ---");
-
+            if (account == null){
+                return;
+            }
             // 获取对象所有字段名
             for (DataField dataField : list) {
                 try {
-                    Object value = account.getField(dataField.getField());
-                    System.out.println(dataField.getField()  + ": " + (value != null ? value.toString()  : "null"));
+                    System.out.println(dataField.getField()  + ": " + account.getField(dataField.getField()));
                 } catch (Exception e) {
                     System.out.println(dataField.getField()  + ": [权限不足或字段不存在]");
                 }
             }
-            System.out.println("old_owner_id__c: " + (account.getField("old_owner_id__c") != null ? account.getField("old_owner_id__c").toString()  : "null"));
-            System.out.println("old_sfdc_id__c: " + (account.getField("old_sfdc_id__c") != null ? account.getField("old_sfdc_id__c").toString()  : "null"));
+            System.out.println("old_owner_id__c: " + account.getField("old_owner_id__c"));
+            System.out.println("old_sfdc_id__c: " + account.getField("old_sfdc_id__c"));
 
         }
     }
@@ -762,18 +754,21 @@ public class DataImportServiceImpl implements DataImportService {
             HashMap<String, String> map = new HashMap<>();
             SObject account = accounts[i];
             System.out.println("---  Account[" + i + "] ---");
+            if (account == null){
+                return null;
+            }
             // 获取对象所有字段名
             for (DataField dataField : list) {
                 try {
                     Object value = account.getField(dataField.getField());
                     map.put(dataField.getField(),String.valueOf(value));
-                    System.out.println(dataField.getField()  + ": " + (value != null ? value.toString()  : "null"));
+                    System.out.println(dataField.getField()  + ": " + value);
                 } catch (Exception e) {
                     System.out.println(dataField.getField()  + ": [权限不足或字段不存在]");
                 }
             }
-            System.out.println("old_owner_id__c: " + (account.getField("old_owner_id__c") != null ? account.getField("old_owner_id__c").toString()  : "null"));
-            System.out.println("old_sfdc_id__c: " + (account.getField("old_sfdc_id__c") != null ? account.getField("old_sfdc_id__c").toString()  : "null"));
+            System.out.println("old_owner_id__c: " + account.getField("old_owner_id__c"));
+            System.out.println("old_sfdc_id__c: " + account.getField("old_sfdc_id__c"));
             map.put("old_owner_id__c",String.valueOf(account.getField("old_owner_id__c")));
             map.put("old_sfdc_id__c",String.valueOf(account.getField("old_sfdc_id__c")));
             arrayList.add(map);

src/main/java/com/celnet/datadump/service/impl/FileServiceImpl.java

@@ -171,6 +171,7 @@ public class FileServiceImpl implements FileService {
                 // 获取未存储的附件id
                 List<Map<String, Object>> list = customMapper.list("Id, " + name, api, extraSql + " AND is_dump = false limit 1000");
                 if (CollectionUtils.isEmpty(list)) {
+                    log.info("无需要下载文件数据！！！");
                     break;
                 }
                 String finalName = name;

src/main/java/com/celnet/datadump/util/DataUtil.java

@@ -31,6 +31,10 @@ import javax.annotation.PostConstruct;
 import java.math.BigDecimal;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
 import java.util.*;
 
 
@@ -393,12 +397,29 @@ public class DataUtil {
         }
     }
 
+    public static Object fieldTypeBulkToSf(DataField dataField) throws ParseException {
+        switch (dataField.getSfType()) {
+            case "int":
+                return 1;
+            case "double":
+            case "currency":
+            case "percent":
+                return new BigDecimal("1");
+            case "boolean":
+                return true;
+            case "email":
+                return "kris.dong@163.com";
+            case "date":
+            case "datetime":
+                return "2020-01-01T00:00:00+00:00";
+            default:
+                return "初始"+ UUID.randomUUID();
+        }
+    }
+
+
     public static Object localBulkDataToSfData(String fieldType, String data) throws ParseException {
-        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
-        SimpleDateFormat sd = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
-        Date date;
-        //date转Calendar类型
-        Calendar calendar = Calendar.getInstance();
+
         switch (fieldType) {
             case "int":
                 return Integer.parseInt(data);
@@ -411,14 +432,9 @@ public class DataUtil {
             case "date":
                 return data+"T08:00:00Z";
             case "datetime":
-                try {
-                    date = sd.parse(data);
-                }catch (ParseException e){
-                    //解决当时间秒为0时，转换秒精度丢失问题
-                    date = sd.parse(data+":00");
-                }
-                calendar.setTime(date);
-                return calendar;
+                LocalDateTime localDateTime = LocalDateTime.parse(data,  DateTimeFormatter.ISO_LOCAL_DATE_TIME);
+                ZonedDateTime zonedDateTime = localDateTime.atZone(ZoneId.systemDefault());
+                return zonedDateTime.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME);
             case "time":
                 return adjustHour(data);
             default: