【feat】保留Document上传下载代码

src/main/java/com/celnet/datadump/controller/FileManagerController.java

@@ -110,6 +110,8 @@ public class FileManagerController {
             }
             if ("Attachment".equals(api)){
                 fileService.uploadFileToAttachment(api, blobField, param.getSingleThread());
+            }else if("Document".equals(api)){
+                fileService.uploadFileToDocument(api, blobField, param.getSingleThread());
             }else {
                 fileService.uploadFile(api, blobField, param.getSingleThread());
             }

src/main/java/com/celnet/datadump/global/Const.java

@@ -50,12 +50,12 @@ public class Const {
     /**
      * sf 下载文件链接
      */
-    public static final String SF_FILE_URL = "/services/data/v56.0/sobjects/%s/%s/%s";
+    public static final String SF_FILE_URL = "/services/data/v55.0/sobjects/%s/%s/%s";
 
     /**
      * sf 上传文件链接
      */
-    public static final String SF_UPLOAD_FILE_URL = "/services/data/v56.0/sobjects/%s";
+    public static final String SF_UPLOAD_FILE_URL = "/services/data/v55.0/sobjects/%s";
 
     /**
      * sf 获取富文本图片链接

src/main/java/com/celnet/datadump/job/DumpFileJob.java

@@ -122,6 +122,8 @@ public class DumpFileJob {
             }
             if ("Attachment".equals(api)){
                 fileService.uploadFileToAttachment(api, blobField, param.getSingleThread());
+            }else if("Document".equals(api)){
+                fileService.uploadFileToDocument(api, blobField, param.getSingleThread());
             }else {
                 fileService.uploadFile(api, blobField, param.getSingleThread());
             }

src/main/java/com/celnet/datadump/service/FileService.java

@@ -35,4 +35,7 @@ public interface FileService {
     void uploadFile(String api, String field, Boolean singleThread);
 
     void uploadFileToAttachment(String api, String field, Boolean singleThread);
+
+    void uploadFileToDocument(String api, String field, Boolean singleThread);
+
 }

src/main/java/com/celnet/datadump/service/impl/CommonServiceImpl.java

@@ -1097,7 +1097,19 @@ public class CommonServiceImpl implements CommonService {
      * @param fields    字段名称list
      */
     private static void fileExtraFieldBuild(String apiName, List<Map<String, Object>> list, List<DataField> fieldList, List<String> fields) {
-        {
+        if ("Document".equals(apiName)){
+            DataField dataField = new DataField();
+            dataField.setApi(apiName);
+            dataField.setField("url");
+            dataField.setName("文件路径");
+            fieldList.add(dataField);
+            fields.add("url");
+            Map<String, Object> map = Maps.newHashMap();
+            map.put("type", "text");
+            map.put("comment", "文件路径");
+            map.put("name", "localUrl");
+            list.add(map);
+        }else {
             DataField dataField = new DataField();
             dataField.setApi(apiName);
             dataField.setField("url");
@@ -1206,7 +1218,7 @@ public class CommonServiceImpl implements CommonService {
                         for (int j = 0; j < saveResults.length; j++) {
                             if (!saveResults[j].getSuccess()) {
                                 String format = String.format("数据导入 error, api name: %s, \nparam: %s, \ncause:\n%s", api, JSON.toJSONString(DataDumpParam.getFilter()), com.alibaba.fastjson.JSON.toJSONString(saveResults[j]));
-                                EmailUtil.send("DataDump ContentDocumentLink ERROR", format);
+                                log.error(format);
                             } else {
                                 List<Map<String, Object>> dList = new ArrayList<>();
                                 Map<String, Object> linkMap = new HashMap<>();

src/main/java/com/celnet/datadump/service/impl/DataImportNewServiceImpl.java

@@ -868,12 +868,23 @@ public class DataImportNewServiceImpl implements DataImportNewService {
     public ReturnT<String> dumpDocumentLinkJob(String paramStr) throws Exception {
         String api = "ContentDocumentLink";
         PartnerConnection partnerConnection = salesforceConnect.createConnect();
-        List<Map<String, Object>> list = customMapper.list("Id", "ContentDocument", "new_id is not null");
+        Integer count = customMapper.countBySQL("ContentDocument", "where new_id is not null");
-        DescribeSObjectResult dsr = partnerConnection.describeSObject(api);
+        log.info("ContentDocument Total size：" + count);
-        List<String> fields = customMapper.getFields(api).stream().map(String::toUpperCase).collect(Collectors.toList());
+
-        Field[] dsrFields = dsr.getFields();
+        int page = count%200 == 0 ? count/200 : (count/200) + 1;
-        try {
+
-            if (list != null && !list.isEmpty()) {
+        int totalSize = 0;
+        for (int i = 0; i < page; i++) {
+            List<Map<String, Object>> list = customMapper.list("Id", "ContentDocument", "new_id is not null order by Id limit " + i * 200 + ",200");
+            if (list.isEmpty()){
+                break;
+            }
+            totalSize = totalSize +list.size();
+            log.info("dumpContentDocumentLink By ContentDocument Now size：" + totalSize);
+            DescribeSObjectResult dsr = partnerConnection.describeSObject(api);
+            List<String> fields = customMapper.getFields(api).stream().map(String::toUpperCase).collect(Collectors.toList());
+            Field[] dsrFields = dsr.getFields();
+            try {
                 for (Map<String, Object> map : list) {
                     String contentDocumentId = (String) map.get("Id");
                     String sql = "SELECT Id, LinkedEntityId, LinkedEntity.Type, ContentDocumentId, Visibility, ShareType, SystemModstamp, IsDeleted FROM ContentDocumentLink where ContentDocumentId = '" + contentDocumentId + "'";
@@ -883,17 +894,13 @@ public class DataImportNewServiceImpl implements DataImportNewService {
                     objects = DataUtil.toJsonArray(records, dsrFields);
                     commonService.saveOrUpdate(api, fields, records, objects, true);
                 }
-
+            } catch (Throwable e) {
+                log.error("dumpDocumentLinkJob error message:{}", e.getMessage());
+                return ReturnT.FAIL;
             }
-        } catch (Exception e) {
-            log.error("getDocumentLink error api:{}, data:{}", api, com.alibaba.fastjson2.JSON.toJSONString(list), e);
-            return ReturnT.FAIL;
-        } catch (Throwable e) {
-            log.error("getDocumentLink error api:{}, data:{}", api, com.alibaba.fastjson2.JSON.toJSONString(list), e);
-            TimeUnit.MINUTES.sleep(1);
-            return ReturnT.FAIL;
         }
-        return null;
+        log.info("dumpDocumentLink Success !!! ");
+        return ReturnT.SUCCESS;
     }
 
     /**
@@ -911,7 +918,7 @@ public class DataImportNewServiceImpl implements DataImportNewService {
                 //批量插入200一次
                 int page = count % 200 == 0 ? count / 200 : (count / 200) + 1;
                 for (int i = 0; i < page; i++) {
-                    List<Map<String, Object>> linkList = customMapper.list("Id,LinkedEntityId,ContentDocumentId,LinkedEntity_Type,ShareType,Visibility", api, "ShareType = 'V' and new_id = '0' order by Id asc limit 200");
+                    List<Map<String, Object>> linkList = customMapper.list("Id,LinkedEntityId,ContentDocumentId,LinkedEntity_Type,ShareType,Visibility", api, "ShareType = 'V' and new_id = '0' order by Id limit " + i * 200 + ",200");
                     SObject[] accounts = new SObject[linkList.size()];
                     String[] ids = new String[linkList.size()];
                     int index = 0;
@@ -946,8 +953,8 @@ public class DataImportNewServiceImpl implements DataImportNewService {
                         SaveResult[] saveResults = connection.create(accounts);
                         for (int j = 0; j < saveResults.length; j++) {
                             if (!saveResults[j].getSuccess()) {
-                                String format = String.format("数据导入 error, api name: %s, \nparam: %s, \ncause:\n%s", api, com.alibaba.fastjson2.JSON.toJSONString(DataDumpParam.getFilter()), com.alibaba.fastjson.JSON.toJSONString(saveResults[j]));
+                                String format = String.format("数据导入 error, api name: %s, \nparam: %s, \ncause:\n%s", api, com.alibaba.fastjson2.JSON.toJSONString(DataDumpParam.getFilter()), JSON.toJSONString(saveResults[j]));
-                                EmailUtil.send("DataDump ContentDocumentLink ERROR", format);
+                                log.error(format);
                             } else {
                                 List<Map<String, Object>> dList = new ArrayList<>();
                                 Map<String, Object> linkMap = new HashMap<>();
@@ -958,17 +965,17 @@ public class DataImportNewServiceImpl implements DataImportNewService {
                             }
                         }
                     } catch (Exception e) {
-                        log.error("getDocumentLink error api:{}, data:{}", api, com.alibaba.fastjson2.JSON.toJSONString(accounts), e);
+                        log.error("uploadDocumentLinkJob error message:{}", e.getMessage());
-                        EmailUtil.send("-------测试-----------", com.alibaba.fastjson2.JSON.toJSONString(accounts));
                         throw new RuntimeException(e);
                     }
                 }
             }
         } catch (Exception e) {
-            log.error("getDocumentLink error api:{}, data:{}", api, com.alibaba.fastjson2.JSON.toJSONString(list), e);
+            log.error("uploadDocumentLink error api:{}, data:{}", api, com.alibaba.fastjson2.JSON.toJSONString(list), e);
             return ReturnT.FAIL;
         }
-        return null;
+        log.info("uploadDocumentLink Success !!! ");
+        return ReturnT.SUCCESS;
     }
 
 }

src/main/java/com/celnet/datadump/service/impl/FileServiceImpl.java

@@ -207,7 +207,11 @@ public class FileServiceImpl implements FileService {
                                                 log.error("id: {}, no mapping dump type", id);
                                         }
                                         Map<String, Object> paramMap = Maps.newHashMap();
-                                        paramMap.put("key", "url");
+                                        if ("Document".equals(api)){
+                                            paramMap.put("key", "localUrl");
+                                        }else {
+                                            paramMap.put("key", "url");
+                                        }
                                         paramMap.put("value", filePath);
                                         maps.add(paramMap);
                                     }
@@ -770,4 +774,145 @@ public class FileServiceImpl implements FileService {
             customMapper.update(maps, api, "is_upload = 2");
         }
     }
+
+    @Override
+    public void uploadFileToDocument(String api, String field, Boolean singleThread) {
+        String uploadUrl = null;
+        List<Map<String, Object>> poll = customMapper.list("code,value","org_config",null);
+        for (Map<String, Object> map1 : poll) {
+            if ("FILE_UPLOAD_URL".equals(map1.get("code"))) {
+                uploadUrl = (String) map1.get("value");
+            }
+        }
+        if (StringUtils.isBlank(uploadUrl)) {
+            EmailUtil.send("UploadFile ERROR", "文件上传失败！上传地址未配置");
+            return;
+        }
+        log.info("upload file api:{}, field:{}", api, field);
+        PartnerConnection connect = salesforceTargetConnect.createConnect();
+        String token = connect.getSessionHeader().getSessionId();
+
+        List<Future<?>> futures = Lists.newArrayList();
+        String extraSql = "";
+        if (dataFieldService.hasDeleted(api)) {
+            extraSql += "AND IsDeleted = false ";
+        }
+        if (Const.FILE_TYPE == FileType.SERVER) {
+            // 检测路径是否存在
+            File excel = new File(Const.SERVER_FILE_PATH + "/" + api);
+            if (!excel.exists()) {
+                throw new RuntimeException("找不到文件路径");
+            }
+        }
+        try {
+            // 获取未存储的附件id
+            List<Map<String, Object>> list = customMapper.list("Id,  Name, localUrl, Description", api, "is_upload = 0");
+            for (Map<String, Object> map : list) {
+                String finalUploadUrl = uploadUrl;
+                String id = null;
+                // 上传完毕 更新附件信息
+                List<Map<String, Object>> maps = Lists.newArrayList();
+                boolean isUpload = true;
+                int failCount = 0;
+                CloseableHttpResponse response = null;
+                String respContent = null;
+                while (true) {
+                    try {
+                        id = (String) map.get("Id");
+                        String url_fileName = (String) map.get("localUrl");
+
+                        // 判断路径是否为空
+                        if (StringUtils.isNotBlank(url_fileName)) {
+                            String filePath = Const.SERVER_FILE_PATH + "/" + url_fileName;
+                            File file = new File(filePath);
+                            boolean exists = file.exists();
+                            if (!exists) {
+                                log.info("文件不存在");
+                                break;
+                            }
+                            String fileName = file.getName();
+
+                            // 拼接url
+                            String url = finalUploadUrl + String.format(Const.SF_UPLOAD_FILE_URL, api);
+                            HttpPost httpPost = new HttpPost(url);
+                            httpPost.setHeader("Authorization", "Bearer " + token);
+                            httpPost.setHeader("connection", "keep-alive");
+
+                            JSONObject credentialsJsonParam = new JSONObject();
+                            credentialsJsonParam.put("Name", fileName);
+
+                            MultipartEntityBuilder builder = MultipartEntityBuilder.create();
+                            builder.addTextBody("data", credentialsJsonParam.toJSONString(), ContentType.APPLICATION_JSON);
+                            builder.addBinaryBody("Body", new FileInputStream(file), ContentType.APPLICATION_OCTET_STREAM, fileName);
+                            HttpEntity entity = builder.build();
+                            httpPost.setEntity(entity);
+
+                            CloseableHttpClient httpClient = HttpClients.createDefault();
+                            response = httpClient.execute(httpPost);
+                            if (response != null && response.getStatusLine() != null && response.getStatusLine().getStatusCode() < 400) {
+                                HttpEntity he = response.getEntity();
+                                if (he != null) {
+                                    respContent = EntityUtils.toString(he, "UTF-8");
+                                    String newId = JSONObject.parseObject(respContent).get("id").toString();
+                                    Map<String, Object> paramMap = Maps.newHashMap();
+                                    paramMap.put("key", "new_id");
+                                    paramMap.put("value", newId);
+                                    maps.add(paramMap);
+                                }
+                            } else {
+                                throw new RuntimeException("Document文件上传失败！地址或参数异常！");
+                            }
+                        }
+                        Map<String, Object> paramMap = Maps.newHashMap();
+                        paramMap.put("key", "is_upload");
+                        paramMap.put("value", isUpload);
+                        maps.add(paramMap);
+                        customMapper.updateById(api, maps, id);
+                        TimeUnit.MILLISECONDS.sleep(1);
+                        break;
+                    } catch (Exception throwable) {
+                        log.error("upload file error, id: {}", id, throwable);
+                        failCount++;
+                        if (Const.MAX_FAIL_COUNT < failCount) {
+                            {
+                                Map<String, Object> paramMap = Maps.newHashMap();
+                                paramMap.put("key", "is_upload");
+                                paramMap.put("value", 2);
+                                maps.add(paramMap);
+                            }
+                            customMapper.updateById(api, maps, id);
+                            break;
+                        }
+                        if (response != null) {
+                            try {
+                                response.close();
+                            } catch (IOException e) {
+                                log.error("exception message", e);
+                            }
+                        }
+                        try {
+                            TimeUnit.SECONDS.sleep(30);
+                        } catch (InterruptedException e) {
+                            throw new RuntimeException(e);
+                        }
+                    }
+                }
+            }
+
+            salesforceExecutor.waitForFutures(futures.toArray(new Future<?>[]{}));
+
+            log.info("upload file success api:{}, field:{}", api, field);
+        } catch (Throwable throwable) {
+            log.error("upload file error", throwable);
+            salesforceExecutor.remove(futures.toArray(new Future<?>[]{}));
+        } finally {
+            // 把is_upload为2的重置为0
+            List<Map<String, Object>> maps = Lists.newArrayList();
+            Map<String, Object> paramMap = Maps.newHashMap();
+            paramMap.put("key", "is_upload");
+            paramMap.put("value", 0);
+            maps.add(paramMap);
+            customMapper.update(maps, api, "is_upload = 2");
+        }
+    }
 }

src/main/resources/application-prod.yml

@@ -5,7 +5,7 @@ spring:
     # 携科
     url: jdbc:mysql://127.0.0.1:3306/xieke?useUnicode=true&allowPublicKeyRetrieval=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai
     username: root
-    password: Celnet2025.QY
+    password: root
     # cook
 #    url: jdbc:mysql://127.0.0.1:3306/cook_1?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai
 #    username: root