From 690a6da050d6289568901f79c83ca05143810e8c Mon Sep 17 00:00:00 2001 From: Kris <2893855659@qq.com> Date: Tue, 1 Jul 2025 14:53:23 +0800 Subject: [PATCH] =?UTF-8?q?=E3=80=90feat=E3=80=91=E4=BF=9D=E7=95=99Documen?= =?UTF-8?q?t=E4=B8=8A=E4=BC=A0=E4=B8=8B=E8=BD=BD=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controller/FileManagerController.java | 2 + .../com/celnet/datadump/global/Const.java | 4 +- .../com/celnet/datadump/job/DumpFileJob.java | 2 + .../celnet/datadump/service/FileService.java | 3 + .../service/impl/CommonServiceImpl.java | 16 +- .../impl/DataImportNewServiceImpl.java | 51 +++--- .../service/impl/FileServiceImpl.java | 147 +++++++++++++++++- src/main/resources/application-prod.yml | 2 +- 8 files changed, 199 insertions(+), 28 deletions(-) diff --git a/src/main/java/com/celnet/datadump/controller/FileManagerController.java b/src/main/java/com/celnet/datadump/controller/FileManagerController.java index 56b4fdd..441c6f5 100644 --- a/src/main/java/com/celnet/datadump/controller/FileManagerController.java +++ b/src/main/java/com/celnet/datadump/controller/FileManagerController.java @@ -110,6 +110,8 @@ public class FileManagerController { } if ("Attachment".equals(api)){ fileService.uploadFileToAttachment(api, blobField, param.getSingleThread()); + }else if("Document".equals(api)){ + fileService.uploadFileToDocument(api, blobField, param.getSingleThread()); }else { fileService.uploadFile(api, blobField, param.getSingleThread()); } diff --git a/src/main/java/com/celnet/datadump/global/Const.java b/src/main/java/com/celnet/datadump/global/Const.java index 3ac7f34..38fb39d 100644 --- a/src/main/java/com/celnet/datadump/global/Const.java +++ b/src/main/java/com/celnet/datadump/global/Const.java @@ -50,12 +50,12 @@ public class Const { /** * sf 下载文件链接 */ - public static final String SF_FILE_URL = "/services/data/v56.0/sobjects/%s/%s/%s"; + public static final String SF_FILE_URL = "/services/data/v55.0/sobjects/%s/%s/%s"; /** * sf 上传文件链接 */ - public static final String SF_UPLOAD_FILE_URL = "/services/data/v56.0/sobjects/%s"; + public static final String SF_UPLOAD_FILE_URL = "/services/data/v55.0/sobjects/%s"; /** * sf 获取富文本图片链接 diff --git a/src/main/java/com/celnet/datadump/job/DumpFileJob.java b/src/main/java/com/celnet/datadump/job/DumpFileJob.java index df4c5d8..bb86d01 100644 --- a/src/main/java/com/celnet/datadump/job/DumpFileJob.java +++ b/src/main/java/com/celnet/datadump/job/DumpFileJob.java @@ -122,6 +122,8 @@ public class DumpFileJob { } if ("Attachment".equals(api)){ fileService.uploadFileToAttachment(api, blobField, param.getSingleThread()); + }else if("Document".equals(api)){ + fileService.uploadFileToDocument(api, blobField, param.getSingleThread()); }else { fileService.uploadFile(api, blobField, param.getSingleThread()); } diff --git a/src/main/java/com/celnet/datadump/service/FileService.java b/src/main/java/com/celnet/datadump/service/FileService.java index ef2e04d..431320e 100644 --- a/src/main/java/com/celnet/datadump/service/FileService.java +++ b/src/main/java/com/celnet/datadump/service/FileService.java @@ -35,4 +35,7 @@ public interface FileService { void uploadFile(String api, String field, Boolean singleThread); void uploadFileToAttachment(String api, String field, Boolean singleThread); + + void uploadFileToDocument(String api, String field, Boolean singleThread); + } diff --git a/src/main/java/com/celnet/datadump/service/impl/CommonServiceImpl.java b/src/main/java/com/celnet/datadump/service/impl/CommonServiceImpl.java index c7ffcce..6e4cfba 100644 --- a/src/main/java/com/celnet/datadump/service/impl/CommonServiceImpl.java +++ b/src/main/java/com/celnet/datadump/service/impl/CommonServiceImpl.java @@ -1097,7 +1097,19 @@ public class CommonServiceImpl implements CommonService { * @param fields 字段名称list */ private static void fileExtraFieldBuild(String apiName, List> list, List fieldList, List fields) { - { + if ("Document".equals(apiName)){ + DataField dataField = new DataField(); + dataField.setApi(apiName); + dataField.setField("url"); + dataField.setName("文件路径"); + fieldList.add(dataField); + fields.add("url"); + Map map = Maps.newHashMap(); + map.put("type", "text"); + map.put("comment", "文件路径"); + map.put("name", "localUrl"); + list.add(map); + }else { DataField dataField = new DataField(); dataField.setApi(apiName); dataField.setField("url"); @@ -1206,7 +1218,7 @@ public class CommonServiceImpl implements CommonService { for (int j = 0; j < saveResults.length; j++) { if (!saveResults[j].getSuccess()) { String format = String.format("数据导入 error, api name: %s, \nparam: %s, \ncause:\n%s", api, JSON.toJSONString(DataDumpParam.getFilter()), com.alibaba.fastjson.JSON.toJSONString(saveResults[j])); - EmailUtil.send("DataDump ContentDocumentLink ERROR", format); + log.error(format); } else { List> dList = new ArrayList<>(); Map linkMap = new HashMap<>(); diff --git a/src/main/java/com/celnet/datadump/service/impl/DataImportNewServiceImpl.java b/src/main/java/com/celnet/datadump/service/impl/DataImportNewServiceImpl.java index 129bb48..bc90baa 100644 --- a/src/main/java/com/celnet/datadump/service/impl/DataImportNewServiceImpl.java +++ b/src/main/java/com/celnet/datadump/service/impl/DataImportNewServiceImpl.java @@ -868,12 +868,23 @@ public class DataImportNewServiceImpl implements DataImportNewService { public ReturnT dumpDocumentLinkJob(String paramStr) throws Exception { String api = "ContentDocumentLink"; PartnerConnection partnerConnection = salesforceConnect.createConnect(); - List> list = customMapper.list("Id", "ContentDocument", "new_id is not null"); - DescribeSObjectResult dsr = partnerConnection.describeSObject(api); - List fields = customMapper.getFields(api).stream().map(String::toUpperCase).collect(Collectors.toList()); - Field[] dsrFields = dsr.getFields(); - try { - if (list != null && !list.isEmpty()) { + Integer count = customMapper.countBySQL("ContentDocument", "where new_id is not null"); + log.info("ContentDocument Total size：" + count); + + int page = count%200 == 0 ? count/200 : (count/200) + 1; + + int totalSize = 0; + for (int i = 0; i < page; i++) { + List> list = customMapper.list("Id", "ContentDocument", "new_id is not null order by Id limit " + i * 200 + ",200"); + if (list.isEmpty()){ + break; + } + totalSize = totalSize +list.size(); + log.info("dumpContentDocumentLink By ContentDocument Now size：" + totalSize); + DescribeSObjectResult dsr = partnerConnection.describeSObject(api); + List fields = customMapper.getFields(api).stream().map(String::toUpperCase).collect(Collectors.toList()); + Field[] dsrFields = dsr.getFields(); + try { for (Map map : list) { String contentDocumentId = (String) map.get("Id"); String sql = "SELECT Id, LinkedEntityId, LinkedEntity.Type, ContentDocumentId, Visibility, ShareType, SystemModstamp, IsDeleted FROM ContentDocumentLink where ContentDocumentId = '" + contentDocumentId + "'"; @@ -883,17 +894,13 @@ public class DataImportNewServiceImpl implements DataImportNewService { objects = DataUtil.toJsonArray(records, dsrFields); commonService.saveOrUpdate(api, fields, records, objects, true); } - + } catch (Throwable e) { + log.error("dumpDocumentLinkJob error message:{}", e.getMessage()); + return ReturnT.FAIL; } - } catch (Exception e) { - log.error("getDocumentLink error api:{}, data:{}", api, com.alibaba.fastjson2.JSON.toJSONString(list), e); - return ReturnT.FAIL; - } catch (Throwable e) { - log.error("getDocumentLink error api:{}, data:{}", api, com.alibaba.fastjson2.JSON.toJSONString(list), e); - TimeUnit.MINUTES.sleep(1); - return ReturnT.FAIL; } - return null; + log.info("dumpDocumentLink Success !!! "); + return ReturnT.SUCCESS; } /** @@ -911,7 +918,7 @@ public class DataImportNewServiceImpl implements DataImportNewService { //批量插入200一次 int page = count % 200 == 0 ? count / 200 : (count / 200) + 1; for (int i = 0; i < page; i++) { - List> linkList = customMapper.list("Id,LinkedEntityId,ContentDocumentId,LinkedEntity_Type,ShareType,Visibility", api, "ShareType = 'V' and new_id = '0' order by Id asc limit 200"); + List> linkList = customMapper.list("Id,LinkedEntityId,ContentDocumentId,LinkedEntity_Type,ShareType,Visibility", api, "ShareType = 'V' and new_id = '0' order by Id limit " + i * 200 + ",200"); SObject[] accounts = new SObject[linkList.size()]; String[] ids = new String[linkList.size()]; int index = 0; @@ -946,8 +953,8 @@ public class DataImportNewServiceImpl implements DataImportNewService { SaveResult[] saveResults = connection.create(accounts); for (int j = 0; j < saveResults.length; j++) { if (!saveResults[j].getSuccess()) { - String format = String.format("数据导入 error, api name: %s, \nparam: %s, \ncause:\n%s", api, com.alibaba.fastjson2.JSON.toJSONString(DataDumpParam.getFilter()), com.alibaba.fastjson.JSON.toJSONString(saveResults[j])); - EmailUtil.send("DataDump ContentDocumentLink ERROR", format); + String format = String.format("数据导入 error, api name: %s, \nparam: %s, \ncause:\n%s", api, com.alibaba.fastjson2.JSON.toJSONString(DataDumpParam.getFilter()), JSON.toJSONString(saveResults[j])); + log.error(format); } else { List> dList = new ArrayList<>(); Map linkMap = new HashMap<>(); @@ -958,17 +965,17 @@ public class DataImportNewServiceImpl implements DataImportNewService { } } } catch (Exception e) { - log.error("getDocumentLink error api:{}, data:{}", api, com.alibaba.fastjson2.JSON.toJSONString(accounts), e); - EmailUtil.send("-------测试-----------", com.alibaba.fastjson2.JSON.toJSONString(accounts)); + log.error("uploadDocumentLinkJob error message:{}", e.getMessage()); throw new RuntimeException(e); } } } } catch (Exception e) { - log.error("getDocumentLink error api:{}, data:{}", api, com.alibaba.fastjson2.JSON.toJSONString(list), e); + log.error("uploadDocumentLink error api:{}, data:{}", api, com.alibaba.fastjson2.JSON.toJSONString(list), e); return ReturnT.FAIL; } - return null; + log.info("uploadDocumentLink Success !!! "); + return ReturnT.SUCCESS; } } diff --git a/src/main/java/com/celnet/datadump/service/impl/FileServiceImpl.java b/src/main/java/com/celnet/datadump/service/impl/FileServiceImpl.java index 18ed38e..46b52c9 100644 --- a/src/main/java/com/celnet/datadump/service/impl/FileServiceImpl.java +++ b/src/main/java/com/celnet/datadump/service/impl/FileServiceImpl.java @@ -207,7 +207,11 @@ public class FileServiceImpl implements FileService { log.error("id: {}, no mapping dump type", id); } Map paramMap = Maps.newHashMap(); - paramMap.put("key", "url"); + if ("Document".equals(api)){ + paramMap.put("key", "localUrl"); + }else { + paramMap.put("key", "url"); + } paramMap.put("value", filePath); maps.add(paramMap); } @@ -770,4 +774,145 @@ public class FileServiceImpl implements FileService { customMapper.update(maps, api, "is_upload = 2"); } } + + @Override + public void uploadFileToDocument(String api, String field, Boolean singleThread) { + String uploadUrl = null; + List> poll = customMapper.list("code,value","org_config",null); + for (Map map1 : poll) { + if ("FILE_UPLOAD_URL".equals(map1.get("code"))) { + uploadUrl = (String) map1.get("value"); + } + } + if (StringUtils.isBlank(uploadUrl)) { + EmailUtil.send("UploadFile ERROR", "文件上传失败！上传地址未配置"); + return; + } + log.info("upload file api:{}, field:{}", api, field); + PartnerConnection connect = salesforceTargetConnect.createConnect(); + String token = connect.getSessionHeader().getSessionId(); + + List> futures = Lists.newArrayList(); + String extraSql = ""; + if (dataFieldService.hasDeleted(api)) { + extraSql += "AND IsDeleted = false "; + } + if (Const.FILE_TYPE == FileType.SERVER) { + // 检测路径是否存在 + File excel = new File(Const.SERVER_FILE_PATH + "/" + api); + if (!excel.exists()) { + throw new RuntimeException("找不到文件路径"); + } + } + try { + // 获取未存储的附件id + List> list = customMapper.list("Id, Name, localUrl, Description", api, "is_upload = 0"); + for (Map map : list) { + String finalUploadUrl = uploadUrl; + String id = null; + // 上传完毕 更新附件信息 + List> maps = Lists.newArrayList(); + boolean isUpload = true; + int failCount = 0; + CloseableHttpResponse response = null; + String respContent = null; + while (true) { + try { + id = (String) map.get("Id"); + String url_fileName = (String) map.get("localUrl"); + + // 判断路径是否为空 + if (StringUtils.isNotBlank(url_fileName)) { + String filePath = Const.SERVER_FILE_PATH + "/" + url_fileName; + File file = new File(filePath); + boolean exists = file.exists(); + if (!exists) { + log.info("文件不存在"); + break; + } + String fileName = file.getName(); + + // 拼接url + String url = finalUploadUrl + String.format(Const.SF_UPLOAD_FILE_URL, api); + HttpPost httpPost = new HttpPost(url); + httpPost.setHeader("Authorization", "Bearer " + token); + httpPost.setHeader("connection", "keep-alive"); + + JSONObject credentialsJsonParam = new JSONObject(); + credentialsJsonParam.put("Name", fileName); + + MultipartEntityBuilder builder = MultipartEntityBuilder.create(); + builder.addTextBody("data", credentialsJsonParam.toJSONString(), ContentType.APPLICATION_JSON); + builder.addBinaryBody("Body", new FileInputStream(file), ContentType.APPLICATION_OCTET_STREAM, fileName); + HttpEntity entity = builder.build(); + httpPost.setEntity(entity); + + CloseableHttpClient httpClient = HttpClients.createDefault(); + response = httpClient.execute(httpPost); + if (response != null && response.getStatusLine() != null && response.getStatusLine().getStatusCode() < 400) { + HttpEntity he = response.getEntity(); + if (he != null) { + respContent = EntityUtils.toString(he, "UTF-8"); + String newId = JSONObject.parseObject(respContent).get("id").toString(); + Map paramMap = Maps.newHashMap(); + paramMap.put("key", "new_id"); + paramMap.put("value", newId); + maps.add(paramMap); + } + } else { + throw new RuntimeException("Document文件上传失败！地址或参数异常！"); + } + } + Map paramMap = Maps.newHashMap(); + paramMap.put("key", "is_upload"); + paramMap.put("value", isUpload); + maps.add(paramMap); + customMapper.updateById(api, maps, id); + TimeUnit.MILLISECONDS.sleep(1); + break; + } catch (Exception throwable) { + log.error("upload file error, id: {}", id, throwable); + failCount++; + if (Const.MAX_FAIL_COUNT < failCount) { + { + Map paramMap = Maps.newHashMap(); + paramMap.put("key", "is_upload"); + paramMap.put("value", 2); + maps.add(paramMap); + } + customMapper.updateById(api, maps, id); + break; + } + if (response != null) { + try { + response.close(); + } catch (IOException e) { + log.error("exception message", e); + } + } + try { + TimeUnit.SECONDS.sleep(30); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } + } + + salesforceExecutor.waitForFutures(futures.toArray(new Future[]{})); + + log.info("upload file success api:{}, field:{}", api, field); + } catch (Throwable throwable) { + log.error("upload file error", throwable); + salesforceExecutor.remove(futures.toArray(new Future[]{})); + } finally { + // 把is_upload为2的重置为0 + List> maps = Lists.newArrayList(); + Map paramMap = Maps.newHashMap(); + paramMap.put("key", "is_upload"); + paramMap.put("value", 0); + maps.add(paramMap); + customMapper.update(maps, api, "is_upload = 2"); + } + } } diff --git a/src/main/resources/application-prod.yml b/src/main/resources/application-prod.yml index 3c37f1e..de5eac2 100644 --- a/src/main/resources/application-prod.yml +++ b/src/main/resources/application-prod.yml @@ -5,7 +5,7 @@ spring: # 携科 url: jdbc:mysql://127.0.0.1:3306/xieke?useUnicode=true&allowPublicKeyRetrieval=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai username: root - password: Celnet2025.QY + password: root # cook # url: jdbc:mysql://127.0.0.1:3306/cook_1?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai # username: root