kris/data-dump
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

【feat】调整文件上传下载，将上传下载链接配置在数据库

Browse Source
This commit is contained in:
Kris 2025-06-17 15:41:28 +08:00
parent 4697cd95a1
commit e532b21465
6 changed files with 93 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
files/start.sh
View File

@ -1,8 +1,13 @@
# 程序配置
# xxl job数据库链接
#cook
#dbUrl="jdbc:mysql://127.0.0.1:3306/data-dump-xxl-job?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&serverTimezone=Asia/Shanghai"
#dbUsername="root"
#dbPassword="celnet@2025.bln"
#携科
dbUrl="jdbc:mysql://127.0.0.1:3306/data-dump-xxl-job?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&serverTimezone=Asia/Shanghai"
dbUsername="root"
dbPassword="celnet@2025.bln"
dbPassword="Celnet2025.QY"
#dbUrl="jdbc:mysql://127.0.0.113306/data-dump-xxl-job?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&serverTimezone=Asia/Shanghai"
#dbUsername="msg"
#dbPassword="msg@2021"

26
src/main/java/com/celnet/datadump/service/impl/FileManagerServiceImpl.java
View File

@ -51,11 +51,6 @@ public class FileManagerServiceImpl implements FileManagerService {
@Autowired
private SalesforceExecutor salesforceExecutor;
@Value(value = "${sf.file-download-url}")
private String downloadUrl;
@Value(value = "${sf.file-upload-url}")
private String uploadUrl;
@ -169,6 +164,16 @@ public class FileManagerServiceImpl implements FileManagerService {
Map<String, String> headers = Maps.newHashMap();
headers.put("Authorization", "Bearer " + token);
headers.put("connection", "keep-alive");
String downloadUrl = null;
List<Map<String, Object>> poll = customMapper.list("code,value","org_config",null);
for (Map<String, Object> map1 : poll) {
if ("FILE_DOWNLOAD_URL".equals(map1.get("code"))) {
downloadUrl = (String) map1.get("value");
}else {
EmailUtil.send("DumpFile ERROR", "文件下载失败！下载地址未配置");
return;
}
}
String url = downloadUrl + String.format(Const.SF_RICH_TEXT_FILE_URL, objectApi, recordId, fieldApi, fieldId);
try {
@ -297,6 +302,17 @@ public class FileManagerServiceImpl implements FileManagerService {
}
ids.add(id);
}
String uploadUrl = null;
List<Map<String, Object>> poll = customMapper.list("code,value","org_config",null);
for (Map<String, Object> map1 : poll) {
if ("FILE_UPLOAD_URL".equals(map1.get("code"))) {
uploadUrl = (String) map1.get("value");
}else {
EmailUtil.send("UploadFile ERROR", "文件上传失败！上传地址未配置");
return;
}
}
// 拼接url
String url = uploadUrl + String.format(Const.SF_UPLOAD_RICH_TEXT_FILE_URL, objectApi, recordId);

51
src/main/java/com/celnet/datadump/service/impl/FileServiceImpl.java
View File

@ -1,5 +1,6 @@
package com.celnet.datadump.service.impl;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
@ -10,10 +11,12 @@ import com.celnet.datadump.config.SalesforceTargetConnect;
import com.celnet.datadump.entity.DataObject;
import com.celnet.datadump.enums.FileType;
import com.celnet.datadump.global.Const;
import com.celnet.datadump.param.DataDumpParam;
import com.celnet.datadump.param.FileTransformParam;
import com.celnet.datadump.global.SystemConfigCode;
import com.celnet.datadump.mapper.CustomMapper;
import com.celnet.datadump.service.*;
import com.celnet.datadump.util.EmailUtil;
import com.celnet.datadump.util.HttpUtil;
import com.celnet.datadump.util.OssUtil;
import com.google.common.collect.Lists;
@ -75,12 +78,6 @@ public class FileServiceImpl implements FileService {
@Autowired
private CommonService commonService;
@Value(value = "${sf.file-download-url}")
private String downloadUrl;
@Value(value = "${sf.file-upload-url}")
private String uploadUrl;
@Override
public void verifyFile(String api, String field) {
log.info("verify file api:{}, field:{}", api, field);
@ -137,6 +134,16 @@ public class FileServiceImpl implements FileService {
@Override
public void dumpFile(String api, String field, Boolean singleThread) {
String downloadUrl = null;
List<Map<String, Object>> poll = customMapper.list("code,value","org_config",null);
for (Map<String, Object> map1 : poll) {
if ("FILE_DOWNLOAD_URL".equals(map1.get("code"))) {
downloadUrl = (String) map1.get("value");
}else {
EmailUtil.send("DumpFile ERROR", "文件下载失败！下载地址未配置");
return;
}
}
log.info("dump file api:{}, field:{}", api, field);
PartnerConnection connect = salesforceConnect.createConnect();
String token = connect.getSessionHeader().getSessionId();
@ -167,6 +174,7 @@ public class FileServiceImpl implements FileService {
}
String finalName = name;
for (Map<String, Object> map : list) {
String finalDownloadUrl = downloadUrl;
Future<?> future = salesforceExecutor.execute(() -> {
String id = null;
// 上传完毕 更新附件信息
@ -182,7 +190,7 @@ public class FileServiceImpl implements FileService {
if (StringUtils.isNotBlank(fileName)) {
String filePath = api + "/" + id + "_" + fileName;
// 拼接url
String url = downloadUrl + String.format(Const.SF_FILE_URL, api, id, field);
String url = finalDownloadUrl + String.format(Const.SF_FILE_URL, api, id, field);
Response response = HttpUtil.doGet(url, null, headers);
if (response.body() != null) {
InputStream inputStream = response.body().byteStream();
@ -423,6 +431,18 @@ public class FileServiceImpl implements FileService {
@Override
public void uploadFile(String api, String field, Boolean singleThread) {
String uploadUrl = null;
List<Map<String, Object>> poll = customMapper.list("code,value","org_config",null);
for (Map<String, Object> map1 : poll) {
if ("FILE_UPLOAD_URL".equals(map1.get("code"))) {
uploadUrl = (String) map1.get("value");
}else {
EmailUtil.send("UploadFile ERROR", "文件上传失败！上传地址未配置");
return;
}
}
log.info("upload file api:{}, field:{}", api, field);
PartnerConnection connect = salesforceTargetConnect.createConnect();
String token = connect.getSessionHeader().getSessionId();
@ -449,6 +469,7 @@ public class FileServiceImpl implements FileService {
if (CollectionUtils.isEmpty(list)) {
continue;
}
String finalUploadUrl = uploadUrl;
Future<?> future = salesforceExecutor.execute(() -> {
String newDocumentId = null;
for (Map<String, Object> map : list) {
@ -477,7 +498,7 @@ public class FileServiceImpl implements FileService {
break;
}
// 拼接url
String url = uploadUrl + String.format(Const.SF_UPLOAD_FILE_URL, api);
String url = finalUploadUrl + String.format(Const.SF_UPLOAD_FILE_URL, api);
HttpPost httpPost = new HttpPost(url);
httpPost.setHeader("Authorization", "Bearer " + token);
httpPost.setHeader("connection", "keep-alive");
@ -592,6 +613,17 @@ public class FileServiceImpl implements FileService {
@Override
public void uploadFileToAttachment(String api, String field, Boolean singleThread) {
String uploadUrl = null;
List<Map<String, Object>> poll = customMapper.list("code,value","org_config",null);
for (Map<String, Object> map1 : poll) {
if ("FILE_UPLOAD_URL".equals(map1.get("code"))) {
uploadUrl = (String) map1.get("value");
}else {
EmailUtil.send("UploadFile ERROR", "文件上传失败！上传地址未配置");
return;
}
}
log.info("upload file api:{}, field:{}", api, field);
PartnerConnection connect = salesforceTargetConnect.createConnect();
String token = connect.getSessionHeader().getSessionId();
@ -612,6 +644,7 @@ public class FileServiceImpl implements FileService {
// 获取未存储的附件id
List<Map<String, Object>> list = customMapper.list("Id, ParentId, Name, url, Description, Parent_type", api, "is_upload = 0");
for (Map<String, Object> map : list) {
String finalUploadUrl = uploadUrl;
Future<?> future = salesforceExecutor.execute(() -> {
String id = null;
// 上传完毕 更新附件信息
@ -649,7 +682,7 @@ public class FileServiceImpl implements FileService {
Map<String, Object> lMap = customMapper.getById("new_id",parentType, parentId);
// 拼接url
String url = uploadUrl + String.format(Const.SF_UPLOAD_FILE_URL, api);
String url = finalUploadUrl + String.format(Const.SF_UPLOAD_FILE_URL, api);
HttpPost httpPost = new HttpPost(url);
httpPost.setHeader("Authorization", "Bearer " + token);
httpPost.setHeader("connection", "keep-alive");

11
src/main/resources/application-dev.yml
View File

@ -2,12 +2,15 @@ spring:
datasource:
type: com.zaxxer.hikari.HikariDataSource
driver-class-name: com.mysql.cj.jdbc.Driver
url: jdbc:mysql://127.0.0.1:3306/cook?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai
# 携科
url: jdbc:mysql://127.0.0.1:3306/xieke?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai
username: root
password: celnet@2025.bln
# url: jdbc:mysql://183.6.105.131:13306/zhonghe_test?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai
password: Celnet2025.QY
# cook
# url: jdbc:mysql://127.0.0.1:3306/cook?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai
# username: root
# password: Celnet@2022
# password: celnet@2025.bln
mail:
host: smtp.163.com
port: 465

13
src/main/resources/application-prod.yml
View File

@ -2,11 +2,14 @@ spring:
datasource:
type: com.zaxxer.hikari.HikariDataSource
driver-class-name: com.mysql.cj.jdbc.Driver
url: jdbc:mysql://127.0.0.1:3306/cook?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true
# 携科
url: jdbc:mysql://127.0.0.1:3306/xieke?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai
username: root
password: Celnet2025.QY
# cook
# url: jdbc:mysql://127.0.0.1:3306/cook?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai
# username: root
# password: celnet@2025.bln
password: root
mail:
host: smtp.mxhichina.com
port: 465
@ -18,8 +21,8 @@ spring:
#sf webservice配置
sf:
# 附件下载url
file-download-url: https://cookmedicalasia.my.salesforce.com
file-upload-url: https://cookchina--sandbox.sandbox.my.sfcrmproducts.cn
file-download-url: https://d2000000079c7eaa.lightning.force.com
file-upload-url: https://steco-process.lightning.sfcrmapps.cn
#线程数
executor-size: 5
list:

10
src/main/resources/application-test.yml
View File

@ -2,9 +2,15 @@ spring:
datasource:
type: com.zaxxer.hikari.HikariDataSource
driver-class-name: com.mysql.cj.jdbc.Driver
url: jdbc:mysql://127.0.0.1:3306/cook?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true
# 携科
url: jdbc:mysql://127.0.0.1:3306/xieke?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai
username: root
password: celnet@2025.bln
password: Celnet2025.QY
# cook
# url: jdbc:mysql://127.0.0.1:3306/cook?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai
# username: root
# password: celnet@2025.bln
mail:
host: smtp.163.com
port: 465