kris/data-dump
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

【feat】保留Document上传下载代码

Browse Source
This commit is contained in:
Kris 2025-07-01 14:53:23 +08:00
parent 7ccfb5ec45
commit 690a6da050
8 changed files with 199 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/com/celnet/datadump/controller/FileManagerController.java
View File

@ -110,6 +110,8 @@ public class FileManagerController {
}
if ("Attachment".equals(api)){
fileService.uploadFileToAttachment(api, blobField, param.getSingleThread());
}else if("Document".equals(api)){
fileService.uploadFileToDocument(api, blobField, param.getSingleThread());
}else {
fileService.uploadFile(api, blobField, param.getSingleThread());
}

4
src/main/java/com/celnet/datadump/global/Const.java
View File

@ -50,12 +50,12 @@ public class Const {
/**
* sf 下载文件链接
*/
public static final String SF_FILE_URL = "/services/data/v56.0/sobjects/%s/%s/%s";
public static final String SF_FILE_URL = "/services/data/v55.0/sobjects/%s/%s/%s";
/**
* sf 上传文件链接
*/
public static final String SF_UPLOAD_FILE_URL = "/services/data/v56.0/sobjects/%s";
public static final String SF_UPLOAD_FILE_URL = "/services/data/v55.0/sobjects/%s";
/**
* sf 获取富文本图片链接

2
src/main/java/com/celnet/datadump/job/DumpFileJob.java
View File

@ -122,6 +122,8 @@ public class DumpFileJob {
}
if ("Attachment".equals(api)){
fileService.uploadFileToAttachment(api, blobField, param.getSingleThread());
}else if("Document".equals(api)){
fileService.uploadFileToDocument(api, blobField, param.getSingleThread());
}else {
fileService.uploadFile(api, blobField, param.getSingleThread());
}

3
src/main/java/com/celnet/datadump/service/FileService.java
View File

@ -35,4 +35,7 @@ public interface FileService {
void uploadFile(String api, String field, Boolean singleThread);
void uploadFileToAttachment(String api, String field, Boolean singleThread);
void uploadFileToDocument(String api, String field, Boolean singleThread);
}

16
src/main/java/com/celnet/datadump/service/impl/CommonServiceImpl.java
View File

@ -1097,7 +1097,19 @@ public class CommonServiceImpl implements CommonService {
* @param fields 字段名称list
*/
private static void fileExtraFieldBuild(String apiName, List<Map<String, Object>> list, List<DataField> fieldList, List<String> fields) {
{
if ("Document".equals(apiName)){
DataField dataField = new DataField();
dataField.setApi(apiName);
dataField.setField("url");
dataField.setName("文件路径");
fieldList.add(dataField);
fields.add("url");
Map<String, Object> map = Maps.newHashMap();
map.put("type", "text");
map.put("comment", "文件路径");
map.put("name", "localUrl");
list.add(map);
}else {
DataField dataField = new DataField();
dataField.setApi(apiName);
dataField.setField("url");
@ -1206,7 +1218,7 @@ public class CommonServiceImpl implements CommonService {
for (int j = 0; j < saveResults.length; j++) {
if (!saveResults[j].getSuccess()) {
String format = String.format("数据导入 error, api name: %s, \nparam: %s, \ncause:\n%s", api, JSON.toJSONString(DataDumpParam.getFilter()), com.alibaba.fastjson.JSON.toJSONString(saveResults[j]));
EmailUtil.send("DataDump ContentDocumentLink ERROR", format);
log.error(format);
} else {
List<Map<String, Object>> dList = new ArrayList<>();
Map<String, Object> linkMap = new HashMap<>();

41
src/main/java/com/celnet/datadump/service/impl/DataImportNewServiceImpl.java
View File

@ -868,12 +868,23 @@ public class DataImportNewServiceImpl implements DataImportNewService {
public ReturnT<String> dumpDocumentLinkJob(String paramStr) throws Exception {
String api = "ContentDocumentLink";
PartnerConnection partnerConnection = salesforceConnect.createConnect();
List<Map<String, Object>> list = customMapper.list("Id", "ContentDocument", "new_id is not null");
Integer count = customMapper.countBySQL("ContentDocument", "where new_id is not null");
log.info("ContentDocument Total size：" + count);
int page = count%200 == 0 ? count/200 : (count/200) + 1;
int totalSize = 0;
for (int i = 0; i < page; i++) {
List<Map<String, Object>> list = customMapper.list("Id", "ContentDocument", "new_id is not null order by Id limit " + i * 200 + ",200");
if (list.isEmpty()){
break;
}
totalSize = totalSize +list.size();
log.info("dumpContentDocumentLink By ContentDocument Now size：" + totalSize);
DescribeSObjectResult dsr = partnerConnection.describeSObject(api);
List<String> fields = customMapper.getFields(api).stream().map(String::toUpperCase).collect(Collectors.toList());
Field[] dsrFields = dsr.getFields();
try {
if (list != null && !list.isEmpty()) {
for (Map<String, Object> map : list) {
String contentDocumentId = (String) map.get("Id");
String sql = "SELECT Id, LinkedEntityId, LinkedEntity.Type, ContentDocumentId, Visibility, ShareType, SystemModstamp, IsDeleted FROM ContentDocumentLink where ContentDocumentId = '" + contentDocumentId + "'";
@ -883,17 +894,13 @@ public class DataImportNewServiceImpl implements DataImportNewService {
objects = DataUtil.toJsonArray(records, dsrFields);
commonService.saveOrUpdate(api, fields, records, objects, true);
}
}
} catch (Exception e) {
log.error("getDocumentLink error api:{}, data:{}", api, com.alibaba.fastjson2.JSON.toJSONString(list), e);
return ReturnT.FAIL;
} catch (Throwable e) {
log.error("getDocumentLink error api:{}, data:{}", api, com.alibaba.fastjson2.JSON.toJSONString(list), e);
TimeUnit.MINUTES.sleep(1);
log.error("dumpDocumentLinkJob error message:{}", e.getMessage());
return ReturnT.FAIL;
}
return null;
}
log.info("dumpDocumentLink Success !!! ");
return ReturnT.SUCCESS;
}
/**
@ -911,7 +918,7 @@ public class DataImportNewServiceImpl implements DataImportNewService {
//批量插入200一次
int page = count % 200 == 0 ? count / 200 : (count / 200) + 1;
for (int i = 0; i < page; i++) {
List<Map<String, Object>> linkList = customMapper.list("Id,LinkedEntityId,ContentDocumentId,LinkedEntity_Type,ShareType,Visibility", api, "ShareType = 'V' and new_id = '0' order by Id asc limit 200");
List<Map<String, Object>> linkList = customMapper.list("Id,LinkedEntityId,ContentDocumentId,LinkedEntity_Type,ShareType,Visibility", api, "ShareType = 'V' and new_id = '0' order by Id limit " + i * 200 + ",200");
SObject[] accounts = new SObject[linkList.size()];
String[] ids = new String[linkList.size()];
int index = 0;
@ -946,8 +953,8 @@ public class DataImportNewServiceImpl implements DataImportNewService {
SaveResult[] saveResults = connection.create(accounts);
for (int j = 0; j < saveResults.length; j++) {
if (!saveResults[j].getSuccess()) {
String format = String.format("数据导入 error, api name: %s, \nparam: %s, \ncause:\n%s", api, com.alibaba.fastjson2.JSON.toJSONString(DataDumpParam.getFilter()), com.alibaba.fastjson.JSON.toJSONString(saveResults[j]));
EmailUtil.send("DataDump ContentDocumentLink ERROR", format);
String format = String.format("数据导入 error, api name: %s, \nparam: %s, \ncause:\n%s", api, com.alibaba.fastjson2.JSON.toJSONString(DataDumpParam.getFilter()), JSON.toJSONString(saveResults[j]));
log.error(format);
} else {
List<Map<String, Object>> dList = new ArrayList<>();
Map<String, Object> linkMap = new HashMap<>();
@ -958,17 +965,17 @@ public class DataImportNewServiceImpl implements DataImportNewService {
}
}
} catch (Exception e) {
log.error("getDocumentLink error api:{}, data:{}", api, com.alibaba.fastjson2.JSON.toJSONString(accounts), e);
EmailUtil.send("-------测试-----------", com.alibaba.fastjson2.JSON.toJSONString(accounts));
log.error("uploadDocumentLinkJob error message:{}", e.getMessage());
throw new RuntimeException(e);
}
}
}
} catch (Exception e) {
log.error("getDocumentLink error api:{}, data:{}", api, com.alibaba.fastjson2.JSON.toJSONString(list), e);
log.error("uploadDocumentLink error api:{}, data:{}", api, com.alibaba.fastjson2.JSON.toJSONString(list), e);
return ReturnT.FAIL;
}
return null;
log.info("uploadDocumentLink Success !!! ");
return ReturnT.SUCCESS;
}
}

145
src/main/java/com/celnet/datadump/service/impl/FileServiceImpl.java
View File

@ -207,7 +207,11 @@ public class FileServiceImpl implements FileService {
log.error("id: {}, no mapping dump type", id);
}
Map<String, Object> paramMap = Maps.newHashMap();
if ("Document".equals(api)){
paramMap.put("key", "localUrl");
}else {
paramMap.put("key", "url");
}
paramMap.put("value", filePath);
maps.add(paramMap);
}
@ -770,4 +774,145 @@ public class FileServiceImpl implements FileService {
customMapper.update(maps, api, "is_upload = 2");
}
}
@Override
public void uploadFileToDocument(String api, String field, Boolean singleThread) {
String uploadUrl = null;
List<Map<String, Object>> poll = customMapper.list("code,value","org_config",null);
for (Map<String, Object> map1 : poll) {
if ("FILE_UPLOAD_URL".equals(map1.get("code"))) {
uploadUrl = (String) map1.get("value");
}
}
if (StringUtils.isBlank(uploadUrl)) {
EmailUtil.send("UploadFile ERROR", "文件上传失败！上传地址未配置");
return;
}
log.info("upload file api:{}, field:{}", api, field);
PartnerConnection connect = salesforceTargetConnect.createConnect();
String token = connect.getSessionHeader().getSessionId();
List<Future<?>> futures = Lists.newArrayList();
String extraSql = "";
if (dataFieldService.hasDeleted(api)) {
extraSql += "AND IsDeleted = false ";
}
if (Const.FILE_TYPE == FileType.SERVER) {
// 检测路径是否存在
File excel = new File(Const.SERVER_FILE_PATH + "/" + api);
if (!excel.exists()) {
throw new RuntimeException("找不到文件路径");
}
}
try {
// 获取未存储的附件id
List<Map<String, Object>> list = customMapper.list("Id, Name, localUrl, Description", api, "is_upload = 0");
for (Map<String, Object> map : list) {
String finalUploadUrl = uploadUrl;
String id = null;
// 上传完毕 更新附件信息
List<Map<String, Object>> maps = Lists.newArrayList();
boolean isUpload = true;
int failCount = 0;
CloseableHttpResponse response = null;
String respContent = null;
while (true) {
try {
id = (String) map.get("Id");
String url_fileName = (String) map.get("localUrl");
// 判断路径是否为空
if (StringUtils.isNotBlank(url_fileName)) {
String filePath = Const.SERVER_FILE_PATH + "/" + url_fileName;
File file = new File(filePath);
boolean exists = file.exists();
if (!exists) {
log.info("文件不存在");
break;
}
String fileName = file.getName();
// 拼接url
String url = finalUploadUrl + String.format(Const.SF_UPLOAD_FILE_URL, api);
HttpPost httpPost = new HttpPost(url);
httpPost.setHeader("Authorization", "Bearer " + token);
httpPost.setHeader("connection", "keep-alive");
JSONObject credentialsJsonParam = new JSONObject();
credentialsJsonParam.put("Name", fileName);
MultipartEntityBuilder builder = MultipartEntityBuilder.create();
builder.addTextBody("data", credentialsJsonParam.toJSONString(), ContentType.APPLICATION_JSON);
builder.addBinaryBody("Body", new FileInputStream(file), ContentType.APPLICATION_OCTET_STREAM, fileName);
HttpEntity entity = builder.build();
httpPost.setEntity(entity);
CloseableHttpClient httpClient = HttpClients.createDefault();
response = httpClient.execute(httpPost);
if (response != null && response.getStatusLine() != null && response.getStatusLine().getStatusCode() < 400) {
HttpEntity he = response.getEntity();
if (he != null) {
respContent = EntityUtils.toString(he, "UTF-8");
String newId = JSONObject.parseObject(respContent).get("id").toString();
Map<String, Object> paramMap = Maps.newHashMap();
paramMap.put("key", "new_id");
paramMap.put("value", newId);
maps.add(paramMap);
}
} else {
throw new RuntimeException("Document文件上传失败！地址或参数异常！");
}
}
Map<String, Object> paramMap = Maps.newHashMap();
paramMap.put("key", "is_upload");
paramMap.put("value", isUpload);
maps.add(paramMap);
customMapper.updateById(api, maps, id);
TimeUnit.MILLISECONDS.sleep(1);
break;
} catch (Exception throwable) {
log.error("upload file error, id: {}", id, throwable);
failCount++;
if (Const.MAX_FAIL_COUNT < failCount) {
{
Map<String, Object> paramMap = Maps.newHashMap();
paramMap.put("key", "is_upload");
paramMap.put("value", 2);
maps.add(paramMap);
}
customMapper.updateById(api, maps, id);
break;
}
if (response != null) {
try {
response.close();
} catch (IOException e) {
log.error("exception message", e);
}
}
try {
TimeUnit.SECONDS.sleep(30);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
}
}
}
salesforceExecutor.waitForFutures(futures.toArray(new Future<?>[]{}));
log.info("upload file success api:{}, field:{}", api, field);
} catch (Throwable throwable) {
log.error("upload file error", throwable);
salesforceExecutor.remove(futures.toArray(new Future<?>[]{}));
} finally {
// 把is_upload为2的重置为0
List<Map<String, Object>> maps = Lists.newArrayList();
Map<String, Object> paramMap = Maps.newHashMap();
paramMap.put("key", "is_upload");
paramMap.put("value", 0);
maps.add(paramMap);
customMapper.update(maps, api, "is_upload = 2");
}
}
}

2
src/main/resources/application-prod.yml
View File

@ -5,7 +5,7 @@ spring:
# 携科
url: jdbc:mysql://127.0.0.1:3306/xieke?useUnicode=true&allowPublicKeyRetrieval=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai
username: root
password: Celnet2025.QY
password: root
# cook
# url: jdbc:mysql://127.0.0.1:3306/cook_1?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai
# username: root