【feat】 富文本图片替换优化

This commit is contained in:
Kris 2025-11-07 10:00:46 +08:00
parent e72501f17d
commit 2e6ec65cd2
4 changed files with 237 additions and 96 deletions

View File

@ -1 +0,0 @@
<result-list xmlns="http://www.force.com/2009/06/asyncapi/dataload"><result>752e2000002qkjy</result></result-list>
Can't render this file because it contains an unexpected character in line 1 and column 20.

View File

@ -62,7 +62,7 @@ public class RichTextLog implements Serializable {
private String newFileId;
@ApiModelProperty("创建时间")
@TableField(value = "created_time", fill = FieldFill.INSERT)
private Date createdTime;
@TableField(value = "createdDate", fill = FieldFill.INSERT)
private Date createdDate;
}

View File

@ -32,6 +32,7 @@ import lombok.extern.slf4j.Slf4j;
import okhttp3.Response;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections4.Get;
import org.apache.commons.compress.utils.IOUtils;
import org.apache.commons.lang.time.DateUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@ -47,6 +48,7 @@ import org.apache.http.util.EntityUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import java.io.*;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
@ -58,8 +60,6 @@ import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.Base64;
@Service
@Slf4j
public class DataImportNewServiceImpl implements DataImportNewService {
@ -102,6 +102,23 @@ public class DataImportNewServiceImpl implements DataImportNewService {
private static final String TEMP_FILE_PATH = "verify/";
private static String FILE_UPLOAD_URL = "";
private static String FILE_DOWNLOAD_URL = "";
@PostConstruct
public void init() {
List<Map<String, Object>> poll = customMapper.list("code,value","org_config",null);
for (Map<String, Object> map1 : poll) {
if ("FILE_UPLOAD_URL".equals(map1.get("code"))) {
FILE_UPLOAD_URL = (String) map1.get("value");
}
if ("FILE_DOWNLOAD_URL".equals(map1.get("code"))) {
FILE_DOWNLOAD_URL = (String) map1.get("value");
}
}
}
static {
// 检测路径是否存在 不存在则创建
File excel = new File(TEMP_FILE_PATH);
@ -110,9 +127,6 @@ public class DataImportNewServiceImpl implements DataImportNewService {
}
}
@Autowired
private EmailUtil emailUtil;
/**
* Get返写个人客户联系人入口
*/
@ -1414,12 +1428,23 @@ public class DataImportNewServiceImpl implements DataImportNewService {
}
if (Const.FILE_TYPE == FileType.SERVER) {
Date defaultBeginDate = DataUtil.DEFAULT_BEGIN_DATE;
Calendar calendar = Calendar.getInstance();
int currentYear = calendar.get(Calendar.YEAR);
calendar.setTime(defaultBeginDate);
int beginYear = calendar.get(Calendar.YEAR);
// 检测路径是否存在 不存在则创建
File excel = new File(Const.SERVER_FILE_PATH + "/" + api);
if (!excel.exists()) {
boolean mkdir = excel.mkdir();
if (!mkdir) {
log.info("创建文件存储目录失败!");
File baseDir = new File(Const.SERVER_FILE_PATH + "/" + api);
if (!baseDir.exists()) {
baseDir.mkdirs();
}
// 创建从开始年份到当前年份的子目录
for (int year = beginYear; year <= currentYear; year++) {
File yearDir = new File(Const.SERVER_FILE_PATH + "/" + api + "" + year);
if (!yearDir.exists()) {
yearDir.mkdir();
}
}
}
@ -1502,7 +1527,7 @@ public class DataImportNewServiceImpl implements DataImportNewService {
log.info("api{} 批次:{}-开始时间:{}-结束时间:{}",api, i, beginDateStr, endDateStr);
// 获取未存储的附件id
List<Map<String, Object>> list = customMapper.list("Id, " + name, api, " is_dump = 0 and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr +"'"+ extraSql + "limit 200");
List<Map<String, Object>> list = customMapper.list("Id,CreatedDate, " + name, api, " is_dump = 0 and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr +"'"+ extraSql + "limit 200");
for (Map<String, Object> map : list) {
int failCount = 0;
@ -1517,9 +1542,11 @@ public class DataImportNewServiceImpl implements DataImportNewService {
}else {
fileName = (String) map.get("Name");
}
String year = map.get("CreatedDate").toString().substring(0, 4);
// 判断路径是否为空
if (StringUtils.isNotBlank(fileName)) {
String filePath = api + "/" + id + "_" + fileName;
String filePath = api + "-" + year + "/" + id + "_" + fileName;
// 拼接url
String url = downloadUrl + String.format(Const.SF_FILE_URL, api, id, field);
@ -2675,6 +2702,16 @@ public class DataImportNewServiceImpl implements DataImportNewService {
endDateStr = DateUtil.format(endDate, "yyyy-MM-dd HH:mm:ss");
}
}
// 检测路径是否存在 不存在则创建
File excel = new File(Const.SERVER_FILE_PATH + "/richTextImages" );
if (!excel.exists()) {
log.info("创建API目录: {}", excel.getAbsolutePath());
boolean mkdir = excel.mkdir();
if (!mkdir) {
log.info("创建文件存储目录失败!");
}
}
// 获取连接
PartnerConnection sourceConnection = salesforceConnect.createConnect();
PartnerConnection targetConnection = salesforceTargetConnect.createConnect();
@ -2740,6 +2777,10 @@ public class DataImportNewServiceImpl implements DataImportNewService {
// 处理富文本中的图片
String updatedContent = processRichTextImages(richTextContent, sourceId, api, field, sourceConnection, targetConnection);
if (updatedContent == null){
return ReturnT.FAIL;
}
// 如果内容有变化则更新目标组织中的记录
if (!richTextContent.equals(updatedContent)) {
@ -2748,6 +2789,7 @@ public class DataImportNewServiceImpl implements DataImportNewService {
sObject.setType(api);
sObject.setId(targetId);
sObject.setField(field, updatedContent);
sObject.setField("Id", targetId);
accounts[index] = sObject;
index ++;
@ -2817,8 +2859,7 @@ public class DataImportNewServiceImpl implements DataImportNewService {
private String processRichTextImages(String richTextContent, String sourceRecordId, String api, String field,
PartnerConnection sourceConnection, PartnerConnection targetConnection) throws Exception {
// 匹配富文本中包含refid的img标签
Pattern imgPattern = Pattern.compile("<img[^>]*refid=\"([^\"]+)\"[^>]*/?>", Pattern.CASE_INSENSITIVE);
Matcher matcher = imgPattern.matcher(richTextContent);
Pattern imgPattern = Pattern.compile("<img[^>]*src\\s*=\\s*\"[^\"]*refid=([^\"]+)(?:&quot;|&amp;|&|\\\"|\"|')?[^>]*/?>", Pattern.CASE_INSENSITIVE); Matcher matcher = imgPattern.matcher(richTextContent);
StringBuffer updatedContent = new StringBuffer();
Map<String, String> imageIdMap = new ConcurrentHashMap<>();
@ -2837,19 +2878,11 @@ public class DataImportNewServiceImpl implements DataImportNewService {
}
try {
// 查询源组织中的ContentDocumentLink记录
String cdlQuery = "SELECT Id, ContentDocumentId FROM ContentDocumentLink WHERE LinkedEntityId = '" + sourceRecordId + "' AND ContentDocumentId = '" + refId + "'";
QueryResult cdlResult = sourceConnection.query(cdlQuery);
if (cdlResult.getRecords() != null && cdlResult.getRecords().length > 0) {
// 构建源图片下载URL (参考Python脚本中的imageUrl构建方式)
String downloadUrl = sourceConnection.getConfig().getServiceEndpoint() +
String downloadUrl = FILE_DOWNLOAD_URL +
String.format("/services/data/v55.0/sobjects/%s/%s/richTextImageFields/%s/%s",
api, sourceRecordId, field, refId);
// 下载源图片文件
InputStream inputStream = downloadImageFile(sourceConnection, downloadUrl);
if (inputStream != null) {
// 获取图片的alt属性作为标题
String title = "image";
Pattern altPattern = Pattern.compile("alt=\"([^\"]*)\"", Pattern.CASE_INSENSITIVE);
@ -2857,17 +2890,25 @@ public class DataImportNewServiceImpl implements DataImportNewService {
if (altMatcher.find()) {
title = altMatcher.group(1);
}
// 下载源图片文件
String downloadImageFile = downloadImageFile(sourceConnection, downloadUrl, title);
if (downloadImageFile != null) {
// 上传图片到目标组织
String newContentVersionId = uploadImageFile(targetConnection, inputStream, title);
String newContentVersionId = uploadImageFile(targetConnection, downloadImageFile, title);
if (newContentVersionId != null) {
// 获取新的ContentDocumentId
String newContentDocumentId = getDocumentId(targetConnection, newContentVersionId);
Map<String, String> documentId = getDocumentId(targetConnection, newContentVersionId);
String newContentDocumentId = documentId.get("ContentDocumentId");
String newOwnId = documentId.get("OwnerId");
if (newContentDocumentId != null) {
imageIdMap.put(refId, newContentDocumentId);
// 构建新的img标签 (参考Python脚本中的richTextUrl构建方式)
String newImageUrl = targetConnection.getConfig().getServiceEndpoint() +
String newImageUrl =
"/sfc/servlet.shepherd/version/download/" + newContentVersionId;
String updatedImgTag = "<img src=\"" + newImageUrl + "\" refid=\"" + newContentDocumentId + "\"/>";
@ -2878,7 +2919,7 @@ public class DataImportNewServiceImpl implements DataImportNewService {
api,
field,
newContentDocumentId,
null, // recordOwnerId 可以从记录中获取但此处简化处理
newOwnId, // recordOwnerId 可以从记录中获取但此处简化处理
richTextContent,
refId, // 原图片链接
newImageUrl, // 新图片链接
@ -2887,7 +2928,8 @@ public class DataImportNewServiceImpl implements DataImportNewService {
continue;
}
}
}
}else {
return null;
}
// 如果找不到对应关系保留原始标签
@ -2909,8 +2951,8 @@ public class DataImportNewServiceImpl implements DataImportNewService {
* @return 图片文件数据
* @throws Exception 异常
*/
private InputStream downloadImageFile(PartnerConnection connection, String downloadUrl) throws Exception {
InputStream inputStream = null;
private String downloadImageFile(PartnerConnection connection, String downloadUrl,String title) throws Exception {
String imageServer = null;
try {
String token = connection.getSessionHeader().getSessionId();
@ -2919,47 +2961,84 @@ public class DataImportNewServiceImpl implements DataImportNewService {
headers.put("connection", "keep-alive");
Response response = HttpUtil.doGet(downloadUrl, null, headers);
assert response.body() != null;
inputStream = response.body().byteStream();
if (response.body() != null && response.code() == 200){
InputStream inputStream = response.body().byteStream();
imageServer = dumpToRichTextImageServer(title, response, inputStream);
}else {
throw new RuntimeException("下载图片文件时发生错误URL: " + downloadUrl + ";错误信息:" + response.message());
}
} catch (Exception e) {
log.error("下载图片文件时发生错误URL: {}", downloadUrl, e);
return null;
}
return inputStream;
return imageServer;
}
/**
* 保存富文本图片到本地服务器
* @param response
* @param inputStream
* @throws IOException
*/
private String dumpToRichTextImageServer(String name, Response response, InputStream inputStream) throws IOException {
String path = Const.SERVER_FILE_PATH + "/richTextImages/" + name;
long offset = 0L;
RandomAccessFile accessFile = new RandomAccessFile(path, "rw");
while (true) {
try {
// 保存到本地
byte[] buf = new byte[8192];
int len = 0;
if (offset > 0) {
inputStream.skip(offset);
accessFile.seek(offset);
}
while ((len = inputStream.read(buf)) != -1) {
accessFile.write(buf, 0, len);
offset += len;
}
break;
} catch (Exception e) {
if (offset <= 0) {
throw e;
}
response.close();
}
}
accessFile.close();
return path;
}
/**
* 上传图片文件到目标组织
* @param connection 目标组织连接
* @param inputStream 文件数据
* @param title 文件标题
= * @param title 文件标题
* @return 新的ContentVersion ID
* @throws Exception 异常
*/
private String uploadImageFile(PartnerConnection connection, InputStream inputStream, String title) throws Exception {
private String uploadImageFile(PartnerConnection connection, String filePath, String title) throws Exception {
String newId = null;
try {
int failCount = 0;
String token = connection.getSessionHeader().getSessionId();
String uploadUrl = connection.getConfig().getServiceEndpoint() +
String uploadUrl = FILE_UPLOAD_URL +
"/services/data/v55.0/sobjects/ContentVersion";
HttpPost httpPost = new HttpPost(uploadUrl);
Map<String, String> headers = Maps.newHashMap();
headers.put("Authorization", "Bearer " + token);
headers.put("connection", "keep-alive");
httpPost.setHeader("Authorization", "Bearer " + token);
httpPost.setHeader("connection", "keep-alive");
com.alibaba.fastjson.JSONObject credentialsJsonParam = new com.alibaba.fastjson.JSONObject();
credentialsJsonParam.put("title", title);
credentialsJsonParam.put("pathOnClient", title + ".jpg");
credentialsJsonParam.put("pathOnClient", title);
MultipartEntityBuilder builder = MultipartEntityBuilder.create();
builder.addTextBody("entity_content", credentialsJsonParam.toJSONString(), ContentType.APPLICATION_JSON);
builder.addBinaryBody("VersionData", inputStream, ContentType.APPLICATION_OCTET_STREAM, title + ".jpg");
builder.addBinaryBody("VersionData", new FileInputStream(filePath), ContentType.APPLICATION_OCTET_STREAM, title + ".jpg");
HttpEntity entity = builder.build();
httpPost.setEntity(entity);
@ -3019,18 +3098,22 @@ public class DataImportNewServiceImpl implements DataImportNewService {
}
/**
* 获取ContentDocumentId
* 获取ContentDocumentId和OwnerId
* @param connection 连接
* @param contentVersionId ContentVersion ID
* @return ContentDocumentId
* @return 包含ContentDocumentId和OwnerId的Map
* @throws Exception 异常
*/
private String getDocumentId(PartnerConnection connection, String contentVersionId) throws Exception {
String soql = "SELECT ContentDocumentId FROM ContentVersion WHERE Id = '" + contentVersionId + "'";
private Map<String, String> getDocumentId(PartnerConnection connection, String contentVersionId) throws Exception {
String soql = "SELECT ContentDocumentId,OwnerId FROM ContentVersion WHERE Id = '" + contentVersionId + "'";
QueryResult queryResult = connection.query(soql);
Map<String, String> result = new HashMap<>();
if (queryResult.getRecords() != null && queryResult.getRecords().length > 0) {
SObject record = queryResult.getRecords()[0];
return (String) record.getField("ContentDocumentId");
result.put("ContentDocumentId", (String) record.getField("ContentDocumentId"));
result.put("OwnerId", (String) record.getField("OwnerId"));
return result;
}
return null;
}

View File

@ -8,6 +8,7 @@ import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.celnet.datadump.config.SalesforceConnect;
import com.celnet.datadump.config.SalesforceExecutor;
import com.celnet.datadump.config.SalesforceTargetConnect;
import com.celnet.datadump.entity.DataField;
import com.celnet.datadump.entity.DataObject;
import com.celnet.datadump.enums.FileType;
import com.celnet.datadump.global.Const;
@ -16,6 +17,7 @@ import com.celnet.datadump.param.FileTransformParam;
import com.celnet.datadump.global.SystemConfigCode;
import com.celnet.datadump.mapper.CustomMapper;
import com.celnet.datadump.service.*;
import com.celnet.datadump.util.DataUtil;
import com.celnet.datadump.util.EmailUtil;
import com.celnet.datadump.util.HttpUtil;
import com.celnet.datadump.util.OssUtil;
@ -45,10 +47,8 @@ import org.springframework.stereotype.Service;
import java.io.*;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.time.LocalDateTime;
import java.util.*;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
@ -154,10 +154,24 @@ public class FileServiceImpl implements FileService {
extraSql += "AND IsDeleted = false ";
}
if (Const.FILE_TYPE == FileType.SERVER) {
Date defaultBeginDate = DataUtil.DEFAULT_BEGIN_DATE;
Calendar calendar = Calendar.getInstance();
int currentYear = calendar.get(Calendar.YEAR);
calendar.setTime(defaultBeginDate);
int beginYear = calendar.get(Calendar.YEAR);
// 检测路径是否存在 不存在则创建
File excel = new File(Const.SERVER_FILE_PATH + "/" + api);
if (!excel.exists()) {
boolean mkdir = excel.mkdir();
File baseDir = new File(Const.SERVER_FILE_PATH + "/" + api);
if (!baseDir.exists()) {
baseDir.mkdirs();
}
// 创建从开始年份到当前年份的子目录
for (int year = beginYear; year <= currentYear; year++) {
File yearDir = new File(Const.SERVER_FILE_PATH + "/" + api + "" + year);
if (!yearDir.exists()) {
yearDir.mkdir();
}
}
}
try {
@ -171,7 +185,7 @@ public class FileServiceImpl implements FileService {
long num = 0;
while (true) {
// 获取未存储的附件id
List<Map<String, Object>> list = customMapper.list("Id, " + name, api, extraSql + " AND is_dump = 0 limit 1000");
List<Map<String, Object>> list = customMapper.list("Id,CreatedDate," + name, api, extraSql + " AND is_dump = 0 limit 1000");
if (CollectionUtils.isEmpty(list)) {
log.info("无需要下载文件数据!!!");
break;
@ -194,9 +208,12 @@ public class FileServiceImpl implements FileService {
fileName = (String) map.get("Name");
}
log.info("------------文件名:"+id + "_" + fileName);
Object createdDateObj = map.get("CreatedDate");
String year = createdDateObj.toString().substring(0, 4);
// 判断路径是否为空
if (StringUtils.isNotBlank(fileName)) {
String filePath = api + "/" + id + "_" + fileName;
String filePath = api + "-" + year + "/" + id + "_" + fileName;
// 拼接url
String url = finalDownloadUrl + String.format(Const.SF_FILE_URL, api, id, field);
Response response = HttpUtil.doGet(url, null, headers);
@ -468,13 +485,29 @@ public class FileServiceImpl implements FileService {
throw new RuntimeException("找不到文件路径");
}
}
String selectField = "Id, url, PathOnClient, Title, ContentDocumentId, VersionNumber";
QueryWrapper<DataField> wrapper = new QueryWrapper<>();
wrapper.eq("api", api);
wrapper.eq("is_createable",1);
List<DataField> dataFields = dataFieldService.list(wrapper);
ArrayList<String> addFieldList = new ArrayList<>();
for (DataField dataField : dataFields) {
if (!selectField.contains(dataField.getField())){
selectField += "," + dataField.getField();
addFieldList.add(dataField.getField());
}
}
try {
List<Map<String, Object>> documentList = customMapper.list("Id", "ContentDocument", "new_id is null");
for (Map<String, Object> documentMap : documentList) {
boolean index = false;
String documentId = (String) documentMap.get("Id");
// 获取未存储的附件id
List<Map<String, Object>> list = customMapper.list("Id, url, PathOnClient, Title, ContentDocumentId, VersionNumber", api, "ContentDocumentId = '" + documentId + "' and new_id is null ORDER BY VersionNumber ASC");
List<Map<String, Object>> list = customMapper.list(selectField, api, "ContentDocumentId = '" + documentId + "' and new_id is null ORDER BY VersionNumber ASC");
if (CollectionUtils.isEmpty(list)) {
continue;
}
@ -520,6 +553,11 @@ public class FileServiceImpl implements FileService {
if (newDocumentId != null) {
credentialsJsonParam.put("ContentDocumentId", newDocumentId);
}
if (!addFieldList.isEmpty()){
for (String s : addFieldList) {
credentialsJsonParam.put(s, map.get(s));
}
}
MultipartEntityBuilder builder = MultipartEntityBuilder.create();
builder.addTextBody("entity_content", credentialsJsonParam.toJSONString(), ContentType.APPLICATION_JSON);
builder.addBinaryBody("VersionData", new FileInputStream(file), ContentType.APPLICATION_OCTET_STREAM, fileName);
@ -655,9 +693,25 @@ public class FileServiceImpl implements FileService {
throw new RuntimeException("找不到文件路径");
}
}
String selectField = "Id,url,ParentId,Parent_Type,OwnerId,Description,CreatedById,Name";
QueryWrapper<DataField> wrapper = new QueryWrapper<>();
wrapper.eq("api", api);
wrapper.eq("is_createable",1);
List<DataField> dataFields = dataFieldService.list(wrapper);
ArrayList<String> addFieldList = new ArrayList<>();
for (DataField dataField : dataFields) {
if (!selectField.contains(dataField.getField())){
selectField += "," + dataField.getField();
addFieldList.add(dataField.getField());
}
}
try {
// 获取未存储的附件id
List<Map<String, Object>> list = customMapper.list("*", api, "is_upload = 0");
List<Map<String, Object>> list = customMapper.list(selectField, api, "is_upload = 0");
for (Map<String, Object> map : list) {
String finalUploadUrl = uploadUrl;
Future<?> future = salesforceExecutor.execute(() -> {
@ -678,8 +732,7 @@ public class FileServiceImpl implements FileService {
String description = (String) map.get("Description");
String createdById = (String) map.get("CreatedById");
String fileName = (String) map.get("Name");
log.info("id: {}, url_fileName: {}, parentId: {}, parentType: {}, ownerId: {}, description: {}, createdById: {}",
id, url_fileName, parentId, parentType, ownerId, description, createdById);
// 判断路径是否为空
if (StringUtils.isNotBlank(url_fileName)) {
String filePath = Const.SERVER_FILE_PATH + "/" + url_fileName;
@ -771,6 +824,12 @@ public class FileServiceImpl implements FileService {
// credentialsJsonParam.put("CreatedDate", map.get("CreatedDate"));
credentialsJsonParam.put("CreatedById", cMap.get("new_id"));
if (!addFieldList.isEmpty()){
for (String s : addFieldList) {
credentialsJsonParam.put(s, map.get(s));
}
}
MultipartEntityBuilder builder = MultipartEntityBuilder.create();
builder.addTextBody("data", credentialsJsonParam.toJSONString(), ContentType.APPLICATION_JSON);
builder.addBinaryBody("Body", new FileInputStream(file), ContentType.APPLICATION_OCTET_STREAM, fileName);