kris/data-dump
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: kris:84d6fbd4014361e16de97f56cdf73cbc1ae96e83
Branches Tags
kris:master
kris:develop_开发版
kris:develop_20251009_V2
kris:develop_20250826
kris:develop_百思_20250801
kris:develop_亚数_20250731
kris:develop_百思_20250729
kris:develop_携科
kris:develop
kris:master_back
..
compare: kris:2457665b145e7b7107aca68c4866b3a6f47bbd8a
Branches Tags
kris:develop_开发版
kris:develop_20251009_V2
kris:develop_20250826
kris:develop_百思_20250801
kris:develop_亚数_20250731
kris:develop_百思_20250729
kris:develop_携科
kris:develop
kris:master
kris:master_back

No commits in common. "84d6fbd4014361e16de97f56cdf73cbc1ae96e83" and "2457665b145e7b7107aca68c4866b3a6f47bbd8a" have entirely different histories.
84d6fbd401 ... 2457665b14

2 changed files with 84 additions and 244 deletions
Show Stats Expand all files Collapse all files

82
src/main/java/com/celnet/datadump/service/impl/CommonServiceImpl.java
View File

@ -1017,9 +1017,9 @@ public class CommonServiceImpl implements CommonService {
log.info("检测到富文本字段: {}，API: {}", field.getName(), apiName);
dataField.setIsHtmlFormatted(true);
Map<String, Object> htmlMap = Maps.newHashMap();
htmlMap.put("type", "varchar(255)");
htmlMap.put("comment", field.getName()+ "_back");
htmlMap.put("name", "new_id");
htmlMap.put("type", "text");
htmlMap.put("comment", field.getLabel()+ "备份字段");
htmlMap.put("name", field.getName()+ "_back");
list.add(htmlMap);
}
if (field.getReferenceTo().length > 1){
@ -1343,6 +1343,7 @@ public class CommonServiceImpl implements CommonService {
String blobField = null;
List<DataPicklist> dataPicklists = Lists.newArrayList();
for (Field field : dsr.getFields()) {
// 过滤字段
if (Const.TABLE_FILTERS.contains(field.getName())) {
continue;
@ -1362,48 +1363,47 @@ public class CommonServiceImpl implements CommonService {
list.add(map);
DataField dataField = new DataField();
dataField.setApi(apiName);
dataField.setSfType(sfType);
dataField.setField(field.getName());
dataField.setName(field.getLabel());
dataField.setIsCreateable(field.getCreateable());
dataField.setIsUpdateable(field.getUpdateable());
dataField.setIsNillable(field.getNillable());
dataField.setIsDefaultedOnCreate(field.getDefaultedOnCreate());
String join = null;
// 会有非常多映射关系的字段在 这里置空不管
if (field.getReferenceTo().length <= 3) {
join = StringUtils.join(field.getReferenceTo(), ",");
} else {
log.info("referenceTo too long set null, api:{}, field:{}, reference to:{}", apiName, field.getName(), StringUtils.join(field.getReferenceTo(), ","));
}
if (field.getReferenceTo().length > 1){
if (field.getName().contains("Id") && !"OwnerId".equals(field.getName()) && !"UserOrGroupId".equals(field.getName())){
LinkConfig linkConfig = new LinkConfig();
linkConfig.setApi(apiName);
linkConfig.setLabel(label);
linkConfig.setField(field.getName());
linkConfig.setLinkField(StringUtils.replace(field.getName(), "Id", "_Type"));
linkConfigService.save(linkConfig);
}
}
// picklist保存到picklist表
if ("picklist".equalsIgnoreCase(sfType)) {
join = "data_picklist";
PicklistEntry[] picklistValues = field.getPicklistValues();
if (ArrayUtils.isNotEmpty(picklistValues)) {
for (PicklistEntry picklistValue : picklistValues) {
DataPicklist dataPicklist = new DataPicklist();
dataPicklist.setApi(apiName);
dataPicklist.setField(field.getName());
dataPicklist.setLabel(picklistValue.getLabel());
dataPicklist.setValue(picklistValue.getValue());
dataPicklists.add(dataPicklist);
}
}
}
dataField.setReferenceTo(join);
dataField.setReferenceTo(field.getReferenceTo() != null && field.getReferenceTo().length > 0 ? field.getReferenceTo()[0] : null);
dataField.setLeftIndex(field.getLeftSideReferenceTo() != null && field.getLeftSideReferenceTo().length > 0 ? field.getLeftSideReferenceTo()[0] : null);
dataField.setRightIndex(field.getRightSideReferenceTo() != null && field.getRightSideReferenceTo().length > 0 ? field.getRightSideReferenceTo()[0] : null);
dataField.setOnList(field.isListVisible());
dataField.setSfType(sfType);
dataField.setCreateable(field.isCreateable());
dataField.setUpdateable(field.isUpdateable());
dataField.setHtmlFormatted(field.isHtmlFormatted());
dataField.setNillable(field.isNillable());
dataField.setDefaultedOnCreate(field.isDefaultedOnCreate());
fieldList.add(dataField);
fields.add(field.getName());
// 如果是富文本字段，添加备份字段
if (field.isHtmlFormatted()) {
String backupFieldName = field.getName() + "_back";
Map<String, Object> backupMap = Maps.newHashMap();
backupMap.put("type", DataUtil.fieldTypeToMysql(field));
backupMap.put("comment", field.getLabel() + " 备份");
backupMap.put("name", backupFieldName);
list.add(backupMap);
DataField backupDataField = new DataField();
backupDataField.setApi(apiName);
backupDataField.setField(backupFieldName);
backupDataField.setName(field.getLabel() + " 备份");
backupDataField.setReferenceTo(field.getReferenceTo() != null && field.getReferenceTo().length > 0 ? field.getReferenceTo()[0] : null);
backupDataField.setLeftIndex(field.getLeftSideReferenceTo() != null && field.getLeftSideReferenceTo().length > 0 ? field.getLeftSideReferenceTo()[0] : null);
backupDataField.setRightIndex(field.getRightSideReferenceTo() != null && field.getRightSideReferenceTo().length > 0 ? field.getRightSideReferenceTo()[0] : null);
backupDataField.setOnList(field.isListVisible());
backupDataField.setSfType(sfType);
backupDataField.setCreateable(field.isCreateable());
backupDataField.setUpdateable(field.isUpdateable());
backupDataField.setHtmlFormatted(field.isHtmlFormatted());
backupDataField.setNillable(field.isNillable());
backupDataField.setDefaultedOnCreate(field.isDefaultedOnCreate());
fieldList.add(backupDataField);
fields.add(backupFieldName);
}
}
// 存在ownerId字段
String extraField = "";

246
src/main/java/com/celnet/datadump/service/impl/DataImportNewServiceImpl.java
View File

@ -31,7 +31,6 @@ import com.xxl.job.core.util.DateUtil;
import lombok.extern.slf4j.Slf4j;
import okhttp3.Response;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections4.Get;
import org.apache.commons.lang.time.DateUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@ -57,9 +56,6 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.Base64;
@Service
@Slf4j
public class DataImportNewServiceImpl implements DataImportNewService {
@ -968,7 +964,7 @@ public class DataImportNewServiceImpl implements DataImportNewService {
.eq("sync_end_date", endDate)
.set("sf_update_num", targetCount);
dataBatchService.update(updateQw2);
log.info("数据更新完成，API: {}，总更新数: {}，开始时间: {}，结束时间: {}", api, targetCount, beginDateStr, endDateStr);
}
@ -2417,7 +2413,7 @@ public class DataImportNewServiceImpl implements DataImportNewService {
String sql = "";
if (!param.getIds().isEmpty()){
sql = "where Record in (" + param.getIds().stream().map(id -> "'" + id + "'").collect(Collectors.joining(",")) + ")";
sql = "where Record in (" + param.getIds().stream().map(id -> "'" + id + "'").collect(Collectors.joining(",")) + ")";
}else {
Date beginDate = param.getBeginCreateDate();
Date endDate = param.getEndCreateDate();
@ -2667,15 +2663,13 @@ public class DataImportNewServiceImpl implements DataImportNewService {
qw.eq("data_work", true);
List<DataObject> objectList = dataObjectService.list(qw);
Map<String, Object> infoFlag = customMapper.list("code,value","system_config","code ='"+SystemConfigCode.INFO_FLAG+"'").get(0);
for (DataObject dataObject : objectList) {
String api = dataObject.getName();
QueryWrapper<DataField> qw1 = new QueryWrapper<>();
qw1.eq("api", api);
qw1.eq("isHtmlFormatted", true);
List<DataField> fields = dataFieldService.list(qw1);
for (DataField dataField : fields) {
String field = dataField.getField();
String sqlCondition = "new_id is not null and " + field + " like '%<img%refid%'";
@ -2698,15 +2692,11 @@ public class DataImportNewServiceImpl implements DataImportNewService {
int totalPages = (count + pageSize - 1) / pageSize;
for (int page = 0; page < totalPages; page++) {
List<Map<String, Object>> records = customMapper.list("Id, new_id, " + field, api,
sqlCondition + " order by Id limit " + (page * pageSize) + "," + pageSize);
sqlCondition + " order by Id limit " + (page * pageSize) + "," + pageSize);
log.info("处理对象 {} 字段 {} 第 {}/{} 页，本页记录数: {}", api, field, (page + 1), totalPages, records.size());
SObject[] accounts = new SObject[records.size()];
int index = 0;
for (Map<String, Object> record : records) {
try {
String sourceId = (String) record.get("Id");
@ -2719,62 +2709,30 @@ public class DataImportNewServiceImpl implements DataImportNewService {
// 处理富文本中的图片
String updatedContent = processRichTextImages(richTextContent, sourceId, api, field, sourceConnection, targetConnection);
// 如果内容有变化，则更新目标组织中的记录
if (!richTextContent.equals(updatedContent)) {
// 更新目标组织中的记录
SObject sObject = new SObject();
sObject.setType(api);
sObject.setId(targetId);
sObject.setField(field, updatedContent);
accounts[index] = sObject;
index ++;
List<Map<String, Object>> maps = new ArrayList<>();
Map<String, Object> linkMap = new HashMap<>();
linkMap.put("key", field + "_back");
linkMap.put("value", richTextContent);
maps.add(linkMap);
customMapper.updateByNewId(maps,api, targetId);
// 同时更新备份字段
sObject.setField(field + "_back", richTextContent);
SaveResult[] results = targetConnection.update(new SObject[]{sObject});
if (results != null && results.length > 0 && results[0].getSuccess()) {
log.info("成功更新记录 {} 的富文本内容", targetId);
} else {
log.error("更新记录 {} 的富文本内容失败: {}", targetId,
results != null && results.length > 0 ? JSON.toJSONString(results[0].getErrors()) : "未知错误");
}
}
} catch (Exception e) {
log.error("处理记录 {} 的富文本内容时发生错误: {}", record.get("Id"), e.getMessage(), e);
}
}
if (infoFlag != null && "1".equals(infoFlag.get("value"))){
printlnAccountsDetails(accounts,fields);
}
SaveResult[] saveResults = targetConnection.update(accounts);
for (SaveResult saveResult : saveResults) {
if (!saveResult.getSuccess()) {
List<Map<String, Object>> maps = new ArrayList<>();
Map<String, Object> linkMap = new HashMap<>();
linkMap.put("key", "is_update");
linkMap.put("value", 2);
maps.add(linkMap);
Map<String, Object> linkMap1 = new HashMap<>();
linkMap1.put("key", "error_message");
linkMap1.put("value", JSON.toJSONString(saveResult.getErrors()));
maps.add(linkMap1);
customMapper.updateByNewId(maps,api, saveResult.getId());
String format = String.format("数据更新 error, api name: %s, \ncause:\n%s", api, JSON.toJSONString(saveResult));
log.info(format);
}else {
List<Map<String, Object>> maps = new ArrayList<>();
Map<String, Object> linkMap = new HashMap<>();
linkMap.put("key", "is_update");
linkMap.put("value", 1);
maps.add(linkMap);
Map<String, Object> linkMap1 = new HashMap<>();
linkMap1.put("key", "error_message");
linkMap1.put("value", null);
maps.add(linkMap1);
customMapper.updateByNewId(maps,api, saveResult.getId());
}
}
}
}
}
@ -2786,7 +2744,7 @@ public class DataImportNewServiceImpl implements DataImportNewService {
/**
* 处理富文本中的图片
* @param richTextContent 原始富文本内容
* @param sourceRecordId 源记录ID
* @param recordId 记录ID
* @param api 对象API名称
* @param field 字段名
* @param sourceConnection 源组织连接
@ -2794,20 +2752,20 @@ public class DataImportNewServiceImpl implements DataImportNewService {
* @return 处理后的富文本内容
* @throws Exception 异常
*/
private String processRichTextImages(String richTextContent, String sourceRecordId, String api, String field,
PartnerConnection sourceConnection, PartnerConnection targetConnection) throws Exception {
private String processRichTextImages(String richTextContent, String recordId, String api, String field,
PartnerConnection sourceConnection, PartnerConnection targetConnection) throws Exception {
// 匹配富文本中包含refid的img标签
Pattern imgPattern = Pattern.compile("<img[^>]*refid=\"([^\"]+)\"[^>]*/?>", Pattern.CASE_INSENSITIVE);
Pattern imgPattern = Pattern.compile("<img[^>]*refid=\"([^\"]+)\"[^>]*>", Pattern.CASE_INSENSITIVE);
Matcher matcher = imgPattern.matcher(richTextContent);
StringBuffer updatedContent = new StringBuffer();
Map<String, String> imageIdMap = new ConcurrentHashMap<>();
// 遍历所有匹配的img标签
while (matcher.find()) {
String imgTag = matcher.group(0);
String refId = matcher.group(1);
// 检查是否已经处理过这个refId
if (imageIdMap.containsKey(refId)) {
String newRefId = imageIdMap.get(refId);
@ -2815,61 +2773,27 @@ public class DataImportNewServiceImpl implements DataImportNewService {
matcher.appendReplacement(updatedContent, Matcher.quoteReplacement(updatedImgTag));
continue;
}
try {
// 查询源组织中的ContentDocumentLink记录
String cdlQuery = "SELECT Id, ContentDocumentId FROM ContentDocumentLink WHERE LinkedEntityId = '" + sourceRecordId + "' AND ContentDocumentId = '" + refId + "'";
String cdlQuery = "SELECT Id, ContentDocumentId FROM ContentDocumentLink WHERE LinkedEntityId = '" + recordId + "' AND ContentDocumentId = '" + refId + "'";
QueryResult cdlResult = sourceConnection.query(cdlQuery);
if (cdlResult.getRecords() != null && cdlResult.getRecords().length > 0) {
// 构建源图片下载URL (参考Python脚本中的imageUrl构建方式)
String downloadUrl = sourceConnection.getConfig().getServiceEndpoint() +
String.format("/services/data/v55.0/sobjects/%s/%s/richTextImageFields/%s/%s",
api, sourceRecordId, field, refId);
// 下载源图片文件
InputStream inputStream = downloadImageFile(sourceConnection, downloadUrl);
if (inputStream != null) {
// 获取图片的alt属性作为标题
String title = "image";
Pattern altPattern = Pattern.compile("alt=\"([^\"]*)\"", Pattern.CASE_INSENSITIVE);
Matcher altMatcher = altPattern.matcher(imgTag);
if (altMatcher.find()) {
title = altMatcher.group(1);
}
// 上传图片到目标组织
String newContentVersionId = uploadImageFile(targetConnection, inputStream, title);
if (newContentVersionId != null) {
// 获取新的ContentDocumentId
String newContentDocumentId = getDocumentId(targetConnection, newContentVersionId);
if (newContentDocumentId != null) {
imageIdMap.put(refId, newContentDocumentId);
// 构建新的img标签 (参考Python脚本中的richTextUrl构建方式)
String newImageUrl = targetConnection.getConfig().getServiceEndpoint() +
"/sfc/servlet.shepherd/version/download/" + newContentVersionId;
String updatedImgTag = "<img src=\"" + newImageUrl + "\" refid=\"" + newContentDocumentId + "\"/>";
matcher.appendReplacement(updatedContent, Matcher.quoteReplacement(updatedImgTag));
// 记录替换日志
richTextLogService.logRichTextImageReplace(
api,
field,
newContentDocumentId,
null, // recordOwnerId 可以从记录中获取，但此处简化处理
richTextContent,
refId, // 原图片链接
newImageUrl, // 新图片链接
newContentDocumentId // 新文件ID
);
continue;
}
// 获取ContentVersion ID
String contentVersionId = getContentVersionId(sourceConnection, refId);
if (contentVersionId != null) {
// 获取目标组织中对应的ContentDocumentId
String newContentDocumentId = getTargetContentDocumentId(targetConnection, contentVersionId);
if (newContentDocumentId != null) {
imageIdMap.put(refId, newContentDocumentId);
String updatedImgTag = imgTag.replace("refid=\"" + refId + "\"", "refid=\"" + newContentDocumentId + "\"");
matcher.appendReplacement(updatedContent, Matcher.quoteReplacement(updatedImgTag));
continue;
}
}
}
// 如果找不到对应关系，保留原始标签
matcher.appendReplacement(updatedContent, Matcher.quoteReplacement(imgTag));
} catch (Exception e) {
@ -2878,94 +2802,10 @@ public class DataImportNewServiceImpl implements DataImportNewService {
}
}
matcher.appendTail(updatedContent);
return updatedContent.toString();
}
/**
* 从源组织下载图片文件
* @param connection 源组织连接
* @param downloadUrl 下载URL
* @return 图片文件数据
* @throws Exception 异常
*/
private InputStream downloadImageFile(PartnerConnection connection, String downloadUrl) throws Exception {
InputStream inputStream = null;
try {
String token = connection.getSessionHeader().getSessionId();
Map<String, String> headers = Maps.newHashMap();
headers.put("Authorization", "Bearer " + token);
headers.put("connection", "keep-alive");
Response response = HttpUtil.doGet(downloadUrl, null, headers);
assert response.body() != null;
inputStream = response.body().byteStream();
} catch (Exception e) {
log.error("下载图片文件时发生错误，URL: {}", downloadUrl, e);
return null;
}
return inputStream;
}
/**
* 上传图片文件到目标组织
* @param connection 目标组织连接
* @param inputStream 文件数据
* @param title 文件标题
* @return 新的ContentVersion ID
* @throws Exception 异常
*/
private String uploadImageFile(PartnerConnection connection, InputStream inputStream, String title) throws Exception {
String newId = null;
try {
int failCount = 0;
String token = connection.getSessionHeader().getSessionId();
String uploadUrl = connection.getConfig().getServiceEndpoint() +
"/services/data/v55.0/sobjects/ContentVersion";
HttpPost httpPost = new HttpPost(uploadUrl);
Map<String, String> headers = Maps.newHashMap();
headers.put("Authorization", "Bearer " + token);
headers.put("connection", "keep-alive");
com.alibaba.fastjson.JSONObject credentialsJsonParam = new com.alibaba.fastjson.JSONObject();
credentialsJsonParam.put("title", title);
credentialsJsonParam.put("pathOnClient", title + ".jpg");
MultipartEntityBuilder builder = MultipartEntityBuilder.create();
builder.addTextBody("entity_content", credentialsJsonParam.toJSONString(), ContentType.APPLICATION_JSON);
builder.addBinaryBody("VersionData", inputStream, ContentType.APPLICATION_OCTET_STREAM, title + ".jpg");
HttpEntity entity = builder.build();
httpPost.setEntity(entity);
CloseableHttpClient httpClient = HttpClients.createDefault();
CloseableHttpResponse response = null;
String respContent = null;
while (failCount < Const.MAX_FAIL_COUNT) {
response = httpClient.execute(httpPost);
if (response != null && response.getStatusLine() != null && response.getStatusLine().getStatusCode() < 400 && response.getEntity() != null) {
HttpEntity he = response.getEntity();
respContent = EntityUtils.toString(he, "UTF-8");
newId = String.valueOf(com.alibaba.fastjson.JSONObject.parseObject(respContent).get("id"));
break;
} else {
failCount++;
log.error("富文本图片替换图片文件上传失败，返回体信息：" + JSON.toJSONString(response));
}
}
} catch (Exception e) {
log.error("上传图片文件时发生错误，标题: {}", title, e);
return null;
}
return newId;
}
/**
* 获取ContentVersion ID
* @param connection Salesforce连接
@ -2981,7 +2821,7 @@ public class DataImportNewServiceImpl implements DataImportNewService {
}
return null;
}
/**
* 获取目标组织中的ContentDocumentId
* @param connection 目标组织连接
@ -3106,7 +2946,7 @@ public class DataImportNewServiceImpl implements DataImportNewService {
JSONObject jsonObject = objects.getJSONObject(i);
String id = jsonObject.getString(Const.ID);
List<Map<String, Object>> maps = buildUpdateMaps(api, jsonObject, stringList);
if (maps!=null && !maps.isEmpty()) {
log.info("准备更新记录，API: {}, ID: {}, 匹配字段信息: {}", api, id, maps);
customMapper.updateNewIdByFields(api, maps, id);
@ -3121,7 +2961,7 @@ public class DataImportNewServiceImpl implements DataImportNewService {
for (String key : stringList) {
Map<String, Object> paramMap = null;
switch (api) {
case "AccountContactRelation":
paramMap = handleAccountContactRelation(key, jsonObject);
@ -3158,12 +2998,12 @@ public class DataImportNewServiceImpl implements DataImportNewService {
log.info("普通字段处理，API: {}, 字段: {}, 值: {}", api, key, jsonObject.getString(key));
break;
}
if (!paramMap.isEmpty()) {
maps.add(paramMap);
}
}
return maps;
}