kris/data-dump
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

【feat】20250819版本

Browse Source
This commit is contained in:
Kris 2025-08-19 11:28:14 +08:00
parent b2610842d9
commit 12e4a5a5e5
20 changed files with 1359 additions and 654 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
src/main/java/com/celnet/datadump/controller/JobController.java
View File

@ -19,6 +19,8 @@ import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.util.List;
/**
* @author Red
* @description
@ -297,17 +299,33 @@ public class JobController {
// 参数转换
param.setBeginCreateDate(param.getBeginDate());
param.setEndCreateDate(param.getEndDate());
switch (param.getApi()){
case "Contact":
return dataImportNewService.getPersonContact(param);
case "User":
// return dataImportNewService.getUser(param);
return ReturnT.FAIL;
default:
log.info("当前对象：{}，不存在反写NewId逻辑！！！" , param.getApi());
return ReturnT.FAIL;
if (StringUtils.isBlank(param.getApi())) {
return new ReturnT<>(500, "api不能为空！");
}
List<String> apis = DataUtil.toIdList(param.getApi());
for (String api : apis) {
switch (api){
case "Contact":
dataImportNewService.getPersonContact(param);
case "User":
case "BrandTemplate":
case "EmailTemplate":
case "Folder":
case "Group":
case "RecordType":
case "AccountContactRelation":
case "TaskRelation":
case "EventRelation":
case "CollaborationGroupMember":
case "FeedAttachment":
dataImportNewService.getNewIdByField(api,null);
default:
log.info("当前对象：{}，不存在反写NewId逻辑！！！" , param.getApi());
}
}
return ReturnT.SUCCESS;
}
/**

48
src/main/java/com/celnet/datadump/job/DataDumpNewJob.java
View File

@ -17,6 +17,8 @@ import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.List;
/**
* 迁移任务 (新)
* 2024/06/12
@ -89,13 +91,13 @@ public class DataDumpNewJob {
}
/**
* 写入个人客户联系人old_id，返写new_id
* 返写NewId
* @param paramStr 参数json
* @return result
*/
@XxlJob("getPersonContactJob")
public ReturnT<String> getPersonContactJob(String paramStr) throws Exception {
log.info("getPersonContactJob execute start ..................");
@XxlJob("getNewId")
public ReturnT<String> getNewId(String paramStr) throws Exception {
log.info("getNewId execute start ..................");
SalesforceParam param = new SalesforceParam();
try {
if (StringUtils.isNotBlank(paramStr)) {
@ -108,7 +110,43 @@ public class DataDumpNewJob {
param.setBeginCreateDate(param.getBeginDate());
param.setEndCreateDate(param.getEndDate());
return dataImportNewService.getPersonContact(param);
if (StringUtils.isBlank(param.getApi())) {
return new ReturnT<>(500, "api不能为空！");
}
List<String> apis = DataUtil.toIdList(param.getApi());
for (String api : apis) {
switch (api){
case "Contact":
dataImportNewService.getPersonContact(param);
case "User":
dataImportNewService.getNewIdByField(api,"Username");
case "BrandTemplate":
case "EmailTemplate":
case "Folder":
dataImportNewService.getNewIdByField(api,"DeveloperName");
case "Group":
dataImportNewService.getNewIdByField(api,"DeveloperName,Type");
case "RecordType":
dataImportNewService.getNewIdByField(api,"DeveloperName,SObjectType");
case "AccountContactRelation":
dataImportNewService.getNewIdByField(api,"AccountId,ContactId,IsPrimary");
case "TaskRelation":
dataImportNewService.getNewIdByField(api,"TaskId,RelationId,RelationType");
case "EventRelation":
dataImportNewService.getNewIdByField(api,"EventId,RelationId,RelationType");
case "CollaborationGroupMember":
dataImportNewService.getNewIdByField(api,"CollaborationGroupId,MemberId,CollaborationRole");
case "FeedAttachment":
dataImportNewService.getNewIdByField(api,"FeedItemId,AttachmentId");
case "Task":
dataImportNewService.getTaskNewId(param);
default:
log.info("当前对象：{}，不存在反写NewId逻辑！！！" , param.getApi());
}
}
return ReturnT.SUCCESS;
}
/**

9
src/main/java/com/celnet/datadump/mapper/CustomMapper.java
View File

@ -82,6 +82,14 @@ public interface CustomMapper {
*/
public void update(@Param("maps") List<Map<String, Object>> maps, @Param("tableName") String api, @Param("sql") String sql);
/**
* 根据字段进行更新
* @param maps
* @param api
* @param newId
*/
public void updateByField(@Param("maps") List<Map<String, Object>> maps, @Param("tableName") String api, @Param("newId") String newId);
/**
* 插入方法
*
@ -139,6 +147,7 @@ public interface CustomMapper {
*/
public Map<String, Object> getById(@Param("select") String select, @Param("api") String api, @Param("id") String id);
public Map<String, Object> getByNewId(@Param("select") String select, @Param("api") String api, @Param("id") String id);
public Integer countBySQL(@Param("api") String api, @Param("sql") String sql);

3
src/main/java/com/celnet/datadump/mapper/SalesforceMapper.java
View File

@ -33,4 +33,7 @@ public interface SalesforceMapper {
*/
public void listOrderById(@Param("param") SalesforceParam param);
public void listOrderByIdNew(@Param("param") SalesforceParam param);
}

8
src/main/java/com/celnet/datadump/param/DataDumpByParentParam.java
View File

@ -2,6 +2,8 @@ package com.celnet.datadump.param;
import lombok.Data;
import java.util.Date;
/**
* @author Red
* @description
@ -28,4 +30,10 @@ public class DataDumpByParentParam {
* 父类字段
*/
private String parentField;
private Date beginModifyDate;
private String type;
}

7
src/main/java/com/celnet/datadump/service/CommonService.java
View File

@ -68,12 +68,5 @@ public interface CommonService {
*/
ReturnT<String> createLinkTypeField(SalesforceParam param) throws Exception;
/**
* 更新关联类型
* @param param 参数
* @return ReturnT
* @throws Exception exception
*/
ReturnT<String> updateLinkType(SalesforceParam param) throws Exception;
}

4
src/main/java/com/celnet/datadump/service/DataDumpSpecialService.java
View File

@ -1,5 +1,6 @@
package com.celnet.datadump.service;
import com.celnet.datadump.entity.DataObject;
import com.celnet.datadump.param.DataDumpSpecialParam;
import com.sforce.soap.partner.PartnerConnection;
import com.xxl.job.core.biz.model.ReturnT;
@ -15,4 +16,7 @@ public interface DataDumpSpecialService {
ReturnT<String> dataDumpSpecial(DataDumpSpecialParam param) throws Throwable;
Future<?> getData(DataDumpSpecialParam param, PartnerConnection connect);
Future<?> getDataNew(DataDumpSpecialParam param, PartnerConnection connect, DataObject dataObject);
}

6
src/main/java/com/celnet/datadump/service/DataFieldService.java
View File

@ -22,4 +22,10 @@ public interface DataFieldService extends IService<DataField> {
public void tableRelation();
String returnDateField(String api);
String returnUpdateDateField(String api);
Boolean hasCreatedDate(String api);
}

9
src/main/java/com/celnet/datadump/service/DataImportNewService.java
View File

@ -4,12 +4,9 @@ import com.celnet.datadump.entity.DataObject;
import com.celnet.datadump.param.SalesforceParam;
import com.xxl.job.core.biz.model.ReturnT;
import java.util.List;
import java.util.concurrent.Future;
public interface DataImportNewService {
ReturnT<String> getPersonContact(SalesforceParam param) throws Exception;
void getPersonContact(SalesforceParam param) throws Exception;
ReturnT<String> immigrationUpdateNew(SalesforceParam param) throws Exception;
@ -31,4 +28,8 @@ public interface DataImportNewService {
ReturnT<String> dumpContentFolderMemberJob(SalesforceParam param) throws Exception;
void getNewIdByField(String api, String queryFields) throws Exception;
void getTaskNewId(SalesforceParam param) throws Exception;
}

77
src/main/java/com/celnet/datadump/service/impl/CommonServiceImpl.java
View File

@ -214,6 +214,15 @@ public class CommonServiceImpl implements CommonService {
dataReport.setApis(list.stream().map(DataObject::getName).collect(Collectors.joining(",")));
dataReportService.save(dataReport);
for (DataObject dataObject : list) {
//无创建时间对象
if (!hasCreatedDate(dataObject.getName())) {
DataDumpSpecialParam dataDumpSpecialParam = new DataDumpSpecialParam();
dataDumpSpecialParam.setApi(dataObject.getName());
Future<?> future = dataDumpSpecialService.getDataNew(dataDumpSpecialParam, salesforceConnect.createConnect(),dataObject);
// 等待当前所有线程执行完成
salesforceExecutor.waitForFutures(future);
continue;
}
Future<?> future = salesforceExecutor.execute(() -> {
try {
Date updateTime = new Date();
@ -226,7 +235,6 @@ public class CommonServiceImpl implements CommonService {
salesforceParam.setUpdateField(dataObject.getUpdateField());
}
dumpDataNew(salesforceParam, dataReport,dataObject);
dataObject.setLastUpdateDate(updateTime);
dataObjectService.updateById(dataObject);
@ -988,7 +996,7 @@ public class CommonServiceImpl implements CommonService {
log.info("referenceTo too long set null, api:{}, field:{}, reference to:{}", apiName, field.getName(), StringUtils.join(field.getReferenceTo(), ","));
}
if (field.getReferenceTo().length > 1){
if (field.getName().contains("Id") && !"OwnerId".equals(field.getName())){
if (field.getName().contains("Id") && !"OwnerId".equals(field.getName()) && !"UserOrGroupId".equals(field.getName())){
LinkConfig linkConfig = new LinkConfig();
linkConfig.setApi(apiName);
linkConfig.setLabel(label);
@ -1134,7 +1142,7 @@ public class CommonServiceImpl implements CommonService {
update.setName(apiName);
update.setLastUpdateDate(endCreateDate);
update.setBlobField(blobField);
if (count>0){
if (count>0 || apiName.endsWith("Share")){
update.setIsEditable(false);
}
update.setKeyPrefix(dsr.getKeyPrefix());
@ -1202,6 +1210,8 @@ public class CommonServiceImpl implements CommonService {
return one != null;
}
/**
* 存在附件的表 特殊构建
*
@ -1293,7 +1303,7 @@ public class CommonServiceImpl implements CommonService {
//表内数据总量
Integer count = customMapper.countBySQL(api, "where ShareType = 'V' and new_id = '0'");
//批量插入200一次
int page = count % 200 == 0 ? count / 200 : (count / 200) + 1;
for (int i = 0; i < page; i++) {
List<Map<String, Object>> linkList = customMapper.list("Id,LinkedEntityId,ContentDocumentId,LinkedEntity_Type,ShareType,Visibility", api, "ShareType = 'V' and new_id = '0' order by Id asc limit 200");
@ -1416,64 +1426,7 @@ public class CommonServiceImpl implements CommonService {
linkConfigService.updateById(config);
}
}catch (Exception e){
log.error(e.getMessage());
}
return ReturnT.SUCCESS;
}
@Override
public ReturnT<String> updateLinkType(SalesforceParam param) throws Exception {
try {
List<String> apis;
if (StringUtils.isBlank(param.getApi())) {
apis = dataObjectService.list().stream().map(DataObject::getName).collect(Collectors.toList());
} else {
apis = DataUtil.toIdList(param.getApi());
}
QueryWrapper<LinkConfig> wrapper = new QueryWrapper<>();
wrapper.eq("is_link","1").eq("is_create",1);
if (!apis.isEmpty()){
wrapper.in("api", apis);
}
Map<String, String> resultMap = dataObjectService.list().stream()
.collect(Collectors.toMap(
DataObject::getKeyPrefix,
DataObject::getName,
(existing, replacement) -> replacement));
List<LinkConfig> list = linkConfigService.list(wrapper);
for (LinkConfig config : list) {
//若是该对象已经存在数据，则遍历写入值
Integer dbNum = customMapper.countBySQL(config.getApi(),null);
log.info("表api：{} 存在" +dbNum+ "条数据！赋值当前表多态类型字段：{}", config.getApi(),config.getLinkField());
if (dbNum >0 ) {
int page = dbNum % 2000 == 0 ? dbNum / 2000 : (dbNum / 2000) + 1;
for (int i = 0; i < page; i++) {
log.info("表api：{}，字段：{}，批次：{}，执行数据更新!", config.getApi(),config.getLinkField(), i);
List<Map<String, Object>> mapList = customMapper.list("*", config.getApi(), "1=1 order by Id limit " + i * 2000 + ",2000");
List<Map<String, Object>> updateMapList = new ArrayList<>();
for (int j = 1; j <= mapList.size(); j++) {
Map<String, Object> map = mapList.get(j - 1);
if (map.get(config.getField()) != null){
String type = resultMap.get(map.get(config.getField()).toString().substring(0, 3));
Map<String, Object> paramMap = Maps.newHashMap();
paramMap.put("key", config.getLinkField());
paramMap.put("value", type);
updateMapList.add(paramMap);
customMapper.updateById(config.getApi(), updateMapList, String.valueOf(mapList.get(j - 1).get("Id") != null?mapList.get(j - 1).get("Id") : mapList.get(j - 1).get("id")));
}
}
}
}
}
}catch (Exception e){
log.error(e.getMessage());
log.error(JSON.toJSONString(e));
}
return ReturnT.SUCCESS;
}

19
src/main/java/com/celnet/datadump/service/impl/DataDumpByParentServiceImpl.java
View File

@ -3,13 +3,16 @@ package com.celnet.datadump.service.impl;
import com.alibaba.fastjson2.JSON;
import com.alibaba.fastjson2.JSONArray;
import com.alibaba.fastjson2.JSONObject;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.celnet.datadump.config.SalesforceConnect;
import com.celnet.datadump.entity.DataObject;
import com.celnet.datadump.global.Const;
import com.celnet.datadump.param.DataDumpByParentParam;
import com.celnet.datadump.param.SalesforceParam;
import com.celnet.datadump.mapper.CustomMapper;
import com.celnet.datadump.service.CommonService;
import com.celnet.datadump.service.DataDumpByParentService;
import com.celnet.datadump.service.DataObjectService;
import com.celnet.datadump.util.DataUtil;
import com.celnet.datadump.util.EmailUtil;
import com.celnet.datadump.util.SqlUtil;
@ -29,6 +32,7 @@ import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
@ -48,6 +52,8 @@ public class DataDumpByParentServiceImpl implements DataDumpByParentService {
private CommonService commonService;
@Autowired
private SalesforceConnect salesforceConnect;
@Autowired
private DataObjectService dataObjectService;
@Override
public ReturnT<String> dataDumpByParent(DataDumpByParentParam param) throws Throwable {
@ -62,6 +68,15 @@ public class DataDumpByParentServiceImpl implements DataDumpByParentService {
// 检测子表 不存在就生成 但不生成批次
commonService.checkApi(param.getApi(), false);
// 遍历父表
if (!param.getType().equals("1")) {
QueryWrapper<DataObject> qw = new QueryWrapper<>();
qw.eq("name", param.getApi());
DataObject update = dataObjectService.getOne(qw);
param.setBeginModifyDate(update.getLastUpdateDate());
update.setLastUpdateDate(new Date());
dataObjectService.updateById(update);
}
while (true) {
List<String> strings = customMapper.listById(param.getParentField(), param.getParentApi(), "order by " + param.getParentField() + " asc limit " + page + "," + size);
if (CollectionUtils.isEmpty(strings)) {
@ -72,6 +87,7 @@ public class DataDumpByParentServiceImpl implements DataDumpByParentService {
Map<String, Object> map = Maps.newHashMap();
SalesforceParam salesforceParam = new SalesforceParam();
salesforceParam.setApi(param.getApi());
salesforceParam.setBeginModifyDate(param.getBeginModifyDate());
salesforceParam.setSql(param.getField() + " IN ('" + StringUtils.join(strings, "','") + "')");
String maxId = null;
Field[] dsrFields;
@ -93,11 +109,12 @@ public class DataDumpByParentServiceImpl implements DataDumpByParentService {
int count = 0, failCount = 0;
// 遍历子表
while (true) {
try {
// 判断是否存在要排除的id
salesforceParam.setMaxId(maxId);
map.put("param", salesforceParam);
String sql = SqlUtil.showSql("com.celnet.datadump.mapper.SalesforceMapper.listOrderById", map);
String sql = SqlUtil.showSql("com.celnet.datadump.mapper.SalesforceMapper.listOrderByIdNew", map);
log.info("query sql: {}", sql);
XxlJobLogger.log("query sql: {}", sql);
QueryResult queryResult = connect.queryAll(sql);

76
src/main/java/com/celnet/datadump/service/impl/DataDumpSpecialServiceImpl.java
View File

@ -219,6 +219,82 @@ public class DataDumpSpecialServiceImpl implements DataDumpSpecialService {
}, 0, 0);
}
@Override
public Future<?> getDataNew(DataDumpSpecialParam param, PartnerConnection connect,DataObject dataObject) {
String api = param.getApi();
return salesforceExecutor.execute(() -> {
try {
Map<String, Object> map = Maps.newHashMap();
SalesforceParam salesforceParam = new SalesforceParam();
salesforceParam.setApi(api);
salesforceParam.setIdField(param.getField());
String maxId = null;
Field[] dsrFields;
List<String> fields = Lists.newArrayList();
// 获取sf字段
{
DescribeSObjectResult dsr = connect.describeSObject(api);
dsrFields = dsr.getFields();
for (Field field : dsrFields) {
// 不查询文件
if ("base64".equalsIgnoreCase(field.getType().toString())) {
continue;
}
fields.add(field.getName());
}
salesforceParam.setSelect(StringUtils.join(fields, ","));
}
JSONArray objects = null;
int count = 0, failCount = 0;
// 遍历
while (true) {
try {
// 判断是否存在要排除的id
salesforceParam.setMaxId(maxId);
salesforceParam.setBeginModifyDate(dataObject.getLastUpdateDate());
map.put("param", salesforceParam);
String sql = SqlUtil.showSql("com.celnet.datadump.mapper.SalesforceMapper.listOrderByIdNew", map);
log.info("query sql: {}", sql);
XxlJobLogger.log("query sql: {}", sql);
QueryResult queryResult = connect.queryAll(sql);
if (ObjectUtils.isEmpty(queryResult) || ObjectUtils.isEmpty(queryResult.getRecords())) {
break;
}
SObject[] records = queryResult.getRecords();
objects = DataUtil.toJsonArray(records, dsrFields);
maxId = ((JSONObject) objects.get(objects.size() - 1)).getString(Const.ID);
// 存储更新
commonService.saveOrUpdate(api, fields, records, objects, true);
count += records.length;
TimeUnit.MILLISECONDS.sleep(1);
log.info("dump success count: {}", count);
XxlJobLogger.log("dump success count: {}", count);
failCount = 0;
objects = null;
} catch (InterruptedException e) {
throw e;
} catch (Throwable throwable) {
failCount++;
log.error("dataDumpSpecial error api:{}, data:{}", api, JSON.toJSONString(objects), throwable);
if (failCount > Const.MAX_FAIL_COUNT) {
throwable.addSuppressed(new Exception("dataDumpSpecial error data:" + JSON.toJSONString(objects)));
throw throwable;
}
TimeUnit.MINUTES.sleep(1);
}
}
updateDataBatch(connect, api);
dataObject.setLastUpdateDate(new Date());
dataObjectService.updateById(dataObject);
} catch (Throwable throwable) {
String format = String.format("数据特殊表迁移 error, api name: %s, \nparam: %s, \ncause:\n%s", api, JSON.toJSONString(param), throwable);
EmailUtil.send("DataDump ERROR", format);
log.error("dataDumpSpecial error ", throwable);
throw new RuntimeException(throwable);
}
}, 0, 0);
}
/**
* 更新batch
*

50
src/main/java/com/celnet/datadump/service/impl/DataFieldServiceImpl.java
View File

@ -5,6 +5,7 @@ import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.celnet.datadump.entity.DataField;
import com.celnet.datadump.global.Const;
import com.celnet.datadump.mapper.DataFieldMapper;
import com.celnet.datadump.service.DataFieldService;
import org.springframework.beans.factory.annotation.Autowired;
@ -23,6 +24,7 @@ import java.util.*;
@Service
public class DataFieldServiceImpl extends ServiceImpl<DataFieldMapper, DataField> implements DataFieldService {
@Override
public Boolean hasDeleted(String api) {
QueryWrapper<DataField> qw = new QueryWrapper<>();
@ -60,7 +62,55 @@ public class DataFieldServiceImpl extends ServiceImpl<DataFieldMapper, DataField
System.out.println("自循环表："+JSONObject.toJSONString(owner));
}
@Override
public String returnDateField(String api) {
QueryWrapper<DataField> queryWrapper = new QueryWrapper<>();
queryWrapper.eq("api", api);
List<DataField> list = list(queryWrapper);
Set<String> fieldNames = new HashSet<>();
for (DataField dataField : list) {
fieldNames.add(dataField.getField());
}
if (fieldNames.contains("CreatedDate")) {
return "CreatedDate";
} else if (fieldNames.contains("SystemModstamp")) {
return "SystemModstamp";
} else if (fieldNames.contains("LastModifiedDate")) {
return "LastModifiedDate";
} else {
return "";
}
}
@Override
public String returnUpdateDateField(String api) {
QueryWrapper<DataField> queryWrapper = new QueryWrapper<>();
queryWrapper.eq("api", api);
List<DataField> list = list(queryWrapper);
Set<String> fieldNames = new HashSet<>();
for (DataField dataField : list) {
fieldNames.add(dataField.getField());
}
if (fieldNames.contains("SystemModstamp")) {
return "SystemModstamp";
} else if (fieldNames.contains("LastModifiedDate")) {
return "LastModifiedDate";
} else {
return "";
}
}
@Override
public Boolean hasCreatedDate(String api) {
QueryWrapper<DataField> qw = new QueryWrapper<>();
qw.eq("api", api).eq("field", Const.CREATED_DATE).last("limit 1");
DataField one = getOne(qw);
return one != null;
}
public void getTableLevel(int[][] connections,List<DataField> list) {

131
src/main/java/com/celnet/datadump/service/impl/DataImportBatchServiceImpl.java
View File

@ -376,7 +376,6 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
if (StringUtils.isNotBlank(linkfield)){
reference = data.get(j-1).get(linkfield)!=null?data.get(j-1).get(linkfield).toString():null;
}
if (reference == null){
continue;
}
@ -419,8 +418,6 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
}
JobInfo salesforceInsertJob = null;
JobInfo salesforceQueryJob = null;
try {
salesforceInsertJob = BulkUtil.createJob(bulkConnection, api, OperationEnum.insert);
@ -428,40 +425,13 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
BulkUtil.awaitCompletion(bulkConnection, salesforceInsertJob, batchInfos);
List<String> resultIds = checkInsertResults(bulkConnection, salesforceInsertJob, batchInfos, api, ids);
String idStr = "(";
for (String id : resultIds) {
idStr += "'" + id + "',"; // 拼接每个ID
}
if (idStr.endsWith(",")) { // 如果最后一个字符是逗号，说明循环正常结束
idStr = idStr.substring(0, idStr.length() - 1); // 去掉最后一个多余的逗号
}
idStr += ")"; // 添加右括号
String query = "SELECT Id,old_sfdc_id__c FROM " + api + " where Id in " + idStr;
salesforceQueryJob = BulkUtil.createJob(bulkConnection, api, OperationEnum.queryAll);
ByteArrayInputStream bout = new ByteArrayInputStream(query.getBytes());
BatchInfo info = bulkConnection.createBatchFromStream(salesforceQueryJob, bout);
List<BatchInfo> batchQueryInfos = new ArrayList<>();
batchQueryInfos.add(info);
BulkUtil.awaitCompletion(bulkConnection, salesforceQueryJob, batchQueryInfos);
sfNum = sfNum + checkQueryResultsNew(bulkConnection, salesforceQueryJob, info, api);
sfNum = sfNum + checkInsertResults(bulkConnection, salesforceInsertJob, batchInfos, api, ids);
} catch (Exception e) {
log.error("createdNewIdBatch error api:{}", api, e);
throw e;
}finally {
BulkUtil.closeJob(bulkConnection, salesforceInsertJob.getId());
BulkUtil.closeJob(bulkConnection, salesforceQueryJob.getId());
}
}
@ -484,10 +454,10 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
/**
* 读写Insert结果
*/
public List<String> checkInsertResults(BulkConnection connection, JobInfo job,
public int checkInsertResults(BulkConnection connection, JobInfo job,
List<BatchInfo> batchInfoList,String api,String[] ids)
throws AsyncApiException, IOException {
ArrayList<String> strings = new ArrayList<>();
int index = 0;
// batchInfoList was populated when batches were created and submitted
for (BatchInfo b : batchInfoList) {
CSVReader rdr =
@ -497,7 +467,7 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
List<String> row;
while ((row = rdr.nextRecord()) != null) {
Map<String, String> resultInfo = new HashMap<>();
Map<String, String> resultInfo = new HashMap<String, String>();
for (int i = 0; i < resultCols; i++) {
resultInfo.put(resultHeader.get(i), row.get(i));
}
@ -506,12 +476,22 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
String id = resultInfo.get("Id");
String error = resultInfo.get("Error");
if (insertStatus && created) {
strings.add(id);
List<Map<String, Object>> maps = new ArrayList<>();
Map<String, Object> m = new HashMap<>();
m.put("key", "new_id");
m.put("value", id);
maps.add(m);
customMapper.updateById(api, maps, ids[index]);
log.info("Created Success row with id " + id);
} else if (!insertStatus) {
log.error("Created Fail with error: " + error);
}
index ++;
}
}
return index;
}
return strings;
}
public Integer checkQueryResultsNew(BulkConnection connection, JobInfo job, BatchInfo batchInfo,String api) throws AsyncApiException, IOException {
int num = 0;
@ -787,7 +767,8 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
log.error("总Update数据 count:{}；总批次:{}；-开始时间:{}；-结束时间:{}；-api:{}；", count,page, beginDateStr, endDateStr, api);
for (int i = 0; i < page; i++) {
List<Map<String, Object>> mapList = customMapper.list("*", api, "new_id is not null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "' order by Id asc limit " + i * 10000 + ",10000");
List<Map<String, Object>> mapList = customMapper.list("*", api, "new_id is not null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "' order by Id asc limit " + i * 5000 + ",5000");
log.error("总Update数据 count:{}；当前批次:{}；-开始时间:{}；-结束时间:{}；-api:{}；", count,i, beginDateStr, endDateStr, api);
@ -812,18 +793,18 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
} else if ("reference".equals(dataField.getSfType()) && map.get(dataField.getField()) != null ) {
if (!"null".equals(value) && StringUtils.isNotEmpty(value)) {
//判断reference_to内是否包含User字符串
if (dataField.getReferenceTo() != null && (dataField.getReferenceTo().contains(",User") || dataField.getReferenceTo().contains("User,"))) {
reference_to = "User";
}
//引用类型字段
String linkfield = fieldMap.get(dataField.getField());
if (StringUtils.isNotBlank(linkfield)){
reference_to = map.get(linkfield)!=null?map.get(linkfield).toString():null;
if (StringUtils.isNotBlank(linkfield) && map.get(linkfield)!=null){
reference_to = map.get(linkfield).toString();
}
if (reference_to == null){
continue;
}
//判断reference_to内是否包含User字符串
if (reference_to.contains(",User") || reference_to.contains("User,")) {
reference_to = "User";
}
Map<String, Object> m = customMapper.getById("new_id", reference_to, value);
if (m != null && !m.isEmpty()) {
account.put(field, m.get("new_id"));
@ -863,7 +844,7 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
sfNum = sfNum + checkUpdateResults(bulkConnection, salesforceUpdateJob, batchInfos,api);
} catch (Throwable e) {
log.info(e.getMessage());
log.info(JSON.toJSONString(e));
throw e;
}finally {
@ -1199,20 +1180,19 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
if ("reference".equals(dataField.getSfType()) && data.get(j - 1).get(dataField.getField()) != null){
//引用类型
String reference_to = dataField.getReferenceTo();
//判断reference_to内是否包含User字符串
if (dataField.getReferenceTo() != null && (dataField.getReferenceTo().contains(",User") || dataField.getReferenceTo().contains("User,"))) {
reference_to = "User";
}
//引用类型字段
String linkfield = fieldMap.get(dataField.getField());
if (StringUtils.isNotBlank(linkfield)){
reference_to = data.get(j-1).get(linkfield)!=null?data.get(j-1).get(linkfield).toString():null;
if (StringUtils.isNotBlank(linkfield) && data.get(j-1).get(linkfield)!=null){
reference_to = data.get(j-1).get(linkfield).toString();
}
if (reference_to == null){
continue;
}
if (reference_to.contains(",User") || reference_to.contains("User,")) {
reference_to = "User";
}
Map<String, Object> m = customMapper.getById("new_id", reference_to, data.get(j - 1).get(dataField.getField()).toString());
if (m != null && !m.isEmpty()) {
account.put(dataField.getField(), m.get("new_id"));
@ -1243,50 +1223,20 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
JobInfo salesforceInsertJob = null;
JobInfo salesforceQueryJob = null;
try {
salesforceInsertJob = BulkUtil.createJob(bulkConnection, api, OperationEnum.insert);
List<BatchInfo> batchInfos = CsvConverterUtil.writeToCsvNew(insertList, UUID.randomUUID().toString(),list,true,bulkConnection,salesforceInsertJob);
BulkUtil.awaitCompletion(bulkConnection, salesforceInsertJob, batchInfos);
List<String> resultIds = checkInsertResults(bulkConnection, salesforceInsertJob, batchInfos, api, ids);
String idStr = "(";
for (String id : resultIds) {
idStr += "'" + id + "',"; // 拼接每个ID
}
if (idStr.endsWith(",")) { // 如果最后一个字符是逗号，说明循环正常结束
idStr = idStr.substring(0, idStr.length() - 1); // 去掉最后一个多余的逗号
}
idStr += ")"; // 添加右括号
String query = "SELECT Id,old_sfdc_id__c FROM " + api + " where Id in " + idStr;
salesforceQueryJob = BulkUtil.createJob(bulkConnection, api, OperationEnum.queryAll);
ByteArrayInputStream bout = new ByteArrayInputStream(query.getBytes());
BatchInfo info = bulkConnection.createBatchFromStream(salesforceQueryJob, bout);
List<BatchInfo> batchQueryInfos = new ArrayList<>();
batchQueryInfos.add(info);
BulkUtil.awaitCompletion(bulkConnection, salesforceQueryJob, batchQueryInfos);
sfNum = sfNum + checkQueryResultsNew(bulkConnection, salesforceQueryJob, info, api);
sfNum = sfNum + checkInsertResults(bulkConnection, salesforceInsertJob, batchInfos, api, ids);
} catch (Exception e) {
log.error("insertSingleData error api:{}", api, e);
log.error("createdNewIdBatch error api:{}", api, e);
throw e;
}finally {
BulkUtil.closeJob(bulkConnection, salesforceInsertJob.getId());
BulkUtil.closeJob(bulkConnection, salesforceQueryJob.getId());
}
}
@ -1369,20 +1319,19 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
if ("reference".equals(dataField.getSfType()) && data.get(j - 1).get(dataField.getField()) != null){
//引用类型
String reference_to = dataField.getReferenceTo();
//判断reference_to内是否包含User字符串
if (dataField.getReferenceTo() != null && (dataField.getReferenceTo().contains(",User") || dataField.getReferenceTo().contains("User,"))) {
reference_to = "User";
}
//引用类型字段
String linkfield = fieldMap.get(dataField.getField());
if (StringUtils.isNotBlank(linkfield)){
reference_to = data.get(j-1).get(linkfield)!=null?data.get(j-1).get(linkfield).toString():null;
if (StringUtils.isNotBlank(linkfield) && data.get(j-1).get(linkfield)!=null){
reference_to = data.get(j-1).get(linkfield).toString();
}
if (reference_to == null){
continue;
}
if (reference_to.contains(",User") || reference_to.contains("User,")) {
reference_to = "User";
}
Map<String, Object> m = customMapper.getById("new_id", reference_to, data.get(j - 1).get(dataField.getField()).toString());
if (m != null && !m.isEmpty()) {
account.put(dataField.getField(), m.get("new_id"));

764
src/main/java/com/celnet/datadump/service/impl/DataImportNewServiceImpl.java
View File

File diff suppressed because it is too large Load Diff

60
src/main/java/com/celnet/datadump/service/impl/DataImportServiceImpl.java
View File

@ -255,6 +255,7 @@ public class DataImportServiceImpl implements DataImportService {
}
private void manualCreatedNewId(SalesforceParam param, PartnerConnection partnerConnection) throws Exception {
Map<String, Object> infoFlag = customMapper.list("code,value","system_config","code ='"+SystemConfigCode.INFO_FLAG+"'").get(0);
String api = param.getApi();
QueryWrapper<DataField> dbQw = new QueryWrapper<>();
@ -272,7 +273,6 @@ public class DataImportServiceImpl implements DataImportService {
}
//表内数据总量
Integer count = customMapper.countBySQL(api, "where new_id is null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "'");
log.error("总Insert数据 count:{}；-开始时间:{}；-结束时间:{}；-api:{}；", count, beginDateStr, endDateStr, api);
if (count == 0) {
return;
}
@ -291,21 +291,37 @@ public class DataImportServiceImpl implements DataImportService {
));
}
//批量插入200一次
int page = count%200 == 0 ? count/200 : (count/200) + 1;
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
DataObject update = new DataObject();
update.setName(api);
for (int i = 0; i < page; i++) {
try {
List<Map<String, Object>> data = customMapper.list("*", api, "new_id is null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "' limit 200");
int size = data.size();
log.info("执行api:{}, 执行page:{}, 执行size:{}", api, i+1, size);
SObject[] accounts = new SObject[size];
String[] ids = new String[size];
for (int j = 1; j <= size; j++) {
int num = count%10000 == 0 ? count/10000 : (count/10000) + 1;
log.info("总Insert数据 count:{}；总批次:{}；-开始时间:{}；-结束时间:{}；-api:{}；", count, num,beginDateStr, endDateStr, api);
for (int z = 0; z < num; z++) {
List<Map<String, Object>> mapsList = customMapper.list("*", api, "new_id is null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "' limit 10000");
log.info("总Insert数据 count:{}；当前批次:{}；-开始时间:{}；-结束时间:{}；-api:{}；", count, z,beginDateStr, endDateStr, api);
int size = mapsList.size();
int page = size%200 == 0 ? size/200 : (size/200) + 1;
for (int i = 0; i < page; i++) {
int startIndex = 200 * i;
int endIndex = Math.min(startIndex + 200, size);
if (startIndex >= endIndex) {
break;
}
List<Map<String, Object>> mapList = mapsList.subList(startIndex, endIndex);
int sized = mapList.size();
SObject[] accounts = new SObject[sized];
String[] ids = new String[sized];
try {
for (int j = 1; j <= sized; j++) {
SObject account = new SObject();
account.setType(api);
//找出sf对象必填字段，并且给默认值
@ -318,10 +334,10 @@ public class DataImportServiceImpl implements DataImportService {
//object类型转Date类型
Date date;
try {
date = sdf.parse(String.valueOf(data.get(j - 1).get("CreatedDate")));
date = sdf.parse(String.valueOf(mapList.get(j - 1).get("CreatedDate")));
}catch (ParseException e){
//解决当时间秒为0时，转换秒精度丢失问题
date = sdf.parse(data.get(j - 1).get("CreatedDate")+":00");
date = sdf.parse(mapList.get(j - 1).get("CreatedDate")+":00");
}
Calendar calendar = Calendar.getInstance();
calendar.setTime(date);
@ -329,7 +345,7 @@ public class DataImportServiceImpl implements DataImportService {
continue;
}
if ("CreatedById".equals(dataField.getField()) && dataField.getIsCreateable()){
Map<String, Object> CreatedByIdMap = customMapper.getById("new_id", "User", data.get(j-1).get("CreatedById").toString());
Map<String, Object> CreatedByIdMap = customMapper.getById("new_id", "User", mapList.get(j-1).get("CreatedById").toString());
if(CreatedByIdMap != null ){
account.setField("CreatedById", CreatedByIdMap.get("new_id"));
}
@ -337,14 +353,14 @@ public class DataImportServiceImpl implements DataImportService {
}
if (dataField.getIsCreateable() != null && dataField.getIsCreateable() && !dataField.getIsNillable() && !dataField.getIsDefaultedOnCreate()) {
if ("reference".equals(dataField.getSfType()) && data.get(j - 1).get(dataField.getField()) != null){
if ("reference".equals(dataField.getSfType()) && mapList.get(j - 1).get(dataField.getField()) != null){
//引用类型
String reference = dataField.getReferenceTo();
//引用类型字段
String linkfield = fieldMap.get(dataField.getField());
if (StringUtils.isNotBlank(linkfield)){
reference = data.get(j-1).get(linkfield)!=null?data.get(j-1).get(linkfield).toString():null;
reference = mapList.get(j-1).get(linkfield)!=null?mapList.get(j-1).get(linkfield).toString():null;
}
if (reference == null){
@ -375,9 +391,9 @@ public class DataImportServiceImpl implements DataImportService {
}
}
ids[j-1] = data.get(j-1).get("Id").toString();
ids[j-1] = mapList.get(j-1).get("Id").toString();
accounts[j-1] = account;
if (i*200+j == count){
if (i*200+j == size){
break;
}
}
@ -406,6 +422,8 @@ public class DataImportServiceImpl implements DataImportService {
log.info(JSON.toJSONString(e));
}
}
}
SalesforceParam countParam = new SalesforceParam();
countParam.setApi(api);
countParam.setBeginCreateDate(beginDate);
@ -643,7 +661,7 @@ public class DataImportServiceImpl implements DataImportService {
}
int targetCount = 0;
//批量插入200一次
int page = count%200 == 0 ? count/200 : (count/200) + 1;
for (int i = 0; i < page; i++) {
List<Map<String, Object>> mapList = customMapper.list("*", api, "new_id is not null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "' order by Id asc limit " + i * 200 + ",200");
@ -816,7 +834,7 @@ public class DataImportServiceImpl implements DataImportService {
int targetCount = 0;
try {
//批量插入200一次
int page = count % 200 == 0 ? count / 200 : (count / 200) + 1;
for (int i = 0; i < page; i++) {
TimeUnit.MILLISECONDS.sleep(1);

23
src/main/java/com/celnet/datadump/service/impl/FileServiceImpl.java
View File

@ -93,7 +93,7 @@ public class FileServiceImpl implements FileService {
extraSqlTmp += "AND Id > '" + maxId + "'";
}
// 获取未存储的附件id
List<Map<String, Object>> list = customMapper.list("Id, url", api, extraSqlTmp + " AND is_dump = 1 order by id asc limit 10");
List<Map<String, Object>> list = customMapper.list("Id, url", api, extraSqlTmp + " AND is_dump = 1 order by id asc limit 200");
if (CollectionUtils.isEmpty(list)) {
break;
}
@ -112,7 +112,7 @@ public class FileServiceImpl implements FileService {
List<Map<String, Object>> maps = Lists.newArrayList();
Map<String, Object> paramMap = Maps.newHashMap();
paramMap.put("key", "is_dump");
paramMap.put("value", false);
paramMap.put("value", 0);
maps.add(paramMap);
customMapper.updateById(api, maps, id);
}
@ -124,7 +124,7 @@ public class FileServiceImpl implements FileService {
successNum += list.size();
}
log.info("dump file verify success, api:{}, success num:{}, retry dump num:{}", api, successNum, num);
XxlJobLogger.log("dump file verify success, api:{}, success num:{}, retry dump num:{}", api, successNum, num);
EmailUtil.send("DumpFile Verify Success", "api: " + api + ", success num: " + successNum + ", retry dump num: " + num);
// 存在需重下的文件 执行一遍
if (num > 0) {
dumpFile(api, field, true);
@ -694,9 +694,17 @@ public class FileServiceImpl implements FileService {
break;
}
Map<String, Object> lMap = customMapper.getById("new_id",parentType, parentId);
Map<String, Object> lMap = customMapper.getById("new_id","User", String.valueOf(map.get("OwnerId")));
if(lMap == null) {
String format = "文件ID：" + id + ",对应关联对象类型：" + parentType + "，对应关联对象ID：" + parentId + "数据不存在！！！！";
String format = "文件ID：" + id + ",对应关联对象类型：" + parentType + "，对应关联对象ID：" + map.get("OwnerId") + "数据不存在！！！！";
log.info(format);
EmailUtil.send("DataDump ERROR", format);
break;
}
Map<String, Object> uMap = customMapper.getById("new_id",parentType, parentId);
if(uMap == null) {
String format = "文件ID：" + id + ",对应用户ID：" + parentId + "数据不存在！！！！";
log.info(format);
EmailUtil.send("DataDump ERROR", format);
break;
@ -711,6 +719,10 @@ public class FileServiceImpl implements FileService {
JSONObject credentialsJsonParam = new JSONObject();
credentialsJsonParam.put("parentId", lMap.get("new_id"));
credentialsJsonParam.put("Name", fileName);
credentialsJsonParam.put("Description", map.get("Description"));
credentialsJsonParam.put("OwnerId", uMap.get("new_id"));
credentialsJsonParam.put("CreatedDate", map.get("CreatedDate"));
credentialsJsonParam.put("CreatedById", map.get("CreatedById"));
MultipartEntityBuilder builder = MultipartEntityBuilder.create();
builder.addTextBody("data", credentialsJsonParam.toJSONString(), ContentType.APPLICATION_JSON);
@ -731,6 +743,7 @@ public class FileServiceImpl implements FileService {
maps.add(paramMap);
}
} else {
log.info("文件ID：" + id + "上传失败，返回信息：" + response);
throw new RuntimeException();
}
}

15
src/main/java/com/celnet/datadump/util/DataUtil.java
View File

@ -127,6 +127,21 @@ public class DataUtil {
return Arrays.asList(ids.replaceAll(StringUtils.SPACE, StringUtils.EMPTY).split(","));
}
public static Set<String> toIdSet(String ids) {
if (StringUtils.isBlank(ids)) {
return Collections.emptySet();
}
Set<String> set = new HashSet<>();
String[] split = ids.replaceAll(StringUtils.SPACE, StringUtils.EMPTY).split(",");
for (String id : split) {
if (StringUtils.isNotBlank(id)) {
set.add(id);
}
}
return set;
}
private static final int MINUTE = 60;
private static final int HOUR = 24;

22
src/main/resources/mapper/CustomMapper.xml
View File

@ -70,6 +70,16 @@
</where>
</update>
<update id="updateByField">
UPDATE `${tableName}`
SET new_id = #{newId}
<where>
<foreach item="map" collection="maps" separator=" AND ">
${map.key} = #{map.value}
</foreach>
</where>
</update>
<insert id="save">
insert into
`${tableName}`
@ -174,6 +184,18 @@
</where>
</select>
<select id="getByNewId" resultType="Map">
SELECT
${select}
FROM
`${api}`
<where>
<if test="id != null">
new_id = #{id}
</if>
</where>
</select>
<select id="countBySQL" resultType="int">
SELECT
count(1)

32
src/main/resources/mapper/SalesforceMapper.xml
View File

@ -147,4 +147,36 @@
</if>
</select>
<select id="listOrderByIdNew">
select ${param.select} from ${param.api}
<where>
<if test="param != null">
<if test="param.maxId != null">
AND id > #{param.maxId}
</if>
<if test="param.isDeleted != null">
AND IsDeleted = #{param.isDeleted}
</if>
<if test="param.sql != null">
AND ${param.sql}
</if>
<!-- 判断 param.api 是否以 "Share" 结尾 -->
<if test="param.api != null and param.api.endsWith('Share')">
AND RowCause = 'Manual'
</if>
<if test="param.beginModifyDate != null and !param.api.endsWith('Share')">
AND SystemModstamp >= #{param.beginModifyDate}
</if>
<if test="param.beginModifyDate != null and param.api.endsWith('Share')">
AND LastModifiedDate >= #{param.beginModifyDate}
</if>
</if>
</where>
<if test="param != null">
<if test="param.idField != null">
order by ${param.idField} asc
</if>
</if>
</select>
</mapper>