kris/data-dump
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: kris:4fc53bf74ad7074bf6a0073a4191673fab35341b
Branches Tags
kris:master
kris:develop_携科
kris:develop
kris:master_back
..
compare: kris:e7209d968463d9209c9c7bc1f6b6d83778e13d7d
Branches Tags
kris:develop_携科
kris:develop
kris:master
kris:master_back

No commits in common. "4fc53bf74ad7074bf6a0073a4191673fab35341b" and "e7209d968463d9209c9c7bc1f6b6d83778e13d7d" have entirely different histories.
4fc53bf74a ... e7209d9684

6 changed files with 59 additions and 73 deletions
Show Stats Expand all files Collapse all files

28
src/main/java/com/celnet/datadump/service/impl/DataImportBatchServiceImpl.java
View File

@ -247,7 +247,7 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
maxIndex.ne("name", api);
Map<String, Object> map = dataObjectService.getMap(maxIndex);
//如果必填lookup字段没有值，跳过
update.setDataIndex(Integer.parseInt(map.get("data_index").toString()) + 1);
update.setDataIndex(Integer.parseInt(map.get("data_index").toString()+1));
dataObjectService.updateById(update);
String message = "api:" + api + "的引用对象:" + reference + "不存在数据！";
String format = String.format("数据导入 error, api name: %s, \nparam: %s, \ncause:\n%s", api, com.alibaba.fastjson2.JSON.toJSONString(param, DataDumpParam.getFilter()), message);
@ -298,12 +298,13 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
List<BatchInfo> batchInfos = BulkUtil.createBatchesFromCSVFile(bulkConnection, salesforceInsertJob, fullPath);
BulkUtil.closeJob(bulkConnection, salesforceInsertJob.getId());
BulkUtil.awaitCompletion(bulkConnection, salesforceInsertJob, batchInfos);
sfNum = sfNum + checkInsertResults(bulkConnection, salesforceInsertJob, batchInfos, api, ids);
BulkUtil.closeJob(bulkConnection, salesforceInsertJob.getId());
new File(fullPath).delete();
} catch (Exception e) {
log.error("manualCreatedNewId error api:{}", api, e);
throw e;
@ -527,8 +528,8 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
} else if (!DataUtil.isUpdate(field) || (dataField.getIsCreateable() != null && !dataField.getIsCreateable())) {
continue;
} else if (StringUtils.isNotBlank(reference_to) && !"data_picklist".equals(reference_to)) {
if ( StringUtils.isNotEmpty(String.valueOf(map.get(field))) && !"OwnerId".equals(field)
&& !"Owner_Type".equals(field)) {
if (!"null".equals(map.get(field)) && null != map.get(field) && !"OwnerId".equals(field)
|| !"Owner_Type".equals(field)) {
//判断reference_to内是否包含User字符串
if (reference_to.contains("User")) {
reference_to = "User";
@ -537,15 +538,22 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
if (m != null && !m.isEmpty()) {
account.put(field, m.get("new_id"));
}else {
String message = "对象类型:" + api + "的数据:"+ map.get("Id") +"的引用对象:" + dataField.getReferenceTo() + "的数据:"+ map.get(field) +"不存在！";
EmailUtil.send("DataDump ERROR", message);
QueryWrapper<DataObject> maxIndex = new QueryWrapper<>();
maxIndex.select("IFNULL(max(data_index),0) as data_index");
maxIndex.ne("name", api);
Map<String, Object> mapTo = dataObjectService.getMap(maxIndex);
//如果必填lookup字段没有值，跳过
update.setDataIndex(Integer.parseInt(mapTo.get("data_index").toString()+1));
dataObjectService.updateById(update);
String message = "对象类型:" + api + "的数据:"+ m.get("Id") +"的引用对象:" + dataField.getReferenceTo() + "的数据:"+ map.get(field) +"不存在！"; String format = String.format("数据导入 error, api name: %s, \nparam: %s, \ncause:\n%s", api, com.alibaba.fastjson2.JSON.toJSONString(param, DataDumpParam.getFilter()), message);
EmailUtil.send("DataDump ERROR", format);
log.info(message);
return;
}
}
} else {
if (map.get(field) != null && StringUtils.isNotBlank(dataField.getSfType())) {
account.put(field, DataUtil.localBulkDataToSfData(dataField.getSfType(), String.valueOf(map.get(field))));
account.put(field, DataUtil.localDataToSfData(dataField.getSfType(), String.valueOf(map.get(field))));
}else {
account.put(field, map.get(field));
}
@ -565,12 +573,12 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
List<BatchInfo> batchInfos = BulkUtil.createBatchesFromCSVFile(bulkConnection, salesforceInsertJob, fullPath);
BulkUtil.closeJob(bulkConnection, salesforceInsertJob.getId());
BulkUtil.awaitCompletion(bulkConnection, salesforceInsertJob, batchInfos);
sfNum = sfNum + checkUpdateResults(bulkConnection, salesforceInsertJob, batchInfos,api);
BulkUtil.closeJob(bulkConnection, salesforceInsertJob.getId());
new File(fullPath).delete();
} catch (Throwable e) {
log.info(e.getMessage());

31
src/main/java/com/celnet/datadump/service/impl/DataImportNewServiceImpl.java
View File

@ -742,35 +742,48 @@ public class DataImportNewServiceImpl implements DataImportNewService {
String field = dataField.getField();
String reference_to = dataField.getReferenceTo();
String value = String.valueOf(map.get(field));
//根据旧sfid查找引用对象新sfid
if (field.equals("Id")) {
account.setId(String.valueOf(map.get("new_id")));
} else if (!DataUtil.isUpdate(field) || (dataField.getIsCreateable() != null && !dataField.getIsCreateable())) {
continue;
} else if (StringUtils.isNotBlank(reference_to) && !"data_picklist".equals(reference_to)) {
if (!"null".equals(value) && StringUtils.isNotEmpty(value) && (!"OwnerId".equals(field)
&& !"Owner_Type".equals(field))) {
if (!"null".equals(map.get(field)) && null != map.get(field) && !"OwnerId".equals(field)
|| !"Owner_Type".equals(field)) {
//判断reference_to内是否包含User字符串
if (reference_to.contains("User")) {
reference_to = "User";
}
Map<String, Object> m = customMapper.getById("new_id", reference_to, value);
Map<String, Object> m = customMapper.getById("new_id", reference_to, String.valueOf(map.get(field)));
if (m != null && !m.isEmpty()) {
account.setField(field, m.get("new_id"));
}else {
String message = "对象类型:" + api + "的数据:"+ map.get("Id") +"的引用对象:" + dataField.getReferenceTo() + "的数据:"+ map.get(field) +"不存在！";
EmailUtil.send("DataDump ERROR", message);
QueryWrapper<DataObject> maxIndex = new QueryWrapper<>();
maxIndex.select("IFNULL(max(data_index),0) as data_index");
maxIndex.ne("name", api);
Map<String, Object> mapTo = dataObjectService.getMap(maxIndex);
//如果必填lookup字段没有值，跳过
update.setDataIndex(Integer.parseInt(mapTo.get("data_index").toString()) +1);
dataObjectService.updateById(update);
String message = "对象类型:" + api + "的数据:"+ m.get("Id") +"的引用对象:" + dataField.getReferenceTo() + "的数据:"+ map.get(field) +"不存在！"; String format = String.format("数据导入 error, api name: %s, \nparam: %s, \ncause:\n%s", api, com.alibaba.fastjson2.JSON.toJSONString(param, DataDumpParam.getFilter()), message);
EmailUtil.send("DataDump ERROR", format);
log.info(message);
return;
}
}
} else {
if (map.get(field) != null && StringUtils.isNotBlank(dataField.getSfType())) {
account.setField(field, DataUtil.localDataToSfData(dataField.getSfType(), value));
account.setField(field, DataUtil.localDataToSfData(dataField.getSfType(), String.valueOf(map.get(field))));
}else {
account.setField(field, map.get(field));
if (api.equals("Account")){
if ("1".equals(map.get("IsPersonAccount")) && field.equals("Name")){
continue;
}else if("0".equals(map.get("IsPersonAccount")) && field.equals("LastName")){
continue;
}
} else {
account.setField(field, map.get(field));
}
}
}
}

3
src/main/java/com/celnet/datadump/service/impl/DataImportServiceImpl.java
View File

@ -309,8 +309,9 @@ public class DataImportServiceImpl implements DataImportService {
maxIndex.select("IFNULL(max(data_index),0) as data_index");
maxIndex.ne("name", api);
Map<String, Object> map = dataObjectService.getMap(maxIndex);
//如果必填lookup字段没有值，跳过
update.setDataIndex(Integer.parseInt(map.get("data_index").toString()) + 1);
update.setDataIndex(Integer.parseInt(map.get("data_index").toString()+1));
dataObjectService.updateById(update);
return;
}else{

14
src/main/java/com/celnet/datadump/service/impl/FileManagerServiceImpl.java
View File

@ -169,12 +169,11 @@ public class FileManagerServiceImpl implements FileManagerService {
for (Map<String, Object> map1 : poll) {
if ("FILE_DOWNLOAD_URL".equals(map1.get("code"))) {
downloadUrl = (String) map1.get("value");
}else {
EmailUtil.send("DumpFile ERROR", "文件下载失败！下载地址未配置");
return;
}
}
if (StringUtils.isNotBlank(downloadUrl)) {
EmailUtil.send("DumpFile ERROR", "文件下载失败！下载地址未配置");
return;
}
String url = downloadUrl + String.format(Const.SF_RICH_TEXT_FILE_URL, objectApi, recordId, fieldApi, fieldId);
try {
@ -308,12 +307,11 @@ public class FileManagerServiceImpl implements FileManagerService {
for (Map<String, Object> map1 : poll) {
if ("FILE_UPLOAD_URL".equals(map1.get("code"))) {
uploadUrl = (String) map1.get("value");
}else {
EmailUtil.send("UploadFile ERROR", "文件上传失败！上传地址未配置");
return;
}
}
if (StringUtils.isBlank(uploadUrl)) {
EmailUtil.send("UploadFile ERROR", "文件上传失败！上传地址未配置");
return;
}
// 拼接url
String url = uploadUrl + String.format(Const.SF_UPLOAD_RICH_TEXT_FILE_URL, objectApi, recordId);

23
src/main/java/com/celnet/datadump/service/impl/FileServiceImpl.java
View File

@ -139,12 +139,11 @@ public class FileServiceImpl implements FileService {
for (Map<String, Object> map1 : poll) {
if ("FILE_DOWNLOAD_URL".equals(map1.get("code"))) {
downloadUrl = (String) map1.get("value");
}else {
EmailUtil.send("DumpFile ERROR", "文件下载失败！下载地址未配置");
return;
}
}
if (StringUtils.isNotBlank(downloadUrl)) {
EmailUtil.send("DumpFile ERROR", "文件下载失败！下载地址未配置");
return;
}
log.info("dump file api:{}, field:{}", api, field);
PartnerConnection connect = salesforceConnect.createConnect();
String token = connect.getSessionHeader().getSessionId();
@ -432,17 +431,17 @@ public class FileServiceImpl implements FileService {
@Override
public void uploadFile(String api, String field, Boolean singleThread) {
String uploadUrl = null;
List<Map<String, Object>> poll = customMapper.list("code,value","org_config",null);
for (Map<String, Object> map1 : poll) {
if ("FILE_UPLOAD_URL".equals(map1.get("code"))) {
uploadUrl = (String) map1.get("value");
}else {
EmailUtil.send("UploadFile ERROR", "文件上传失败！上传地址未配置");
return;
}
}
if (StringUtils.isBlank(uploadUrl)) {
EmailUtil.send("UploadFile ERROR", "文件上传失败！上传地址未配置");
return;
}
log.info("upload file api:{}, field:{}", api, field);
PartnerConnection connect = salesforceTargetConnect.createConnect();
@ -619,12 +618,12 @@ public class FileServiceImpl implements FileService {
for (Map<String, Object> map1 : poll) {
if ("FILE_UPLOAD_URL".equals(map1.get("code"))) {
uploadUrl = (String) map1.get("value");
}else {
EmailUtil.send("UploadFile ERROR", "文件上传失败！上传地址未配置");
return;
}
}
if (StringUtils.isBlank(uploadUrl)) {
EmailUtil.send("UploadFile ERROR", "文件上传失败！上传地址未配置");
return;
}
log.info("upload file api:{}, field:{}", api, field);
PartnerConnection connect = salesforceTargetConnect.createConnect();
String token = connect.getSessionHeader().getSessionId();

33
src/main/java/com/celnet/datadump/util/DataUtil.java
View File

@ -392,39 +392,6 @@ public class DataUtil {
}
}
public static Object localBulkDataToSfData(String fieldType, String data) throws ParseException {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
SimpleDateFormat sd = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
Date date;
//date转Calendar类型
Calendar calendar = Calendar.getInstance();
switch (fieldType) {
case "int":
return Integer.parseInt(data);
case "double":
case "currency":
case "percent":
return new BigDecimal(data);
case "boolean":
return Boolean.valueOf(data);
case "date":
return data+"T08:00:00Z";
case "datetime":
try {
date = sd.parse(data);
}catch (ParseException e){
//解决当时间秒为0时，转换秒精度丢失问题
date = sd.parse(data+":00");
}
calendar.setTime(date);
return calendar;
case "time":
return adjustHour(data);
default:
return data;
}
}
public static Object localDataToSfData(String fieldType, String data) throws ParseException {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
SimpleDateFormat sd = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");