kris/data-dump
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

【feat】 数据更新同步，增加更新是否失败，以及失败信息

Browse Source
This commit is contained in:
Kris 2025-08-27 19:09:27 +08:00
parent 97e21e30dc
commit fef14ca0b7
4 changed files with 42 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/com/celnet/datadump/mapper/CustomMapper.java
View File

@ -88,7 +88,7 @@ public interface CustomMapper {
* @param api
* @param newId
*/
public void updateByField(@Param("maps") List<Map<String, Object>> maps, @Param("tableName") String api, @Param("newId") String newId);
public void updateByNewId(@Param("maps") List<Map<String, Object>> maps, @Param("tableName") String api, @Param("newId") String newId);
/**
* 插入方法

14
src/main/java/com/celnet/datadump/service/impl/CommonServiceImpl.java
View File

@ -1079,6 +1079,7 @@ public class CommonServiceImpl implements CommonService {
map.put("comment", "新sfid");
map.put("name", "new_id");
list.add(map);
fields.add("new_id");
// 构建索引
@ -1108,6 +1109,19 @@ public class CommonServiceImpl implements CommonService {
map2.put("name", "all_data");
list.add(map2);
}
//是否更新
Map<String, Object> map3 = Maps.newHashMap();
map3.put("type", "tinyint(1) DEFAULT 0");
map3.put("comment", "是否更新");
map3.put("name", "is_update");
list.add(map3);
//是否更新
Map<String, Object> map4 = Maps.newHashMap();
map4.put("type", "text");
map4.put("comment", "更新错误信息");
map4.put("name", "error_message");
list.add(map4);
customMapper.createTable(apiName, label, list, index);
// 生成字段映射

36
src/main/java/com/celnet/datadump/service/impl/DataImportNewServiceImpl.java
View File

@ -738,20 +738,20 @@ public class DataImportNewServiceImpl implements DataImportNewService {
if (1 == param.getType()) {
if (api.endsWith("Share")){
sql = "where RowCause = 'Manual' and new_id is not null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "'";
sql2 = "RowCause = 'Manual' and new_id is not null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "' order by Id asc limit ";
sql = "where RowCause = 'Manual' and is_update != 1 and new_id is not null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "'";
sql2 = "RowCause = 'Manual' and is_update != 1 and new_id is not null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "' order by Id asc limit ";
}else {
sql = "where new_id is not null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "'";
sql2 = "new_id is not null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "' order by Id asc limit ";
sql = "where new_id is not null and is_update != 1 and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "'";
sql2 = "new_id is not null and is_update != 1 and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "' order by Id asc limit ";
}
}else {
String updateDateField = dataFieldService.returnUpdateDateField(api);
if (api.endsWith("Share")){
sql = "where RowCause = 'Manual' and new_id is not null and "+updateDateField+" >= '" + beginDateStr + "' ";
sql2 = "RowCause = 'Manual' and new_id is not null and "+updateDateField+" >= '" + beginDateStr + "' order by Id asc limit ";
sql = "where RowCause = 'Manual' and is_update != 1 and new_id is not null and "+updateDateField+" >= '" + beginDateStr + "' ";
sql2 = "RowCause = 'Manual' and is_update != 1 and new_id is not null and "+updateDateField+" >= '" + beginDateStr + "' order by Id asc limit ";
}else {
sql = "where new_id is not null and "+updateDateField+" >= '" + beginDateStr + "' ";
sql2 = "new_id is not null and "+updateDateField+" >= '" + beginDateStr + "' order by Id asc limit ";
sql = "where new_id is not null and and is_update != 1 "+updateDateField+" >= '" + beginDateStr + "' ";
sql2 = "new_id is not null and is_update != 1 and "+updateDateField+" >= '" + beginDateStr + "' order by Id asc limit ";
}
}
//表内数据总量
@ -881,11 +881,27 @@ public class DataImportNewServiceImpl implements DataImportNewService {
SaveResult[] saveResults = partnerConnection.update(accounts);
for (SaveResult saveResult : saveResults) {
if (!saveResult.getSuccess()) {
List<Map<String, Object>> maps = new ArrayList<>();
Map<String, Object> linkMap = new HashMap<>();
linkMap.put("key", "is_update");
linkMap.put("value", 2);
maps.add(linkMap);
Map<String, Object> linkMap1 = new HashMap<>();
linkMap1.put("key", "error_message");
linkMap1.put("value", saveResult.getErrors());
maps.add(linkMap1);
customMapper.updateByNewId(maps,api, saveResult.getId());
Map<String, String> map = returnErrorAccountsDetails(accounts, list, saveResult.getId());
String format = String.format("数据更新 error, api name: %s, \nparam: %s, \ncause:\n%s, \n数据实体类:\n%s", api, com.alibaba.fastjson2.JSON.toJSONString(param, DataDumpParam.getFilter()), JSON.toJSONString(saveResult),JSON.toJSONString(map));
EmailUtil.send("DataDump ERROR", format);
log.info(format);
return;
}else {
List<Map<String, Object>> maps = new ArrayList<>();
Map<String, Object> linkMap = new HashMap<>();
linkMap.put("key", "is_update");
linkMap.put("value", 1);
maps.add(linkMap);
customMapper.updateByNewId(maps,api, saveResult.getId());
targetCount ++;
}
}
@ -2654,7 +2670,7 @@ public class DataImportNewServiceImpl implements DataImportNewService {
maps.add(paramMap);
}
}
customMapper.updateByField(maps, api, id);
customMapper.updateByNewId(maps, api, id);
} catch (Throwable throwable) {
if (throwable.toString().contains("interrupt")) {
log.error("may interrupt error:", throwable);

2
src/main/resources/mapper/CustomMapper.xml
View File

@ -77,7 +77,7 @@
</where>
</update>
<update id="updateByField">
<update id="updateByNewId">
UPDATE `${tableName}`
SET new_id = #{newId}
<where>