kris/data-dump
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

【feat】 优化

Browse Source 
(cherry picked from commit 8c1f7511d6)
This commit is contained in:
Kris 2025-10-23 15:09:03 +08:00
parent 95fae12de8
commit f653813c5c
7 changed files with 34 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
files/start.sh
View File

@ -5,7 +5,7 @@
#dbUsername="root"
#dbPassword="celnet@2025.bln"
#携科
dbUrl="jdbc:mysql://127.0.0.1:3306/data-dump-xxl-job?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&useSSL=false&serverTimezone=Asia/Shanghai"
dbUrl="jdbc:mysql://127.0.0.1:3306/data-dump-xxl-job?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&serverTimezone=Asia/Shanghai&useSSL=false"
dbUsername="root"
dbPassword="Celnet2025.QY"
#其它

10
src/main/java/com/celnet/datadump/service/impl/CommonServiceImpl.java
View File

@ -934,14 +934,10 @@ public class CommonServiceImpl implements CommonService {
TimeUnit.MILLISECONDS.sleep(1);
} catch (InterruptedException e) {
throw e;
} catch (Throwable throwable) {
if (throwable.toString().contains("interrupt")) {
log.error("may interrupt error:", throwable);
throw new InterruptedException();
}
throwable.addSuppressed(new Exception("saveOrUpdate error data:" + JSON.toJSONString(jsonObject)));
} catch (Exception e) {
log.info("存量任务异常！" + "对象API:" + api + "存量数据下载异常！" ,e);
EmailUtil.send("存量任务异常！", "对象API:" + api + "存量数据下载异常！");
throw throwable;
throw new RuntimeException(e);
}
}
return existsIds.size();

2
src/main/java/com/celnet/datadump/service/impl/DataImportBatchServiceImpl.java
View File

@ -900,7 +900,7 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
if (map.get(field) != null && StringUtils.isNotBlank(dataField.getSfType())) {
account.put(field, DataUtil.localBulkDataToSfData(dataField.getSfType(), String.valueOf(map.get(field))));
}else {
account.put(field, map.get(field));
account.put(field, value);
}
}
}

37
src/main/java/com/celnet/datadump/service/impl/DataVerifyServiceImpl.java
View File

@ -647,18 +647,17 @@ public class DataVerifyServiceImpl implements DataVerifyService {
ArrayList<String> arrayList = new ArrayList<>();
String currentFilePath = generateObjectQualityCheckFilePath(objectApi + "_part" + fileIndex);
arrayList.add(currentFilePath);
try {
objectWriter = EasyExcel.write(currentFilePath).build();
for (DataBatch batch : batches) {
objectWriter = EasyExcel.write(currentFilePath).build();
for (DataBatch batch : batches) {
String startDate = DateFormatUtils.format(batch.getSyncStartDate(), "yyyy-MM-dd HH:mm:ss");
String endDate = DateFormatUtils.format(batch.getSyncEndDate(), "yyyy-MM-dd HH:mm:ss");
log.info("开始处理数据批次: 批次ID={} 批次总量(大于十万只校验十万)={} 批次开始时间={} 批次结束时间={} 对象API={} 忽略开始时间={}", batch.getId(),batch.getFirstSfNum(),startDate,endDate, objectApi, ignoreBeginDate);
try {
// 为每个批次创建新的detail sheet
detailSheet = createDetailSheet(objectWriter, dataObject);
String startDate = DateFormatUtils.format(batch.getSyncStartDate(), "yyyy-MM-dd HH:mm:ss");
String endDate = DateFormatUtils.format(batch.getSyncEndDate(), "yyyy-MM-dd HH:mm:ss");
log.info("开始处理数据批次: 批次ID={} 批次总量(大于十万只校验十万)={} 批次开始时间={} 批次结束时间={} 对象API={} 忽略开始时间={}", batch.getId(),batch.getFirstSfNum(),startDate,endDate, objectApi, ignoreBeginDate);
// 分页处理记录
int page = 0;
int totalRecordNum = 0;
@ -937,16 +936,20 @@ public class DataVerifyServiceImpl implements DataVerifyService {
// 写入Excel的summarySheet
synchronized (this) {
excelWriter.write(Collections.singletonList(row), summarySheet);
}
}
} catch (Exception e){
log.error("API={},处理异常：{}",objectApi, e.getMessage(),e);
}finally {
if (objectWriter != null) {
objectWriter.finish();
}
} catch (Exception e){
// 写入批次汇总结果
List<String> row = Lists.newArrayList();
row.add(String.valueOf(batch.getId()));
row.add(objectApi);
row.add(startDate);
row.add(endDate);
row.add("校验异常");
log.error("API={},处理异常：{}",objectApi, e.getMessage(),e);
}
}
if (sendEmail && hasError){
sendObjectQualityCheckEmailList(objectApi, arrayList);
}else {

2
src/main/java/com/celnet/datadump/util/CsvConverterUtil.java
View File

@ -215,7 +215,7 @@ public class CsvConverterUtil {
for (int i = 0; i < header.length; i++) {
// 将每个值转换为字符串（如果为null则转为空字符串）
Object value = jsonObject.get(header[i]);
row[i] = value == null ? "" : value.toString();
row[i] = String.valueOf(value);
}
csvWriter.writeLine(row);
}

7
src/main/java/com/celnet/datadump/util/DataUtil.java
View File

@ -494,8 +494,13 @@ public class DataUtil {
case "date":
return data+"T08:00:00Z";
case "datetime":
// 解析输入的日期时间字符串为LocalDateTime对象
LocalDateTime localDateTime = LocalDateTime.parse(data);
OffsetDateTime offsetDateTime = localDateTime.atOffset(ZoneOffset.ofHours(0));
// 将LocalDateTime增加8小时以补偿时区差异
LocalDateTime plusEightHours = localDateTime.plusHours(8);
// 转换为UTC时区的OffsetDateTime对象
OffsetDateTime offsetDateTime = plusEightHours.atOffset(ZoneOffset.ofHours(0));
// 使用ISO标准格式化器将OffsetDateTime格式化为字符串并返回
return offsetDateTime.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME);
case "time":
return adjustHour(data);

4
src/main/resources/mapper/CustomMapper.xml
View File

@ -107,7 +107,7 @@
`${tableName}`
SET
<foreach item="map" collection="maps" open="" separator="," close="">
${map.key} = #{map.value}
`${map.key}` = #{map.value}
</foreach>
WHERE
id = #{id}
@ -159,7 +159,7 @@
insert into
`${tableName}`
<foreach item="map" collection="maps" open="(" separator="," close=")">
${map.key}
`${map.key}`
</foreach>
values
<foreach item="map" collection="maps" open="(" separator="," close=")">