【feat】 优化
This commit is contained in:
parent
acc8e2ad46
commit
8c1f7511d6
@ -5,7 +5,7 @@
|
||||
#dbUsername="root"
|
||||
#dbPassword="celnet@2025.bln"
|
||||
#携科
|
||||
dbUrl="jdbc:mysql://127.0.0.1:3306/data-dump-xxl-job?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&useSSL=false&serverTimezone=Asia/Shanghai"
|
||||
dbUrl="jdbc:mysql://127.0.0.1:3306/data-dump-xxl-job?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&serverTimezone=Asia/Shanghai&useSSL=false"
|
||||
dbUsername="root"
|
||||
dbPassword="Celnet2025.QY"
|
||||
#其它
|
||||
|
||||
@ -934,14 +934,10 @@ public class CommonServiceImpl implements CommonService {
|
||||
TimeUnit.MILLISECONDS.sleep(1);
|
||||
} catch (InterruptedException e) {
|
||||
throw e;
|
||||
} catch (Throwable throwable) {
|
||||
if (throwable.toString().contains("interrupt")) {
|
||||
log.error("may interrupt error:", throwable);
|
||||
throw new InterruptedException();
|
||||
}
|
||||
throwable.addSuppressed(new Exception("saveOrUpdate error data:" + JSON.toJSONString(jsonObject)));
|
||||
} catch (Exception e) {
|
||||
log.info("存量任务异常！" + "对象API:" + api + "存量数据下载异常！" ,e);
|
||||
EmailUtil.send("存量任务异常！", "对象API:" + api + "存量数据下载异常！");
|
||||
throw throwable;
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
return existsIds.size();
|
||||
|
||||
@ -900,7 +900,7 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
|
||||
if (map.get(field) != null && StringUtils.isNotBlank(dataField.getSfType())) {
|
||||
account.put(field, DataUtil.localBulkDataToSfData(dataField.getSfType(), String.valueOf(map.get(field))));
|
||||
}else {
|
||||
account.put(field, map.get(field));
|
||||
account.put(field, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -647,18 +647,17 @@ public class DataVerifyServiceImpl implements DataVerifyService {
|
||||
ArrayList<String> arrayList = new ArrayList<>();
|
||||
String currentFilePath = generateObjectQualityCheckFilePath(objectApi + "_part" + fileIndex);
|
||||
arrayList.add(currentFilePath);
|
||||
try {
|
||||
objectWriter = EasyExcel.write(currentFilePath).build();
|
||||
|
||||
for (DataBatch batch : batches) {
|
||||
objectWriter = EasyExcel.write(currentFilePath).build();
|
||||
|
||||
for (DataBatch batch : batches) {
|
||||
String startDate = DateFormatUtils.format(batch.getSyncStartDate(), "yyyy-MM-dd HH:mm:ss");
|
||||
String endDate = DateFormatUtils.format(batch.getSyncEndDate(), "yyyy-MM-dd HH:mm:ss");
|
||||
|
||||
log.info("开始处理数据批次: 批次ID={} 批次总量(大于十万只校验十万)={} 批次开始时间={} 批次结束时间={} 对象API={} 忽略开始时间={}", batch.getId(),batch.getFirstSfNum(),startDate,endDate, objectApi, ignoreBeginDate);
|
||||
|
||||
try {
|
||||
// 为每个批次创建新的detail sheet
|
||||
detailSheet = createDetailSheet(objectWriter, dataObject);
|
||||
|
||||
String startDate = DateFormatUtils.format(batch.getSyncStartDate(), "yyyy-MM-dd HH:mm:ss");
|
||||
String endDate = DateFormatUtils.format(batch.getSyncEndDate(), "yyyy-MM-dd HH:mm:ss");
|
||||
|
||||
log.info("开始处理数据批次: 批次ID={} 批次总量(大于十万只校验十万)={} 批次开始时间={} 批次结束时间={} 对象API={} 忽略开始时间={}", batch.getId(),batch.getFirstSfNum(),startDate,endDate, objectApi, ignoreBeginDate);
|
||||
|
||||
// 分页处理记录
|
||||
int page = 0;
|
||||
int totalRecordNum = 0;
|
||||
@ -937,16 +936,20 @@ public class DataVerifyServiceImpl implements DataVerifyService {
|
||||
// 写入Excel的summarySheet
|
||||
synchronized (this) {
|
||||
excelWriter.write(Collections.singletonList(row), summarySheet);
|
||||
}
|
||||
}
|
||||
} catch (Exception e){
|
||||
log.error("API={},处理异常：{}",objectApi, e.getMessage(),e);
|
||||
}finally {
|
||||
if (objectWriter != null) {
|
||||
objectWriter.finish();
|
||||
}
|
||||
} catch (Exception e){
|
||||
// 写入批次汇总结果
|
||||
List<String> row = Lists.newArrayList();
|
||||
row.add(String.valueOf(batch.getId()));
|
||||
row.add(objectApi);
|
||||
row.add(startDate);
|
||||
row.add(endDate);
|
||||
row.add("校验异常");
|
||||
log.error("API={},处理异常：{}",objectApi, e.getMessage(),e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if (sendEmail && hasError){
|
||||
sendObjectQualityCheckEmailList(objectApi, arrayList);
|
||||
}else {
|
||||
|
||||
@ -215,7 +215,7 @@ public class CsvConverterUtil {
|
||||
for (int i = 0; i < header.length; i++) {
|
||||
// 将每个值转换为字符串（如果为null则转为空字符串）
|
||||
Object value = jsonObject.get(header[i]);
|
||||
row[i] = value == null ? "" : value.toString();
|
||||
row[i] = String.valueOf(value);
|
||||
}
|
||||
csvWriter.writeLine(row);
|
||||
}
|
||||
|
||||
@ -494,8 +494,13 @@ public class DataUtil {
|
||||
case "date":
|
||||
return data+"T08:00:00Z";
|
||||
case "datetime":
|
||||
// 解析输入的日期时间字符串为LocalDateTime对象
|
||||
LocalDateTime localDateTime = LocalDateTime.parse(data);
|
||||
OffsetDateTime offsetDateTime = localDateTime.atOffset(ZoneOffset.ofHours(0));
|
||||
// 将LocalDateTime增加8小时以补偿时区差异
|
||||
LocalDateTime plusEightHours = localDateTime.plusHours(8);
|
||||
// 转换为UTC时区的OffsetDateTime对象
|
||||
OffsetDateTime offsetDateTime = plusEightHours.atOffset(ZoneOffset.ofHours(0));
|
||||
// 使用ISO标准格式化器将OffsetDateTime格式化为字符串并返回
|
||||
return offsetDateTime.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME);
|
||||
case "time":
|
||||
return adjustHour(data);
|
||||
|
||||
@ -107,7 +107,7 @@
|
||||
`${tableName}`
|
||||
SET
|
||||
<foreach item="map" collection="maps" open="" separator="," close="">
|
||||
${map.key} = #{map.value}
|
||||
`${map.key}` = #{map.value}
|
||||
</foreach>
|
||||
WHERE
|
||||
id = #{id}
|
||||
@ -159,7 +159,7 @@
|
||||
insert into
|
||||
`${tableName}`
|
||||
<foreach item="map" collection="maps" open="(" separator="," close=")">
|
||||
${map.key}
|
||||
`${map.key}`
|
||||
</foreach>
|
||||
values
|
||||
<foreach item="map" collection="maps" open="(" separator="," close=")">
|
||||
|
||||
Loading…
Reference in New Issue
Block a user