diff --git a/files/start.sh b/files/start.sh index 132098a..c86780c 100644 --- a/files/start.sh +++ b/files/start.sh @@ -5,7 +5,7 @@ #dbUsername="root" #dbPassword="celnet@2025.bln" #携科 -dbUrl="jdbc:mysql://127.0.0.1:3306/data-dump-xxl-job?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&useSSL=false&serverTimezone=Asia/Shanghai" +dbUrl="jdbc:mysql://127.0.0.1:3306/data-dump-xxl-job?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&serverTimezone=Asia/Shanghai&useSSL=false" dbUsername="root" dbPassword="Celnet2025.QY" #其它 diff --git a/src/main/java/com/celnet/datadump/service/impl/CommonServiceImpl.java b/src/main/java/com/celnet/datadump/service/impl/CommonServiceImpl.java index bd427ea..5338929 100644 --- a/src/main/java/com/celnet/datadump/service/impl/CommonServiceImpl.java +++ b/src/main/java/com/celnet/datadump/service/impl/CommonServiceImpl.java @@ -934,14 +934,10 @@ public class CommonServiceImpl implements CommonService { TimeUnit.MILLISECONDS.sleep(1); } catch (InterruptedException e) { throw e; - } catch (Throwable throwable) { - if (throwable.toString().contains("interrupt")) { - log.error("may interrupt error:", throwable); - throw new InterruptedException(); - } - throwable.addSuppressed(new Exception("saveOrUpdate error data:" + JSON.toJSONString(jsonObject))); + } catch (Exception e) { + log.info("存量任务异常！" + "对象API:" + api + "存量数据下载异常！" ,e); EmailUtil.send("存量任务异常！", "对象API:" + api + "存量数据下载异常！"); - throw throwable; + throw new RuntimeException(e); } } return existsIds.size(); diff --git a/src/main/java/com/celnet/datadump/service/impl/DataImportBatchServiceImpl.java b/src/main/java/com/celnet/datadump/service/impl/DataImportBatchServiceImpl.java index 25d8e64..b2529c7 100644 --- a/src/main/java/com/celnet/datadump/service/impl/DataImportBatchServiceImpl.java +++ b/src/main/java/com/celnet/datadump/service/impl/DataImportBatchServiceImpl.java @@ -900,7 +900,7 @@ public class DataImportBatchServiceImpl implements DataImportBatchService { if (map.get(field) != null && StringUtils.isNotBlank(dataField.getSfType())) { account.put(field, DataUtil.localBulkDataToSfData(dataField.getSfType(), String.valueOf(map.get(field)))); }else { - account.put(field, map.get(field)); + account.put(field, value); } } } diff --git a/src/main/java/com/celnet/datadump/service/impl/DataVerifyServiceImpl.java b/src/main/java/com/celnet/datadump/service/impl/DataVerifyServiceImpl.java index cf6fd6d..11a28c4 100644 --- a/src/main/java/com/celnet/datadump/service/impl/DataVerifyServiceImpl.java +++ b/src/main/java/com/celnet/datadump/service/impl/DataVerifyServiceImpl.java @@ -647,18 +647,17 @@ public class DataVerifyServiceImpl implements DataVerifyService { ArrayList arrayList = new ArrayList<>(); String currentFilePath = generateObjectQualityCheckFilePath(objectApi + "_part" + fileIndex); arrayList.add(currentFilePath); - try { - objectWriter = EasyExcel.write(currentFilePath).build(); - - for (DataBatch batch : batches) { + objectWriter = EasyExcel.write(currentFilePath).build(); + + for (DataBatch batch : batches) { + String startDate = DateFormatUtils.format(batch.getSyncStartDate(), "yyyy-MM-dd HH:mm:ss"); + String endDate = DateFormatUtils.format(batch.getSyncEndDate(), "yyyy-MM-dd HH:mm:ss"); + + log.info("开始处理数据批次: 批次ID={} 批次总量(大于十万只校验十万)={} 批次开始时间={} 批次结束时间={} 对象API={} 忽略开始时间={}", batch.getId(),batch.getFirstSfNum(),startDate,endDate, objectApi, ignoreBeginDate); + + try { // 为每个批次创建新的detail sheet detailSheet = createDetailSheet(objectWriter, dataObject); - - String startDate = DateFormatUtils.format(batch.getSyncStartDate(), "yyyy-MM-dd HH:mm:ss"); - String endDate = DateFormatUtils.format(batch.getSyncEndDate(), "yyyy-MM-dd HH:mm:ss"); - - log.info("开始处理数据批次: 批次ID={} 批次总量(大于十万只校验十万)={} 批次开始时间={} 批次结束时间={} 对象API={} 忽略开始时间={}", batch.getId(),batch.getFirstSfNum(),startDate,endDate, objectApi, ignoreBeginDate); - // 分页处理记录 int page = 0; int totalRecordNum = 0; @@ -937,16 +936,20 @@ public class DataVerifyServiceImpl implements DataVerifyService { // 写入Excel的summarySheet synchronized (this) { excelWriter.write(Collections.singletonList(row), summarySheet); - } - } - } catch (Exception e){ - log.error("API={},处理异常：{}",objectApi, e.getMessage(),e); - }finally { - if (objectWriter != null) { - objectWriter.finish(); + } + } catch (Exception e){ + // 写入批次汇总结果 + List row = Lists.newArrayList(); + row.add(String.valueOf(batch.getId())); + row.add(objectApi); + row.add(startDate); + row.add(endDate); + row.add("校验异常"); + log.error("API={},处理异常：{}",objectApi, e.getMessage(),e); } } + if (sendEmail && hasError){ sendObjectQualityCheckEmailList(objectApi, arrayList); }else { diff --git a/src/main/java/com/celnet/datadump/util/CsvConverterUtil.java b/src/main/java/com/celnet/datadump/util/CsvConverterUtil.java index 1496afb..2d06d2a 100644 --- a/src/main/java/com/celnet/datadump/util/CsvConverterUtil.java +++ b/src/main/java/com/celnet/datadump/util/CsvConverterUtil.java @@ -215,7 +215,7 @@ public class CsvConverterUtil { for (int i = 0; i < header.length; i++) { // 将每个值转换为字符串（如果为null则转为空字符串） Object value = jsonObject.get(header[i]); - row[i] = value == null ? "" : value.toString(); + row[i] = String.valueOf(value); } csvWriter.writeLine(row); } diff --git a/src/main/java/com/celnet/datadump/util/DataUtil.java b/src/main/java/com/celnet/datadump/util/DataUtil.java index 5eb4c6b..5300df3 100644 --- a/src/main/java/com/celnet/datadump/util/DataUtil.java +++ b/src/main/java/com/celnet/datadump/util/DataUtil.java @@ -494,8 +494,13 @@ public class DataUtil { case "date": return data+"T08:00:00Z"; case "datetime": + // 解析输入的日期时间字符串为LocalDateTime对象 LocalDateTime localDateTime = LocalDateTime.parse(data); - OffsetDateTime offsetDateTime = localDateTime.atOffset(ZoneOffset.ofHours(0)); + // 将LocalDateTime增加8小时以补偿时区差异 + LocalDateTime plusEightHours = localDateTime.plusHours(8); + // 转换为UTC时区的OffsetDateTime对象 + OffsetDateTime offsetDateTime = plusEightHours.atOffset(ZoneOffset.ofHours(0)); + // 使用ISO标准格式化器将OffsetDateTime格式化为字符串并返回 return offsetDateTime.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME); case "time": return adjustHour(data); diff --git a/src/main/resources/mapper/CustomMapper.xml b/src/main/resources/mapper/CustomMapper.xml index 48a1e38..f4e6558 100644 --- a/src/main/resources/mapper/CustomMapper.xml +++ b/src/main/resources/mapper/CustomMapper.xml @@ -107,7 +107,7 @@ `${tableName}` SET - ${map.key} = #{map.value} + `${map.key}` = #{map.value} WHERE id = #{id} @@ -159,7 +159,7 @@ insert into `${tableName}` - ${map.key} + `${map.key}` values