【feat】 优化

files/start.sh

@@ -5,7 +5,7 @@
 #dbUsername="root"
 #dbPassword="celnet@2025.bln"
 #携科
-dbUrl="jdbc:mysql://127.0.0.1:3306/data-dump-xxl-job?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&useSSL=false&serverTimezone=Asia/Shanghai"
+dbUrl="jdbc:mysql://127.0.0.1:3306/data-dump-xxl-job?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&serverTimezone=Asia/Shanghai&useSSL=false"
 dbUsername="root"
 dbPassword="Celnet2025.QY"
 #其它

src/main/java/com/celnet/datadump/service/impl/CommonServiceImpl.java

@@ -934,14 +934,10 @@ public class CommonServiceImpl implements CommonService {
                 TimeUnit.MILLISECONDS.sleep(1);
             } catch (InterruptedException e) {
                 throw e;
-            } catch (Throwable throwable) {
+            } catch (Exception e) {
-                if (throwable.toString().contains("interrupt")) {
+                log.info("存量任务异常！" + "对象API:" + api + "存量数据下载异常！" ,e);
-                    log.error("may interrupt error:", throwable);
-                    throw new InterruptedException();
-                }
-                throwable.addSuppressed(new Exception("saveOrUpdate error data:" + JSON.toJSONString(jsonObject)));
                 EmailUtil.send("存量任务异常！", "对象API:" + api + "存量数据下载异常！");
-                throw throwable;
+                throw new RuntimeException(e);
             }
         }
         return existsIds.size();

src/main/java/com/celnet/datadump/service/impl/DataImportBatchServiceImpl.java

@@ -900,7 +900,7 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
                         if (map.get(field) != null && StringUtils.isNotBlank(dataField.getSfType())) {
                             account.put(field, DataUtil.localBulkDataToSfData(dataField.getSfType(), String.valueOf(map.get(field))));
                         }else {
-                            account.put(field, map.get(field));
+                            account.put(field, value);
                         }
                     }
                 }

src/main/java/com/celnet/datadump/service/impl/DataVerifyServiceImpl.java

@@ -647,18 +647,17 @@ public class DataVerifyServiceImpl implements DataVerifyService {
             ArrayList<String> arrayList = new ArrayList<>();
             String currentFilePath = generateObjectQualityCheckFilePath(objectApi + "_part" + fileIndex);
             arrayList.add(currentFilePath);
-            try {
+            objectWriter = EasyExcel.write(currentFilePath).build();
-                objectWriter = EasyExcel.write(currentFilePath).build();
 
-                for (DataBatch batch : batches) {
+            for (DataBatch batch : batches) {
+                String startDate = DateFormatUtils.format(batch.getSyncStartDate(), "yyyy-MM-dd HH:mm:ss");
+                String endDate = DateFormatUtils.format(batch.getSyncEndDate(), "yyyy-MM-dd HH:mm:ss");
+
+                log.info("开始处理数据批次: 批次ID={} 批次总量(大于十万只校验十万)={} 批次开始时间={} 批次结束时间={} 对象API={} 忽略开始时间={}", batch.getId(),batch.getFirstSfNum(),startDate,endDate, objectApi, ignoreBeginDate);
+
+                try {
                     // 为每个批次创建新的detail sheet
                     detailSheet = createDetailSheet(objectWriter, dataObject);
-
-                    String startDate = DateFormatUtils.format(batch.getSyncStartDate(), "yyyy-MM-dd HH:mm:ss");
-                    String endDate = DateFormatUtils.format(batch.getSyncEndDate(), "yyyy-MM-dd HH:mm:ss");
-
-                    log.info("开始处理数据批次: 批次ID={} 批次总量(大于十万只校验十万)={} 批次开始时间={} 批次结束时间={} 对象API={} 忽略开始时间={}", batch.getId(),batch.getFirstSfNum(),startDate,endDate, objectApi, ignoreBeginDate);
-
                     // 分页处理记录
                     int page = 0;
                     int totalRecordNum = 0;
@@ -938,15 +937,19 @@ public class DataVerifyServiceImpl implements DataVerifyService {
                     synchronized (this) {
                         excelWriter.write(Collections.singletonList(row), summarySheet);
                     }
-                }
+                } catch (Exception e){
-            } catch (Exception e){
+                    // 写入批次汇总结果
-                log.error("API={},处理异常：{}",objectApi, e.getMessage(),e);
+                    List<String> row = Lists.newArrayList();
-            }finally {
+                    row.add(String.valueOf(batch.getId()));
-                if (objectWriter != null) {
+                    row.add(objectApi);
-                    objectWriter.finish();
+                    row.add(startDate);
+                    row.add(endDate);
+                    row.add("校验异常");
+                    log.error("API={},处理异常：{}",objectApi, e.getMessage(),e);
                 }
             }
 
+
             if (sendEmail && hasError){
                 sendObjectQualityCheckEmailList(objectApi, arrayList);
             }else {

src/main/java/com/celnet/datadump/util/CsvConverterUtil.java

@@ -215,7 +215,7 @@ public class CsvConverterUtil {
                 for (int i = 0; i < header.length;  i++) {
                     // 将每个值转换为字符串（如果为null则转为空字符串）
                     Object value = jsonObject.get(header[i]);
-                    row[i] = value == null ? "" : value.toString();
+                    row[i] = String.valueOf(value);
                 }
                 csvWriter.writeLine(row);
             }

src/main/java/com/celnet/datadump/util/DataUtil.java

@@ -494,8 +494,13 @@ public class DataUtil {
             case "date":
                 return data+"T08:00:00Z";
             case "datetime":
+                // 解析输入的日期时间字符串为LocalDateTime对象
                 LocalDateTime localDateTime = LocalDateTime.parse(data);
-                OffsetDateTime offsetDateTime = localDateTime.atOffset(ZoneOffset.ofHours(0));
+                // 将LocalDateTime增加8小时以补偿时区差异
+                LocalDateTime plusEightHours = localDateTime.plusHours(8);
+                // 转换为UTC时区的OffsetDateTime对象
+                OffsetDateTime offsetDateTime = plusEightHours.atOffset(ZoneOffset.ofHours(0));
+                // 使用ISO标准格式化器将OffsetDateTime格式化为字符串并返回
                 return offsetDateTime.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME);
             case "time":
                 return adjustHour(data);

src/main/resources/mapper/CustomMapper.xml

@@ -107,7 +107,7 @@
         `${tableName}`
         SET
         <foreach item="map" collection="maps" open="" separator="," close="">
-            ${map.key} = #{map.value}
+            `${map.key}` = #{map.value}
         </foreach>
         WHERE
         id = #{id}
@@ -159,7 +159,7 @@
         insert into
         `${tableName}`
         <foreach item="map" collection="maps" open="(" separator="," close=")">
-            ${map.key}
+            `${map.key}`
         </foreach>
         values
         <foreach item="map" collection="maps" open="(" separator="," close=")">