【feat】 数据质量校验：1、所有字段校验 2、校验数据量动态配置，写入配置表

(cherry picked from commit 4a969e53b9)

src/main/java/com/celnet/datadump/global/SystemConfigCode.java

@@ -45,6 +45,12 @@ public class SystemConfigCode {
      */
     public static final String BATCH_PROCESS_COUNT = "BATCH_PROCESS_COUNT";
 
+    /**
+     * 数据质量校验最大值
+     */
+    public static final String TOTAL_VERIFY_DATE = "TOTAL_VERIFY_DATE";
+
+
     public static final String BATCH_TYPE_WEEK = "WEEK";
     public static final String BATCH_TYPE_MONTH = "MONTH";
     public static final String BATCH_TYPE_YEAR = "YEAR";

src/main/java/com/celnet/datadump/service/impl/DataVerifyServiceImpl.java

@@ -15,6 +15,7 @@ import com.celnet.datadump.config.SalesforceTargetConnect;
 import com.celnet.datadump.entity.*;
 import com.celnet.datadump.enums.FileType;
 import com.celnet.datadump.global.Const;
+import com.celnet.datadump.global.SystemConfigCode;
 import com.celnet.datadump.param.DataVerifyParam;
 import com.celnet.datadump.param.SalesforceParam;
 import com.celnet.datadump.mapper.CustomMapper;
@@ -41,6 +42,7 @@ import org.checkerframework.checker.units.qual.A;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
+import javax.annotation.PostConstruct;
 import java.io.File;
 import java.util.*;
 import java.util.concurrent.Future;
@@ -73,16 +75,35 @@ public class DataVerifyServiceImpl implements DataVerifyService {
     private SalesforceTargetConnect salesforceTargetConnect;
     @Autowired
     private DataObjectService dataObjectService;
+    @Autowired
+    private SystemConfigService systemConfigService;
 
     private static final Integer MAX_FAIL_COUNT = 3;
     private static final Integer MAX_BATCH_RECORDS = 200;
     private static final Integer MAX_PAGE_RECORDS = 10000;
-    private static final Integer MAX_TOTAL_RECORDS = 100000;
+    private Integer MAX_TOTAL_RECORDS = 100000;
     private static final Integer MAX_EXCEL_ROWS = 1000000; // 设置一个安全的最大行数，留一些空间给表头等
 
     private static final String TEMP_FILE_PATH = Const.SERVER_FILE_PATH + "/excel/";
 
 
+    @PostConstruct
+    public void init() {
+        QueryWrapper<SystemConfig> queryWrapper = new QueryWrapper<>();
+        queryWrapper.eq("code", SystemConfigCode.TOTAL_VERIFY_DATE);
+        SystemConfig systemConfig = systemConfigService.getOne(queryWrapper);
+        if (systemConfig != null && systemConfig.getValue() != null) {
+            try {
+                MAX_TOTAL_RECORDS = Integer.parseInt(systemConfig.getValue());
+                log.info("数据质量校验最大值已设置为: {}", MAX_TOTAL_RECORDS);
+            } catch (NumberFormatException e) {
+                log.warn("无法解析数据质量校验最大值配置，使用默认值: {}", MAX_TOTAL_RECORDS);
+            }
+        } else {
+            log.info("未找到数据质量校验最大值配置，使用默认值: {}", MAX_TOTAL_RECORDS);
+        }
+    }
+
 
     static {
         if (Const.FILE_TYPE == FileType.SERVER) {
@@ -1888,8 +1909,7 @@ private void sendIncrementalQualityCheckEmail(DataVerifyParam param, String file
 
     private List<DataField> getVerifiableFields(String objectApi) {
         return dataFieldService.list(new  QueryWrapper<DataField>()
-                .eq("api", objectApi)
+                .eq("api", objectApi));
-                .and(wrapper -> wrapper.eq("is_createable",  1).or().eq("is_updateable", 1)));
     }
 
     private List<DataBatch> getDataBatches(String objectApi) {