diff --git a/src/main/java/com/celnet/datadump/global/SystemConfigCode.java b/src/main/java/com/celnet/datadump/global/SystemConfigCode.java index a4f8075..a6d4199 100644 --- a/src/main/java/com/celnet/datadump/global/SystemConfigCode.java +++ b/src/main/java/com/celnet/datadump/global/SystemConfigCode.java @@ -45,6 +45,12 @@ public class SystemConfigCode { */ public static final String BATCH_PROCESS_COUNT = "BATCH_PROCESS_COUNT"; + /** + * 数据质量校验最大值 + */ + public static final String TOTAL_VERIFY_DATE = "TOTAL_VERIFY_DATE"; + + public static final String BATCH_TYPE_WEEK = "WEEK"; public static final String BATCH_TYPE_MONTH = "MONTH"; public static final String BATCH_TYPE_YEAR = "YEAR"; diff --git a/src/main/java/com/celnet/datadump/service/impl/DataVerifyServiceImpl.java b/src/main/java/com/celnet/datadump/service/impl/DataVerifyServiceImpl.java index 372634c..c43a2cf 100644 --- a/src/main/java/com/celnet/datadump/service/impl/DataVerifyServiceImpl.java +++ b/src/main/java/com/celnet/datadump/service/impl/DataVerifyServiceImpl.java @@ -15,6 +15,7 @@ import com.celnet.datadump.config.SalesforceTargetConnect; import com.celnet.datadump.entity.*; import com.celnet.datadump.enums.FileType; import com.celnet.datadump.global.Const; +import com.celnet.datadump.global.SystemConfigCode; import com.celnet.datadump.param.DataVerifyParam; import com.celnet.datadump.param.SalesforceParam; import com.celnet.datadump.mapper.CustomMapper; @@ -41,6 +42,7 @@ import org.checkerframework.checker.units.qual.A; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import javax.annotation.PostConstruct; import java.io.File; import java.util.*; import java.util.concurrent.Future; @@ -73,16 +75,35 @@ public class DataVerifyServiceImpl implements DataVerifyService { private SalesforceTargetConnect salesforceTargetConnect; @Autowired private DataObjectService dataObjectService; + @Autowired + private SystemConfigService systemConfigService; private static final Integer MAX_FAIL_COUNT = 3; private static final Integer MAX_BATCH_RECORDS = 200; private static final Integer MAX_PAGE_RECORDS = 10000; - private static final Integer MAX_TOTAL_RECORDS = 100000; + private Integer MAX_TOTAL_RECORDS = 100000; private static final Integer MAX_EXCEL_ROWS = 1000000; // 设置一个安全的最大行数,留一些空间给表头等 private static final String TEMP_FILE_PATH = Const.SERVER_FILE_PATH + "/excel/"; + @PostConstruct + public void init() { + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("code", SystemConfigCode.TOTAL_VERIFY_DATE); + SystemConfig systemConfig = systemConfigService.getOne(queryWrapper); + if (systemConfig != null && systemConfig.getValue() != null) { + try { + MAX_TOTAL_RECORDS = Integer.parseInt(systemConfig.getValue()); + log.info("数据质量校验最大值已设置为: {}", MAX_TOTAL_RECORDS); + } catch (NumberFormatException e) { + log.warn("无法解析数据质量校验最大值配置,使用默认值: {}", MAX_TOTAL_RECORDS); + } + } else { + log.info("未找到数据质量校验最大值配置,使用默认值: {}", MAX_TOTAL_RECORDS); + } + } + static { if (Const.FILE_TYPE == FileType.SERVER) { @@ -1888,8 +1909,7 @@ private void sendIncrementalQualityCheckEmail(DataVerifyParam param, String file private List getVerifiableFields(String objectApi) { return dataFieldService.list(new QueryWrapper() - .eq("api", objectApi) - .and(wrapper -> wrapper.eq("is_createable", 1).or().eq("is_updateable", 1))); + .eq("api", objectApi)); } private List getDataBatches(String objectApi) {