kris/data-dump
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

【feat】 数据质量校验：1、所有字段校验 2、校验数据量动态配置，写入配置表

Browse Source
This commit is contained in:
Kris 2025-10-22 10:46:52 +08:00
parent 32047b2287
commit 4a969e53b9
2 changed files with 29 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/main/java/com/celnet/datadump/global/SystemConfigCode.java
View File

@ -45,6 +45,12 @@ public class SystemConfigCode {
*/
public static final String BATCH_PROCESS_COUNT = "BATCH_PROCESS_COUNT";
/**
* 数据质量校验最大值
*/
public static final String TOTAL_VERIFY_DATE = "TOTAL_VERIFY_DATE";
public static final String BATCH_TYPE_WEEK = "WEEK";
public static final String BATCH_TYPE_MONTH = "MONTH";
public static final String BATCH_TYPE_YEAR = "YEAR";

26
src/main/java/com/celnet/datadump/service/impl/DataVerifyServiceImpl.java
View File

@ -15,6 +15,7 @@ import com.celnet.datadump.config.SalesforceTargetConnect;
import com.celnet.datadump.entity.*;
import com.celnet.datadump.enums.FileType;
import com.celnet.datadump.global.Const;
import com.celnet.datadump.global.SystemConfigCode;
import com.celnet.datadump.param.DataVerifyParam;
import com.celnet.datadump.param.SalesforceParam;
import com.celnet.datadump.mapper.CustomMapper;
@ -41,6 +42,7 @@ import org.checkerframework.checker.units.qual.A;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import java.io.File;
import java.util.*;
import java.util.concurrent.Future;
@ -73,16 +75,35 @@ public class DataVerifyServiceImpl implements DataVerifyService {
private SalesforceTargetConnect salesforceTargetConnect;
@Autowired
private DataObjectService dataObjectService;
@Autowired
private SystemConfigService systemConfigService;
private static final Integer MAX_FAIL_COUNT = 3;
private static final Integer MAX_BATCH_RECORDS = 200;
private static final Integer MAX_PAGE_RECORDS = 10000;
private static final Integer MAX_TOTAL_RECORDS = 100000;
private Integer MAX_TOTAL_RECORDS = 100000;
private static final Integer MAX_EXCEL_ROWS = 1000000; // 设置一个安全的最大行数，留一些空间给表头等
private static final String TEMP_FILE_PATH = Const.SERVER_FILE_PATH + "/excel/";
@PostConstruct
public void init() {
QueryWrapper<SystemConfig> queryWrapper = new QueryWrapper<>();
queryWrapper.eq("code", SystemConfigCode.TOTAL_VERIFY_DATE);
SystemConfig systemConfig = systemConfigService.getOne(queryWrapper);
if (systemConfig != null && systemConfig.getValue() != null) {
try {
MAX_TOTAL_RECORDS = Integer.parseInt(systemConfig.getValue());
log.info("数据质量校验最大值已设置为: {}", MAX_TOTAL_RECORDS);
} catch (NumberFormatException e) {
log.warn("无法解析数据质量校验最大值配置，使用默认值: {}", MAX_TOTAL_RECORDS);
}
} else {
log.info("未找到数据质量校验最大值配置，使用默认值: {}", MAX_TOTAL_RECORDS);
}
}
static {
if (Const.FILE_TYPE == FileType.SERVER) {
@ -1888,8 +1909,7 @@ private void sendIncrementalQualityCheckEmail(DataVerifyParam param, String file
private List<DataField> getVerifiableFields(String objectApi) {
return dataFieldService.list(new QueryWrapper<DataField>()
.eq("api", objectApi)
.and(wrapper -> wrapper.eq("is_createable", 1).or().eq("is_updateable", 1)));
.eq("api", objectApi));
}
private List<DataBatch> getDataBatches(String objectApi) {