kris/data-dump
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[feat] bigobject下载

Browse Source
This commit is contained in:
Kris 2025-09-25 19:58:57 +08:00
parent 6c8f75f57f
commit afb7abd999
2 changed files with 13 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
src/main/java/com/celnet/datadump/service/impl/CommonBatchServiceImpl.java
View File

@ -189,13 +189,6 @@ public class CommonBatchServiceImpl implements CommonBatchService {
log.info("开始导出BigObject数据, API: {}", api); log.info("开始导出BigObject数据, API: {}", api);
// 检测路径是否存在 不存在则创建
File excel = new File( param.getApi() + "/");
if (!excel.exists()) {
System.out.println("创建目录: " + param.getApi() + "/");
boolean mkdir = excel.mkdir();
}
List<DataField> fieldList = dataFieldService.list(new QueryWrapper<DataField>().eq("api", param.getApi())); List<DataField> fieldList = dataFieldService.list(new QueryWrapper<DataField>().eq("api", param.getApi()));
String fieldStr = fieldList.stream() String fieldStr = fieldList.stream()
.map(DataField::getField) .map(DataField::getField)
@ -222,22 +215,27 @@ public class CommonBatchServiceImpl implements CommonBatchService {
QueryResultList queryResultList = bulkConnect.getQueryResultList(job.getId(), batchInfo.getId()); QueryResultList queryResultList = bulkConnect.getQueryResultList(job.getId(), batchInfo.getId());
log.info("获取查询结果列表, 结果数量: {}", queryResultList.getResult().length); log.info("获取查询结果列表, 结果数量: {}", queryResultList.getResult().length);
// 检测路径是否存在 不存在则创建
Path apiPath = Paths.get(param.getApi());
if (!Files.exists(apiPath)) {
log.info("创建目录: {}", apiPath);
Files.createDirectories(apiPath);
}
// 使用确定性命名策略，避免使用随机时间戳 // 使用确定性命名策略，避免使用随机时间戳
String fileName = param.getApi() + "/" + DateFormatUtils.format(new Date(), "yyyyMMddHHmmss"); String fileName = param.getApi() + "/" + DateFormatUtils.format(new Date(), "yyyyMMddHHmmss");
final long maxFileSize = 50 * 1024 * 1024; // 50MB final long maxFileSize = 50 * 1024 * 1024; // 50MB
int fileIndex = 0; int fileIndex = 0;
String currentFilePath = fileName + "_" + fileIndex + ".csv"; Path currentFilePath = Paths.get(fileName + "_" + fileIndex + ".csv");
log.info("开始写入数据到文件 : {}", fileName); log.info("开始写入数据到文件 : {}", fileName);
BufferedWriter bufferedWriter = null; BufferedWriter bufferedWriter = null;
FileWriter fileWriter = null; FileWriter fileWriter = null;
try { try {
fileWriter = new FileWriter(currentFilePath, true); fileWriter = new FileWriter(currentFilePath.toFile(), true);
bufferedWriter = new BufferedWriter(fileWriter); bufferedWriter = new BufferedWriter(fileWriter);
long currentFileSize = 0; long currentFileSize = 0;
@ -264,12 +262,12 @@ public class CommonBatchServiceImpl implements CommonBatchService {
fileWriter.close(); fileWriter.close();
} }
log.info("当前文件已达到大小限制, 创建新文件. 文件路径: {}, 数据行数: {}", currentFilePath, lineCount); log.info("当前文件已达到大小限制, 创建新文件. 文件路径: {}, 数据行数: {}", currentFilePath.toString(), lineCount);
// 创建新文件 // 创建新文件
fileIndex++; fileIndex++;
currentFilePath = fileName + "_" + fileIndex + ".csv"; currentFilePath = Paths.get(fileName + "_" + fileIndex + ".csv");
fileWriter = new FileWriter(currentFilePath, true); fileWriter = new FileWriter(currentFilePath.toFile(), true);
bufferedWriter = new BufferedWriter(fileWriter); bufferedWriter = new BufferedWriter(fileWriter);
currentFileSize = 0; currentFileSize = 0;
lineCount = 0; lineCount = 0;

2
src/main/java/com/celnet/datadump/service/impl/DataVerifyServiceImpl.java
View File

@ -1327,7 +1327,7 @@ public class DataVerifyServiceImpl implements DataVerifyService {
private List<DataBatch> getDataBatches(String objectApi) { private List<DataBatch> getDataBatches(String objectApi) {
return dataBatchService.list(new QueryWrapper<DataBatch>() return dataBatchService.list(new QueryWrapper<DataBatch>()
.eq("name", objectApi) .eq("name", objectApi).isNotNull("first_sync_date")
.orderByDesc("sync_start_date")); .orderByDesc("sync_start_date"));
} }
} }