kris/data-dump
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

【feat】 bigobject文件下载初版

Browse Source
This commit is contained in:
Kris 2025-09-26 11:41:07 +08:00
parent ff6f9c0ee1
commit 89dcbb48bd
1 changed files with 29 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
src/main/java/com/celnet/datadump/service/impl/CommonBatchServiceImpl.java
View File

@ -10,6 +10,7 @@ import com.celnet.datadump.config.SalesforceConnect;
import com.celnet.datadump.config.SalesforceExecutor;
import com.celnet.datadump.config.SalesforceTargetConnect;
import com.celnet.datadump.entity.*;
import com.celnet.datadump.enums.FileType;
import com.celnet.datadump.global.Const;
import com.celnet.datadump.global.SystemConfigCode;
import com.celnet.datadump.mapper.CustomMapper;
@ -142,21 +143,17 @@ public class CommonBatchServiceImpl implements CommonBatchService {
@Override
public ReturnT<String> dumpBigObject(SalesforceParam param) throws Exception {
List<Future<?>> futures = Lists.newArrayList();
try {
if (StringUtils.isNotBlank(param.getApi())) {
// 手动任务
ReturnT<String> result = manualBigObjectDump(param, futures);
if (result != null) {
return result;
}
// 手动任务
ReturnT<String> result = manualBigObjectDump(param);
if (result != null) {
return result;
}
return ReturnT.SUCCESS;
} catch (Throwable throwable) {
salesforceExecutor.remove(futures.toArray(new Future<?>[]{}));
log.error("dump error", throwable);
throw throwable;
}
return ReturnT.SUCCESS;
}
@Override
@ -178,7 +175,7 @@ public class CommonBatchServiceImpl implements CommonBatchService {
}
}
private ReturnT<String> manualBigObjectDump(SalesforceParam param, List<Future<?>> futures) throws Exception {
private ReturnT<String> manualBigObjectDump(SalesforceParam param) throws Exception {
String api = param.getApi();
@ -215,43 +212,48 @@ public class CommonBatchServiceImpl implements CommonBatchService {
QueryResultList queryResultList = bulkConnect.getQueryResultList(job.getId(), batchInfo.getId());
log.info("获取查询结果列表, 结果数量: {}", queryResultList.getResult().length);
// 检测路径是否存在 不存在则创建
Path apiPath = Paths.get(param.getApi());
if (!Files.exists(apiPath)) {
Files.createDirectories(apiPath);
log.info("创建目录: {}", apiPath.toAbsolutePath());
if (Const.FILE_TYPE == FileType.SERVER) {
// 检测路径是否存在 不存在则创建
File excel = new File(Const.SERVER_FILE_PATH + "/" + api);
if (!excel.exists()) {
log.info("创建API目录: {}", excel.getAbsolutePath());
boolean mkdir = excel.mkdir();
if (!mkdir) {
log.info("创建文件存储目录失败！");
}
}
}
// 使用确定性命名策略，避免使用随机时间戳
String fileName = param.getApi() + "/" + DateFormatUtils.format(new Date(), "yyyyMMddHHmmss");
String fileName = Const.SERVER_FILE_PATH + "/" + api+ "/" + DateFormatUtils.format(new Date(), "yyyyMMddHHmmss");
final long maxFileSize = 50 * 1024 * 1024; // 50MB
int fileIndex = 0;
Path currentFilePath = Paths.get(fileName + "_" + fileIndex + ".csv");
log.info("开始写入数据到文件 : {}", fileName);
BufferedWriter bufferedWriter = null;
FileWriter fileWriter = null;
try {
fileWriter = new FileWriter(currentFilePath.toFile(), true);
bufferedWriter = new BufferedWriter(fileWriter);
long currentFileSize = 0;
int resultIndex = 0;
for (final String resultId : queryResultList.getResult()) {
log.info("处理结果ID: {}, 进度: {}/{}", resultId, ++resultIndex, queryResultList.getResult().length);
InputStream resultStream = bulkConnect.getQueryResultStream(job.getId(), batchInfo.getId(), resultId);
try (BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(resultStream, StandardCharsets.UTF_8))) {
String line;
int lineCount = 0;
while ((line = bufferedReader.readLine()) != null) {
lineCount++;
byte[] lineBytes = (line + "\n").getBytes(StandardCharsets.UTF_8);
// 如果添加这行会超出文件大小限制，则创建新文件
if (currentFileSize + lineBytes.length > maxFileSize && currentFileSize > 0) {
// 关闭当前文件
@ -261,9 +263,9 @@ public class CommonBatchServiceImpl implements CommonBatchService {
if (fileWriter != null) {
fileWriter.close();
}
log.info("当前文件已达到大小限制, 创建新文件. 文件路径: {}, 数据行数: {}", currentFilePath.toString(), lineCount);
// 创建新文件
fileIndex++;
currentFilePath = Paths.get(fileName + "_" + fileIndex + ".csv");
@ -272,7 +274,7 @@ public class CommonBatchServiceImpl implements CommonBatchService {
currentFileSize = 0;
lineCount = 0;
}
// 写入数据
bufferedWriter.write(line + "\n");
currentFileSize += lineBytes.length;