kris/data-dump
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

【feat】 优化调整bulk批次为每批次10000条

Browse Source
This commit is contained in:
Kris 2025-06-09 10:36:59 +08:00
parent f221ee3f69
commit d087e1b8c5
1 changed files with 8 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/main/java/com/celnet/datadump/service/impl/DataImportBatchServiceImpl.java
View File

@ -200,13 +200,13 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
if (count == 0) { if (count == 0) {
return; return;
} }
//批量插入2000一次 //批量插入10000一次
int page = count%2000 == 0 ? count/2000 : (count/2000) + 1; int page = count%10000 == 0 ? count/10000 : (count/10000) + 1;
//总插入数 //总插入数
int sfNum = 0; int sfNum = 0;
for (int i = 0; i < page; i++) { for (int i = 0; i < page; i++) {
List<JSONObject> data = customMapper.listJsonObject("*", api, "new_id is null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "' limit 2000"); List<JSONObject> data = customMapper.listJsonObject("*", api, "new_id is null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "' limit 10000");
int size = data.size(); int size = data.size();
log.info("执行api:{}, 执行page:{}, 执行size:{}", api, i+1, size); log.info("执行api:{}, 执行page:{}, 执行size:{}", api, i+1, size);
@ -262,7 +262,7 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
// 转换为UTC时间并格式化 // 转换为UTC时间并格式化
LocalDateTime localDateTime = LocalDateTime.parse(String.valueOf(data.get(j - 1).get("CreatedDate")), inputFormatter); LocalDateTime localDateTime = LocalDateTime.parse(String.valueOf(data.get(j - 1).get("CreatedDate")), inputFormatter);
ZonedDateTime utcDateTime = localDateTime.atZone(ZoneId.of("UTC")); ZonedDateTime utcDateTime = localDateTime.atZone(ZoneId.of("UTC")).minusHours(8) ;
String convertedTime = utcDateTime.format(outputFormatter); String convertedTime = utcDateTime.format(outputFormatter);
@ -276,7 +276,7 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
ids[j-1] = data.get(j-1).get("Id").toString(); ids[j-1] = data.get(j-1).get("Id").toString();
insertList.add(account); insertList.add(account);
if (i*2000+j == count){ if (i*10000+j == count){
break; break;
} }
} }
@ -497,10 +497,10 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
} }
// 总更新数 // 总更新数
int sfNum = 0; int sfNum = 0;
// 批量更新2000一次 // 批量更新10000一次
int page = count%2000 == 0 ? count/2000 : (count/2000) + 1; int page = count%10000 == 0 ? count/10000 : (count/10000) + 1;
for (int i = 0; i < page; i++) { for (int i = 0; i < page; i++) {
List<Map<String, Object>> mapList = customMapper.list("*", api, "new_id is not null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "' order by Id asc limit " + i * 2000 + ",2000"); List<Map<String, Object>> mapList = customMapper.list("*", api, "new_id is not null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "' order by Id asc limit " + i * 10000 + ",10000");
List<JSONObject> updateList = new ArrayList<>(); List<JSONObject> updateList = new ArrayList<>();