【fix】 优化文件批量下载
This commit is contained in:
parent
1e346e19ac
commit
15e755ff4b
@ -238,14 +238,6 @@ public class CommonServiceImpl implements CommonService {
|
||||
}
|
||||
// 等待当前所有线程执行完成
|
||||
salesforceExecutor.waitForFutures(futures.toArray(new Future<?>[]{}));
|
||||
// 存在附件的开始dump
|
||||
list.stream().filter(t -> StringUtils.isNotBlank(t.getBlobField())).forEach(t -> {
|
||||
try {
|
||||
fileService.dumpFile(t.getName(), t.getBlobField(), true);
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
});
|
||||
return ReturnT.SUCCESS;
|
||||
} catch (Throwable throwable) {
|
||||
salesforceExecutor.remove(futures.toArray(new Future<?>[]{}));
|
||||
|
||||
@ -1035,6 +1035,7 @@ public class DataImportNewServiceImpl implements DataImportNewService {
|
||||
}
|
||||
|
||||
List<Map<String, Object>> poll = customMapper.list("code,value","org_config",null);
|
||||
|
||||
for (Map<String, Object> map1 : poll) {
|
||||
if ("FILE_DOWNLOAD_URL".equals(map1.get("code"))) {
|
||||
downloadUrl = (String) map1.get("value");
|
||||
@ -1072,12 +1073,23 @@ public class DataImportNewServiceImpl implements DataImportNewService {
|
||||
}).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
if (Const.FILE_TYPE == FileType.SERVER) {
|
||||
// 检测路径是否存在 不存在则创建
|
||||
File excel = new File(Const.SERVER_FILE_PATH + "/" + api);
|
||||
if (!excel.exists()) {
|
||||
boolean mkdir = excel.mkdir();
|
||||
if (!mkdir) {
|
||||
log.info("创建文件存储目录失败！");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 手动任务优先执行
|
||||
for (SalesforceParam salesforceParam : salesforceParams) {
|
||||
String finalDownloadUrl = downloadUrl;
|
||||
Future<?> future = salesforceExecutor.execute(() -> {
|
||||
try {
|
||||
saveFile(salesforceParam, connect, finalDownloadUrl,dataObject.getName(),dataObject.getExtraField());
|
||||
saveFile(salesforceParam, connect, finalDownloadUrl,dataObject.getName(),dataObject.getBlobField());
|
||||
} catch (Throwable throwable) {
|
||||
log.error("salesforceExecutor error", throwable);
|
||||
throw new RuntimeException(throwable);
|
||||
@ -1089,6 +1101,7 @@ public class DataImportNewServiceImpl implements DataImportNewService {
|
||||
salesforceExecutor.waitForFutures(futures.toArray(new Future<?>[]{}));
|
||||
update.setDataWork(0);
|
||||
} catch (InterruptedException e) {
|
||||
salesforceExecutor.remove(futures.toArray(new Future<?>[]{}));
|
||||
throw e;
|
||||
} catch (Throwable e) {
|
||||
throw new RuntimeException(e);
|
||||
@ -1106,18 +1119,13 @@ public class DataImportNewServiceImpl implements DataImportNewService {
|
||||
/**
|
||||
* 下载文件
|
||||
*/
|
||||
private void saveFile(SalesforceParam param, PartnerConnection Connection ,String downloadUrl, String api, String field) {
|
||||
private void saveFile(SalesforceParam param, PartnerConnection Connection ,String downloadUrl, String api, String field) throws Exception {
|
||||
|
||||
String extraSql = "";
|
||||
if (dataFieldService.hasDeleted(api)) {
|
||||
extraSql += "AND IsDeleted = false ";
|
||||
}
|
||||
if (Const.FILE_TYPE == FileType.SERVER) {
|
||||
// 检测路径是否存在 不存在则创建
|
||||
File excel = new File(Const.SERVER_FILE_PATH + "/" + api);
|
||||
if (!excel.exists()) {
|
||||
boolean mkdir = excel.mkdir();
|
||||
}
|
||||
extraSql += " AND IsDeleted = false ";
|
||||
}
|
||||
|
||||
Date beginDate = param.getBeginCreateDate();
|
||||
Date endDate = param.getEndCreateDate();
|
||||
String beginDateStr = DateUtil.format(beginDate, "yyyy-MM-dd HH:mm:ss");
|
||||
@ -1125,27 +1133,32 @@ public class DataImportNewServiceImpl implements DataImportNewService {
|
||||
|
||||
String token = Connection.getSessionHeader().getSessionId();
|
||||
|
||||
try {
|
||||
String name = getName(api);
|
||||
log.info("api:{},field:{},开始时间:{},结束时间：{}", api, field,beginDateStr,endDateStr);
|
||||
|
||||
Map<String, String> headers = Maps.newHashMap();
|
||||
headers.put("Authorization", "Bearer " + token);
|
||||
headers.put("connection", "keep-alive");
|
||||
long num = 0;
|
||||
while (true) {
|
||||
// 获取未存储的附件id
|
||||
List<Map<String, Object>> list = customMapper.list("Id, " + name, api, " is_dump = false and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + extraSql + "' limit 10");
|
||||
if (CollectionUtils.isEmpty(list)) {
|
||||
break;
|
||||
|
||||
Integer count = customMapper.countBySQL(api, "where is_dump = 0 and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr +"'"+ extraSql );
|
||||
|
||||
log.error("总文件数 count:{}；-开始时间:{}；-结束时间:{}；-api:{}；", count, beginDateStr, endDateStr, api);
|
||||
|
||||
if (count == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
int page = count%200 == 0 ? count/200 : (count/200) + 1;
|
||||
//总插入数
|
||||
for (int i = 0; i < page; i++) {
|
||||
|
||||
// 获取未存储的附件id
|
||||
List<Map<String, Object>> list = customMapper.list("Id, " + name, api, " is_dump = 0 and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr +"'"+ extraSql + "limit " + i * 200 + ",200");
|
||||
|
||||
for (Map<String, Object> map : list) {
|
||||
String id = null;
|
||||
// 上传完毕 更新附件信息
|
||||
List<Map<String, Object>> maps = Lists.newArrayList();
|
||||
boolean isDump = true;
|
||||
int failCount = 0;
|
||||
while (true) {
|
||||
try {
|
||||
id = (String) map.get("Id");
|
||||
String fileName = (String) map.get(name);
|
||||
@ -1155,9 +1168,12 @@ public class DataImportNewServiceImpl implements DataImportNewService {
|
||||
String filePath = api + "/" + id + "_" + fileName;
|
||||
// 拼接url
|
||||
String url = downloadUrl + String.format(Const.SF_FILE_URL, api, id, field);
|
||||
|
||||
log.info("文件下载请求地址：{}",url);
|
||||
Response response = HttpUtil.doGet(url, null, headers);
|
||||
if (response.body() != null) {
|
||||
if (response.body() != null && response.code() == 200) {
|
||||
InputStream inputStream = response.body().byteStream();
|
||||
log.info("文件下载返回状态码：{}，返回信息：{}", response.code(),response.message());
|
||||
switch (Const.FILE_TYPE) {
|
||||
case OSS:
|
||||
// 上传到oss
|
||||
@ -1177,52 +1193,30 @@ public class DataImportNewServiceImpl implements DataImportNewService {
|
||||
}
|
||||
paramMap.put("value", filePath);
|
||||
maps.add(paramMap);
|
||||
}else {
|
||||
log.error("文件下载失败！, id: "+ id + "，返回体信息：" + response.message());
|
||||
EmailUtil.send("File Dump ERROR", "文件下载失败！, id: "+ id + "，返回体信息：" + response.message());
|
||||
}
|
||||
}
|
||||
Map<String, Object> paramMap = Maps.newHashMap();
|
||||
paramMap.put("key", "is_dump");
|
||||
paramMap.put("value", isDump);
|
||||
paramMap.put("value", true);
|
||||
maps.add(paramMap);
|
||||
customMapper.updateById(api, maps, id);
|
||||
TimeUnit.MILLISECONDS.sleep(1);
|
||||
break;
|
||||
} catch (Throwable throwable) {
|
||||
log.error("dump file error, id: {}", id, throwable);
|
||||
failCount++;
|
||||
if (Const.MAX_FAIL_COUNT < failCount) {
|
||||
{
|
||||
} catch (InterruptedException interruptedException){
|
||||
return;
|
||||
} catch (Exception e) {
|
||||
log.error("文件下载失败！, id: {}, 错误信息：{}", id ,e.getMessage());
|
||||
Map<String, Object> paramMap = Maps.newHashMap();
|
||||
paramMap.put("key", "is_dump");
|
||||
paramMap.put("value", 2);
|
||||
maps.add(paramMap);
|
||||
}
|
||||
customMapper.updateById(api, maps, id);
|
||||
break;
|
||||
}
|
||||
try {
|
||||
TimeUnit.SECONDS.sleep(30);
|
||||
} catch (InterruptedException e) {
|
||||
throw new RuntimeException(e);
|
||||
EmailUtil.send("File Dump ERROR", "文件下载失败！, id: "+ id + "，错误信息：" + e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
num += list.size();
|
||||
log.info("dump file count api:{}, field:{}, num:{}", api, field, num);
|
||||
}
|
||||
log.info("dump file success api:{}, field:{}, num:{}", api, field, num);
|
||||
|
||||
} catch (Throwable throwable) {
|
||||
log.error("dump file error", throwable);
|
||||
} finally {
|
||||
// 把is_dump为2的重置为0
|
||||
List<Map<String, Object>> maps = Lists.newArrayList();
|
||||
Map<String, Object> paramMap = Maps.newHashMap();
|
||||
paramMap.put("key", "is_dump");
|
||||
paramMap.put("value", 0);
|
||||
maps.add(paramMap);
|
||||
customMapper.update(maps, api, "is_dump = 2");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -90,10 +90,7 @@
|
||||
AND id IN <foreach item="id" collection="param.ids" open="(" separator="," close=")">#{id}</foreach>
|
||||
</if>
|
||||
<if test="param.beginModifyDate != null">
|
||||
AND LastModifiedDate >= #{param.beginModifyDate}
|
||||
</if>
|
||||
<if test="param.beginModifyDate != null">
|
||||
AND LastModifiedDate >= #{param.beginModifyDate}
|
||||
AND SystemModstamp >= #{param.beginModifyDate}
|
||||
</if>
|
||||
<if test="param.isDeleted != null">
|
||||
AND IsDeleted = #{param.isDeleted}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user