kris/data-dump
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

【fix】 调整share相关对象，及文件下载

Browse Source
This commit is contained in:
Kris 2025-07-30 16:37:19 +08:00
parent 56dafe6274
commit 62dd9db39b
2 changed files with 23 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/java/com/celnet/datadump/service/impl/DataImportBatchServiceImpl.java
View File

@ -924,7 +924,7 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
update.setDataLock(1); update.setDataLock(1);
dataObjectService.updateById(update); dataObjectService.updateById(update);
if (api.contains("Share")){ if (api.endsWith("Share")){
insertSingleShareData(api,bulkConnection); insertSingleShareData(api,bulkConnection);
continue; continue;
} }
@ -1002,7 +1002,7 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
update.setDataLock(1); update.setDataLock(1);
dataObjectService.updateById(update); dataObjectService.updateById(update);
if (api.contains("Share")){ if (api.endsWith("Share")){
insertSingleShareData(api,bulkConnection); insertSingleShareData(api,bulkConnection);
continue; continue;
} }

35
src/main/java/com/celnet/datadump/service/impl/DataImportNewServiceImpl.java
View File

@ -726,7 +726,7 @@ public class DataImportNewServiceImpl implements DataImportNewService {
} }
if (1 == param.getType()) { if (1 == param.getType()) {
if (api.contains("Share")){ if (api.endsWith("Share")){
sql = "where RowCause = 'Manual' and new_id is not null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "'"; sql = "where RowCause = 'Manual' and new_id is not null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "'";
sql2 = "RowCause = 'Manual' and new_id is not null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "' order by Id asc limit "; sql2 = "RowCause = 'Manual' and new_id is not null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "' order by Id asc limit ";
}else { }else {
@ -734,7 +734,7 @@ public class DataImportNewServiceImpl implements DataImportNewService {
sql2 = "new_id is not null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "' order by Id asc limit "; sql2 = "new_id is not null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "' order by Id asc limit ";
} }
}else { }else {
if (api.contains("Share")){ if (api.endsWith("Share")){
sql = "where RowCause = 'Manual' and new_id is not null and LastModifiedDate >= '" + beginDateStr + "' "; sql = "where RowCause = 'Manual' and new_id is not null and LastModifiedDate >= '" + beginDateStr + "' ";
sql2 = "RowCause = 'Manual' and new_id is not null and LastModifiedDate >= '" + beginDateStr + "' order by Id asc limit "; sql2 = "RowCause = 'Manual' and new_id is not null and LastModifiedDate >= '" + beginDateStr + "' order by Id asc limit ";
}else { }else {
@ -1155,7 +1155,6 @@ public class DataImportNewServiceImpl implements DataImportNewService {
if (Const.CONTENT_VERSION.equalsIgnoreCase(api)) { if (Const.CONTENT_VERSION.equalsIgnoreCase(api)) {
name = "Title,FileExtension"; name = "Title,FileExtension";
} }
log.info("api:{},field:{},开始时间:{},结束时间：{}", api, field,beginDateStr,endDateStr);
Map<String, String> headers = Maps.newHashMap(); Map<String, String> headers = Maps.newHashMap();
headers.put("Authorization", "Bearer " + token); headers.put("Authorization", "Bearer " + token);
@ -1163,18 +1162,20 @@ public class DataImportNewServiceImpl implements DataImportNewService {
Integer count = customMapper.countBySQL(api, "where is_dump = 0 and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr +"'"+ extraSql ); Integer count = customMapper.countBySQL(api, "where is_dump = 0 and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr +"'"+ extraSql );
log.error("总文件数 count:{}；-开始时间:{}；-结束时间:{}；-api:{}；", count, beginDateStr, endDateStr, api); log.info("api：{}；总文件数 count:{}；-开始时间:{}；-结束时间:{}；",api, count, beginDateStr, endDateStr);
if (count == 0) { if (count == 0) {
return; return;
} }
int page = count%200 == 0 ? count/200 : (count/200) + 1; int page = count%1000 == 0 ? count/1000 : (count/1000) + 1;
//总插入数 //总插入数
for (int i = 0; i < page; i++) { for (int i = 0; i < page; i++) {
log.info("api：{} 批次:{}；-开始时间:{}；-结束时间:{}；",api, i, beginDateStr, endDateStr);
// 获取未存储的附件id // 获取未存储的附件id
List<Map<String, Object>> list = customMapper.list("Id, " + name, api, " is_dump = 0 and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr +"'"+ extraSql + "limit " + i * 200 + ",200"); List<Map<String, Object>> list = customMapper.list("Id, " + name, api, " is_dump = 0 and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr +"'"+ extraSql + "limit " + i * 1000 + ",1000");
for (Map<String, Object> map : list) { for (Map<String, Object> map : list) {
String id = null; String id = null;
@ -1188,23 +1189,23 @@ public class DataImportNewServiceImpl implements DataImportNewService {
}else { }else {
fileName = (String) map.get("Name"); fileName = (String) map.get("Name");
} }
log.info("------------文件名:" + id + "_" + fileName); // log.info("------------文件名:" + id + "_" + fileName);
// 判断路径是否为空 // 判断路径是否为空
if (StringUtils.isNotBlank(fileName)) { if (StringUtils.isNotBlank(fileName)) {
String filePath = api + "/" + id + "_" + fileName; String filePath = api + "/" + id + "_" + fileName;
// 拼接url // 拼接url
String url = downloadUrl + String.format(Const.SF_FILE_URL, api, id, field); String url = downloadUrl + String.format(Const.SF_FILE_URL, api, id, field);
log.info("文件下载请求地址：{}",url); // log.info("文件下载请求地址：{}",url);
Response response = HttpUtil.doGet(url, null, headers); Response response = HttpUtil.doGet(url, null, headers);
if (response.body() != null && response.code() == 200) { if (response.body() != null && response.code() == 200) {
InputStream inputStream = response.body().byteStream(); InputStream inputStream = response.body().byteStream();
log.info("文件下载返回状态码：{}，返回信息：{}", response.code(),response.message()); // log.info("文件下载返回状态码：{}，返回信息：{}", response.code(),response.message());
switch (Const.FILE_TYPE) { switch (Const.FILE_TYPE) {
case OSS: case OSS:
// 上传到oss // 上传到oss
OssUtil.upload(inputStream, filePath); OssUtil.upload(inputStream, filePath);
case SERVER: default:
dumpToServer(headers, id, filePath, url, response, inputStream); dumpToServer(headers, id, filePath, url, response, inputStream);
} }
Map<String, Object> paramMap = Maps.newHashMap(); Map<String, Object> paramMap = Maps.newHashMap();
@ -1227,6 +1228,7 @@ public class DataImportNewServiceImpl implements DataImportNewService {
customMapper.updateById(api, maps, id); customMapper.updateById(api, maps, id);
TimeUnit.MILLISECONDS.sleep(1); TimeUnit.MILLISECONDS.sleep(1);
} catch (InterruptedException interruptedException){ } catch (InterruptedException interruptedException){
log.info("文件下载手动终止！");
return; return;
} catch (Exception e) { } catch (Exception e) {
log.error("文件下载失败！, id: {}, 错误信息：{}", id ,e.getMessage()); log.error("文件下载失败！, id: {}, 错误信息：{}", id ,e.getMessage());
@ -1401,7 +1403,7 @@ public class DataImportNewServiceImpl implements DataImportNewService {
Integer count = customMapper.countBySQL(api, "where is_dump = 1 and is_upload = 0 and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr +"'"+ extraSql ); Integer count = customMapper.countBySQL(api, "where is_dump = 1 and is_upload = 0 and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr +"'"+ extraSql );
log.error("总文件数 count:{}；-开始时间:{}；-结束时间:{}；-api:{}；", count, beginDateStr, endDateStr, api); log.info("总文件数 count:{}；-开始时间:{}；-结束时间:{}；-api:{}；", count, beginDateStr, endDateStr, api);
if (count == 0) { if (count == 0) {
return; return;
@ -1537,7 +1539,7 @@ public class DataImportNewServiceImpl implements DataImportNewService {
Integer count = customMapper.countBySQL("ContentDocument", "where new_id is null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr +"'"+ extraSql ); Integer count = customMapper.countBySQL("ContentDocument", "where new_id is null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr +"'"+ extraSql );
log.error("总文件数 count:{}；-开始时间:{}；-结束时间:{}；-api:{}；", count, beginDateStr, endDateStr, api); log.info("总文件数 count:{}；-开始时间:{}；-结束时间:{}；-api:{}；", count, beginDateStr, endDateStr, api);
if (count == 0) { if (count == 0) {
return; return;
@ -1863,8 +1865,13 @@ public class DataImportNewServiceImpl implements DataImportNewService {
String sql1 = null; String sql1 = null;
String sql2 = null; String sql2 = null;
if (beginDateStr != null){ if (beginDateStr != null){
sql1 = "where SystemModstamp >= " + beginDateStr; if (api.endsWith("Share")){
sql2 = "SystemModstamp >= "+ beginDateStr +" order by Id limit " ; sql1 = "where lastModifiedDate >= '" + beginDateStr + "'";
sql2 = "lastModifiedDate >= '"+ beginDateStr +"' order by Id limit " ;
}else {
sql1 = "where SystemModstamp >= '" + beginDateStr + "'";
sql2 = "SystemModstamp >= '"+ beginDateStr +"' order by Id limit " ;
}
}else { }else {
sql1 = null; sql1 = null;
sql2 = "1=1 order by Id limit " ; sql2 = "1=1 order by Id limit " ;