kris/data-dump
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

【fix】 调整share相关对象，及文件下载

Browse Source
This commit is contained in:
Kris 2025-07-30 16:37:19 +08:00
parent 56dafe6274
commit 62dd9db39b
2 changed files with 23 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/java/com/celnet/datadump/service/impl/DataImportBatchServiceImpl.java
View File

@ -924,7 +924,7 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
update.setDataLock(1);
dataObjectService.updateById(update);
if (api.contains("Share")){
if (api.endsWith("Share")){
insertSingleShareData(api,bulkConnection);
continue;
}
@ -1002,7 +1002,7 @@ public class DataImportBatchServiceImpl implements DataImportBatchService {
update.setDataLock(1);
dataObjectService.updateById(update);
if (api.contains("Share")){
if (api.endsWith("Share")){
insertSingleShareData(api,bulkConnection);
continue;
}

35
src/main/java/com/celnet/datadump/service/impl/DataImportNewServiceImpl.java
View File

@ -726,7 +726,7 @@ public class DataImportNewServiceImpl implements DataImportNewService {
}
if (1 == param.getType()) {
if (api.contains("Share")){
if (api.endsWith("Share")){
sql = "where RowCause = 'Manual' and new_id is not null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "'";
sql2 = "RowCause = 'Manual' and new_id is not null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "' order by Id asc limit ";
}else {
@ -734,7 +734,7 @@ public class DataImportNewServiceImpl implements DataImportNewService {
sql2 = "new_id is not null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "' order by Id asc limit ";
}
}else {
if (api.contains("Share")){
if (api.endsWith("Share")){
sql = "where RowCause = 'Manual' and new_id is not null and LastModifiedDate >= '" + beginDateStr + "' ";
sql2 = "RowCause = 'Manual' and new_id is not null and LastModifiedDate >= '" + beginDateStr + "' order by Id asc limit ";
}else {
@ -1155,7 +1155,6 @@ public class DataImportNewServiceImpl implements DataImportNewService {
if (Const.CONTENT_VERSION.equalsIgnoreCase(api)) {
name = "Title,FileExtension";
}
log.info("api:{},field:{},开始时间:{},结束时间：{}", api, field,beginDateStr,endDateStr);
Map<String, String> headers = Maps.newHashMap();
headers.put("Authorization", "Bearer " + token);
@ -1163,18 +1162,20 @@ public class DataImportNewServiceImpl implements DataImportNewService {
Integer count = customMapper.countBySQL(api, "where is_dump = 0 and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr +"'"+ extraSql );
log.error("总文件数 count:{}；-开始时间:{}；-结束时间:{}；-api:{}；", count, beginDateStr, endDateStr, api);
log.info("api：{}；总文件数 count:{}；-开始时间:{}；-结束时间:{}；",api, count, beginDateStr, endDateStr);
if (count == 0) {
return;
}
int page = count%200 == 0 ? count/200 : (count/200) + 1;
int page = count%1000 == 0 ? count/1000 : (count/1000) + 1;
//总插入数
for (int i = 0; i < page; i++) {
log.info("api：{} 批次:{}；-开始时间:{}；-结束时间:{}；",api, i, beginDateStr, endDateStr);
// 获取未存储的附件id
List<Map<String, Object>> list = customMapper.list("Id, " + name, api, " is_dump = 0 and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr +"'"+ extraSql + "limit " + i * 200 + ",200");
List<Map<String, Object>> list = customMapper.list("Id, " + name, api, " is_dump = 0 and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr +"'"+ extraSql + "limit " + i * 1000 + ",1000");
for (Map<String, Object> map : list) {
String id = null;
@ -1188,23 +1189,23 @@ public class DataImportNewServiceImpl implements DataImportNewService {
}else {
fileName = (String) map.get("Name");
}
log.info("------------文件名:" + id + "_" + fileName);
// log.info("------------文件名:" + id + "_" + fileName);
// 判断路径是否为空
if (StringUtils.isNotBlank(fileName)) {
String filePath = api + "/" + id + "_" + fileName;
// 拼接url
String url = downloadUrl + String.format(Const.SF_FILE_URL, api, id, field);
log.info("文件下载请求地址：{}",url);
// log.info("文件下载请求地址：{}",url);
Response response = HttpUtil.doGet(url, null, headers);
if (response.body() != null && response.code() == 200) {
InputStream inputStream = response.body().byteStream();
log.info("文件下载返回状态码：{}，返回信息：{}", response.code(),response.message());
// log.info("文件下载返回状态码：{}，返回信息：{}", response.code(),response.message());
switch (Const.FILE_TYPE) {
case OSS:
// 上传到oss
OssUtil.upload(inputStream, filePath);
case SERVER:
default:
dumpToServer(headers, id, filePath, url, response, inputStream);
}
Map<String, Object> paramMap = Maps.newHashMap();
@ -1227,6 +1228,7 @@ public class DataImportNewServiceImpl implements DataImportNewService {
customMapper.updateById(api, maps, id);
TimeUnit.MILLISECONDS.sleep(1);
} catch (InterruptedException interruptedException){
log.info("文件下载手动终止！");
return;
} catch (Exception e) {
log.error("文件下载失败！, id: {}, 错误信息：{}", id ,e.getMessage());
@ -1401,7 +1403,7 @@ public class DataImportNewServiceImpl implements DataImportNewService {
Integer count = customMapper.countBySQL(api, "where is_dump = 1 and is_upload = 0 and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr +"'"+ extraSql );
log.error("总文件数 count:{}；-开始时间:{}；-结束时间:{}；-api:{}；", count, beginDateStr, endDateStr, api);
log.info("总文件数 count:{}；-开始时间:{}；-结束时间:{}；-api:{}；", count, beginDateStr, endDateStr, api);
if (count == 0) {
return;
@ -1537,7 +1539,7 @@ public class DataImportNewServiceImpl implements DataImportNewService {
Integer count = customMapper.countBySQL("ContentDocument", "where new_id is null and CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr +"'"+ extraSql );
log.error("总文件数 count:{}；-开始时间:{}；-结束时间:{}；-api:{}；", count, beginDateStr, endDateStr, api);
log.info("总文件数 count:{}；-开始时间:{}；-结束时间:{}；-api:{}；", count, beginDateStr, endDateStr, api);
if (count == 0) {
return;
@ -1863,8 +1865,13 @@ public class DataImportNewServiceImpl implements DataImportNewService {
String sql1 = null;
String sql2 = null;
if (beginDateStr != null){
sql1 = "where SystemModstamp >= " + beginDateStr;
sql2 = "SystemModstamp >= "+ beginDateStr +" order by Id limit " ;
if (api.endsWith("Share")){
sql1 = "where lastModifiedDate >= '" + beginDateStr + "'";
sql2 = "lastModifiedDate >= '"+ beginDateStr +"' order by Id limit " ;
}else {
sql1 = "where SystemModstamp >= '" + beginDateStr + "'";
sql2 = "SystemModstamp >= '"+ beginDateStr +"' order by Id limit " ;
}
}else {
sql1 = null;
sql2 = "1=1 order by Id limit " ;