From fbc049f619f46ae7faaa5595bf2918066a831f9a Mon Sep 17 00:00:00 2001 From: Kris <2893855659@qq.com> Date: Tue, 29 Jul 2025 18:31:08 +0800 Subject: [PATCH] =?UTF-8?q?=E3=80=90fix=E3=80=91=20=E8=B0=83=E6=95=B4?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E4=B8=8B=E8=BD=BD=EF=BC=8C=E6=9B=B4=E6=96=B0?= =?UTF-8?q?=E5=85=B3=E8=81=94=E7=B1=BB=E5=9E=8B=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit 514fb41a370806b8caa9febdeb6a44d351493bfc) --- .../impl/DataImportNewServiceImpl.java | 76 +++++++++++++------ 1 file changed, 53 insertions(+), 23 deletions(-) diff --git a/src/main/java/com/celnet/datadump/service/impl/DataImportNewServiceImpl.java b/src/main/java/com/celnet/datadump/service/impl/DataImportNewServiceImpl.java index ad972a6..e718228 100644 --- a/src/main/java/com/celnet/datadump/service/impl/DataImportNewServiceImpl.java +++ b/src/main/java/com/celnet/datadump/service/impl/DataImportNewServiceImpl.java @@ -1204,10 +1204,8 @@ public class DataImportNewServiceImpl implements DataImportNewService { case OSS: // 上传到oss OssUtil.upload(inputStream, filePath); - case SERVER: - dumpToServer(headers, id, filePath, url, response, inputStream); default: - log.error("id: {}, no mapping dump type", id); + dumpToServer(headers, id, filePath, url, response, inputStream); } Map paramMap = Maps.newHashMap(); if ("Document".equals(api)) { @@ -1753,6 +1751,8 @@ public class DataImportNewServiceImpl implements DataImportNewService { } // 等待当前所有线程执行完成 salesforceExecutor.waitForFutures(futures.toArray(new Future[]{})); + } catch (InterruptedException interruptedException){ + return null; } catch (Exception e) { throw e; } @@ -1777,35 +1777,64 @@ public class DataImportNewServiceImpl implements DataImportNewService { beginDateStr = DateUtil.format(beginDate, "yyyy-MM-dd HH:mm:ss"); endDateStr = DateUtil.format(endDate, "yyyy-MM-dd HH:mm:ss"); } - // 表内数据总量 - Integer count = customMapper.countBySQL(param.getApi(), "where CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "'"); - log.info("表api：{} 存在" +count+ "条数据！", param.getApi()); + String sql = "CreatedDate >= '" + beginDateStr + "' and CreatedDate < '" + endDateStr + "'"; + String allFiled = "Id"; + + if (linkConfigs.size() == 1){ + for (LinkConfig linkConfig : linkConfigs) { + sql = sql + " and " + linkConfig.getField() + " is not null"; + allFiled = allFiled + "," + linkConfig.getField(); + } + }else { + String info = null; + for (LinkConfig linkConfig : linkConfigs) { + if (StringUtils.isBlank(info)){ + info = linkConfig.getField() + " is not null" ; + }else { + info = info + " or " + linkConfig.getField() + " is not null"; + } + allFiled = allFiled + "," + linkConfig.getField(); + } + sql = sql +" and ("+ info +")"; + } + + // 表内数据总量 + Integer count = customMapper.countBySQL(param.getApi(), "where " + sql); + + log.info("表api：{} 存在" +count+ "条需更新数据！开始时间:{},结束时间：{}", param.getApi(), beginDateStr, endDateStr); if (count >0 ) { - int page = count % 2000 == 0 ? count / 2000 : (count / 2000) + 1; + try { + int page = count % 2000 == 0 ? count / 2000 : (count / 2000) + 1; - for (int i = 0; i < page; i++) { + for (int i = 0; i < page; i++) { - log.info("表api：{}，数据量：{}，执行数据更新!", param.getApi(), 2000* (i+1)); + log.info("表api：{}，批次：{}，单批次数：2000，开始时间：{}，结束时间：{}，执行数据更新!", param.getApi(), i,beginDateStr, endDateStr); - List> mapList = customMapper.list("*", param.getApi(), "1=1 order by Id limit " + i * 2000 + ",2000"); + List> mapList = customMapper.list(allFiled, param.getApi(), sql + " order by Id limit " + i * 2000 + ",2000"); - List> updateMapList = new ArrayList<>(); - for (int j = 1; j <= mapList.size(); j++) { - Map map = mapList.get(j - 1); - for (LinkConfig config : linkConfigs) { - if (map.get(config.getField()) != null){ - String type = resultMap.get(map.get(config.getField()).toString().substring(0, 3)); - Map paramMap = Maps.newHashMap(); - paramMap.put("key", config.getLinkField()); - paramMap.put("value", type); - updateMapList.add(paramMap); + for (int j = 1; j <= mapList.size(); j++) { + List> updateMapList = new ArrayList<>(); + Map map = mapList.get(j - 1); + for (LinkConfig config : linkConfigs) { + if (map.get(config.getField()) != null){ + String type = resultMap.get(map.get(config.getField()).toString().substring(0, 3)); + Map paramMap = Maps.newHashMap(); + paramMap.put("key", config.getLinkField()); + paramMap.put("value", type); + updateMapList.add(paramMap); + } + } + if (!updateMapList.isEmpty()) { + customMapper.updateById(param.getApi(), updateMapList, String.valueOf(mapList.get(j - 1).get("Id") != null?mapList.get(j - 1).get("Id") : mapList.get(j - 1).get("id"))); + }else { + log.error("对象：{}的Id为：{}数据不存在可更新的关联类型！！！！",param.getApi(),String.valueOf(mapList.get(j - 1).get("Id") != null?mapList.get(j - 1).get("Id") : mapList.get(j - 1).get("id"))); } } - customMapper.updateById(param.getApi(), updateMapList, String.valueOf(mapList.get(j - 1).get("Id") != null?mapList.get(j - 1).get("Id") : mapList.get(j - 1).get("id"))); - } + }catch (Exception e){ + log.info(e.getMessage()); } } @@ -1886,8 +1915,9 @@ public class DataImportNewServiceImpl implements DataImportNewService { List> mapList = customMapper.list("*", api, sql2 + i * 2000 + ",2000" ); - List> updateMapList = new ArrayList<>(); + for (int j = 1; j <= mapList.size(); j++) { + List> updateMapList = new ArrayList<>(); Map map = mapList.get(j - 1); for (LinkConfig config : linkConfigs) { if (map.get(config.getField()) != null){