package com.celnet.datadump.util; import cn.hutool.core.lang.UUID; import com.alibaba.fastjson2.JSONArray; import com.alibaba.fastjson2.JSONObject; import com.celnet.datadump.entity.DataField; import com.celnet.datadump.entity.SystemConfig; import com.celnet.datadump.global.Const; import com.celnet.datadump.mapper.CustomMapper; import com.celnet.datadump.param.SalesforceParam; import com.celnet.datadump.global.SystemConfigCode; import com.celnet.datadump.service.SystemConfigService; import com.google.common.collect.Lists; import com.sforce.soap.partner.Field; import com.sforce.soap.partner.sobject.SObject; import com.sforce.ws.bind.XmlObject; import com.sforce.ws.types.Time; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.map.CaseInsensitiveMap; import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.DateFormatUtils; import org.apache.commons.lang3.time.DateUtils; import org.apache.commons.lang3.time.FastDateFormat; import org.apache.commons.lang3.time.FastTimeZone; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; import java.math.BigDecimal; import java.text.ParseException; import java.text.SimpleDateFormat; import java.time.LocalDateTime; import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.*; /** * @author Red * @description 数据处理工具 * @date 2022/11/16 */ @Slf4j @Component public class DataUtil { @Autowired private SystemConfigService systemConfigService; private static final FastDateFormat SF_DATE_FORMAT = FastDateFormat.getInstance(Const.SF_DATE_FORMAT, FastTimeZone.getGmtTimeZone()); private static final FastDateFormat SF_TIME_FORMAT = FastDateFormat.getInstance(Const.SF_TIME_FORMAT, FastTimeZone.getGmtTimeZone()); /** * sf field to mysql field * * @param field sf filed type * @return mysql filed type */ public static String fieldTypeToMysql(Field field) { String type = field.getType().toString(); int length = field.getLength(); String result = type; switch (type) { case "id": case "reference": result = "varchar(" + length + ")"; break; case "string": // Name字段用varchar方便加索引 if ("Name".equalsIgnoreCase(field.getName())) { result = "varchar(" + length + ")"; } else { result = "text(" + length + ")"; } break; case "url": case "email": case "phone": case "picklist": case "textarea": case "multipicklist": case "address": case "combobox": case "location": case "encryptedstring": case "anyType": result = "text(" + length + ")"; break; case "date": case "datetime": case "boolean": result = type; break; case "time": result = "time(3)"; break; case "long": case "int": result = "int(" + length + ")"; break; case "double": case "currency": case "percent": int scale = field.getScale(); result = "double(" + 18 + "," + scale + ")"; break; case "base64": result = "longblob"; break; default: result = "text(" + length + ")"; break; } return result; } /** * 分割英文逗号字符串 并去除所有空格 * * @param ids id串 * @return list */ public static List toIdList(String ids) { if (StringUtils.isBlank(ids)) { return Lists.newArrayList(); } return Arrays.asList(ids.replaceAll(StringUtils.SPACE, StringUtils.EMPTY).split(",")); } private static final int MINUTE = 60; private static final int HOUR = 24; /** * 计算两个时间差 * * @param date1 日期1 * @param date2 日期2 * @return 返回日期格式 xxdxxhxxmin */ public static String calTime(Date date1, Date date2) { if (date1 == null || date2 == null) { return null; } long min = Math.abs(date2.getTime() - date1.getTime()) / 1000 / 60; StringBuilder stringBuilder = new StringBuilder(); if (min >= MINUTE) { long h = min / MINUTE; long realMin = min % MINUTE; if (h >= HOUR) { stringBuilder.append(h / HOUR).append("d"); long realH = h % HOUR; if (realH > 0) { stringBuilder.append(realH).append("h"); } } else { stringBuilder.append(h).append("h"); } if (realMin > 0) { stringBuilder.append(realMin).append("min"); } } else { stringBuilder.append(min).append("min"); } return stringBuilder.toString(); } public static Date DEFAULT_BEGIN_DATE = null; @PostConstruct public void init() throws ParseException { SystemConfig systemConfig = systemConfigService.getById(SystemConfigCode.DEFAULT_START_DATE); if (systemConfig != null) { DEFAULT_BEGIN_DATE = DateUtils.parseDate(systemConfig.getValue(), "yyyy-MM-dd", "yyyy-MM-dd HH:mm:ss"); } } /** * 分割任务 * * @param param 参数 * @return list */ public static List splitTask(SalesforceParam param) { if (CollectionUtils.isNotEmpty(param.getIds())) { return Lists.newArrayList(param); } Date now = new Date(); DateUtils.truncate(now, Calendar.DATE); Date beginCreateDate = param.getBeginCreateDate() != null ? param.getBeginCreateDate() : DEFAULT_BEGIN_DATE; // 如果为空 取当天零点 作为最大时间 Date endCreateDate = null; try { endCreateDate = param.getEndCreateDate() != null ? param.getEndCreateDate() : DateUtils.parseDate(DateFormatUtils.format(now, "yyyy-MM-dd"), "yyyy-MM-dd"); } catch (Throwable ignored) { } Date startDate = beginCreateDate; Date lastDay = null; List list = Lists.newArrayList(); int batch = 1; do { lastDay = getYearLastDay(endCreateDate, startDate); SalesforceParam clone = param.clone(); clone.setBeginCreateDate(startDate); clone.setEndCreateDate(lastDay); clone.setBatch(batch); batch++; list.add(clone); startDate = lastDay; } while (lastDay.compareTo(endCreateDate) < 0); return list; } /** * 返回本周最后一天 如果大于结束日期 返回结束日期 * * @param endDate 结束日期 * @param startDate 开始日期 * @return 本周最后一天 */ public static Date getWeekLastDay(Date endDate, Date startDate) { Date weekLastDay; Calendar cal = DateUtils.toCalendar(startDate); cal.set(Calendar.DAY_OF_WEEK, cal.getActualMaximum(Calendar.DAY_OF_WEEK)); cal.add(Calendar.DAY_OF_WEEK, 1); weekLastDay = cal.getTime(); if (weekLastDay.compareTo(endDate) > 0) { weekLastDay = endDate; } return weekLastDay; } /** * 返回本月最后一天 如果大于结束日期 返回结束日期 * * @param endDate 结束日期 * @param startDate 开始日期 * @return 本月最后一天 */ public static Date getMonthLastDay(Date endDate, Date startDate) { Date monthLastDay; Calendar cal = DateUtils.toCalendar(startDate); cal.set(Calendar.DAY_OF_MONTH, cal.getActualMaximum(Calendar.DAY_OF_MONTH)); cal.add(Calendar.DAY_OF_MONTH, 1); monthLastDay = cal.getTime(); if (monthLastDay.compareTo(endDate) > 0) { monthLastDay = endDate; } return monthLastDay; } /** * 返回本周最后一天 如果大于结束日期 返回结束日期 * * @param endDate 结束日期 * @param startDate 开始日期 * @return 本周最后一天 */ public static Date getYearLastDay(Date endDate, Date startDate) { Date yearLastDay; startDate = DateUtils.addYears(startDate, 1); Calendar cal = DateUtils.toCalendar(startDate); cal.set(Calendar.YEAR, cal.get(Calendar.YEAR)); yearLastDay = cal.getTime(); if (yearLastDay.compareTo(endDate) > 0) { yearLastDay = endDate; } return yearLastDay; } /** * sObject 数组转 JSONObject * * @param sObject sObject * @param fields 字段 * @return JSONObject */ public static JSONObject toJsonObject(SObject sObject, Field[] fields) throws ParseException { Iterator children = sObject.getChildren(); JSONObject jsonObject = new JSONObject(); CaseInsensitiveMap map = new CaseInsensitiveMap(); for (Field field : fields) { map.put(field.getName(), field.getType().toString()); } while (children.hasNext()) { XmlObject next = children.next(); String name = next.getName().getLocalPart(); Object value = next.getValue(); // 全部当string处理 if (next instanceof SObject) { SObject sObjectChild = (SObject) next; Iterator children1 = sObjectChild.getChildren(); while (children1.hasNext()) { XmlObject childNext = children1.next(); String childName = childNext.getName().getLocalPart(); Object childValue = childNext.getValue(); // partner会多一个type 值为对象名称 这里过滤掉 if (Const.TYPE_FILTER.equals(childName)) { continue; } if (childValue == null) { continue; } jsonObject.put(name+"_"+childName, childValue); } } // partner会多一个type 值为对象名称 这里过滤掉 if (Const.TYPE_FILTER.equals(name)) { continue; } if (value == null) { continue; } String type = (String) map.get(name); if (type == null) { continue; } switch (type) { case "date": jsonObject.put(name, DateUtils.parseDate((String) value, "yyyy-MM-dd")); break; case "datetime": jsonObject.put(name, SF_DATE_FORMAT.parse((String) value)); break; case "time": jsonObject.put(name, SF_TIME_FORMAT.parse((String) value)); break; case "boolean": jsonObject.put(name, BooleanUtils.toBoolean((String) value)); break; case "long": case "int": jsonObject.put(name, Integer.parseInt((String) value)); break; case "double": case "currency": case "percent": jsonObject.put(name, Double.parseDouble((String) value)); break; case "base64": // 文件不处理 统一存oss或者server break; default: if (value instanceof GregorianCalendar) { jsonObject.put(name, DateFormatUtils.format(((GregorianCalendar) value), "yyyy-MM-dd HH:mm:ss")); } else if (value instanceof Time){ jsonObject.put(name, DateFormatUtils.format((((Time) value).getTimeInMillis()), "HH:mm:ss")); }else { jsonObject.put(name, String.valueOf(value)); } break; } } return jsonObject; } /** * sObject 数组转 JSONArray * * @param records sObject 数组 * @return JSONArray */ public static JSONArray toJsonArray(SObject[] records, Field[] fields) throws ParseException { JSONArray jsonArray = new JSONArray(); for (SObject sObject : records) { jsonArray.add(toJsonObject(sObject, fields)); } return jsonArray; } public static Object fieldTypeToSf(DataField dataField) throws ParseException { String fieldType = dataField.getSfType(); SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); Date date = sdf.parse("2023-03-23"); switch (fieldType) { case "int": return 1; case "double": case "currency": case "percent": return new BigDecimal("1"); case "boolean": return true; case "email": return "jax.li@celnet.com.cn"; case "date": //字符串转日期 return date; case "datetime": return date; default: return "初始"+ UUID.randomUUID(); } } public static Object fieldTypeBulkToSf(DataField dataField) throws ParseException { switch (dataField.getSfType()) { case "int": return 1; case "double": case "currency": case "percent": return new BigDecimal("1"); case "boolean": return true; case "email": return "kris.dong@163.com"; case "date": case "datetime": return "2020-01-01T00:00:00+00:00"; default: return "初始"+ UUID.randomUUID(); } } public static Object localBulkDataToSfData(String fieldType, String data) throws ParseException { switch (fieldType) { case "int": return Integer.parseInt(data); case "double": case "currency": case "percent": return new BigDecimal(data); case "boolean": return Boolean.valueOf(data); case "date": return data+"T08:00:00Z"; case "datetime": LocalDateTime localDateTime = LocalDateTime.parse(data, DateTimeFormatter.ISO_LOCAL_DATE_TIME); ZonedDateTime zonedDateTime = localDateTime.atZone(ZoneId.systemDefault()); return zonedDateTime.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME); case "time": return adjustHour(data); default: return data; } } public static Object localDataToSfData(String fieldType, String data) throws ParseException { SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); SimpleDateFormat sd = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); Date date; //date转Calendar类型 Calendar calendar = Calendar.getInstance(); switch (fieldType) { case "int": return Integer.parseInt(data); case "double": case "currency": case "percent": return new BigDecimal(data); case "boolean": return Boolean.valueOf(data); case "date": try { return sdf.parse(data+"T08:00:00Z"); } catch (ParseException e) { log.error("exception message", e); } case "datetime": try { date = sd.parse(data); }catch (ParseException e){ //解决当时间秒为0时,转换秒精度丢失问题 date = sd.parse(data+":00"); } calendar.setTime(date); return calendar; case "time": return adjustHour(data); default: return data; } } public static String adjustHour(String timeStr) { // 提取小时部分并转换为整数 int hour = Integer.parseInt(timeStr.substring(0, 2)); // 减去8小时并处理跨天逻辑 int adjustedHour = hour - 8; if (adjustedHour < 0) { adjustedHour += 24; // 处理负数情况(跨天) } // 格式化为两位字符串(自动补零) String newHour = String.format("%02d", adjustedHour); // 拼接原始字符串的剩余部分(分钟+秒+毫秒) return newHour + timeStr.substring(2); } public static boolean isUpdate(String field){ switch (field) { case "LastModifiedDate": case "IsDeleted": case "SystemModstamp": case "CreatedById": case "CreatedDate": case "LastModifiedById": case "LastViewedDate": case "LastReferencedDate": case "IsConverted": return false; default: return true; } } }