517 lines
18 KiB
Java
517 lines
18 KiB
Java
package com.celnet.datadump.util;
|
||
|
||
import cn.hutool.core.lang.UUID;
|
||
import com.alibaba.fastjson2.JSONArray;
|
||
import com.alibaba.fastjson2.JSONObject;
|
||
import com.celnet.datadump.entity.DataField;
|
||
import com.celnet.datadump.entity.SystemConfig;
|
||
import com.celnet.datadump.global.Const;
|
||
import com.celnet.datadump.mapper.CustomMapper;
|
||
import com.celnet.datadump.param.SalesforceParam;
|
||
import com.celnet.datadump.global.SystemConfigCode;
|
||
import com.celnet.datadump.service.SystemConfigService;
|
||
import com.google.common.collect.Lists;
|
||
import com.sforce.soap.partner.Field;
|
||
import com.sforce.soap.partner.sobject.SObject;
|
||
import com.sforce.ws.bind.XmlObject;
|
||
import com.sforce.ws.types.Time;
|
||
import lombok.extern.slf4j.Slf4j;
|
||
import org.apache.commons.collections.CollectionUtils;
|
||
import org.apache.commons.collections.map.CaseInsensitiveMap;
|
||
import org.apache.commons.lang3.BooleanUtils;
|
||
import org.apache.commons.lang3.StringUtils;
|
||
import org.apache.commons.lang3.time.DateFormatUtils;
|
||
import org.apache.commons.lang3.time.DateUtils;
|
||
import org.apache.commons.lang3.time.FastDateFormat;
|
||
import org.apache.commons.lang3.time.FastTimeZone;
|
||
import org.springframework.beans.factory.annotation.Autowired;
|
||
import org.springframework.stereotype.Component;
|
||
|
||
import javax.annotation.PostConstruct;
|
||
import java.math.BigDecimal;
|
||
import java.text.ParseException;
|
||
import java.text.SimpleDateFormat;
|
||
import java.time.LocalDateTime;
|
||
import java.time.ZoneId;
|
||
import java.time.ZonedDateTime;
|
||
import java.time.format.DateTimeFormatter;
|
||
import java.util.*;
|
||
|
||
|
||
/**
|
||
* @author Red
|
||
* @description 数据处理工具
|
||
* @date 2022/11/16
|
||
*/
|
||
@Slf4j
|
||
@Component
|
||
public class DataUtil {
|
||
|
||
@Autowired
|
||
private SystemConfigService systemConfigService;
|
||
private static final FastDateFormat SF_DATE_FORMAT = FastDateFormat.getInstance(Const.SF_DATE_FORMAT, FastTimeZone.getGmtTimeZone());
|
||
private static final FastDateFormat SF_TIME_FORMAT = FastDateFormat.getInstance(Const.SF_TIME_FORMAT, FastTimeZone.getGmtTimeZone());
|
||
|
||
/**
|
||
* sf field to mysql field
|
||
*
|
||
* @param field sf filed type
|
||
* @return mysql filed type
|
||
*/
|
||
public static String fieldTypeToMysql(Field field) {
|
||
String type = field.getType().toString();
|
||
int length = field.getLength();
|
||
String result = type;
|
||
switch (type) {
|
||
case "id":
|
||
case "reference":
|
||
result = "varchar(" + length + ")";
|
||
break;
|
||
case "string":
|
||
// Name字段用varchar方便加索引
|
||
if ("Name".equalsIgnoreCase(field.getName())) {
|
||
result = "varchar(" + length + ")";
|
||
} else {
|
||
result = "text(" + length + ")";
|
||
}
|
||
break;
|
||
case "url":
|
||
case "email":
|
||
case "phone":
|
||
case "picklist":
|
||
case "textarea":
|
||
case "multipicklist":
|
||
case "address":
|
||
case "combobox":
|
||
case "location":
|
||
case "encryptedstring":
|
||
case "anyType":
|
||
result = "text(" + length + ")";
|
||
break;
|
||
case "date":
|
||
case "datetime":
|
||
case "boolean":
|
||
result = type;
|
||
break;
|
||
case "time":
|
||
result = "time(3)";
|
||
break;
|
||
case "long":
|
||
case "int":
|
||
result = "int(" + length + ")";
|
||
break;
|
||
case "double":
|
||
case "currency":
|
||
case "percent":
|
||
int scale = field.getScale();
|
||
result = "double(" + 18 + "," + scale + ")";
|
||
break;
|
||
case "base64":
|
||
result = "longblob";
|
||
break;
|
||
default:
|
||
result = "text(" + length + ")";
|
||
break;
|
||
}
|
||
return result;
|
||
}
|
||
|
||
/**
|
||
* 分割英文逗号字符串 并去除所有空格
|
||
*
|
||
* @param ids id串
|
||
* @return list
|
||
*/
|
||
public static List<String> toIdList(String ids) {
|
||
if (StringUtils.isBlank(ids)) {
|
||
return Lists.newArrayList();
|
||
}
|
||
return Arrays.asList(ids.replaceAll(StringUtils.SPACE, StringUtils.EMPTY).split(","));
|
||
}
|
||
|
||
private static final int MINUTE = 60;
|
||
private static final int HOUR = 24;
|
||
|
||
/**
|
||
* 计算两个时间差
|
||
*
|
||
* @param date1 日期1
|
||
* @param date2 日期2
|
||
* @return 返回日期格式 xxdxxhxxmin
|
||
*/
|
||
public static String calTime(Date date1, Date date2) {
|
||
if (date1 == null || date2 == null) {
|
||
return null;
|
||
}
|
||
long min = Math.abs(date2.getTime() - date1.getTime()) / 1000 / 60;
|
||
StringBuilder stringBuilder = new StringBuilder();
|
||
if (min >= MINUTE) {
|
||
long h = min / MINUTE;
|
||
long realMin = min % MINUTE;
|
||
if (h >= HOUR) {
|
||
stringBuilder.append(h / HOUR).append("d");
|
||
long realH = h % HOUR;
|
||
if (realH > 0) {
|
||
stringBuilder.append(realH).append("h");
|
||
}
|
||
} else {
|
||
stringBuilder.append(h).append("h");
|
||
}
|
||
if (realMin > 0) {
|
||
stringBuilder.append(realMin).append("min");
|
||
}
|
||
} else {
|
||
stringBuilder.append(min).append("min");
|
||
}
|
||
return stringBuilder.toString();
|
||
}
|
||
|
||
public static Date DEFAULT_BEGIN_DATE = null;
|
||
|
||
@PostConstruct
|
||
public void init() throws ParseException {
|
||
SystemConfig systemConfig = systemConfigService.getById(SystemConfigCode.DEFAULT_START_DATE);
|
||
if (systemConfig != null) {
|
||
DEFAULT_BEGIN_DATE = DateUtils.parseDate(systemConfig.getValue(), "yyyy-MM-dd", "yyyy-MM-dd HH:mm:ss");
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 分割任务
|
||
*
|
||
* @param param 参数
|
||
* @return list
|
||
*/
|
||
public static List<SalesforceParam> splitTask(SalesforceParam param) {
|
||
if (CollectionUtils.isNotEmpty(param.getIds())) {
|
||
return Lists.newArrayList(param);
|
||
}
|
||
Date now = new Date();
|
||
DateUtils.truncate(now, Calendar.DATE);
|
||
Date beginCreateDate = param.getBeginCreateDate() != null ? param.getBeginCreateDate() : DEFAULT_BEGIN_DATE;
|
||
// 如果为空 取当天零点 作为最大时间
|
||
Date endCreateDate = null;
|
||
try {
|
||
endCreateDate = param.getEndCreateDate() != null ? param.getEndCreateDate() : DateUtils.parseDate(DateFormatUtils.format(now, "yyyy-MM-dd"), "yyyy-MM-dd");
|
||
} catch (Throwable ignored) {
|
||
}
|
||
|
||
Date startDate = beginCreateDate;
|
||
Date lastDay = null;
|
||
List<SalesforceParam> list = Lists.newArrayList();
|
||
int batch = 1;
|
||
do {
|
||
lastDay = getYearLastDay(endCreateDate, startDate);
|
||
|
||
SalesforceParam clone = param.clone();
|
||
clone.setBeginCreateDate(startDate);
|
||
clone.setEndCreateDate(lastDay);
|
||
clone.setBatch(batch);
|
||
batch++;
|
||
list.add(clone);
|
||
startDate = lastDay;
|
||
} while (lastDay.compareTo(endCreateDate) < 0);
|
||
return list;
|
||
}
|
||
|
||
/**
|
||
* 返回本周最后一天 如果大于结束日期 返回结束日期
|
||
*
|
||
* @param endDate 结束日期
|
||
* @param startDate 开始日期
|
||
* @return 本周最后一天
|
||
*/
|
||
public static Date getWeekLastDay(Date endDate, Date startDate) {
|
||
Date weekLastDay;
|
||
Calendar cal = DateUtils.toCalendar(startDate);
|
||
cal.set(Calendar.DAY_OF_WEEK, cal.getActualMaximum(Calendar.DAY_OF_WEEK));
|
||
cal.add(Calendar.DAY_OF_WEEK, 1);
|
||
weekLastDay = cal.getTime();
|
||
if (weekLastDay.compareTo(endDate) > 0) {
|
||
weekLastDay = endDate;
|
||
}
|
||
return weekLastDay;
|
||
}
|
||
|
||
/**
|
||
* 返回本月最后一天 如果大于结束日期 返回结束日期
|
||
*
|
||
* @param endDate 结束日期
|
||
* @param startDate 开始日期
|
||
* @return 本月最后一天
|
||
*/
|
||
public static Date getMonthLastDay(Date endDate, Date startDate) {
|
||
Date monthLastDay;
|
||
Calendar cal = DateUtils.toCalendar(startDate);
|
||
cal.set(Calendar.DAY_OF_MONTH, cal.getActualMaximum(Calendar.DAY_OF_MONTH));
|
||
cal.add(Calendar.DAY_OF_MONTH, 1);
|
||
monthLastDay = cal.getTime();
|
||
if (monthLastDay.compareTo(endDate) > 0) {
|
||
monthLastDay = endDate;
|
||
}
|
||
return monthLastDay;
|
||
}
|
||
|
||
/**
|
||
* 返回本周最后一天 如果大于结束日期 返回结束日期
|
||
*
|
||
* @param endDate 结束日期
|
||
* @param startDate 开始日期
|
||
* @return 本周最后一天
|
||
*/
|
||
public static Date getYearLastDay(Date endDate, Date startDate) {
|
||
Date yearLastDay;
|
||
startDate = DateUtils.addYears(startDate, 1);
|
||
Calendar cal = DateUtils.toCalendar(startDate);
|
||
cal.set(Calendar.YEAR, cal.get(Calendar.YEAR));
|
||
yearLastDay = cal.getTime();
|
||
if (yearLastDay.compareTo(endDate) > 0) {
|
||
yearLastDay = endDate;
|
||
}
|
||
return yearLastDay;
|
||
}
|
||
|
||
/**
|
||
* sObject 数组转 JSONObject
|
||
*
|
||
* @param sObject sObject
|
||
* @param fields 字段
|
||
* @return JSONObject
|
||
*/
|
||
public static JSONObject toJsonObject(SObject sObject, Field[] fields) throws ParseException {
|
||
Iterator<XmlObject> children = sObject.getChildren();
|
||
JSONObject jsonObject = new JSONObject();
|
||
CaseInsensitiveMap map = new CaseInsensitiveMap();
|
||
for (Field field : fields) {
|
||
map.put(field.getName(), field.getType().toString());
|
||
}
|
||
while (children.hasNext()) {
|
||
XmlObject next = children.next();
|
||
String name = next.getName().getLocalPart();
|
||
Object value = next.getValue();
|
||
// 全部当string处理
|
||
if (next instanceof SObject) {
|
||
SObject sObjectChild = (SObject) next;
|
||
Iterator<XmlObject> children1 = sObjectChild.getChildren();
|
||
while (children1.hasNext()) {
|
||
XmlObject childNext = children1.next();
|
||
String childName = childNext.getName().getLocalPart();
|
||
Object childValue = childNext.getValue();
|
||
// partner会多一个type 值为对象名称 这里过滤掉
|
||
if (Const.TYPE_FILTER.equals(childName)) {
|
||
continue;
|
||
}
|
||
if (childValue == null) {
|
||
continue;
|
||
}
|
||
jsonObject.put(name+"_"+childName, childValue);
|
||
}
|
||
}
|
||
// partner会多一个type 值为对象名称 这里过滤掉
|
||
if (Const.TYPE_FILTER.equals(name)) {
|
||
continue;
|
||
}
|
||
if (value == null) {
|
||
continue;
|
||
}
|
||
String type = (String) map.get(name);
|
||
if (type == null) {
|
||
continue;
|
||
}
|
||
switch (type) {
|
||
case "date":
|
||
jsonObject.put(name, DateUtils.parseDate((String) value, "yyyy-MM-dd"));
|
||
break;
|
||
case "datetime":
|
||
jsonObject.put(name, SF_DATE_FORMAT.parse((String) value));
|
||
break;
|
||
case "time":
|
||
jsonObject.put(name, SF_TIME_FORMAT.parse((String) value));
|
||
break;
|
||
case "boolean":
|
||
jsonObject.put(name, BooleanUtils.toBoolean((String) value));
|
||
break;
|
||
case "long":
|
||
case "int":
|
||
jsonObject.put(name, Integer.parseInt((String) value));
|
||
break;
|
||
case "double":
|
||
case "currency":
|
||
case "percent":
|
||
jsonObject.put(name, Double.parseDouble((String) value));
|
||
break;
|
||
case "base64":
|
||
// 文件不处理 统一存oss或者server
|
||
break;
|
||
default:
|
||
if (value instanceof GregorianCalendar) {
|
||
jsonObject.put(name, DateFormatUtils.format(((GregorianCalendar) value), "yyyy-MM-dd HH:mm:ss"));
|
||
} else if (value instanceof Time){
|
||
jsonObject.put(name, DateFormatUtils.format((((Time) value).getTimeInMillis()), "HH:mm:ss"));
|
||
}else {
|
||
jsonObject.put(name, String.valueOf(value));
|
||
}
|
||
break;
|
||
}
|
||
}
|
||
return jsonObject;
|
||
}
|
||
|
||
|
||
/**
|
||
* sObject 数组转 JSONArray
|
||
*
|
||
* @param records sObject 数组
|
||
* @return JSONArray
|
||
*/
|
||
public static JSONArray toJsonArray(SObject[] records, Field[] fields) throws ParseException {
|
||
JSONArray jsonArray = new JSONArray();
|
||
for (SObject sObject : records) {
|
||
jsonArray.add(toJsonObject(sObject, fields));
|
||
}
|
||
return jsonArray;
|
||
}
|
||
|
||
public static Object fieldTypeToSf(DataField dataField) throws ParseException {
|
||
String fieldType = dataField.getSfType();
|
||
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
|
||
Date date = sdf.parse("2023-03-23");
|
||
switch (fieldType) {
|
||
case "int":
|
||
return 1;
|
||
case "double":
|
||
case "currency":
|
||
case "percent":
|
||
return new BigDecimal("1");
|
||
case "boolean":
|
||
return true;
|
||
case "email":
|
||
return "jax.li@celnet.com.cn";
|
||
case "date":
|
||
//字符串转日期
|
||
return date;
|
||
case "datetime":
|
||
return date;
|
||
default:
|
||
return "初始"+ UUID.randomUUID();
|
||
}
|
||
}
|
||
|
||
public static Object fieldTypeBulkToSf(DataField dataField) throws ParseException {
|
||
switch (dataField.getSfType()) {
|
||
case "int":
|
||
return 1;
|
||
case "double":
|
||
case "currency":
|
||
case "percent":
|
||
return new BigDecimal("1");
|
||
case "boolean":
|
||
return true;
|
||
case "email":
|
||
return "kris.dong@163.com";
|
||
case "date":
|
||
case "datetime":
|
||
return "2020-01-01T00:00:00+00:00";
|
||
default:
|
||
return "初始"+ UUID.randomUUID();
|
||
}
|
||
}
|
||
|
||
|
||
public static Object localBulkDataToSfData(String fieldType, String data) throws ParseException {
|
||
|
||
switch (fieldType) {
|
||
case "int":
|
||
return Integer.parseInt(data);
|
||
case "double":
|
||
case "currency":
|
||
case "percent":
|
||
return new BigDecimal(data);
|
||
case "boolean":
|
||
return Boolean.valueOf(data);
|
||
case "date":
|
||
return data+"T08:00:00Z";
|
||
case "datetime":
|
||
LocalDateTime localDateTime = LocalDateTime.parse(data, DateTimeFormatter.ISO_LOCAL_DATE_TIME);
|
||
ZonedDateTime zonedDateTime = localDateTime.atZone(ZoneId.systemDefault());
|
||
return zonedDateTime.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME);
|
||
case "time":
|
||
return adjustHour(data);
|
||
default:
|
||
return data;
|
||
}
|
||
}
|
||
|
||
public static Object localDataToSfData(String fieldType, String data) throws ParseException {
|
||
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
|
||
SimpleDateFormat sd = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
|
||
Date date;
|
||
//date转Calendar类型
|
||
Calendar calendar = Calendar.getInstance();
|
||
switch (fieldType) {
|
||
case "int":
|
||
return Integer.parseInt(data);
|
||
case "double":
|
||
case "currency":
|
||
case "percent":
|
||
return new BigDecimal(data);
|
||
case "boolean":
|
||
return Boolean.valueOf(data);
|
||
case "date":
|
||
try {
|
||
return sdf.parse(data+"T08:00:00Z");
|
||
} catch (ParseException e) {
|
||
log.error("exception message", e);
|
||
}
|
||
case "datetime":
|
||
try {
|
||
date = sd.parse(data);
|
||
}catch (ParseException e){
|
||
//解决当时间秒为0时，转换秒精度丢失问题
|
||
date = sd.parse(data+":00");
|
||
}
|
||
calendar.setTime(date);
|
||
return calendar;
|
||
case "time":
|
||
return adjustHour(data);
|
||
default:
|
||
return data;
|
||
}
|
||
}
|
||
|
||
public static String adjustHour(String timeStr) {
|
||
// 提取小时部分并转换为整数
|
||
int hour = Integer.parseInt(timeStr.substring(0, 2));
|
||
|
||
// 减去8小时并处理跨天逻辑
|
||
int adjustedHour = hour - 8;
|
||
if (adjustedHour < 0) {
|
||
adjustedHour += 24; // 处理负数情况（跨天）
|
||
}
|
||
|
||
// 格式化为两位字符串（自动补零）
|
||
String newHour = String.format("%02d", adjustedHour);
|
||
|
||
// 拼接原始字符串的剩余部分（分钟+秒+毫秒）
|
||
return newHour + timeStr.substring(2);
|
||
}
|
||
|
||
public static boolean isUpdate(String field){
|
||
switch (field) {
|
||
case "LastModifiedDate":
|
||
case "IsDeleted":
|
||
case "SystemModstamp":
|
||
case "CreatedById":
|
||
case "CreatedDate":
|
||
case "LastModifiedById":
|
||
case "LastViewedDate":
|
||
case "LastReferencedDate":
|
||
case "IsConverted":
|
||
return false;
|
||
default:
|
||
return true;
|
||
}
|
||
}
|
||
|
||
}
|