data-dump/src/main/java/com/celnet/datadump/util/DataUtil.java

package com.celnet.datadump.util;

import cn.hutool.core.lang.UUID;
import com.alibaba.fastjson2.JSONArray;
import com.alibaba.fastjson2.JSONObject;
import com.celnet.datadump.entity.DataField;
import com.celnet.datadump.entity.SystemConfig;
import com.celnet.datadump.global.Const;
import com.celnet.datadump.mapper.CustomMapper;
import com.celnet.datadump.param.SalesforceParam;
import com.celnet.datadump.global.SystemConfigCode;
import com.celnet.datadump.service.SystemConfigService;
import com.google.common.collect.Lists;
import com.sforce.soap.partner.Field;
import com.sforce.soap.partner.sobject.SObject;
import com.sforce.ws.bind.XmlObject;
import com.sforce.ws.types.Time;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.map.CaseInsensitiveMap;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.commons.lang3.time.FastDateFormat;
import org.apache.commons.lang3.time.FastTimeZone;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import javax.annotation.PostConstruct;
import java.math.BigDecimal;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;


/**
 * @author Red
 * @description 数据处理工具
 * @date 2022/11/16
 */
@Slf4j
@Component
public class DataUtil {

    @Autowired
    private SystemConfigService systemConfigService;
    private static final FastDateFormat SF_DATE_FORMAT = FastDateFormat.getInstance(Const.SF_DATE_FORMAT, FastTimeZone.getGmtTimeZone());
    private static final FastDateFormat SF_TIME_FORMAT = FastDateFormat.getInstance(Const.SF_TIME_FORMAT, FastTimeZone.getGmtTimeZone());

    /**
     * sf field to mysql field
     *
     * @param field sf filed type
     * @return mysql filed type
     */
    public static String fieldTypeToMysql(Field field) {
        String type = field.getType().toString();
        int length = field.getLength();
        String result = type;
        switch (type) {
            case "id":
            case "reference":
                result = "varchar(" + length + ")";
                break;
            case "string":
                // Name字段用varchar方便加索引
                if ("Name".equalsIgnoreCase(field.getName())) {
                    result = "varchar(" + length + ")";
                } else {
                    result = "text(" + length + ")";
                }
                break;
            case "url":
            case "email":
            case "phone":
            case "picklist":
            case "textarea":
            case "multipicklist":
            case "address":
            case "combobox":
            case "location":
            case "encryptedstring":
            case "anyType":
                result = "text(" + length + ")";
                break;
            case "date":
            case "datetime":
            case "boolean":
                result = type;
                break;
            case "time":
                result = "time(3)";
                break;
            case "long":
            case "int":
                result = "int(" + length + ")";
                break;
            case "double":
            case "currency":
            case "percent":
                int scale = field.getScale();
                result = "double(" + 18 + "," + scale + ")";
                break;
            case "base64":
                result = "longblob";
                break;
            default:
                result = "text(" + length + ")";
                break;
        }
        return result;
    }

    /**
     * 分割英文逗号字符串 并去除所有空格
     *
     * @param ids id串
     * @return list
     */
    public static List<String> toIdList(String ids) {
        if (StringUtils.isBlank(ids)) {
            return Lists.newArrayList();
        }
        return Arrays.asList(ids.replaceAll(StringUtils.SPACE, StringUtils.EMPTY).split(","));
    }

    private static final int MINUTE = 60;
    private static final int HOUR = 24;

    /**
     * 计算两个时间差
     *
     * @param date1 日期1
     * @param date2 日期2
     * @return 返回日期格式 xxdxxhxxmin
     */
    public static String calTime(Date date1, Date date2) {
        if (date1 == null || date2 == null) {
            return null;
        }
        long min = Math.abs(date2.getTime() - date1.getTime()) / 1000 / 60;
        StringBuilder stringBuilder = new StringBuilder();
        if (min >= MINUTE) {
            long h = min / MINUTE;
            long realMin = min % MINUTE;
            if (h >= HOUR) {
                stringBuilder.append(h / HOUR).append("d");
                long realH = h % HOUR;
                if (realH > 0) {
                    stringBuilder.append(realH).append("h");
                }
            } else {
                stringBuilder.append(h).append("h");
            }
            if (realMin > 0) {
                stringBuilder.append(realMin).append("min");
            }
        } else {
            stringBuilder.append(min).append("min");
        }
        return stringBuilder.toString();
    }

    public static Date DEFAULT_BEGIN_DATE = null;

    @PostConstruct
    public void init() throws ParseException {
        SystemConfig systemConfig = systemConfigService.getById(SystemConfigCode.DEFAULT_START_DATE);
        if (systemConfig != null) {
            DEFAULT_BEGIN_DATE = DateUtils.parseDate(systemConfig.getValue(), "yyyy-MM-dd", "yyyy-MM-dd HH:mm:ss");
        }
    }

    /**
     * 分割任务
     *
     * @param param 参数
     * @return list
     */
    public static List<SalesforceParam> splitTask(SalesforceParam param) {
        if (CollectionUtils.isNotEmpty(param.getIds())) {
            return Lists.newArrayList(param);
        }
        Date now = new Date();
        DateUtils.truncate(now, Calendar.DATE);
        Date beginCreateDate = param.getBeginCreateDate() != null ? param.getBeginCreateDate() : DEFAULT_BEGIN_DATE;
        // 如果为空 取当天零点 作为最大时间
        Date endCreateDate = null;
        try {
            endCreateDate = param.getEndCreateDate() != null ? param.getEndCreateDate() : DateUtils.parseDate(DateFormatUtils.format(now, "yyyy-MM-dd"), "yyyy-MM-dd");
        } catch (Throwable ignored) {
        }

        Date startDate = beginCreateDate;
        Date lastDay = null;
        List<SalesforceParam> list = Lists.newArrayList();
        int batch = 1;
        do {
            lastDay = getYearLastDay(endCreateDate, startDate);

            SalesforceParam clone = param.clone();
            clone.setBeginCreateDate(startDate);
            clone.setEndCreateDate(lastDay);
            clone.setBatch(batch);
            batch++;
            list.add(clone);
            startDate = lastDay;
        } while (lastDay.compareTo(endCreateDate) < 0);
        return list;
    }

    /**
     * 返回本周最后一天 如果大于结束日期 返回结束日期
     *
     * @param endDate   结束日期
     * @param startDate 开始日期
     * @return 本周最后一天
     */
    public static Date getWeekLastDay(Date endDate, Date startDate) {
        Date weekLastDay;
        Calendar cal = DateUtils.toCalendar(startDate);
        cal.set(Calendar.DAY_OF_WEEK, cal.getActualMaximum(Calendar.DAY_OF_WEEK));
        cal.add(Calendar.DAY_OF_WEEK, 1);
        weekLastDay = cal.getTime();
        if (weekLastDay.compareTo(endDate) > 0) {
            weekLastDay = endDate;
        }
        return weekLastDay;
    }

    /**
     * 返回本月最后一天 如果大于结束日期 返回结束日期
     *
     * @param endDate   结束日期
     * @param startDate 开始日期
     * @return 本月最后一天
     */
    public static Date getMonthLastDay(Date endDate, Date startDate) {
        Date monthLastDay;
        Calendar cal = DateUtils.toCalendar(startDate);
        cal.set(Calendar.DAY_OF_MONTH, cal.getActualMaximum(Calendar.DAY_OF_MONTH));
        cal.add(Calendar.DAY_OF_MONTH, 1);
        monthLastDay = cal.getTime();
        if (monthLastDay.compareTo(endDate) > 0) {
            monthLastDay = endDate;
        }
        return monthLastDay;
    }

    /**
     * 返回本周最后一天 如果大于结束日期 返回结束日期
     *
     * @param endDate   结束日期
     * @param startDate 开始日期
     * @return 本周最后一天
     */
    public static Date getYearLastDay(Date endDate, Date startDate) {
        Date yearLastDay;
        startDate = DateUtils.addYears(startDate, 1);
        Calendar cal = DateUtils.toCalendar(startDate);
        cal.set(Calendar.YEAR, cal.get(Calendar.YEAR));
        yearLastDay = cal.getTime();
        if (yearLastDay.compareTo(endDate) > 0) {
            yearLastDay = endDate;
        }
        return yearLastDay;
    }

    /**
     * sObject 数组转 JSONObject
     *
     * @param sObject sObject
     * @param fields  字段
     * @return JSONObject
     */
    public static JSONObject toJsonObject(SObject sObject, Field[] fields) throws ParseException {
        Iterator<XmlObject> children = sObject.getChildren();
        JSONObject jsonObject = new JSONObject();
        CaseInsensitiveMap map = new CaseInsensitiveMap();
        for (Field field : fields) {
            map.put(field.getName(), field.getType().toString());
        }
        while (children.hasNext()) {
            XmlObject next = children.next();
            String name = next.getName().getLocalPart();
            Object value = next.getValue();
            // 全部当string处理
            if (next instanceof SObject) {
                SObject sObjectChild = (SObject) next;
                Iterator<XmlObject> children1 = sObjectChild.getChildren();
                while (children1.hasNext()) {
                    XmlObject childNext = children1.next();
                    String childName = childNext.getName().getLocalPart();
                    Object childValue = childNext.getValue();
                    // partner会多一个type 值为对象名称 这里过滤掉
                    if (Const.TYPE_FILTER.equals(childName)) {
                        continue;
                    }
                    if (childValue == null) {
                        continue;
                    }
                    jsonObject.put(name+"_"+childName, childValue);
                }
            }
            // partner会多一个type 值为对象名称 这里过滤掉
            if (Const.TYPE_FILTER.equals(name)) {
                continue;
            }
            if (value == null) {
                continue;
            }
            String type = (String) map.get(name);
            if (type == null) {
                continue;
            }
            switch (type) {
                case "date":
                    jsonObject.put(name, DateUtils.parseDate((String) value, "yyyy-MM-dd"));
                    break;
                case "datetime":
                    jsonObject.put(name, SF_DATE_FORMAT.parse((String) value));
                    break;
                case "time":
                    jsonObject.put(name, SF_TIME_FORMAT.parse((String) value));
                    break;
                case "boolean":
                    jsonObject.put(name, BooleanUtils.toBoolean((String) value));
                    break;
                case "long":
                case "int":
                    jsonObject.put(name, Integer.parseInt((String) value));
                    break;
                case "double":
                case "currency":
                case "percent":
                    jsonObject.put(name, Double.parseDouble((String) value));
                    break;
                case "base64":
                    // 文件不处理 统一存oss或者server
                    break;
                default:
                    if (value instanceof GregorianCalendar) {
                        jsonObject.put(name, DateFormatUtils.format(((GregorianCalendar) value), "yyyy-MM-dd HH:mm:ss"));
                    } else if (value instanceof Time){
                        jsonObject.put(name, DateFormatUtils.format((((Time) value).getTimeInMillis()), "HH:mm:ss"));
                    }else {
                        jsonObject.put(name,  String.valueOf(value));
                    }
                    break;
            }
        }
        return jsonObject;
    }


    /**
     * sObject 数组转 JSONArray
     *
     * @param records sObject 数组
     * @return JSONArray
     */
    public static JSONArray toJsonArray(SObject[] records, Field[] fields) throws ParseException {
        JSONArray jsonArray = new JSONArray();
        for (SObject sObject : records) {
            jsonArray.add(toJsonObject(sObject, fields));
        }
        return jsonArray;
    }

    public static Object fieldTypeToSf(DataField dataField) throws ParseException {
        String fieldType = dataField.getSfType();
        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
        Date date = sdf.parse("2023-03-23");
        switch (fieldType) {
            case "int":
                return 1;
            case "double":
            case "currency":
            case "percent":
                return new BigDecimal("1");
            case "boolean":
                return true;
            case "email":
                return "jax.li@celnet.com.cn";
            case "date":
                //字符串转日期
                return date;
            case "datetime":
                return date;
            default:
                return "初始"+ UUID.randomUUID();
        }
    }

    public static Object localBulkDataToSfData(String fieldType, String data) throws ParseException {
        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
        SimpleDateFormat sd = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
        Date date;
        //date转Calendar类型
        Calendar calendar = Calendar.getInstance();
        switch (fieldType) {
            case "int":
                return Integer.parseInt(data);
            case "double":
            case "currency":
            case "percent":
                return new BigDecimal(data);
            case "boolean":
                return Boolean.valueOf(data);
            case "date":
                return data+"T08:00:00Z";
            case "datetime":
                try {
                    date = sd.parse(data);
                }catch (ParseException e){
                    //解决当时间秒为0时，转换秒精度丢失问题
                    date = sd.parse(data+":00");
                }
                calendar.setTime(date);
                return calendar;
            case "time":
                return adjustHour(data);
            default:
                return data;
        }
    }

    public static Object localDataToSfData(String fieldType, String data) throws ParseException {
        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
        SimpleDateFormat sd = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
        Date date;
        //date转Calendar类型
        Calendar calendar = Calendar.getInstance();
        switch (fieldType) {
            case "int":
                return Integer.parseInt(data);
            case "double":
            case "currency":
            case "percent":
                return new BigDecimal(data);
            case "boolean":
                return Boolean.valueOf(data);
            case "date":
                try {
                    return sdf.parse(data+"T08:00:00Z");
                } catch (ParseException e) {
                    log.error("exception message", e); 
                }
            case "datetime":
                try {
                    date = sd.parse(data);
                }catch (ParseException e){
                    //解决当时间秒为0时，转换秒精度丢失问题
                    date = sd.parse(data+":00");
                }
                calendar.setTime(date);
                return calendar;
            case "time":
                return adjustHour(data);
            default:
                return data;
        }
    }

    public static String adjustHour(String timeStr) {
        // 提取小时部分并转换为整数
        int hour = Integer.parseInt(timeStr.substring(0,  2));

        // 减去8小时并处理跨天逻辑
        int adjustedHour = hour - 8;
        if (adjustedHour < 0) {
            adjustedHour += 24;  // 处理负数情况（跨天）
        }

        // 格式化为两位字符串（自动补零）
        String newHour = String.format("%02d",  adjustedHour);

        // 拼接原始字符串的剩余部分（分钟+秒+毫秒）
        return newHour + timeStr.substring(2);
    }

    public static boolean isUpdate(String field){
        switch (field) {
            case "LastModifiedDate":
            case "IsDeleted":
            case "SystemModstamp":
            case "CreatedById":
            case "CreatedDate":
            case "LastModifiedById":
            case "LastViewedDate":
            case "LastReferencedDate":
            case "IsConverted":
                return false;
            default:
                return true;
        }
    }

}