584 lines
21 KiB
Java
584 lines
21 KiB
Java
package com.celnet.datadump.util;
|
||
|
||
import cn.hutool.core.io.FileUtil;
|
||
import cn.hutool.core.io.IoUtil;
|
||
|
||
import cn.hutool.core.text.csv.CsvUtil;
|
||
import cn.hutool.core.text.csv.CsvWriteConfig;
|
||
import cn.hutool.core.text.csv.CsvWriter;
|
||
import cn.hutool.core.util.CharsetUtil;
|
||
import cn.hutool.json.JSONObject;
|
||
import com.alibaba.fastjson.JSON;
|
||
import com.celnet.datadump.entity.DataField;
|
||
import com.celnet.datadump.entity.DataObject;
|
||
import com.opencsv.CSVReader;
|
||
import com.opencsv.CSVWriter;
|
||
import com.opencsv.exceptions.CsvException;
|
||
import com.sforce.async.AsyncApiException;
|
||
import com.sforce.async.BatchInfo;
|
||
import com.sforce.async.BulkConnection;
|
||
import com.sforce.async.JobInfo;
|
||
import lombok.extern.slf4j.Slf4j;
|
||
import org.apache.commons.csv.CSVFormat;
|
||
import org.apache.commons.csv.CSVPrinter;
|
||
|
||
import java.io.*;
|
||
import java.nio.charset.StandardCharsets;
|
||
import java.nio.file.Files;
|
||
import java.nio.file.Path;
|
||
import java.nio.file.Paths;
|
||
import java.util.*;
|
||
import java.util.stream.Stream;
|
||
|
||
/**
|
||
* csv格式转换工具类
|
||
*/
|
||
@Slf4j
|
||
public class CsvConverterUtil {
|
||
|
||
/**
|
||
* 将JSONObject（代表一行）或JSONArray（代表多行）写入CSV UTF-8格式文件,后调用方法上传
|
||
* @param jsonList JSON数据
|
||
*/
|
||
public static String writeToCsv(List<JSONObject> jsonList, String fileName, List<DataField> list, Boolean isEditable) {
|
||
|
||
// 1. 创建目标目录（不存在则创建）
|
||
File targetDir = FileUtil.mkdir("data-dump/dataFile");
|
||
|
||
// 2. 构建完整文件路径
|
||
String fullPath = targetDir.getAbsolutePath() + File.separator + fileName + ".csv";
|
||
|
||
CsvWriter csvWriter = CsvUtil.getWriter(fullPath, CharsetUtil.CHARSET_GBK, false);
|
||
|
||
String[] header = null;
|
||
|
||
if (isEditable){
|
||
header = Stream.concat(
|
||
list.stream().filter(dataField ->(dataField.getIsCreateable() != null && dataField.getIsCreateable()))
|
||
.map(DataField::getField),
|
||
Stream.of("Id","old_owner_id__c", "old_sfdc_id__c")
|
||
).toArray(String[]::new);
|
||
}else {
|
||
header = list.stream().filter(dataField ->(dataField.getIsCreateable() != null && dataField.getIsCreateable()))
|
||
.map(DataField::getField).toArray(String[]::new);
|
||
}
|
||
|
||
// 2. 写入表头（必须使用 String[]）
|
||
csvWriter.writeHeaderLine(header);
|
||
|
||
// 遍历数据列表
|
||
for (JSONObject jsonObject : jsonList) {
|
||
// 按表头顺序获取值
|
||
String[] row = new String[header.length];
|
||
for (int i = 0; i < header.length; i++) {
|
||
// 将每个值转换为字符串（如果为null则转为空字符串）
|
||
Object value = jsonObject.get(header[i]);
|
||
row[i] = value == null ? "" : value.toString();
|
||
}
|
||
csvWriter.writeLine(row);
|
||
}
|
||
|
||
// 关闭writer（在try-with-resources中可省略，但这里我们显式关闭）
|
||
csvWriter.close();
|
||
|
||
return fullPath;
|
||
}
|
||
|
||
/**
|
||
* 将JSON数据写入CSV(UTF-8)文件并上传
|
||
* @param jsonList JSON数据列表
|
||
* @param fileName 文件名(不含扩展名)
|
||
* @param fields 字段定义
|
||
* @param isEditable 是否编辑模式
|
||
* @return 批处理信息列表
|
||
*/
|
||
public static List<BatchInfo> writeToCsvNew(List<JSONObject> jsonList,
|
||
String fileName,
|
||
List<DataField> fields,
|
||
Boolean isEditable,
|
||
BulkConnection connection,
|
||
JobInfo jobInfo) {
|
||
|
||
// 1. 创建目标目录（不存在则创建）
|
||
File targetDir = FileUtil.mkdir("data-dump/dataFile");
|
||
|
||
// 2. 构建完整文件路径
|
||
String fullPath = targetDir.getAbsolutePath() + File.separator + fileName + ".csv";
|
||
|
||
CsvWriter csvWriter = CsvUtil.getWriter(fullPath, CharsetUtil.CHARSET_UTF_8, false);
|
||
|
||
String[] header = null;
|
||
|
||
File file = null;
|
||
|
||
if (isEditable){
|
||
header = Stream.concat(
|
||
fields.stream().filter(dataField ->(dataField.getIsCreateable() != null && dataField.getIsCreateable()))
|
||
.map(DataField::getField),
|
||
Stream.of("Id","old_owner_id__c", "old_sfdc_id__c")
|
||
).toArray(String[]::new);
|
||
}else {
|
||
header = fields.stream().filter(dataField ->(dataField.getIsCreateable() != null && dataField.getIsCreateable()))
|
||
.map(DataField::getField).toArray(String[]::new);
|
||
}
|
||
|
||
List<BatchInfo> batchInfos = new ArrayList<>();
|
||
|
||
try {
|
||
|
||
// 2. 写入表头（必须使用 String[]）
|
||
csvWriter.writeHeaderLine(header);
|
||
|
||
// 遍历数据列表
|
||
for (JSONObject jsonObject : jsonList) {
|
||
// 按表头顺序获取值
|
||
String[] row = new String[header.length];
|
||
for (int i = 0; i < header.length; i++) {
|
||
// 将每个值转换为字符串（如果为null则转为空字符串）
|
||
Object value = jsonObject.get(header[i]);
|
||
row[i] = value == null ? "" : value.toString();
|
||
}
|
||
csvWriter.writeLine(row);
|
||
}
|
||
|
||
// 关闭writer（在try-with-resources中可省略，但这里我们显式关闭）
|
||
csvWriter.close();
|
||
|
||
file = new File(fullPath);
|
||
|
||
InputStream inputStream = Files.newInputStream(file.toPath());
|
||
|
||
BatchInfo batch = connection.createBatchFromStream(jobInfo, inputStream);
|
||
|
||
batchInfos.add(batch);
|
||
|
||
} catch (IOException e) {
|
||
log.error("CSV文件操作失败: {}", fullPath, e);
|
||
file.deleteOnExit();
|
||
} catch (AsyncApiException e) {
|
||
log.error("Bulk API上传失败: {}", e.getMessage(), e);
|
||
file.deleteOnExit();
|
||
}
|
||
|
||
return batchInfos;
|
||
}
|
||
|
||
/**
|
||
* 将JSON数据写入CSV(UTF-8)文件并上传
|
||
* @param jsonList JSON数据列表
|
||
* @param fileName 文件名(不含扩展名)
|
||
* @param fields 字段定义
|
||
* @param isEditable 是否编辑模式
|
||
* @return 批处理信息列表
|
||
*/
|
||
public static BatchInfo writeToCsvNewOne(List<JSONObject> jsonList,
|
||
String fileName,
|
||
List<DataField> fields,
|
||
Boolean isEditable,
|
||
BulkConnection connection,
|
||
JobInfo jobInfo) {
|
||
|
||
// 1. 创建目标目录（不存在则创建）
|
||
File targetDir = FileUtil.mkdir("data-dump/dataFile");
|
||
|
||
// 2. 构建完整文件路径
|
||
String fullPath = targetDir.getAbsolutePath() + File.separator + fileName + ".csv";
|
||
|
||
CsvWriter csvWriter = CsvUtil.getWriter(fullPath, CharsetUtil.CHARSET_UTF_8, false);
|
||
|
||
String[] header = null;
|
||
|
||
File file = null;
|
||
|
||
if (isEditable){
|
||
header = Stream.concat(
|
||
fields.stream().map(DataField::getField),
|
||
Stream.of("old_owner_id__c", "old_sfdc_id__c")
|
||
).toArray(String[]::new);
|
||
}else {
|
||
header = fields.stream().map(DataField::getField).toArray(String[]::new);
|
||
}
|
||
|
||
BatchInfo batch = null;
|
||
|
||
try {
|
||
|
||
// 2. 写入表头（必须使用 String[]）
|
||
csvWriter.writeHeaderLine(header);
|
||
|
||
// 遍历数据列表
|
||
for (JSONObject jsonObject : jsonList) {
|
||
// 按表头顺序获取值
|
||
String[] row = new String[header.length];
|
||
for (int i = 0; i < header.length; i++) {
|
||
// 将每个值转换为字符串（如果为null则转为空字符串）
|
||
Object value = jsonObject.get(header[i]);
|
||
row[i] = String.valueOf(value);
|
||
}
|
||
csvWriter.writeLine(row);
|
||
}
|
||
|
||
// 关闭writer（在try-with-resources中可省略，但这里我们显式关闭）
|
||
csvWriter.close();
|
||
|
||
file = new File(fullPath);
|
||
|
||
InputStream inputStream = Files.newInputStream(file.toPath());
|
||
|
||
batch = connection.createBatchFromStream(jobInfo, inputStream);
|
||
|
||
} catch (IOException e) {
|
||
log.error("CSV文件操作失败: {}", fullPath, e);
|
||
file.deleteOnExit();
|
||
} catch (AsyncApiException e) {
|
||
log.error("Bulk API上传失败: {}", e.getMessage(), e);
|
||
file.deleteOnExit();
|
||
}
|
||
|
||
return batch;
|
||
}
|
||
|
||
public static BatchInfo writeToCsvOne(List<JSONObject> jsonList,
|
||
String fileName,
|
||
List<DataField> fields,
|
||
Boolean isEditable,
|
||
BulkConnection connection,
|
||
JobInfo jobInfo,String[] ids) {
|
||
|
||
// 1. 创建目标目录（不存在则创建）
|
||
File targetDir = FileUtil.mkdir("data-dump/dataFile");
|
||
|
||
// 2. 构建完整文件路径
|
||
String fullPath = targetDir.getAbsolutePath() + File.separator + fileName + ".csv";
|
||
|
||
CsvWriter csvWriter = CsvUtil.getWriter(fullPath, CharsetUtil.CHARSET_UTF_8, false);
|
||
|
||
String[] header = null;
|
||
|
||
File file = null;
|
||
|
||
if (isEditable){
|
||
header = Stream.concat(
|
||
fields.stream().filter(dataField ->(dataField.getIsCreateable() != null && dataField.getIsCreateable()))
|
||
.map(DataField::getField),
|
||
Stream.of("old_owner_id__c", "old_sfdc_id__c")
|
||
).toArray(String[]::new);
|
||
}else {
|
||
header = fields.stream().filter(dataField ->(dataField.getIsCreateable() != null && dataField.getIsCreateable()))
|
||
.map(DataField::getField).toArray(String[]::new);
|
||
}
|
||
|
||
BatchInfo batch = null;
|
||
|
||
try {
|
||
|
||
// 2. 写入表头（必须使用 String[]）
|
||
csvWriter.writeHeaderLine(header);
|
||
|
||
int index = 0;
|
||
|
||
// 遍历数据列表
|
||
for (JSONObject jsonObject : jsonList) {
|
||
// 按表头顺序获取值
|
||
String[] row = new String[header.length];
|
||
for (int i = 0; i < header.length; i++) {
|
||
// 将每个值转换为字符串（如果为null则转为空字符串）
|
||
Object value = jsonObject.get(header[i]);
|
||
row[i] = String.valueOf(value);
|
||
}
|
||
ids[index] = jsonObject.get("Id").toString();
|
||
|
||
csvWriter.writeLine(row);
|
||
|
||
index ++;
|
||
}
|
||
|
||
// 关闭writer（在try-with-resources中可省略，但这里我们显式关闭）
|
||
csvWriter.close();
|
||
|
||
file = new File(fullPath);
|
||
|
||
InputStream inputStream = Files.newInputStream(file.toPath());
|
||
|
||
batch = connection.createBatchFromStream(jobInfo, inputStream);
|
||
|
||
} catch (IOException e) {
|
||
log.error("CSV文件操作失败: {}", fullPath, e);
|
||
file.deleteOnExit();
|
||
} catch (AsyncApiException e) {
|
||
log.error("Bulk API上传失败: {}", e.getMessage(), e);
|
||
file.deleteOnExit();
|
||
}
|
||
|
||
return batch;
|
||
}
|
||
|
||
public static String exportToCsv(List<Map<String, Object>> data, String fileName) throws IOException {
|
||
// 1. 创建目标目录（不存在则创建）
|
||
File targetDir = FileUtil.mkdir("data-dump/dataFile");
|
||
|
||
// 2. 构建完整文件路径
|
||
String fullPath = targetDir.getAbsolutePath() + File.separator + fileName + ".csv";
|
||
|
||
CsvWriter csvWriter = CsvUtil.getWriter(fullPath, CharsetUtil.CHARSET_UTF_8, false);
|
||
|
||
// 1. 提取表头（保持顺序）
|
||
Set<String> headers = new LinkedHashSet<>();
|
||
for (Map<String, Object> map : data) {
|
||
headers.addAll(map.keySet());
|
||
}
|
||
String[] headerArray = headers.toArray(new String[0]);
|
||
|
||
// 2. 写入表头（必须使用 String[]）
|
||
csvWriter.writeLine(headerArray);
|
||
|
||
// 3. 写入数据行（需要将 Object 转换为 String）
|
||
for (Map<String, Object> map : data) {
|
||
// 按表头顺序构建行数据
|
||
String[] row = new String[headerArray.length];
|
||
for (int i = 0; i < headerArray.length; i++) {
|
||
Object value = map.get(headerArray[i]);
|
||
// 处理空值和特殊字符
|
||
row[i] = convertToCsvValue(value);
|
||
}
|
||
csvWriter.writeLine(row); // 使用 String[] 参数
|
||
}
|
||
|
||
return fullPath;
|
||
}
|
||
|
||
|
||
|
||
/**
|
||
* @Description: convert list<Map<>> to csv
|
||
* @Param: list<Map<>>,pathName
|
||
* @return:
|
||
*/
|
||
public static String saveOpenToDataDump(List<Map<String,Object>> list,String fileName) throws IOException {
|
||
|
||
// 1. 创建目标目录（不存在则创建）
|
||
File targetDir = FileUtil.mkdir("data-dump/dataFile");
|
||
|
||
// 2. 构建完整文件路径
|
||
String fullPath = targetDir.getAbsolutePath() + File.separator + fileName + ".csv";
|
||
|
||
List<String> headerList = new ArrayList<>();
|
||
for (String s : list.get(0).keySet()) {
|
||
headerList.add(s);
|
||
}
|
||
String[] csvHeader = headerList.toArray(new String[headerList.size()]);
|
||
|
||
FileWriter out = new FileWriter(fullPath);
|
||
|
||
try (CSVPrinter printer = new CSVPrinter(out, CSVFormat.DEFAULT
|
||
.withHeader(csvHeader))) {
|
||
for(Map<String,Object> map:list) {
|
||
List<String> valueList = new ArrayList<>();
|
||
for(String s:headerList)
|
||
valueList.add(String.valueOf(map.get(s)));
|
||
String[] csvValue = valueList.toArray(new String[valueList.size()]);
|
||
printer.printRecord(csvValue);
|
||
}
|
||
}
|
||
|
||
out.close();
|
||
|
||
return fullPath;
|
||
}
|
||
|
||
|
||
|
||
/**
|
||
* 保存 CSV 文件到 data-dump/dataFile 目录
|
||
* @param data 数据集合
|
||
* @param fileName 文件名（无需后缀，自动添加.csv）
|
||
* @return 完整的文件路径
|
||
*/
|
||
public static String saveToDataDump(List<Map<String, Object>> data, String fileName) {
|
||
// 1. 创建目标目录（不存在则创建）
|
||
File targetDir = FileUtil.mkdir("data-dump/dataFile");
|
||
|
||
// 2. 构建完整文件路径
|
||
String fullPath = targetDir.getAbsolutePath() + File.separator + fileName + ".csv";
|
||
|
||
// 3. 写入CSV文件
|
||
try (CsvWriter csvWriter = new CsvWriter(
|
||
new File(fullPath),
|
||
CharsetUtil.CHARSET_UTF_8, // UTF-8编码
|
||
false, // 非追加模式
|
||
new CsvWriteConfig().setFieldSeparator(',') // 逗号分隔符
|
||
)) {
|
||
// 1. 提取表头（保持顺序）
|
||
Set<String> headers = new LinkedHashSet<>();
|
||
for (Map<String, Object> map : data) {
|
||
headers.addAll(map.keySet());
|
||
}
|
||
String[] headerArray = headers.toArray(new String[0]);
|
||
|
||
// 2. 写入表头（必须使用 String[]）
|
||
csvWriter.writeLine(headerArray);
|
||
|
||
// 3. 写入数据行（需要将 Object 转换为 String）
|
||
for (Map<String, Object> map : data) {
|
||
// 按表头顺序构建行数据
|
||
String[] row = new String[headerArray.length];
|
||
for (int i = 0; i < headerArray.length; i++) {
|
||
Object value = map.get(headerArray[i]);
|
||
// 处理空值和特殊字符
|
||
row[i] = convertToCsvValue(value);
|
||
}
|
||
csvWriter.writeLine(row); // 使用 String[] 参数
|
||
}
|
||
// 4. 释放资源并返回结果
|
||
IoUtil.close(csvWriter);
|
||
}
|
||
return fullPath;
|
||
}
|
||
|
||
public static InputStream convertListToCsv(List<Map<String, Object>> data) {
|
||
if (data == null || data.isEmpty()) {
|
||
return null;
|
||
}
|
||
|
||
// 使用 StringWriter 作为缓冲区
|
||
StringWriter stringWriter = new StringWriter();
|
||
CsvWriter csvWriter = CsvUtil.getWriter(stringWriter, CsvWriteConfig.defaultConfig());
|
||
|
||
// 1. 提取表头（保持顺序）
|
||
Set<String> headers = new LinkedHashSet<>();
|
||
for (Map<String, Object> map : data) {
|
||
headers.addAll(map.keySet());
|
||
}
|
||
String[] headerArray = headers.toArray(new String[0]);
|
||
|
||
// 2. 写入表头（必须使用 String[]）
|
||
csvWriter.writeLine(headerArray);
|
||
|
||
// 3. 写入数据行（需要将 Object 转换为 String）
|
||
for (Map<String, Object> map : data) {
|
||
// 按表头顺序构建行数据
|
||
String[] row = new String[headerArray.length];
|
||
for (int i = 0; i < headerArray.length; i++) {
|
||
Object value = map.get(headerArray[i]);
|
||
// 处理空值和特殊字符
|
||
row[i] = convertToCsvValue(value);
|
||
}
|
||
csvWriter.writeLine(row); // 使用 String[] 参数
|
||
}
|
||
|
||
// 4. 释放资源并返回结果
|
||
IoUtil.close(csvWriter);
|
||
return new ByteArrayInputStream(csvWriter.toString().getBytes(StandardCharsets.UTF_8));
|
||
}
|
||
|
||
// 处理特殊值和空值
|
||
private static String convertToCsvValue(Object value) {
|
||
if (value == null) {
|
||
return "";
|
||
}
|
||
// Hutool 会自动处理逗号、引号等特殊字符
|
||
return value.toString();
|
||
}
|
||
|
||
public static List<String> extractIdColumn(InputStream inputStream) throws IOException, CsvException {
|
||
|
||
List<String> idList = new ArrayList<>();
|
||
|
||
try (CSVReader reader = new CSVReader(new InputStreamReader(inputStream))) {
|
||
// 读取CSV表头
|
||
String[] headers = reader.readNext();
|
||
int idIndex = -1;
|
||
|
||
// 查找id列的索引位置
|
||
for (int i = 0; i < headers.length; i++) {
|
||
if ("id".equalsIgnoreCase(headers[i].trim())) {
|
||
idIndex = i;
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (idIndex == -1) {
|
||
throw new IllegalArgumentException("CSV文件中未找到id列");
|
||
}
|
||
|
||
// 逐行读取数据
|
||
String[] nextRecord;
|
||
while ((nextRecord = reader.readNext()) != null) {
|
||
if (idIndex < nextRecord.length) {
|
||
if (!"Id".equals(nextRecord[idIndex])){
|
||
idList.add(nextRecord[idIndex]);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
return idList;
|
||
}
|
||
|
||
public static InputStream convertToCsvStream(List<Map<String, Object>> data) throws IOException {
|
||
if (data == null || data.isEmpty()) {
|
||
return new ByteArrayInputStream(new byte[0]);
|
||
}
|
||
|
||
// 获取所有列头（保持顺序）
|
||
Set<String> headers = new LinkedHashSet<>();
|
||
for (Map<String, Object> map : data) {
|
||
headers.addAll(map.keySet());
|
||
}
|
||
List<String> headerList = new ArrayList<>(headers);
|
||
|
||
try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
|
||
Writer writer = new OutputStreamWriter(outputStream, StandardCharsets.UTF_8)) {
|
||
|
||
// 写入CSV头
|
||
writeCsvLine(writer, headerList);
|
||
|
||
// 写入数据行
|
||
for (Map<String, Object> row : data) {
|
||
List<String> values = new ArrayList<>();
|
||
for (String header : headerList) {
|
||
Object value = row.get(header);
|
||
values.add(value != null ? escapeCsv(value.toString()) : "");
|
||
}
|
||
writeCsvLine(writer, values);
|
||
}
|
||
|
||
writer.flush();
|
||
return new ByteArrayInputStream(outputStream.toByteArray());
|
||
}
|
||
}
|
||
|
||
private static void writeCsvLine(Writer writer, List<String> values) throws IOException {
|
||
if (values.isEmpty()) {
|
||
writer.write("\n");
|
||
return;
|
||
}
|
||
|
||
for (int i = 0; i < values.size(); i++) {
|
||
writer.write(values.get(i));
|
||
if (i < values.size() - 1) {
|
||
writer.write(',');
|
||
}
|
||
}
|
||
writer.write('\n');
|
||
}
|
||
|
||
private static String escapeCsv(String value) {
|
||
if (value == null) {
|
||
return "";
|
||
}
|
||
|
||
// 检查是否需要转义（包含特殊字符）
|
||
boolean needsEscape = value.contains(",")
|
||
|| value.contains("\"")
|
||
|| value.contains("\n")
|
||
|| value.contains("\r");
|
||
|
||
if (!needsEscape) {
|
||
return value;
|
||
}
|
||
|
||
// 转义双引号并包裹整个字段
|
||
return "\"" + value.replace("\"", "\"\"") + "\"";
|
||
}
|
||
}
|