331 lines
11 KiB
Java
331 lines
11 KiB
Java
|
package com.celnet.datadump.util;
|
|||
|
|
|
|||
|
|
import cn.hutool.core.io.FileUtil;
|
|||
|
|
import cn.hutool.core.io.IoUtil;
|
|||
|
|
|
|||
|
|
import cn.hutool.core.text.csv.CsvUtil;
|
|||
|
|
import cn.hutool.core.text.csv.CsvWriteConfig;
|
|||
|
|
import cn.hutool.core.text.csv.CsvWriter;
|
|||
|
|
import cn.hutool.core.util.CharsetUtil;
|
|||
|
|
import cn.hutool.json.JSONObject;
|
|||
|
|
import com.opencsv.CSVReader;
|
|||
|
|
import com.opencsv.exceptions.CsvException;
|
|||
|
|
import org.apache.commons.csv.CSVFormat;
|
|||
|
|
import org.apache.commons.csv.CSVPrinter;
|
|||
|
|
|
|||
|
|
import java.io.*;
|
|||
|
|
import java.nio.charset.StandardCharsets;
|
|||
|
|
import java.util.*;
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* csv格式转换工具类
|
|||
|
|
*/
|
|||
|
|
public class CsvConverterUtil {
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* 将JSONObject（代表一行）或JSONArray（代表多行）写入CSV文件
|
|||
|
|
* @param jsonList JSON数据
|
|||
|
|
*/
|
|||
|
|
public static String writeToCsv(List<JSONObject> jsonList, String fileName) {
|
|||
|
|
|
|||
|
|
// 1. 创建目标目录（不存在则创建）
|
|||
|
|
File targetDir = FileUtil.mkdir("data-dump/dataFile");
|
|||
|
|
|
|||
|
|
// 2. 构建完整文件路径
|
|||
|
|
String fullPath = targetDir.getAbsolutePath() + File.separator + fileName + ".csv";
|
|||
|
|
|
|||
|
|
CsvWriter csvWriter = CsvUtil.getWriter(fullPath, CharsetUtil.CHARSET_GBK, false);
|
|||
|
|
|
|||
|
|
String[] header = jsonList.get(0).keySet().toArray(new String[0]);
|
|||
|
|
|
|||
|
|
// 2. 写入表头（必须使用 String[]）
|
|||
|
|
csvWriter.writeHeaderLine(header);
|
|||
|
|
|
|||
|
|
// 遍历数据列表
|
|||
|
|
for (JSONObject jsonObject : jsonList) {
|
|||
|
|
// 按表头顺序获取值
|
|||
|
|
String[] row = new String[header.length];
|
|||
|
|
for (int i = 0; i < header.length; i++) {
|
|||
|
|
// 将每个值转换为字符串（如果为null则转为空字符串）
|
|||
|
|
Object value = jsonObject.get(header[i]);
|
|||
|
|
row[i] = value == null ? "" : value.toString();
|
|||
|
|
}
|
|||
|
|
csvWriter.writeLine(row);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 关闭writer（在try-with-resources中可省略，但这里我们显式关闭）
|
|||
|
|
csvWriter.close();
|
|||
|
|
|
|||
|
|
return fullPath;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
public static String exportToCsv(List<Map<String, Object>> data, String fileName) throws IOException {
|
|||
|
|
// 1. 创建目标目录（不存在则创建）
|
|||
|
|
File targetDir = FileUtil.mkdir("data-dump/dataFile");
|
|||
|
|
|
|||
|
|
// 2. 构建完整文件路径
|
|||
|
|
String fullPath = targetDir.getAbsolutePath() + File.separator + fileName + ".csv";
|
|||
|
|
|
|||
|
|
CsvWriter csvWriter = CsvUtil.getWriter(fullPath, CharsetUtil.CHARSET_UTF_8, false);
|
|||
|
|
|
|||
|
|
// 1. 提取表头（保持顺序）
|
|||
|
|
Set<String> headers = new LinkedHashSet<>();
|
|||
|
|
for (Map<String, Object> map : data) {
|
|||
|
|
headers.addAll(map.keySet());
|
|||
|
|
}
|
|||
|
|
String[] headerArray = headers.toArray(new String[0]);
|
|||
|
|
|
|||
|
|
// 2. 写入表头（必须使用 String[]）
|
|||
|
|
csvWriter.writeLine(headerArray);
|
|||
|
|
|
|||
|
|
// 3. 写入数据行（需要将 Object 转换为 String）
|
|||
|
|
for (Map<String, Object> map : data) {
|
|||
|
|
// 按表头顺序构建行数据
|
|||
|
|
String[] row = new String[headerArray.length];
|
|||
|
|
for (int i = 0; i < headerArray.length; i++) {
|
|||
|
|
Object value = map.get(headerArray[i]);
|
|||
|
|
// 处理空值和特殊字符
|
|||
|
|
row[i] = convertToCsvValue(value);
|
|||
|
|
}
|
|||
|
|
csvWriter.writeLine(row); // 使用 String[] 参数
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return fullPath;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* @Description: convert list<Map<>> to csv
|
|||
|
|
* @Param: list<Map<>>,pathName
|
|||
|
|
* @return:
|
|||
|
|
*/
|
|||
|
|
public static String saveOpenToDataDump(List<Map<String,Object>> list,String fileName) throws IOException {
|
|||
|
|
|
|||
|
|
// 1. 创建目标目录（不存在则创建）
|
|||
|
|
File targetDir = FileUtil.mkdir("data-dump/dataFile");
|
|||
|
|
|
|||
|
|
// 2. 构建完整文件路径
|
|||
|
|
String fullPath = targetDir.getAbsolutePath() + File.separator + fileName + ".csv";
|
|||
|
|
|
|||
|
|
List<String> headerList = new ArrayList<>();
|
|||
|
|
for (String s : list.get(0).keySet()) {
|
|||
|
|
headerList.add(s);
|
|||
|
|
}
|
|||
|
|
String[] csvHeader = headerList.toArray(new String[headerList.size()]);
|
|||
|
|
|
|||
|
|
FileWriter out = new FileWriter(fullPath);
|
|||
|
|
|
|||
|
|
try (CSVPrinter printer = new CSVPrinter(out, CSVFormat.DEFAULT
|
|||
|
|
.withHeader(csvHeader))) {
|
|||
|
|
for(Map<String,Object> map:list) {
|
|||
|
|
List<String> valueList = new ArrayList<>();
|
|||
|
|
for(String s:headerList)
|
|||
|
|
valueList.add(String.valueOf(map.get(s)));
|
|||
|
|
String[] csvValue = valueList.toArray(new String[valueList.size()]);
|
|||
|
|
printer.printRecord(csvValue);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
out.close();
|
|||
|
|
|
|||
|
|
return fullPath;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* 保存 CSV 文件到 data-dump/dataFile 目录
|
|||
|
|
* @param data 数据集合
|
|||
|
|
* @param fileName 文件名（无需后缀，自动添加.csv）
|
|||
|
|
* @return 完整的文件路径
|
|||
|
|
*/
|
|||
|
|
public static String saveToDataDump(List<Map<String, Object>> data, String fileName) {
|
|||
|
|
// 1. 创建目标目录（不存在则创建）
|
|||
|
|
File targetDir = FileUtil.mkdir("data-dump/dataFile");
|
|||
|
|
|
|||
|
|
// 2. 构建完整文件路径
|
|||
|
|
String fullPath = targetDir.getAbsolutePath() + File.separator + fileName + ".csv";
|
|||
|
|
|
|||
|
|
// 3. 写入CSV文件
|
|||
|
|
try (CsvWriter csvWriter = new CsvWriter(
|
|||
|
|
new File(fullPath),
|
|||
|
|
CharsetUtil.CHARSET_UTF_8, // UTF-8编码
|
|||
|
|
false, // 非追加模式
|
|||
|
|
new CsvWriteConfig().setFieldSeparator(',') // 逗号分隔符
|
|||
|
|
)) {
|
|||
|
|
// 1. 提取表头（保持顺序）
|
|||
|
|
Set<String> headers = new LinkedHashSet<>();
|
|||
|
|
for (Map<String, Object> map : data) {
|
|||
|
|
headers.addAll(map.keySet());
|
|||
|
|
}
|
|||
|
|
String[] headerArray = headers.toArray(new String[0]);
|
|||
|
|
|
|||
|
|
// 2. 写入表头（必须使用 String[]）
|
|||
|
|
csvWriter.writeLine(headerArray);
|
|||
|
|
|
|||
|
|
// 3. 写入数据行（需要将 Object 转换为 String）
|
|||
|
|
for (Map<String, Object> map : data) {
|
|||
|
|
// 按表头顺序构建行数据
|
|||
|
|
String[] row = new String[headerArray.length];
|
|||
|
|
for (int i = 0; i < headerArray.length; i++) {
|
|||
|
|
Object value = map.get(headerArray[i]);
|
|||
|
|
// 处理空值和特殊字符
|
|||
|
|
row[i] = convertToCsvValue(value);
|
|||
|
|
}
|
|||
|
|
csvWriter.writeLine(row); // 使用 String[] 参数
|
|||
|
|
}
|
|||
|
|
// 4. 释放资源并返回结果
|
|||
|
|
IoUtil.close(csvWriter);
|
|||
|
|
}
|
|||
|
|
return fullPath;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
public static InputStream convertListToCsv(List<Map<String, Object>> data) {
|
|||
|
|
if (data == null || data.isEmpty()) {
|
|||
|
|
return null;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 使用 StringWriter 作为缓冲区
|
|||
|
|
StringWriter stringWriter = new StringWriter();
|
|||
|
|
CsvWriter csvWriter = CsvUtil.getWriter(stringWriter, CsvWriteConfig.defaultConfig());
|
|||
|
|
|
|||
|
|
// 1. 提取表头（保持顺序）
|
|||
|
|
Set<String> headers = new LinkedHashSet<>();
|
|||
|
|
for (Map<String, Object> map : data) {
|
|||
|
|
headers.addAll(map.keySet());
|
|||
|
|
}
|
|||
|
|
String[] headerArray = headers.toArray(new String[0]);
|
|||
|
|
|
|||
|
|
// 2. 写入表头（必须使用 String[]）
|
|||
|
|
csvWriter.writeLine(headerArray);
|
|||
|
|
|
|||
|
|
// 3. 写入数据行（需要将 Object 转换为 String）
|
|||
|
|
for (Map<String, Object> map : data) {
|
|||
|
|
// 按表头顺序构建行数据
|
|||
|
|
String[] row = new String[headerArray.length];
|
|||
|
|
for (int i = 0; i < headerArray.length; i++) {
|
|||
|
|
Object value = map.get(headerArray[i]);
|
|||
|
|
// 处理空值和特殊字符
|
|||
|
|
row[i] = convertToCsvValue(value);
|
|||
|
|
}
|
|||
|
|
csvWriter.writeLine(row); // 使用 String[] 参数
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 4. 释放资源并返回结果
|
|||
|
|
IoUtil.close(csvWriter);
|
|||
|
|
return new ByteArrayInputStream(csvWriter.toString().getBytes(StandardCharsets.UTF_8));
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 处理特殊值和空值
|
|||
|
|
private static String convertToCsvValue(Object value) {
|
|||
|
|
if (value == null) {
|
|||
|
|
return "";
|
|||
|
|
}
|
|||
|
|
// Hutool 会自动处理逗号、引号等特殊字符
|
|||
|
|
return value.toString();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
public static List<String> extractIdColumn(InputStream inputStream) throws IOException, CsvException {
|
|||
|
|
|
|||
|
|
List<String> idList = new ArrayList<>();
|
|||
|
|
|
|||
|
|
try (CSVReader reader = new CSVReader(new InputStreamReader(inputStream))) {
|
|||
|
|
// 读取CSV表头
|
|||
|
|
String[] headers = reader.readNext();
|
|||
|
|
int idIndex = -1;
|
|||
|
|
|
|||
|
|
// 查找id列的索引位置
|
|||
|
|
for (int i = 0; i < headers.length; i++) {
|
|||
|
|
if ("id".equalsIgnoreCase(headers[i].trim())) {
|
|||
|
|
idIndex = i;
|
|||
|
|
break;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if (idIndex == -1) {
|
|||
|
|
throw new IllegalArgumentException("CSV文件中未找到id列");
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 逐行读取数据
|
|||
|
|
String[] nextRecord;
|
|||
|
|
while ((nextRecord = reader.readNext()) != null) {
|
|||
|
|
if (idIndex < nextRecord.length) {
|
|||
|
|
if (!"Id".equals(nextRecord[idIndex])){
|
|||
|
|
idList.add(nextRecord[idIndex]);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return idList;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
public static InputStream convertToCsvStream(List<Map<String, Object>> data) throws IOException {
|
|||
|
|
if (data == null || data.isEmpty()) {
|
|||
|
|
return new ByteArrayInputStream(new byte[0]);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 获取所有列头（保持顺序）
|
|||
|
|
Set<String> headers = new LinkedHashSet<>();
|
|||
|
|
for (Map<String, Object> map : data) {
|
|||
|
|
headers.addAll(map.keySet());
|
|||
|
|
}
|
|||
|
|
List<String> headerList = new ArrayList<>(headers);
|
|||
|
|
|
|||
|
|
try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
|
|||
|
|
Writer writer = new OutputStreamWriter(outputStream, StandardCharsets.UTF_8)) {
|
|||
|
|
|
|||
|
|
// 写入CSV头
|
|||
|
|
writeCsvLine(writer, headerList);
|
|||
|
|
|
|||
|
|
// 写入数据行
|
|||
|
|
for (Map<String, Object> row : data) {
|
|||
|
|
List<String> values = new ArrayList<>();
|
|||
|
|
for (String header : headerList) {
|
|||
|
|
Object value = row.get(header);
|
|||
|
|
values.add(value != null ? escapeCsv(value.toString()) : "");
|
|||
|
|
}
|
|||
|
|
writeCsvLine(writer, values);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
writer.flush();
|
|||
|
|
return new ByteArrayInputStream(outputStream.toByteArray());
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
private static void writeCsvLine(Writer writer, List<String> values) throws IOException {
|
|||
|
|
if (values.isEmpty()) {
|
|||
|
|
writer.write("\n");
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
for (int i = 0; i < values.size(); i++) {
|
|||
|
|
writer.write(values.get(i));
|
|||
|
|
if (i < values.size() - 1) {
|
|||
|
|
writer.write(',');
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
writer.write('\n');
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
private static String escapeCsv(String value) {
|
|||
|
|
if (value == null) {
|
|||
|
|
return "";
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 检查是否需要转义（包含特殊字符）
|
|||
|
|
boolean needsEscape = value.contains(",")
|
|||
|
|
|| value.contains("\"")
|
|||
|
|
|| value.contains("\n")
|
|||
|
|
|| value.contains("\r");
|
|||
|
|
|
|||
|
|
if (!needsEscape) {
|
|||
|
|
return value;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 转义双引号并包裹整个字段
|
|||
|
|
return "\"" + value.replace("\"", "\"\"") + "\"";
|
|||
|
|
}
|
|||
|
|
}
|