268 lines
8.6 KiB
Java
268 lines
8.6 KiB
Java
package com.sforce.async;
|
|
|
|
import java.io.BufferedReader;
|
|
import java.io.IOException;
|
|
import java.io.InputStream;
|
|
import java.io.InputStreamReader;
|
|
import java.io.Reader;
|
|
import java.io.StreamTokenizer;
|
|
import java.io.UnsupportedEncodingException;
|
|
import java.util.ArrayList;
|
|
import java.util.Arrays;
|
|
import java.util.List;
|
|
|
|
public class CSVReader {
|
|
private StreamTokenizer parser;
|
|
|
|
private char[] separators;
|
|
|
|
private boolean ignoreBlankRecords = true;
|
|
|
|
private int maxSizeOfIndividualCell = 131072;
|
|
|
|
private int maxColumnsPerRow = 5000;
|
|
|
|
private int maxRowSizeInCharacters = 400000;
|
|
|
|
private int maxFileSizeInCharacters = 10000000;
|
|
|
|
private int maxRowsInFile = 10001;
|
|
|
|
private int fileSizeInCharacters = 0;
|
|
|
|
private int rowsInFile = 0;
|
|
|
|
private int maxFieldCount;
|
|
|
|
boolean atEOF;
|
|
|
|
public CSVReader(Reader input) {
|
|
this(new BufferedReader(input));
|
|
}
|
|
|
|
public CSVReader(Reader input, char customizedSeparator) {
|
|
this(new BufferedReader(input), customizedSeparator);
|
|
}
|
|
|
|
public CSVReader(Reader input, char[] customizedSeparators) {
|
|
this(new BufferedReader(input), customizedSeparators);
|
|
}
|
|
|
|
public CSVReader(InputStream input) {
|
|
this(new InputStreamReader(input));
|
|
}
|
|
|
|
public CSVReader(InputStream input, char customizedSeparator) throws UnsupportedEncodingException {
|
|
this(new InputStreamReader(input), customizedSeparator);
|
|
}
|
|
|
|
public CSVReader(InputStream input, char[] customizedSeparators) throws UnsupportedEncodingException {
|
|
this(new InputStreamReader(input), customizedSeparators);
|
|
}
|
|
|
|
public CSVReader(InputStream input, String enc) throws UnsupportedEncodingException {
|
|
this(new InputStreamReader(input, enc));
|
|
}
|
|
|
|
public CSVReader(InputStream input, String enc, char customizedSeparator) throws UnsupportedEncodingException {
|
|
this(new InputStreamReader(input, enc), customizedSeparator);
|
|
}
|
|
|
|
public CSVReader(InputStream input, String enc, char[] customizedSeparators) throws UnsupportedEncodingException {
|
|
this(new InputStreamReader(input, enc), customizedSeparators);
|
|
}
|
|
|
|
public CSVReader(BufferedReader input) {
|
|
this(input, ',');
|
|
}
|
|
|
|
public CSVReader(BufferedReader input, char customizedSeparator) {
|
|
this(input, new char[] { customizedSeparator });
|
|
}
|
|
|
|
public CSVReader(BufferedReader input, char[] customizedSeparators) {
|
|
Arrays.sort(customizedSeparators);
|
|
this.separators = customizedSeparators;
|
|
this.parser = new StreamTokenizer(input);
|
|
this.parser.ordinaryChars(0, 255);
|
|
this.parser.wordChars(0, 255);
|
|
this.parser.ordinaryChar(34);
|
|
byte b;
|
|
int i;
|
|
char[] arrayOfChar;
|
|
for (i = (arrayOfChar = customizedSeparators).length, b = 0; b < i; ) {
|
|
char customizedSeparator = arrayOfChar[b];
|
|
this.parser.ordinaryChar(customizedSeparator);
|
|
b++;
|
|
}
|
|
this.parser.eolIsSignificant(true);
|
|
this.parser.whitespaceChars(10, 10);
|
|
this.parser.whitespaceChars(13, 13);
|
|
this.atEOF = false;
|
|
}
|
|
|
|
private void checkRecordExceptions(List<String> line) throws IOException {
|
|
int rowSizeInCharacters = 0;
|
|
if (line != null) {
|
|
for (String value : line) {
|
|
if (value != null)
|
|
rowSizeInCharacters += value.length();
|
|
}
|
|
if (rowSizeInCharacters > this.maxRowSizeInCharacters)
|
|
throw new CSVParseException("Exceeded max length for one record: " + rowSizeInCharacters +
|
|
". Max length for one record should be less than or equal to " +
|
|
this.maxRowSizeInCharacters, this.parser.lineno());
|
|
this.fileSizeInCharacters += rowSizeInCharacters;
|
|
if (this.fileSizeInCharacters > this.maxFileSizeInCharacters)
|
|
throw new CSVParseException("Exceeded max file size: " + this.fileSizeInCharacters +
|
|
". Max file size in characters should be less than or equal to " +
|
|
this.maxFileSizeInCharacters, this.parser.lineno());
|
|
this.rowsInFile++;
|
|
if (this.rowsInFile > this.maxRowsInFile)
|
|
throw new CSVParseException("Exceeded number of records : " + this.rowsInFile +
|
|
". Number of records should be less than or equal to " + this.maxRowsInFile,
|
|
this.parser.lineno());
|
|
}
|
|
}
|
|
|
|
public ArrayList<String> nextRecord() throws IOException {
|
|
ArrayList<String> record = nextRecordLocal();
|
|
if (this.ignoreBlankRecords)
|
|
while (record != null) {
|
|
boolean emptyLine = false;
|
|
if (record.size() == 0) {
|
|
emptyLine = true;
|
|
} else if (record.size() == 1) {
|
|
String val = record.get(0);
|
|
if (val == null || val.length() == 0)
|
|
emptyLine = true;
|
|
}
|
|
if (emptyLine) {
|
|
record = nextRecordLocal();
|
|
continue;
|
|
}
|
|
break;
|
|
}
|
|
checkRecordExceptions(record);
|
|
return record;
|
|
}
|
|
|
|
private ArrayList<String> nextRecordLocal() throws IOException {
|
|
if (this.atEOF)
|
|
return null;
|
|
ArrayList<String> record = new ArrayList<>(this.maxFieldCount);
|
|
StringBuilder fieldValue = null;
|
|
while (true) {
|
|
int token = this.parser.nextToken();
|
|
if (token == -1) {
|
|
addField(record, fieldValue);
|
|
this.atEOF = true;
|
|
break;
|
|
}
|
|
if (token == 10) {
|
|
addField(record, fieldValue);
|
|
break;
|
|
}
|
|
if (token == -3) {
|
|
if (fieldValue != null)
|
|
throw new CSVParseException(String.format("Unknown error, near %s", new Object[] { fieldValue }), this.parser.lineno());
|
|
fieldValue = new StringBuilder(this.parser.sval);
|
|
continue;
|
|
}
|
|
if (Arrays.binarySearch(this.separators, (char)token) >= 0) {
|
|
addField(record, fieldValue);
|
|
fieldValue = null;
|
|
continue;
|
|
}
|
|
if (token == 34) {
|
|
int nextToken;
|
|
if (fieldValue != null)
|
|
throw new CSVParseException("Found unescaped quote. A value with quote should be within a quote",
|
|
this.parser.lineno());
|
|
while (true) {
|
|
token = this.parser.nextToken();
|
|
if (token == -1) {
|
|
this.atEOF = true;
|
|
throw new CSVParseException("EOF reached before closing an opened quote", this.parser.lineno());
|
|
}
|
|
if (token == 10) {
|
|
fieldValue = appendFieldValue(fieldValue, "\n");
|
|
continue;
|
|
}
|
|
if (token == -3) {
|
|
fieldValue = appendFieldValue(fieldValue, this.parser.sval);
|
|
continue;
|
|
}
|
|
if (Arrays.binarySearch(this.separators, (char)token) >= 0) {
|
|
fieldValue = appendFieldValue(fieldValue, token);
|
|
continue;
|
|
}
|
|
if (token == 34) {
|
|
nextToken = this.parser.nextToken();
|
|
if (nextToken == 34) {
|
|
fieldValue = appendFieldValue(fieldValue, nextToken);
|
|
continue;
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
if (nextToken == -3)
|
|
throw new CSVParseException("Not expecting more text after end quote", this.parser.lineno());
|
|
this.parser.pushBack();
|
|
}
|
|
}
|
|
if (record.size() > this.maxFieldCount)
|
|
this.maxFieldCount = record.size();
|
|
return record;
|
|
}
|
|
|
|
private StringBuilder appendFieldValue(StringBuilder fieldValue, int token) throws CSVParseException {
|
|
return appendFieldValue(fieldValue, (char)token);
|
|
}
|
|
|
|
private StringBuilder appendFieldValue(StringBuilder fieldValue, String token) throws CSVParseException {
|
|
if (fieldValue == null)
|
|
fieldValue = new StringBuilder();
|
|
fieldValue.append(token);
|
|
if (token.length() > this.maxSizeOfIndividualCell)
|
|
throw new CSVParseException("Exceeded max field size: " + token.length(), this.parser.lineno());
|
|
return fieldValue;
|
|
}
|
|
|
|
private void addField(ArrayList<String> record, StringBuilder fieldValue) throws CSVParseException {
|
|
record.add((fieldValue == null) ? null : fieldValue.toString());
|
|
if (record.size() > this.maxColumnsPerRow)
|
|
throw new CSVParseException("Exceeded max number of columns per record : " + this.maxColumnsPerRow,
|
|
this.parser.lineno());
|
|
}
|
|
|
|
public int getMaxRowsInFile() {
|
|
return this.maxRowsInFile;
|
|
}
|
|
|
|
public void setMaxRowsInFile(int newMax) {
|
|
this.maxRowsInFile = newMax;
|
|
}
|
|
|
|
public void setMaxCharsInFile(int newMax) {
|
|
this.maxFileSizeInCharacters = newMax;
|
|
}
|
|
|
|
public static class CSVParseException extends IOException {
|
|
final int recordNumber;
|
|
|
|
CSVParseException(String message, int lineno) {
|
|
super(message);
|
|
this.recordNumber = lineno;
|
|
}
|
|
|
|
CSVParseException(int i) {
|
|
this.recordNumber = i;
|
|
}
|
|
|
|
public int getRecordNumber() {
|
|
return this.recordNumber;
|
|
}
|
|
}
|
|
}
|