The map type.
* @param map The Map to populate.
* @return the given map.
* @since 1.9.0
diff --git a/src/main/java/org/apache/commons/csv/Constants.java b/src/main/java/org/apache/commons/csv/Constants.java
index 0b9476e1c..9dd276ecc 100644
--- a/src/main/java/org/apache/commons/csv/Constants.java
+++ b/src/main/java/org/apache/commons/csv/Constants.java
@@ -40,7 +40,7 @@ final class Constants {
/** RFC 4180 defines line breaks as CRLF. */
static final String CRLF = "\r\n";
- static final Character DOUBLE_QUOTE_CHAR = Character.valueOf('"'); // Explicit (un)boxing is intentional.
+ static final Character DOUBLE_QUOTE_CHAR = Character.valueOf('"'); // Explicit boxing is intentional.
static final String EMPTY = "";
diff --git a/src/main/java/org/apache/commons/csv/DuplicateHeaderMode.java b/src/main/java/org/apache/commons/csv/DuplicateHeaderMode.java
index 01989d664..8087f16ee 100644
--- a/src/main/java/org/apache/commons/csv/DuplicateHeaderMode.java
+++ b/src/main/java/org/apache/commons/csv/DuplicateHeaderMode.java
@@ -20,7 +20,7 @@
package org.apache.commons.csv;
/**
- * Determines how duplicate header fields should be handled
+ * Enumerates how duplicate header fields should be handled
* if {@link CSVFormat.Builder#setHeader(Class)} is not null.
*
* @since 1.10.0
diff --git a/src/main/java/org/apache/commons/csv/ExtendedBufferedReader.java b/src/main/java/org/apache/commons/csv/ExtendedBufferedReader.java
index 8dcda6517..20c1ef544 100644
--- a/src/main/java/org/apache/commons/csv/ExtendedBufferedReader.java
+++ b/src/main/java/org/apache/commons/csv/ExtendedBufferedReader.java
@@ -37,26 +37,30 @@
/**
* A special buffered reader which supports sophisticated read access.
*
- * In particular the reader supports a look-ahead option, which allows you to see the next char returned by
- * {@link #read()}. This reader also tracks how many characters have been read with {@link #getPosition()}.
+ * In particular the reader supports a look-ahead option, which allows you to see the next char returned by {@link #read()}. This reader also tracks how many
+ * characters have been read with {@link #getPosition()}.
*
*/
final class ExtendedBufferedReader extends UnsynchronizedBufferedReader {
/** The last char returned */
private int lastChar = UNDEFINED;
+
private int lastCharMark = UNDEFINED;
/** The count of EOLs (CR/LF/CRLF) seen so far */
private long lineNumber;
+
private long lineNumberMark;
/** The position, which is the number of characters read so far */
private long position;
+
private long positionMark;
/** The number of bytes read so far. */
private long bytesRead;
+
private long bytesReadMark;
/** Encoder for calculating the number of bytes for each character read. */
@@ -70,12 +74,11 @@ final class ExtendedBufferedReader extends UnsynchronizedBufferedReader {
}
/**
- * Constructs a new instance with the specified reader, character set,
- * and byte tracking option. Initializes an encoder if byte tracking is enabled
- * and a character set is provided.
+ * Constructs a new instance with the specified reader, character set, and byte tracking option. Initializes an encoder if byte tracking is enabled and a
+ * character set is provided.
*
- * @param reader the reader supports a look-ahead option.
- * @param charset the character set for encoding, or {@code null} if not applicable.
+ * @param reader the reader supports a look-ahead option.
+ * @param charset the character set for encoding, or {@code null} if not applicable.
* @param trackBytes {@code true} to enable byte tracking; {@code false} to disable it.
*/
ExtendedBufferedReader(final Reader reader, final Charset charset, final boolean trackBytes) {
@@ -86,8 +89,7 @@ final class ExtendedBufferedReader extends UnsynchronizedBufferedReader {
/**
* Closes the stream.
*
- * @throws IOException
- * If an I/O error occurs
+ * @throws IOException If an I/O error occurs
*/
@Override
public void close() throws IOException {
@@ -105,26 +107,35 @@ long getBytesRead() {
return this.bytesRead;
}
+ private long getEncodedCharLength(final char[] buf, final int offset, final int length) throws CharacterCodingException {
+ long len = 0;
+ int previous = lastChar;
+ for (int i = offset; i < offset + length; i++) {
+ len += getEncodedCharLength(previous, buf[i]);
+ previous = buf[i];
+ }
+ return len;
+ }
+
/**
- * Gets the byte length of the given character based on the original Unicode
- * specification, which defined characters as fixed-width 16-bit entities.
+ * Gets the byte length of the given character based on the original Unicode specification, which defined characters as fixed-width 16-bit entities.
*
* The Unicode characters are divided into two main ranges:
*
- * - U+0000 to U+FFFF (Basic Multilingual Plane, BMP):
- *
- * - Represented using a single 16-bit {@code char}.
- * - Includes UTF-8 encodings of 1-byte, 2-byte, and some 3-byte characters.
- *
- *
- * - U+10000 to U+10FFFF (Supplementary Characters):
- *
- * - Represented as a pair of {@code char}s:
- * - The first {@code char} is from the high-surrogates range (\uD800-\uDBFF).
- * - The second {@code char} is from the low-surrogates range (\uDC00-\uDFFF).
- * - Includes UTF-8 encodings of some 3-byte characters and all 4-byte characters.
- *
- *
+ * - U+0000 to U+FFFF (Basic Multilingual Plane, BMP):
+ *
+ * - Represented using a single 16-bit {@code char}.
+ * - Includes UTF-8 encodings of 1-byte, 2-byte, and some 3-byte characters.
+ *
+ *
+ * - U+10000 to U+10FFFF (Supplementary Characters):
+ *
+ * - Represented as a pair of {@code char}s:
+ * - The first {@code char} is from the high-surrogates range (\uD800-\uDBFF).
+ * - The second {@code char} is from the low-surrogates range (\uDC00-\uDFFF).
+ * - Includes UTF-8 encodings of some 3-byte characters and all 4-byte characters.
+ *
+ *
*
*
* @param current the current character to process.
@@ -132,8 +143,12 @@ long getBytesRead() {
* @throws CharacterCodingException if the character cannot be encoded.
*/
private int getEncodedCharLength(final int current) throws CharacterCodingException {
+ return getEncodedCharLength(lastChar, current);
+ }
+
+ private int getEncodedCharLength(final int previous, final int current) throws CharacterCodingException {
final char cChar = (char) current;
- final char lChar = (char) lastChar;
+ final char lChar = (char) previous;
if (!Character.isSurrogate(cChar)) {
return encoder.encode(CharBuffer.wrap(new char[] { cChar })).limit();
}
@@ -148,10 +163,9 @@ private int getEncodedCharLength(final int current) throws CharacterCodingExcept
}
/**
- * Returns the last character that was read as an integer (0 to 65535). This will be the last character returned by
- * any of the read methods. This will not include a character read using the {@link #peek()} method. If no
- * character has been read then this will return {@link Constants#UNDEFINED}. If the end of the stream was reached
- * on the last read then this will return {@link IOUtils#EOF}.
+ * Returns the last character that was read as an integer (0 to 65535). This will be the last character returned by any of the read methods. This will not
+ * include a character read using the {@link #peek()} method. If no character has been read then this will return {@link Constants#UNDEFINED}. If the end of
+ * the stream was reached on the last read then this will return {@link IOUtils#EOF}.
*
* @return the last character that was read
*/
@@ -193,11 +207,10 @@ public void mark(final int readAheadLimit) throws IOException {
@Override
public int read() throws IOException {
final int current = super.read();
- if (current == CR || current == LF && lastChar != CR ||
- current == EOF && lastChar != CR && lastChar != LF && lastChar != EOF) {
+ if (current == CR || current == LF && lastChar != CR || current == EOF && lastChar != CR && lastChar != LF && lastChar != EOF) {
lineNumber++;
}
- if (encoder != null) {
+ if (encoder != null && current != EOF) {
this.bytesRead += getEncodedCharLength(current);
}
lastChar = current;
@@ -211,6 +224,9 @@ public int read(final char[] buf, final int offset, final int length) throws IOE
return 0;
}
final int len = super.read(buf, offset, length);
+ if (encoder != null && len > 0) {
+ this.bytesRead += getEncodedCharLength(buf, offset, len);
+ }
if (len > 0) {
for (int i = offset; i < offset + len; i++) {
final char ch = buf[i];
@@ -231,8 +247,7 @@ public int read(final char[] buf, final int offset, final int length) throws IOE
}
/**
- * Gets the next line, dropping the line terminator(s). This method should only be called when processing a
- * comment, otherwise, information can be lost.
+ * Gets the next line, dropping the line terminator(s). This method should only be called when processing a comment, otherwise, information can be lost.
*
* Increments {@link #lineNumber} and updates {@link #position}.
*
@@ -272,5 +287,4 @@ public void reset() throws IOException {
bytesRead = bytesReadMark;
super.reset();
}
-
}
diff --git a/src/main/java/org/apache/commons/csv/Lexer.java b/src/main/java/org/apache/commons/csv/Lexer.java
index 3d00fe0bf..fe964480a 100644
--- a/src/main/java/org/apache/commons/csv/Lexer.java
+++ b/src/main/java/org/apache/commons/csv/Lexer.java
@@ -23,6 +23,7 @@
import java.io.Closeable;
import java.io.IOException;
+import java.util.Arrays;
import org.apache.commons.io.IOUtils;
@@ -68,8 +69,8 @@ final class Lexer implements Closeable {
/**
* Appends the next escaped character to the token's content.
*
- * @param token the current token
- * @throws IOException on stream access error
+ * @param token the current token.
+ * @throws IOException on stream access error.
* @throws CSVException Thrown on invalid input.
*/
private void appendNextEscapedCharacterToToken(final Token token) throws IOException {
@@ -89,7 +90,7 @@ private void appendNextEscapedCharacterToToken(final Token token) throws IOExcep
* Closes resources.
*
* @throws IOException
- * If an I/O error occurs
+ * If an I/O error occurs.
*/
@Override
public void close() throws IOException {
@@ -97,27 +98,27 @@ public void close() throws IOException {
}
/**
- * Gets the number of bytes read
+ * Gets the number of bytes read.
*
- * @return the number of bytes read
+ * @return the number of bytes read.
*/
long getBytesRead() {
return reader.getBytesRead();
}
/**
- * Returns the current character position
+ * Gets the current character position.
*
- * @return the current character position
+ * @return the current character position.
*/
long getCharacterPosition() {
return reader.getPosition();
}
/**
- * Returns the current line number
+ * Gets the current line number.
*
- * @return the current line number
+ * @return the current line number.
*/
long getCurrentLineNumber() {
return reader.getLineNumber();
@@ -136,7 +137,7 @@ boolean isCommentStart(final int ch) {
}
/**
- * Determine whether the next characters constitute a delimiter through {@link ExtendedBufferedReader#peek(char[])}.
+ * Tests whether the next characters constitute a delimiter through {@link ExtendedBufferedReader#peek(char[])}.
*
* @param ch
* the current character.
@@ -152,6 +153,7 @@ boolean isDelimiter(final int ch) throws IOException {
isLastTokenDelimiter = true;
return true;
}
+ Arrays.fill(delimiterBuf, '\0');
reader.peek(delimiterBuf);
for (int i = 0; i < delimiterBuf.length; i++) {
if (delimiterBuf[i] != delimiter[i + 1]) {
@@ -190,6 +192,7 @@ boolean isEscape(final int ch) {
* @throws IOException If an I/O error occurs.
*/
boolean isEscapeDelimiter() throws IOException {
+ Arrays.fill(escapeDelimiterBuf, '\0');
reader.peek(escapeDelimiterBuf);
if (escapeDelimiterBuf[0] != delimiter[0]) {
return false;
@@ -214,7 +217,7 @@ boolean isQuoteChar(final int ch) {
/**
* Tests if the current character represents the start of a line: a CR, LF, or is at the start of the file.
*
- * @param ch the character to check
+ * @param ch the character to check.
* @return true if the character is at the start of a line.
*/
boolean isStartOfLine(final int ch) {
@@ -274,15 +277,22 @@ Token nextToken(final Token token) throws IOException {
}
// Important: make sure a new char gets consumed in each iteration
while (token.type == Token.Type.INVALID) {
+ // isDelimiter consumes the trailing characters of a multi-character delimiter as a side effect, so it must
+ // only be evaluated once per character. Remember a match found while skipping whitespace below.
+ boolean delimiter = false;
// ignore whitespaces at beginning of a token
if (ignoreSurroundingSpaces) {
- while (Character.isWhitespace((char) c) && !isDelimiter(c) && !eol) {
+ while (Character.isWhitespace((char) c) && !eol) {
+ if (isDelimiter(c)) {
+ delimiter = true;
+ break;
+ }
c = reader.read();
eol = readEndOfLine(c);
}
}
// ok, start of token reached: encapsulated, or token
- if (isDelimiter(c)) {
+ if (delimiter || isDelimiter(c)) {
// empty token return TOKEN("")
token.type = Token.Type.TOKEN;
} else if (eol) {
@@ -400,10 +410,10 @@ private Token parseEncapsulatedToken(final Token token) throws IOException {
* An unescaped delimiter has been reached (TOKEN)
*
*
- * @param token the current token
- * @param ch the current character
- * @return the filled token
- * @throws IOException on stream access error
+ * @param token the current token.
+ * @param ch the current character.
+ * @return the filled token.
+ * @throws IOException on stream access error.
* @throws CSVException Thrown on invalid input.
*/
private Token parseSimpleToken(final Token token, final int ch) throws IOException {
@@ -442,7 +452,7 @@ private Token parseSimpleToken(final Token token, final int ch) throws IOExcepti
/**
* Greedily accepts \n, \r and \r\n This checker consumes silently the second control-character...
*
- * @return true if the given or next character is a line-terminator
+ * @return true if the given or next character is a line-terminator.
*/
boolean readEndOfLine(final int ch) throws IOException {
// check if we have \r\n...
diff --git a/src/main/java/org/apache/commons/csv/QuoteMode.java b/src/main/java/org/apache/commons/csv/QuoteMode.java
index 79bb1b34e..ae64ab486 100644
--- a/src/main/java/org/apache/commons/csv/QuoteMode.java
+++ b/src/main/java/org/apache/commons/csv/QuoteMode.java
@@ -19,7 +19,7 @@
package org.apache.commons.csv;
/**
- * Defines quoting behavior.
+ * Enumerates quoting behavior.
*
* @see CSVFormat.Builder#setQuoteMode(QuoteMode)
*/
diff --git a/src/media/commons-logo-component-100.xcf b/src/media/commons-logo-component-100.xcf
new file mode 100644
index 000000000..77d92f277
Binary files /dev/null and b/src/media/commons-logo-component-100.xcf differ
diff --git a/src/media/commons-logo-component.xcf b/src/media/commons-logo-component.xcf
new file mode 100644
index 000000000..3670221da
Binary files /dev/null and b/src/media/commons-logo-component.xcf differ
diff --git a/src/media/logo-large.xcf b/src/media/logo-large.xcf
deleted file mode 100644
index 7bb07af3e..000000000
Binary files a/src/media/logo-large.xcf and /dev/null differ
diff --git a/src/media/logo.png b/src/media/logo.png
index 582fba980..93bb6c014 100644
Binary files a/src/media/logo.png and b/src/media/logo.png differ
diff --git a/src/media/logo.xcf b/src/media/logo.xcf
deleted file mode 100644
index ac6376f71..000000000
Binary files a/src/media/logo.xcf and /dev/null differ
diff --git a/src/site/resources/images/logo.png b/src/site/resources/images/logo.png
index 582fba980..93bb6c014 100644
Binary files a/src/site/resources/images/logo.png and b/src/site/resources/images/logo.png differ
diff --git a/src/site/xdoc/index.xml b/src/site/xdoc/index.xml
index af49476f9..ac5b8cfa9 100644
--- a/src/site/xdoc/index.xml
+++ b/src/site/xdoc/index.xml
@@ -24,12 +24,10 @@ limitations under the License.
-
Commons CSV reads and writes files in variations of the Comma Separated Value (CSV) format.
Read the documentation starting with the Javadoc Overview.
-
-
-
-
+
+
+ Apache Commons CSV requires Java 8 or above.
+
+
+
The commons developer mailing list is the main channel of communication for contributors. Please remember that the lists are shared between all commons components, so prefix your email by [csv].
@@ -87,7 +103,6 @@ For previous releases, see the TagList report.
If you'd like to offer up pull requests via GitHub rather than applying patches to JIRA, we have a GitHub mirror.
-
-
-
diff --git a/src/site/xdoc/security.xml b/src/site/xdoc/security.xml
index ab0056049..47edf5d11 100644
--- a/src/site/xdoc/security.xml
+++ b/src/site/xdoc/security.xml
@@ -47,5 +47,10 @@
+
\ No newline at end of file
diff --git a/src/test/java/org/apache/commons/csv/CSVDuplicateHeaderTest.java b/src/test/java/org/apache/commons/csv/CSVDuplicateHeaderTest.java
index cc47b999a..2f518a120 100644
--- a/src/test/java/org/apache/commons/csv/CSVDuplicateHeaderTest.java
+++ b/src/test/java/org/apache/commons/csv/CSVDuplicateHeaderTest.java
@@ -19,13 +19,16 @@
package org.apache.commons.csv;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
@@ -285,11 +288,11 @@ void testCSVFormat(final DuplicateHeaderMode duplicateHeaderMode,
.setHeader(headers);
if (valid) {
final CSVFormat format = builder.get();
- Assertions.assertEquals(duplicateHeaderMode, format.getDuplicateHeaderMode(), "DuplicateHeaderMode");
- Assertions.assertEquals(allowMissingColumnNames, format.getAllowMissingColumnNames(), "AllowMissingColumnNames");
- Assertions.assertArrayEquals(headers, format.getHeader(), "Header");
+ assertEquals(duplicateHeaderMode, format.getDuplicateHeaderMode(), "DuplicateHeaderMode");
+ assertEquals(allowMissingColumnNames, format.getAllowMissingColumnNames(), "AllowMissingColumnNames");
+ assertArrayEquals(headers, format.getHeader(), "Header");
} else {
- Assertions.assertThrows(IllegalArgumentException.class, builder::get);
+ assertThrows(IllegalArgumentException.class, builder::get);
}
}
@@ -327,10 +330,10 @@ void testCSVParser(final DuplicateHeaderMode duplicateHeaderMode,
try (CSVParser parser = CSVParser.parse(input, format)) {
// Parser ignores null headers
final List expected = Arrays.stream(headers).filter(s -> s != null).collect(Collectors.toList());
- Assertions.assertEquals(expected, parser.getHeaderNames(), "HeaderNames");
+ assertEquals(expected, parser.getHeaderNames(), "HeaderNames");
}
} else {
- Assertions.assertThrows(IllegalArgumentException.class, () -> CSVParser.parse(input, format));
+ assertThrows(IllegalArgumentException.class, () -> CSVParser.parse(input, format));
}
}
}
diff --git a/src/test/java/org/apache/commons/csv/CSVFormatTest.java b/src/test/java/org/apache/commons/csv/CSVFormatTest.java
index d1d19f755..ed20898de 100644
--- a/src/test/java/org/apache/commons/csv/CSVFormatTest.java
+++ b/src/test/java/org/apache/commons/csv/CSVFormatTest.java
@@ -26,6 +26,7 @@
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNotSame;
import static org.junit.jupiter.api.Assertions.assertNull;
@@ -48,7 +49,6 @@
import java.util.Objects;
import org.apache.commons.csv.CSVFormat.Builder;
-import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
/**
@@ -64,16 +64,16 @@ public enum Header {
Name, Email, Phone
}
- private static void assertNotEquals(final Object right, final Object left) {
- Assertions.assertNotEquals(right, left);
- Assertions.assertNotEquals(left, right);
+ private static void assertNotEqualsFlip(final Object right, final Object left) {
+ assertNotEquals(right, left);
+ assertNotEquals(left, right);
}
private static CSVFormat copy(final CSVFormat format) {
return format.builder().setDelimiter(format.getDelimiter()).get();
}
- private void assertNotEquals(final String name, final String type, final Object left, final Object right) {
+ private void assertNotEqualsHash(final String name, final String type, final Object left, final Object right) {
if (left.equals(right) || right.equals(left)) {
fail("Objects must not compare equal for " + name + "(" + type + ")");
}
@@ -198,8 +198,8 @@ void testDuplicateHeaderElementsTrueContainsEmpty3() {
void testEquals() {
final CSVFormat right = CSVFormat.DEFAULT;
final CSVFormat left = copy(right);
- Assertions.assertNotEquals(null, right);
- Assertions.assertNotEquals("A String Instance", right);
+ assertNotEquals(null, right);
+ assertNotEquals("A String Instance", right);
assertEquals(right, right);
assertEquals(right, left);
assertEquals(left, right);
@@ -212,7 +212,7 @@ void testEqualsCommentStart() {
final CSVFormat right = CSVFormat.newFormat('\'').builder().setQuote('"').setCommentMarker('#').setQuoteMode(QuoteMode.ALL).get();
final CSVFormat left = right.builder().setCommentMarker('!').get();
- assertNotEquals(right, left);
+ assertNotEqualsFlip(right, left);
}
@SuppressWarnings("deprecation")
@@ -221,7 +221,7 @@ void testEqualsCommentStart_Deprecated() {
final CSVFormat right = CSVFormat.newFormat('\'').withQuote('"').withCommentMarker('#').withQuoteMode(QuoteMode.ALL);
final CSVFormat left = right.withCommentMarker('!');
- assertNotEquals(right, left);
+ assertNotEqualsFlip(right, left);
}
@Test
@@ -229,7 +229,7 @@ void testEqualsDelimiter() {
final CSVFormat right = CSVFormat.newFormat('!');
final CSVFormat left = CSVFormat.newFormat('?');
- assertNotEquals(right, left);
+ assertNotEqualsFlip(right, left);
}
@Test
@@ -237,7 +237,7 @@ void testEqualsEscape() {
final CSVFormat right = CSVFormat.newFormat('\'').builder().setQuote('"').setCommentMarker('#').setEscape('+').setQuoteMode(QuoteMode.ALL).get();
final CSVFormat left = right.builder().setEscape('!').get();
- assertNotEquals(right, left);
+ assertNotEqualsFlip(right, left);
}
@SuppressWarnings("deprecation")
@@ -246,7 +246,7 @@ void testEqualsEscape_Deprecated() {
final CSVFormat right = CSVFormat.newFormat('\'').withQuote('"').withCommentMarker('#').withEscape('+').withQuoteMode(QuoteMode.ALL);
final CSVFormat left = right.withEscape('!');
- assertNotEquals(right, left);
+ assertNotEqualsFlip(right, left);
}
@Test
@@ -262,49 +262,49 @@ void testEqualsHash() throws Exception {
case "boolean": {
final Object defTrue = method.invoke(CSVFormat.DEFAULT, Boolean.TRUE);
final Object defFalse = method.invoke(CSVFormat.DEFAULT, Boolean.FALSE);
- assertNotEquals(name, type, defTrue, defFalse);
+ assertNotEqualsHash(name, type, defTrue, defFalse);
break;
}
case "char": {
final Object a = method.invoke(CSVFormat.DEFAULT, 'a');
final Object b = method.invoke(CSVFormat.DEFAULT, 'b');
- assertNotEquals(name, type, a, b);
+ assertNotEqualsHash(name, type, a, b);
break;
}
case "java.lang.Character": {
final Object a = method.invoke(CSVFormat.DEFAULT, new Object[] { null });
final Object b = method.invoke(CSVFormat.DEFAULT, Character.valueOf('d'));
- assertNotEquals(name, type, a, b);
+ assertNotEqualsHash(name, type, a, b);
break;
}
case "java.lang.String": {
final Object a = method.invoke(CSVFormat.DEFAULT, new Object[] { null });
final Object b = method.invoke(CSVFormat.DEFAULT, "e");
- assertNotEquals(name, type, a, b);
+ assertNotEqualsHash(name, type, a, b);
break;
}
case "java.lang.String[]": {
final Object a = method.invoke(CSVFormat.DEFAULT, new Object[] { new String[] { null, null } });
final Object b = method.invoke(CSVFormat.DEFAULT, new Object[] { new String[] { "f", "g" } });
- assertNotEquals(name, type, a, b);
+ assertNotEqualsHash(name, type, a, b);
break;
}
case "org.apache.commons.csv.QuoteMode": {
final Object a = method.invoke(CSVFormat.DEFAULT, QuoteMode.MINIMAL);
final Object b = method.invoke(CSVFormat.DEFAULT, QuoteMode.ALL);
- assertNotEquals(name, type, a, b);
+ assertNotEqualsHash(name, type, a, b);
break;
}
case "org.apache.commons.csv.DuplicateHeaderMode": {
final Object a = method.invoke(CSVFormat.DEFAULT, DuplicateHeaderMode.ALLOW_ALL);
final Object b = method.invoke(CSVFormat.DEFAULT, DuplicateHeaderMode.DISALLOW);
- assertNotEquals(name, type, a, b);
+ assertNotEqualsHash(name, type, a, b);
break;
}
case "java.lang.Object[]": {
final Object a = method.invoke(CSVFormat.DEFAULT, new Object[] { new Object[] { null, null } });
final Object b = method.invoke(CSVFormat.DEFAULT, new Object[] { new Object[] { new Object(), new Object() } });
- assertNotEquals(name, type, a, b);
+ assertNotEqualsHash(name, type, a, b);
break;
}
default:
@@ -327,7 +327,7 @@ void testEqualsHeader() {
.setIgnoreEmptyLines(true).setIgnoreSurroundingSpaces(true).setQuote('"').setQuoteMode(QuoteMode.ALL).get();
final CSVFormat left = right.builder().setHeader("Three", "Two", "One").get();
- assertNotEquals(right, left);
+ assertNotEqualsFlip(right, left);
}
@SuppressWarnings("deprecation")
@@ -337,7 +337,7 @@ void testEqualsHeader_Deprecated() {
.withIgnoreEmptyLines().withIgnoreSurroundingSpaces().withQuote('"').withQuoteMode(QuoteMode.ALL);
final CSVFormat left = right.withHeader("Three", "Two", "One");
- assertNotEquals(right, left);
+ assertNotEqualsFlip(right, left);
}
@Test
@@ -346,7 +346,7 @@ void testEqualsIgnoreEmptyLines() {
.setIgnoreSurroundingSpaces(true).setQuote('"').setQuoteMode(QuoteMode.ALL).get();
final CSVFormat left = right.builder().setIgnoreEmptyLines(false).get();
- assertNotEquals(right, left);
+ assertNotEqualsFlip(right, left);
}
@SuppressWarnings("deprecation")
@@ -356,7 +356,7 @@ void testEqualsIgnoreEmptyLines_Deprecated() {
.withQuote('"').withQuoteMode(QuoteMode.ALL);
final CSVFormat left = right.withIgnoreEmptyLines(false);
- assertNotEquals(right, left);
+ assertNotEqualsFlip(right, left);
}
@Test
@@ -365,7 +365,7 @@ void testEqualsIgnoreSurroundingSpaces() {
.setQuoteMode(QuoteMode.ALL).get();
final CSVFormat left = right.builder().setIgnoreSurroundingSpaces(false).get();
- assertNotEquals(right, left);
+ assertNotEqualsFlip(right, left);
}
@SuppressWarnings("deprecation")
@@ -375,7 +375,7 @@ void testEqualsIgnoreSurroundingSpaces_Deprecated() {
.withQuoteMode(QuoteMode.ALL);
final CSVFormat left = right.withIgnoreSurroundingSpaces(false);
- assertNotEquals(right, left);
+ assertNotEqualsFlip(right, left);
}
@Test
@@ -383,7 +383,7 @@ void testEqualsLeftNoQuoteRightQuote() {
final CSVFormat left = CSVFormat.newFormat(',').builder().setQuote(null).get();
final CSVFormat right = left.builder().setQuote('#').get();
- assertNotEquals(left, right);
+ assertNotEqualsFlip(left, right);
}
@SuppressWarnings("deprecation")
@@ -392,7 +392,15 @@ void testEqualsLeftNoQuoteRightQuote_Deprecated() {
final CSVFormat left = CSVFormat.newFormat(',').withQuote(null);
final CSVFormat right = left.withQuote('#');
- assertNotEquals(left, right);
+ assertNotEqualsFlip(left, right);
+ }
+
+ @Test
+ void testEqualsMaxRows() {
+ final CSVFormat right = CSVFormat.DEFAULT.builder().setMaxRows(10).get();
+ final CSVFormat left = CSVFormat.DEFAULT.builder().setMaxRows(1000).get();
+ assertNotEqualsFlip(right, left);
+ assertNotEquals(right.hashCode(), left.hashCode());
}
@Test
@@ -418,7 +426,7 @@ void testEqualsNullString() {
.setIgnoreSurroundingSpaces(true).setQuote('"').setQuoteMode(QuoteMode.ALL).setNullString("null").get();
final CSVFormat left = right.builder().setNullString("---").get();
- assertNotEquals(right, left);
+ assertNotEqualsFlip(right, left);
}
@SuppressWarnings("deprecation")
@@ -428,7 +436,7 @@ void testEqualsNullString_Deprecated() {
.withIgnoreSurroundingSpaces().withQuote('"').withQuoteMode(QuoteMode.ALL).withNullString("null");
final CSVFormat left = right.withNullString("---");
- assertNotEquals(right, left);
+ assertNotEqualsFlip(right, left);
}
@Test
@@ -495,7 +503,7 @@ void testEqualsOne() {
assertFalse(csvFormatTwo.isCommentMarkerSet());
assertNotSame(csvFormatTwo, csvFormatOne);
- Assertions.assertNotEquals(csvFormatTwo, csvFormatOne);
+ assertNotEquals(csvFormatTwo, csvFormatOne);
assertEquals('\\', (char) csvFormatOne.getEscapeCharacter());
assertNull(csvFormatOne.getQuoteMode());
@@ -554,10 +562,10 @@ void testEqualsOne() {
assertNotSame(csvFormatOne, csvFormatTwo);
assertNotSame(csvFormatTwo, csvFormatOne);
- Assertions.assertNotEquals(csvFormatOne, csvFormatTwo);
- Assertions.assertNotEquals(csvFormatTwo, csvFormatOne);
+ assertNotEquals(csvFormatOne, csvFormatTwo);
+ assertNotEquals(csvFormatTwo, csvFormatOne);
- Assertions.assertNotEquals(csvFormatTwo, csvFormatOne);
+ assertNotEquals(csvFormatTwo, csvFormatOne);
}
@@ -566,7 +574,7 @@ void testEqualsQuoteChar() {
final CSVFormat right = CSVFormat.newFormat('\'').builder().setQuote('"').get();
final CSVFormat left = right.builder().setQuote('!').get();
- assertNotEquals(right, left);
+ assertNotEqualsFlip(right, left);
}
@SuppressWarnings("deprecation")
@@ -575,7 +583,7 @@ void testEqualsQuoteChar_Deprecated() {
final CSVFormat right = CSVFormat.newFormat('\'').withQuote('"');
final CSVFormat left = right.withQuote('!');
- assertNotEquals(right, left);
+ assertNotEqualsFlip(right, left);
}
@Test
@@ -583,7 +591,7 @@ void testEqualsQuotePolicy() {
final CSVFormat right = CSVFormat.newFormat('\'').builder().setQuote('"').setQuoteMode(QuoteMode.ALL).get();
final CSVFormat left = right.builder().setQuoteMode(QuoteMode.MINIMAL).get();
- assertNotEquals(right, left);
+ assertNotEqualsFlip(right, left);
}
@SuppressWarnings("deprecation")
@@ -592,7 +600,7 @@ void testEqualsQuotePolicy_Deprecated() {
final CSVFormat right = CSVFormat.newFormat('\'').withQuote('"').withQuoteMode(QuoteMode.ALL);
final CSVFormat left = right.withQuoteMode(QuoteMode.MINIMAL);
- assertNotEquals(right, left);
+ assertNotEqualsFlip(right, left);
}
@Test
@@ -601,7 +609,7 @@ void testEqualsRecordSeparator() {
.setIgnoreSurroundingSpaces(true).setQuote('"').setQuoteMode(QuoteMode.ALL).get();
final CSVFormat left = right.builder().setRecordSeparator(LF).get();
- assertNotEquals(right, left);
+ assertNotEqualsFlip(right, left);
}
@SuppressWarnings("deprecation")
@@ -611,7 +619,7 @@ void testEqualsRecordSeparator_Deprecated() {
.withIgnoreSurroundingSpaces().withQuote('"').withQuoteMode(QuoteMode.ALL);
final CSVFormat left = right.withRecordSeparator(LF);
- assertNotEquals(right, left);
+ assertNotEqualsFlip(right, left);
}
void testEqualsSkipHeaderRecord() {
@@ -619,7 +627,7 @@ void testEqualsSkipHeaderRecord() {
.setIgnoreSurroundingSpaces(true).setQuote('"').setQuoteMode(QuoteMode.ALL).setNullString("null").setSkipHeaderRecord(true).get();
final CSVFormat left = right.builder().setSkipHeaderRecord(false).get();
- assertNotEquals(right, left);
+ assertNotEqualsFlip(right, left);
}
@SuppressWarnings("deprecation")
@@ -629,7 +637,7 @@ void testEqualsSkipHeaderRecord_Deprecated() {
.withIgnoreSurroundingSpaces().withQuote('"').withQuoteMode(QuoteMode.ALL).withNullString("null").withSkipHeaderRecord();
final CSVFormat left = right.withSkipHeaderRecord(false);
- assertNotEquals(right, left);
+ assertNotEqualsFlip(right, left);
}
@Test
@@ -691,7 +699,7 @@ void testEqualsWithNull() {
assertNull(csvFormat.getQuoteCharacter());
assertTrue(csvFormat.isNullStringSet());
- Assertions.assertNotEquals(null, csvFormat);
+ assertNotEquals(null, csvFormat);
}
@@ -801,7 +809,7 @@ void testHashCodeAndWithIgnoreHeaderCase() {
assertTrue(csvFormatTwo.getIgnoreHeaderCase()); // now different
assertFalse(csvFormatTwo.getTrailingDelimiter());
- Assertions.assertNotEquals(csvFormatTwo, csvFormat); // CSV-244 - should not be equal
+ assertNotEquals(csvFormatTwo, csvFormat); // CSV-244 - should not be equal
assertFalse(csvFormatTwo.getAllowMissingColumnNames());
assertFalse(csvFormatTwo.getTrim());
@@ -958,6 +966,23 @@ void testPrintWithQuotes() throws IOException {
assertEquals("\"\"\"a,b,c\r\nx,y,z\"", out.toString());
}
+ /**
+ * Tests CSV-326.
+ */
+ @Test
+ void testPrintWithQuotesEscapeBeforeQuote() throws IOException {
+ final CSVFormat format = CSVFormat.DEFAULT.builder()
+ .setEscape('\\')
+ .setQuote('"')
+ .get();
+ final String value = "\\\"";
+ final Appendable out = new StringBuilder();
+ format.print(new StringReader(value), out, true);
+ try (CSVParser parser = CSVParser.parse(out.toString(), format)) {
+ assertEquals(value, parser.getRecords().get(0).get(0));
+ }
+ }
+
@Test
void testQuoteCharSameAsCommentStartThrowsException() {
assertThrows(IllegalArgumentException.class, () -> CSVFormat.DEFAULT.builder().setQuote('!').setCommentMarker('!').get());
@@ -993,6 +1018,35 @@ void testQuoteCharSameAsDelimiterThrowsException_Deprecated() {
assertThrows(IllegalArgumentException.class, () -> CSVFormat.DEFAULT.withQuote('!').withDelimiter('!'));
}
+ @Test
+ void testQuotedNullStringTracksQuoteCharacter() throws IOException {
+ final StringBuilder out = new StringBuilder();
+ // @formatter:off
+ final Builder builder = CSVFormat.DEFAULT.builder();
+ final CSVFormat format = builder
+ .setQuoteMode(QuoteMode.ALL)
+ .setNullString("NULL")
+ .get();
+ // @formatter:on
+ format.print(null, out, true);
+ assertEquals("\"NULL\"", out.toString());
+ // set
+ out.setLength(0);
+ builder.setQuote('\'');
+ builder.get().print(null, out, true);
+ assertEquals("'NULL'", out.toString());
+ // reset
+ out.setLength(0);
+ builder.setQuote((Character) null);
+ builder.get().print(null, out, true);
+ assertEquals("\"NULL\"", out.toString());
+ // reset, reverse setter order
+ out.setLength(0);
+ builder.setNullString(null).setQuote((Character) null).setNullString("NULL");
+ builder.get().print(null, out, true);
+ assertEquals("\"NULL\"", out.toString());
+ }
+
@Test
void testQuoteModeNoneShouldReturnMeaningfulExceptionMessage() {
final Exception exception = assertThrows(IllegalArgumentException.class, () ->
@@ -1156,7 +1210,7 @@ void testToStringAndWithCommentMarkerTakingCharacter() {
assertNotSame(csvFormat, csvFormatTwo);
assertNotSame(csvFormatTwo, csvFormat);
- Assertions.assertNotEquals(csvFormatTwo, csvFormat);
+ assertNotEquals(csvFormatTwo, csvFormat);
assertNull(csvFormat.getEscapeCharacter());
assertTrue(csvFormat.isQuoteCharacterSet());
@@ -1215,9 +1269,9 @@ void testToStringAndWithCommentMarkerTakingCharacter() {
assertNotSame(csvFormat, csvFormatTwo);
assertNotSame(csvFormatTwo, csvFormat);
- Assertions.assertNotEquals(csvFormat, csvFormatTwo);
+ assertNotEquals(csvFormat, csvFormatTwo);
- Assertions.assertNotEquals(csvFormatTwo, csvFormat);
+ assertNotEquals(csvFormatTwo, csvFormat);
assertEquals("Delimiter=<,> QuoteChar=<\"> CommentStart= RecordSeparator=<\r\n> EmptyLines:ignored SkipHeaderRecord:false",
csvFormatTwo.toString());
@@ -1403,7 +1457,7 @@ void testWithHeaderComments() {
assertNotSame(csvFormat, csvFormatTwo);
assertNotSame(csvFormatTwo, csvFormat);
- Assertions.assertNotEquals(csvFormatTwo, csvFormat); // CSV-244 - should not be equal
+ assertNotEquals(csvFormatTwo, csvFormat); // CSV-244 - should not be equal
final String string = csvFormatTwo.format(objectArray);
@@ -1465,9 +1519,9 @@ void testWithHeaderComments() {
assertNotSame(csvFormatTwo, csvFormat);
assertNotNull(string);
- Assertions.assertNotEquals(csvFormat, csvFormatTwo); // CSV-244 - should not be equal
+ assertNotEquals(csvFormat, csvFormatTwo); // CSV-244 - should not be equal
- Assertions.assertNotEquals(csvFormatTwo, csvFormat); // CSV-244 - should not be equal
+ assertNotEquals(csvFormatTwo, csvFormat); // CSV-244 - should not be equal
assertEquals(",,,,,,,", string);
}
diff --git a/src/test/java/org/apache/commons/csv/CSVParserTest.java b/src/test/java/org/apache/commons/csv/CSVParserTest.java
index d9dd4e545..6d9bdd9e8 100644
--- a/src/test/java/org/apache/commons/csv/CSVParserTest.java
+++ b/src/test/java/org/apache/commons/csv/CSVParserTest.java
@@ -465,6 +465,31 @@ void testDuplicateHeadersNotAllowed() {
() -> CSVParser.parse("a,b,a\n1,2,3\nx,y,z", CSVFormat.DEFAULT.withHeader().withAllowDuplicateHeaderNames(false)));
}
+ /**
+ * With {@code ignoreSurroundingSpaces} enabled and a multi-character delimiter whose first character is whitespace,
+ * the empty field at the delimiter boundary must survive. The delimiter look-ahead is consumed while skipping
+ * leading whitespace, so re-evaluating it would drop the empty field and merge the following field's value.
+ */
+ @Test
+ void testEmptyFieldBeforeWhitespacePrefixedMultiCharacterDelimiter() throws IOException {
+ final CSVFormat format = CSVFormat.DEFAULT.builder().setDelimiter(" |").setIgnoreSurroundingSpaces(true).get();
+ try (CSVParser parser = CSVParser.parse(" |a", format)) {
+ final List records = parser.getRecords();
+ assertEquals(1, records.size());
+ assertValuesEquals(new String[] { "", "a" }, records.get(0));
+ }
+ try (CSVParser parser = CSVParser.parse("a | |b", format)) {
+ final List records = parser.getRecords();
+ assertEquals(1, records.size());
+ assertValuesEquals(new String[] { "a", "", "b" }, records.get(0));
+ }
+ try (CSVParser parser = CSVParser.parse("a | |b |", format)) {
+ final List records = parser.getRecords();
+ assertEquals(1, records.size());
+ assertValuesEquals(new String[] { "a", "", "b", "" }, records.get(0));
+ }
+ }
+
@Test
void testEmptyFile() throws Exception {
try (CSVParser parser = CSVParser.parse(Paths.get("src/test/resources/org/apache/commons/csv/empty.txt"), StandardCharsets.UTF_8,
@@ -648,6 +673,100 @@ void testForEach() throws Exception {
}
}
+ @Test
+ void testGetBytePositionMultiCharacterDelimiter() throws IOException {
+ final String code = "aa[|]bb\ncc[|]dd\n";
+ final CSVFormat format = CSVFormat.DEFAULT.builder().setDelimiter("[|]").get();
+ try (CSVParser parser = CSVParser.builder()
+ .setReader(new StringReader(code))
+ .setFormat(format)
+ .setCharset(StandardCharsets.UTF_8)
+ .setTrackBytes(true)
+ .get()) {
+ final Iterator it = parser.iterator();
+ final CSVRecord first = it.next();
+ final CSVRecord second = it.next();
+ assertEquals(0, first.getBytePosition());
+ assertEquals(8, second.getBytePosition());
+ }
+ }
+
+ /**
+ * Tests CSV-329.
+ */
+ @Test
+ void testGetBytePositionMultiCharacterDelimiterWithSupplementaryCharacter() throws IOException {
+ final String delimiter = "xπ";
+ final String code = "axπb\ncxπd\n";
+ final CSVFormat format = CSVFormat.DEFAULT.builder().setDelimiter(delimiter).get();
+ try (CSVParser parser = CSVParser.builder()
+ .setReader(new StringReader(code))
+ .setFormat(format)
+ .setCharset(UTF_8)
+ .setTrackBytes(true)
+ .get()) {
+ final CSVRecord first = parser.nextRecord();
+ final CSVRecord second = parser.nextRecord();
+ assertNotNull(first);
+ assertNotNull(second);
+ assertValuesEquals(new String[] { "a", "b" }, first);
+ assertValuesEquals(new String[] { "c", "d" }, second);
+ assertEquals(0, first.getBytePosition());
+ assertEquals("axπb\n".getBytes(UTF_8).length, second.getBytePosition());
+ }
+ }
+
+ @Test
+ void testGetBytePositionWithCharacterOffsetAndMultiBytePrefix() throws Exception {
+ final String row0 = "Γ©,x\n";
+ final Charset charset = UTF_8;
+ // row0 char count is 4
+ assertEquals(4, row0.length());
+ // row0 byte count is 5
+ final int record1ByteOffset = row0.getBytes(charset).length;
+ assertEquals(5, record1ByteOffset);
+ final String row1 = "b,c\n";
+ final String rows = row0 + row1;
+ final long record1CharOffset = row0.length();
+ final long expectedByteOffset = row0.getBytes(charset).length;
+ try (CSVParser parser = CSVParser.builder()
+ .setReader(new StringReader(row1))
+ .setFormat(CSVFormat.DEFAULT)
+ .setCharset(charset)
+ .setTrackBytes(true)
+ .setByteOffset(record1ByteOffset)
+ .setCharacterOffset(record1CharOffset)
+ .setRecordNumber(2) // not relevant but a better use case example.
+ .get()) {
+ final CSVRecord record = parser.nextRecord();
+ assertNotNull(record);
+ assertEquals(4, record.getCharacterPosition());
+ assertEquals(record1CharOffset, record.getCharacterPosition());
+ assertEquals(expectedByteOffset, record.getBytePosition());
+ }
+ }
+
+ @Test
+ void testGetBytePositionWithSingleByteCharset() throws IOException {
+ // A single-byte charset cannot encode U+FFFF, the char value of the EOF sentinel.
+ // Byte counting must skip the EOF read so a valid file parses without throwing.
+ final String code = "a,b\nc,d\n";
+ try (CSVParser parser = CSVParser.builder()
+ .setReader(new StringReader(code))
+ .setFormat(CSVFormat.DEFAULT)
+ .setCharset(StandardCharsets.ISO_8859_1)
+ .setTrackBytes(true)
+ .get()) {
+ final CSVRecord first = parser.nextRecord();
+ final CSVRecord second = parser.nextRecord();
+ assertNotNull(first);
+ assertNotNull(second);
+ assertNull(parser.nextRecord());
+ assertEquals(0, first.getBytePosition());
+ assertEquals(4, second.getBytePosition());
+ }
+ }
+
@Test
void testGetHeaderComment_HeaderComment1() throws IOException {
try (CSVParser parser = CSVParser.parse(CSV_INPUT_HEADER_COMMENT, FORMAT_AUTO_HEADER)) {
@@ -917,6 +1036,23 @@ void testGetRecordsMaxRows(final long maxRows) throws IOException {
}
}
+ /**
+ * Tests CSV-327.
+ */
+ @Test
+ void testGetRecordsMaxRowsWithRecordNumberOffset() throws IOException {
+ try (CSVParser parser = CSVParser.builder()
+ .setReader(new StringReader("a,b\nc,d\n"))
+ .setFormat(CSVFormat.DEFAULT.builder().setMaxRows(1).get())
+ .setRecordNumber(2)
+ .get()) {
+ final List records = parser.getRecords();
+ assertEquals(1, records.size());
+ assertEquals(2, records.get(0).getRecordNumber());
+ assertValuesEquals(new String[] { "a", "b" }, records.get(0));
+ }
+ }
+
@Test
void testGetRecordThreeBytesRead() throws Exception {
final String code = "id,date,val5,val4\n" +
@@ -1603,6 +1739,50 @@ void testParsingPrintedEmptyFirstColumn(final CSVFormat.Predefined format) throw
}
}
+ /**
+ * A truncated escaped multi-character delimiter at EOF must stay literal data and not be completed from a stale
+ * escape delimiter look-ahead.
+ */
+ @Test
+ void testPartialEscapedMultiCharacterDelimiterAtEOF() throws IOException {
+ final CSVFormat format = CSVFormat.DEFAULT.builder().setDelimiter("[|]").setEscape('!').get();
+ try (CSVParser parser = format.parse(new StringReader("x![!|!]y![!|"))) {
+ final CSVRecord record = parser.nextRecord();
+ assertEquals("x[|]y![!|", record.get(0));
+ assertEquals(1, record.size());
+ }
+ }
+
+ /**
+ * Tests CSV-324.
+ */
+ @Test
+ void testPartialMultiCharacterDelimiterAtEOF() throws IOException {
+ final CSVFormat format = CSVFormat.DEFAULT.builder().setDelimiter("[|]").get();
+ try (CSVParser parser = format.parse(new StringReader("a[|]b[|"))) {
+ final CSVRecord record = parser.nextRecord();
+ assertEquals("a", record.get(0));
+ assertEquals("b[|", record.get(1));
+ assertEquals(2, record.size());
+ }
+ }
+
+ /**
+ * A truncated multi-character delimiter at EOF must not be completed from the look-ahead buffer left dirty by an
+ * earlier non-matching peek in the same token.
+ */
+ @Test
+ void testPartialMultiCharacterDelimiterAtEOFAfterMismatch() throws IOException {
+ final CSVFormat format = CSVFormat.DEFAULT.builder().setDelimiter("[|]").get();
+ // The "[a]" peek leaves ']' in the look-ahead buffer; the trailing "[|" must not match "[|]".
+ final String recordString = "x[a][|";
+ try (CSVParser parser = format.parse(new StringReader(recordString))) {
+ final CSVRecord record = parser.nextRecord();
+ assertEquals(recordString, record.get(0));
+ assertEquals(1, record.size());
+ }
+ }
+
@Test
void testProvidedHeader() throws Exception {
final Reader in = new StringReader("a,b,c\n1,2,3\nx,y,z");
@@ -1794,6 +1974,23 @@ void testTrailingDelimiter() throws Exception {
}
}
+ @Test
+ void testTrailingDelimiterKeepsQuotedEmptyLastField() throws Exception {
+ final CSVFormat format = CSVFormat.DEFAULT.builder().setTrailingDelimiter(true).get();
+ try (CSVParser parser = CSVParser.parse("a,b,\"\"", format)) {
+ final CSVRecord record = parser.iterator().next();
+ assertEquals(3, record.size());
+ assertEquals("a", record.get(0));
+ assertEquals("b", record.get(1));
+ assertEquals("", record.get(2));
+ }
+ // An unquoted trailing delimiter still drops the empty field.
+ try (CSVParser parser = CSVParser.parse("a,b,", format)) {
+ final CSVRecord record = parser.iterator().next();
+ assertEquals(2, record.size());
+ }
+ }
+
@Test
void testTrim() throws Exception {
final Reader in = new StringReader("a,a,a\n\" 1 \",\" 2 \",\" 3 \"\nx,y,z");
diff --git a/src/test/java/org/apache/commons/csv/CSVPrinterTest.java b/src/test/java/org/apache/commons/csv/CSVPrinterTest.java
index 1ff791010..9ae80c1e5 100644
--- a/src/test/java/org/apache/commons/csv/CSVPrinterTest.java
+++ b/src/test/java/org/apache/commons/csv/CSVPrinterTest.java
@@ -569,6 +569,57 @@ void testEscapeBackslash5() throws IOException {
assertEquals("\\\\", sw.toString());
}
+ @Test
+ void testEscapeCommentMarkerFirstChar() throws IOException {
+ // No quoting available in escape mode, so a leading comment marker must be escaped or the
+ // record reads back as a comment and is dropped. Mirrors the quoting fix for QuoteMode.MINIMAL.
+ final CSVFormat format = CSVFormat.DEFAULT.builder().setQuote(null).setEscape('\\').setCommentMarker(';').get();
+ final StringWriter sw = new StringWriter();
+ final String col1 = ";comment-like";
+ try (CSVPrinter printer = new CSVPrinter(sw, format)) {
+ printer.printRecord(col1, "b");
+ printer.printRecord(new StringReader(col1), new StringReader("b"));
+ // The marker past the first character does not start a comment and is left alone.
+ printer.printRecord("a;b", ";c");
+ }
+ final String string = sw.toString();
+ assertEquals("\\;comment-like,b" + RECORD_SEPARATOR +
+ "\\;comment-like,b" + RECORD_SEPARATOR +
+ "a;b,\\;c" + RECORD_SEPARATOR, string);
+ // The emitted records must read back as the original values, none parsed as a comment.
+ try (CSVParser parser = CSVParser.parse(string, format)) {
+ final List records = parser.getRecords();
+ assertEquals(3, records.size());
+ assertEquals(col1, records.get(0).get(0));
+ assertEquals("b", records.get(0).get(1));
+ assertEquals(col1, records.get(1).get(0));
+ assertEquals("b", records.get(1).get(1));
+ assertEquals("a;b", records.get(2).get(0));
+ assertEquals(";c", records.get(2).get(1));
+ }
+ }
+
+ @Test
+ void testEscapeCommentMarkerFirstCharWithQuoteModeNone() throws IOException {
+ final CSVFormat format = CSVFormat.DEFAULT.builder().setEscape('\\').setQuoteMode(QuoteMode.NONE).setCommentMarker(';').get();
+ final StringWriter sw = new StringWriter();
+ final String col1 = ";bar";
+ try (CSVPrinter printer = new CSVPrinter(sw, format)) {
+ printer.printRecord(col1, "b");
+ printer.printRecord(new StringReader(col1), new StringReader("b"));
+ }
+ final String string = sw.toString();
+ assertEquals("\\;bar,b" + RECORD_SEPARATOR + "\\;bar,b" + RECORD_SEPARATOR, string);
+ try (CSVParser parser = CSVParser.parse(string, format)) {
+ final List records = parser.getRecords();
+ assertEquals(2, records.size());
+ for (final CSVRecord record : records) {
+ assertEquals(col1, record.get(0));
+ assertEquals("b", record.get(1));
+ }
+ }
+ }
+
@Test
void testEscapeNull1() throws IOException {
final StringWriter sw = new StringWriter();
@@ -1798,6 +1849,28 @@ void testQuoteAll() throws IOException {
}
}
+ @Test
+ void testQuoteCharEscapedWithQuoteModeNone() throws IOException {
+ final CSVFormat format = CSVFormat.DEFAULT.builder().setQuote('"').setEscape('?').setQuoteMode(QuoteMode.NONE).get();
+ final StringWriter sw = new StringWriter();
+ final String col1 = "\"abc";
+ final String col2 = "x\"y";
+ try (CSVPrinter printer = new CSVPrinter(sw, format)) {
+ printer.printRecord(col1, col2);
+ printer.printRecord(new StringReader(col1), new StringReader(col2));
+ }
+ assertEquals("?\"abc,x?\"y" + RECORD_SEPARATOR + "?\"abc,x?\"y" + RECORD_SEPARATOR, sw.toString());
+ // The emitted records must read back as the original values.
+ try (CSVParser parser = CSVParser.parse(sw.toString(), format)) {
+ final List records = parser.getRecords();
+ assertEquals(2, records.size());
+ for (final CSVRecord record : records) {
+ assertEquals(col1, record.get(0));
+ assertEquals(col2, record.get(1));
+ }
+ }
+ }
+
@Test
void testQuoteCommaFirstChar() throws IOException {
final StringWriter sw = new StringWriter();
@@ -1807,6 +1880,34 @@ void testQuoteCommaFirstChar() throws IOException {
}
}
+ @Test
+ void testQuoteCommentMarkerFirstChar() throws IOException {
+ final CSVFormat format = CSVFormat.DEFAULT.builder().setCommentMarker(';').get();
+ final StringWriter sw = new StringWriter();
+ final String col1 = ";comment-like";
+ try (CSVPrinter printer = new CSVPrinter(sw, format)) {
+ // A real comment is written with the marker, unquoted.
+ printer.printComment("a real comment");
+ // A value starting with the marker is quoted, so it does not read back as a comment.
+ printer.printRecord(col1, "b");
+ // The marker past the first character does not start a comment, so only the leading-marker value is quoted.
+ printer.printRecord("a;b", ";c");
+ }
+ final String string = sw.toString();
+ assertEquals("; a real comment" + RECORD_SEPARATOR +
+ "\";comment-like\",b" + RECORD_SEPARATOR +
+ "a;b,\";c\"" + RECORD_SEPARATOR, string);
+ // The comment is dropped on read; both data records survive intact.
+ try (CSVParser parser = CSVParser.parse(string, format)) {
+ final List records = parser.getRecords();
+ assertEquals(2, records.size());
+ assertEquals(col1, records.get(0).get(0));
+ assertEquals("b", records.get(0).get(1));
+ assertEquals("a;b", records.get(1).get(0));
+ assertEquals(";c", records.get(1).get(1));
+ }
+ }
+
@Test
void testQuoteNonNumeric() throws IOException {
final StringWriter sw = new StringWriter();
diff --git a/src/test/java/org/apache/commons/csv/ExtendedBufferedReaderTest.java b/src/test/java/org/apache/commons/csv/ExtendedBufferedReaderTest.java
index 056b8a9c9..b8d9b9f19 100644
--- a/src/test/java/org/apache/commons/csv/ExtendedBufferedReaderTest.java
+++ b/src/test/java/org/apache/commons/csv/ExtendedBufferedReaderTest.java
@@ -26,6 +26,7 @@
import static org.junit.jupiter.api.Assertions.assertNull;
import java.io.StringReader;
+import java.nio.charset.StandardCharsets;
import org.junit.jupiter.api.Test;
@@ -104,6 +105,19 @@ void testReadingInDifferentBuffer() throws Exception {
}
}
+ @Test
+ void testReadingSupplementaryCharacterTracksBytes() throws Exception {
+ final String input = "π";
+ final char[] buffer = new char[input.length()];
+ try (ExtendedBufferedReader reader = new ExtendedBufferedReader(new StringReader(input), StandardCharsets.UTF_8, true)) {
+ assertEquals(input.length(), reader.read(buffer, 0, buffer.length));
+ assertArrayEquals(input.toCharArray(), buffer);
+ assertEquals(input.getBytes(StandardCharsets.UTF_8).length, reader.getBytesRead());
+ assertEquals(input.length(), reader.getPosition());
+ assertEquals(input.charAt(input.length() - 1), reader.getLastChar());
+ }
+ }
+
@Test
void testReadLine() throws Exception {
try (ExtendedBufferedReader br = createBufferedReader("")) {
diff --git a/src/test/java/org/apache/commons/csv/LexerTest.java b/src/test/java/org/apache/commons/csv/LexerTest.java
index e54e93365..a76f6e513 100644
--- a/src/test/java/org/apache/commons/csv/LexerTest.java
+++ b/src/test/java/org/apache/commons/csv/LexerTest.java
@@ -216,6 +216,25 @@ void testDelimiterIsWhitespace() throws IOException {
}
}
+ /**
+ * With {@code ignoreSurroundingSpaces} enabled and a multi-character delimiter whose first character is whitespace,
+ * the side-effecting {@link Lexer#isDelimiter(int)} must only be evaluated once per character, otherwise the
+ * delimiter is consumed in the whitespace-skip loop and the empty field at the boundary is dropped.
+ */
+ @Test
+ void testEmptyTokenBeforeWhitespacePrefixedMultiCharacterDelimiter() throws IOException {
+ final CSVFormat format = CSVFormat.DEFAULT.builder().setDelimiter(" |").setIgnoreSurroundingSpaces(true).get();
+ try (Lexer lexer = createLexer(" |a", format)) {
+ assertNextToken(TOKEN, "", lexer);
+ assertNextToken(EOF, "a", lexer);
+ }
+ try (Lexer lexer = createLexer("a | |b", format)) {
+ assertNextToken(TOKEN, "a", lexer);
+ assertNextToken(TOKEN, "", lexer);
+ assertNextToken(EOF, "b", lexer);
+ }
+ }
+
@Test
void testEOFWithoutClosingQuote() throws Exception {
final String code = "a,\"b";
@@ -409,6 +428,44 @@ void testNextToken6() throws IOException {
}
}
+ /**
+ * A truncated escaped multi-character delimiter at EOF must not be accepted by reusing the previous escape delimiter
+ * look-ahead in {@link Lexer#isEscapeDelimiter()}.
+ */
+ @Test
+ void testPartialEscapedMultiCharacterDelimiterAtEOF() throws IOException {
+ final CSVFormat format = CSVFormat.DEFAULT.builder().setDelimiter("[|]").setEscape('!').get();
+ try (Lexer lexer = createLexer("x![!|!]y![!|", format)) {
+ assertNextToken(EOF, "x[|]y![!|", lexer);
+ }
+ }
+
+ /**
+ * Tests CSV-324.
+ */
+ @Test
+ void testPartialMultiCharacterDelimiterAtEOF() throws IOException {
+ final CSVFormat format = CSVFormat.DEFAULT.builder().setDelimiter("[|]").get();
+ try (Lexer lexer = createLexer("a[|]b[|", format)) {
+ assertNextToken(TOKEN, "a", lexer);
+ assertNextToken(EOF, "b[|", lexer);
+ }
+ }
+
+ /**
+ * A truncated multi-character delimiter at EOF must not be accepted by reusing the look-ahead buffer left dirty by an
+ * earlier non-matching peek in the same token (CSV-324 only cleared the buffer once per token).
+ */
+ @Test
+ void testPartialMultiCharacterDelimiterAtEOFAfterMismatch() throws IOException {
+ final CSVFormat format = CSVFormat.DEFAULT.builder().setDelimiter("[|]").get();
+ // The "[a]" peek leaves ']' in the look-ahead buffer; the trailing "[|" must not match "[|]".
+ final String recordString = "x[a][|";
+ try (Lexer lexer = createLexer(recordString, format)) {
+ assertNextToken(EOF, recordString, lexer);
+ }
+ }
+
@Test
void testReadEscapeBackspace() throws IOException {
try (Lexer lexer = createLexer("b", CSVFormat.DEFAULT.withEscape('\b'))) {
diff --git a/src/test/java/org/apache/commons/csv/issues/JiraCsv227Test.java b/src/test/java/org/apache/commons/csv/issues/JiraCsv227Test.java
new file mode 100644
index 000000000..2b9e335a8
--- /dev/null
+++ b/src/test/java/org/apache/commons/csv/issues/JiraCsv227Test.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.commons.csv.issues;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.io.IOException;
+
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVPrinter;
+import org.apache.commons.csv.QuoteMode;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests https://issues.apache.org/jira/browse/CSV-227
+ */
+class JiraCsv227Test {
+
+ @Test
+ public void test() throws IOException {
+ final StringBuilder out = new StringBuilder();
+ try (CSVPrinter printer = new CSVPrinter(out, CSVFormat.DEFAULT.withQuoteMode(QuoteMode.MINIMAL))) {
+ printer.printRecord("γ
γ
γ·γΉ", "γ
γ
γ·γΉ", "", "test2");
+ printer.printRecord("νκΈ3", "hello3", "3νκΈ3", "test3");
+ printer.printRecord("", "hello4", "", "test4");
+ }
+ // γ
γ
γ·γΉ,γ
γ
γ·γΉ,,test2
+ // νκΈ3,hello3,3νκΈ3,test3
+ // "",hello4,,test4
+ assertEquals("γ
γ
γ·γΉ,γ
γ
γ·γΉ,,test2\r\nνκΈ3,hello3,3νκΈ3,test3\r\n\"\",hello4,,test4\r\n", out.toString());
+ }
+}