Merge pull request apache#611 from rootvector2/delimiter-buffer-clear

Clear delimiter buffer before each peek in isDelimiter

Author: garydgregory

src/main/java/org/apache/commons/csv/Lexer.java

@@ -153,6 +153,7 @@ boolean isDelimiter(final int ch) throws IOException {
             isLastTokenDelimiter = true;
             return true;
         }
+        Arrays.fill(delimiterBuf, '\0');
         reader.peek(delimiterBuf);
         for (int i = 0; i < delimiterBuf.length; i++) {
             if (delimiterBuf[i] != delimiter[i + 1]) {
@@ -274,7 +275,6 @@ Token nextToken(final Token token) throws IOException {
             token.type = Token.Type.COMMENT;
             return token;
         }
-        Arrays.fill(delimiterBuf, '\0');
         // Important: make sure a new char gets consumed in each iteration
         while (token.type == Token.Type.INVALID) {
             // ignore whitespaces at beginning of a token

src/test/java/org/apache/commons/csv/CSVParserTest.java

@@ -1696,6 +1696,21 @@ void testPartialMultiCharacterDelimiterAtEOF() throws IOException {
         }
     }
 
+    /**
+     * A truncated multi-character delimiter at EOF must not be completed from the look-ahead buffer left dirty by an
+     * earlier non-matching peek in the same token.
+     */
+    @Test
+    void testPartialMultiCharacterDelimiterAtEOFAfterMismatch() throws IOException {
+        final CSVFormat format = CSVFormat.DEFAULT.builder().setDelimiter("[|]").get();
+        // The "[a]" peek leaves ']' in the look-ahead buffer; the trailing "[|" must not match "[|]".
+        try (CSVParser parser = format.parse(new StringReader("x[a][|"))) {
+            final CSVRecord record = parser.nextRecord();
+            assertEquals("x[a][|", record.get(0));
+            assertEquals(1, record.size());
+        }
+    }
+
     @Test
     void testProvidedHeader() throws Exception {
         final Reader in = new StringReader("a,b,c\n1,2,3\nx,y,z");

src/test/java/org/apache/commons/csv/LexerTest.java

@@ -433,6 +433,19 @@ void testPartialMultiCharacterDelimiterAtEOF() throws IOException {
         }
     }
 
+    /**
+     * A truncated multi-character delimiter at EOF must not be accepted by reusing the look-ahead buffer left dirty by an
+     * earlier non-matching peek in the same token (CSV-324 only cleared the buffer once per token).
+     */
+    @Test
+    void testPartialMultiCharacterDelimiterAtEOFAfterMismatch() throws IOException {
+        final CSVFormat format = CSVFormat.DEFAULT.builder().setDelimiter("[|]").get();
+        // The "[a]" peek leaves ']' in the look-ahead buffer; the trailing "[|" must not match "[|]".
+        try (Lexer lexer = createLexer("x[a][|", format)) {
+            assertNextToken(EOF, "x[a][|", lexer);
+        }
+    }
+
     @Test
     void testReadEscapeBackspace() throws IOException {
         try (Lexer lexer = createLexer("b", CSVFormat.DEFAULT.withEscape('\b'))) {