Merge pull request apache#603 from garydgregory/fix/CSV-324_clean_delim_buffer

Lexer.isDelimiter() accepts a partial multi-character delimiter at EOF.

Author: garydgregory

src/main/java/org/apache/commons/csv/Lexer.java

@@ -23,6 +23,7 @@
 
 import java.io.Closeable;
 import java.io.IOException;
+import java.util.Arrays;
 
 import org.apache.commons.io.IOUtils;
 
@@ -272,6 +273,7 @@ Token nextToken(final Token token) throws IOException {
             token.type = Token.Type.COMMENT;
             return token;
         }
+        Arrays.fill(delimiterBuf, '\0');
         // Important: make sure a new char gets consumed in each iteration
         while (token.type == Token.Type.INVALID) {
             // ignore whitespaces at beginning of a token

src/test/java/org/apache/commons/csv/CSVParserTest.java

@@ -1621,6 +1621,20 @@ void testParsingPrintedEmptyFirstColumn(final CSVFormat.Predefined format) throw
         }
     }
 
+    /**
+     * Tests <a href="https://issues.apache.org/jira/browse/CSV-324">CSV-324</a>.
+     */
+    @Test
+    void testPartialMultiCharacterDelimiterAtEOF() throws IOException {
+        final CSVFormat format = CSVFormat.DEFAULT.builder().setDelimiter("[|]").get();
+        try (CSVParser parser = format.parse(new StringReader("a[|]b[|"))) {
+            final CSVRecord record = parser.nextRecord();
+            assertEquals("a", record.get(0));
+            assertEquals("b[|", record.get(1));
+            assertEquals(2, record.size());
+        }
+    }
+
     @Test
     void testProvidedHeader() throws Exception {
         final Reader in = new StringReader("a,b,c\n1,2,3\nx,y,z");

src/test/java/org/apache/commons/csv/LexerTest.java

@@ -409,6 +409,18 @@ void testNextToken6() throws IOException {
         }
     }
 
+    /**
+     * Tests <a href="https://issues.apache.org/jira/browse/CSV-324">CSV-324</a>.
+     */
+    @Test
+    void testPartialMultiCharacterDelimiterAtEOF() throws IOException {
+        final CSVFormat format = CSVFormat.DEFAULT.builder().setDelimiter("[|]").get();
+        try (Lexer lexer = createLexer("a[|]b[|", format)) {
+            assertNextToken(TOKEN, "a", lexer);
+            assertNextToken(EOF, "b[|", lexer);
+        }
+    }
+
     @Test
     void testReadEscapeBackspace() throws IOException {
         try (Lexer lexer = createLexer("b", CSVFormat.DEFAULT.withEscape('\b'))) {