Test with empty lines

git-svn-id: https://svn.apache.org/repos/asf/commons/proper/csv/trunk@1306651 13f79535-47bb-0310-9956-ffa450edef68

Author: sebbASF

src/test/java/org/apache/commons/csv/CSVLexerTest.java

@@ -89,6 +89,42 @@ public void testNextToken2() throws IOException {
 
     }
 
+    // multiline including comments (and empty lines)
+    @Test
+    public void testNextToken2EmptyLines() throws IOException {
+        final String code = 
+                "1,2,3,\n"+                // 1
+                "a,b x,c#no-comment\n"+    // 2
+                "#foo\n"+                  // 3
+                "\n"+                      // 4
+                "d,e,#no-comment\n"+       // 5
+                "# penultimate comment\n"+ // 6
+                "# Final comment\n";       // 7
+        CSVFormat format = CSVFormat.DEFAULT.withCommentStart('#').withEmptyLinesIgnored(false);
+        assertFalse("Should not ignore empty lines", format.isEmptyLinesIgnored());
+        
+        Lexer parser = getLexer(code, format);
+
+
+        assertTokenEquals(TOKEN, "1", parser.nextToken(new Token()));
+        assertTokenEquals(TOKEN, "2", parser.nextToken(new Token()));
+        assertTokenEquals(TOKEN, "3", parser.nextToken(new Token()));
+        assertTokenEquals(EORECORD, "", parser.nextToken(new Token()));             // 1
+        assertTokenEquals(TOKEN, "a", parser.nextToken(new Token()));
+        assertTokenEquals(TOKEN, "b x", parser.nextToken(new Token()));
+        assertTokenEquals(EORECORD, "c#no-comment", parser.nextToken(new Token())); // 2
+        assertTokenEquals(COMMENT, "", parser.nextToken(new Token()));              // 3
+        assertTokenEquals(EORECORD, "", parser.nextToken(new Token()));             // 4
+        assertTokenEquals(TOKEN, "d", parser.nextToken(new Token()));
+        assertTokenEquals(TOKEN, "e", parser.nextToken(new Token()));
+        assertTokenEquals(EORECORD, "#no-comment", parser.nextToken(new Token()));  // 5
+        assertTokenEquals(COMMENT, "", parser.nextToken(new Token()));              // 6
+        assertTokenEquals(COMMENT, "", parser.nextToken(new Token()));              // 7
+        assertTokenEquals(EOF, "", parser.nextToken(new Token()));
+        assertTokenEquals(EOF, "", parser.nextToken(new Token()));
+
+    }
+
     // simple token with escaping
     @Test
     public void testNextToken3() throws IOException {