Separate testing empty line handling from comment recognition

git-svn-id: https://svn.apache.org/repos/asf/commons/proper/csv/trunk@1465722 13f79535-47bb-0310-9956-ffa450edef68

Author: britter

src/test/java/org/apache/commons/csv/CSVLexerTest.java

@@ -73,41 +73,60 @@ public void testIgnoreSurroundingSpacesAreDeleted() throws IOException {
     @Test
     public void testIgnoreEmptyLines() throws IOException {
         final String code =
-                "1,2,3,\n"+                // 1
+                "first,line,\n"+
                 "\n"+
                 "\n"+
-                "a,b x,c#no-comment\n"+    // 2
+                "second,line\n"+
                 "\n"+
                 "\n"+
-                "# foo \n"+                // 3
-                "\n"+                      // 4
-                "d,e,#no-comment\n"+       // 5
+                "third line \n"+
                 "\n"+
                 "\n"+
-                "# penultimate comment\n"+ // 6
+                "last, line \n"+
                 "\n"+
                 "\n"+
-                "# Final comment\n";       // 7
-        final CSVFormat format = CSVFormat.newBuilder().withCommentStart('#').withIgnoreEmptyLines(true).build();
+                "\n";
+        final CSVFormat format = CSVFormat.newBuilder().withIgnoreEmptyLines(true).build();
         final Lexer parser = getLexer(code, format);
 
-        assertThat(parser.nextToken(new Token()), matches(TOKEN, "1"));
-        assertThat(parser.nextToken(new Token()), matches(TOKEN, "2"));
-        assertThat(parser.nextToken(new Token()), matches(TOKEN, "3"));
-        assertThat(parser.nextToken(new Token()), matches(EORECORD, ""));             // 1
-        assertThat(parser.nextToken(new Token()), matches(TOKEN, "a"));
-        assertThat(parser.nextToken(new Token()), matches(TOKEN, "b x"));
-        assertThat(parser.nextToken(new Token()), matches(EORECORD, "c#no-comment")); // 2
-        assertThat(parser.nextToken(new Token()), matches(COMMENT, "foo"));              // 3
-        // 4 empty line, ignored                                                    // 4
-        assertThat(parser.nextToken(new Token()), matches(TOKEN, "d"));
-        assertThat(parser.nextToken(new Token()), matches(TOKEN, "e"));
-        assertThat(parser.nextToken(new Token()), matches(EORECORD, "#no-comment"));  // 5
-        assertThat(parser.nextToken(new Token()), matches(COMMENT, "penultimate comment"));              // 6
-        assertThat(parser.nextToken(new Token()), matches(COMMENT, "Final comment"));              // 7
+        assertThat(parser.nextToken(new Token()), matches(TOKEN, "first"));
+        assertThat(parser.nextToken(new Token()), matches(TOKEN, "line"));
+        assertThat(parser.nextToken(new Token()), matches(EORECORD, ""));
+        assertThat(parser.nextToken(new Token()), matches(TOKEN, "second"));
+        assertThat(parser.nextToken(new Token()), matches(EORECORD, "line"));
+        assertThat(parser.nextToken(new Token()), matches(EORECORD, "third line "));
+        assertThat(parser.nextToken(new Token()), matches(TOKEN, "last"));
+        assertThat(parser.nextToken(new Token()), matches(EORECORD, " line "));
         assertThat(parser.nextToken(new Token()), matches(EOF, ""));
         assertThat(parser.nextToken(new Token()), matches(EOF, ""));
+    }
 
+    @Test
+    public void testComments() throws IOException {
+        final String code =
+                "first,line,\n"+
+                "second,line,tokenWith#no-comment\n"+
+                "# comment line \n"+
+                "third,line,#no-comment\n"+
+                "# penultimate comment\n"+
+                "# Final comment\n";
+        final CSVFormat format = CSVFormat.newBuilder().withCommentStart('#').build();
+        final Lexer parser = getLexer(code, format);
+
+        assertThat(parser.nextToken(new Token()), matches(TOKEN, "first"));
+        assertThat(parser.nextToken(new Token()), matches(TOKEN, "line"));
+        assertThat(parser.nextToken(new Token()), matches(EORECORD, ""));
+        assertThat(parser.nextToken(new Token()), matches(TOKEN, "second"));
+        assertThat(parser.nextToken(new Token()), matches(TOKEN, "line"));
+        assertThat(parser.nextToken(new Token()), matches(EORECORD, "tokenWith#no-comment"));
+        assertThat(parser.nextToken(new Token()), matches(COMMENT, "comment line"));
+        assertThat(parser.nextToken(new Token()), matches(TOKEN, "third"));
+        assertThat(parser.nextToken(new Token()), matches(TOKEN, "line"));
+        assertThat(parser.nextToken(new Token()), matches(EORECORD, "#no-comment"));
+        assertThat(parser.nextToken(new Token()), matches(COMMENT, "penultimate comment"));
+        assertThat(parser.nextToken(new Token()), matches(COMMENT, "Final comment"));
+        assertThat(parser.nextToken(new Token()), matches(EOF, ""));
+        assertThat(parser.nextToken(new Token()), matches(EOF, ""));
     }
 
     // multiline including comments (and empty lines)