Improve lexer and token coverage (apache#67)

* Improve Lexer and Token coverage

* fix the imports

Author: dota17

src/test/java/org/apache/commons/csv/LexerTest.java

@@ -32,7 +32,7 @@
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
-
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import java.io.IOException;
 import java.io.StringReader;
 
@@ -389,4 +389,45 @@ public void testEscapingAtEOF() throws Exception {
             assertThrows(IOException.class, () -> lexer.nextToken(new Token()));
         }
     }
+
+    @Test
+    public void testTrimTrailingSpacesZeroLength() throws Exception {
+        final StringBuilder buffer = new StringBuilder("");
+        final Lexer lexer = createLexer(buffer.toString(), CSVFormat.DEFAULT);
+        lexer.trimTrailingSpaces(buffer);
+        assertThat(lexer.nextToken(new Token()), matches(EOF, ""));
+    }
+
+    @Test
+    public void testReadEscapeTab() throws IOException {
+        try (final Lexer lexer = createLexer("t", CSVFormat.DEFAULT.withEscape('\t'))) {
+            final int ch = lexer.readEscape();
+            assertThat(lexer.nextToken(new Token()), matches(EOF, ""));
+            assertEquals(TAB, ch);
+        }
+    }
+
+    @Test
+    public void testReadEscapeBackspace() throws IOException {
+        try (final Lexer lexer = createLexer("b", CSVFormat.DEFAULT.withEscape('\b'))) {
+            final int ch = lexer.readEscape();
+            assertEquals(BACKSPACE, ch);
+        }
+    }
+
+    @Test
+    public void testReadEscapeFF() throws IOException {
+        try (final Lexer lexer = createLexer("f", CSVFormat.DEFAULT.withEscape('\f'))) {
+            final int ch = lexer.readEscape();
+            assertEquals(FF, ch);
+        }
+    }
+
+    @Test
+    public void testIsMetaCharCommentStart() throws IOException {
+        try (final Lexer lexer = createLexer("#", CSVFormat.DEFAULT.withCommentMarker('#'))) {
+            final int ch = lexer.readEscape();
+            assertEquals('#', ch);
+        }
+    }
 }

src/test/java/org/apache/commons/csv/TokenMatchersTest.java

@@ -22,6 +22,7 @@
 import static org.apache.commons.csv.TokenMatchers.matches;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -67,4 +68,10 @@ public void testMatches() {
         assertFalse(matches(Token.Type.EORECORD, "not the content").matches(token));
     }
 
+    @Test
+    public void testToString() {
+        assertTrue(matches(Token.Type.TOKEN, "content").matches(token));
+        assertEquals("TOKEN", token.type.name());
+        assertEquals("TOKEN [content]", token.toString());
+    }
 }