Move Token into separate file for more flexibility

git-svn-id: https://svn.apache.org/repos/asf/commons/proper/csv/trunk@1303505 13f79535-47bb-0310-9956-ffa450edef68

Author: sebbASF

src/main/java/org/apache/commons/csv/CSVLexer.java

@@ -19,58 +19,17 @@
 
 import java.io.IOException;
 
-import static org.apache.commons.csv.CSVLexer.Token.Type.*;
+import static org.apache.commons.csv.Token.Type.*;
 
 class CSVLexer {
 
-    /** length of the initial token (content-)buffer */
-    private static final int INITIAL_TOKEN_LENGTH = 50;
-    
     private final StringBuilder wsBuf = new StringBuilder();
 
     private final CSVFormat format;
 
     /** The input stream */
     private final ExtendedBufferedReader in;
 
-    /**
-     * Token is an internal token representation.
-     * <p/>
-     * It is used as contract between the lexer and the parser.
-     */
-    static class Token {
-
-        enum Type {
-            /** Token has no valid content, i.e. is in its initialized state. */
-            INVALID,
-            
-            /** Token with content, at beginning or in the middle of a line. */
-            TOKEN,
-            
-            /** Token (which can have content) when end of file is reached. */
-            EOF,
-            
-            /** Token with content when end of a line is reached. */
-            EORECORD
-        }
-        
-        /** Token type */
-        Type type = INVALID;
-        
-        /** The content buffer. */
-        StringBuilder content = new StringBuilder(INITIAL_TOKEN_LENGTH);
-        
-        /** Token ready flag: indicates a valid token with content (ready for the parser). */
-        boolean isReady;
-
-        Token reset() {
-            content.setLength(0);
-            type = INVALID;
-            isReady = false;
-            return this;
-        }
-    }
-
     CSVLexer(CSVFormat format, ExtendedBufferedReader in) {
         this.format = format;
         this.in = in;

src/main/java/org/apache/commons/csv/CSVParser.java

@@ -27,9 +27,8 @@
 import java.util.Map;
 import java.util.NoSuchElementException;
 
-import org.apache.commons.csv.CSVLexer.Token;
 
-import static org.apache.commons.csv.CSVLexer.Token.Type.*;
+import static org.apache.commons.csv.Token.Type.*;
 
 /**
  * Parses CSV files according to the specified configuration.

src/test/java/org/apache/commons/csv/CSVLexerTest.java

@@ -20,10 +20,9 @@
 import java.io.IOException;
 import java.io.StringReader;
 
-import org.apache.commons.csv.CSVLexer.Token;
 import org.junit.Test;
 
-import static org.apache.commons.csv.CSVLexer.Token.Type.*;
+import static org.apache.commons.csv.Token.Type.*;
 import static org.junit.Assert.*;
 
 public class CSVLexerTest {