[CSV-131] Save positions of records to enable random access. The floor is open for code review and further discussion based on the comments in the Jira.

garydgregory · garydgregory · commit e28e28e1f27c · 2014-10-29T05:44:40.000Z
git-svn-id: https://svn.apache.org/repos/asf/commons/proper/csv/trunk@1635052 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/src/changes/changes.xml b/src/changes/changes.xml
@@ -39,6 +39,7 @@
   </properties>
   <body>
     <release version="1.1" date="2014-mm-dd" description="Feature and bug fix release">
+      <action issue="CSV-131" type="add" dev="ggregory" due-to="Holger Stratmann">Save positions of records to enable random access</action>
       <action issue="CSV-130" type="fix" dev="ggregory" due-to="Sergei Lebedev">CSVFormat#withHeader doesn't work well with #printComment, add withHeaderComments(String...)</action>
       <action issue="CSV-128" type="fix" dev="ggregory">CSVFormat.EXCEL should ignore empty header names</action>
       <action issue="CSV-129" type="add" dev="ggregory">Add CSVFormat#with 0-arg methods matching boolean arg methods</action>
diff --git a/src/main/java/org/apache/commons/csv/CSVParser.java b/src/main/java/org/apache/commons/csv/CSVParser.java
@@ -219,6 +219,12 @@ public static CSVParser parse(final URL url, final Charset charset, final CSVFor
     private final List<String> record = new ArrayList<String>();
 
     private long recordNumber;
+    
+    /**
+     * Lexer offset if the parser does not start parsing at the beginning of the source. Usually used in combination
+     * with {@link #setNextRecordNumber(long)}
+     */
+    private long characterOffset;
 
     private final Token reusableToken = new Token();
 
@@ -295,6 +301,43 @@ public Map<String, Integer> getHeaderMap() {
         return this.headerMap == null ? null : new LinkedHashMap<String, Integer>(this.headerMap);
     }
 
+    /**
+     * Sets the record number to be assigned to the next record read.
+     * <p>
+     * Use this if the reader is not positioned at the first record when you create the parser. For example, the first
+     * record read might be the 51st record in the source file.
+     * </p>
+     * <p>
+     * If you want the records to also have the correct character position referring to the underlying source, call
+     * {@link #setNextCharacterPosition(long)}.
+     * </p>
+     * 
+     * @param nextRecordNumber
+     *            the next record number
+     * @since 1.1
+     */
+    public void setNextRecordNumber(long nextRecordNumber) {
+        this.recordNumber = nextRecordNumber - 1;
+    }
+    
+    /**
+     * Sets the current position in the source stream regardless of where the parser and lexer start reading.
+     * <p>
+     * For example: We open a file and seek to position 5434 in order to start reading at record 42. In order to have
+     * the parser assign the correct characterPosition to records, we call this method.
+     * </p>
+     * <p>
+     * If you want the records to also have the correct record numbers, call {@link #setNextRecordNumber(long)}
+     * </p>
+     * 
+     * @param position
+     *            the new character position
+     * @since 1.1
+     */
+    public void setNextCharacterPosition(long position) {
+        this.characterOffset = position - lexer.getCharacterPosition();
+    }
+    
     /**
      * Returns the current record number in the input stream.
      *
@@ -445,6 +488,7 @@ CSVRecord nextRecord() throws IOException {
         CSVRecord result = null;
         this.record.clear();
         StringBuilder sb = null;
+        final long startCharPosition = lexer.getCharacterPosition() + this.characterOffset;
         do {
             this.reusableToken.reset();
             this.lexer.nextToken(this.reusableToken);
@@ -480,7 +524,7 @@ CSVRecord nextRecord() throws IOException {
             this.recordNumber++;
             final String comment = sb == null ? null : sb.toString();
             result = new CSVRecord(this.record.toArray(new String[this.record.size()]), this.headerMap, comment,
-                    this.recordNumber);
+                    this.recordNumber, startCharPosition);
         }
         return result;
     }
diff --git a/src/main/java/org/apache/commons/csv/CSVRecord.java b/src/main/java/org/apache/commons/csv/CSVRecord.java
@@ -36,6 +36,8 @@ public final class CSVRecord implements Serializable, Iterable<String> {
 
     private static final long serialVersionUID = 1L;
 
+    private final long characterPosition;
+
     /** The accumulated comments (if any) */
     private final String comment;
 
@@ -44,15 +46,16 @@ public final class CSVRecord implements Serializable, Iterable<String> {
 
     /** The record number. */
     private final long recordNumber;
-
+    
     /** The values of the record */
     private final String[] values;
 
-    CSVRecord(final String[] values, final Map<String, Integer> mapping, final String comment, final long recordNumber) {
+    CSVRecord(final String[] values, final Map<String, Integer> mapping, final String comment, final long recordNumber, long characterPosition) {
         this.recordNumber = recordNumber;
         this.values = values != null ? values : EMPTY_STRING_ARRAY;
         this.mapping = mapping;
         this.comment = comment;
+        this.characterPosition = characterPosition;
     }
 
     /**
@@ -109,6 +112,16 @@ public String get(final String name) {
         }
     }
 
+    /**
+     * Returns the start position of this record as a character position in the source stream. This may or may not
+     * correspond to the byte position depending on the character set.
+     *
+     * @return the position of this record in the source stream.
+     */
+    public long getCharacterPosition() {
+        return characterPosition;
+    }
+    
     /**
      * Returns the comment for this record, if any.
      *
diff --git a/src/test/java/org/apache/commons/csv/CSVParserTest.java b/src/test/java/org/apache/commons/csv/CSVParserTest.java
@@ -299,22 +299,23 @@ public void testEmptyLineBehaviourExcel() throws Exception {
         }
     }
 
-//    @Test
-//    public void testStartWithEmptyLinesThenHeaders() throws Exception {
-//        final String[] codes = { "\r\n\r\n\r\nhello,\r\n\r\n\r\n", "hello,\n\n\n", "hello,\"\"\r\n\r\n\r\n", "hello,\"\"\n\n\n" };
-//        final String[][] res = { { "hello", "" }, { "" }, // Excel format does not ignore empty lines
-//                { "" } };
-//        for (final String code : codes) {
-//            final CSVParser parser = CSVParser.parse(code, CSVFormat.EXCEL);
-//            final List<CSVRecord> records = parser.getRecords();
-//            assertEquals(res.length, records.size());
-//            assertTrue(records.size() > 0);
-//            for (int i = 0; i < res.length; i++) {
-//                assertArrayEquals(res[i], records.get(i).values());
-//            }
-//            parser.close();
-//        }
-//    }
+    // @Test
+    // public void testStartWithEmptyLinesThenHeaders() throws Exception {
+    // final String[] codes = { "\r\n\r\n\r\nhello,\r\n\r\n\r\n", "hello,\n\n\n", "hello,\"\"\r\n\r\n\r\n",
+    // "hello,\"\"\n\n\n" };
+    // final String[][] res = { { "hello", "" }, { "" }, // Excel format does not ignore empty lines
+    // { "" } };
+    // for (final String code : codes) {
+    // final CSVParser parser = CSVParser.parse(code, CSVFormat.EXCEL);
+    // final List<CSVRecord> records = parser.getRecords();
+    // assertEquals(res.length, records.size());
+    // assertTrue(records.size() > 0);
+    // for (int i = 0; i < res.length; i++) {
+    // assertArrayEquals(res[i], records.get(i).values());
+    // }
+    // parser.close();
+    // }
+    // }
 
     @Test
     public void testEndOfFileBehaviorCSV() throws Exception {
@@ -474,6 +475,16 @@ public void testGetLineNumberWithLF() throws Exception {
         this.validateLineNumbers(String.valueOf(LF));
     }
 
+    @Test
+    public void testGetRecordPositionWithCRLF() throws Exception {
+        this.validateRecordPosition(CRLF);
+    }
+
+    @Test
+    public void testGetRecordPositionWithLF() throws Exception {
+        this.validateRecordPosition(String.valueOf(LF));
+    }
+
     @Test
     public void testGetOneLine() throws IOException {
         final CSVParser parser = CSVParser.parse(CSV_INPUT_1, CSVFormat.DEFAULT);
@@ -902,4 +913,65 @@ private void validateRecordNumbers(final String lineSeparator) throws IOExceptio
         parser.close();
     }
 
+    private void validateRecordPosition(final String lineSeparator) throws IOException {
+        final String nl = lineSeparator; // used as linebreak in values for better distinction
+
+        String code = "a,b,c" + lineSeparator + "1,2,3" + lineSeparator +
+        // to see if recordPosition correctly points to the enclosing quote
+                "'A" + nl + "A','B" + nl + "B',CC" + lineSeparator +
+                // unicode test... not very relevant while operating on strings instead of bytes, but for
+                // completeness...
+                "\u00c4,\u00d6,\u00dc" + lineSeparator + "EOF,EOF,EOF";
+
+        final CSVFormat format = CSVFormat.newFormat(',').withQuote('\'').withRecordSeparator(lineSeparator);
+        CSVParser parser = CSVParser.parse(code, format);
+
+        CSVRecord record;
+        assertEquals(0, parser.getRecordNumber());
+
+        assertNotNull(record = parser.nextRecord());
+        assertEquals(1, record.getRecordNumber());
+        assertEquals(code.indexOf('a'), record.getCharacterPosition());
+
+        assertNotNull(record = parser.nextRecord());
+        assertEquals(2, record.getRecordNumber());
+        assertEquals(code.indexOf('1'), record.getCharacterPosition());
+
+        assertNotNull(record = parser.nextRecord());
+        final long positionRecord3 = record.getCharacterPosition();
+        assertEquals(3, record.getRecordNumber());
+        assertEquals(code.indexOf("'A"), record.getCharacterPosition());
+        assertEquals("A" + lineSeparator + "A", record.get(0));
+        assertEquals("B" + lineSeparator + "B", record.get(1));
+        assertEquals("CC", record.get(2));
+
+        assertNotNull(record = parser.nextRecord());
+        assertEquals(4, record.getRecordNumber());
+        assertEquals(code.indexOf('\u00c4'), record.getCharacterPosition());
+
+        assertNotNull(record = parser.nextRecord());
+        assertEquals(5, record.getRecordNumber());
+        assertEquals(code.indexOf("EOF"), record.getCharacterPosition());
+
+        parser.close();
+
+        // now try to read starting at record 3
+        parser = CSVParser.parse(code.substring((int) positionRecord3), format);
+        parser.setNextRecordNumber(3);
+        parser.setNextCharacterPosition(positionRecord3);
+
+        assertNotNull(record = parser.nextRecord());
+        assertEquals(3, record.getRecordNumber());
+        assertEquals(code.indexOf("'A"), record.getCharacterPosition());
+        assertEquals("A" + lineSeparator + "A", record.get(0));
+        assertEquals("B" + lineSeparator + "B", record.get(1));
+        assertEquals("CC", record.get(2));
+
+        assertNotNull(record = parser.nextRecord());
+        assertEquals(4, record.getRecordNumber());
+        assertEquals(code.indexOf('\u00c4'), record.getCharacterPosition());
+        assertEquals("\u00c4", record.get(0));
+
+        parser.close();
+    }
 }
diff --git a/src/test/java/org/apache/commons/csv/CSVRecordTest.java b/src/test/java/org/apache/commons/csv/CSVRecordTest.java
@@ -45,12 +45,12 @@ private enum EnumFixture { UNKNOWN_COLUMN }
     @Before
     public void setUp() throws Exception {
         values = new String[] { "A", "B", "C" };
-        record = new CSVRecord(values, null, null, 0);
+        record = new CSVRecord(values, null, null, 0, -1);
         header = new HashMap<String, Integer>();
         header.put("first", Integer.valueOf(0));
         header.put("second", Integer.valueOf(1));
         header.put("third", Integer.valueOf(2));
-        recordWithHeader = new CSVRecord(values, header, null, 0);
+        recordWithHeader = new CSVRecord(values, header, null, 0, -1);
     }
 
     @Test
