[CSV-196] Track byte information of the source

Author: mattsunsjf

src/main/java/org/apache/commons/csv/CSVParser.java

@@ -348,11 +348,16 @@ public CSVParser(final Reader reader, final CSVFormat format) throws IOException
     @SuppressWarnings("resource")
     public CSVParser(final Reader reader, final CSVFormat format, final long characterOffset, final long recordNumber)
             throws IOException {
+        this(reader, format, characterOffset, recordNumber, null);
+    }
+
+    public CSVParser(final Reader reader, final CSVFormat format, final long characterOffset,
+            final long recordNumber, String encoding) throws IOException {
         Assertions.notNull(reader, "reader");
         Assertions.notNull(format, "format");
 
         this.format = format;
-        this.lexer = new Lexer(format, new ExtendedBufferedReader(reader));
+        this.lexer = new Lexer(format, new ExtendedBufferedReader(reader, encoding));
         this.headerMap = this.initializeHeader();
         this.characterOffset = characterOffset;
         this.recordNumber = recordNumber - 1;
@@ -581,6 +586,7 @@ CSVRecord nextRecord() throws IOException {
         this.recordList.clear();
         StringBuilder sb = null;
         final long startCharPosition = lexer.getCharacterPosition() + this.characterOffset;
+        final long startCharByte = lexer.getBytesRead() + this.characterOffset;
         do {
             this.reusableToken.reset();
             this.lexer.nextToken(this.reusableToken);
@@ -616,7 +622,7 @@ CSVRecord nextRecord() throws IOException {
             this.recordNumber++;
             final String comment = sb == null ? null : sb.toString();
             result = new CSVRecord(this.recordList.toArray(new String[this.recordList.size()]), this.headerMap, comment,
-                    this.recordNumber, startCharPosition);
+                    this.recordNumber, startCharPosition, startCharByte);
         }
         return result;
     }

src/main/java/org/apache/commons/csv/CSVRecord.java

@@ -36,6 +36,8 @@ public final class CSVRecord implements Serializable, Iterable<String> {
 
     private final long characterPosition;
 
+    private final long characterByte;
+
     /** The accumulated comments (if any) */
     private final String comment;
 
@@ -48,13 +50,25 @@ public final class CSVRecord implements Serializable, Iterable<String> {
     /** The values of the record */
     private final String[] values;
 
+
+    CSVRecord(final String[] values, final Map<String, Integer> mapping, final String comment, final long recordNumber,
+            final long characterPosition, final long characterByte) {
+        this.recordNumber = recordNumber;
+        this.values = values != null ? values : EMPTY_STRING_ARRAY;
+        this.mapping = mapping;
+        this.comment = comment;
+        this.characterPosition = characterPosition;
+        this.characterByte = characterByte;
+    }
+
     CSVRecord(final String[] values, final Map<String, Integer> mapping, final String comment, final long recordNumber,
             final long characterPosition) {
         this.recordNumber = recordNumber;
         this.values = values != null ? values : EMPTY_STRING_ARRAY;
         this.mapping = mapping;
         this.comment = comment;
         this.characterPosition = characterPosition;
+        this.characterByte = 0L;
     }
 
     /**
@@ -121,6 +135,15 @@ public long getCharacterPosition() {
         return characterPosition;
     }
 
+    /**
+     * Returns the start byte of this record as a character byte in the source stream.
+     *
+     * @return the start byte of this record as a character byte in the source stream.
+     */
+    public long getCharacterByte() {
+        return characterByte;
+    }
+
     /**
      * Returns the comment for this record, if any.
      * Note that comments are attached to the following record.

src/main/java/org/apache/commons/csv/ExtendedBufferedReader.java

@@ -24,7 +24,11 @@
 
 import java.io.BufferedReader;
 import java.io.IOException;
+import java.io.InputStreamReader;
 import java.io.Reader;
+import java.nio.charset.CharsetEncoder;
+import java.nio.charset.Charset;
+import java.nio.CharBuffer;
 
 /**
  * A special buffered reader which supports sophisticated read access.
@@ -46,12 +50,25 @@ final class ExtendedBufferedReader extends BufferedReader {
 
     private boolean closed;
 
+    /** The number of bytes read so far */
+    private long bytesRead;
+
+    /** Encoder used to calculate the bytes of characters */
+    CharsetEncoder encoder;
+
     /**
      * Created extended buffered reader using default buffer-size
      */
     ExtendedBufferedReader(final Reader reader) {
         super(reader);
     }
+    
+    ExtendedBufferedReader(final Reader reader, String encoding) {
+        super(reader);
+        if (encoding != null) {
+            encoder = Charset.forName(encoding).newEncoder();
+        }
+    }
 
     @Override
     public int read() throws IOException {
@@ -61,6 +78,9 @@ public int read() throws IOException {
         }
         lastChar = current;
         this.position++;
+        if (encoder != null) {
+            this.bytesRead += encoder.encode(CharBuffer.wrap(new char[] { (char)current })).limit();
+        }
         return lastChar;
     }
 
@@ -170,6 +190,15 @@ long getPosition() {
         return this.position;
     }
 
+    /**
+     * Gets the number of bytes read by the reader.
+     *
+     * @return the number of bytes read by the read
+     */
+    long getBytesRead() {
+        return this.bytesRead;
+    }
+
     public boolean isClosed() {
         return closed;
     }

src/main/java/org/apache/commons/csv/Lexer.java

@@ -316,6 +316,15 @@ long getCharacterPosition() {
         return reader.getPosition();
     }
 
+    /**
+     * Returns the number of bytes read
+     *
+     * @return the number of bytes read
+     */
+    long getBytesRead() {
+        return reader.getBytesRead();
+    }
+
     // TODO escape handling needs more work
     /**
      * Handle an escape sequence.

src/test/java/org/apache/commons/csv/CSVParserTest.java

@@ -560,6 +560,42 @@ public void testGetRecordPositionWithCRLF() throws Exception {
     public void testGetRecordPositionWithLF() throws Exception {
         this.validateRecordPosition(String.valueOf(LF));
     }
+    
+    @Test
+    public void testGetRecordBytesRead() throws Exception {
+        String code = "id,date,val5,val4\n"
+                + "11111111111111,'4017-09-01',きちんと節分近くには咲いてる～,v4\n"
+                + "22222222222222,'4017-01-01',おはよう私の友人～,v4\n"
+                + "33333333333333,'4017-01-01',きる自然の力ってすごいな～,v4\n";
+        final CSVFormat format = CSVFormat.newFormat(',').withQuote('\'');
+        CSVParser parser = new CSVParser(new StringReader(code), format, 0L, 1L, "UTF-8");
+
+        CSVRecord record;
+        assertEquals(0, parser.getRecordNumber());
+
+        assertNotNull(record = parser.nextRecord());
+        assertEquals(1, record.getRecordNumber());
+        assertEquals(code.indexOf('i'), record.getCharacterByte());
+        assertEquals(code.indexOf('i'), record.getCharacterPosition());
+
+        assertNotNull(record = parser.nextRecord());
+        assertEquals(2, record.getRecordNumber());
+        assertEquals(code.indexOf('1'), record.getCharacterByte());
+        assertEquals(code.indexOf('1'), record.getCharacterPosition());
+
+        assertNotNull(record = parser.nextRecord());
+        assertEquals(3, record.getRecordNumber());
+        assertEquals(code.indexOf('2'), record.getCharacterPosition());
+        assertEquals(code.indexOf('2'), record.getCharacterPosition());
+
+        assertNotNull(record = parser.nextRecord());
+        assertEquals(4, record.getRecordNumber());
+        assertEquals(code.indexOf('3'), record.getCharacterPosition());
+        assertEquals(code.indexOf('3'), record.getCharacterPosition());
+
+        parser.close();
+
+    }
 
     @Test
     public void testGetRecords() throws IOException {

src/test/java/org/apache/commons/csv/issues/JiraCsv196Test.java

@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.commons.csv.issues;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVParser;
+import org.apache.commons.csv.CSVRecord;
+import org.junit.Test;
+import org.junit.Assert;
+
+public class JiraCsv196Test {
+
+    @Test
+    public void parse() throws IOException {
+        final CSVFormat format = CSVFormat.newFormat(',').withQuote('\'');
+        CSVParser parser = new CSVParser(getTestInput(), format, 0L, 1L, "UTF-8");
+        long[] charByteKey = {0, 89, 242, 395};
+        int idx = 0;
+        for (CSVRecord record : parser) {
+            Assert.assertEquals(charByteKey[idx++], record.getCharacterByte());
+        }
+        parser.close();
+    }
+
+    private Reader getTestInput() {
+        final InputStream is = ClassLoader.getSystemClassLoader().getResourceAsStream("CSV-196/sample1.csv");
+        return new InputStreamReader(is);
+    }
+
+}

src/test/resources/CSV-196/sample1.csv

@@ -0,0 +1,4 @@
+id,date,val1,val2,val3,val4,val5,val6,val7,val8,val9,val10,val11,val12,val13,val14,val15
+00000000000001,2017-01-01,きちんと節分近くには咲いてる。自然の力ってすごいな～,v2,v3,v4,v5,v6,v7,v8,v9,v10,v11,v12,v13,v14,v15
+00000000000002,2017-01-01,きちんと節分近くには咲いてる。自然の力ってすごいな～,v2,v3,v4,v5,v6,v7,v8,v9,v10,v11,v12,v13,v14,v15
+00000000000003,2017-01-01,きちんと節分近くには咲いてる。自然の力ってすごいな～,v2,v3,v4,v5,v6,v7,v8,v9,v10,v11,v12,v13,v14,v15