Merge pull request #604 from garydgregory/fix/CSV-325_CSVParser_setByteOffset

[CSV-325] CSVParser applies characterOffset to bytePosition

Author: garydgregory

src/main/java/org/apache/commons/csv/CSVParser.java

@@ -154,6 +154,7 @@ public final class CSVParser implements Iterable<CSVRecord>, Closeable {
     public static class Builder extends AbstractStreamBuilder<CSVParser, Builder> {
 
         private CSVFormat format;
+        private long byteOffset = -1;
         private long characterOffset;
         private long recordNumber = 1;
         private boolean trackBytes;
@@ -171,10 +172,27 @@ public CSVParser get() throws IOException {
         }
 
         /**
-         * Sets the lexer offset when the parser does not start parsing at the beginning of the source.
+         * Sets the lexer byte offset when the parser does not start parsing at the beginning of the source.
+         * <p>
+         * By default, the value is {@code -1}, which reuses the character offset for the byte offset.
+         * </p>
          *
-         * @param characterOffset the lexer offset.
+         * @param byteOffset the lexer byte offset.
          * @return {@code this} instance.
+         * @see #setCharacterOffset(long)
+         * @since 1.15.0
+         */
+        public Builder setByteOffset(final long byteOffset) {
+            this.byteOffset = byteOffset;
+            return asThis();
+        }
+
+        /**
+         * Sets the lexer character offset when the parser does not start parsing at the beginning of the source.
+         *
+         * @param characterOffset the lexer character offset.
+         * @return {@code this} instance.
+         * @see #setByteOffset(long)
          */
         public Builder setCharacterOffset(final long characterOffset) {
             this.characterOffset = characterOffset;
@@ -465,6 +483,12 @@ public static CSVParser parse(final URL url, final Charset charset, final CSVFor
      */
     private long recordNumber;
 
+    /**
+     * Lexer offset when the parser does not start parsing at the beginning of the source. Usually used in combination
+     * with {@link #recordNumber}.
+     */
+    private final long byteOffset;
+
     /**
      * Lexer offset when the parser does not start parsing at the beginning of the source. Usually used in combination
      * with {@link #recordNumber}.
@@ -485,6 +509,7 @@ private CSVParser(final Builder builder) throws IOException {
         this.lexer = new Lexer(format, new ExtendedBufferedReader(builder.getReader(), builder.getCharset(), builder.trackBytes));
         this.csvRecordIterator = new CSVRecordIterator();
         this.headers = createHeaders();
+        this.byteOffset = builder.byteOffset != -1 ? builder.byteOffset : builder.characterOffset;
         this.characterOffset = builder.characterOffset;
         this.recordNumber = builder.recordNumber - 1;
     }
@@ -870,7 +895,7 @@ CSVRecord nextRecord() throws IOException {
         recordList.clear();
         StringBuilder sb = null;
         final long startCharPosition = lexer.getCharacterPosition() + characterOffset;
-        final long startBytePosition = lexer.getBytesRead() + characterOffset;
+        final long startBytePosition = lexer.getBytesRead() + byteOffset;
         do {
             reusableToken.reset();
             lexer.nextToken(reusableToken);

src/test/java/org/apache/commons/csv/CSVParserTest.java

@@ -666,6 +666,36 @@ void testGetBytePositionMultiCharacterDelimiter() throws IOException {
         }
     }
 
+    @Test
+    void testGetBytePositionWithCharacterOffsetAndMultiBytePrefix() throws Exception {
+        final String row0 = "é,x\n";
+        final Charset charset = UTF_8;
+        // row0 char count is 4
+        assertEquals(4, row0.length());
+        // row0 byte count is 5
+        final int record1ByteOffset = row0.getBytes(charset).length;
+        assertEquals(5, record1ByteOffset);
+        final String row1 = "b,c\n";
+        final String rows = row0 + row1;
+        final long record1CharOffset = row0.length();
+        final long expectedByteOffset = row0.getBytes(charset).length;
+        try (CSVParser parser = CSVParser.builder()
+                .setReader(new StringReader(row1))
+                .setFormat(CSVFormat.DEFAULT)
+                .setCharset(charset)
+                .setTrackBytes(true)
+                .setByteOffset(record1ByteOffset)
+                .setCharacterOffset(record1CharOffset)
+                .setRecordNumber(2) // not relevant but a better use case example.
+                .get()) {
+            final CSVRecord record = parser.nextRecord();
+            assertNotNull(record);
+            assertEquals(4, record.getCharacterPosition());
+            assertEquals(record1CharOffset, record.getCharacterPosition());
+            assertEquals(expectedByteOffset, record.getBytePosition());
+        }
+    }
+
     @Test
     void testGetHeaderComment_HeaderComment1() throws IOException {
         try (CSVParser parser = CSVParser.parse(CSV_INPUT_HEADER_COMMENT, FORMAT_AUTO_HEADER)) {