CODEC-301: Reduce byte[] allocations by reusing buffers

* Reduces byte[] allocations from 280MB to <4MB when reading a 133MB base64 stream. Messured with JFR.
* Keep reusing inital buffer when decoding BaseN
* Attempt to fill up the user-provided buffer
    Previously we only filled up to a maximum of 8KB - encoding-overhead (e.g. 6KB for Base64) even if the provided
    buffer was bigger.
* Reuse hasData method for checking pos/readPos markers

Author: apinske

src/main/java/org/apache/commons/codec/binary/BaseNCodec.java

@@ -394,7 +394,7 @@ protected BaseNCodec(final int unencodedBlockSize, final int encodedBlockSize,
      * @return The amount of buffered data available for reading.
      */
     int available(final Context context) {  // package protected for access from I/O streams
-        return context.buffer != null ? context.pos - context.readPos : 0;
+        return hasData(context) ? context.pos - context.readPos : 0;
     }
 
     /**
@@ -632,7 +632,7 @@ public long getEncodedLength(final byte[] pArray) {
      * @return true if there is data still available for reading.
      */
     boolean hasData(final Context context) {  // package protected for access from I/O streams
-        return context.buffer != null;
+        return context.pos > context.readPos;
     }
 
     /**
@@ -711,12 +711,16 @@ public boolean isStrictDecoding() {
      * @return The number of bytes successfully extracted into the provided byte[] array.
      */
     int readResults(final byte[] b, final int bPos, final int bAvail, final Context context) {
-        if (context.buffer != null) {
+        if (hasData(context)) {
             final int len = Math.min(available(context), bAvail);
             System.arraycopy(context.buffer, context.readPos, b, bPos, len);
             context.readPos += len;
-            if (context.readPos >= context.pos) {
-                context.buffer = null; // so hasData() will return false, and this method can return -1
+            if (!hasData(context)) {
+                // All data read.
+                // Reset position markers but do not set buffer to null to allow its reuse.
+                // hasData(context) will still return false, and this method will return 0 until
+                // more data is available, or -1 if EOF.
+                context.pos = context.readPos = 0;
             }
             return len;
         }

src/main/java/org/apache/commons/codec/binary/BaseNCodecInputStream.java

@@ -39,12 +39,15 @@ public class BaseNCodecInputStream extends FilterInputStream {
 
     private final byte[] singleByte = new byte[1];
 
+    private final byte[] buf;
+
     private final Context context = new Context();
 
     protected BaseNCodecInputStream(final InputStream input, final BaseNCodec baseNCodec, final boolean doEncode) {
         super(input);
         this.doEncode = doEncode;
         this.baseNCodec = baseNCodec;
+        this.buf = new byte[doEncode ? 4096 : 8192];
     }
 
     /**
@@ -169,17 +172,24 @@ public int read(final byte array[], final int offset, final int len) throws IOEx
          -----
          This is a fix for CODEC-101
         */
-        while (readLen == 0) {
+        // Attempt to read the request length
+        while (readLen < len) {
             if (!baseNCodec.hasData(context)) {
-                final byte[] buf = new byte[doEncode ? 4096 : 8192];
+                // Obtain more data.
+                // buf is reused across calls to read to avoid repeated allocations
                 final int c = in.read(buf);
                 if (doEncode) {
                     baseNCodec.encode(buf, 0, c, context);
                 } else {
                     baseNCodec.decode(buf, 0, c, context);
                 }
             }
-            readLen = baseNCodec.readResults(array, offset, len, context);
+            final int read = baseNCodec.readResults(array, offset + readLen, len - readLen, context);
+            if (read < 0) {
+                // Return the amount read or EOF
+                return readLen != 0 ? readLen : -1;
+            }
+            readLen += read;
         }
         return readLen;
     }

src/test/java/org/apache/commons/codec/binary/Base64InputStreamTest.java

@@ -408,6 +408,30 @@ public void testMarkSupported() throws Exception {
         }
     }
 
+    /**
+     * Tests read using different buffer sizes
+     *
+     * @throws Exception
+     *             for some failure scenarios.
+     */
+    @Test
+    public void testReadMultipleBufferSizes() throws Exception {
+        final byte[][] randomData = BaseNTestData.randomData(new Base64(0, null, false), 1024 * 64);
+        byte[] encoded = randomData[1];
+        byte[] decoded = randomData[0];
+        final ByteArrayInputStream bin = new ByteArrayInputStream(encoded);
+        final ByteArrayOutputStream out = new ByteArrayOutputStream();
+        try (final Base64InputStream in = new Base64InputStream(bin)) {
+            for (int i : new int[] { 4 * 1024, 4 * 1024, 8 * 1024, 8 * 1024, 16 * 1024, 16 * 1024, 8 * 1024 }) {
+                final byte[] buf = new byte[i];
+                int bytesRead = in.read(buf);
+                assertEquals(i, bytesRead);
+                out.write(buf, 0, bytesRead);
+            }
+        }
+        assertArrayEquals(decoded, out.toByteArray());
+    }
+
     /**
      * Tests read returning 0
      *