+ * This variant allocates the target array immediately and attempts to fill it in one pass. + * It assumes that {@code size} is correct. + * If the stream ends prematurely, an {@link EOFException} is thrown. + *
+ * + *+ * Important: This method does not defend against corrupted + * or untrusted {@code size} values. + * For untrusted input, use {@link #toByteArray(InputStream, int, int)} instead, + * which validates that the stream contains at least {@code size} bytes before allocating the target array. + *
+ * + * @param input the {@link InputStream} to read; must not be {@code null}. + * @param size the exact number of bytes to read; must be {@code >= 0}. + * @return a new byte array of length {@code size}. + * @throws IllegalArgumentException if {@code size} is negative. + * @throws EOFException if the stream ends before {@code size} bytes are read. + * @throws IOException if an I/O error occurs while reading. + * @throws NullPointerException if {@code input} is {@code null}. * @since 2.1 */ public static byte[] toByteArray(final InputStream input, final int size) throws IOException { - if (size == 0) { - return EMPTY_BYTE_ARRAY; + Objects.requireNonNull(input, "input"); + if (size < 0) { + throw new IllegalArgumentException("Size must be equal or greater than zero: " + size); } - return toByteArray(Objects.requireNonNull(input, "input")::read, size); + return toByteArray(input::read, size); } /** - * Gets contents of an {@link InputStream} as a {@code byte[]}. - * Use this method instead of {@link #toByteArray(InputStream)} - * when {@link InputStream} size is known. - * NOTE: the method checks that the length can safely be cast to an int without truncation - * before using {@link IOUtils#toByteArray(InputStream, int)} to read into the byte array. - * (Arrays can have no more than Integer.MAX_VALUE entries anyway.) + * Reads exactly {@code size} bytes from the given {@link InputStream} into a new {@code byte[]}. * - * @param input the {@link InputStream} to read. - * @param size the size of {@link InputStream} to read, where 0 < {@code size} <= min(Integer.MAX_VALUE, length of input stream). - * @return byte [] the requested byte array, of length {@code size}. - * @throws IOException if an I/O error occurs or {@link InputStream} length is less than {@code size}. - * @throws IllegalArgumentException if size is less than zero or size is greater than Integer.MAX_VALUE. - * @see IOUtils#toByteArray(InputStream, int) + *+ * This is a convenience overload of {@link #toByteArray(InputStream, int, int)} that accepts a + * {@code long} size parameter. The value is checked to ensure it does not exceed + * {@link Integer#MAX_VALUE} before being safely converted to {@code int}. + *
+ * + *+ * All behavior, validation rules, and exceptions are otherwise identical to + * {@link #toByteArray(InputStream, int, int)}. + *
+ * + * @param input the {@link InputStream} to read; must not be {@code null}. + * @param size the exact number of bytes to read; must be {@code >= 0} and {@code <= Integer.MAX_VALUE}. + * @return a new byte array of length {@code size}. + * @throws IllegalArgumentException if {@code size} is negative or greater than {@link Integer#MAX_VALUE}. + * @throws EOFException if the stream ends before {@code size} bytes are read. + * @throws IOException if an I/O error occurs while reading. + * @throws NullPointerException if {@code input} is {@code null}. + * @see #toByteArray(InputStream, int, int) * @since 2.1 */ public static byte[] toByteArray(final InputStream input, final long size) throws IOException { @@ -2699,6 +2723,62 @@ public static byte[] toByteArray(final InputStream input, final long size) throw return toByteArray(input, (int) size); } + /** + * Reads exactly {@code size} bytes from the given {@link InputStream} into a new {@code byte[]}. + * + *+ * This variant validates that the stream actually contains {@code size} bytes. + * It is suitable for untrusted input because it prevents oversized allocations when the provided {@code size} + * is corrupted or malicious. + *
+ * + *- * This method buffers the input internally, so there is no need to use a {@link BufferedInputStream}. - *
+ * Reads all remaining bytes from the given {@link InputStream} into a new {@code byte[]}. * - * @param inputStream the {@link InputStream} to read. - * @return the requested byte array. - * @throws NullPointerException if the InputStream is {@code null}. - * @throws IOException if an I/O error occurs or reading more than {@link Integer#MAX_VALUE} occurs. + *The method accumulates the data in temporary buffers and returns a single array + * containing the entire contents once the end of the stream is reached.
+ * + * @param input the {@link InputStream} to read; must not be {@code null}. + * @return a new byte array. + * @throws IllegalArgumentException if the size of the stream is greater than {@code Integer.MAX_VALUE}. + * @throws IOException if an I/O error occurs while reading. + * @throws NullPointerException if {@code input} is {@code null}. */ - public static byte[] toByteArray(final InputStream inputStream) throws IOException { + public static byte[] toByteArray(final InputStream input) throws IOException { // We use a ThresholdingOutputStream to avoid reading AND writing more than Integer.MAX_VALUE. try (UnsynchronizedByteArrayOutputStream ubaOutput = UnsynchronizedByteArrayOutputStream.builder().get(); ThresholdingOutputStream thresholdOutput = new ThresholdingOutputStream(Integer.MAX_VALUE, os -> { throw new IllegalArgumentException(String.format("Cannot read more than %,d into a byte array", Integer.MAX_VALUE)); }, os -> ubaOutput)) { - copy(inputStream, thresholdOutput); + copy(input, thresholdOutput); return ubaOutput.toByteArray(); } } @@ -2662,18 +2663,8 @@ public static byte[] toByteArray(final InputStream inputStream) throws IOExcepti /** * Reads exactly {@code size} bytes from the given {@link InputStream} into a new {@code byte[]}. * - *- * This variant allocates the target array immediately and attempts to fill it in one pass. - * It assumes that {@code size} is correct. - * If the stream ends prematurely, an {@link EOFException} is thrown. - *
- * - *- * Important: This method does not defend against corrupted - * or untrusted {@code size} values. - * For untrusted input, use {@link #toByteArray(InputStream, int, int)} instead, - * which validates that the stream contains at least {@code size} bytes before allocating the target array. - *
+ *The method allocates a single array of the requested size and fills it directly + * from the stream.
* * @param input the {@link InputStream} to read; must not be {@code null}. * @param size the exact number of bytes to read; must be {@code >= 0}. @@ -2691,21 +2682,13 @@ public static byte[] toByteArray(final InputStream input, final int size) throws /** * Reads exactly {@code size} bytes from the given {@link InputStream} into a new {@code byte[]}. * - *- * This is a convenience overload of {@link #toByteArray(InputStream, int, int)} that accepts a - * {@code long} size parameter. The value is checked to ensure it does not exceed - * {@link Integer#MAX_VALUE} before being safely converted to {@code int}. - *
- * - *- * All behavior, validation rules, and exceptions are otherwise identical to - * {@link #toByteArray(InputStream, int, int)}. - *
+ *The method allocates a single array of the requested size and fills it directly + * from the stream.
* * @param input the {@link InputStream} to read; must not be {@code null}. * @param size the exact number of bytes to read; must be {@code >= 0} and {@code <= Integer.MAX_VALUE}. * @return a new byte array of length {@code size}. - * @throws IllegalArgumentException if {@code size} is negative or greater than {@link Integer#MAX_VALUE}. + * @throws IllegalArgumentException if {@code size} is negative or does not fit into an int. * @throws EOFException if the stream ends before {@code size} bytes are read. * @throws IOException if an I/O error occurs while reading. * @throws NullPointerException if {@code input} is {@code null}. @@ -2722,20 +2705,8 @@ public static byte[] toByteArray(final InputStream input, final long size) throw /** * Reads exactly {@code size} bytes from the given {@link InputStream} into a new {@code byte[]}. * - *- * This variant validates that the stream actually contains {@code size} bytes. - * It is suitable for untrusted input because it prevents oversized allocations when the provided {@code size} - * is corrupted or malicious. - *
- * - *The method accumulates the data in temporary buffers of size at most {@code bufferSize} + * and returns a single array containing the entire contents once the end of the stream is reached.
* * @param input the {@link InputStream} to read; must not be {@code null}. * @param size the exact number of bytes to read; must be {@code >= 0}. From fe39b777c447f2eeb185ae2520ca6b547d84e719 Mon Sep 17 00:00:00 2001 From: "Piotr P. Karwasz"The method accumulates the data in temporary buffers and returns a single array * containing the entire contents once the end of the stream is reached.
* - * @param input the {@link InputStream} to read; must not be {@code null}. + * @param inputStream the {@link InputStream} to read; must not be {@code null}. * @return a new byte array. * @throws IllegalArgumentException if the size of the stream is greater than {@code Integer.MAX_VALUE}. * @throws IOException if an I/O error occurs while reading. - * @throws NullPointerException if {@code input} is {@code null}. + * @throws NullPointerException if {@code inputStream} is {@code null}. */ - public static byte[] toByteArray(final InputStream input) throws IOException { + public static byte[] toByteArray(final InputStream inputStream) throws IOException { // We use a ThresholdingOutputStream to avoid reading AND writing more than Integer.MAX_VALUE. try (UnsynchronizedByteArrayOutputStream ubaOutput = UnsynchronizedByteArrayOutputStream.builder().get(); ThresholdingOutputStream thresholdOutput = new ThresholdingOutputStream(Integer.MAX_VALUE, os -> { throw new IllegalArgumentException(String.format("Cannot read more than %,d into a byte array", Integer.MAX_VALUE)); }, os -> ubaOutput)) { - copy(input, thresholdOutput); + copy(inputStream, thresholdOutput); return ubaOutput.toByteArray(); } } From 97d37a94de8c935ea8e4961e354a1cd1d5c49b6a Mon Sep 17 00:00:00 2001 From: "Piotr P. Karwasz"The method accumulates the data in temporary buffers and returns a single array - * containing the entire contents once the end of the stream is reached.
- * * @param inputStream the {@link InputStream} to read; must not be {@code null}. * @return a new byte array. * @throws IllegalArgumentException if the size of the stream is greater than {@code Integer.MAX_VALUE}. @@ -2663,9 +2660,6 @@ public static byte[] toByteArray(final InputStream inputStream) throws IOExcepti /** * Reads exactly {@code size} bytes from the given {@link InputStream} into a new {@code byte[]}. * - *The method allocates a single array of the requested size and fills it directly - * from the stream.
- * * @param input the {@link InputStream} to read; must not be {@code null}. * @param size the exact number of bytes to read; must be {@code >= 0}. * @return a new byte array of length {@code size}. @@ -2682,9 +2676,6 @@ public static byte[] toByteArray(final InputStream input, final int size) throws /** * Reads exactly {@code size} bytes from the given {@link InputStream} into a new {@code byte[]}. * - *The method allocates a single array of the requested size and fills it directly - * from the stream.
- * * @param input the {@link InputStream} to read; must not be {@code null}. * @param size the exact number of bytes to read; must be {@code >= 0} and {@code <= Integer.MAX_VALUE}. * @return a new byte array of length {@code size}. @@ -2705,8 +2696,12 @@ public static byte[] toByteArray(final InputStream input, final long size) throw /** * Reads exactly {@code size} bytes from the given {@link InputStream} into a new {@code byte[]}. * - *The method accumulates the data in temporary buffers of size at most {@code bufferSize} - * and returns a single array containing the entire contents once the end of the stream is reached.
+ *When reading from an untrusted stream, this variant lowers the risk of + * {@link OutOfMemoryError} by allocating data in buffers of up to {@code bufferSize} + * bytes rather than in one large array.
+ * + *Note, however, that this approach requires additional temporary memory + * compared to {@link #toByteArray(InputStream, int)}.
* * @param input the {@link InputStream} to read; must not be {@code null}. * @param size the exact number of bytes to read; must be {@code >= 0}. From cbfa307f526f8c6fac4744fea135de16edbdb1b0 Mon Sep 17 00:00:00 2001 From: "Piotr P. Karwasz"+ * This value is somewhat arbitrary, currently aligned with the value used by + * Python + * for copying streams. + *
+ */ + private static final int DEFAULT_CHUNK_SIZE = 128 * 1024; + /** * Returns the given InputStream if it is already a {@link BufferedInputStream}, otherwise creates a * BufferedInputStream from the given InputStream. @@ -2640,26 +2654,34 @@ public static BufferedReader toBufferedReader(final Reader reader, final int siz /** * Reads all the bytes from an input stream in a byte array. * - * @param inputStream the {@link InputStream} to read; must not be {@code null}. - * @return a new byte array. - * @throws IllegalArgumentException if the size of the stream is greater than {@code Integer.MAX_VALUE}. - * @throws IOException if an I/O error occurs while reading. - * @throws NullPointerException if {@code inputStream} is {@code null}. + *The memory used by this method is proportional to the number + * of bytes read, which is only limited by {@link Integer#MAX_VALUE}. This makes it unsuitable for + * processing large input streams, unless sufficient heap space is available.
+ * + * @param inputStream The {@link InputStream} to read; must not be {@code null}. + * @return A new byte array. + * @throws IllegalArgumentException If the size of the stream is greater than the maximum array size. + * @throws IOException If an I/O error occurs while reading. + * @throws NullPointerException If {@code inputStream} is {@code null}. */ public static byte[] toByteArray(final InputStream inputStream) throws IOException { - // We use a ThresholdingOutputStream to avoid reading AND writing more than Integer.MAX_VALUE. - try (UnsynchronizedByteArrayOutputStream ubaOutput = UnsynchronizedByteArrayOutputStream.builder().get(); - ThresholdingOutputStream thresholdOutput = new ThresholdingOutputStream(Integer.MAX_VALUE, os -> { - throw new IllegalArgumentException(String.format("Cannot read more than %,d into a byte array", Integer.MAX_VALUE)); - }, os -> ubaOutput)) { - copy(inputStream, thresholdOutput); - return ubaOutput.toByteArray(); + final UnsynchronizedByteArrayOutputStream output = + copyToOutputStream(inputStream, MAX_ARRAY_LENGTH + 1, DEFAULT_CHUNK_SIZE); + if (output.size() > MAX_ARRAY_LENGTH) { + throw new IllegalArgumentException( + String.format("Cannot read more than %,d into a byte array", MAX_ARRAY_LENGTH)); } + return output.toByteArray(); } /** * Reads exactly {@code size} bytes from the given {@link InputStream} into a new {@code byte[]}. * + *The memory used by this method is proportional to the number + * of bytes read and limited by the specified {@code size}. This makes it suitable for + * processing large input streams, provided that sufficient heap space is + * available.
+ * * @param input the {@link InputStream} to read; must not be {@code null}. * @param size the exact number of bytes to read; must be {@code >= 0}. * @return a new byte array of length {@code size}. @@ -2670,12 +2692,17 @@ public static byte[] toByteArray(final InputStream inputStream) throws IOExcepti * @since 2.1 */ public static byte[] toByteArray(final InputStream input, final int size) throws IOException { - return toByteArray(Objects.requireNonNull(input, "input")::read, size); + return toByteArray(input, size, DEFAULT_CHUNK_SIZE); } /** * Reads exactly {@code size} bytes from the given {@link InputStream} into a new {@code byte[]}. * + *The memory used by this method is proportional to the number + * of bytes read and limited by the specified {@code size}. This makes it suitable for + * processing large input streams, provided that sufficient heap space is + * available.
+ * * @param input the {@link InputStream} to read; must not be {@code null}. * @param size the exact number of bytes to read; must be {@code >= 0} and {@code <= Integer.MAX_VALUE}. * @return a new byte array of length {@code size}. @@ -2696,46 +2723,63 @@ public static byte[] toByteArray(final InputStream input, final long size) throw /** * Reads exactly {@code size} bytes from the given {@link InputStream} into a new {@code byte[]}. * - *When reading from an untrusted stream, this variant lowers the risk of - * {@link OutOfMemoryError} by allocating data in buffers of up to {@code bufferSize} - * bytes rather than in one large array.
+ *The memory used by this method is proportional to the number + * of bytes read and limited by the specified {@code size}. This makes it suitable for + * processing large input streams, provided that sufficient heap space is + * available.
* - *Note, however, that this approach requires additional temporary memory - * compared to {@link #toByteArray(InputStream, int)}.
+ *This method processes the input stream in successive chunks of up to + * {@code chunkSize} bytes.
* * @param input the {@link InputStream} to read; must not be {@code null}. * @param size the exact number of bytes to read; must be {@code >= 0}. * The actual bytes read are validated to equal {@code size}. - * @param bufferSize the buffer size for incremental reading; must be {@code > 0}. + * @param chunkSize The chunk size for incremental reading; must be {@code > 0}. * @return a new byte array of length {@code size}. - * @throws IllegalArgumentException if {@code size} is negative or {@code bufferSize <= 0}. + * @throws IllegalArgumentException if {@code size} is negative or {@code chunkSize <= 0}. * @throws EOFException if the stream ends before {@code size} bytes are read. * @throws IOException if an I/O error occurs while reading. * @throws NullPointerException if {@code input} is {@code null}. * @since 2.21.0 */ - public static byte[] toByteArray(final InputStream input, final int size, final int bufferSize) throws IOException { + public static byte[] toByteArray(final InputStream input, final int size, final int chunkSize) throws IOException { Objects.requireNonNull(input, "input"); - if (bufferSize <= 0) { - throw new IllegalArgumentException("Buffer size must be greater than zero: " + bufferSize); + if (chunkSize <= 0) { + throw new IllegalArgumentException("Chunk size must be greater than zero: " + chunkSize); } - if (size <= bufferSize) { + if (size <= chunkSize) { // throws if size < 0 return toByteArray(input::read, size); } + final UnsynchronizedByteArrayOutputStream output = copyToOutputStream(input, size, chunkSize); + if (output.size() != size) { + throw new EOFException("Unexpected read size, current: " + output.size() + ", expected: " + size); + } + return output.toByteArray(); + } + + /** + * Copies up to {@code size} bytes from the given {@link InputStream} into a new {@link UnsynchronizedByteArrayOutputStream}. + * + * + * @param input The {@link InputStream} to read; must not be {@code null}. + * @param limit The maximum number of bytes to read; must be {@code >= 0}. + * The actual bytes read are validated to equal {@code size}. + * @param bufferSize The buffer size of the output stream; must be {@code > 0}. + * @return a ByteArrayOutputStream containing the read bytes. + */ + private static UnsynchronizedByteArrayOutputStream copyToOutputStream( + final InputStream input, final long limit, final int bufferSize) throws IOException { try (UnsynchronizedByteArrayOutputStream output = UnsynchronizedByteArrayOutputStream.builder() .setBufferSize(bufferSize) .get(); InputStream boundedInput = BoundedInputStream.builder() - .setMaxCount(size) + .setMaxCount(limit) .setPropagateClose(false) .setInputStream(input) .get()) { output.write(boundedInput); - if (output.size() != size) { - throw new EOFException("Unexpected read size, current: " + output.size() + ", expected: " + size); - } - return output.toByteArray(); + return output; } } @@ -2756,13 +2800,9 @@ static byte[] toByteArray(final IOTriFunction+ * The constant is copied from OpenJDK's {@link jdk.internal.util.ArraysSupport#SOFT_MAX_ARRAY_LENGTH}. + *
*/ - private static final int MAX_ARRAY_LENGTH = Integer.MAX_VALUE - 8; + private static final int SOFT_MAX_ARRAY_LENGTH = Integer.MAX_VALUE - 8; /* * Default maximum chunk size used when copying large streams into a byte array. - *- * This value is somewhat arbitrary, currently aligned with the value used by - * Python - * for copying streams. - *
*/ private static final int DEFAULT_CHUNK_SIZE = 128 * 1024; @@ -2666,10 +2664,10 @@ public static BufferedReader toBufferedReader(final Reader reader, final int siz */ public static byte[] toByteArray(final InputStream inputStream) throws IOException { final UnsynchronizedByteArrayOutputStream output = - copyToOutputStream(inputStream, MAX_ARRAY_LENGTH + 1, DEFAULT_CHUNK_SIZE); - if (output.size() > MAX_ARRAY_LENGTH) { + copyToOutputStream(inputStream, SOFT_MAX_ARRAY_LENGTH + 1, DEFAULT_CHUNK_SIZE); + if (output.size() > SOFT_MAX_ARRAY_LENGTH) { throw new IllegalArgumentException( - String.format("Cannot read more than %,d into a byte array", MAX_ARRAY_LENGTH)); + String.format("Cannot read more than %,d into a byte array", SOFT_MAX_ARRAY_LENGTH)); } return output.toByteArray(); } From 29f365be5d485dce7d03f0d0384dbf36fd40cc9d Mon Sep 17 00:00:00 2001 From: "Piotr P. Karwasz"The memory used by this method is proportional to the number - * of bytes read and limited by the specified {@code size}. This makes it suitable for - * processing large input streams, provided that sufficient heap space is - * available.
+ *This variant provides no safeguards against allocating very large arrays. + * For large streams, prefer {@link #toByteArray(InputStream, int, int)}, + * which enforces stricter memory usage constraints.
* * @param input the {@link InputStream} to read; must not be {@code null}. * @param size the exact number of bytes to read; must be {@code >= 0}. @@ -2688,16 +2687,15 @@ public static byte[] toByteArray(final InputStream inputStream) throws IOExcepti * @since 2.1 */ public static byte[] toByteArray(final InputStream input, final int size) throws IOException { - return toByteArray(input, size, DEFAULT_CHUNK_SIZE); + return toByteArray(input::read, size); } /** * Reads exactly {@code size} bytes from the given {@link InputStream} into a new {@code byte[]}. * - *The memory used by this method is proportional to the number - * of bytes read and limited by the specified {@code size}. This makes it suitable for - * processing large input streams, provided that sufficient heap space is - * available.
+ *This variant provides no safeguards against allocating very large arrays. + * For large streams, prefer {@link #toByteArray(InputStream, int, int)}, + * which enforces stricter memory usage constraints.
* * @param input the {@link InputStream} to read; must not be {@code null}. * @param size the exact number of bytes to read; must be {@code >= 0} and {@code <= Integer.MAX_VALUE}. From 4aba097d871332ca1ac537abb5f6880c0bf0ffc3 Mon Sep 17 00:00:00 2001 From: "Piotr P. Karwasz"The memory used by this method is proportional to the number - * of bytes read, which is only limited by {@link Integer#MAX_VALUE}. This makes it unsuitable for - * processing large input streams, unless sufficient heap space is available.
+ * of bytes read, which is only limited by {@link Integer#MAX_VALUE}. Only streams + * which fit into a single byte array with roughly 2 GiB limit can be processed + * with this method. * * @param inputStream The {@link InputStream} to read; must not be {@code null}. * @return A new byte array. @@ -2658,6 +2659,7 @@ public static BufferedReader toBufferedReader(final Reader reader, final int siz * @throws NullPointerException If {@code inputStream} is {@code null}. */ public static byte[] toByteArray(final InputStream inputStream) throws IOException { + // Using SOFT_MAX_ARRAY_LENGTH guarantees that size() will not overflow final UnsynchronizedByteArrayOutputStream output = copyToOutputStream(inputStream, SOFT_MAX_ARRAY_LENGTH + 1, DEFAULT_BUFFER_SIZE); if (output.size() > SOFT_MAX_ARRAY_LENGTH) { throw new IllegalArgumentException(String.format("Cannot read more than %,d into a byte array", SOFT_MAX_ARRAY_LENGTH)); From e6e2a4d6e1dafb74ef8162116183862fdd9f8d70 Mon Sep 17 00:00:00 2001 From: "Piotr P. Karwasz"This variant provides no safeguards against allocating very large arrays. - * For large streams, prefer {@link #toByteArray(InputStream, int, int)}, + *
This variant always allocates the whole requested array size, + * for a dynamic growing variant use {@link #toByteArray(InputStream, int, int)}, * which enforces stricter memory usage constraints.
* * @param input the {@link InputStream} to read; must not be {@code null}. @@ -2690,8 +2690,8 @@ public static byte[] toByteArray(final InputStream input, final int size) throws /** * Reads exactly {@code size} bytes from the given {@link InputStream} into a new {@code byte[]}. * - *This variant provides no safeguards against allocating very large arrays. - * For large streams, prefer {@link #toByteArray(InputStream, int, int)}, + *
This variant always allocates the whole requested array size, + * for a dynamic growing variant use {@link #toByteArray(InputStream, int, int)}, * which enforces stricter memory usage constraints.
* * @param input the {@link InputStream} to read; must not be {@code null}. From 91636d30fad8814a8339624288c40bf133de6b96 Mon Sep 17 00:00:00 2001 From: Gary Gregory- * The constant is copied from OpenJDK's {@link jdk.internal.util.ArraysSupport#SOFT_MAX_ARRAY_LENGTH}. + * The constant is copied from OpenJDK's {@link jdk.internal.util.ArraysSupport#SOFT_MAX_ARRAY_LENGTH}. *
*/ private static final int SOFT_MAX_ARRAY_LENGTH = Integer.MAX_VALUE - 8;