IO-382 Chunked IO for large arrays

sebbASF · sebbASF · commit 0a8ea398409f · 2013-06-04T00:03:06.000Z
git-svn-id: https://svn.apache.org/repos/asf/commons/proper/io/trunk@1489238 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/src/changes/changes.xml b/src/changes/changes.xml
@@ -47,6 +47,11 @@ The <action> type attribute can be add,update,fix,remove.
   <body>
     <!-- The release date is the date RC is cut -->
     <release version="2.5" date="2013-??-??" description="New features and bug fixes.">    
+      <action issue="IO-382" dev="sebb" type="add">
+         Chunked IO for large arrays.
+         Added writeChunked(byte[], OutputStream) and writeChunked(char[] Writer)
+         Added ChunkedOutputStream, ChunkedWriter
+      </action>
       <action issue="IO-385" dev="sebb" type="fix">
          FileUtils.doCopyFile can potentially loop for ever
          Exit loop if no data to copy
diff --git a/src/main/java/org/apache/commons/io/IOUtils.java b/src/main/java/org/apache/commons/io/IOUtils.java
@@ -1339,6 +1339,32 @@ public static void write(final byte[] data, final OutputStream output)
         }
     }
 
+    /**
+     * Writes bytes from a <code>byte[]</code> to an <code>OutputStream</code> using chunked writes.
+     * This is intended for writing very large byte arrays which might otherwise cause excessive
+     * memory usage if the native code has to allocate a copy.
+     *
+     * @param data  the byte array to write, do not modify during output,
+     * null ignored
+     * @param output  the <code>OutputStream</code> to write to
+     * @throws NullPointerException if output is null
+     * @throws IOException if an I/O error occurs
+     * @since 2.5
+     */
+    public static void writeChunked(final byte[] data, final OutputStream output)
+            throws IOException {
+        if (data != null) {
+            int bytes = data.length;
+            int offset = 0;
+            while(bytes > 0) {
+                int chunk = Math.min(bytes, DEFAULT_BUFFER_SIZE);
+                output.write(data, offset, chunk);
+                bytes -= chunk;
+                offset += chunk;
+            }
+        }
+    }
+
     /**
      * Writes bytes from a <code>byte[]</code> to chars on a <code>Writer</code>
      * using the default character encoding of the platform.
@@ -1420,6 +1446,31 @@ public static void write(final char[] data, final Writer output) throws IOExcept
         }
     }
 
+    /**
+     * Writes chars from a <code>char[]</code> to a <code>Writer</code> using chunked writes.
+     * This is intended for writing very large byte arrays which might otherwise cause excessive
+     * memory usage if the native code has to allocate a copy.
+     *
+     * @param data  the char array to write, do not modify during output,
+     * null ignored
+     * @param output  the <code>Writer</code> to write to
+     * @throws NullPointerException if output is null
+     * @throws IOException if an I/O error occurs
+     * @since 2.5
+     */
+    public static void writeChunked(final char[] data, final Writer output) throws IOException {
+        if (data != null) {
+            int bytes = data.length;
+            int offset = 0;
+            while(bytes > 0) {
+                int chunk = Math.min(bytes, DEFAULT_BUFFER_SIZE);
+                output.write(data, offset, chunk);
+                bytes -= chunk;
+                offset += chunk;
+            }
+        }
+    }
+
     /**
      * Writes chars from a <code>char[]</code> to bytes on an
      * <code>OutputStream</code>.
@@ -2825,4 +2876,5 @@ public static void readFully(final ReadableByteChannel input, final ByteBuffer b
             throw new EOFException("Length to read: " + expected + " actual: " + actual);
         }
     }
+    
 }
diff --git a/src/main/java/org/apache/commons/io/output/ChunkedOutputStream.java b/src/main/java/org/apache/commons/io/output/ChunkedOutputStream.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.commons.io.output;
+
+import java.io.FilterOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * OutputStream which breaks larger output blocks into chunks.
+ * Native code may need to copy the input array; if the write buffer
+ * is very large this can cause OOME.
+ *
+ * @since 2.5
+ */
+public class ChunkedOutputStream extends FilterOutputStream {
+
+    /**
+     * The default chunk size to use, i.e. {@value} bytes.
+     */
+    private static final int DEFAULT_CHUNK_SIZE = 1024 * 4;
+
+    /**
+     * The maximum chunk size to us when writing data arrays
+     */
+    private final int chunkSize;
+
+    /**
+     * Creates a new stream that uses the specified chunk size.
+     *
+     * @param stream the stream to wrap
+     * @param chunkSize the chunk size to use; must be a positive number.
+     * @throws IllegalArgumentException if the chunk size is &lt;= 0
+     */
+    public ChunkedOutputStream(final OutputStream stream, int chunkSize) {
+       super(stream);
+       if (chunkSize <= 0) {
+           throw new IllegalArgumentException();
+       }
+       this.chunkSize = chunkSize;
+    }
+
+    /**
+     * Creates a new stream that uses a chunk size of {@link #DEFAULT_CHUNK_SIZE}
+     * @param stream the stream to wrap
+     */
+    public ChunkedOutputStream(final OutputStream stream) {
+        this(stream, DEFAULT_CHUNK_SIZE);
+    }
+
+    /**
+     * writes the data buffer in chunks to the underlying stream
+     */
+    @Override
+    public void write(byte[] data, int srcOffset, int length) throws IOException {
+        int bytes = length;
+        int dstOffset = srcOffset;
+        while(bytes > 0) {
+            int chunk = Math.min(bytes, chunkSize);
+            out.write(data, dstOffset, chunk);
+            bytes -= chunk;
+            dstOffset += chunk;
+        }
+    }
+
+}
diff --git a/src/main/java/org/apache/commons/io/output/ChunkedWriter.java b/src/main/java/org/apache/commons/io/output/ChunkedWriter.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.commons.io.output;
+
+import java.io.FilterWriter;
+import java.io.IOException;
+import java.io.Writer;
+
+/**
+ * OutputStream which breaks larger output blocks into chunks.
+ * Native code may need to copy the input array; if the write buffer
+ * is very large this can cause OOME.
+ *
+ * @since 2.5
+ */
+public class ChunkedWriter extends FilterWriter {
+
+    /**
+     * The default chunk size to use, i.e. {@value} bytes.
+     */
+    private static final int DEFAULT_CHUNK_SIZE = 1024 * 4;
+
+    /**
+     * The maximum chunk size to us when writing data arrays
+     */
+    private final int chunkSize;
+
+    /**
+     * Creates a new writer that uses the specified chunk size.
+     *
+     * @param writer the writer to wrap
+     * @param chunkSize the chunk size to use; must be a positive number.
+     * @throws IllegalArgumentException if the chunk size is &lt;= 0
+     */
+    public ChunkedWriter(final Writer writer, int chunkSize) {
+       super(writer);
+       if (chunkSize <= 0) {
+           throw new IllegalArgumentException();
+       }
+       this.chunkSize = chunkSize;
+    }
+
+    /**
+     * Creates a new writer that uses a chunk size of {@link #DEFAULT_CHUNK_SIZE}
+     * @param writer the writer to wrap
+     */
+    public ChunkedWriter(final Writer writer) {
+        this(writer, DEFAULT_CHUNK_SIZE);
+    }
+
+    /**
+     * writes the data buffer in chunks to the underlying writer
+     */
+    @Override
+    public void write(char[] data, int srcOffset, int length) throws IOException {
+        int bytes = length;
+        int dstOffset = srcOffset;
+        while(bytes > 0) {
+            int chunk = Math.min(bytes, chunkSize);
+            out.write(data, dstOffset, chunk);
+            bytes -= chunk;
+            dstOffset += chunk;
+        }
+    }
+
+}
