Skip to content

Commit fc28a71

Browse files
sebbASFsebbASF
committed
IO-339 MaxBytesInputStream to limit size of bytes read
git-svn-id: https://svn.apache.org/repos/asf/commons/proper/io/trunk@1483389 13f79535-47bb-0310-9956-ffa450edef68
1 parent 982ecd8 commit fc28a71

3 files changed

Lines changed: 316 additions & 0 deletions

File tree

‎src/changes/changes.xml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,9 @@ The <action> type attribute can be add,update,fix,remove.
4747
<body>
4848
<!-- The release date is the date RC is cut -->
4949
<release version="2.5" date="2013-??-??" description="New features and bug fixes.">
50+
<action issue="IO-339" dev="sebb" type="add" due-to="Ken Weiner">
51+
MaxBytesInputStream to limit size of bytes read
52+
</action>
5053
<action issue="IO-381" dev="ggregory" type="add">
5154
Add FileUtils.copyInputStreamToFile API with option to leave the source open.
5255
See copyInputStreamToFile(final InputStream source, final File destination, boolean closeSource)

‎src/main/java/org/apache/commons/io/input/MaxBytesInputStream.java‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*
17+
*/
18+
19+
package org.apache.commons.io.input;
20+
21+
import java.io.IOException;
22+
import java.io.InputStream;
23+
24+
/**
25+
* An input stream that will end when the amount of bytes read reaches
26+
* the configured number of maxBytes.
27+
* This is useful for preventing OutOfMemoryExceptions when downloading
28+
* very large files in cases where getting partial content is acceptable.
29+
*/
30+
public class MaxBytesInputStream extends CountingInputStream {
31+
private final long maxBytes;
32+
33+
public MaxBytesInputStream(InputStream is, long maxBytes) {
34+
super(is);
35+
this.maxBytes = maxBytes;
36+
}
37+
38+
@Override
39+
public int read() throws IOException {
40+
if (getByteCount() < this.maxBytes) {
41+
return super.read();
42+
}
43+
return -1;
44+
}
45+
46+
@Override
47+
public int read(byte[] b) throws IOException {
48+
return this.read(b, 0, b.length);
49+
}
50+
51+
@Override
52+
public int read(byte[] b, int off, int len) throws IOException {
53+
long remain = this.maxBytes - getByteCount();
54+
if (remain > 0) {
55+
if (remain < len) { // avoid reading past max
56+
return super.read(b, off, (int) remain);
57+
} else {
58+
return super.read(b, off, len);
59+
}
60+
}
61+
return -1;
62+
}
63+
}

‎src/test/java/org/apache/commons/io/input/MaxBytesInputStreamTest.java‎

Lines changed: 250 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,250 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package org.apache.commons.io.input;
18+
19+
import java.io.ByteArrayInputStream;
20+
import java.io.IOException;
21+
import java.io.OutputStream;
22+
23+
import junit.framework.TestCase;
24+
25+
import org.apache.commons.io.IOUtils;
26+
import org.apache.commons.io.output.NullOutputStream;
27+
28+
/**
29+
* Tests the MaxBytesInputStream.
30+
*
31+
* @version $Id$
32+
*/
33+
public class MaxBytesInputStreamTest extends TestCase {
34+
35+
public MaxBytesInputStreamTest(final String name) {
36+
super(name);
37+
}
38+
39+
public void testCounting() throws Exception {
40+
final String text = "A piece of text";
41+
final byte[] bytes = text.getBytes();
42+
final ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
43+
final MaxBytesInputStream cis = new MaxBytesInputStream(bais, text.length());
44+
45+
// have to declare this larger as we're going to read
46+
// off the end of the stream and input stream seems
47+
// to do bounds checking
48+
final byte[] result = new byte[21];
49+
50+
final byte[] ba = new byte[5];
51+
int found = cis.read(ba);
52+
System.arraycopy(ba, 0, result, 0, 5);
53+
assertEquals( found, cis.getCount() );
54+
55+
final int value = cis.read();
56+
found++;
57+
result[5] = (byte)value;
58+
assertEquals( found, cis.getCount() );
59+
60+
found += cis.read(result, 6, 5);
61+
assertEquals( found, cis.getCount() );
62+
63+
found += cis.read(result, 11, 10); // off the end
64+
assertEquals( found, cis.getCount() );
65+
66+
// trim to get rid of the 6 empty values
67+
final String textResult = new String(result).trim();
68+
assertEquals(textResult, text);
69+
cis.close();
70+
}
71+
72+
73+
/**
74+
* Test for files > 2GB in size - see issue IO-84
75+
*/
76+
public void testLargeFiles_IO84() throws Exception {
77+
final long size = (long)Integer.MAX_VALUE + (long)1;
78+
final NullInputStream mock = new NullInputStream(size);
79+
final MaxBytesInputStream cis = new MaxBytesInputStream(mock, size);
80+
final OutputStream out = new NullOutputStream();
81+
82+
// Test integer methods
83+
IOUtils.copyLarge(cis, out);
84+
try {
85+
cis.getCount();
86+
fail("Expected getCount() to throw an ArithmeticException");
87+
} catch (final ArithmeticException ae) {
88+
// expected result
89+
}
90+
try {
91+
cis.resetCount();
92+
fail("Expected resetCount() to throw an ArithmeticException");
93+
} catch (final ArithmeticException ae) {
94+
// expected result
95+
}
96+
97+
mock.close();
98+
99+
// Test long methods
100+
IOUtils.copyLarge(cis, out);
101+
assertEquals("getByteCount()", size, cis.getByteCount());
102+
assertEquals("resetByteCount()", size, cis.resetByteCount());
103+
}
104+
105+
public void testResetting() throws Exception {
106+
final String text = "A piece of text";
107+
final byte[] bytes = text.getBytes();
108+
final ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
109+
final MaxBytesInputStream cis = new MaxBytesInputStream(bais, bytes.length);
110+
111+
final byte[] result = new byte[bytes.length];
112+
113+
int found = cis.read(result, 0, 5);
114+
assertEquals( found, cis.getCount() );
115+
116+
final int count = cis.resetCount();
117+
found = cis.read(result, 6, 5);
118+
assertEquals( found, count );
119+
cis.close();
120+
}
121+
122+
public void testZeroLength1() throws Exception {
123+
final ByteArrayInputStream bais = new ByteArrayInputStream(new byte[0]);
124+
final MaxBytesInputStream cis = new MaxBytesInputStream(bais, 0);
125+
126+
final int found = cis.read();
127+
assertEquals(-1, found);
128+
assertEquals(0, cis.getCount());
129+
cis.close();
130+
}
131+
132+
public void testZeroLength2() throws Exception {
133+
final ByteArrayInputStream bais = new ByteArrayInputStream(new byte[0]);
134+
final MaxBytesInputStream cis = new MaxBytesInputStream(bais, 0);
135+
136+
final byte[] result = new byte[10];
137+
138+
final int found = cis.read(result);
139+
assertEquals(-1, found);
140+
assertEquals(0, cis.getCount());
141+
cis.close();
142+
}
143+
144+
public void testZeroLength3() throws Exception {
145+
final ByteArrayInputStream bais = new ByteArrayInputStream(new byte[0]);
146+
final MaxBytesInputStream cis = new MaxBytesInputStream(bais, 0);
147+
148+
final byte[] result = new byte[10];
149+
150+
final int found = cis.read(result, 0, 5);
151+
assertEquals(-1, found);
152+
assertEquals(0, cis.getCount());
153+
cis.close();
154+
}
155+
156+
public void testEOF1() throws Exception {
157+
final ByteArrayInputStream bais = new ByteArrayInputStream(new byte[2]);
158+
final MaxBytesInputStream cis = new MaxBytesInputStream(bais, 2);
159+
160+
int found = cis.read();
161+
assertEquals(0, found);
162+
assertEquals(1, cis.getCount());
163+
found = cis.read();
164+
assertEquals(0, found);
165+
assertEquals(2, cis.getCount());
166+
found = cis.read();
167+
assertEquals(-1, found);
168+
assertEquals(2, cis.getCount());
169+
cis.close();
170+
}
171+
172+
public void testEOF2() throws Exception {
173+
final ByteArrayInputStream bais = new ByteArrayInputStream(new byte[2]);
174+
final MaxBytesInputStream cis = new MaxBytesInputStream(bais, 2);
175+
176+
final byte[] result = new byte[10];
177+
178+
final int found = cis.read(result);
179+
assertEquals(2, found);
180+
assertEquals(2, cis.getCount());
181+
cis.close();
182+
}
183+
184+
public void testEOF3() throws Exception {
185+
final ByteArrayInputStream bais = new ByteArrayInputStream(new byte[2]);
186+
final MaxBytesInputStream cis = new MaxBytesInputStream(bais, 2);
187+
188+
final byte[] result = new byte[10];
189+
190+
final int found = cis.read(result, 0, 5);
191+
assertEquals(2, found);
192+
assertEquals(2, cis.getCount());
193+
cis.close();
194+
}
195+
196+
public void testMaxZero() throws Exception {
197+
final ByteArrayInputStream bais = new ByteArrayInputStream(new byte[2]);
198+
final MaxBytesInputStream cis = new MaxBytesInputStream(bais, 0);
199+
200+
final byte[] result = new byte[10];
201+
202+
final int found = cis.read(result, 0, 5);
203+
assertEquals(-1, found);
204+
assertEquals(0, cis.getCount());
205+
cis.close();
206+
}
207+
208+
public void testMax1() throws Exception {
209+
final ByteArrayInputStream bais = new ByteArrayInputStream(new byte[20]);
210+
final int max = 4;
211+
final MaxBytesInputStream cis = new MaxBytesInputStream(bais, max);
212+
213+
final byte[] result = new byte[10];
214+
215+
final int found = cis.read(result, 0, 5);
216+
assertEquals(max, found);
217+
assertEquals(max, cis.getCount());
218+
cis.close();
219+
}
220+
221+
public void testMax1b() throws Exception {
222+
final ByteArrayInputStream bais = new ByteArrayInputStream(new byte[20]);
223+
final int max = 4;
224+
final MaxBytesInputStream cis = new MaxBytesInputStream(bais, max);
225+
226+
final byte[] result = new byte[10];
227+
228+
final int found = cis.read(result);
229+
assertEquals(max, found);
230+
assertEquals(max, cis.getCount());
231+
cis.close();
232+
}
233+
234+
public void testSkipping() throws IOException {
235+
final String text = "Hello World!";
236+
final byte[] bytes = text.getBytes();
237+
final ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
238+
final MaxBytesInputStream cis = new MaxBytesInputStream(bais, 100);
239+
240+
assertEquals(6,cis.skip(6));
241+
assertEquals(6,cis.getCount());
242+
final byte[] result = new byte[6];
243+
cis.read(result);
244+
245+
assertEquals("World!",new String(result));
246+
assertEquals(12,cis.getCount());
247+
cis.close();
248+
}
249+
250+
}

0 commit comments

Comments
 (0)