diff --git a/src/main/java/org/archive/io/RecordingOutputStream.java b/src/main/java/org/archive/io/RecordingOutputStream.java index 95e444cc..fe05701c 100644 --- a/src/main/java/org/archive/io/RecordingOutputStream.java +++ b/src/main/java/org/archive/io/RecordingOutputStream.java @@ -21,6 +21,7 @@ import it.unimi.dsi.fastutil.io.FastBufferedOutputStream; +import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; @@ -89,12 +90,6 @@ public class RecordingOutputStream extends OutputStream { /** flag to disable recording */ private boolean recording; - /** - * Reusable buffer for FastBufferedOutputStream - */ - protected byte[] bufStreamBuf = - new byte [ FastBufferedOutputStream.DEFAULT_BUFFER_SIZE ]; - /** * True if we're to digest content. */ @@ -206,15 +201,18 @@ public void open(OutputStream wrappedStream) throws IOException { } clearForReuse(); this.out = wrappedStream; + startTime = System.currentTimeMillis(); + } + + protected OutputStream ensureDiskStream() throws FileNotFoundException { if (this.diskStream == null) { - // TODO: Fix so we only make file when its actually needed. FileOutputStream fis = new FileOutputStream(this.backingFilename); - - this.diskStream = new RecyclingFastBufferedOutputStream(fis, bufStreamBuf); + this.diskStream = new FastBufferedOutputStream(fis); } - startTime = System.currentTimeMillis(); + return this.diskStream; } + public void write(int b) throws IOException { if(position= this.buffer.length) { - // TODO: Its possible to call write w/o having first opened a - // stream. Protect ourselves against this. - assert this.diskStream != null: "Diskstream is null"; - this.diskStream.write(b); + this.ensureDiskStream().write(b); } else { this.buffer[(int) this.position] = (byte) b; } @@ -357,12 +352,7 @@ private void record(byte[] b, int off, int len) throws IOException { */ private void tailRecord(byte[] b, int off, int len) throws IOException { if(this.position >= this.buffer.length){ - // TODO: Its possible to call write w/o having first opened a - // stream. Lets protect ourselves against this. - if (this.diskStream == null) { - throw new IOException("diskstream is null"); - } - this.diskStream.write(b, off, len); + this.ensureDiskStream().write(b, off, len); this.position += len; } else { assert this.buffer != null: "Buffer is null"; diff --git a/src/main/java/org/archive/io/RecyclingFastBufferedOutputStream.java b/src/main/java/org/archive/io/RecyclingFastBufferedOutputStream.java deleted file mode 100644 index a3b76e46..00000000 --- a/src/main/java/org/archive/io/RecyclingFastBufferedOutputStream.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.io; - -import it.unimi.dsi.fastutil.io.FastBufferedOutputStream; - -import java.io.OutputStream; - -/** - * FastBufferedOutputStream that accepts a passed-in buffer (avoiding - * reallocation). - */ -public class RecyclingFastBufferedOutputStream extends FastBufferedOutputStream { - public RecyclingFastBufferedOutputStream( final OutputStream os, final byte[] buffer ) { - super(os); - this.buffer = buffer; - avail = buffer.length; - } -} - - diff --git a/src/main/java/org/archive/io/WriterPoolMember.java b/src/main/java/org/archive/io/WriterPoolMember.java index 6ea6b295..85d44e5d 100644 --- a/src/main/java/org/archive/io/WriterPoolMember.java +++ b/src/main/java/org/archive/io/WriterPoolMember.java @@ -19,6 +19,8 @@ package org.archive.io; +import it.unimi.dsi.fastutil.io.FastBufferedOutputStream; + import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -80,9 +82,6 @@ public abstract class WriterPoolMember implements ArchiveFileConstants { /** Counting stream for metering */ protected MiserOutputStream countOut = null; - /** reusable buffer for recycling scenarios */ - protected byte[] rebuf; - protected WriterPoolSettings settings; private final String extension; @@ -209,10 +208,7 @@ protected String createFile(final File file) throws IOException { close(); this.f = file; FileOutputStream fos = new FileOutputStream(this.f); - if(rebuf==null) { - rebuf = new byte[settings.getWriteBufferSize()]; - } - this.countOut = new MiserOutputStream(new RecyclingFastBufferedOutputStream(fos,rebuf),settings.getFrequentFlushes()); + this.countOut = new MiserOutputStream(new FastBufferedOutputStream(fos),settings.getFrequentFlushes()); this.out = this.countOut; logger.fine("Opened " + this.f.getAbsolutePath()); return this.f.getName();