Set in constructor. Should support at least 1 byte mark/reset. + * Make it protected so subclasses have access. + */ + protected InputStream in = null; + + /** + * Maximum amount of recoverable exceptions in a row. + * If more than this amount in a row, we'll let out the exception rather + * than go back in for yet another retry. + */ + public static final int MAX_ALLOWED_RECOVERABLES = 10; + + + /** + * The Record currently being read. + * + * Keep this ongoing reference so we'll close the record even if the caller + * doesn't. + */ + private ArchiveRecord currentRecord = null; + + /** + * Descriptive string for the Archive file we're going against: + * full path, url, etc. -- depends on context in which file was made. + */ + private String identifier = null; + + /** + * Archive file version. + */ + private String version = null; + + + protected ArchiveReader() { + super(); + } + + /** + * Convenience method used by subclass constructors. + * @param i Identifier for Archive file this reader goes against. + */ + protected void initialize(final String i) { + setReaderIdentifier(i); + } + + /** + * Convenience method for constructors. + * + * @param f File to read. + * @param offset Offset at which to start reading. + * @return InputStream to read from. + * @throws IOException If failed open or fail to get a memory + * mapped byte buffer on file. + */ + protected InputStream getInputStream(final File f, final long offset) + throws IOException { + FileInputStream fin = new FileInputStream(f); + return new BufferedInputStream(fin); + } + + public boolean isCompressed() { + return this.compressed; + } + + /** + * Get record at passed offset. + * + * @param offset Byte index into file at which a record starts. + * @return An Archive Record reference. + * @throws IOException + */ + public ArchiveRecord get(long offset) throws IOException { + cleanupCurrentRecord(); + long posn = positionForRecord(in); + if(offset>=posn) { + in.skip(offset-posn); + } else { + throw new UnsupportedOperationException("no reverse seeking: at "+posn+" requested "+offset); + } + return createArchiveRecord(this.in, offset); + } + + /** + * @return Return Archive Record created against current offset. + * @throws IOException + */ + public ArchiveRecord get() throws IOException { + return createArchiveRecord(this.in, positionForRecord(in)); + } + + public void close() throws IOException { + if (this.in != null) { + this.in.close(); + this.in = null; + } + } + + /** + * Cleanout the current record if there is one. + * @throws IOException + */ + protected void cleanupCurrentRecord() throws IOException { + if (this.currentRecord != null) { + this.currentRecord.close(); + gotoEOR(this.currentRecord); + this.currentRecord = null; + } + } + + /** + * Return an Archive Record homed on offset into + * is. + * @param is Stream to read Record from. + * @param offset Offset to find Record at. + * @return ArchiveRecord instance. + * @throws IOException + */ + protected abstract ArchiveRecord createArchiveRecord(InputStream is, + long offset) + throws IOException; + + /** + * Skip over any trailing new lines at end of the record so we're lined up + * ready to read the next. + * @param record + * @throws IOException + */ + protected abstract void gotoEOR(ArchiveRecord record) throws IOException; + + public abstract String getFileExtension(); + public abstract String getDotFileExtension(); + + /** + * @return Version of this Archive file. + */ + public String getVersion() { + return this.version; + } + + /** + * Validate the Archive file. + * + * This method iterates over the file throwing exception if it fails + * to successfully parse any record. + * + *

Assumes the stream is at the start of the file. + * @return List of all read Archive Headers. + * + * @throws IOException + */ + public List validate() throws IOException { + return validate(-1); + } + + /** + * Validate the Archive file. + * + * This method iterates over the file throwing exception if it fails + * to successfully parse. + * + *

We start validation from wherever we are in the stream. + * + * @param numRecords Number of records expected. Pass -1 if number is + * unknown. + * + * @return List of all read metadatas. As we validate records, we add + * a reference to the read metadata. + * + * @throws IOException + */ + public List validate(int numRecords) + throws IOException { + List hdrList = new ArrayList(); + int recordCount = 0; + setStrict(true); + for (Iterator i = iterator(); i.hasNext();) { + recordCount++; + ArchiveRecord r = i.next(); + if (r.getHeader().getLength() <= 0 + && r.getHeader().getMimetype(). + equals(MimetypeUtils.NO_TYPE_MIMETYPE)) { + throw new IOException("record content is empty."); + } + r.close(); + hdrList.add(r.getHeader()); + } + + if (numRecords != -1) { + if (recordCount != numRecords) { + throw new IOException("Count of records, " + + Integer.toString(recordCount) + + " is not equal to expected " + + Integer.toString(numRecords)); + } + } + + return hdrList; + } + + /** + * Test Archive file is valid. + * Assumes the stream is at the start of the file. Be aware that this + * method makes a pass over the whole file. + * @return True if file can be successfully parsed. + */ + public boolean isValid() { + boolean valid = false; + try { + validate(); + valid = true; + } catch(Exception e) { + // File is not valid if exception thrown parsing. + valid = false; + } + + return valid; + } + + /** + * @return Returns the strict. + */ + public boolean isStrict() { + return this.strict; + } + + /** + * @param s The strict to set. + */ + public void setStrict(boolean s) { + this.strict = s; + } + + /** + * @param d True if we're to digest. + */ + public void setDigest(boolean d) { + this.digest = d; + } + + /** + * @return True if we're digesting as we read. + */ + public boolean isDigest() { + return this.digest; + } + + protected Logger getLogger() { + return Logger.getLogger(this.getClass().getName()); + } + + /** + * Returns an ArchiveRecord iterator. + * Of note, on IOException, especially if ZipException reading compressed + * ARCs, rather than fail the iteration, try moving to the next record. + * If {@link ArchiveReader#strict} is not set, this will usually succeed. + * @return An iterator over ARC records. + */ + public Iterator iterator() { + // Eat up any record outstanding. + try { + cleanupCurrentRecord(); + } catch (IOException e) { + throw new RuntimeException(e); + } + + return new ArchiveRecordIterator(); + } + + protected void setCompressed(boolean compressed) { + this.compressed = compressed; + } + + /** + * @return The current ARC record or null if none. + * After construction has the arcfile header record. + * @see #get() + */ + protected ArchiveRecord getCurrentRecord() { + return this.currentRecord; + } + + protected ArchiveRecord currentRecord(final ArchiveRecord r) { + this.currentRecord = r; + return r; + } + + protected InputStream getIn() { + return in; + } + + protected void setIn(InputStream in) { + this.in = in; + } + + protected void setVersion(String version) { + this.version = version; + } + + public String getReaderIdentifier() { + return this.identifier; + } + + protected void setReaderIdentifier(final String i) { + this.identifier = i; + } + + /** + * Log on stderr. + * Logging should go via the logging system. This method + * bypasses the logging system going direct to stderr. + * Should not generally be used. Its used for rare messages + * that come of cmdline usage of ARCReader ERRORs and WARNINGs. + * Override if using ARCReader in a context where no stderr or + * where you'd like to redirect stderr to other than System.err. + * @param level Level to log message at. + * @param message Message to log. + */ + public void logStdErr(Level level, String message) { + System.err.println(level.toString() + " " + message); + } + +// /** +// * Add buffering to RandomAccessInputStream. +// */ +// protected class RandomAccessBufferedInputStream +// extends BufferedInputStream implements RepositionableStream { +// +// public RandomAccessBufferedInputStream(RandomAccessInputStream is) +// throws IOException { +// super(is); +// } +// +// public RandomAccessBufferedInputStream(RandomAccessInputStream is, int size) +// throws IOException { +// super(is, size); +// } +// +// public long position() throws IOException { +// // Current position is the underlying files position +// // minus the amount thats in the buffer yet to be read. +// return ((RandomAccessInputStream)this.in).position() - +// (this.count - this.pos); +// } +// +// public void position(long position) throws IOException { +// // Force refill of buffer whenever there's been a seek. +// this.pos = 0; +// this.count = 0; +// ((RandomAccessInputStream)this.in).position(position); +// } +// +// public int available() throws IOException { +// // Avoid overflow on large datastreams +// long amount = (long)in.available() + (long)(count - pos); +// return (amount >= Integer.MAX_VALUE)? Integer.MAX_VALUE: (int)amount; +// } +// } + + /** + * Inner ArchiveRecord Iterator class. + * Throws RuntimeExceptions in {@link #hasNext()} and {@link #next()} if + * trouble pulling record from underlying stream. + * @author stack + */ + protected class ArchiveRecordIterator implements Iterator { + private final Logger logger = + Logger.getLogger(this.getClass().getName()); + /** + * @return True if we have more records to read. + * @exception RuntimeException Can throw an IOException wrapped in a + * RuntimeException if a problem reading underlying stream (Corrupted + * gzip, etc.). + */ + public boolean hasNext() { + // Call close on any extant record. This will scoot us past + // any content not yet read. + try { + cleanupCurrentRecord(); + } catch (IOException e) { + if (isStrict()) { + throw new RuntimeException(e); + } + if (e instanceof EOFException) { + logger.warning("Premature EOF cleaning up " + + currentRecord.getHeader().toString() + ": " + + e.getMessage()); + return false; + } + // If not strict, try going again. We might be able to skip + // over the bad record. + logger.log(Level.WARNING,"Trying skip of failed record cleanup of " + + currentRecord.getHeader().toString() + ": " + + e.getMessage(), e); + } + return innerHasNext(); + } + + protected boolean innerHasNext(){ + try { + getIn().mark(1); + int c = getIn().read(); + getIn().reset(); + return c > -1; + } catch (IOException e) { + logger.log(Level.WARNING,"problem probing for more content",e); + return false; + } + } + + /** + * Tries to move to next record if we get + * {@link RecoverableIOException}. If not strict + * tries to move to next record if we get an + * {@link IOException}. + * @return Next object. + * @exception RuntimeException Throws a runtime exception, + * usually a wrapping of an IOException, if trouble getting + * a record (Throws exception rather than return null). + */ + public ArchiveRecord next() { + long offset = -1; + try { + offset = positionForRecord(getIn()); + return exceptionNext(); + } catch (IOException e) { + if (!isStrict()) { + // Retry though an IOE. Maybe we will succeed reading + // subsequent record. + try { + if (hasNext()) { + getLogger().warning("Bad Record. Trying skip " + + "(Record start " + offset + "): " + + e.getMessage()); + return exceptionNext(); + } + // Else we are at last record. Iterator#next is + // expecting value. We do not have one. Throw exception. + throw new RuntimeException("Retried but no next " + + "record (Record start " + offset + ")", e); + } catch (IOException e1) { + throw new RuntimeException("After retry (Offset " + + offset + ")", e1); + } + } + throw new RuntimeException("(Record start " + offset + ")", e); + } + } + + /** + * A next that throws exceptions and has handling of + * recoverable exceptions moving us to next record. Can call + * hasNext which itself may throw exceptions. + * @return Next record. + * @throws IOException + * @throws RuntimeException Thrown when we've reached maximum + * retries. + */ + protected ArchiveRecord exceptionNext() + throws IOException, RuntimeException { + ArchiveRecord result = null; + IOException ioe = null; + for (int i = MAX_ALLOWED_RECOVERABLES; i > 0 && + result == null; i--) { + ioe = null; + try { + result = innerNext(); + } catch (RecoverableIOException e) { + ioe = e; + getLogger().warning(e.getMessage()); + if (hasNext()) { + continue; + } + // No records left. Throw exception rather than + // return null. The caller is expecting to get + // back a record since they've just called + // hasNext. + break; + } + } + if (ioe != null) { + // Then we did MAX_ALLOWED_RECOVERABLES retries. Throw + // the recoverable ioe wrapped in a RuntimeException so + // it goes out pass checks for IOE. + throw new RuntimeException("Retried " + + MAX_ALLOWED_RECOVERABLES + " times in a row", ioe); + } + return result; + } + + protected ArchiveRecord innerNext() throws IOException { + return get(positionForRecord(getIn())); + } + + public void remove() { + throw new UnsupportedOperationException(); + } + } + + protected static long positionForRecord(InputStream in) { + return (in instanceof GZIPMembersInputStream) + ? ((GZIPMembersInputStream)in).getCurrentMemberStart() + : ((CountingInputStream)in).getCount(); + } + + protected static String stripExtension(final String name, + final String ext) { + return (!name.endsWith(ext))? name: + name.substring(0, name.length() - ext.length()); + } + + /** + * @return short name of Archive file. + */ + public String getFileName() { + return (new File(getReaderIdentifier())).getName(); + } + + /** + * @return short name of Archive file. + */ + public String getStrippedFileName() { + return getStrippedFileName(getFileName(), + getDotFileExtension()); + } + + /** + * @param name Name of ARCFile. + * @param dotFileExtension '.arc' or '.warc', etc. + * @return short name of Archive file. + */ + public static String getStrippedFileName(String name, + final String dotFileExtension) { + name = stripExtension(name, + ArchiveFileConstants.DOT_COMPRESSED_FILE_EXTENSION); + return stripExtension(name, dotFileExtension); + } + + /** + * @param value Value to test. + * @return True if value is 'true', else false. + */ + protected static boolean getTrueOrFalse(final String value) { + if (value == null || value.length() <= 0) { + return false; + } + return Boolean.TRUE.toString().equals(value.toLowerCase()); + } + + /** + * @param format Format to use outputting. + * @throws IOException + * @throws java.text.ParseException + * @return True if handled. + */ + protected boolean output(final String format) + throws IOException, java.text.ParseException { + boolean result = true; + // long start = System.currentTimeMillis(); + + // Write output as pseudo-CDX file. See + // http://www.archive.org/web/researcher/cdx_legend.php + // and http://www.archive.org/web/researcher/example_cdx.php. + // Hash is hard-coded straight SHA-1 hash of content. + if (format.equals(DUMP)) { + // No point digesting dumping. + setDigest(false); + dump(false); + } else if (format.equals(GZIP_DUMP)) { + // No point digesting dumping. + setDigest(false); + dump(true); + } else if (format.equals(CDX)) { + cdxOutput(false); + } else if (format.equals(CDX_FILE)) { + cdxOutput(true); + } else { + result = false; + } + return result; + } + + protected void cdxOutput(boolean toFile) + throws IOException { + BufferedWriter cdxWriter = null; + if (toFile) { + String cdxFilename = stripExtension(getReaderIdentifier(), + DOT_COMPRESSED_FILE_EXTENSION); + cdxFilename = stripExtension(cdxFilename, getDotFileExtension()); + cdxFilename += ('.' + CDX); + cdxWriter = new BufferedWriter(new FileWriter(cdxFilename)); + } + + String header = "CDX b e a m s c " + ((isCompressed()) ? "V" : "v") + + " n g"; + if (toFile) { + cdxWriter.write(header); + cdxWriter.newLine(); + } else { + System.out.println(header); + } + + String strippedFileName = getStrippedFileName(); + try { + for (Iterator ii = iterator(); ii.hasNext();) { + ArchiveRecord r = ii.next(); + if (toFile) { + cdxWriter.write(r.outputCdx(strippedFileName)); + cdxWriter.newLine(); + } else { + System.out.println(r.outputCdx(strippedFileName)); + } + } + } finally { + if (toFile) { + cdxWriter.close(); + } + } + } + + /** + * Output passed record using passed format specifier. + * @param format What format to use outputting. + * @throws IOException + * @return True if handled. + */ + public boolean outputRecord(final String format) + throws IOException { + boolean result = true; + if (format.equals(CDX)) { + System.out.println(get().outputCdx(getStrippedFileName())); + } else if(format.equals(ArchiveFileConstants.DUMP)) { + // No point digesting if dumping content. + setDigest(false); + get().dump(); + } else { + result = false; + } + return result; + } + + /** + * Dump this file on STDOUT + * @throws compress True if dumped output is compressed. + * @throws IOException + * @throws java.text.ParseException + */ + public abstract void dump(final boolean compress) + throws IOException, java.text.ParseException; + + /** + * @return an ArchiveReader that will delete a local file on close. Used + * when we bring Archive files local and need to clean up afterward. + */ + public abstract ArchiveReader getDeleteFileOnCloseReader(final File f); + + /** + * Output passed record using passed format specifier. + * @param r ARCReader instance to output. + * @param format What format to use outputting. + * @throws IOException + */ + protected static void outputRecord(final ArchiveReader r, + final String format) + throws IOException { + if (!r.outputRecord(format)) { + throw new IOException("Unsupported format" + + " (or unsupported on a single record): " + format); + } + } + + /** + * @return Base Options object filled out with help, digest, strict, etc. + * options. + */ + protected static Options getOptions() { + Options options = new Options(); + options.addOption(new Option("h","help", false, + "Prints this message and exits.")); + options.addOption(new Option("o","offset", true, + "Outputs record at this offset into file.")); + options.addOption(new Option("d","digest", true, + "Pass true|false. Expensive. Default: true (SHA-1).")); + options.addOption(new Option("s","strict", false, + "Strict mode. Fails parse if incorrectly formatted file.")); + options.addOption(new Option("f","format", true, + "Output options: 'cdx', cdxfile', 'dump', 'gzipdump'," + + "'or 'nohead'. Default: 'cdx'.")); + return options; + } +} \ No newline at end of file diff --git a/src/main/java/org/archive/io/ArchiveReaderFactory.java b/src/main/java/org/archive/io/ArchiveReaderFactory.java new file mode 100644 index 00000000..17f14d3a --- /dev/null +++ b/src/main/java/org/archive/io/ArchiveReaderFactory.java @@ -0,0 +1,301 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.io; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLConnection; + +import org.archive.io.arc.ARCReaderFactory; +import org.archive.io.warc.WARCReaderFactory; +import org.archive.net.md5.Md5URLConnection; +import org.archive.net.rsync.RsyncURLConnection; +import org.archive.url.UsableURI; +import org.archive.util.FileUtils; + + +/** + * Factory that returns an Archive file Reader. + * Returns Readers for ARCs or WARCs. + * @author stack + * @version $Date$ $Revision$ + */ +public class ArchiveReaderFactory implements ArchiveFileConstants { + // Static block to enable S3 URLs + static { + if (System.getProperty("java.protocol.handler.pkgs") != null) { + System.setProperty("java.protocol.handler.pkgs", + System.getProperty("java.protocol.handler.pkgs") + + "|" + "org.archive.net"); + } else { + System.setProperty("java.protocol.handler.pkgs", "org.archive.net"); + } + } + + private static final ArchiveReaderFactory factory = + new ArchiveReaderFactory(); + + /** + * Shutdown any public access to default constructor. + */ + protected ArchiveReaderFactory() { + super(); + } + + /** + * Get an Archive file Reader on passed path or url. + * Does primitive heuristic figuring if path or URL. + * @param arcFileOrUrl File path or URL pointing at an Archive file. + * @return An Archive file Reader. + * @throws IOException + * @throws MalformedURLException + * @throws IOException + */ + public static ArchiveReader get(final String arcFileOrUrl) + throws MalformedURLException, IOException { + return ArchiveReaderFactory.factory.getArchiveReader(arcFileOrUrl); + } + + protected ArchiveReader getArchiveReader(final String arcFileOrUrl) + throws MalformedURLException, IOException { + return getArchiveReader(arcFileOrUrl, 0); + } + + protected ArchiveReader getArchiveReader(final String arcFileOrUrl, + final long offset) + throws MalformedURLException, IOException { + return UsableURI.hasScheme(arcFileOrUrl) && arcFileOrUrl.indexOf(":")>1? + get(new URL(arcFileOrUrl), offset): + get(new File(arcFileOrUrl), offset); + } + + /** + * @param f An Archive file to read. + * @return An ArchiveReader + * @throws IOException + */ + public static ArchiveReader get(final File f) throws IOException { + return ArchiveReaderFactory.factory.getArchiveReader(f); + } + + protected ArchiveReader getArchiveReader(final File f) + throws IOException { + return getArchiveReader(f, 0); + } + + /** + * @param f An Archive file to read. + * @param offset Have returned Reader set to start reading at this offset. + * @return An ArchiveReader + * @throws IOException + */ + public static ArchiveReader get(final File f, final long offset) + throws IOException { + return ArchiveReaderFactory.factory.getArchiveReader(f, offset); + } + + protected ArchiveReader getArchiveReader(final File f, + final long offset) + throws IOException { + if (ARCReaderFactory.isARCSuffix(f.getName())) { + return ARCReaderFactory.get(f, true, offset); + } else if (WARCReaderFactory.isWARCSuffix(f.getName())) { + return WARCReaderFactory.get(f, offset); + } + throw new IOException("Unknown file extension (Not ARC nor WARC): " + + f.getName()); + } + + /** + * Wrap a Reader around passed Stream. + * @param s Identifying String for this Stream used in error messages. + * Must be a string that ends with the name of the file we're to put + * an ArchiveReader on. This code looks at file endings to figure + * whether to return an ARC or WARC reader. + * @param is Stream. Stream will be wrapped with implementation of + * RepositionableStream unless already supported. + * @param atFirstRecord Are we at first Record? + * @return ArchiveReader. + * @throws IOException + */ + public static ArchiveReader get(final String s, final InputStream is, + final boolean atFirstRecord) + throws IOException { + return ArchiveReaderFactory.factory.getArchiveReader(s, is, + atFirstRecord); + } + + protected ArchiveReader getArchiveReader(final String id, + final InputStream is, final boolean atFirstRecord) + throws IOException { + final InputStream stream = is; + if (ARCReaderFactory.isARCSuffix(id)) { + return ARCReaderFactory.get(id, stream, atFirstRecord); + } else if (WARCReaderFactory.isWARCSuffix(id)) { + return WARCReaderFactory.get(id, stream, atFirstRecord); + } + throw new IOException("Unknown extension (Not ARC nor WARC): " + id); + } + + /** + * Get an Archive Reader aligned at offset. + * This version of get will not bring the file local but will try to + * stream across the net making an HTTP 1.1 Range request on remote + * http server (RFC1435 Section 14.35). + * @param u HTTP URL for an Archive file. + * @param offset Offset into file at which to start fetching. + * @return An ArchiveReader aligned at offset. + * @throws IOException + */ + public static ArchiveReader get(final URL u, final long offset) + throws IOException { + return ArchiveReaderFactory.factory.getArchiveReader(u, offset); + } + + protected ArchiveReader getArchiveReader(final URL f, final long offset) + throws IOException { + // Get URL connection. + URLConnection connection = f.openConnection(); + if (connection instanceof HttpURLConnection) { + addUserAgent((HttpURLConnection)connection); + } + if (offset != 0) { + // Use a Range request (Assumes HTTP 1.1 on other end). If + // length >= 0, add open-ended range header to the request. Else, + // because end-byte is inclusive, subtract 1. + connection.addRequestProperty("Range", "bytes=" + offset + "-"); + // TODO: should actually verify that server respected 'Range' request + // (spec allows them to ignore; 206 response or Content-Range header + // should be present if Range satisfied; multipart/byteranges could be + // a problem). + } + + return getArchiveReader(f.toString(), connection.getInputStream(), (offset == 0)); + } + + /** + * Get an ARCReader. + * Pulls the ARC local into whereever the System Property + * java.io.tmpdir points. It then hands back an ARCReader that + * points at this local copy. A close on this ARCReader instance will + * remove the local copy. + * @param u An URL that points at an ARC. + * @return An ARCReader. + * @throws IOException + */ + public static ArchiveReader get(final URL u) + throws IOException { + return ArchiveReaderFactory.factory.getArchiveReader(u); + } + + protected ArchiveReader getArchiveReader(final URL u) + throws IOException { + // If url represents a local file then return file it points to. + if (u.getPath() != null) { + // TODO: Add scheme check and host check. + File f = new File(u.getPath()); + if (f.exists()) { + return get(f, 0); + } + } + + String scheme = u.getProtocol(); + if (scheme.startsWith("http") || scheme.equals("s3")) { + // Try streaming if http or s3 URLs rather than copying local + // and then reading (Passing an offset will get us an Reader + // that wraps a Stream). + return get(u, 0); + } + + return makeARCLocal(u.openConnection()); + } + + protected ArchiveReader makeARCLocal(final URLConnection connection) + throws IOException { + File localFile = null; + if (connection instanceof HttpURLConnection) { + // If http url connection, bring down the resource local. + String p = connection.getURL().getPath(); + int index = p.lastIndexOf('/'); + if (index >= 0) { + // Name file for the file we're making local. + localFile = File.createTempFile("",p.substring(index + 1)); + if (localFile.exists()) { + // If file of same name already exists in TMPDIR, then + // clean it up (Assuming only reason a file of same name in + // TMPDIR is because we failed a previous download). + localFile.delete(); + } + } else { + localFile = File.createTempFile(ArchiveReader.class.getName(), + ".tmp"); + } + addUserAgent((HttpURLConnection)connection); + connection.connect(); + try { + FileUtils.readFullyToFile(connection.getInputStream(), localFile); + } catch (IOException ioe) { + localFile.delete(); + throw ioe; + } + } else if (connection instanceof RsyncURLConnection) { + // Then, connect and this will create a local file. + // See implementation of the rsync handler. + connection.connect(); + localFile = ((RsyncURLConnection)connection).getFile(); + } else if (connection instanceof Md5URLConnection) { + // Then, connect and this will create a local file. + // See implementation of the md5 handler. + connection.connect(); + localFile = ((Md5URLConnection)connection).getFile(); + } else { + throw new UnsupportedOperationException("No support for " + + connection); + } + + ArchiveReader reader = null; + try { + reader = get(localFile, 0); + } catch (IOException e) { + localFile.delete(); + throw e; + } + + // Return a delegate that does cleanup of downloaded file on close. + return reader.getDeleteFileOnCloseReader(localFile); + } + + protected void addUserAgent(final HttpURLConnection connection) { + connection.addRequestProperty("User-Agent", this.getClass().getName()); + } + + /** + * @param f File to test. + * @return True if f is compressed. + * @throws IOException + */ + protected boolean isCompressed(final File f) throws IOException { + return f.getName().toLowerCase(). + endsWith(DOT_COMPRESSED_FILE_EXTENSION); + } +} \ No newline at end of file diff --git a/src/main/java/org/archive/io/ArchiveRecord.java b/src/main/java/org/archive/io/ArchiveRecord.java new file mode 100644 index 00000000..63bfe628 --- /dev/null +++ b/src/main/java/org/archive/io/ArchiveRecord.java @@ -0,0 +1,409 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.io; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.logging.Level; + +import org.archive.util.Base32; + +/** + * Archive file Record. + * @author stack + * @version $Date$ $Version$ + */ +public abstract class ArchiveRecord extends InputStream { + + /** + * Minimal http response or request header length. + * + * I've seen in arcs content length of 1 with no header. + */ + protected static final long MIN_HTTP_HEADER_LENGTH = + Math.min("HTTP/1.1 200 OK\r\n".length(), "GET / HTTP/1.0\n\r".length()); + + protected ArchiveRecordHeader header = null; + + /** + * Stream to read this record from. + * + * Stream can only be read sequentially. Will only return this records' + * content returning a -1 if you try to read beyond the end of the current + * record. + * + *

Streams can be markable or not. If they are, we'll be able to roll + * back when we've read too far. If not markable, assumption is that + * the underlying stream is managing our not reading too much (This pertains + * to the skipping over the end of the ARCRecord. See {@link #skip()}. + */ + protected InputStream in = null; + + /** + * Position w/i the Record content, within in. + * This position is relative within this Record. Its not same as the + * Archive file position. + */ + protected long position = 0; + + /** + * Set flag when we've reached the end-of-record. + */ + protected boolean eor = false; + + /** + * Compute digest on what we read and add to metadata when done. + * + * Currently hardcoded as sha-1. TODO: Remove when archive records + * digest or else, add a facility that allows the arc reader to + * compare the calculated digest to that which is recorded in + * the arc. + * + *

Protected instead of private so subclasses can update and complete + * the digest. + */ + protected MessageDigest digest = null; + private String digestStr = null; + + protected boolean strict = false; + + + /** + * Constructor. + * + * @param in Stream cue'd up to be at the start of the record this instance + * is to represent. + * @throws IOException + */ + public ArchiveRecord(InputStream in) + throws IOException { + this(in, null, 0, true, false); + } + + /** + * Constructor. + * + * @param in Stream cue'd up to be at the start of the record this instance + * is to represent. + * @param header Header data. + * @throws IOException + */ + public ArchiveRecord(InputStream in, ArchiveRecordHeader header) + throws IOException { + this(in, header, 0, true, false); + } + + /** + * Constructor. + * + * @param in Stream cue'd up to be at the start of the record this instance + * is to represent. + * @param header Header data. + * @param bodyOffset Offset into the body. Usually 0. + * @param digest True if we're to calculate digest for this record. Not + * digesting saves about ~15% of cpu during an ARC parse. + * @param strict Be strict parsing (Parsing stops if ARC inproperly + * formatted). + * @throws IOException + */ + public ArchiveRecord(InputStream in, ArchiveRecordHeader header, + int bodyOffset, boolean digest, boolean strict) + throws IOException { + this.in = in; + this.header = header; + this.position = bodyOffset; + if (digest) { + try { + this.digest = MessageDigest.getInstance("SHA1"); + } catch (NoSuchAlgorithmException e) { + // Convert to IOE because thats more amenable to callers + // -- they are dealing with it anyways. + throw new IOException(e.getMessage()); + } + } + this.strict = strict; + } + + public boolean markSupported() { + return false; + } + + /** + * @return Header data for this record. + */ + public ArchiveRecordHeader getHeader() { + return this.header; + } + + protected void setHeader(ArchiveRecordHeader header) { + this.header = header; + } + + /** + * Calling close on a record skips us past this record to the next record + * in the stream. + * + * It does not actually close the stream. The underlying steam is probably + * being used by the next arc record. + * + * @throws IOException + */ + public void close() throws IOException { + if (this.in != null) { + skip(); + this.in = null; + if (this.digest != null) { + this.digestStr = Base32.encode(this.digest.digest()); + } + } + } + + /** + * @return Next character in this Record content else -1 if at EOR. + * @throws IOException + */ + public int read() throws IOException { + int c = -1; + if (available() > 0) { + c = this.in.read(); + if (c == -1) { + throw new IOException("Premature EOF before end-of-record."); + } + if (this.digest != null) { + this.digest.update((byte) c); + } + incrementPosition(); + } + return c; + } + + public int read(byte[] b, int offset, int length) throws IOException { + int read = Math.min(length, available()); + if (read == -1 || read == 0) { + read = -1; + } else { + read = this.in.read(b, offset, read); + if (read == -1) { + String msg = "Premature EOF before end-of-record: " + + getHeader().getHeaderFields(); + if (isStrict()) { + throw new IOException(msg); + } + setEor(true); + System.err.println(Level.WARNING.toString() + " " + msg); + } + if (this.digest != null && read >= 0) { + this.digest.update(b, offset, read); + } + incrementPosition(read); + } + return read; + } + + /** + * This available is not the stream's available. Its an available based on + * what the stated Archive record length is minus what we've read to date. + * + * @return True if bytes remaining in record content. + */ + public int available() { + long amount = getHeader().getLength() - getPosition(); + return (amount > Integer.MAX_VALUE? Integer.MAX_VALUE: (int)amount); + } + + /** + * Skip over this records content. + * + * @throws IOException + */ + protected void skip() throws IOException { + if (this.eor) { + return; + } + + // Read to the end of the body of the record. Exhaust the stream. + // Can't skip direct to end because underlying stream may be compressed + // and we're calculating the digest for the record. + int r = available(); + while (r > 0 && !this.eor) { + skip(r); + r = available(); + } + } + + public long skip(long n) throws IOException { + final int SKIP_BUFFERSIZE = 1024 * 4; + byte[] b = new byte[SKIP_BUFFERSIZE]; + long total = 0; + for (int read = 0; (total < n) && (read != -1);) { + read = Math.min(SKIP_BUFFERSIZE, (int) (n - total)); + // TODO: Interesting is that reading from compressed stream, we only + // read about 500 characters at a time though we ask for 4k. + // Look at this sometime. + read = read(b, 0, read); + if (read <= 0) { + read = -1; + } else { + total += read; + } + } + return total; + } + + /** + * @return Returns the strict. + */ + public boolean isStrict() { + return this.strict; + } + + /** + * @param strict The strict to set. + */ + public void setStrict(boolean strict) { + this.strict = strict; + } + + protected InputStream getIn() { + return this.in; + } + + public String getDigestStr() { + return this.digestStr; + } + + protected void incrementPosition() { + this.position++; + } + + protected void incrementPosition(final long incr) { + this.position += incr; + } + + public long getPosition() { + return this.position; + } + + protected boolean isEor() { + return eor; + } + + protected void setEor(boolean eor) { + this.eor = eor; + } + + protected String getStatusCode4Cdx(final ArchiveRecordHeader h) { + return "-"; + } + + protected String getIp4Cdx(final ArchiveRecordHeader h) { + return "-"; + } + + protected String getDigest4Cdx(final ArchiveRecordHeader h) { + return getDigestStr() == null? "-": getDigestStr(); + } + + protected String getMimetype4Cdx(final ArchiveRecordHeader h) { + return h.getMimetype(); + } + + protected String outputCdx(final String strippedFileName) + throws IOException { + // Read the whole record so we get out a hash. Should be safe calling + // close on already closed Record. + close(); + ArchiveRecordHeader h = getHeader(); + StringBuilder buffer = + new StringBuilder(ArchiveFileConstants.CDX_LINE_BUFFER_SIZE); + buffer.append(h.getDate()); + buffer.append(ArchiveFileConstants.SINGLE_SPACE); + buffer.append(getIp4Cdx(h)); + buffer.append(ArchiveFileConstants.SINGLE_SPACE); + buffer.append(h.getUrl()); + buffer.append(ArchiveFileConstants.SINGLE_SPACE); + buffer.append(getMimetype4Cdx(h)); + buffer.append(ArchiveFileConstants.SINGLE_SPACE); + buffer.append(getStatusCode4Cdx(h)); + buffer.append(ArchiveFileConstants.SINGLE_SPACE); + buffer.append(getDigest4Cdx(h)); + buffer.append(ArchiveFileConstants.SINGLE_SPACE); + buffer.append(h.getOffset()); + buffer.append(ArchiveFileConstants.SINGLE_SPACE); + buffer.append(h.getLength()); + buffer.append(ArchiveFileConstants.SINGLE_SPACE); + buffer.append(strippedFileName != null? strippedFileName: '-'); + return buffer.toString(); + } + + /** + * Writes output on STDOUT. + * @throws IOException + */ + public void dump() + throws IOException { + dump(System.out); + } + + /** + * Writes output on passed os. + * @throws IOException + */ + public void dump(final OutputStream os) + throws IOException { + final byte [] outputBuffer = new byte [16*1024]; + int read = outputBuffer.length; + while ((read = read(outputBuffer, 0, outputBuffer.length)) != -1) { + os.write(outputBuffer, 0, read); + } + os.flush(); + } + + /** + * Is it likely that this record contains headers? + * This method will return true if the body is a http response that includes + * http response headers or the body is a http request that includes request + * headers, etc. Be aware that headers in content are distinct from + * {@link ArchiveRecordHeader} 'headers'. + * @return True if this Record's content has headers: + */ + public boolean hasContentHeaders() { + final String url = getHeader().getUrl(); + if (url == null) { + return false; + } + + if (!url.toLowerCase().startsWith("http")) { + return false; + } + + if (getHeader().getLength() <= MIN_HTTP_HEADER_LENGTH) { + return false; + } + + return true; + } + + protected void setBodyOffset(int bodyOffset) { + this.position = bodyOffset; + } +} diff --git a/src/main/java/org/archive/io/ArchiveRecordHeader.java b/src/main/java/org/archive/io/ArchiveRecordHeader.java new file mode 100644 index 00000000..953537b1 --- /dev/null +++ b/src/main/java/org/archive/io/ArchiveRecordHeader.java @@ -0,0 +1,111 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.io; + +import java.util.Map; +import java.util.Set; + +/** + * Archive Record Header. + * @author stack + * @version $Date$ $Version$ + */ +public interface ArchiveRecordHeader { + /** + * Get the time when the record was created. + * @return Date in 14 digit time format (UTC). + * @see org.archive.util.ArchiveUtils#parse14DigitDate(String) + */ + public abstract String getDate(); + + /** + * @return Return length of record. + */ + public abstract long getLength(); + + /** + * @return Return Content-Length of the contents of the record + */ + public abstract long getContentLength(); + + + /** + * @return Record subject-url. + */ + public abstract String getUrl(); + + /** + * @return Record mimetype. + */ + public abstract String getMimetype(); + + /** + * @return Record version. + */ + public abstract String getVersion(); + + /** + * @return Offset into Archive file at which this record begins. + */ + public abstract long getOffset(); + + /** + * @param key Key to use looking up field value. + * @return value for passed key of null if no such entry. + */ + public abstract Object getHeaderValue(final String key); + + /** + * @return Header field name keys. + */ + public abstract Set getHeaderFieldKeys(); + + /** + * @return Map of header fields. + */ + public abstract Map getHeaderFields(); + + /** + * @return Returns identifier for current Archive file. Be aware this + * may not be a file name or file path. It may just be an URL. Depends + * on how Archive file was made. + */ + public abstract String getReaderIdentifier(); + + /** + * @return Identifier for the record. If ARC, the URL + date. If WARC, + * the GUID assigned. + */ + public abstract String getRecordIdentifier(); + + /** + * @return Returns digest as String for this record. Only available after + * the record has been read in totality. + */ + public abstract String getDigest(); + + /** + * Offset at which the content begins. + * For ARCs, its used to delimit where http headers end and content begins. + * For WARCs, its end of Named Fields before payload starts. + */ + public int getContentBegin(); + + public abstract String toString(); +} \ No newline at end of file diff --git a/src/main/java/org/archive/io/ArraySeekInputStream.java b/src/main/java/org/archive/io/ArraySeekInputStream.java new file mode 100644 index 00000000..5b30747e --- /dev/null +++ b/src/main/java/org/archive/io/ArraySeekInputStream.java @@ -0,0 +1,106 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.io; + +import java.io.IOException; + + +/** + * A repositionable stream backed by an array. + * + * @author pjack + */ +public class ArraySeekInputStream extends SeekInputStream { + + + /** + * The array of bytes to read from. + */ + private byte[] array; + + + /** + * The offset in the array of the next byte to read. + */ + private int offset; + + + /** + * Constructor. Note that changes to the given array will be reflected + * in the stream. + * + * @param array The array to read bytes from. + */ + public ArraySeekInputStream(byte[] array) { + this.array = array; + this.offset = 0; + } + + + @Override + public int read() { + if (offset >= array.length) { + return -1; + } + int r = array[offset] & 0xFF; + offset++; + return r; + } + + + @Override + public int read(byte[] buf, int ofs, int len) { + if (offset >= array.length) { + return 0; + } + len = Math.min(len, array.length - offset); + System.arraycopy(array, offset, buf, ofs, len); + offset += len; + return len; + } + + + @Override + public int read(byte[] buf) { + return read(buf, 0, buf.length); + } + + + /** + * Returns the position of the stream. + */ + public long position() { + return offset; + } + + + /** + * Repositions the stream. + * + * @param p the new position for the stream + * @throws IOException if the given position is out of bounds + */ + public void position(long p) throws IOException { + if ((p < 0) || (p > array.length)) { + throw new IOException("Invalid position: " + p); + } + offset = (int)p; + } + +} diff --git a/src/main/java/org/archive/io/BufferedSeekInputStream.java b/src/main/java/org/archive/io/BufferedSeekInputStream.java new file mode 100644 index 00000000..2fdc72b7 --- /dev/null +++ b/src/main/java/org/archive/io/BufferedSeekInputStream.java @@ -0,0 +1,217 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.io; + + +import java.io.IOException; + + +/** + * Buffers data from some other SeekInputStream. + * + * @author pjack + */ +public class BufferedSeekInputStream extends SeekInputStream { + + + /** + * The underlying input stream. + */ + final private SeekInputStream input; + + + /** + * The buffered data. + */ + final private byte[] buffer; + + + /** + * The maximum offset of valid data in the buffer. Usually the same + * as buffer.length, but may be shorter if we're in the last region + * of the stream. + */ + private int maxOffset; + + + /** + * The offset of within the buffer of the next byte to read. + */ + private int offset; + + + /** + * Constructor. + * + * @param input the underlying input stream + * @param capacity the size of the buffer + * @throws IOException if an IO occurs filling the first buffer + */ + public BufferedSeekInputStream(SeekInputStream input, int capacity) + throws IOException { + this.input = input; + this.buffer = new byte[capacity]; + buffer(); + } + + /** + * Fills the buffer. + * + * @throws IOException if an IO error occurs + */ + private void buffer() throws IOException { + int remaining = buffer.length; + while (remaining > 0) { + int r = input.read(buffer, buffer.length - remaining, remaining); + if (r <= 0) { + // Not enough information to fill the buffer + offset = 0; + maxOffset = buffer.length - remaining; + return; + } + remaining -= r; + } + maxOffset = buffer.length; + offset = 0; + } + + + /** + * Ensures that the buffer is valid. + * + * @throws IOException if an IO error occurs + */ + private void ensureBuffer() throws IOException { + if (offset >= maxOffset) { + buffer(); + } + } + + + /** + * Returns the number of unread bytes in the current buffer. + * + * @return the remaining bytes + */ + private int remaining() { + return maxOffset - offset; + } + + + @Override + public int read() throws IOException { + ensureBuffer(); + if (maxOffset == 0) { + return -1; + } + int ch = buffer[offset] & 0xFF; + offset++; + return ch; + } + + + @Override + public int read(byte[] buf, int ofs, int len) throws IOException { + ensureBuffer(); + if (maxOffset == 0) { + return 0; + } + len = Math.min(len, remaining()); + System.arraycopy(buffer, offset, buf, ofs, len); + offset += len; + return len; + } + + + @Override + public int read(byte[] buf) throws IOException { + return read(buf, 0, buf.length); + } + + + @Override + public long skip(long c) throws IOException { + ensureBuffer(); + if (maxOffset == 0) { + return 0; + } + int count = (c > Integer.MAX_VALUE) ? Integer.MAX_VALUE : (int)c; + int skip = Math.min(count, remaining()); + offset += skip; + return skip; + } + + + /** + * Returns the stream's current position. + * + * @return the current position + */ + public long position() throws IOException { + return input.position() - buffer.length + offset; + } + + + /** + * Seeks to the given position. This method avoids re-filling the buffer + * if at all possible. + * + * @param p the position to set + * @throws IOException if an IO error occurs + */ + public void position(long p) throws IOException { + long blockStart = (input.position() - maxOffset) + / buffer.length * buffer.length; + long blockEnd = blockStart + maxOffset; + if ((p >= blockStart) && (p < blockEnd)) { + // Desired position is somewhere inside current buffer + long adj = p - blockStart; + offset = (int)adj; + return; + } + positionDirect(p); + } + + + /** + * Positions the underlying stream at the given position, then refills + * the buffer. + * + * @param p the position to set + * @throws IOException if an IO error occurs + */ + private void positionDirect(long p) throws IOException { + long newBlockStart = p / buffer.length * buffer.length; + input.position(newBlockStart); + buffer(); + offset = (int)(p % buffer.length); + } + + /** + * Close the stream, including the wrapped input stream. + */ + public void close() throws IOException { + super.close(); + if(this.input!=null) { + this.input.close(); + } + } + + +} diff --git a/src/main/java/org/archive/io/CharSubSequence.java b/src/main/java/org/archive/io/CharSubSequence.java new file mode 100644 index 00000000..1e89da56 --- /dev/null +++ b/src/main/java/org/archive/io/CharSubSequence.java @@ -0,0 +1,90 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.io; + + +/** + * Provides a subsequence view onto a CharSequence. + * + * @author gojomo + * @version $Revision$, $Date$ + */ +public class CharSubSequence implements CharSequence { + + protected CharSequence inner; + protected int start; + protected int end; + + public CharSubSequence(CharSequence inner, int start, int end) { + if (end < start) { + throw new IllegalArgumentException("Start " + start + " is > " + + " than end " + end); + } + + if (end < 0 || start < 0) { + throw new IllegalArgumentException("Start " + start + " or end " + + end + " is < 0."); + } + + if (inner == null) { + throw new NullPointerException("Passed charsequence is null."); + } + + this.inner = inner; + this.start = start; + this.end = end; + } + + /* + * (non-Javadoc) + * @see java.lang.CharSequence#length() + */ + public int length() { + return this.end - this.start; + } + + /* + * (non-Javadoc) + * @see java.lang.CharSequence#charAt(int) + */ + public char charAt(int index) { + return this.inner.charAt(this.start + index); + } + + /* + * (non-Javadoc) + * @see java.lang.CharSequence#subSequence(int, int) + */ + public CharSequence subSequence(int begin, int finish) { + return new CharSubSequence(this, begin, finish); + } + + /* + * (non-Javadoc) + * @see java.lang.CharSequence#toString() + */ + public String toString() { + StringBuffer sb = new StringBuffer(length()); + // could use StringBuffer.append(CharSequence) if willing to do 1.5 & up + for (int i = 0;i filenames; + + /* (non-Javadoc) + * @see java.io.InputStream#read() + */ + public int read() throws IOException { + int c = super.read(); + if( c == -1 && filenames.hasNext() ) { + cueStream(); + return read(); + } + return c; + } + /* (non-Javadoc) + * @see java.io.InputStream#read(byte[], int, int) + */ + public int read(byte[] b, int off, int len) throws IOException { + int c = super.read(b, off, len); + if( c == -1 && filenames.hasNext() ) { + cueStream(); + return read(b,off,len); + } + return c; + } + /* (non-Javadoc) + * @see java.io.InputStream#read(byte[]) + */ + public int read(byte[] b) throws IOException { + int c = super.read(b); + if( c == -1 && filenames.hasNext() ) { + cueStream(); + return read(b); + } + return c; + } + + /* (non-Javadoc) + * @see java.io.InputStream#skip(long) + */ + public long skip(long n) throws IOException { + long s = super.skip(n); + if( s files) throws IOException { + super(null); + filenames = files.iterator(); + cueStream(); + } + + private void cueStream() throws IOException { + if(filenames.hasNext()) { + this.in = new FileInputStream(filenames.next()); + } + } + +} diff --git a/src/main/java/org/archive/io/CompositeFileReader.java b/src/main/java/org/archive/io/CompositeFileReader.java new file mode 100644 index 00000000..14b56219 --- /dev/null +++ b/src/main/java/org/archive/io/CompositeFileReader.java @@ -0,0 +1,40 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.io; + +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.List; + + +/** + * @author gojomo + */ +public class CompositeFileReader extends InputStreamReader { + + /** + * @param filenames + * @throws IOException + */ + public CompositeFileReader(List filenames) throws IOException { + super(new CompositeFileInputStream(filenames)); + } + +} diff --git a/src/main/java/org/archive/io/Endian.java b/src/main/java/org/archive/io/Endian.java new file mode 100644 index 00000000..f6d89aaa --- /dev/null +++ b/src/main/java/org/archive/io/Endian.java @@ -0,0 +1,125 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.io; + + +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; + + +/** + * Reads integers stored in big or little endian streams. + * + * @author pjack + */ +public class Endian { + + + /** + * Static utility class. + */ + private Endian() { + } + + + /** + * Reads the next little-endian unsigned 16 bit integer from the + * given stream. + * + * @param input the input stream to read from + * @return the next 16-bit little-endian integer + * @throws IOException if an IO error occurs + */ + public static char littleChar(InputStream input) throws IOException { + int lo = input.read(); + if (lo < 0) { + throw new EOFException(); + } + int hi = input.read(); + if (hi < 0) { + throw new EOFException(); + } + return (char)((hi << 8) | lo); + } + + + /** + * Reads the next little-endian signed 16-bit integer from the + * given stream. + * + * @param input the input stream to read from + * @return the next 16-bit little-endian integer + * @throws IOException if an IO error occurs + */ + public static short littleShort(InputStream input) throws IOException { + return (short)littleChar(input); + } + + + /** + * Reads the next little-endian signed 32-bit integer from the + * given stream. + * + * @param input the input stream to read from + * @return the next 32-bit little-endian integer + * @throws IOException if an IO error occurs + */ + public static int littleInt(InputStream input) throws IOException { + char lo = littleChar(input); + char hi = littleChar(input); + return (hi << 16) | lo; + } + + + /** + * Reads the next big-endian unsigned 16 bit integer from the + * given stream. + * + * @param input the input stream to read from + * @return the next 16-bit big-endian integer + * @throws IOException if an IO error occurs + */ + public static char bigChar(InputStream input) throws IOException { + int hi = input.read(); + if (hi < 0) { + throw new EOFException(); + } + int lo = input.read(); + if (lo < 0) { + throw new EOFException(); + } + return (char)((hi << 8) | lo); + } + + + /** + * Reads the next big-endian signed 32-bit integer from the + * given stream. + * + * @param input the input stream to read from + * @return the next 32-bit big-endian integer + * @throws IOException if an IO error occurs + */ + public static int bigInt(InputStream input) throws IOException { + char hi = bigChar(input); + char lo = bigChar(input); + return (hi << 16) | lo; + } +} diff --git a/src/main/java/org/archive/io/GZIPMembersInputStream.java b/src/main/java/org/archive/io/GZIPMembersInputStream.java new file mode 100644 index 00000000..35fb9e90 --- /dev/null +++ b/src/main/java/org/archive/io/GZIPMembersInputStream.java @@ -0,0 +1,38 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.io; + +import java.io.IOException; +import java.io.InputStream; + +/** + * @deprecated use {@link org.archive.util.zip.GZIPMembersInputStream} + */ +@Deprecated +public class GZIPMembersInputStream extends org.archive.util.zip.GZIPMembersInputStream { + + public GZIPMembersInputStream(InputStream in) throws IOException { + super(in); + } + + public GZIPMembersInputStream(InputStream in, int size) throws IOException { + super(in, size); + } + +} \ No newline at end of file diff --git a/src/main/java/org/archive/io/GenerationFileHandler.java b/src/main/java/org/archive/io/GenerationFileHandler.java new file mode 100644 index 00000000..c1ce8d79 --- /dev/null +++ b/src/main/java/org/archive/io/GenerationFileHandler.java @@ -0,0 +1,200 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.io; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; +import java.util.logging.FileHandler; +import java.util.logging.Formatter; +import java.util.logging.LogRecord; + +import org.archive.util.FileUtils; + + +/** + * FileHandler with support for rotating the current file to + * an archival name with a specified integer suffix, and + * provision of a new replacement FileHandler with the current + * filename. + * + * @author gojomo + */ +public class GenerationFileHandler extends FileHandler { + private LinkedList filenameSeries = new LinkedList(); + private boolean shouldManifest = false; + + /** + * @return Returns the filenameSeries. + */ + public List getFilenameSeries() { + return filenameSeries; + } + + /** + * Constructor. + * @param pattern + * @param append + * @param shouldManifest + * @throws IOException + * @throws SecurityException + */ + public GenerationFileHandler(String pattern, boolean append, + boolean shouldManifest) + throws IOException, SecurityException { + super(pattern, append); + filenameSeries.addFirst(pattern); + this.shouldManifest = shouldManifest; + } + + /** + * @param filenameSeries + * @param shouldManifest + * @throws IOException + */ + public GenerationFileHandler(LinkedList filenameSeries, + boolean shouldManifest) + throws IOException { + super((String)filenameSeries.getFirst(), false); // Never append in this case + this.filenameSeries = filenameSeries; + this.shouldManifest = shouldManifest; + } + + /** + * Move the current file to a new filename with the storeSuffix in place + * of the activeSuffix; continuing logging to a new file under the + * original filename. + * + * @param storeSuffix Suffix to put in place of activeSuffix + * @param activeSuffix Suffix to replace with storeSuffix. + * @return GenerationFileHandler instance. + * @throws IOException + */ + public GenerationFileHandler rotate(String storeSuffix, + String activeSuffix) + throws IOException { + return rotate(storeSuffix, activeSuffix, false); + } + + public GenerationFileHandler rotate(String storeSuffix, + String activeSuffix, boolean mergeOld) throws IOException { + close(); + String filename = (String) filenameSeries.getFirst(); + if (!filename.endsWith(activeSuffix)) { + throw new FileNotFoundException("Active file does not have" + + " expected suffix"); + } + String storeFilename = filename.substring(0, filename.length() + - activeSuffix.length()) + + storeSuffix; + File activeFile = new File(filename); + File storeFile = new File(storeFilename); + FileUtils.moveAsideIfExists(storeFile); + + if (mergeOld) { + File fileToAppendTo = new File(filenameSeries.getLast()); + for (int i = filenameSeries.size() - 2; i >= 0; i--) { + File f = new File(filenameSeries.get(i)); + FileUtils.appendTo(fileToAppendTo, f); + f.delete(); + } + filenameSeries.clear(); + filenameSeries.add(filename); + if (!fileToAppendTo.renameTo(storeFile)) { + throw new IOException("Unable to move " + fileToAppendTo + " to " + + storeFilename); + } + } else { + if (!activeFile.renameTo(storeFile)) { + throw new IOException("Unable to move " + filename + " to " + + storeFilename); + } + } + filenameSeries.add(1, storeFilename); + GenerationFileHandler newGfh = new GenerationFileHandler( + filenameSeries, shouldManifest); + newGfh.setFormatter(this.getFormatter()); + return newGfh; + } + + /** + * @return True if should manifest. + */ + public boolean shouldManifest() { + return this.shouldManifest; + } + + /** + * Constructor-helper that rather than clobbering any existing + * file, moves it aside with a timestamp suffix. + * + * @param filename + * @param append + * @param shouldManifest + * @return + * @throws SecurityException + * @throws IOException + */ + public static GenerationFileHandler makeNew(String filename, boolean append, boolean shouldManifest) throws SecurityException, IOException { + FileUtils.moveAsideIfExists(new File(filename)); + return new GenerationFileHandler(filename, append, shouldManifest); + } + + @Override + public void publish(LogRecord record) { + // when possible preformat outside synchronized superclass method + // (our most involved UriProcessingFormatter can cache result) + Formatter f = getFormatter(); + if(!(f instanceof Preformatter)) { + super.publish(record); + } else { + try { + ((Preformatter)f).preformat(record); + super.publish(record); + } finally { + ((Preformatter)f).clear(); + } + } + } +// +// TODO: determine if there's another way to have this optimization without +// negative impact on log-following (esp. in web UI) +// /** +// * Flush only 1/100th of the usual once-per-record, to reduce the time +// * spent holding the synchronization lock. (Flush is primarily called in +// * a superclass's synchronized publish()). +// * +// * The eventual close calls a direct flush on the target writer, so all +// * rotates/ends will ultimately be fully flushed. +// * +// * @see java.util.logging.StreamHandler#flush() +// */ +// @Override +// public synchronized void flush() { +// flushCount++; +// if(flushCount==100) { +// super.flush(); +// flushCount=0; +// } +// } +// int flushCount; + +} \ No newline at end of file diff --git a/src/main/java/org/archive/io/GenericReplayCharSequence.java b/src/main/java/org/archive/io/GenericReplayCharSequence.java new file mode 100644 index 00000000..1af3922b --- /dev/null +++ b/src/main/java/org/archive/io/GenericReplayCharSequence.java @@ -0,0 +1,412 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.io; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.nio.CharBuffer; +import java.nio.channels.FileChannel; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.Charset; +import java.text.NumberFormat; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.commons.io.IOUtils; +import org.archive.util.DevUtils; + +import com.google.common.base.Charsets; +import com.google.common.primitives.Ints; + +/** + * (Replay)CharSequence view on recorded streams. + * + * For small streams, use {@link InMemoryReplayCharSequence}. + * + *

Call {@link close()} on this class when done to clean up resources. + * + * @contributor stack + * @contributor nlevitt + * @version $Revision$, $Date$ + */ +public class GenericReplayCharSequence implements ReplayCharSequence { + + protected static Logger logger = Logger + .getLogger(GenericReplayCharSequence.class.getName()); + + /** + * Name of the encoding we use writing out concatenated decoded prefix + * buffer and decoded backing file. + * + *

This define is also used as suffix for the file that holds the + * decodings. The name of the file that holds the decoding is the name + * of the backing file w/ this encoding for a suffix. + * + *

See Encoding. + */ + public static final Charset WRITE_ENCODING = Charsets.UTF_16BE; + + private static final long MAP_MAX_BYTES = 64 * 1024 * 1024; // 64M + + /** + * When the memory map moves away from the beginning of the file + * (to the "right") in order to reach a certain index, it will + * map up to this many bytes preceding (to the left of) the target character. + * Consequently it will map up to + * MAP_MAX_BYTES - MAP_TARGET_LEFT_PADDING + * bytes to the right of the target. + */ + private static final long MAP_TARGET_LEFT_PADDING_BYTES = (long) (MAP_MAX_BYTES * 0.01); + + /** + * Total length of character stream to replay minus the HTTP headers + * if present. + * + * If the backing file is larger than Integer.MAX_VALUE (i.e. 2gb), + * only the first Integer.MAX_VALUE characters are available through this API. + * We're overriding java.lang.CharSequence so that we can use + * java.util.regex directly on the data, and the CharSequence + * API uses int for the length and index. + */ + protected int length; + + /** counter of decoding exceptions for report at end */ + protected long decodingExceptions = 0; + protected CharacterCodingException codingException = null; + + /** + * Byte offset into the file where the memory mapped portion begins. + */ + private long mapByteOffset; + + // XXX do we need to keep the input stream around? + private FileInputStream backingFileIn = null; + + private FileChannel backingFileChannel = null; + + private long bytesPerChar; + + private CharBuffer mappedBuffer = null; + + /** + * File that has decoded content. + * + * Keep it around so we can remove on close. + */ + private File decodedFile = null; + + /* + * This portion of the CharSequence precedes what's in the backing file. In + * cases where we decodeToFile(), this is always empty, because we decode + * the entire input stream. + */ + private CharBuffer prefixBuffer = null; + + private boolean isOpen = true; + + protected Charset charset = null; + + /** + * Constructor. + * + * @param contentReplayInputStream inputStream of content + * @param charset Encoding to use reading the passed prefix + * buffer and backing file. Must not be null. + * @param backingFilename Path to backing file with content in excess of + * whats in buffer. + * + * @throws IOException + */ + public GenericReplayCharSequence(InputStream contentReplayInputStream, + int prefixMax, + String backingFilename, + Charset charset) throws IOException { + super(); + logger.fine("characterEncoding=" + charset + " backingFilename=" + + backingFilename); + + if(charset==null) { + charset = ReplayCharSequence.FALLBACK_CHARSET; + } + // decodes only up to Integer.MAX_VALUE characters + decode(contentReplayInputStream, prefixMax, backingFilename, charset); + + this.bytesPerChar = 2; + + if(length>prefixBuffer.position()) { + this.backingFileIn = new FileInputStream(decodedFile); + this.backingFileChannel = backingFileIn.getChannel(); + this.mapByteOffset = 0; + updateMemoryMappedBuffer(); + } + } + + private void updateMemoryMappedBuffer() { + long charLength = (long) this.length() - (long) prefixBuffer.limit(); // in characters + long mapSize = Math.min((charLength * bytesPerChar) - mapByteOffset, MAP_MAX_BYTES); + logger.fine("updateMemoryMappedBuffer: mapOffset=" + + NumberFormat.getInstance().format(mapByteOffset) + + " mapSize=" + NumberFormat.getInstance().format(mapSize)); + try { + // TODO: stress-test without these possibly-costly requests! +// System.gc(); +// System.runFinalization(); + // TODO: Confirm the READ_ONLY works. I recall it not working. + // The buffers seem to always say that the buffer is writable. + mappedBuffer = backingFileChannel.map( + FileChannel.MapMode.READ_ONLY, mapByteOffset, mapSize) + .asReadOnlyBuffer().asCharBuffer(); + } catch (IOException e) { + // TODO convert this to a runtime error? + DevUtils.logger.log(Level.SEVERE, + " backingFileChannel.map() mapByteOffset=" + mapByteOffset + + " mapSize=" + mapSize + "\n" + "decodedFile=" + + decodedFile + " length=" + length + "\n" + + DevUtils.extraInfo(), e); + throw new RuntimeException(e); + } + } + + /** + * Converts the first Integer.MAX_VALUE characters from the + * file backingFilename from encoding encoding to + * encoding WRITE_ENCODING and saves as + * this.decodedFile, which is named backingFilename + * + "." + WRITE_ENCODING. + * + * @throws IOException + */ + protected void decode(InputStream inStream, int prefixMax, + String backingFilename, Charset charset) throws IOException { + + this.charset = charset; + + // TODO: consider if BufferedReader is helping any + // TODO: consider adding TBW 'LimitReader' to stop reading at + // Integer.MAX_VALUE characters because of charAt(int) limit + BufferedReader reader = new BufferedReader(new InputStreamReader( + inStream, charset)); + + logger.fine("backingFilename=" + backingFilename + " encoding=" + + charset + " decodedFile=" + decodedFile); + + this.prefixBuffer = CharBuffer.allocate(prefixMax); + + long count = 0; + while(count < prefixMax) { + int read = reader.read(prefixBuffer); + if(read<0) { + break; + } + count += read; + } + + int ch = reader.read(); + if(ch >= 0) { + count++; + + // more to decode to file overflow + this.decodedFile = new File(backingFilename + "." + WRITE_ENCODING); + + FileOutputStream fos; + try { + fos = new FileOutputStream(this.decodedFile); + } catch (FileNotFoundException e) { + // Windows workaround attempt + System.gc(); + System.runFinalization(); + this.decodedFile = new File(decodedFile.getAbsolutePath()+".win"); + logger.info("Windows 'file with a user-mapped section open' " + + "workaround gc/finalization/name-extension performed."); + // try again + fos = new FileOutputStream(this.decodedFile); + } + + Writer writer = new OutputStreamWriter(fos,WRITE_ENCODING); + writer.write(ch); + count += IOUtils.copyLarge(reader, writer); + writer.close(); + reader.close(); + } + + this.length = Ints.saturatedCast(count); + if(count>Integer.MAX_VALUE) { + logger.warning("input stream is longer than Integer.MAX_VALUE=" + + NumberFormat.getInstance().format(Integer.MAX_VALUE) + + " characters -- only first " + + NumberFormat.getInstance().format(Integer.MAX_VALUE) + + " are accessible through this GenericReplayCharSequence"); + } + + logger.fine("decode: decoded " + count + " characters" + + ((decodedFile==null) ? "" + : " ("+(count-prefixBuffer.length())+" to "+decodedFile+")")); + } + + /** + * Get character at passed absolute position. + * @param index Index into content + * @return Character at offset index. + */ + public char charAt(int index) { + if (index < 0 || index >= this.length()) { + throw new IndexOutOfBoundsException("index=" + index + + " - should be between 0 and length()=" + this.length()); + } + + // is it in the buffer + if (index < prefixBuffer.limit()) { + return prefixBuffer.get(index); + } + + // otherwise we gotta get it from disk via memory map + long charFileIndex = (long) index - (long) prefixBuffer.limit(); + long charFileLength = (long) this.length() - (long) prefixBuffer.limit(); // in characters + if (charFileIndex * bytesPerChar < mapByteOffset) { + logger.log(Level.WARNING,"left-fault; probably don't want to use CharSequence that far backward"); + } + if (charFileIndex * bytesPerChar < mapByteOffset + || charFileIndex - (mapByteOffset / bytesPerChar) >= mappedBuffer.limit()) { + // fault + /* + * mapByteOffset is bounded by 0 and file size +/- size of the map, + * and starts as close to fileIndex - + * MAP_TARGET_LEFT_PADDING_BYTES as it can while also not + * being smaller than it needs to be. + */ + mapByteOffset = Math.min(charFileIndex * bytesPerChar - MAP_TARGET_LEFT_PADDING_BYTES, + charFileLength * bytesPerChar - MAP_MAX_BYTES); + mapByteOffset = Math.max(0, mapByteOffset); + updateMemoryMappedBuffer(); + } + + return mappedBuffer.get((int)(charFileIndex-(mapByteOffset/bytesPerChar))); + } + + public CharSequence subSequence(int start, int end) { + return new CharSubSequence(this, start, end); + } + + private void deleteFile(File fileToDelete) { + deleteFile(fileToDelete, null); + } + + private void deleteFile(File fileToDelete, final Exception e) { + if (e != null) { + // Log why the delete to help with debug of + // java.io.FileNotFoundException: + // ....tt53http.ris.UTF-16BE. + logger.severe("Deleting " + fileToDelete + " because of " + + e.toString()); + } + if (fileToDelete != null && fileToDelete.exists()) { + logger.fine("deleting file: " + fileToDelete); + fileToDelete.delete(); + } + } + + + @Override + public boolean isOpen() { + return this.isOpen; + } + + public void close() throws IOException { + this.isOpen = false; + + logger.fine("closing"); + + if (this.backingFileChannel != null && this.backingFileChannel.isOpen()) { + this.backingFileChannel.close(); + } + if (backingFileIn != null) { + backingFileIn.close(); + } + + deleteFile(this.decodedFile); + + // clear decodedFile -- so that double-close (as in finalize()) won't + // delete a later instance with same name see bug [ 1218961 ] + // "failed get of replay" in ExtractorHTML... usu: UTF-16BE + this.decodedFile = null; + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#finalize() + */ + protected void finalize() throws Throwable { + super.finalize(); + logger.fine("finalizing"); + close(); + } + + /** + * Convenience method for getting a substring. + * + * @deprecated please use subSequence() and then toString() directly + */ + public String substring(int offset, int len) { + return subSequence(offset, offset + len).toString(); + } + + public String toString() { + StringBuilder sb = new StringBuilder(this.length()); + sb.append(this); + return sb.toString(); + } + + public int length() { + return length; + } + + /* (non-Javadoc) + * @see org.archive.io.ReplayCharSequence#getDecodeExceptionCount() + */ + @Override + public long getDecodeExceptionCount() { + return decodingExceptions; + } + + + /* (non-Javadoc) + * @see org.archive.io.ReplayCharSequence#getCodingException() + */ + @Override + public CharacterCodingException getCodingException() { + return codingException; + } + + /* (non-Javadoc) + * @see org.archive.io.ReplayCharSequence#getCharset() + */ + public Charset getCharset() { + return charset; + } +} \ No newline at end of file diff --git a/src/main/java/org/archive/io/GzipHeader.java b/src/main/java/org/archive/io/GzipHeader.java new file mode 100644 index 00000000..6b8263bc --- /dev/null +++ b/src/main/java/org/archive/io/GzipHeader.java @@ -0,0 +1,26 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.io; + +/** + * @deprecated use {@link org.archive.util.zip.GzipHeader} + */ +@Deprecated +public class GzipHeader extends org.archive.util.zip.GzipHeader { +} diff --git a/src/main/java/org/archive/io/HeaderedArchiveRecord.java b/src/main/java/org/archive/io/HeaderedArchiveRecord.java new file mode 100644 index 00000000..3cce595b --- /dev/null +++ b/src/main/java/org/archive/io/HeaderedArchiveRecord.java @@ -0,0 +1,423 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.io; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.PrintStream; + +import org.apache.commons.httpclient.Header; +import org.apache.commons.httpclient.HttpParser; +import org.apache.commons.httpclient.StatusLine; +import org.apache.commons.httpclient.util.EncodingUtil; +import org.archive.io.arc.ARCConstants; +import org.archive.util.LaxHttpParser; + +/** + * An ArchiveRecord whose content has a preamble of RFC822-like headers: e.g. + * The ArchiveRecord is a http response that leads off with http response + * headers. Use this ArchiveRecord Decorator to get at the content headers and + * the header/content demarcation. + * + * @author stack + * @author Olaf Freyer + */ +public class HeaderedArchiveRecord extends ArchiveRecord { + private int contentHeadersLength = -1; + private int statusCode = -1; + + /** + * Http header bytes. + * + * If non-null and bytes available, give out its contents before we + * go back to the underlying stream. + */ + private InputStream contentHeaderStream = null; + + /** + * Content headers. + * + * Only available after the reading of headers. + */ + private Header [] contentHeaders = null; + + + public HeaderedArchiveRecord(final ArchiveRecord ar) throws IOException { + super(ar); + } + + public HeaderedArchiveRecord(final ArchiveRecord ar, + final boolean readContentHeader) throws IOException { + super(ar); + if (readContentHeader) { + this.contentHeaderStream = readContentHeaders(); + } + } + + /** + * Skip over the the content headers if present. + * + * Subsequent reads will get the body. + * + *

Calling this method in the midst of reading the header + * will make for strange results. Otherwise, safe to call + * at any time though before reading any of the record + * content is only time that it makes sense. + * + *

After calling this method, you can call + * {@link #getContentHeaders()} to get the read http header. + * + * @throws IOException + */ + public void skipHttpHeader() throws IOException { + if (this.contentHeaderStream == null) { + return; + } + // Empty the contentHeaderStream + for (int available = this.contentHeaderStream.available(); + this.contentHeaderStream != null + && (available = this.contentHeaderStream.available()) > 0;) { + // We should be in this loop once only we should only do this + // buffer allocation once. + byte[] buffer = new byte[available]; + // The read nulls out httpHeaderStream when done with it so + // need check for null in the loop control line. + read(buffer, 0, available); + } + } + + public void dumpHttpHeader() throws IOException { + dumpHttpHeader(System.out); + } + + public void dumpHttpHeader(final PrintStream stream) throws IOException { + if (this.contentHeaderStream == null) { + return; + } + // Dump the httpHeaderStream to STDOUT + for (int available = this.contentHeaderStream.available(); + this.contentHeaderStream != null + && (available = this.contentHeaderStream.available()) > 0;) { + // We should be in this loop only once and should do this + // buffer allocation once. + byte[] buffer = new byte[available]; + // The read nulls out httpHeaderStream when done with it so + // need check for null in the loop control line. + int read = read(buffer, 0, available); + stream.write(buffer, 0, read); + } + } + + /** + * Read header if present. Technique borrowed from HttpClient HttpParse + * class. Using http parser code for now. Later move to more generic header + * parsing code if there proves a need. + * + * @return ByteArrayInputStream with the http header in it or null if no + * http header. + * @throws IOException + */ + private InputStream readContentHeaders() throws IOException { + // If judged a record that doesn't have an http header, return + // immediately. + if (!hasContentHeaders()) { + return null; + } + byte [] statusBytes = LaxHttpParser.readRawLine(getIn()); + int eolCharCount = getEolCharsCount(statusBytes); + if (eolCharCount <= 0) { + throw new IOException("Failed to read raw lie where one " + + " was expected: " + new String(statusBytes)); + } + String statusLine = EncodingUtil.getString(statusBytes, 0, + statusBytes.length - eolCharCount, ARCConstants.DEFAULT_ENCODING); + if (statusLine == null) { + throw new NullPointerException("Expected status line is null"); + } + // TODO: Tighten up this test. + boolean isHttpResponse = StatusLine.startsWithHTTP(statusLine); + boolean isHttpRequest = false; + if (!isHttpResponse) { + isHttpRequest = statusLine.toUpperCase().startsWith("GET") || + !statusLine.toUpperCase().startsWith("POST"); + } + if (!isHttpResponse && !isHttpRequest) { + throw new UnexpectedStartLineIOException("Failed parse of " + + "status line: " + statusLine); + } + this.statusCode = isHttpResponse? + (new StatusLine(statusLine)).getStatusCode(): -1; + + // Save off all bytes read. Keep them as bytes rather than + // convert to strings so we don't have to worry about encodings + // though this should never be a problem doing http headers since + // its all supposed to be ascii. + ByteArrayOutputStream baos = + new ByteArrayOutputStream(statusBytes.length + 4 * 1024); + baos.write(statusBytes); + + // Now read rest of the header lines looking for the separation + // between header and body. + for (byte [] lineBytes = null; true;) { + lineBytes = LaxHttpParser.readRawLine(getIn()); + eolCharCount = getEolCharsCount(lineBytes); + if (eolCharCount <= 0) { + throw new IOException("Failed reading headers: " + + ((lineBytes != null)? new String(lineBytes): null)); + } + // Save the bytes read. + baos.write(lineBytes); + if ((lineBytes.length - eolCharCount) <= 0) { + // We've finished reading the http header. + break; + } + } + + byte [] headerBytes = baos.toByteArray(); + // Save off where content body, post content headers, starts. + this.contentHeadersLength = headerBytes.length; + ByteArrayInputStream bais = + new ByteArrayInputStream(headerBytes); + if (!bais.markSupported()) { + throw new IOException("ByteArrayInputStream does not support mark"); + } + bais.mark(headerBytes.length); + // Read the status line. Don't let it into the parseHeaders function. + // It doesn't know what to do with it. + bais.read(statusBytes, 0, statusBytes.length); + this.contentHeaders = LaxHttpParser.parseHeaders(bais, + ARCConstants.DEFAULT_ENCODING); + bais.reset(); + return bais; + } + + public static class UnexpectedStartLineIOException + extends RecoverableIOException { + private static final long serialVersionUID = 1L; + + public UnexpectedStartLineIOException(final String reason) { + super(reason); + } + } + + /** + * @param bytes Array of bytes to examine for an EOL. + * @return Count of end-of-line characters or zero if none. + */ + private int getEolCharsCount(byte [] bytes) { + int count = 0; + if (bytes != null && bytes.length >=1 && + bytes[bytes.length - 1] == '\n') { + count++; + if (bytes.length >=2 && bytes[bytes.length -2] == '\r') { + count++; + } + } + return count; + } + + /** + * @return If headers are for a http response AND the headers have been + * read, return status code. Else return -1. + */ + public int getStatusCode() { + return this.statusCode; + } + + /** + * @return Returns length of content headers or -1 if headers have + * not yet been read. + */ + public int getContentHeadersLength() { + return this.contentHeadersLength; + } + + public Header[] getContentHeaders() { + return contentHeaders; + } + + /** + * @return Next character in this ARCRecord's content else -1 if at end of + * this record. + * @throws IOException + */ + public int read() throws IOException { + int c = -1; + if (this.contentHeaderStream != null && + (this.contentHeaderStream.available() > 0)) { + // If http header, return bytes from it before we go to underlying + // stream. + c = this.contentHeaderStream.read(); + // If done with the header stream, null it out. + if (this.contentHeaderStream.available() <= 0) { + this.contentHeaderStream = null; + } + // do not increment position - + // the underlying ArchiveRecord stream allready did this + // incrementPosition(); + } else { + c = super.read(); + } + return c; + } + + public int read(byte [] b, int offset, int length) throws IOException { + int read = -1; + if (this.contentHeaderStream != null && + (this.contentHeaderStream.available() > 0)) { + // If http header, return bytes from it before we go to underlying + // stream. + read = Math.min(length, this.contentHeaderStream.available()); + if (read == 0) { + read = -1; + } else { + read = this.contentHeaderStream.read(b, offset, read); + } + // If done with the header stream, null it out. + if (this.contentHeaderStream.available() <= 0) { + this.contentHeaderStream = null; + } + // do not increment position - + // the underlying ArchiveRecord stream allready did this + //incrementPosition(); + } else { + read = super.read(b, offset, length); + } + return read; + } + + @Override + public int available() { + return ((ArchiveRecord)this.in).available(); + } + + @Override + public void close() throws IOException { + ((ArchiveRecord)this.in).close(); + } + + @Override + public void dump() throws IOException { + ((ArchiveRecord)this.in).dump(); + } + + @Override + public void dump(OutputStream os) throws IOException { + ((ArchiveRecord)this.in).dump(os); + } + + @Override + protected String getDigest4Cdx(ArchiveRecordHeader h) { + return ((ArchiveRecord)this.in).getDigest4Cdx(h); + } + + @Override + public String getDigestStr() { + return ((ArchiveRecord)this.in).getDigestStr(); + } + + @Override + public ArchiveRecordHeader getHeader() { + return ((ArchiveRecord)this.in).getHeader(); + } + + @Override + protected String getIp4Cdx(ArchiveRecordHeader h) { + return ((ArchiveRecord)this.in).getIp4Cdx(h); + } + + @Override + protected String getMimetype4Cdx(ArchiveRecordHeader h) { + return ((ArchiveRecord)this.in).getMimetype4Cdx(h); + } + + @Override + public long getPosition() { + return ((ArchiveRecord)this.in).getPosition(); + } + + @Override + protected String getStatusCode4Cdx(ArchiveRecordHeader h) { + return ((ArchiveRecord)this.in).getStatusCode4Cdx(h); + } + + @Override + public boolean hasContentHeaders() { + return ((ArchiveRecord)this.in).hasContentHeaders(); + } + + @Override + protected void incrementPosition() { + ((ArchiveRecord)this.in).incrementPosition(); + } + + @Override + protected void incrementPosition(long incr) { + ((ArchiveRecord)this.in).incrementPosition(incr); + } + + @Override + protected boolean isEor() { + return ((ArchiveRecord)this.in).isEor(); + } + + @Override + public boolean isStrict() { + return ((ArchiveRecord)this.in).isStrict(); + } + + @Override + public boolean markSupported() { + return ((ArchiveRecord)this.in).markSupported(); + } + + @Override + protected String outputCdx(String strippedFileName) throws IOException { + return ((ArchiveRecord)this.in).outputCdx(strippedFileName); + } + + @Override + protected void setEor(boolean eor) { + ((ArchiveRecord)this.in).setEor(eor); + } + + @Override + protected void setHeader(ArchiveRecordHeader header) { + ((ArchiveRecord)this.in).setHeader(header); + } + + @Override + public void setStrict(boolean strict) { + ((ArchiveRecord)this.in).setStrict(strict); + } + + @Override + protected void skip() throws IOException { + ((ArchiveRecord)this.in).skip(); + } + + @Override + public long skip(long n) throws IOException { + return ((ArchiveRecord)this.in).skip(n); + } +} diff --git a/src/main/java/org/archive/io/LoudObjectOutputStream.java b/src/main/java/org/archive/io/LoudObjectOutputStream.java new file mode 100644 index 00000000..959c2620 --- /dev/null +++ b/src/main/java/org/archive/io/LoudObjectOutputStream.java @@ -0,0 +1,63 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.io; + +import java.io.IOException; +import java.io.ObjectOutputStream; +import java.io.OutputStream; +import java.util.HashSet; +import java.util.Set; +import java.util.logging.Logger; + +/** + * ObjectOutputStream that logs class name of each object that is written + * to the stream. Useful for tracking down sources of NotSerializableException. + * + * @author pjack + * + */ +public class LoudObjectOutputStream extends ObjectOutputStream { + + + final private static Logger LOGGER = Logger.getLogger( + LoudObjectOutputStream.class.getName()); + + // Only log each class name once + private Set alreadyLogged = new HashSet(); + + public LoudObjectOutputStream(OutputStream out) throws IOException { + super(out); + this.enableReplaceObject(true); + } + + + @Override + protected Object replaceObject(Object obj) throws IOException { + if (obj != null) { + String name = obj.getClass().getName(); + if (alreadyLogged.add(name)) { + LOGGER.info("WROTE: " + name); + } + } + return obj; + } + + +} diff --git a/src/main/java/org/archive/io/MiserOutputStream.java b/src/main/java/org/archive/io/MiserOutputStream.java new file mode 100644 index 00000000..f10ac9ca --- /dev/null +++ b/src/main/java/org/archive/io/MiserOutputStream.java @@ -0,0 +1,82 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.io; + +import java.io.FilterOutputStream; +import java.io.IOException; +import java.io.OutputStream; + +/** + * A filter stream that both counts bytes written, and optionally swallows + * flush() requests. + * + * @contributor gojomo + */ +public class MiserOutputStream extends FilterOutputStream { + protected long count; + protected boolean passFlushes; + + /** + * Wraps another output stream, counting the number of bytes written. + * + * @param out the output stream to be wrapped + */ + public MiserOutputStream(OutputStream out) { + this(out,true); + } + + /** + * Wraps another output stream, counting the number of bytes written. + * + * @param out the output stream to be wrapped + */ + public MiserOutputStream(OutputStream out, boolean passFlushes) { + super(out); + this.passFlushes = passFlushes; + } + + /** Returns the number of bytes written. */ + public long getCount() { + return count; + } + + @Override public void write(byte[] b, int off, int len) throws IOException { + out.write(b, off, len); + count += len; + } + + @Override public void write(int b) throws IOException { + out.write(b); + count++; + } + + @Override + public void close() throws IOException { + passFlushes = true; + super.close(); + } + + @Override + public void flush() throws IOException { + if(passFlushes) { + super.flush(); + } + } +} diff --git a/src/main/java/org/archive/io/NoGzipMagicException.java b/src/main/java/org/archive/io/NoGzipMagicException.java new file mode 100644 index 00000000..27d1058a --- /dev/null +++ b/src/main/java/org/archive/io/NoGzipMagicException.java @@ -0,0 +1,26 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.io; + +/** + * @deprecated use {@link org.archive.util.zip.NoGzipMagicException} + */ +@Deprecated +public class NoGzipMagicException extends org.archive.util.zip.NoGzipMagicException { +} diff --git a/src/main/java/org/archive/io/ObjectPlusFilesInputStream.java b/src/main/java/org/archive/io/ObjectPlusFilesInputStream.java new file mode 100644 index 00000000..892860ed --- /dev/null +++ b/src/main/java/org/archive/io/ObjectPlusFilesInputStream.java @@ -0,0 +1,143 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.io; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.ObjectInputStream; +import java.util.Iterator; +import java.util.LinkedList; + +import org.archive.util.FileUtils; + + +/** + * Enhanced ObjectOutputStream with support for restoring + * files that had been saved, in parallel with object + * serialization. + * + * @author gojomo + * + */ +public class ObjectPlusFilesInputStream extends ObjectInputStream { + protected LinkedList auxiliaryDirectoryStack = new LinkedList(); + protected LinkedList postRestoreTasks = new LinkedList(); + + /** + * Instantiate over the given stream and using the supplied + * auxiliary storage directory. + * + * @param in + * @param storeDir + * @throws IOException + */ + public ObjectPlusFilesInputStream(InputStream in, File storeDir) + throws IOException { + super(in); + auxiliaryDirectoryStack.addFirst(storeDir); + } + + /** + * Push another default storage directory for use + * until popped. + * + * @param dir + */ + public void pushAuxiliaryDirectory(String dir) { + auxiliaryDirectoryStack. + addFirst(new File(getAuxiliaryDirectory(), dir)); + } + + /** + * Discard the top auxiliary directory. + */ + public void popAuxiliaryDirectory() { + auxiliaryDirectoryStack.removeFirst(); + } + + /** + * Return the top auxiliary directory, from + * which saved files are restored. + * + * @return Auxillary directory. + */ + public File getAuxiliaryDirectory() { + return (File)auxiliaryDirectoryStack.getFirst(); + } + + /** + * Restore a file from storage, using the name and length + * info on the serialization stream and the file from the + * current auxiliary directory, to the given File. + * + * @param destination + * @throws IOException + */ + public void restoreFile(File destination) throws IOException { + String nameAsStored = readUTF(); + long lengthAtStoreTime = readLong(); + File storedFile = new File(getAuxiliaryDirectory(),nameAsStored); + FileUtils.copyFile(storedFile, destination, lengthAtStoreTime); + } + + /** + * Restore a file from storage, using the name and length + * info on the serialization stream and the file from the + * current auxiliary directory, to the given File. + * + * @param directory + * @throws IOException + */ + public void restoreFileTo(File directory) throws IOException { + String nameAsStored = readUTF(); + long lengthAtStoreTime = readLong(); + File storedFile = new File(getAuxiliaryDirectory(),nameAsStored); + File destination = new File(directory,nameAsStored); + FileUtils.copyFile(storedFile, destination, lengthAtStoreTime); + } + + /** + * Register a task to be done when the ObjectPlusFilesInputStream + * is closed. + * + * @param task + */ + public void registerFinishTask(Runnable task) { + postRestoreTasks.addFirst(task); + } + + private void doFinishTasks() { + Iterator iter = postRestoreTasks.iterator(); + while(iter.hasNext()) { + ((Runnable)iter.next()).run(); + } + } + + /** + * In addition to default, do any registered cleanup tasks. + * + * @see java.io.InputStream#close() + */ + public void close() throws IOException { + super.close(); + doFinishTasks(); + } +} diff --git a/src/main/java/org/archive/io/ObjectPlusFilesOutputStream.java b/src/main/java/org/archive/io/ObjectPlusFilesOutputStream.java new file mode 100644 index 00000000..224f24e7 --- /dev/null +++ b/src/main/java/org/archive/io/ObjectPlusFilesOutputStream.java @@ -0,0 +1,134 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.io; + +import java.io.File; +import java.io.IOException; +import java.io.ObjectOutputStream; +import java.io.OutputStream; +import java.util.LinkedList; + +import org.archive.util.FileUtils; + + +/** + * Enhanced ObjectOutputStream which maintains (a stack of) auxiliary + * directories and offers convenience methods for serialized objects + * to save their related disk files alongside their serialized version. + * + * @author gojomo + */ +public class ObjectPlusFilesOutputStream extends ObjectOutputStream { + protected LinkedList auxiliaryDirectoryStack = new LinkedList(); + + /** + * Constructor + * + * @param out + * @param topDirectory + * @throws java.io.IOException + */ + public ObjectPlusFilesOutputStream(OutputStream out, File topDirectory) throws IOException { + super(out); + auxiliaryDirectoryStack.addFirst(topDirectory); + } + + /** + * Add another subdirectory for any file-capture needs during the + * current serialization. + * + * @param dir + */ + public void pushAuxiliaryDirectory(String dir) { + auxiliaryDirectoryStack.addFirst(new File(getAuxiliaryDirectory(),dir)); + } + + /** + * Remove the top subdirectory. + * + */ + public void popAuxiliaryDirectory() { + auxiliaryDirectoryStack.removeFirst(); + } + + /** + * Return the current auxiliary directory for storing + * files associated with serialized objects. + * + * @return Auxillary directory. + */ + public File getAuxiliaryDirectory() { + return (File)auxiliaryDirectoryStack.getFirst(); + } + + /** + * Store a snapshot of an object's supporting file to the + * current auxiliary directory. Should only be used for + * files which are strictly appended-to, because it tries + * to use a "hard link" where possible (meaning that + * future edits to the original file's contents will + * also affect the snapshot). + * + * Remembers current file extent to allow a future restore + * to ignore subsequent appended data. + * + * @param file + * @throws IOException + */ + public void snapshotAppendOnlyFile(File file) throws IOException { + // write filename + String name = file.getName(); + writeUTF(name); + // write current file length + writeLong(file.length()); + File auxDir = getAuxiliaryDirectory(); + if(!auxDir.exists()) { + FileUtils.ensureWriteableDirectory(auxDir); + } + File destination = new File(auxDir,name); + hardlinkOrCopy(file, destination); + } + + /** + * Create a backup of this given file, first by trying a "hard + * link", then by using a copy if hard linking is unavailable + * (either because it is unsupported or the origin and checkpoint + * directories are on different volumes). + * + * @param file + * @param destination + * @throws IOException + */ + private void hardlinkOrCopy(File file, File destination) throws IOException { + // For Linux/UNIX, try a hard link first. + Process link = Runtime.getRuntime().exec("ln "+file.getAbsolutePath()+" "+destination.getAbsolutePath()); + // TODO NTFS also supports hard links; add appropriate try + try { + link.waitFor(); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + if(link.exitValue()!=0) { + // hard link failed + FileUtils.copyFile(file,destination); + } + } + +} diff --git a/src/main/java/org/archive/io/OriginSeekInputStream.java b/src/main/java/org/archive/io/OriginSeekInputStream.java new file mode 100644 index 00000000..00605d82 --- /dev/null +++ b/src/main/java/org/archive/io/OriginSeekInputStream.java @@ -0,0 +1,121 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.io; + + +import java.io.IOException; + + +/** + * Alters the origin of some other SeekInputStream. This class allows you + * to completely ignore everything in the underlying stream before a specified + * position, the origin position. + * + *

With the exception of {@link #position()} and {@link position(long)}, + * all of the methods in this class simply delegate to the underlying input + * stream. The position methods adjust the position of the + * underlying stream relative to the origin specified at construction time. + * + * @author pjack + */ +public class OriginSeekInputStream extends SeekInputStream { + + + /** + * The underlying stream. + */ + final private SeekInputStream input; + + + /** + * The origin position. In other words, this.position(0) + * resolves to input.position(start). + */ + final private long origin; + + + /** + * Constructor. + * + * @param input the underlying stream + * @param origin the origin position + * @throws IOException if an IO error occurs + */ + public OriginSeekInputStream(SeekInputStream input, long origin) + throws IOException { + this.input = input; + this.origin = origin; + input.position(origin); + } + + + @Override + public int available() throws IOException { + return input.available(); + } + + + @Override + public int read() throws IOException { + return input.read(); + } + + + @Override + public int read(byte[] buf, int ofs, int len) throws IOException { + return input.read(buf, ofs, len); + } + + + @Override + public int read(byte[] buf) throws IOException { + return input.read(buf); + } + + + @Override + public long skip(long count) throws IOException { + return input.skip(count); + } + + + /** + * Returns the position of the underlying stream relative to the origin. + * + * @return the relative position + * @throws IOException if an IO error occurs + */ + public long position() throws IOException { + return input.position() - origin; + } + + + /** + * Positions the underlying stream relative to the origin. + * In other words, this.position(0) resolves to input.position(origin), + * where input is underlying stream and origin is the origin specified + * at construction time. + * + * @param p the new position for this stream + * @throws IOException if an IO error occurs + */ + public void position(long p) throws IOException { + input.position(p + origin); + } +} diff --git a/src/main/java/org/archive/io/Preformatter.java b/src/main/java/org/archive/io/Preformatter.java new file mode 100644 index 00000000..dcd31bb6 --- /dev/null +++ b/src/main/java/org/archive/io/Preformatter.java @@ -0,0 +1,32 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.io; + +import java.util.logging.LogRecord; + +/** + * Interface indicating a logging Formatter can preformat a record (outside + * the standard-implementation synchronized block) and cache it, returning it + * for the next request for formatting from the same thread. + * @contributor gojomo + */ +public interface Preformatter { + public void preformat(LogRecord record); + public void clear(); +} diff --git a/src/main/java/org/archive/io/RandomAccessInputStream.java b/src/main/java/org/archive/io/RandomAccessInputStream.java new file mode 100644 index 00000000..d8dd260b --- /dev/null +++ b/src/main/java/org/archive/io/RandomAccessInputStream.java @@ -0,0 +1,180 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.io; + + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; + + +/** + * Wraps a RandomAccessFile with an InputStream interface. + * + * @author gojomo + */ +public class RandomAccessInputStream extends SeekInputStream { + + /** + * Reference to the random access file this stream is reading from. + */ + private RandomAccessFile raf = null; + + /** + * When mark is called, save here the current position so we can go back + * on reset. + */ + private long markpos = -1; + + /** + * True if we are to close the underlying random access file when this + * stream is closed. + */ + private boolean sympathyClose; + + /** + * Constructor. + * + * If using this constructor, caller created the RAF and therefore + * its assumed wants to control close of the RAF. The RAF.close + * is not called if this constructor is used on close of this stream. + * + * @param raf RandomAccessFile to wrap. + * @throws IOException + */ + public RandomAccessInputStream(RandomAccessFile raf) + throws IOException { + this(raf, false, 0); + } + + /** + * Constructor. + * + * @param file File to get RAFIS on. Creates an RAF from passed file. + * Closes the created RAF when this stream is closed. + * @throws IOException + */ + public RandomAccessInputStream(final File file) + throws IOException { + this(new RandomAccessFile(file, "r"), true, 0); + } + + /** + * Constructor. + * + * @param file File to get RAFIS on. Creates an RAF from passed file. + * Closes the created RAF when this stream is closed. + * @param offset + * @throws IOException + */ + public RandomAccessInputStream(final File file, final long offset) + throws IOException { + this(new RandomAccessFile(file, "r"), true, offset); + } + + /** + * @param raf RandomAccessFile to wrap. + * @param sympathyClose Set to true if we are to close the RAF + * file when this stream is closed. + * @param offset + * @throws IOException + */ + public RandomAccessInputStream(final RandomAccessFile raf, + final boolean sympathyClose, final long offset) + throws IOException { + super(); + this.sympathyClose = sympathyClose; + this.raf = raf; + if (offset > 0) { + this.raf.seek(offset); + } + } + + /* (non-Javadoc) + * @see java.io.InputStream#read() + */ + public int read() throws IOException { + return this.raf.read(); + } + + /* (non-Javadoc) + * @see java.io.InputStream#read(byte[], int, int) + */ + public int read(byte[] b, int off, int len) throws IOException { + return this.raf.read(b, off, len); + } + + /* (non-Javadoc) + * @see java.io.InputStream#read(byte[]) + */ + public int read(byte[] b) throws IOException { + return this.raf.read(b); + } + + /* (non-Javadoc) + * @see java.io.InputStream#skip(long) + */ + public long skip(long n) throws IOException { + this.raf.seek(this.raf.getFilePointer() + n); + return n; + } + + public long position() throws IOException { + return this.raf.getFilePointer(); + } + + public void position(long position) throws IOException { + this.raf.seek(position); + } + + public int available() throws IOException { + long amount = this.raf.length() - this.position(); + return (amount >= Integer.MAX_VALUE)? Integer.MAX_VALUE: (int)amount; + } + + public boolean markSupported() { + return true; + } + + public synchronized void mark(int readlimit) { + try { + this.markpos = position(); + } catch (IOException e) { + // Set markpos to -1. Will cause exception reset. + this.markpos = -1; + } + } + + public synchronized void reset() throws IOException { + if (this.markpos == -1) { + throw new IOException("Mark has not been set."); + } + position(this.markpos); + } + + public void close() throws IOException { + try { + super.close(); + } finally { + if (this.sympathyClose) { + this.raf.close(); + } + } + } +} \ No newline at end of file diff --git a/src/main/java/org/archive/io/RandomAccessOutputStream.java b/src/main/java/org/archive/io/RandomAccessOutputStream.java new file mode 100644 index 00000000..225f995f --- /dev/null +++ b/src/main/java/org/archive/io/RandomAccessOutputStream.java @@ -0,0 +1,69 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.io; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.RandomAccessFile; + + +/** + * Wraps a RandomAccessFile with OutputStream interface. + * + * @author gojomo + */ +public class RandomAccessOutputStream extends OutputStream { + protected RandomAccessFile raf; + + /** + * Wrap the given RandomAccessFile + */ + public RandomAccessOutputStream(RandomAccessFile raf) { + super(); + this.raf = raf; + } + + /* (non-Javadoc) + * @see java.io.OutputStream#write(int) + */ + public void write(int b) throws IOException { + raf.write(b); + } + + /* (non-Javadoc) + * @see java.io.OutputStream#close() + */ + public void close() throws IOException { + raf.close(); + } + + /* (non-Javadoc) + * @see java.io.OutputStream#write(byte[], int, int) + */ + public void write(byte[] b, int off, int len) throws IOException { + raf.write(b, off, len); + } + + /* (non-Javadoc) + * @see java.io.OutputStream#write(byte[]) + */ + public void write(byte[] b) throws IOException { + raf.write(b); + } +} diff --git a/src/main/java/org/archive/io/ReadSource.java b/src/main/java/org/archive/io/ReadSource.java new file mode 100644 index 00000000..a3c29967 --- /dev/null +++ b/src/main/java/org/archive/io/ReadSource.java @@ -0,0 +1,37 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.io; + +import java.io.Reader; + +/** + * Interface for objects that can provide a Reader view of their + * contents. + * + */ +public interface ReadSource { + /** + * Obtain a Reader. Not named 'getReader' so that it is not + * considered a simple costless read-only property by + * bean-convention introspection tools. + * @return a Reader on this object + */ + Reader obtainReader(); +} diff --git a/src/main/java/org/archive/io/RecorderIOException.java b/src/main/java/org/archive/io/RecorderIOException.java new file mode 100644 index 00000000..07b30061 --- /dev/null +++ b/src/main/java/org/archive/io/RecorderIOException.java @@ -0,0 +1,38 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.io; + +import java.io.IOException; + +/** + * + * @author Gordon Mohr + */ +public class RecorderIOException extends IOException { + + private static final long serialVersionUID = 5907470275350314277L; + + public RecorderIOException() { + super(); + } + + public RecorderIOException(String msg) { + super(msg); + } +} diff --git a/src/main/java/org/archive/io/RecorderLengthExceededException.java b/src/main/java/org/archive/io/RecorderLengthExceededException.java new file mode 100644 index 00000000..8c3e067d --- /dev/null +++ b/src/main/java/org/archive/io/RecorderLengthExceededException.java @@ -0,0 +1,39 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.io; + + +/** + * Indicates a length exception thrown by the Recorder. + * + * @author Gordon Mohr + */ +public class RecorderLengthExceededException +extends RecorderIOException { + + private static final long serialVersionUID = 6655419033414648444L; + + public RecorderLengthExceededException() { + super(); + } + + public RecorderLengthExceededException(String msg) { + super(msg); + } +} diff --git a/src/main/java/org/archive/io/RecorderTimeoutException.java b/src/main/java/org/archive/io/RecorderTimeoutException.java new file mode 100644 index 00000000..32be5b5d --- /dev/null +++ b/src/main/java/org/archive/io/RecorderTimeoutException.java @@ -0,0 +1,37 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.io; + +/** + * Indicates a timeout thrown by the RecordingInputStream. + * + * @author Gordon Mohr + */ +public class RecorderTimeoutException extends RecorderIOException { + + private static final long serialVersionUID = 7433214063765078269L; + + public RecorderTimeoutException() { + super(); + } + + public RecorderTimeoutException(String msg) { + super(msg); + } +} diff --git a/src/main/java/org/archive/io/RecorderTooMuchHeaderException.java b/src/main/java/org/archive/io/RecorderTooMuchHeaderException.java new file mode 100644 index 00000000..23f5d264 --- /dev/null +++ b/src/main/java/org/archive/io/RecorderTooMuchHeaderException.java @@ -0,0 +1,40 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.io; + + +/** + * Indicates a too much header material exception thrown by the Recorder + * (specificially the RecordingOutputStream) + * + * @author Gordon Mohr + */ +public class RecorderTooMuchHeaderException +extends RecorderIOException { + + private static final long serialVersionUID = 3528516034898129150L; + + public RecorderTooMuchHeaderException() { + super(); + } + + public RecorderTooMuchHeaderException(String msg) { + super(msg); + } +} diff --git a/src/main/java/org/archive/io/RecordingInputStream.java b/src/main/java/org/archive/io/RecordingInputStream.java new file mode 100644 index 00000000..b46905ed --- /dev/null +++ b/src/main/java/org/archive/io/RecordingInputStream.java @@ -0,0 +1,355 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.io; + +import java.io.IOException; +import java.io.InputStream; +import java.net.SocketException; +import java.net.SocketTimeoutException; +import java.security.MessageDigest; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.commons.io.IOUtils; + + +/** + * Stream which records all data read from it, which it acquires from a wrapped + * input stream. + * + * Makes use of a RecordingOutputStream for recording because of its being + * file backed so we can write massive amounts of data w/o worrying about + * overflowing memory. + * + * @author gojomo + * + */ +public class RecordingInputStream + extends InputStream { + + protected static Logger logger = + Logger.getLogger("org.archive.io.RecordingInputStream"); + + /** + * Where we are recording to. + */ + private RecordingOutputStream recordingOutputStream; + + /** + * Stream to record. + */ + private InputStream in = null; + + /** + * Reusable buffer to avoid reallocation on each readFullyUntil + */ + protected byte[] drainBuffer = new byte[16*1024]; + + /** + * Create a new RecordingInputStream. + * + * @param bufferSize Size of buffer to use. + * @param backingFilename Name of backing file. + */ + public RecordingInputStream(int bufferSize, String backingFilename) + { + this.recordingOutputStream = new RecordingOutputStream(bufferSize, + backingFilename); + } + + public void open(InputStream wrappedStream) throws IOException { + logger.fine(Thread.currentThread().getName() + " opening " + + wrappedStream + ", " + Thread.currentThread().getName()); + if(isOpen()) { + // error; should not be opening/wrapping in an unclosed + // stream remains open + throw new IOException("RIS already open for " + +Thread.currentThread().getName()); + } + try { + this.in = wrappedStream; + this.recordingOutputStream.open(); + } catch (IOException ioe) { + close(); // ...and rethrow... + throw ioe; + } + } + + public int read() throws IOException { + if (!isOpen()) { + throw new IOException("Stream closed " + + Thread.currentThread().getName()); + } + int b = this.in.read(); + if (b != -1) { + assert this.recordingOutputStream != null: "ROS is null " + + Thread.currentThread().getName(); + this.recordingOutputStream.write(b); + } + return b; + } + + public int read(byte[] b, int off, int len) throws IOException { + if (!isOpen()) { + throw new IOException("Stream closed " + + Thread.currentThread().getName()); + } + int count = this.in.read(b,off,len); + if (count > 0) { + assert this.recordingOutputStream != null: "ROS is null " + + Thread.currentThread().getName(); + this.recordingOutputStream.write(b,off,count); + } + return count; + } + + public int read(byte[] b) throws IOException { + if (!isOpen()) { + throw new IOException("Stream closed " + + Thread.currentThread().getName()); + } + int count = this.in.read(b); + if (count > 0) { + assert this.recordingOutputStream != null: "ROS is null " + + Thread.currentThread().getName(); + this.recordingOutputStream.write(b,0,count); + } + return count; + } + + public void close() throws IOException { + if (logger.isLoggable(Level.FINE)) { + logger.fine(Thread.currentThread().getName() + " closing " + + this.in + ", " + Thread.currentThread().getName()); + } + IOUtils.closeQuietly(this.in); + this.in = null; + IOUtils.closeQuietly(this.recordingOutputStream); + } + + public ReplayInputStream getReplayInputStream() throws IOException { + return this.recordingOutputStream.getReplayInputStream(); + } + + public ReplayInputStream getMessageBodyReplayInputStream() throws IOException { + return this.recordingOutputStream.getMessageBodyReplayInputStream(); + } + + public long readFully() throws IOException { + while(read(drainBuffer) != -1) { + // Empty out stream. + continue; + } + return this.recordingOutputStream.getSize(); + } + + /** + * Read all of a stream (Or read until we timeout or have read to the max). + * @param softMaxLength Maximum length to read; if zero or < 0, then no + * limit. If met, return normally. + * @param hardMaxLength Maximum length to read; if zero or < 0, then no + * limit. If exceeded, throw RecorderLengthExceededException + * @param timeout Timeout in milliseconds for total read; if zero or + * negative, timeout is Long.MAX_VALUE. If exceeded, throw + * RecorderTimeoutException + * @param maxBytesPerMs How many bytes per millisecond. + * @throws IOException failed read. + * @throws RecorderLengthExceededException + * @throws RecorderTimeoutException + * @throws InterruptedException + */ + public void readFullyOrUntil(long softMaxLength) + throws IOException, RecorderLengthExceededException, + RecorderTimeoutException, InterruptedException { + // Check we're open before proceeding. + if (!isOpen()) { + // TODO: should this be a noisier exception-raising error? + return; + } + + long totalBytes = 0L; + long bytesRead = -1L; + long maxToRead = -1; + while (true) { + try { + // read no more than soft max + maxToRead = (softMaxLength <= 0) + ? drainBuffer.length + : Math.min(drainBuffer.length, softMaxLength - totalBytes); + // nor more than hard max + maxToRead = Math.min(maxToRead, recordingOutputStream.getRemainingLength()); + // but always at least 1 (to trigger hard max exception + maxToRead = Math.max(maxToRead, 1); + + bytesRead = read(drainBuffer,0,(int)maxToRead); + if (bytesRead == -1) { + break; + } + totalBytes += bytesRead; + + if (Thread.interrupted()) { + throw new InterruptedException("Interrupted during IO"); + } + } catch (SocketTimeoutException e) { + // A socket timeout is just a transient problem, meaning + // nothing was available in the configured timeout period, + // but something else might become available later. + // Take this opportunity to check the overall + // timeout (below). One reason for this timeout is + // servers that keep up the connection, 'keep-alive', even + // though we asked them to not keep the connection open. + if (logger.isLoggable(Level.FINE)) { + logger.log(Level.FINE, "socket timeout", e); + } + // check for interrupt + if (Thread.interrupted()) { + throw new InterruptedException("Interrupted during IO"); + } + // check for overall timeout + recordingOutputStream.checkLimits(); + } catch (SocketException se) { + throw se; + } catch (NullPointerException e) { + // [ 896757 ] NPEs in Andy's Th-Fri Crawl. + // A crawl was showing NPE's in this part of the code but can + // not reproduce. Adding this rethrowing catch block w/ + // diagnostics to help should we come across the problem in the + // future. + throw new NullPointerException("Stream " + this.in + ", " + + e.getMessage() + " " + Thread.currentThread().getName()); + } + + // if have read 'enough', just finish + if (softMaxLength > 0 && totalBytes >= softMaxLength) { + break; // return + } + } + } + + public long getSize() { + return this.recordingOutputStream.getSize(); + } + + public void markContentBegin() { + this.recordingOutputStream.markMessageBodyBegin(); + } + + public long getContentBegin() { + return this.recordingOutputStream.getMessageBodyBegin(); + } + + public void startDigest() { + this.recordingOutputStream.startDigest(); + } + + /** + * Convenience method for setting SHA1 digest. + */ + public void setSha1Digest() { + this.recordingOutputStream.setSha1Digest(); + } + + /** + * Sets a digest algorithm which may be applied to recorded data. + * As usually only a subset of the recorded data should + * be fed to the digest, you must also call startDigest() + * to begin digesting. + * + * @param algorithm + */ + public void setDigest(String algorithm) { + this.recordingOutputStream.setDigest(algorithm); + } + + /** + * Sets a digest function which may be applied to recorded data. + * As usually only a subset of the recorded data should + * be fed to the digest, you must also call startDigest() + * to begin digesting. + * + * @param md + */ + public void setDigest(MessageDigest md) { + this.recordingOutputStream.setDigest(md); + } + + /** + * Return the digest value for any recorded, digested data. Call + * only after all data has been recorded; otherwise, the running + * digest state is ruined. + * + * @return the digest final value + */ + public byte[] getDigestValue() { + return this.recordingOutputStream.getDigestValue(); + } + + public long getResponseContentLength() { + return this.recordingOutputStream.getResponseContentLength(); + } + + public void closeRecorder() throws IOException { + this.recordingOutputStream.closeRecorder(); + } + + /** + * @return True if we've been opened. + */ + public boolean isOpen() + { + return this.in != null; + } + + @Override + public synchronized void mark(int readlimit) { + this.in.mark(readlimit); + this.recordingOutputStream.mark(); + } + + @Override + public boolean markSupported() { + return this.in.markSupported(); + } + + @Override + public synchronized void reset() throws IOException { + this.in.reset(); + this.recordingOutputStream.reset(); + } + + /** + * Set limits to be enforced by internal recording-out + */ + public void setLimits(long hardMax, long timeoutMs, long maxRateKBps) { + recordingOutputStream.setLimits(hardMax, timeoutMs, maxRateKBps); + } + + /** + * Expose the amount of in-memory buffering used by the internal + * recording stream. + * @return int buffer size + */ + public int getRecordedBufferLength() { + return recordingOutputStream.getBufferLength(); + } + + public void clearForReuse() throws IOException { + recordingOutputStream.clearForReuse(); + } +} diff --git a/src/main/java/org/archive/io/RecordingOutputStream.java b/src/main/java/org/archive/io/RecordingOutputStream.java new file mode 100644 index 00000000..4d0713da --- /dev/null +++ b/src/main/java/org/archive/io/RecordingOutputStream.java @@ -0,0 +1,576 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.io; + +import it.unimi.dsi.fastutil.io.FastBufferedOutputStream; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.logging.Level; +import java.util.logging.Logger; + + +/** + * An output stream that records all writes to wrapped output + * stream. + * + * A RecordingOutputStream can be wrapped around any other + * OutputStream to record all bytes written to it. You can + * then request a ReplayInputStream to read those bytes. + * + *

The RecordingOutputStream uses an in-memory buffer and + * backing disk file to allow it to record streams of + * arbitrary length limited only by available disk space. + * + *

As long as the stream recorded is smaller than the + * in-memory buffer, no disk access will occur. + * + *

Recorded content can be recovered as a ReplayInputStream + * (via getReplayInputStream() or, for only the content after + * the content-begin-mark is set, getContentReplayInputStream() ) + * or as a ReplayCharSequence (via getReplayCharSequence()). + * + *

This class is also used as a straight output stream + * by {@link RecordingInputStream} to which it records all reads. + * {@link RecordingInputStream} is exploiting the file backed buffer + * facility of this class passing null for the stream + * to wrap. TODO: Make a FileBackedOutputStream class that is + * subclassed by RecordingInputStream. + * + * @author gojomo + * + */ +public class RecordingOutputStream extends OutputStream { + protected static Logger logger = + Logger.getLogger(RecordingOutputStream.class.getName()); + + /** + * Size of recording. + * + * Later passed to ReplayInputStream on creation. It uses it to know when + * EOS. + */ + protected long size = 0; + + protected String backingFilename; + protected OutputStream diskStream = null; + + /** + * Buffer we write recordings to. + * + * We write all recordings here first till its full. Thereafter we + * write the backing file. + */ + private byte[] buffer; + + /** current virtual position in the recording */ + private long position; + + /** flag to disable recording */ + private boolean recording; + + /** + * Reusable buffer for FastBufferedOutputStream + */ + protected byte[] bufStreamBuf = + new byte [ FastBufferedOutputStream.DEFAULT_BUFFER_SIZE ]; + + /** + * True if we're to digest content. + */ + private boolean shouldDigest = false; + + /** + * Digest instance. + */ + private MessageDigest digest = null; + + /** + * Define for SHA1 algarithm. + */ + private static final String SHA1 = "SHA1"; + + /** + * Maximum amount of header material to accept without the content + * body beginning -- if more, throw a RecorderTooMuchHeaderException. + * TODO: make configurable? make smaller? + */ + protected static final long MAX_HEADER_MATERIAL = 1024*1024; // 1MB + + // configurable max length, max time limits + /** maximum length of material to record before throwing exception */ + protected long maxLength = Long.MAX_VALUE; + /** maximum time to record before throwing exception */ + protected long timeoutMs = Long.MAX_VALUE; + /** maximum rate to record (adds delays to hit target rate) */ + protected long maxRateBytesPerMs = Long.MAX_VALUE; + /** time recording begins for timeout, rate calculations */ + protected long startTime = Long.MAX_VALUE; + + /** + * When recording HTTP, where the content-body starts. + */ + protected long messageBodyBeginMark; + + /** + * Stream to record. + */ + private OutputStream out = null; + + // mark/reset support + /** furthest position reached before any reset()s */ + private long maxPosition = 0; + /** remembered position to reset() to */ + private long markPosition = 0; + + /** + * Create a new RecordingOutputStream. + * + * @param bufferSize Buffer size to use. + * @param backingFilename Name of backing file to use. + */ + public RecordingOutputStream(int bufferSize, String backingFilename) { + this.buffer = new byte[bufferSize]; + this.backingFilename = backingFilename; + recording = true; + } + + /** + * Wrap the given stream, both recording and passing along any data written + * to this RecordingOutputStream. + * + * @throws IOException If failed creation of backing file. + */ + public void open() throws IOException { + this.open(null); + } + + /** + * Wrap the given stream, both recording and passing along any data written + * to this RecordingOutputStream. + * + * @param wrappedStream Stream to wrap. May be null for case where we + * want to write to a file backed stream only. + * + * @throws IOException If failed creation of backing file. + */ + public void open(OutputStream wrappedStream) throws IOException { + if(isOpen()) { + // error; should not be opening/wrapping in an unclosed + // stream remains open + throw new IOException("ROS already open for " + +Thread.currentThread().getName()); + } + clearForReuse(); + this.out = wrappedStream; + if (this.diskStream == null) { + // TODO: Fix so we only make file when its actually needed. + FileOutputStream fis = new FileOutputStream(this.backingFilename); + + this.diskStream = new RecyclingFastBufferedOutputStream(fis, bufStreamBuf); + } + startTime = System.currentTimeMillis(); + } + + public void write(int b) throws IOException { + if(position< maxPosition) { + if(position+len<=maxPosition) { + // revisiting; do nothing but advance position + position += len; + return; + } + // consume part of the array doing nothing but advancing position + long consumeRange = maxPosition - position; + position += consumeRange; + off += consumeRange; + len -= consumeRange; + } + if(recording) { + record(b, off, len); + } + if (this.out != null) { + this.out.write(b, off, len); + } + checkLimits(); + } + + /** + * Check any enforced limits. + */ + protected void checkLimits() throws RecorderIOException { + // too much material before finding end of headers? + if (messageBodyBeginMark<0) { + // no mark yet + if(position>MAX_HEADER_MATERIAL) { + throw new RecorderTooMuchHeaderException(); + } + } + // overlong? + if(position>maxLength) { + throw new RecorderLengthExceededException(); + } + // taking too long? + long duration = System.currentTimeMillis() - startTime; + duration = Math.max(duration,1); // !divzero + if(duration>timeoutMs) { + throw new RecorderTimeoutException(); + } + // need to throttle reading to hit max configured rate? + if(position/duration > maxRateBytesPerMs) { + long desiredDuration = position / maxRateBytesPerMs; + try { + Thread.sleep(desiredDuration-duration); + } catch (InterruptedException e) { + logger.log(Level.WARNING, + "bandwidth throttling sleep interrupted", e); + } + } + } + + /** + * Record the given byte for later recovery + * + * @param b Int to record. + * + * @exception IOException Failed write to backing file. + */ + private void record(int b) throws IOException { + if (this.shouldDigest) { + this.digest.update((byte)b); + } + if (this.position >= this.buffer.length) { + // TODO: Its possible to call write w/o having first opened a + // stream. Protect ourselves against this. + assert this.diskStream != null: "Diskstream is null"; + this.diskStream.write(b); + } else { + this.buffer[(int) this.position] = (byte) b; + } + this.position++; + } + + /** + * Record the given byte-array range for recovery later + * + * @param b Buffer to record. + * @param off Offset into buffer at which to start recording. + * @param len Length of buffer to record. + * + * @exception IOException Failed write to backing file. + */ + private void record(byte[] b, int off, int len) throws IOException { + if(this.shouldDigest) { + assert this.digest != null: "Digest is null."; + this.digest.update(b, off, len); + } + tailRecord(b, off, len); + } + + /** + * Record without digesting. + * + * @param b Buffer to record. + * @param off Offset into buffer at which to start recording. + * @param len Length of buffer to record. + * + * @exception IOException Failed write to backing file. + */ + private void tailRecord(byte[] b, int off, int len) throws IOException { + if(this.position >= this.buffer.length){ + // TODO: Its possible to call write w/o having first opened a + // stream. Lets protect ourselves against this. + if (this.diskStream == null) { + throw new IOException("diskstream is null"); + } + this.diskStream.write(b, off, len); + this.position += len; + } else { + assert this.buffer != null: "Buffer is null"; + int toCopy = (int)Math.min(this.buffer.length - this.position, len); + assert b != null: "Passed buffer is null"; + System.arraycopy(b, off, this.buffer, (int)this.position, toCopy); + this.position += toCopy; + // TODO verify these are +1 -1 right + if (toCopy < len) { + tailRecord(b, off + toCopy, len - toCopy); + } + } + } + + public void close() throws IOException { + if(messageBodyBeginMark<0) { + // if unset, consider 0 posn as content-start + // (so that a -1 never survives to replay step) + messageBodyBeginMark = 0; + } + if (this.out != null) { + this.out.close(); + this.out = null; + } + closeRecorder(); + } + + protected synchronized void closeDiskStream() + throws IOException { + if (this.diskStream != null) { + this.diskStream.close(); + this.diskStream = null; + } + } + + public void closeRecorder() throws IOException { + recording = false; + closeDiskStream(); // if any + // This setting of size is important. Its passed to ReplayInputStream + // on creation. It uses it to know EOS. + if (this.size == 0) { + this.size = this.position; + } + } + + /* (non-Javadoc) + * @see java.io.OutputStream#flush() + */ + public void flush() throws IOException { + if (this.out != null) { + this.out.flush(); + } + if (this.diskStream != null) { + this.diskStream.flush(); + } + } + + public ReplayInputStream getReplayInputStream() throws IOException { + return getReplayInputStream(0); + } + + public ReplayInputStream getReplayInputStream(long skip) throws IOException { + // If this method is being called, then assumption must be that the + // stream is closed. If it ain't, then the stream gotten won't work + // -- the size will zero so any attempt at a read will get back EOF. + assert this.out == null: "Stream is still open."; + ReplayInputStream replay = new ReplayInputStream(this.buffer, + this.size, this.messageBodyBeginMark, this.backingFilename); + replay.skip(skip); + return replay; + } + + /** + * Return a replay stream, cued up to begining of content + * + * @throws IOException + * @return An RIS. + */ + public ReplayInputStream getMessageBodyReplayInputStream() throws IOException { + return getReplayInputStream(this.messageBodyBeginMark); + } + + public long getSize() { + return this.size; + } + + /** + * Remember the current position as the start of the "message + * body". Useful when recording HTTP traffic as a way to start + * replays after the headers. + */ + public void markMessageBodyBegin() { + this.messageBodyBeginMark = this.position; + startDigest(); + } + + /** + * Return stored message-body-begin-mark (which is also end-of-headers) + */ + public long getMessageBodyBegin() { + return this.messageBodyBeginMark; + } + + /** + * Starts digesting recorded data, if a MessageDigest has been + * set. + */ + public void startDigest() { + if (this.digest != null) { + this.digest.reset(); + this.shouldDigest = true; + } + } + + /** + * Convenience method for setting SHA1 digest. + * @see #setDigest(String) + */ + public void setSha1Digest() { + setDigest(SHA1); + } + + + /** + * Sets a digest function which may be applied to recorded data. + * The difference between calling this method and {@link #setDigest(MessageDigest)} + * is that this method tries to reuse MethodDigest instance if already allocated + * and of appropriate algorithm. + * @param algorithm Message digest algorithm to use. + * @see #setDigest(MessageDigest) + */ + public void setDigest(String algorithm) { + try { + // Reuse extant digest if its sha1 algorithm. + if (this.digest == null || + !this.digest.getAlgorithm().equals(algorithm)) { + setDigest(MessageDigest.getInstance(algorithm)); + } + } catch (NoSuchAlgorithmException e) { + e.printStackTrace(); + } + } + + /** + * Sets a digest function which may be applied to recorded data. + * + * As usually only a subset of the recorded data should + * be fed to the digest, you must also call startDigest() + * to begin digesting. + * + * @param md Message digest function to use. + */ + public void setDigest(MessageDigest md) { + this.digest = md; + } + + /** + * Return the digest value for any recorded, digested data. Call + * only after all data has been recorded; otherwise, the running + * digest state is ruined. + * + * @return the digest final value + */ + public byte[] getDigestValue() { + if(this.digest == null) { + return null; + } + return this.digest.digest(); + } + + public long getResponseContentLength() { + return this.size - this.messageBodyBeginMark; + } + + /** + * @return True if this ROS is open. + */ + public boolean isOpen() { + return this.out != null; + } + + public int getBufferLength() { + return this.buffer.length; + } + + /** + * When used alongside a mark-supporting RecordingInputStream, remember + * a position reachable by a future reset(). + */ + public void mark() { + // remember this position for subsequent reset() + this.markPosition = position; + } + + /** + * When used alongside a mark-supporting RecordingInputStream, reset + * the position to that saved by previous mark(). Until the position + * again reached "new" material, none of the bytes pushed to this + * stream will be digested or recorded. + */ + public void reset() { + // take note of furthest-position-reached to avoid double-recording + maxPosition = Math.max(maxPosition, position); + // reset to previous position + position = markPosition; + } + + /** + * Set limits on length, time, and rate to enforce. + * + * @param length + * @param milliseconds + * @param rateKBps + */ + public void setLimits(long length, long milliseconds, long rateKBps) { + maxLength = (length>0) ? length : Long.MAX_VALUE; + timeoutMs = (milliseconds>0) ? milliseconds : Long.MAX_VALUE; + maxRateBytesPerMs = (rateKBps>0) ? rateKBps*1024/1000 : Long.MAX_VALUE; + } + + /** + * Reset limits to effectively-unlimited defaults + */ + public void resetLimits() { + maxLength = Long.MAX_VALUE; + timeoutMs = Long.MAX_VALUE; + maxRateBytesPerMs = Long.MAX_VALUE; + } + + /** + * Return number of bytes that could be recorded without hitting + * length limit + * + * @return long byte count + */ + public long getRemainingLength() { + return maxLength - position; + } + + public void clearForReuse() throws IOException { + this.out = null; + this.position = 0; + this.markPosition = 0; + this.maxPosition = 0; + this.size = 0; + this.messageBodyBeginMark = -1; + // ensure recording turned on + this.recording = true; + // Always begins false; must use startDigest() to begin + this.shouldDigest = false; + if (this.diskStream != null) { + closeDiskStream(); + } + } +} + diff --git a/src/main/java/org/archive/io/RecoverableIOException.java b/src/main/java/org/archive/io/RecoverableIOException.java new file mode 100644 index 00000000..5ce2251a --- /dev/null +++ b/src/main/java/org/archive/io/RecoverableIOException.java @@ -0,0 +1,83 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.io; + +import java.io.IOException; +import java.io.PrintStream; +import java.io.PrintWriter; + +/** + * A decorator on IOException for IOEs that are likely not fatal or at least + * merit retry. + * @author stack + * @version $Date$, $Revision$ + */ +public class RecoverableIOException extends IOException { + private static final long serialVersionUID = 6194776587381865451L; + private final IOException decoratedIOException; + + public RecoverableIOException(final String message) { + this(new IOException(message)); + } + + public RecoverableIOException(final IOException ioe) { + super(); + this.decoratedIOException = ioe; + } + + public Throwable getCause() { + return this.decoratedIOException.getCause(); + } + + public String getLocalizedMessage() { + return this.decoratedIOException.getLocalizedMessage(); + } + + public String getMessage() { + return this.decoratedIOException.getMessage(); + } + + public StackTraceElement[] getStackTrace() { + return this.decoratedIOException.getStackTrace(); + } + + public synchronized Throwable initCause(Throwable cause) { + return this.decoratedIOException.initCause(cause); + } + + public void printStackTrace() { + this.decoratedIOException.printStackTrace(); + } + + public void printStackTrace(PrintStream s) { + this.decoratedIOException.printStackTrace(s); + } + + public void printStackTrace(PrintWriter s) { + this.decoratedIOException.printStackTrace(s); + } + + public void setStackTrace(StackTraceElement[] stackTrace) { + this.decoratedIOException.setStackTrace(stackTrace); + } + + public String toString() { + return this.decoratedIOException.toString(); + } +} \ No newline at end of file diff --git a/src/main/java/org/archive/io/RecyclingFastBufferedOutputStream.java b/src/main/java/org/archive/io/RecyclingFastBufferedOutputStream.java new file mode 100644 index 00000000..a3b76e46 --- /dev/null +++ b/src/main/java/org/archive/io/RecyclingFastBufferedOutputStream.java @@ -0,0 +1,37 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.io; + +import it.unimi.dsi.fastutil.io.FastBufferedOutputStream; + +import java.io.OutputStream; + +/** + * FastBufferedOutputStream that accepts a passed-in buffer (avoiding + * reallocation). + */ +public class RecyclingFastBufferedOutputStream extends FastBufferedOutputStream { + public RecyclingFastBufferedOutputStream( final OutputStream os, final byte[] buffer ) { + super(os); + this.buffer = buffer; + avail = buffer.length; + } +} + + diff --git a/src/main/java/org/archive/io/ReplayCharSequence.java b/src/main/java/org/archive/io/ReplayCharSequence.java new file mode 100644 index 00000000..aa9b9587 --- /dev/null +++ b/src/main/java/org/archive/io/ReplayCharSequence.java @@ -0,0 +1,77 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.io; + +import java.io.Closeable; +import java.io.IOException; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.Charset; + +import com.google.common.base.Charsets; + + +/** + * CharSequence interface with addition of a {@link #close()} method. + * + * Users of implementations of this interface must call {@link #close()} so + * implementations get a chance at cleaning up after themselves. + * + * @author stack + * @version $Revision$, $Date$ + */ +public interface ReplayCharSequence extends CharSequence, Closeable { + + /** charset to use in replay when declared value + * is absent/illegal/unavailable */ + public Charset FALLBACK_CHARSET = Charsets.ISO_8859_1; // TODO: should this be UTF-8? + + /** + * Call this method when done so implementation has chance to clean up + * resources. + * + * @throws IOException Problem cleaning up file system resources. + */ + public void close() throws IOException; + + /** + * Report count of decoder errors silently eaten during ReplayCharSequence + * use. May be less than the number of individual decoding anomalies in + * underlying content (if decoding method doesn't allow counting individual + * errors). + */ + public long getDecodeExceptionCount(); + + /** + * Return the first coding-exception encountered, if the count > 0. + * @return CharacterCodingException + */ + public CharacterCodingException getCodingException(); + + /** + * @return false if {@link #close()} has been called + */ + public boolean isOpen(); + + /** + * Return the effective Charset used to create this CharSequence from + * (raw byte) source material. + */ + public Charset getCharset(); +} diff --git a/src/main/java/org/archive/io/ReplayInputStream.java b/src/main/java/org/archive/io/ReplayInputStream.java new file mode 100644 index 00000000..fccf5fd3 --- /dev/null +++ b/src/main/java/org/archive/io/ReplayInputStream.java @@ -0,0 +1,325 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.io; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +import org.apache.commons.io.IOUtils; +import org.archive.util.ArchiveUtils; +import org.archive.util.FileUtils; + + +/** + * Replays the bytes recorded from a RecordingInputStream or + * RecordingOutputStream. + * + * This InputStream supports mark and reset. + * + * @author gojomo + */ +public class ReplayInputStream extends SeekInputStream +{ + private static final int DEFAULT_BUFFER_SIZE = 256*1024; // 256KiB + private BufferedSeekInputStream diskStream; + private byte[] buffer; + private long position; + + /** + * Total size of stream content. + * + * Size of data to replay. + */ + private long size = -1; + + /** + * Where the response body starts, if marked + */ + protected long responseBodyStart = -1; + + + /** + * Constructor. + * + * @param buffer Buffer to read from. + * @param size Size of data to replay. + * @param responseBodyStart Start of the response body. + * @param backingFilename Backing file that sits behind the buffer. If + * size > than buffer then we go to backing file to read + * data that is beyond buffer.length. + * + * @throws IOException If we fail to open an input stream on + * backing file. + */ + public ReplayInputStream(byte[] buffer, long size, long responseBodyStart, + String backingFilename) + throws IOException + { + this(buffer, size, backingFilename); + this.responseBodyStart = responseBodyStart; + } + + /** + * Constructor. + * + * @param buffer Buffer to read from. + * @param size Size of data to replay. + * @param backingFilename Backing file that sits behind the buffer. If + * size > than buffer then we go to backing file to read + * data that is beyond buffer.length. + * @throws IOException If we fail to open an input stream on + * backing file. + */ + public ReplayInputStream(byte[] buffer, long size, String backingFilename) + throws IOException + { + this.buffer = buffer; + this.size = size; + if (size > buffer.length) { + setupDiskStream(new File(backingFilename)); + } + } + + protected void setupDiskStream(File backingFile) throws IOException { + RandomAccessInputStream rais = new RandomAccessInputStream(backingFile); + diskStream = new BufferedSeekInputStream(rais, 4096); + } + + protected File backingFile; + + /** + * Create a ReplayInputStream from the given source stream. Requires + * reading the entire stream (and possibly overflowing to a temporary + * file). Primary reason for doing so would be to have a repositionable + * version of the original stream's contents. + * + * If created via this constructor, use the destroy() method to ensure + * prompt deletion of any associated tmp file when done. + * + * @param fillStream + * @throws IOException + */ + public ReplayInputStream(InputStream fillStream) throws IOException { + this.buffer = new byte[DEFAULT_BUFFER_SIZE]; + long count = ArchiveUtils.readFully(fillStream, buffer); + if(fillStream.available()>0) { + this.backingFile = File.createTempFile("tid"+Thread.currentThread().getId(), "ris"); + count += FileUtils.readFullyToFile(fillStream, backingFile); + setupDiskStream(backingFile); + } + this.size = count; + } + + /** + * Close & destroy any internally-generated temporary files. + */ + public void destroy() { + IOUtils.closeQuietly(this); + if(backingFile!=null) { + FileUtils.deleteSoonerOrLater(backingFile); + } + } + + public long setToResponseBodyStart() throws IOException { + position(responseBodyStart); + return this.position; + } + + + /* (non-Javadoc) + * @see java.io.InputStream#read() + */ + public int read() throws IOException { + if (position == size) { + return -1; // EOF + } + if (position < buffer.length) { + // Convert to unsigned int. + int c = buffer[(int) position] & 0xFF; + position++; + return c; + } + int c = diskStream.read(); + if (c >= 0) { + position++; + } + return c; + } + + /* + * (non-Javadoc) + * + * @see java.io.InputStream#read(byte[], int, int) + */ + public int read(byte[] b, int off, int len) throws IOException { + if (position == size) { + return -1; // EOF + } + if (position < buffer.length) { + int toCopy = (int)Math.min(size - position, + Math.min(len, buffer.length - position)); + System.arraycopy(buffer, (int)position, b, off, toCopy); + if (toCopy > 0) { + position += toCopy; + } + return toCopy; + } + // into disk zone + int read = diskStream.read(b,off,len); + if(read>0) { + position += read; + } + return read; + } + + public void readFullyTo(OutputStream os) throws IOException { + byte[] buf = new byte[4096]; + int c = read(buf); + while (c != -1) { + os.write(buf,0,c); + c = read(buf); + } + } + + /* + * Like 'readFullyTo', but only reads the header-part. + * Starts from the beginning each time it is called. + */ + public void readHeaderTo(OutputStream os) throws IOException { + position = 0; + byte[] buf = new byte[(int)responseBodyStart]; + int c = read(buf,0,buf.length); + if(c != -1) { + os.write(buf,0,c); + } + } + + /* + * Like 'readFullyTo', but only reads the content-part. + */ + public void readContentTo(OutputStream os) throws IOException { + setToResponseBodyStart(); + byte[] buf = new byte[4096]; + int c = read(buf); + while (c != -1) { + os.write(buf,0,c); + c = read(buf); + } + } + + /** + * Convenience method to copy content out to target stream. + * @param os stream to write content to + * @param maxSize maximum count of bytes to copy + * @throws IOException + */ + public void readContentTo(OutputStream os, long maxSize) throws IOException { + setToResponseBodyStart(); + byte[] buf = new byte[4096]; + int c = read(buf); + long tot = 0; + while (c != -1 && tot < maxSize) { + os.write(buf,0,c); + c = read(buf); + tot += c; + } + } + + /* (non-Javadoc) + * @see java.io.InputStream#close() + */ + public void close() throws IOException { + super.close(); + if(diskStream != null) { + diskStream.close(); + } + } + + /** + * Total size of stream content. + * @return Returns the size. + */ + public long getSize() + { + return size; + } + + /** + * Total size of header. + * @return the size of the header. + */ + public long getHeaderSize() + { + return responseBodyStart; + } + + /** + * Total size of content. + * @return the size of the content. + */ + public long getContentSize() + { + return size - responseBodyStart; + } + + /** + * @return Amount THEORETICALLY remaining (TODO: Its not theoretical + * seemingly. The class implemetentation depends on it being exact). + */ + public long remaining() { + return size - position; + } + + + /** + * Reposition the stream. + * + * @param p the new position for this stream + * @throws IOException if an IO error occurs + */ + public void position(long p) throws IOException { + if (p < 0) { + throw new IOException("Negative seek offset."); + } + if (p > size) { + throw new IOException("Desired position exceeds size."); + } + if (p < buffer.length) { + // Only seek file if necessary + if (position > buffer.length) { + diskStream.position(0); + } + } else { + diskStream.position(p - buffer.length); + } + this.position = p; + } + + + public long position() throws IOException { + return position; + } + + protected byte[] getBuffer() { + return buffer; + } +} diff --git a/src/main/java/org/archive/io/RepositionableInputStream.java b/src/main/java/org/archive/io/RepositionableInputStream.java new file mode 100644 index 00000000..6f885130 --- /dev/null +++ b/src/main/java/org/archive/io/RepositionableInputStream.java @@ -0,0 +1,133 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.io; + +import it.unimi.dsi.fastutil.io.RepositionableStream; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; + +/** + * Wrapper around an {@link InputStream} to make a primitive Repositionable + * stream. Uses a {@link BufferedInputStream}. Calls mark on every read so + * we'll remember at least the last thing read (You can only backup on the + * last thing read -- not last 2 or 3 things read). Used by + * {@link GzippedInputStream} when reading streams over a network. Wraps a + * HTTP, etc., stream so we can back it up if needs be after the + * GZIP inflater has done a fill of its full buffer though it only needed + * the first few bytes to finish decompressing the current GZIP member. + * + *

TODO: More robust implementation. Tried to use the it.unimi.dsi.io + * FastBufferdInputStream but relies on FileChannel ByteBuffers and if not + * present -- as would be the case reading from a network stream, the main + * application for this instance -- then it expects the underlying stream + * implements RepositionableStream interface so chicken or egg problem. + * @author stack + */ +public class RepositionableInputStream extends BufferedInputStream implements + RepositionableStream { + private long position = 0; + private long markPosition = -1; + + public RepositionableInputStream(InputStream in) { + super(in); + } + + public RepositionableInputStream(InputStream in, int size) { + super(in, size); + } + + public int read(byte[] b) throws IOException { + int read = super.read(b); + if (read != -1) { + position += read; + } + return read; + } + + public synchronized int read(byte[] b, int offset, int ct) + throws IOException { + // Mark the underlying stream so that we'll remember what we are about + // to read unless a mark has been set in this RepositionableStream + // (We have two levels of mark). In this latter case we want the + // underlying stream to preserve its mark position so aligns with + // this RS when eset is called. + if (!isMarked()) { + super.mark((ct > offset)? ct - offset: ct); + } + int read = super.read(b, offset, ct); + if (read != -1) { + position += read; + } + return read; + } + + public int read() throws IOException { + // Mark the underlying stream so that we'll remember what we are about + // to read unless a mark has been set in this RepositionableStream + // (We have two levels of mark). In this latter case we want the + // underlying stream to preserve its mark position so aligns with + // this RS when eset is called. + if (!isMarked()) { + super.mark(1); + } + int c = super.read(); + if (c != -1) { + position++; + } + return c; + } + + public void position(final long offset) { + if (this.position == offset) { + return; + } + int diff = (int)(offset - this.position); + long lowerBound = this.position - this.pos; + long upperBound = lowerBound + this.count; + if (offset < lowerBound || offset >= upperBound) { + throw new IllegalAccessError("Offset goes outside " + + "current this.buf (TODO: Do buffer fills if positive)"); + } + this.position = offset; + this.pos += diff; + // Clear any mark. + this.markPosition = -1; + } + + public void mark(int readlimit) { + this.markPosition = this.position; + super.mark(readlimit); + } + + public void reset() throws IOException { + super.reset(); + this.position = this.markPosition; + this.markPosition = -1; + } + + protected boolean isMarked() { + return this.markPosition != -1; + } + + public long position() { + return this.position; + } +} \ No newline at end of file diff --git a/src/main/java/org/archive/io/SafeSeekInputStream.java b/src/main/java/org/archive/io/SafeSeekInputStream.java new file mode 100644 index 00000000..0d8f83b1 --- /dev/null +++ b/src/main/java/org/archive/io/SafeSeekInputStream.java @@ -0,0 +1,124 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.io; + + +import java.io.IOException; + + +/** + * Enables multiple concurrent streams based on the same underlying stream. + * + * @author pjack + */ +public class SafeSeekInputStream extends SeekInputStream { + + + /** + * The underlying stream. + */ + private SeekInputStream input; + + + /** + * The expected position of the underlying stream. + */ + private long expected; + + + /** + * Constructor. The given stream will be positioned to 0 so that an + * accurate position can be tracked. + * + * @param input the underlying input stream + * @throws IOException if an IO error occurs + */ + public SafeSeekInputStream(SeekInputStream input) throws IOException { + this.input = input; + this.expected = input.position(); + } + + + /** + * Ensures that the underlying stream's position is what we expect to be. + * + * @throws IOException if an IO error occurs + */ + private void ensure() throws IOException { + if (expected != input.position()) { + input.position(expected); + } + } + + + @Override + public int read() throws IOException { + ensure(); + int c = input.read(); + if (c >= 0) { + expected++; + } + return c; + } + + + @Override + public int read(byte[] buf, int ofs, int len) throws IOException { + ensure(); + int r = input.read(buf, ofs, len); + if (r > 0) { + expected += r; + } + return r; + } + + + @Override + public int read(byte[] buf) throws IOException { + ensure(); + int r = input.read(buf); + if (r > 0) { + expected += r; + } + return r; + } + + + @Override + public long skip(long c) throws IOException { + ensure(); + long r = input.skip(c); + if (r > 0) { + expected += r; + } + return r; + } + + + public void position(long p) throws IOException { + input.position(p); + expected = p; + } + + + public long position() throws IOException { + return expected; + } + +} diff --git a/src/main/java/org/archive/io/SeekInputStream.java b/src/main/java/org/archive/io/SeekInputStream.java new file mode 100644 index 00000000..177724ec --- /dev/null +++ b/src/main/java/org/archive/io/SeekInputStream.java @@ -0,0 +1,81 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.io; + + +import it.unimi.dsi.fastutil.io.RepositionableStream; + +import java.io.IOException; +import java.io.InputStream; + + +/** + * Base class for repositionable input streams. + * + * @author pjack + */ +public abstract class SeekInputStream extends InputStream +implements RepositionableStream { + + + /** + * The marked file position. A value less than zero + * indicates that no mark has been set. + */ + private long mark = -1; + + + /** + * Marks the current position of the stream. The limit parameter is + * ignored; the mark will remain valid until reset is called or the + * stream is closed. + * + * @param limit ignored + */ + public void mark(int limit) { + try { + this.mark = position(); + } catch (IOException e) { + mark = -1; + } + } + + + /** + * Resets this stream to its marked position. + * + * @throws IOException if there is no mark, or if an IO error occurs + */ + public void reset() throws IOException { + if (mark < 0) { + throw new IOException("No mark."); + } + position(mark); + } + + + /** + * Returns true, since SeekInputStreams support mark/reset by default. + * + * @return true + */ + public boolean markSupported() { + return true; + } +} diff --git a/src/main/java/org/archive/io/SeekReader.java b/src/main/java/org/archive/io/SeekReader.java new file mode 100644 index 00000000..4abf7847 --- /dev/null +++ b/src/main/java/org/archive/io/SeekReader.java @@ -0,0 +1,84 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.io; + + +import java.io.IOException; +import java.io.Reader; + +import it.unimi.dsi.fastutil.io.RepositionableStream; + + +/** + * Base class for repositionable readers. + * + * @author pjack + */ +public abstract class SeekReader extends Reader +implements RepositionableStream { + + + /** + * The marked file position. A value less than zero + * indicates that no mark has been set. + */ + private long mark = -1; + + + /** + * Marks the current position of the stream. The limit parameter is + * ignored; the mark will remain valid until reset is called or the + * stream is closed. + * + * @param limit ignored + */ + @Override + public void mark(int limit) { + try { + this.mark = position(); + } catch (IOException e) { + mark = -1; + } + } + + + /** + * Resets this stream to its marked position. + * + * @throws IOException if there is no mark, or if an IO error occurs + */ + @Override + public void reset() throws IOException { + if (mark < 0) { + throw new IOException("No mark."); + } + position(mark); + } + + + /** + * Returns true, since SeekInputStreams support mark/reset by default. + * + * @return true + */ + @Override + public boolean markSupported() { + return true; + } +} diff --git a/src/main/java/org/archive/io/SeekReaderCharSequence.java b/src/main/java/org/archive/io/SeekReaderCharSequence.java new file mode 100644 index 00000000..a9b4880f --- /dev/null +++ b/src/main/java/org/archive/io/SeekReaderCharSequence.java @@ -0,0 +1,56 @@ +package org.archive.io; + +import java.io.IOException; + +public class SeekReaderCharSequence implements CharSequence { + + + final private SeekReader reader; + final private int size; + + + public SeekReaderCharSequence(SeekReader reader, int size) { + this.reader = reader; + this.size = size; + } + + + public int length() { + return size; + } + + + public char charAt(int index) { + if ((index < 0) || (index >= length())) { + throw new IndexOutOfBoundsException(Integer.toString(index)); + } + try { + reader.position(index); + int r = reader.read(); + if (r < 0) { + throw new IllegalStateException("EOF"); + } + return (char)reader.read(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + + public CharSequence subSequence(int start, int end) { + return new CharSubSequence(this, start, end); + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + try { + reader.position(0); + for (int ch = reader.read(); ch >= 0; ch = reader.read()) { + sb.append((char)ch); + } + return sb.toString(); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } +} diff --git a/src/main/java/org/archive/io/SinkHandlerLogThread.java b/src/main/java/org/archive/io/SinkHandlerLogThread.java new file mode 100644 index 00000000..0070785e --- /dev/null +++ b/src/main/java/org/archive/io/SinkHandlerLogThread.java @@ -0,0 +1,34 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.io; + + +/** + * Implemented by threads that provide extra information. + * + * TODO: rename class, rename getCurrentProcessorName() + */ +public interface SinkHandlerLogThread { + + String getName(); + String getCurrentProcessorName(); + int getSerialNumber(); + +} diff --git a/src/main/java/org/archive/io/UTF8Bytes.java b/src/main/java/org/archive/io/UTF8Bytes.java new file mode 100644 index 00000000..c280b08d --- /dev/null +++ b/src/main/java/org/archive/io/UTF8Bytes.java @@ -0,0 +1,37 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.io; + +import java.io.UnsupportedEncodingException; + +/** + * Marker Interface for instances that can be serialized as UTF8 bytes. + * TODO: Do we need a UTF8Stream Marker Interface? + * @author stack + * @version $Date$ $Version$ + */ +public interface UTF8Bytes { + public static final String UTF8 = "UTF-8"; + + /** + * @return Instance as UTF-8 bytes. + * @throws UnsupportedEncodingException + */ + public byte [] getUTF8Bytes() throws UnsupportedEncodingException; +} diff --git a/src/main/java/org/archive/io/WriterPool.java b/src/main/java/org/archive/io/WriterPool.java new file mode 100644 index 00000000..2dc385a1 --- /dev/null +++ b/src/main/java/org/archive/io/WriterPool.java @@ -0,0 +1,343 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.io; + +import java.io.File; +import java.io.IOException; +import java.util.Collection; +import java.util.LinkedList; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +/** + * Pool of Writers. + * + * Abstract. Override and pass in the Constructor a factory that creates + * {@link WriterPoolMember} implementations. + * + * @author stack + */ +public abstract class WriterPool { + private final Logger logger = Logger.getLogger(this.getClass().getName()); + + /** + * Used to generate unique filename sequences. + */ + final protected AtomicInteger serialNo; + + /** + * Default maximum active number of files in the pool. + */ + public static final int DEFAULT_MAX_ACTIVE = 1; + + /** Assumed largest possible value of maxActive; pool will have this + * maximum capacity, so dynamic changes beyond this number won't work. */ + protected static final int LARGEST_MAX_ACTIVE = 255; + + /** + * Maximum time to wait on a free file before considering + * making a new one (if not already at max) + */ + public static final int DEFAULT_MAX_WAIT_FOR_IDLE = 500; + + /** + * File settings. + * Keep in data structure rather than as individual values. + */ + protected final WriterPoolSettings settings; + + /** maximum number of writers to create at a time*/ + protected int maxActive; + /** maximum ms to wait before considering creation of a writer */ + protected int maxWait; + /** current count of active writers; only read/mutated in synchronized blocks */ + protected int currentActive = 0; + /** round-robin queue of available writers */ + protected BlockingQueue availableWriters; + + /** system time when writer was last wanted (because one was not ready in time) */ + protected long lastWriterNeededTime; + /** system time when writer was last 'rolled over' (imminent creation of new file) */ + protected long lastWriterRolloverTime; + + /** + * Constructor + * @param serial Used to generate unique filename sequences + * @param factory Factory that knows how to make a {@link WriterPoolMember}. + * @param settings Settings for this pool. + * @param poolMaximumActive + * @param poolMaximumWait + */ + public WriterPool(final AtomicInteger serial, + final WriterPoolSettings settings, + final int poolMaximumActive, final int poolMaximumWait) { + logger.info("Initial configuration:" + + " prefix=" + settings.getPrefix() + + ", template=" + settings.getTemplate() + + ", compress=" + settings.getCompress() + + ", maxSize=" + settings.getMaxFileSizeBytes() + + ", maxActive=" + poolMaximumActive + + ", maxWait=" + poolMaximumWait); + this.settings = settings; + this.maxActive = poolMaximumActive; + this.maxWait = poolMaximumWait; + availableWriters = new ArrayBlockingQueue(LARGEST_MAX_ACTIVE, true); + this.serialNo = serial; + } + + /** + * Check out a {@link WriterPoolMember}. + * + * This method should be followed by a call to + * {@link #returnFile(WriterPoolMember)} or + * {@link #invalidateFile(WriterPoolMember)} else pool starts leaking. + * + * @return Writer checked out of a pool of files or created + * @throws IOException Problem getting Writer from pool (Converted + * from Exception to IOException so this pool can live as a good citizen + * down in depths of ARCSocketFactory). + */ + public WriterPoolMember borrowFile() + throws IOException { + WriterPoolMember writer = null; + while(writer == null) { + try { + writer = availableWriters.poll(maxWait,TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + // nothing to do but proceed + } + if(writer==null) { + writer = makeNewWriterIfAppropriate(); + } + } + return writer; + } + + /** + * Create a new writer instance, if still below maxActive count. + * Remember times to help make later decision when writer should + * be discarded. + * + * @return WriterPoolMember or null if already at max + */ + protected synchronized WriterPoolMember makeNewWriterIfAppropriate() { + long now = System.currentTimeMillis(); + lastWriterNeededTime = now; + if(currentActive < maxActive) { + currentActive++; + lastWriterRolloverTime = now; + return makeWriter(); + } + return null; + } + + /** + * @return new WriterPoolMember of appropriate type + */ + protected abstract WriterPoolMember makeWriter(); + + /** + * Discard a previously-used writer, cleanly closing it and leaving it out + * of the pool. + * @param writer + * @throws IOException + */ + public synchronized void destroyWriter(WriterPoolMember writer) throws IOException { + currentActive--; + writer.close(); + } + /** + * Return a writer, for likely reuse unless (1) writer's current file has + * reached its target size; and (2) there's been no demand for additional + * writers since the last time a new writer-file was rolled-over. In that + * case, the possibly-superfluous writer instance is discarded. + * @param writer Writer to return to the pool. + * @throws IOException Problem returning File to pool. + */ + public void returnFile(WriterPoolMember writer) + throws IOException { + synchronized(this) { + if(writer.isOversize()) { + // maybe retire writer rather than recycle + if(lastWriterNeededTime<=lastWriterRolloverTime) { + // no timeouts waiting for recycled writer since last writer rollover + destroyWriter(writer); + return; + } else { + // reuse writer instance, causing new file to be created + lastWriterRolloverTime = System.currentTimeMillis(); + } + } + } + if(!availableWriters.offer(writer)) { + logger.log(Level.WARNING, "writer unreturnable to available pool; closing early"); + destroyWriter(writer); + } + } + + /** + * Close and discard a writer that experienced a potentially-corrupting + * error. + * @param f writer with problem + * @throws IOException + */ + public synchronized void invalidateFile(WriterPoolMember f) + throws IOException { + try { + destroyWriter(f); + } catch (Exception e) { + // Convert exception. + throw new IOException(e.getMessage()); + } + // It'll have been closed. Rename with an '.invalid' suffix so it + // gets attention. + File file = f.getFile(); + file.renameTo(new File(file.getAbsoluteFile() + + WriterPoolMember.INVALID_SUFFIX)); + } + + /** + * @return Number of {@link WriterPoolMember}s checked out of pool. + * @throws java.lang.UnsupportedOperationException + */ + public synchronized int getNumActive() + throws UnsupportedOperationException { + return currentActive - getNumIdle(); + } + + /** + * @return Number of {@link WriterPoolMember} instances still in the pool. + * @throws java.lang.UnsupportedOperationException + */ + public int getNumIdle() + throws UnsupportedOperationException { + return availableWriters.size(); + } + + /** + * Close all {@link WriterPoolMember}s in pool. + */ + public void close() { + Collection writers = drainAllWriters(); + for (WriterPoolMember writer: writers) { + try { + destroyWriter(writer); + } catch (IOException e) { + logger.log(Level.WARNING,"problem closing writer",e); + } + } + } + + /** + * @return Returns settings. + */ + public WriterPoolSettings getSettings() { + return this.settings; + } + + /** + * @return State of the pool string + */ + protected String getPoolState() { + StringBuffer buffer = new StringBuffer("Active "); + buffer.append(getNumActive()); + buffer.append(" of max "); + buffer.append(maxActive); + buffer.append(", idle "); + buffer.append(getNumIdle()); + return buffer.toString(); + } + + /** + * Returns the atomic integer used to generate serial numbers + * for files. + * + * @return the serial number generator + */ + public AtomicInteger getSerialNo() { + return serialNo; + } + + /** + * Drains all the active writers from {@link #availableWriters}, blocking to + * wait for any writers currently in use to become available. + * + *

+ * When finished with writers, call availableWriters.addAll(...) to put them + * back into the rotation. + * + * @return all the active writers + */ + protected synchronized Collection drainAllWriters() { + LinkedList writers = new LinkedList(); + availableWriters.drainTo(writers); + + while (writers.size() < currentActive) { + try { + WriterPoolMember w = availableWriters.take(); + writers.add(w); + } catch (InterruptedException e) { + logger.severe("caught " + e + " while waiting for writers to free up; returning only " + + writers.size() + " of " + currentActive + " active writers"); + break; + } + } + + return writers; + } + + public void flush() { + Collection writers = drainAllWriters(); + + for (WriterPoolMember writer: writers) { + try { + writer.flush(); + } catch (IOException e) { + logger.log(Level.WARNING, "problem flushing writer " + writer, e); + } + } + + availableWriters.addAll(writers); + } + + public JSONArray jsonStatus() throws JSONException { + Collection writers = drainAllWriters(); + + JSONArray ja = new JSONArray(); + for (WriterPoolMember w: writers) { + JSONObject jo = new JSONObject(); + jo.put("file", w.getFile()); + jo.put("position", w.getPosition()); + ja.put(jo); + } + + availableWriters.addAll(writers); + + return ja; + } +} diff --git a/src/main/java/org/archive/io/WriterPoolMember.java b/src/main/java/org/archive/io/WriterPoolMember.java new file mode 100644 index 00000000..6ea6b295 --- /dev/null +++ b/src/main/java/org/archive/io/WriterPoolMember.java @@ -0,0 +1,487 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.io; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.text.DecimalFormat; +import java.text.NumberFormat; +import java.util.Iterator; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Logger; +import java.util.zip.GZIPOutputStream; + +import org.archive.util.ArchiveUtils; +import org.archive.util.FileUtils; +import org.archive.util.PropertyUtils; + + + +/** + * Member of {@link WriterPool}. + * Implements rotating off files, file naming with some guarantee of + * uniqueness, and position in file. Subclass to pick up functionality for a + * particular Writer type. + * @author stack + * @version $Date$ $Revision$ + */ +public abstract class WriterPoolMember implements ArchiveFileConstants { + private final Logger logger = Logger.getLogger(this.getClass().getName()); + + public static final String UTF8 = "UTF-8"; + + /** + * Default archival-aggregate filename template. + * + * Under usual assumptions -- hostnames aren't shared among crawling hosts; + * processes have unique PIDs and admin ports; timestamps inside one process + * don't repeat (see UniqueTimestampService); clocks are generally + * accurate -- will generate a unique name. + * + * Stands for Internet Archive Heritrix. + */ + public static final String DEFAULT_TEMPLATE = + "${prefix}-${timestamp17}-${serialno}-${heritrix.pid}~${heritrix.hostname}~${heritrix.port}"; + + /** + * Default for file prefix. + */ + public static final String DEFAULT_PREFIX = "WEB"; + + /** + * Reference to file we're currently writing. + */ + protected File f = null; + + /** Output stream for file. */ + protected OutputStream out = null; + /** Counting stream for metering */ + protected MiserOutputStream countOut = null; + + /** reusable buffer for recycling scenarios */ + protected byte[] rebuf; + + protected WriterPoolSettings settings; + private final String extension; + + /** + * Creation date for the current file. + * Set by {@link #createFile()}. + */ + protected String currentTimestamp = "UNSET!!!"; + + protected String currentBasename; + + /** + * A running sequence used making unique file names. + */ + final private AtomicInteger serialNo; + + /** + * Directories round-robin index. + */ + protected static int roundRobinIndex = 0; + + /** + * NumberFormat instance for formatting serial number. + * + * Pads serial number with zeros. + */ + protected static NumberFormat serialNoFormatter = new DecimalFormat("00000"); + + + /** + * Buffer to reuse writing streams. + */ + protected final byte [] scratchbuffer = new byte[4 * 1024]; + + + /** + * Constructor. + * Takes a stream. Use with caution. There is no upperbound check on size. + * Will just keep writing. + * + * @param serialNo used to create unique filename sequences + * @param out Where to write. + * @param file File the out is connected to. + * @param cmprs Compress the content written. + * @param a14DigitDate If null, we'll write current time. + * @throws IOException + */ + protected WriterPoolMember(AtomicInteger serialNo, + final OutputStream out, final File file, + final WriterPoolSettings settings) + throws IOException { + this(serialNo, settings, null); + this.countOut = (out instanceof MiserOutputStream) + ? (MiserOutputStream)out + : new MiserOutputStream(out, settings.getFrequentFlushes()); + this.out = this.countOut; + this.f = file; + } + + /** + * Constructor. + * + * @param serialNo used to create unique filename sequences + * @param dirs Where to drop files. + * @param prefix File prefix to use. + * @param cmprs Compress the records written. + * @param maxSize Maximum size for ARC files written. + * @param template filenaming template to use + * @param extension Extension to give file. + */ + public WriterPoolMember(AtomicInteger serialNo, + final WriterPoolSettings settings, final String extension) { + this.settings = settings; + this.extension = extension; + this.serialNo = serialNo; + } + + /** + * Call this method just before/after any significant write. + * + * Call at the end of the writing of a record or just before we start + * writing a new record. Will close current file and open a new file + * if file size has passed out maxSize. + * + *

Creates and opens a file if none already open. One use of this method + * then is after construction, call this method to add the metadata, then + * call {@link #getPosition()} to find offset of first record. + * + * TODO: perhaps this should be called checkForNewOpen? because it also + * handles initial open, even when not rolling oversize + * + * @exception IOException + */ + public void checkSize() throws IOException { + if (this.out == null || isOversize()) { + createFile(); + } + } + + /** Check if underlying file has already reached its target size. + * @return boolean true if file has reached target size and due to be closed + */ + public boolean isOversize() { + return settings.getMaxFileSizeBytes() != -1 && (this.getPosition() > settings.getMaxFileSizeBytes()); + } + + /** + * Create a new file. + * Rotates off the current Writer and creates a new in its place + * to take subsequent writes. Usually called from {@link #checkSize()}. + * @return Name of file created. + * @throws IOException + */ + protected String createFile() throws IOException { + generateNewBasename(); + String name = currentBasename + '.' + this.extension + + ((settings.getCompress())? DOT_COMPRESSED_FILE_EXTENSION: "") + + OCCUPIED_SUFFIX; + File dir = getNextDirectory(settings.calcOutputDirs()); + return createFile(new File(dir, name)); + } + + protected String createFile(final File file) throws IOException { + close(); + this.f = file; + FileOutputStream fos = new FileOutputStream(this.f); + if(rebuf==null) { + rebuf = new byte[settings.getWriteBufferSize()]; + } + this.countOut = new MiserOutputStream(new RecyclingFastBufferedOutputStream(fos,rebuf),settings.getFrequentFlushes()); + this.out = this.countOut; + logger.fine("Opened " + this.f.getAbsolutePath()); + return this.f.getName(); + } + + /** + * @param dirs List of File objects that point at directories. + * @return Find next directory to write an arc too. If more + * than one, it tries to round-robin through each in turn. + * @throws IOException + */ + protected File getNextDirectory(List dirs) + throws IOException { + if (WriterPoolMember.roundRobinIndex >= dirs.size()) { + WriterPoolMember.roundRobinIndex = 0; + } + File d = null; + try { + d = checkWriteable((File)dirs. + get(WriterPoolMember.roundRobinIndex)); + } catch (IndexOutOfBoundsException e) { + // Dirs list might be altered underneath us. + // If so, we get this exception -- just keep on going. + } + if (d == null && dirs.size() > 1) { + for (Iterator i = dirs.iterator(); d == null && i.hasNext();) { + d = checkWriteable((File)i.next()); + } + } else { + WriterPoolMember.roundRobinIndex++; + } + if (d == null) { + throw new IOException("Directories unusable."); + } + return d; + } + + protected File checkWriteable(File d) { + if (d == null) { + return d; + } + + try { + FileUtils.ensureWriteableDirectory(d); + } catch(IOException e) { + logger.warning("Directory " + d.getPath() + " is not" + + " writeable or cannot be created: " + e.getMessage()); + d = null; + } + return d; + } + + /** + * Generate a new basename by interpolating values in the configured + * template. Values come from local state, other configured values, and + * global system properties. The recommended default template will + * generate a unique basename under reasonable assumptions. + */ + protected void generateNewBasename() { + Properties localProps = new Properties(); + localProps.setProperty("prefix", settings.getPrefix()); + synchronized(this.getClass()) { + // ensure that serialNo and timestamp are minted together (never inverted sort order) + String paddedSerialNumber = WriterPoolMember.serialNoFormatter.format(serialNo.getAndIncrement()); + String timestamp17 = ArchiveUtils.getUnique17DigitDate(); + String timestamp14 = ArchiveUtils.getUnique14DigitDate(); + currentTimestamp = timestamp17; + localProps.setProperty("serialno", paddedSerialNumber); + localProps.setProperty("timestamp17", timestamp17); + localProps.setProperty("timestamp14", timestamp14); + } + currentBasename = PropertyUtils.interpolateWithProperties(settings.getTemplate(), + localProps, System.getProperties()); + } + + + /** + * Get the file name + * + * @return the filename, as if uncompressed + */ + protected String getBaseFilename() { + String name = this.f.getName(); + if (settings.getCompress() && name.endsWith(DOT_COMPRESSED_FILE_EXTENSION)) { + return name.substring(0,name.length() - 3); + } else if(settings.getCompress() && + name.endsWith(DOT_COMPRESSED_FILE_EXTENSION + + OCCUPIED_SUFFIX)) { + return name.substring(0, name.length() - + (3 + OCCUPIED_SUFFIX.length())); + } else { + return name; + } + } + + /** + * Get this file. + * + * Used by junit test to test for creation and when {@link WriterPool} wants + * to invalidate a file. + * + * @return The current file. + */ + public File getFile() { + return this.f; + } + + /** + * Post write tasks. + * + * Has side effects. Will open new file if we're at the upper bound. + * If we're writing compressed files, it will wrap output stream with a + * GZIP writer with side effect that GZIP header is written out on the + * stream. + * + * @exception IOException + */ + protected void preWriteRecordTasks() + throws IOException { + if (this.out == null) { + createFile(); + } + if (settings.getCompress()) { + // Wrap stream in GZIP Writer. + // The below construction immediately writes the GZIP 'default' + // header out on the underlying stream. + this.out = new CompressedStream(this.out); + } + } + + /** + * Post file write tasks. + * If compressed, finishes up compression and flushes stream so any + * subsequent checks get good reading. + * + * @exception IOException + */ + protected void postWriteRecordTasks() + throws IOException { + if (settings.getCompress()) { + CompressedStream o = (CompressedStream)this.out; + o.finish(); + o.flush(); + o.end(); + this.out = o.getWrappedStream(); + } + } + + /** + * Position in raw output (typically, physical file). + * Used making accounting of bytes written. + * @return Position in final media (assuming all flushing completes) + * @throws IOException + */ + public long getPosition() { + return (countOut==null)? 0L : this.countOut.getCount(); + } + + public boolean isCompressed() { + return settings.getCompress(); + } + + protected void write(final byte [] b) throws IOException { + this.out.write(b); + } + + protected void flush() throws IOException { + this.out.flush(); + } + + protected void write(byte[] b, int off, int len) throws IOException { + this.out.write(b, off, len); + } + + protected void write(int b) throws IOException { + this.out.write(b); + } + + /** + * Copy bytes from the provided InputStream to the target file/stream being + * written. + * + * @return number of bytes written (normally equal to {@code enforceLength}) + * @param is + * InputStream to copy bytes from + * @param recordLength + * expected number of bytes to copy + * @param enforceLength + * whether to throw an exception if too many/too few bytes are + * available from stream + * @throws IOException + */ + protected long copyFrom(final InputStream is, final long recordLength, + boolean enforceLength) throws IOException { + int read = scratchbuffer.length; + long tot = 0; + while ((tot < recordLength) + && (read = is.read(scratchbuffer)) != -1) { + int write = read; + // never write more than enforced length + write = (int) Math.min(write, recordLength - tot); + tot += read; + write(scratchbuffer, 0, write); + } + if (enforceLength && tot != recordLength) { + // throw exception if desired for read vs. declared mismatches + throw new IOException("Read " + tot + " but expected " + + recordLength); + } + + return tot; + } + + public void close() throws IOException { + if (this.out == null) { + return; + } + this.out.close(); + this.out = null; + if (this.f != null && this.f.exists()) { + String path = this.f.getAbsolutePath(); + if (path.endsWith(OCCUPIED_SUFFIX)) { + File f = new File(path.substring(0, + path.length() - OCCUPIED_SUFFIX.length())); + if (f.exists() & !f.delete()) { + logger.warning("Failed delete of " + f); + } + if (!this.f.renameTo(f)) { + logger.warning("Failed rename of " + path); + } + this.f = f; + } + + logger.fine("Closed " + this.f.getAbsolutePath() + + ", size " + this.f.length()); + } + } + + protected OutputStream getOutputStream() { + return this.out; + } + + /** + * An override so we get access to underlying output stream. + * and offer an end() that does not accompany closing underlying + * stream. + * @author stack + */ + private class CompressedStream extends GZIPOutputStream { + public CompressedStream(OutputStream out) + throws IOException { + super(out); + } + + /** + * @return Reference to stream being compressed. + */ + OutputStream getWrappedStream() { + return this.out; + } + + /** + * Release the deflater's native process resources, + * which otherwise would not occur until either + * finalization or DeflaterOutputStream.close() + * (which would also close underlying stream). + */ + public void end() { + def.end(); + } + } +} diff --git a/src/main/java/org/archive/io/WriterPoolSettings.java b/src/main/java/org/archive/io/WriterPoolSettings.java new file mode 100644 index 00000000..d0805cdc --- /dev/null +++ b/src/main/java/org/archive/io/WriterPoolSettings.java @@ -0,0 +1,39 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.io; + +import java.io.File; +import java.util.List; + +/** + * Settings object for a {@link WriterPool}. + * Used creating {@link WriterPoolMember}s. + * @author stack + * @version $Date$, $Revision$ + */ +public interface WriterPoolSettings { + public long getMaxFileSizeBytes(); + public String getPrefix(); + public String getTemplate(); + public List calcOutputDirs(); + public boolean getCompress(); + public List getMetadata(); + public boolean getFrequentFlushes(); + public int getWriteBufferSize(); +} \ No newline at end of file diff --git a/src/main/java/org/archive/io/arc/ARC2WCDX.java b/src/main/java/org/archive/io/arc/ARC2WCDX.java new file mode 100644 index 00000000..19010131 --- /dev/null +++ b/src/main/java/org/archive/io/arc/ARC2WCDX.java @@ -0,0 +1,243 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.io.arc; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.util.Date; +import java.util.Iterator; +import java.util.zip.GZIPOutputStream; + +import org.apache.commons.httpclient.Header; +import org.apache.commons.httpclient.HeaderGroup; +import org.apache.commons.httpclient.util.DateParseException; +import org.apache.commons.httpclient.util.DateUtil; +import org.archive.io.ArchiveRecord; +import org.archive.util.ArchiveUtils; +import org.archive.util.SURT; + +/** + * Create a 'Wide' CDX from an ARC. Takes one argument, the path to the ARC. + * Writes .wcdx.gz in same directory. + * + * @author gojomo + */ +public class ARC2WCDX { + final public static String WCDX_VERSION="0.1"; + + public static void main(String[] args) throws IOException { + String arcFilename = args[0]; + createWcdx(arcFilename); + } + + public static Object[] createWcdx(String arcFilename) throws IOException { + ARCReader reader = ARCReaderFactory.get(arcFilename); + Object[] retVal = createWcdx(reader); + reader.close(); + return retVal; + } + + public static Object[] createWcdx(ARCReader reader) { + reader.setDigest(true); + + String wcdxPath = reader.getReaderIdentifier().replaceAll("\\.arc(\\.gz)?$",".wcdx.gz"); + File wcdxFile = new File(wcdxPath+".open"); + PrintStream writer = null; + long count = 0; + try { + writer = new PrintStream(new GZIPOutputStream(new FileOutputStream(wcdxFile))); + + // write header: legend + timestamp + StringBuilder legend = new StringBuilder(); + appendField(legend,"CDX"); + appendField(legend,"surt-uri"); + appendField(legend,"b"); // ARC timestamp + appendField(legend,"http-date"); + appendField(legend,"s"); // status code + appendField(legend,"m"); // media type + appendField(legend,"sha1"); // content sha1 + appendField(legend,"g"); // ARC name + appendField(legend,"V"); // start offset + appendField(legend,"end-offset"); // TODO: implement + appendField(legend,"n"); // ARC record length TODO: verify + appendField(legend,"http-content-length"); + appendField(legend,"http-last-modified"); + appendField(legend,"http-expires"); + appendField(legend,"http-etag"); + appendField(legend,"http-location"); + appendField(legend,"e"); // IP + appendField(legend,"a"); // original URL + // WCDX version+creation time: crude version control + appendField(legend,WCDX_VERSION+"@"+ArchiveUtils.get14DigitDate()); + writer.println(legend.toString()); + + Iterator iter = reader.iterator(); + count = 0; + while(iter.hasNext()) { + ARCRecord record = (ARCRecord) iter.next(); + record.close(); + ARCRecordMetaData h = (ARCRecordMetaData) record.getHeader(); + Header[] httpHeaders = record.getHttpHeaders(); + if(httpHeaders==null) { + httpHeaders = new Header[0]; + } + HeaderGroup hg = new HeaderGroup(); + hg.setHeaders(httpHeaders); + StringBuilder builder = new StringBuilder(); + + // SURT-form URI + appendField(builder,SURT.fromURI(h.getUrl())); + // record timestamp ('b') + appendField(builder,h.getDate()); + // http header date + appendTimeField(builder,hg.getFirstHeader("Date")); + // response code ('s') + appendField(builder,h.getStatusCode()); + // media type ('m') + appendField(builder,h.getMimetype()); + // content checksum (like 'c', but here Base32 SHA1) + appendField(builder,record.getDigestStr()); + // arc name ('g') + appendField(builder,reader.getFileName()); + // compressed start offset ('V') + appendField(builder,h.getOffset()); + + // compressed end offset (?) +// appendField(builder, +// reader.getInputStream() instanceof RepositionableStream +// ? ((GzippedInputStream)reader.getInputStream()).vPosition() +// : "-"); + // TODO; leave unavail for now + appendField(builder, "-"); + + // uncompressed (declared in ARC headerline) record length + appendField(builder,h.getLength()); + // http header content-length + appendField(builder,hg.getFirstHeader("Content-Length")); + + // http header mod-date + appendTimeField(builder,hg.getFirstHeader("Last-Modified")); + // http header expires + appendTimeField(builder,hg.getFirstHeader("Expires")); + + // http header etag + appendField(builder,hg.getFirstHeader("ETag")); + // http header redirect ('Location' header?) + appendField(builder,hg.getFirstHeader("Location")); + // ip ('e') + appendField(builder,h.getIp()); + // original URI + appendField(builder,h.getUrl()); + // TODO MAYBE - a title from inside content? + + writer.println(builder.toString()); + count++; + } + wcdxFile.renameTo(new File(wcdxPath)); + } catch (IOException e) { + // soldier on: but leave '.open' wcdx file as indicator of error + if(!wcdxFile.exists()) { + try { + wcdxFile.createNewFile(); + } catch (IOException e1) { + // TODO Auto-generated catch block + throw new RuntimeException(e1); + } + } + } catch (RuntimeException e) { + // soldier on: but leave '.open' wcdx file as indicator of error + if(!wcdxFile.exists()) { + try { + wcdxFile.createNewFile(); + } catch (IOException e1) { + // TODO Auto-generated catch block + throw new RuntimeException(e1); + } + } + } finally { + if(writer!=null) { + writer.close(); + } + } + + return new Object[] {wcdxPath, count}; + } + + protected static void appendField(StringBuilder builder, Object obj) { + if(builder.length()>0) { + // prepend with delimiter + builder.append(' '); + } + if(obj instanceof Header) { + obj = ((Header)obj).getValue().trim(); + } + + builder.append((obj==null||obj.toString().length()==0)?"-":obj); + } + + protected static void appendTimeField(StringBuilder builder, Object obj) { + if(builder.length()>0) { + // prepend with delimiter + builder.append(' '); + } + if(obj==null) { + builder.append("-"); + return; + } + if(obj instanceof Header) { + String s = ((Header)obj).getValue().trim(); + try { + Date date = DateUtil.parseDate(s); + String d = ArchiveUtils.get14DigitDate(date); + if(d.startsWith("209")) { + d = "199"+d.substring(3); + } + obj = d; + } catch (DateParseException e) { + builder.append('e'); + return; + } + + } + builder.append(obj); + } +} + +//'wide' CDX +//a original url +//b timestamp +//s resp code +//m type +//? content md5 (full 'k'? 'c'? +//g arc name +//V compressed start offset +//? compressed length +//n? uncompressed length +//? mod date +//? expires +//? server 'date' hdr +//? etag +//r redirect ('Location'?) +//e ip +//MAYBE: +//? TITLE from HTML or other format? + + diff --git a/src/main/java/org/archive/io/arc/ARCConstants.java b/src/main/java/org/archive/io/arc/ARCConstants.java new file mode 100644 index 00000000..c44cfef7 --- /dev/null +++ b/src/main/java/org/archive/io/arc/ARCConstants.java @@ -0,0 +1,29 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.io.arc; + + +/** + * Constants used by ARC files and in ARC file processing. + * + * @author stack + * @deprecated + */ +public interface ARCConstants extends org.archive.format.arc.ARCConstants { +} diff --git a/src/main/java/org/archive/io/arc/ARCLocation.java b/src/main/java/org/archive/io/arc/ARCLocation.java new file mode 100644 index 00000000..c6c64437 --- /dev/null +++ b/src/main/java/org/archive/io/arc/ARCLocation.java @@ -0,0 +1,37 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.io.arc; + +/** + * Datastructure to hold ARC record location. + * Used by wayback machine. + * @author stack + */ +public interface ARCLocation { + /** + * @return Returns the ARC filename. Can be full path to ARC, URL to an + * ARC or just the portion of an ARC name that is unique to a collection. + */ + public String getName(); + + /** + * @return Returns the offset into the ARC. + */ + public long getOffset(); +} diff --git a/src/main/java/org/archive/io/arc/ARCReader.java b/src/main/java/org/archive/io/arc/ARCReader.java new file mode 100644 index 00000000..7f85cc2a --- /dev/null +++ b/src/main/java/org/archive/io/arc/ARCReader.java @@ -0,0 +1,553 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.io.arc; + +import java.io.ByteArrayOutputStream; +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Logger; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.cli.PosixParser; +import org.archive.io.ArchiveReader; +import org.archive.io.ArchiveRecord; +import org.archive.io.ArchiveRecordHeader; +import org.archive.io.RecoverableIOException; +import org.archive.io.WriterPoolMember; +import org.archive.util.ArchiveUtils; + + +/** + * Get an iterator on an ARC file or get a record by absolute position. + * + * ARC files are described here: + * Arc + * File Format. + * + *

This class knows how to parse an ARC file. Pass it a file path + * or an URL to an ARC. It can parse ARC Version 1 and 2. + * + *

Iterator returns ARCRecord + * though {@link Iterator#next()} is returning + * java.lang.Object. Cast the return. + * + *

Profiling java.io vs. memory-mapped ByteBufferInputStream shows the + * latter slightly slower -- but not by much. TODO: Test more. Just + * change {@link #getInputStream(File, long)}. + * + * @author stack + * @version $Date$ $Revision$ + */ +public abstract class ARCReader extends ArchiveReader +implements ARCConstants, Closeable { + private final Logger logger = Logger.getLogger(ARCReader.class.getName()); + + /** + * Set to true if we are aligned on first record of Archive file. + * We used depend on offset. If offset was zero, then we were + * aligned on first record. This is no longer necessarily the case when + * Reader is created at an offset into an Archive file: The offset is zero + * but its relative to where we started reading. + */ + private boolean alignedOnFirstRecord = true; + + private boolean parseHttpHeaders = true; + + protected ARCReader() { + super(); + } + + /** + * Skip over any trailing new lines at end of the record so we're lined up + * ready to read the next. + * @param record + * @throws IOException + */ + protected void gotoEOR(ArchiveRecord record) throws IOException { + if (getIn().available() <= 0) { + return; + } + + // Remove any trailing LINE_SEPARATOR + int c = -1; + while (getIn().available() > 0) { + if (getIn().markSupported()) { + getIn().mark(1); + } + c = getIn().read(); + if (c != -1) { + if (c == LINE_SEPARATOR) { + continue; + } + if (getIn().markSupported()) { + // We've overread. We're probably in next record. There is + // no way of telling for sure. It may be dross at end of + // current record. Backup. + getIn().reset(); + break; + } + ArchiveRecordHeader h = (getCurrentRecord() != null)? + record.getHeader(): null; + throw new IOException("Read " + (char)c + + " when only " + LINE_SEPARATOR + " expected. " + + getReaderIdentifier() + ((h != null)? + h.getHeaderFields().toString(): "")); + } + } + } + + /** + * Create new arc record. + * + * Encapsulate housekeeping that has to do w/ creating a new record. + * + *

Call this method at end of constructor to read in the + * arcfile header. Will be problems reading subsequent arc records + * if you don't since arcfile header has the list of metadata fields for + * all records that follow. + * + *

When parsing through ARCs writing out CDX info, we spend about + * 38% of CPU in here -- about 30% of which is in getTokenizedHeaderLine + * -- of which 16% is reading. + * + * @param is InputStream to use. + * @param offset Absolute offset into arc file. + * @return An arc record. + * @throws IOException + */ + protected ARCRecord createArchiveRecord(InputStream is, long offset) + throws IOException { + try { + String version = super.getVersion(); + ARCRecord record = new ARCRecord(is, getReaderIdentifier(), offset, + isDigest(), isStrict(), isParseHttpHeaders(), + isAlignedOnFirstRecord(), version); + if (version != null && super.getVersion() == null) + super.setVersion(version); + currentRecord(record); + } catch (IOException e) { + if (e instanceof RecoverableIOException) { + // Don't mess with RecoverableIOExceptions. Let them out. + throw e; + } + IOException newE = new IOException(e.getMessage() + " (Offset " + + offset + ")."); + newE.setStackTrace(e.getStackTrace()); + throw newE; + } + return (ARCRecord)getCurrentRecord(); + } + + /** + * Returns version of this ARC file. Usually read from first record of ARC. + * If we're reading without having first read the first record -- e.g. + * random access into middle of an ARC -- then version will not have been + * set. For now, we return a default, version 1.1. Later, if more than + * just one version of ARC, we could look at such as the meta line to see + * what version of ARC this is. + * @return Version of this ARC file. + */ + public String getVersion() { + return (super.getVersion() == null)? "1.1": super.getVersion(); + } + + protected boolean isAlignedOnFirstRecord() { + return alignedOnFirstRecord; + } + + protected void setAlignedOnFirstRecord(boolean alignedOnFirstRecord) { + this.alignedOnFirstRecord = alignedOnFirstRecord; + } + + /** + * @return Returns the parseHttpHeaders. + */ + public boolean isParseHttpHeaders() { + return this.parseHttpHeaders; + } + + /** + * @param parse The parseHttpHeaders to set. + */ + public void setParseHttpHeaders(boolean parse) { + this.parseHttpHeaders = parse; + } + + public String getFileExtension() { + return ARC_FILE_EXTENSION; + } + + public String getDotFileExtension() { + return DOT_ARC_FILE_EXTENSION; + } + + protected boolean output(final String format) + throws IOException, java.text.ParseException { + boolean result = super.output(format); + if(!result && (format.equals(NOHEAD) || format.equals(HEADER))) { + throw new IOException(format + + " format only supported for single Records"); + } + return result; + } + + public boolean outputRecord(final String format) throws IOException { + boolean result = super.outputRecord(format); + if (result) { + return result; + } + if (format.equals(NOHEAD)) { + // No point digesting if dumping content. + setDigest(false); + ARCRecord r = (ARCRecord) get(); + r.skipHttpHeader(); + r.dump(); + result = true; + } else if (format.equals(HEADER)) { + // No point digesting if dumping content. + setDigest(false); + ARCRecord r = (ARCRecord) get(); + r.dumpHttpHeader(); + result = true; + } + + return result; + } + + public void dump(final boolean compress) + throws IOException, java.text.ParseException { + // No point digesting if we're doing a dump. + setDigest(false); + boolean firstRecord = true; + ARCWriter writer = null; + for (Iterator ii = iterator(); ii.hasNext();) { + ARCRecord r = (ARCRecord)ii.next(); + // We're to dump the arc on stdout. + // Get the first record's data if any. + ARCRecordMetaData meta = r.getMetaData(); + if (firstRecord) { + firstRecord = false; + // Get an ARCWriter. + ByteArrayOutputStream baos = + new ByteArrayOutputStream(r.available()); + // This is slow but done only once at top of ARC. + while (r.available() > 0) { + baos.write(r.read()); + } + List listOfMetadata = new ArrayList(); + listOfMetadata.add(baos.toString(WriterPoolMember.UTF8)); + // Assume getArc returns full path to file. ARCWriter + // or new File will complain if it is otherwise. + List outDirs = new ArrayList(); + WriterPoolSettingsData settings = + new WriterPoolSettingsData("","",-1L,compress,outDirs,listOfMetadata); + writer = new ARCWriter(new AtomicInteger(), System.out, + new File(meta.getArc()), settings); + continue; + } + + writer.write(meta.getUrl(), meta.getMimetype(), meta.getIp(), + ArchiveUtils.parse14DigitDate(meta.getDate()).getTime(), + (int)meta.getLength(), r); + } + // System.out.println(System.currentTimeMillis() - start); + } + + /** + * @return an ArchiveReader that will delete a local file on close. Used + * when we bring Archive files local and need to clean up afterward. + */ + public ARCReader getDeleteFileOnCloseReader(final File f) { + final ARCReader d = this; + return new ARCReader() { + private final ARCReader delegate = d; + private File archiveFile = f; + + public void close() throws IOException { + this.delegate.close(); + if (this.archiveFile != null) { + if (archiveFile.exists()) { + archiveFile.delete(); + } + this.archiveFile = null; + } + } + + public ArchiveRecord get(long o) throws IOException { + return this.delegate.get(o); + } + + public boolean isDigest() { + return this.delegate.isDigest(); + } + + public boolean isStrict() { + return this.delegate.isStrict(); + } + + public Iterator iterator() { + return this.delegate.iterator(); + } + + public void setDigest(boolean d) { + this.delegate.setDigest(d); + } + + public void setStrict(boolean s) { + this.delegate.setStrict(s); + } + + public List validate() throws IOException { + return this.delegate.validate(); + } + + @Override + public ArchiveRecord get() throws IOException { + return this.delegate.get(); + } + + @Override + public String getVersion() { + return this.delegate.getVersion(); + } + + @Override + public List validate(int noRecords) throws IOException { + return this.delegate.validate(noRecords); + } + + @Override + protected ARCRecord createArchiveRecord(InputStream is, + long offset) + throws IOException { + return this.delegate.createArchiveRecord(is, offset); + } + + @Override + protected void gotoEOR(ArchiveRecord record) throws IOException { + this.delegate.gotoEOR(record); + } + + @Override + public void dump(boolean compress) + throws IOException, java.text.ParseException { + this.delegate.dump(compress); + } + + @Override + public String getDotFileExtension() { + return this.delegate.getDotFileExtension(); + } + + @Override + public String getFileExtension() { + return this.delegate.getFileExtension(); + } + }; + } + + // Static methods follow. + + /** + * + * @param formatter Help formatter instance. + * @param options Usage options. + * @param exitCode Exit code. + */ + private static void usage(HelpFormatter formatter, Options options, + int exitCode) { + formatter.printHelp("java org.archive.io.arc.ARCReader" + + " [--digest=true|false] \\\n" + + " [--format=cdx|cdxfile|dump|gzipdump|header|nohead]" + + " [--offset=#] \\\n[--strict] [--parse] ARC_FILE|ARC_URL", + options); + System.exit(exitCode); + } + + /** + * Write out the arcfile. + * + * @param reader + * @param format Format to use outputting. + * @throws IOException + * @throws java.text.ParseException + */ + protected static void output(ARCReader reader, String format) + throws IOException, java.text.ParseException { + if (!reader.output(format)) { + throw new IOException("Unsupported format: " + format); + } + } + + /** + * Generate a CDX index file for an ARC file. + * + * @param urlOrPath The ARC file to generate a CDX index for + * @throws IOException + * @throws java.text.ParseException + */ + public static void createCDXIndexFile(String urlOrPath) + throws IOException, java.text.ParseException { + ARCReader r = ARCReaderFactory.get(urlOrPath); + r.setStrict(false); + r.setParseHttpHeaders(true); + r.setDigest(true); + output(r, CDX_FILE); + } + + /** + * Command-line interface to ARCReader. + * + * Here is the command-line interface: + *

+     * usage: java org.archive.io.arc.ARCReader [--offset=#] ARCFILE
+     *  -h,--help      Prints this message and exits.
+     *  -o,--offset    Outputs record at this offset into arc file.

+ * % av_procarc hx20040109230030-0.arc.gz | av_ziparc > \
+ *     /tmp/hx20040109230030-0.dat.gz
+ * % av_ripdat /tmp/hx20040109230030-0.dat.gz > /tmp/hx20040109230030-0.cdx
+ * 

filedesc://testWriteRecord-JunitIAH20040110013326-2.arc 0.0.0.0 \\
+     *  20040110013326 text/plain 77
+     * 1 0 InternetArchive
+     * URL IP-address Archive-date Content-type Archive-length
+     * 

+     * usage: java org.archive.io.arc.WARCReader [--offset=#] ARCFILE
+     *  -h,--help      Prints this message and exits.
+     *  -o,--offset    Outputs record at this offset into arc file.

+     field       =  field-name ":" [ field-body ] CRLF
+     
+     field-name  =  1*<any CHAR, excluding CTLs, SPACE, and ":">
+     
+     field-body  =  field-body-contents
+                    [CRLF LWSP-char field-body]
+     
+     field-body-contents =
+                   <the ASCII characters making up the field-body, as
+                    defined in the following sections, and consisting
+                    of combinations of atom, quoted-string, and
+                    specials tokens, or else consisting of texts>
+