CDX: Add CloseableCompositeIterator which iterates in sequence, optimization for zipnum clusters to be loaded sequentially when only looking for last line

Author: ikreymer

src/main/java/org/archive/format/cdx/MultiCDXInputSource.java

@@ -8,6 +8,7 @@
 
 import org.archive.format.gzip.zipnum.ZipNumIndex;
 import org.archive.format.gzip.zipnum.ZipNumParams;
+import org.archive.util.iterator.CloseableCompositeIterator;
 import org.archive.util.iterator.CloseableIterator;
 import org.archive.util.iterator.SortedCompositeIterator;
 
@@ -70,9 +71,36 @@ public CloseableIterator<String> getCDXIterator(String key, String prefix, boole
 		return scitr;
 	}
 
+	public CloseableIterator<String> createSeqIterator(String key, String start, String end, ZipNumParams params)
+	{
+		CloseableCompositeIterator<String> composite = new CloseableCompositeIterator<String>();
+		CloseableIterator<String> iter = null;
+		
+		for (CDXInputSource cdxReader : cdx) {
+			try {
+				iter = cdxReader.getCDXIterator(key, start, end, params);
+				
+				if (!params.isReverse()) {
+					composite.addLast(iter);
+				} else {
+					composite.addFirst(iter);
+				}
+				
+			} catch (IOException io) {
+				LOGGER.warning(io.toString());
+			}
+		}
+		
+		return composite;
+	}
+	
 
 	public CloseableIterator<String> getCDXIterator(String key, String start, String end, ZipNumParams params) throws IOException {
 
+		if (params.isSequential()) {
+			return this.createSeqIterator(key, start, end, params);
+		}
+		
 		SortedCompositeIterator<String> scitr = new SortedCompositeIterator<String>(cdx.size(), params.isReverse() ? reverseComparator : comparator);
 
 		CloseableIterator<String> iter = null;

src/main/java/org/archive/format/gzip/zipnum/ZipNumParams.java

@@ -6,6 +6,7 @@ public class ZipNumParams
 	protected int timestampDedupLength = 0;
 	protected int maxBlocks = 0;
 	private boolean reverse = false;
+	private boolean sequential = false;
 
 	public ZipNumParams()
 	{
@@ -56,4 +57,12 @@ public boolean isReverse() {
 	public void setReverse(boolean reverse) {
 		this.reverse  = reverse;
 	}
+
+	public boolean isSequential() {
+		return sequential;
+	}
+
+	public void setSequential(boolean sequential) {
+		this.sequential = sequential;
+	}
 }

src/main/java/org/archive/util/iterator/CloseableCompositeIterator.java

@@ -0,0 +1,73 @@
+package org.archive.util.iterator;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.LinkedList;
+
+public class CloseableCompositeIterator<E> implements CloseableIterator<E> {
+
+	protected LinkedList<CloseableIterator<E>> iters;
+	protected Iterator<CloseableIterator<E>> iterPtr;
+	protected CloseableIterator<E> currIter;
+	
+    public CloseableCompositeIterator()
+	{
+		iters = new LinkedList<CloseableIterator<E>>();
+	}
+	
+	public void addFirst(CloseableIterator<E> e)
+	{
+		iters.addFirst(e);
+	}
+	
+	public void addLast(CloseableIterator<E> e)
+	{
+		iters.addLast(e);
+	}
+	
+	@Override
+    public boolean hasNext() {
+		
+		if (iterPtr == null) {
+			iterPtr = iters.iterator();
+			currIter = iterPtr.next();
+		}
+		
+		if (currIter == null) {
+			return false;
+		}
+		
+		while (currIter != null) {
+			if (currIter.hasNext()) {
+				return true;
+			}
+			
+			currIter = (iterPtr.hasNext() ? iterPtr.next() : null);
+		}
+		
+		return false;
+    }
+
+	@Override
+    public E next() {
+		return currIter.next();
+    }
+
+	@Override
+    public void remove() {
+		currIter.remove();
+    }
+
+	@Override
+    public void close() throws IOException {
+		for (CloseableIterator<E> e : iters) {
+			if (e != null) {
+				try {
+					e.close();
+				} catch (IOException io) {
+					
+				}
+			}
+		}
+    }
+}