From 0dd8a167e31eb8c9cd592f2290cfd279aa8430f9 Mon Sep 17 00:00:00 2001
From: Gerhard Gossen <gossen@l3s.de>
Date: Wed, 2 Mar 2016 14:17:04 +0100
Subject: [PATCH] Fix some ARC issues

---
 pom.xml                                            |  4 ++--
 .../archive/extract/RealCDXExtractorOutput.java    |  2 +-
 src/main/java/org/archive/io/ArchiveReader.java    |  2 +-
 src/main/java/org/archive/io/arc/ARCRecord.java    |  2 +-
 .../extract/RealCDXExtractorOutputTest.java        | 14 ++++++++++++++
 5 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/pom.xml b/pom.xml
index 7a32de08..5d8fffdd 100644
--- a/pom.xml
+++ b/pom.xml
@@ -98,8 +98,8 @@
 
     <dependency>
       <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-core</artifactId>
-      <version>0.20.2-cdh3u4</version>
+      <artifactId>hadoop-client</artifactId>
+      <version>2.6.0</version>
       <exclusions>
         <exclusion>
           <groupId>commons-httpclient</groupId>
diff --git a/src/main/java/org/archive/extract/RealCDXExtractorOutput.java b/src/main/java/org/archive/extract/RealCDXExtractorOutput.java
index 8ca3ff82..e9922c56 100644
--- a/src/main/java/org/archive/extract/RealCDXExtractorOutput.java
+++ b/src/main/java/org/archive/extract/RealCDXExtractorOutput.java
@@ -314,7 +314,7 @@ static String resolve(String context, String spec) {
 			URL cUrl = new URL(context);
 			URL url = new URL(cUrl, spec);
 			// this constructor escapes its arguments, if necessary
-			URI uri = new URI(url.getProtocol(), url.getHost(), url.getPath(), url.getQuery(), url.getRef());
+			URI uri = new URI(url.getProtocol(), url.getUserInfo(), url.getHost(), url.getPort(), url.getPath(), url.getQuery(), url.getRef());
 			return uri.toASCIIString();
 			
 		} catch (URISyntaxException e) {			
diff --git a/src/main/java/org/archive/io/ArchiveReader.java b/src/main/java/org/archive/io/ArchiveReader.java
index 66056d33..55faef17 100644
--- a/src/main/java/org/archive/io/ArchiveReader.java
+++ b/src/main/java/org/archive/io/ArchiveReader.java
@@ -79,7 +79,7 @@ public abstract class ArchiveReader implements ArchiveFileConstants, Iterable<Ar
      * If more than this amount in a row, we'll let out the exception rather
      * than go back in for yet another retry.
      */
-    public static final int MAX_ALLOWED_RECOVERABLES = 10;
+    public static final int MAX_ALLOWED_RECOVERABLES = 20;
     
 
     /**
diff --git a/src/main/java/org/archive/io/arc/ARCRecord.java b/src/main/java/org/archive/io/arc/ARCRecord.java
index 21bea07c..abe6a2d5 100644
--- a/src/main/java/org/archive/io/arc/ARCRecord.java
+++ b/src/main/java/org/archive/io/arc/ARCRecord.java
@@ -597,7 +597,7 @@ private InputStream readHttpHeader() throws IOException {
         	
         	// If it's actually the status line, break, otherwise continue skipping any
         	// previous header values
-        	if (!statusLine.contains(":") && StatusLine.startsWithHTTP(statusLine)) {
+        	if (StatusLine.startsWithHTTP(statusLine)) {
         		break;
         	}
         	
diff --git a/src/test/java/org/archive/extract/RealCDXExtractorOutputTest.java b/src/test/java/org/archive/extract/RealCDXExtractorOutputTest.java
index 14f8489d..35e5d85f 100644
--- a/src/test/java/org/archive/extract/RealCDXExtractorOutputTest.java
+++ b/src/test/java/org/archive/extract/RealCDXExtractorOutputTest.java
@@ -25,4 +25,18 @@ public void testNoDoubleEscaping() throws Exception {
 	String resolved = RealCDXExtractorOutput.resolve(spec, spec);
 	assertTrue(spec.equals(resolved));
     }
+    
+    public void ignoreTestMoreBrokenUrls() throws Exception {
+	String[] tests = new String[] { "mailto: john.doe@uni-oldenburg.de", 
+	"mailto:john.doe @Informatik.Uni-Oldenburg.DE",
+	"mailto:john.doe@blicher Tbinger Anhang",
+	"mailto:john.doe@uni-oldenburg.de?subject=Antrag auf SAP Zugang",
+	"E:/SmartSource Data Collector/util/content/wt_dcs.gif",
+	"ttp://find.galegroup.com/bncn/infomark.do?serQuery=Locale%28en%2C%2C%29%3AFQE%3D%28JX%2CNone%2C16%29%22Dublin Gazette%22%24&queryType=PH&type=pubIssues&prodId=BBCN&version=1.0&source=library"};
+	for (String test : tests) {
+		String escaped = RealCDXExtractorOutput.resolve(null, test);
+		System.out.println(test + " -> " + escaped);
+		assertTrue(escaped.indexOf(" ") < 0);
+	}
+    }
 }