WarcCdxWriter: extraction of redirect targets for CDX should not be case-sensitive (#18)

- make extraction of HTTP headers not depend on correct casing for:
  - "Location" and "Content-Type" (WarcCdxWriter: "redirect" and "mime")
  - "Content-Type" (support for language detector)
- refactor: header names as constants

Author: sebastian-nagel

src/java/org/commoncrawl/util/LanguageDetector.java

@@ -97,7 +97,8 @@ protected Result detectLanguage(URI uri, Content content) {
 
     String httpContentLanguage = content.getMetadata()
         .get(Response.CONTENT_LANGUAGE);
-    String httpContentType = content.getMetadata().get(Response.CONTENT_TYPE);
+    String httpContentType = WarcWriter.getMeta(content.getMetadata(),
+        Response.CONTENT_TYPE);
 
     Metadata metadata = new Metadata();
     if (httpContentType != null) {
@@ -107,7 +108,7 @@ protected Result detectLanguage(URI uri, Content content) {
     String text;
     byte[] bytes = content.getContent();
     try (AutoDetectReader charsetDetectReader = new AutoDetectReader(
-        new ByteArrayInputStream(bytes), metadata);) {
+        new ByteArrayInputStream(bytes), metadata)) {
       result.charset = charsetDetectReader.getCharset();
       boolean isValidUtf8 = false;
       if (result.charset.equals(StandardCharsets.UTF_8)) {

src/java/org/commoncrawl/util/WarcCdxWriter.java

@@ -18,10 +18,8 @@
 
 import java.io.IOException;
 import java.io.OutputStream;
-import java.net.MalformedURLException;
 import java.net.URI;
 import java.net.URISyntaxException;
-import java.net.URL;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.text.SimpleDateFormat;
@@ -34,6 +32,7 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.nutch.metadata.Metadata;
+import org.apache.nutch.net.protocols.Response;
 import org.apache.nutch.protocol.Content;
 import org.archive.url.WaybackURLKeyMaker;
 import org.slf4j.Logger;
@@ -120,15 +119,7 @@ public URI writeWarcResponseRecord(final URI targetUri, final String ip,
     long length = (countingOut.getByteCount() - offset);
     String redirectLocation = null;
     if (isRedirect(httpStatusCode)) {
-      redirectLocation = content.getMetadata().get("Location");
-      if (redirectLocation != null) {
-        try {
-          redirectLocation = new URL(targetUri.toURL(), redirectLocation)
-              .toURI().toString();
-        } catch (URISyntaxException | MalformedURLException e) {
-          redirectLocation = null;
-        }
-      }
+      redirectLocation = getMeta(content.getMetadata(), "Location");
     }
     writeCdxLine(targetUri, date, offset, length, payloadDigest, content, false,
         redirectLocation, truncated);
@@ -162,21 +153,21 @@ public void writeCdxLine(final URI targetUri, final Date date, long offset,
     if (revisit) {
       data.put("mime", "warc/revisit");
     } else {
-      data.put("mime", cleanMimeType(meta.get("Content-Type")));
+      data.put("mime", cleanMimeType(getMeta(meta, Response.CONTENT_TYPE)));
       data.put("mime-detected", content.getContentType());
     }
-    data.put("status", meta.get("HTTP-Status-Code"));
+    data.put("status", meta.get(WarcWriter.HTTP_STATUS_CODE));
     if (payloadDigest != null) {
       data.put("digest", payloadDigest);
     }
     data.put("length", String.format("%d", length));
     data.put("offset", String.format("%d", offset));
     data.put("filename", warcFilename);
-    String val = meta.get("Detected-Charset");
+    String val = meta.get(WarcWriter.DETECTED_CHARSET);
     if (val != null) {
       data.put("charset", val);
     }
-    val = meta.get("Detected-Language");
+    val = meta.get(WarcWriter.DETECTED_LANGUAGE);
     if (val != null) {
       data.put("languages", val);
     }

src/java/org/commoncrawl/util/WarcRecordWriter.java

@@ -637,7 +637,7 @@ public synchronized void write(Text key, WarcCapture value)
     }
 
     if (generateCdx) {
-      value.content.getMetadata().add("HTTP-Status-Code",
+      value.content.getMetadata().add(WarcWriter.HTTP_STATUS_CODE,
           String.format("%d", httpStatusCode));
     }
 
@@ -660,14 +660,15 @@ public synchronized void write(Text key, WarcCapture value)
       }
       if (generateCdx) {
         if (ldres.charset != null) {
-          value.content.getMetadata().add("Detected-Charset",
+          value.content.getMetadata().add(WarcWriter.DETECTED_CHARSET,
               ldres.charset.name());
         }
         org.commoncrawl.langdetect.cld2.Result lr = ldres.languages;
         if (lr != null) {
           String codes = lr.getLanguageCodesISO639_3(",", true);
           if (codes != null && !codes.isEmpty()) {
-            value.content.getMetadata().add("Detected-Language", codes);
+            value.content.getMetadata().add(WarcWriter.DETECTED_LANGUAGE,
+                codes);
           }
         }
       }

src/java/org/commoncrawl/util/WarcWriter.java

@@ -32,6 +32,7 @@
 import java.util.UUID;
 import java.util.zip.GZIPOutputStream;
 
+import org.apache.nutch.metadata.Metadata;
 import org.apache.nutch.protocol.Content;
 
 public class WarcWriter {
@@ -74,6 +75,11 @@ public class WarcWriter {
   private static final String CRLF = "\r\n";
   private static final String COLONSP = ": ";
 
+  /* Metadata names to pass from WARC to CDX */
+  protected static final String HTTP_STATUS_CODE = "HTTP-Status-Code";
+  protected static final String DETECTED_CHARSET = "Detected-Charset";
+  protected static final String DETECTED_LANGUAGE = "Detected-Language";
+
   private SimpleDateFormat isoDate;
 
   public static class CompressedOutputStream extends GZIPOutputStream {
@@ -374,4 +380,19 @@ public URI getRecordId() {
       throw new RuntimeException(e);
     }
   }
+
+  protected static String getMeta(Metadata metadata, String name) {
+    String value = metadata.get(name);
+    if (value == null) {
+      // check for case variants
+      for (String n : metadata.names()) {
+        if (n.equalsIgnoreCase(name)) {
+          value = metadata.get(n);
+          break;
+        }
+      }
+    }
+    return value;
+  }
+
 }