NUTCH-1011 Remove duplicate slashes from URLs

git-svn-id: https://svn.apache.org/repos/asf/nutch/trunk@1143468 13f79535-47bb-0310-9956-ffa450edef68

Author: Markus Jelsma

CHANGES.txt

@@ -2,6 +2,8 @@ Nutch Change Log
 
 Release 2.0 - Current Development
 
+* NUTCH-1011 Normalize duplicate slashes in URL's (markus)
+
 * NUTCH-1013 Migrate RegexURLNormalizer from Apache ORO to java.util.regex (markus)
 
 * NUTCH-1016 Strip UTF-8 non-character codepoints and add logging for SolrWriter (markus)

conf/regex-normalize.xml.template

@@ -63,4 +63,10 @@
   <substitution></substitution>
 </regex>
 
+<!-- removes duplicate slashes -->
+<regex>
+  <pattern>(?&lt;!:)/{2,}</pattern>
+  <substitution>/</substitution>
+</regex>
+
 </regex-normalize>

src/test/org/apache/nutch/net/TestURLNormalizers.java

@@ -39,6 +39,15 @@ public void testURLNormalizers() {
     } catch (MalformedURLException mue) {
       fail(mue.toString());
     }
+
+    // NUTCH-1011 - Get rid of superfluous slashes
+    try {
+      String normalizedSlashes = normalizers.normalize("http://www.example.org//path/to//somewhere.html", URLNormalizers.SCOPE_DEFAULT);
+      assertEquals(normalizedSlashes, "http://www.example.org/path/to/somewhere.html");
+    } catch (MalformedURLException mue) {
+      fail(mue.toString());
+    }
+
     // check the order
     int pos1 = -1, pos2 = -1;
     URLNormalizer[] impls = normalizers.getURLNormalizers(URLNormalizers.SCOPE_DEFAULT);