moving org.archive.net.PublicSuffixes to ia-web-commons

Author: nlevitt

src/main/java/org/archive/net/PublicSuffixes.java

@@ -0,0 +1,363 @@
+/*
+ *  This file is part of the Heritrix web crawler (crawler.archive.org).
+ *
+ *  Licensed to the Internet Archive (IA) by one or more individual 
+ *  contributors. 
+ *
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.archive.net;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.FileInputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.io.IOUtils;
+import org.archive.util.TextUtils;
+
+/**
+ * Utility class for making use of the information about 'public suffixes' at
+ * http://publicsuffix.org.
+ * 
+ * The public suffix list (once known as 'effective TLDs') was motivated by the
+ * need to decide on which broader domains a subdomain was allowed to set
+ * cookies. For example, a server at 'www.example.com' can set cookies for
+ * 'www.example.com' or 'example.com' but not 'com'. 'www.example.co.uk' can set
+ * cookies for 'www.example.co.uk' or 'example.co.uk' but not 'co.uk' or 'uk'.
+ * The number of rules for all top-level-domains and 2nd- or 3rd- level domains
+ * has become quite long; essentially the broadest domain a subdomain may assign
+ * to is the one that was sold/registered to a specific name registrant.
+ * 
+ * This concept should be useful in other contexts, too. Grouping URIs (or
+ * queues of URIs to crawl) together with others sharing the same registered
+ * suffix may be useful for applying the same rules to all, such as assigning
+ * them to the same queue or crawler in a multi- machine setup.
+ * 
+ * As of Heritrix3, we prefer the term 'Assignment Level Domain' (ALD) 
+ * for such domains, by analogy to 'Top Level Domain' (TLD) or '2nd Level 
+ * Domain' (2LD), etc. 
+ * 
+ * @author Gojomo
+ * 
+ * this version of PublicSuffixes uses suffix-tree data structure for generating less
+ * redundant regular expression. It may be even possible to write a light-weight,
+ * thread-safe matcher based on this class.
+ * @author Kenji Nagahashi 
+ */
+public class PublicSuffixes {
+    protected static Pattern topmostAssignedSurtPrefixPattern;
+    protected static String topmostAssignedSurtPrefixRegex;
+
+    /**
+     * prefix tree node. each Node represents sequence of letters (prefix)
+     * and alternative sequences following it (list of Node's). Nodes in 
+     * {@code branches} are sorted for skip list like lookup and for generating
+     * effective regular expression (see {@link #compareTo(Node)} and {@link #compareTo(char).)
+     * 
+     * as is intended for internal use only, there's no access methods. procedures for updating
+     * prefix tree with new input are defined within this class ({@link #addBranch(CharSequence)}).
+     * 
+     * terminal node could be represented in two different form: 1) Node with zero branches,
+     * or 2) Node with zero-length {@code cs}. So, root node must be initialized with empty (not null)
+     * {@code branches} unless empty string matches the overall pattern. 
+     * {@code cs} must not be null except for root node. 
+     */
+    public static class Node implements Comparable<Node> {
+        protected CharSequence cs;
+        protected List<Node> branches;
+        public Node() {
+            this("", null);
+        }
+        protected Node(CharSequence cs) {
+            this(cs, null);
+        }
+        protected Node(CharSequence cs, List<Node> branches) {
+            this.cs = cs;
+            this.branches = branches;
+        }
+        public void addBranch(CharSequence s) {
+            if (branches == null) {
+                branches = new ArrayList<Node>();
+                branches.add(new Node("", null));
+            }
+            for (int i = 0; i < branches.size(); i++) {
+                Node alt = branches.get(i);
+                if (alt.add(s)) return;
+                if (alt.compareTo(s.charAt(0)) > 0) {
+                    Node alt1 = new Node(s, null);
+                    branches.add(i, alt1);
+                    return;
+                }
+            }
+            Node alt2 = new Node(s, null);
+            branches.add(alt2);
+        }
+        public boolean add(CharSequence s) {
+            int l = Math.min(s.length(), cs.length());
+            int i = 0;
+            while (i < l && s.charAt(i) == cs.charAt(i))
+                i++;
+            // zero-length match holds only when both cs and s are empty.
+            if (i == 0) return cs.length() == 0 && s.length() == 0;
+            if (i < cs.length()) {
+                CharSequence cs0 = cs.subSequence(0, i);
+                CharSequence cs1 = cs.subSequence(i, cs.length());
+                CharSequence cs2 = s.subSequence(i, s.length());
+                cs = cs0;
+                Node alt1 = new Node(cs1, branches);
+                (branches = new ArrayList<Node>()).add(alt1);
+                addBranch(cs2);
+            } else {
+                assert i == cs.length();
+                addBranch(s.subSequence(i, s.length()));
+            }
+            return true;
+        }
+        public int compareTo(Node other) {
+            if (other.cs == null || other.cs.length() == 0)
+                return (cs == null || cs.length() == 0) ? 0 : -1;
+            return compareTo(other.cs.charAt(0));
+        }
+        public int compareTo(char oc) {
+            if (cs == null || cs.length() == 0) return 1;
+            // '!' and '*' must come after ordinary letters, in this order, for regexp
+            // to work as intended.
+            char c = cs.charAt(0);
+            if (c == oc) return 0;
+            if (c == '!') return oc == '*' ? -1 : 1;
+            if (c == '*') return 1;
+            if (oc == '*' || oc == '!') return -1;
+            return Character.valueOf(c).compareTo(oc);
+            // for generating the same regexp as previous version.
+            //return Character.valueOf(oc).compareTo(c);
+        }        
+    }
+    
+    /**
+     * Utility method for dumping a regex String, based on a published public
+     * suffix list, which matches any SURT-form hostname up through the broadest
+     * 'private' (assigned/sold) domain-segment. That is, for any of the
+     * SURT-form hostnames...
+     * 
+     * com,example, com,example,www, com,example,california,www
+     * 
+     * ...the regex will match 'com,example,'.
+     * 
+     * @param args
+     * @throws IOException
+     */
+    public static void main(String args[]) throws IOException {
+        InputStream is;
+        if (args.length == 0 || "=".equals(args[0])) {
+            // use bundled list
+            is = PublicSuffixes.class.getClassLoader().getResourceAsStream(
+            "effective_tld_names.dat");
+        } else {
+            is = new FileInputStream(args[0]);
+        }
+        BufferedReader reader = new BufferedReader(new InputStreamReader(is, "UTF-8"));
+        String regex = getTopmostAssignedSurtPrefixRegex(reader);
+        IOUtils.closeQuietly(is);
+        
+        boolean needsClose = false;
+        BufferedWriter writer;
+        if (args.length >= 2) {
+            // write to specified file
+            writer = new BufferedWriter(new FileWriter(args[1]));
+            needsClose = true;
+        } else {
+            // write to stdout
+            writer = new BufferedWriter(new OutputStreamWriter(System.out));
+        }
+        writer.append(regex);
+        writer.flush();
+        if (needsClose) {
+            writer.close();
+        }
+    }
+    /**
+     * Reads a file of the format promulgated by publicsuffix.org, ignoring
+     * comments and '!' exceptions/notations, converting domain segments to
+     * SURT-ordering. Leaves glob-style '*' wildcarding in place. Returns root
+     * node of SURT-ordered prefix tree.
+     * 
+     * @param reader
+     * @return root of prefix tree node.
+     * @throws IOException
+     */
+    protected static Node readPublishedFileToSurtTrie(BufferedReader reader) throws IOException {
+        // initializing with empty Alt list prevents empty pattern from being
+        // created for the first addBranch()
+        Node alt = new Node(null, new ArrayList<Node>());
+        String line;
+        while ((line = reader.readLine()) != null) {
+            // discard whitespace, empty lines, comments, exceptions
+            line = line.trim();
+            if (line.length() == 0 || line.startsWith("//")) continue;
+            // discard utf8 notation after entry
+            line = line.split("\\s+")[0];
+            // TODO: maybe we don't need to create lower-cased String
+            line = line.toLowerCase();
+            // SURT-order domain segments
+            String[] segs = line.split("\\.");
+            StringBuilder sb = new StringBuilder();
+            for (int i = segs.length - 1; i >= 0; i--) {
+                if (segs[i].length() == 0) continue;
+                sb.append(segs[i]).append(',');
+            }
+            alt.addBranch(sb.toString());
+        }
+        return alt;
+    }
+    /**
+     * utility function for dumping prefix tree structure. intended for debug use.
+     * @param alt root of prefix tree.
+     * @param lv indent level. 0 for root (no indent).
+     * @param out writer to send output to.
+     */
+    public static void dump(Node alt, int lv, PrintWriter out) {
+        for (int i = 0; i < lv; i++)
+            out.print("  ");
+        out.println(alt.cs != null ? ('"'+alt.cs.toString()+'"') : "(null)");
+        if (alt.branches != null) {
+            for (Node br : alt.branches) {
+                dump(br, lv + 1, out);
+            }
+        }
+    }
+    /**
+     * bulids regular expression from prefix-tree {@code alt} into buffer {@code sb}.
+     * @param alt prefix tree root.
+     * @param sb StringBuffer to store regular expression.
+     */
+    protected static void buildRegex(Node alt, StringBuilder sb) {
+        String close = null;
+        if (alt.cs != null) {
+            // actually '!' always be the first character, because it is
+            // always used along with '*'.
+            for (int i = 0; i < alt.cs.length(); i++) {
+                char c = alt.cs.charAt(i);
+                if (c == '!') {
+                    if (close != null)
+                        throw new RuntimeException("more than one '!'");
+                    sb.append("(?=");
+                    close = ")";
+                } else if (c == '*') {
+                    sb.append("[-\\w]+");
+                } else {
+                    sb.append(c);
+                }
+            }
+        }
+        if (alt.branches != null) {
+            // alt.branches.size() should always be > 1
+            if (alt.branches.size() > 1) {
+                sb.append("(?:");
+            }
+            String sep = "";
+            for (Node alt1 : alt.branches) {
+                sb.append(sep); sep = "|";
+                buildRegex(alt1, sb);
+            }
+            if (alt.branches.size() > 1) {
+                sb.append(")");
+            }
+        }
+        if (close != null)
+            sb.append(close);
+    }
+
+    /**
+     * Converts SURT-ordered list of public prefixes into a Java regex which
+     * matches the public-portion "plus one" segment, giving the domain on which
+     * cookies can be set or other policy grouping should occur. Also adds to
+     * regex a fallback matcher that for any new/unknown TLDs assumes the
+     * second-level domain is assignable. (Eg: 'zzz,example,').
+     * 
+     * @param list
+     * @return
+     */
+    private static String surtPrefixRegexFromTrie(Node trie) {
+        StringBuilder regex = new StringBuilder();
+        regex.append("(?ix)^\n");
+        trie.addBranch("*,"); // for new/unknown TLDs
+        buildRegex(trie, regex);
+        regex.append("\n([-\\w]+,)");
+        return regex.toString();
+    }
+
+    public static synchronized Pattern getTopmostAssignedSurtPrefixPattern() {
+        if (topmostAssignedSurtPrefixPattern == null) {
+            topmostAssignedSurtPrefixPattern = Pattern
+                    .compile(getTopmostAssignedSurtPrefixRegex());
+        }
+        return topmostAssignedSurtPrefixPattern;
+    }
+
+    public static synchronized String getTopmostAssignedSurtPrefixRegex() {
+        if (topmostAssignedSurtPrefixRegex == null) {
+            // use bundled list
+            try {
+                BufferedReader reader = new BufferedReader(new InputStreamReader(
+                        PublicSuffixes.class.getClassLoader().getResourceAsStream(
+                        "effective_tld_names.dat"), "UTF-8"));
+                topmostAssignedSurtPrefixRegex = getTopmostAssignedSurtPrefixRegex(reader);
+                IOUtils.closeQuietly(reader);
+            } catch (UnsupportedEncodingException ex) {
+                // should never happen
+                throw new RuntimeException(ex);
+            }
+        }
+        return topmostAssignedSurtPrefixRegex;
+    }
+
+    public static String getTopmostAssignedSurtPrefixRegex(BufferedReader reader) {
+        try {
+            Node trie = readPublishedFileToSurtTrie(reader);
+            return surtPrefixRegexFromTrie(trie);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * Truncate SURT to its topmost assigned domain segment; that is, 
+     * the public suffix plus one segment, but as a SURT-ordered prefix. 
+     * 
+     * if the pattern doesn't match, the passed-in SURT is returned.
+     * 
+     * @param surt SURT to truncate
+     * @return truncated-to-topmost-assigned SURT prefix
+     */
+    public static String reduceSurtToAssignmentLevel(String surt) {
+        Matcher matcher = TextUtils.getMatcher(
+                getTopmostAssignedSurtPrefixRegex(), surt);
+        if (matcher.find()) {
+            surt = matcher.group();
+        }
+        TextUtils.recycleMatcher(matcher);
+        return surt;
+    }
+}