Filters URLs based on a file of regular expressions using host/domains matching first. The + * default policy is to accept a URL if no matches are found. + * + *

Rule Format: * - * Rule Format: - * *

  * Host www.example.org
  *   DenyPath /path/to/be/excluded
@@ -72,48 +68,45 @@
  * Domain example.org
  *   DenyPathQuery /resource/.*?action=exclude
  * 

E.g., for "www.example.com" the rules given above are looked up in the following order: + * *

- *
1. check "www.example.com" whether host-based rules exist and whether one of - * them matches
- *
2. check "www.example.com" for domain-based rules
- *
3. check "example.com" for domain-based rules
- *
4. check "com" for domain-based rules
- *
5. check for global rules ("Domain .")
+ *
6. check "www.example.com" whether host-based rules exist and whether one of them matches + *
7. check "www.example.com" for domain-based rules + *
8. check "example.com" for domain-based rules + *
9. check "com" for domain-based rules + *
10. check for global rules ("Domain .") *

For rules either the URL path (DenyPath) or path and query (DenyPathQuery + * ) are checked whether the given {@link java.util.regex Java Regular expression} is found + * (see {@link java.util.regex.Matcher#find()}) in the URL path (and query). + * + *

Rules are applied in the order of their definition. For better performance, regular + * expressions which are simpler/faster or match more URLs should be defined earlier. + * + *

Comments in the rule file start with the # character and reach until the end of + * the line. + * + *

The rules file is defined via the property urlfilter.fast.file, the default name + * is fast-urlfilter.txt. */ -public class FastURLFilter extends URLFilter implements JSONResource { +public class FastURLFilter extends URLFilter implements JSONResource { - protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + protected static final Logger LOG = + LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); public static final String URLFILTER_FAST_FILE = "urlfilter.fast.file"; private Multimap hostRules = LinkedHashMultimap.create(); @@ -121,302 +114,320 @@ public class FastURLFilter extends URLFilter implements JSONResource { private String resourceFile; - private static final Pattern CATCH_ALL_RULE = Pattern.compile("^\\s*DenyPath(?:Query)?\\s+\\.[*?]\\s*$"); + private static final Pattern CATCH_ALL_RULE = + Pattern.compile("^\\s*DenyPath(?:Query)?\\s+\\.[*?]\\s*$"); private String resourceETAG; public void configure(@SuppressWarnings("rawtypes") Map stormConf, JsonNode filterParams) { - // read from conf first - int refreshRate = ConfUtils.getInt(stormConf, "fast.urlfilter.refresh", -1); - this.resourceFile = ConfUtils.getString(stormConf, "fast.urlfilter.file", null); - - // then from the param file (which needs recompiling in case of change) - if (filterParams != null) { - JsonNode node = filterParams.get("file"); - if (node != null && node.isTextual() && this.resourceFile == null) { - this.resourceFile = node.asText(); - } - node = filterParams.get("refresh"); - if (node != null && node.isInt() && refreshRate == -1) { - refreshRate = node.asInt(); - } - } - - try { - loadJSONResources(); - } catch (Exception e) { - LOG.error("Exception while loading resources", e); - } - - if (refreshRate != -1) { - LOG.info("Filter set to reload from {} every {} sec", getResourceFile(), refreshRate); - new Timer().schedule(new TimerTask() { - public void run() { - LOG.info("Reloading resources"); - try { - loadJSONResources(); - } catch (Exception e) { - LOG.error("Can't load resources", e); - } - } - }, refreshRate * 1000, refreshRate * 1000); - } + // read from conf first + int refreshRate = ConfUtils.getInt(stormConf, "fast.urlfilter.refresh", -1); + this.resourceFile = ConfUtils.getString(stormConf, "fast.urlfilter.file", null); + + // then from the param file (which needs recompiling in case of change) + if (filterParams != null) { + JsonNode node = filterParams.get("file"); + if (node != null && node.isTextual() && this.resourceFile == null) { + this.resourceFile = node.asText(); + } + node = filterParams.get("refresh"); + if (node != null && node.isInt() && refreshRate == -1) { + refreshRate = node.asInt(); + } + } + + try { + loadJSONResources(); + } catch (Exception e) { + LOG.error("Exception while loading resources", e); + } + + if (refreshRate != -1) { + LOG.info("Filter set to reload from {} every {} sec", getResourceFile(), refreshRate); + new Timer() + .schedule( + new TimerTask() { + public void run() { + LOG.info("Reloading resources"); + try { + loadJSONResources(); + } catch (Exception e) { + LOG.error("Can't load resources", e); + } + } + }, + refreshRate * 1000, + refreshRate * 1000); + } } /** * Load the resources from the JSON file in the uber jar or from S3 - * + * * @throws Exception - **/ + */ @Override public void loadJSONResources() throws Exception { - InputStream inputStream = null; - AmazonS3 s3client = null; - try { - if (getResourceFile().startsWith("s3://")) { - // try loading from S3 - s3client = AmazonS3ClientBuilder.standard().build(); - java.net.URI uri = new java.net.URI(getResourceFile()); - - String bucketName = uri.getHost(); - // remove the first "/" - String path = uri.getPath().substring(1); - - // optimisation - avoid a full reload if the resource has not changed - ObjectMetadata metadata = s3client.getObjectMetadata(bucketName, path); - final String ETAG = metadata.getETag(); - if (ETAG != null && ETAG.equals(resourceETAG)) { - LOG.info("Unchanged ETAG for {} - skipping reload", getResourceFile()); - return; - } else { - resourceETAG = ETAG; - } - - final S3Object object = s3client.getObject(new GetObjectRequest(bucketName, path)); - inputStream = object.getObjectContent(); - } else { - inputStream = getClass().getClassLoader().getResourceAsStream(getResourceFile()); - if (inputStream == null) { - LOG.error("Can't load conf from {}", getResourceFile()); - return; - } - } - if (getResourceFile().endsWith(".gz")) { - inputStream = new GZIPInputStream(inputStream); - } - - loadJSONResources(new BufferedInputStream(inputStream)); - } finally { - if (inputStream != null) { - inputStream.close(); - } - if (s3client != null) { - s3client.shutdown(); - } - } + InputStream inputStream = null; + AmazonS3 s3client = null; + try { + if (getResourceFile().startsWith("s3://")) { + // try loading from S3 + s3client = AmazonS3ClientBuilder.standard().build(); + java.net.URI uri = new java.net.URI(getResourceFile()); + + String bucketName = uri.getHost(); + // remove the first "/" + String path = uri.getPath().substring(1); + + // optimisation - avoid a full reload if the resource has not changed + ObjectMetadata metadata = s3client.getObjectMetadata(bucketName, path); + final String ETAG = metadata.getETag(); + if (ETAG != null && ETAG.equals(resourceETAG)) { + LOG.info("Unchanged ETAG for {} - skipping reload", getResourceFile()); + return; + } else { + resourceETAG = ETAG; + } + + final S3Object object = s3client.getObject(new GetObjectRequest(bucketName, path)); + inputStream = object.getObjectContent(); + } else { + inputStream = getClass().getClassLoader().getResourceAsStream(getResourceFile()); + if (inputStream == null) { + LOG.error("Can't load conf from {}", getResourceFile()); + return; + } + } + if (getResourceFile().endsWith(".gz")) { + inputStream = new GZIPInputStream(inputStream); + } + + loadJSONResources(new BufferedInputStream(inputStream)); + } finally { + if (inputStream != null) { + inputStream.close(); + } + if (s3client != null) { + s3client.shutdown(); + } + } } @Override public void loadJSONResources(InputStream inputStream) - throws JsonParseException, JsonMappingException, IOException { - long start = System.currentTimeMillis(); - - try (Reader r = new InputStreamReader(inputStream)) { - reloadRules(r); - } - - long end = System.currentTimeMillis(); - LOG.info("Loaded {} hostrules and {} domain rules in {} msec from {}", hostRules.size(), domainRules.size(), - (end - start), resourceFile); + throws JsonParseException, JsonMappingException, IOException { + long start = System.currentTimeMillis(); + + try (Reader r = new InputStreamReader(inputStream)) { + reloadRules(r); + } + + long end = System.currentTimeMillis(); + LOG.info( + "Loaded {} hostrules and {} domain rules in {} msec from {}", + hostRules.size(), + domainRules.size(), + (end - start), + resourceFile); } @Override public String getResourceFile() { - return resourceFile; + return resourceFile; } @Override public String filter(URL sourceUrl, Metadata sourceMetadata, String urlToFilter) { - synchronized (this) { - URL u; - - try { - u = new URL(urlToFilter); - } catch (Exception e) { - LOG.debug("Rejected {} because failed to parse as URL: {}", urlToFilter, e.getMessage()); - return null; - } - - String hostname = u.getHost(); - - // first check for host-specific rules - for (Rule rule : hostRules.get(hostname)) { - if (rule.match(u)) { - return null; - } - } - - // also look up domain rules for host name - for (Rule rule : domainRules.get(hostname)) { - if (rule.match(u)) { - return null; - } - } - - // check suffixes of host name from longer to shorter: - // subdomains, domain, top-level domain - int start = 0; - int pos; - while ((pos = hostname.indexOf('.', start)) != -1) { - start = pos + 1; - String domain = hostname.substring(start); - for (Rule rule : domainRules.get(domain)) { - if (rule.match(u)) { - return null; - } - } - } - - // finally check "global" rules defined for `Domain .` - for (Rule rule : domainRules.get(".")) { - if (rule.match(u)) { - return null; - } - } - - // no reject rules found - return urlToFilter; - } + synchronized (this) { + URL u; + + try { + u = new URL(urlToFilter); + } catch (Exception e) { + LOG.debug( + "Rejected {} because failed to parse as URL: {}", + urlToFilter, + e.getMessage()); + return null; + } + + String hostname = u.getHost(); + + // first check for host-specific rules + for (Rule rule : hostRules.get(hostname)) { + if (rule.match(u)) { + return null; + } + } + + // also look up domain rules for host name + for (Rule rule : domainRules.get(hostname)) { + if (rule.match(u)) { + return null; + } + } + + // check suffixes of host name from longer to shorter: + // subdomains, domain, top-level domain + int start = 0; + int pos; + while ((pos = hostname.indexOf('.', start)) != -1) { + start = pos + 1; + String domain = hostname.substring(start); + for (Rule rule : domainRules.get(domain)) { + if (rule.match(u)) { + return null; + } + } + } + + // finally check "global" rules defined for `Domain .` + for (Rule rule : domainRules.get(".")) { + if (rule.match(u)) { + return null; + } + } + + // no reject rules found + return urlToFilter; + } } private void reloadRules(Reader rules) throws IOException { - synchronized (this) { - domainRules.clear(); - hostRules.clear(); - - BufferedReader reader = new BufferedReader(rules); - - String current = null; - boolean host = false; - int lineno = 0; - - String line; - try { - while ((line = reader.readLine()) != null) { - lineno++; - line = line.trim(); - - if (line.indexOf("#") != -1) { - // strip comments - line = line.substring(0, line.indexOf("#")).trim(); - } - - if (StringUtils.isBlank(line)) { - continue; - } - - if (line.startsWith("Host")) { - host = true; - current = line.split("\\s+")[1]; - } else if (line.startsWith("Domain")) { - host = false; - current = line.split("\\s+")[1]; - } else { - if (current == null) { - continue; - } - - Rule rule = null; - try { - if (CATCH_ALL_RULE.matcher(line).matches()) { - rule = DenyAllRule.getInstance(); - } else if (line.startsWith("DenyPathQuery")) { - rule = new DenyPathQueryRule(line.split("\\s+")[1]); - } else if (line.startsWith("DenyPath")) { - rule = new DenyPathRule(line.split("\\s+")[1]); - } else { - LOG.warn("Problem reading rule on line {}: {}", lineno, line); - continue; - } - } catch (Exception e) { - LOG.warn("Problem reading rule on line {}: {} - {}", lineno, line, e.getMessage()); - continue; - } - - if (host) { - LOG.trace("Adding host rule [{}] [{}]", current, rule); - hostRules.put(current, rule); - } else { - LOG.trace("Adding domain rule [{}] [{}]", current, rule); - domainRules.put(current, rule); - } - } - } - - } catch (IOException e) { - LOG.warn("Caught exception while reading rules file at line {}: {}", lineno, e.getMessage()); - throw e; - } - } + synchronized (this) { + domainRules.clear(); + hostRules.clear(); + + BufferedReader reader = new BufferedReader(rules); + + String current = null; + boolean host = false; + int lineno = 0; + + String line; + try { + while ((line = reader.readLine()) != null) { + lineno++; + line = line.trim(); + + if (line.indexOf("#") != -1) { + // strip comments + line = line.substring(0, line.indexOf("#")).trim(); + } + + if (StringUtils.isBlank(line)) { + continue; + } + + if (line.startsWith("Host")) { + host = true; + current = line.split("\\s+")[1]; + } else if (line.startsWith("Domain")) { + host = false; + current = line.split("\\s+")[1]; + } else { + if (current == null) { + continue; + } + + Rule rule = null; + try { + if (CATCH_ALL_RULE.matcher(line).matches()) { + rule = DenyAllRule.getInstance(); + } else if (line.startsWith("DenyPathQuery")) { + rule = new DenyPathQueryRule(line.split("\\s+")[1]); + } else if (line.startsWith("DenyPath")) { + rule = new DenyPathRule(line.split("\\s+")[1]); + } else { + LOG.warn("Problem reading rule on line {}: {}", lineno, line); + continue; + } + } catch (Exception e) { + LOG.warn( + "Problem reading rule on line {}: {} - {}", + lineno, + line, + e.getMessage()); + continue; + } + + if (host) { + LOG.trace("Adding host rule [{}] [{}]", current, rule); + hostRules.put(current, rule); + } else { + LOG.trace("Adding domain rule [{}] [{}]", current, rule); + domainRules.put(current, rule); + } + } + } + + } catch (IOException e) { + LOG.warn( + "Caught exception while reading rules file at line {}: {}", + lineno, + e.getMessage()); + throw e; + } + } } public static class Rule { - protected Pattern pattern; + protected Pattern pattern; - Rule() { - } + Rule() {} - public Rule(String regex) { - pattern = Pattern.compile(regex); - } + public Rule(String regex) { + pattern = Pattern.compile(regex); + } - public boolean match(URL url) { - return pattern.matcher(url.toString()).find(); - } + public boolean match(URL url) { + return pattern.matcher(url.toString()).find(); + } - public String toString() { - return pattern.toString(); - } + public String toString() { + return pattern.toString(); + } } public static class DenyPathRule extends Rule { - public DenyPathRule(String regex) { - super(regex); - } - - public boolean match(URL url) { - String haystack = url.getPath(); - return pattern.matcher(haystack).find(); - } + public DenyPathRule(String regex) { + super(regex); + } + + public boolean match(URL url) { + String haystack = url.getPath(); + return pattern.matcher(haystack).find(); + } } /** Rule for DenyPath .* or DenyPath .? */ public static class DenyAllRule extends Rule { - private static Rule instance = new DenyAllRule("."); + private static Rule instance = new DenyAllRule("."); - private DenyAllRule(String regex) { - super(regex); - } + private DenyAllRule(String regex) { + super(regex); + } - public static Rule getInstance() { - return instance; - } + public static Rule getInstance() { + return instance; + } - public boolean match(URL url) { - return true; - } + public boolean match(URL url) { + return true; + } } public static class DenyPathQueryRule extends Rule { - public DenyPathQueryRule(String regex) { - super(regex); - } - - public boolean match(URL url) { - String haystack = url.getFile(); - return pattern.matcher(haystack).find(); - } + public DenyPathQueryRule(String regex) { + super(regex); + } + + public boolean match(URL url) { + String haystack = url.getFile(); + return pattern.matcher(haystack).find(); + } } } diff --git a/src/main/java/org/commoncrawl/stormcrawler/news/ContentDetector.java b/src/main/java/org/commoncrawl/stormcrawler/news/ContentDetector.java index 1c3a4a5..9979781 100644 --- a/src/main/java/org/commoncrawl/stormcrawler/news/ContentDetector.java +++ b/src/main/java/org/commoncrawl/stormcrawler/news/ContentDetector.java @@ -13,33 +13,28 @@ */ package org.commoncrawl.stormcrawler.news; +import com.google.common.primitives.Bytes; import java.nio.charset.StandardCharsets; import java.util.Arrays; -import com.google.common.primitives.Bytes; - public class ContentDetector { protected byte[][][] clues; protected int maxOffset; /** - * Set up detector to detect content sniffing for a set of clue strings in a - * prefix of the binary content. + * Set up detector to detect content sniffing for a set of clue strings in a prefix of the + * binary content. * - * @param clues - * nested list of literal clues. Outer list defines an OR-group, - * inner list contained ANDed clues required to match all, e.g. - * the following definition would match if either - * "clue1" and "and_clue2" are matched, or - * alternatively "or_clue3" is found + * @param clues nested list of literal clues. Outer list defines an OR-group, inner list + * contained ANDed clues required to match all, e.g. the following definition would match if + * either "clue1" and "and_clue2" are matched, or alternatively + * "or_clue3" is found + *

+     *                  { { clue1, and_clue2 }, { or_clue3 } }
+     *                  

-     *            { { clue1, and_clue2 }, { or_clue3 } }
-     *            

  *     {
  *      "class": "org.commoncrawl.stormcrawler.news.bootstrap.FeedLinkParseFilter",
@@ -41,12 +40,10 @@
  */
 public class FeedLinkParseFilter extends LinkParseFilter {
 
-    private static final org.slf4j.Logger LOG = LoggerFactory
-            .getLogger(FeedLinkParseFilter.class);
+    private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(FeedLinkParseFilter.class);
 
     @Override
-    public void filter(String URL, byte[] content, DocumentFragment doc,
-            ParseResult parse) {
+    public void filter(String URL, byte[] content, DocumentFragment doc, ParseResult parse) {
 
         // skip existing links
         logLinks(parse, URL, "Skipped links");
@@ -60,11 +57,8 @@ public void filter(String URL, byte[] content, DocumentFragment doc,
 
     public static void logLinks(ParseResult parse, String URL, String message) {
         if (LOG.isDebugEnabled() && parse.getOutlinks().size() > 0) {
-            if (!message.isEmpty())
-                LOG.debug("{} for {}:", message, URL);
-            for (Outlink outlink : parse.getOutlinks())
-                LOG.debug(outlink.getTargetURL());
+            if (!message.isEmpty()) LOG.debug("{} for {}:", message, URL);
+            for (Outlink outlink : parse.getOutlinks()) LOG.debug(outlink.getTargetURL());
         }
     }
-
 }
diff --git a/src/main/java/org/commoncrawl/stormcrawler/news/bootstrap/NewsSiteMapDetectorBolt.java b/src/main/java/org/commoncrawl/stormcrawler/news/bootstrap/NewsSiteMapDetectorBolt.java
index 1160201..f0af134 100644
--- a/src/main/java/org/commoncrawl/stormcrawler/news/bootstrap/NewsSiteMapDetectorBolt.java
+++ b/src/main/java/org/commoncrawl/stormcrawler/news/bootstrap/NewsSiteMapDetectorBolt.java
@@ -14,15 +14,10 @@
 package org.commoncrawl.stormcrawler.news.bootstrap;
 
 import java.util.Map;
-
 import org.apache.storm.task.OutputCollector;
 import org.apache.storm.task.TopologyContext;
 import org.apache.storm.tuple.Tuple;
 import org.apache.storm.tuple.Values;
-import org.commoncrawl.stormcrawler.news.ContentDetector;
-import org.commoncrawl.stormcrawler.news.NewsSiteMapParserBolt;
-import org.slf4j.LoggerFactory;
-
 import org.apache.stormcrawler.Constants;
 import org.apache.stormcrawler.Metadata;
 import org.apache.stormcrawler.bolt.SiteMapParserBolt;
@@ -31,25 +26,26 @@
 import org.apache.stormcrawler.parse.ParseFilters;
 import org.apache.stormcrawler.parse.ParseResult;
 import org.apache.stormcrawler.persistence.Status;
+import org.commoncrawl.stormcrawler.news.ContentDetector;
+import org.commoncrawl.stormcrawler.news.NewsSiteMapParserBolt;
+import org.slf4j.LoggerFactory;
 
 /**
- * Detector for news
+ * Detector for news
  * sitemaps and also sitemaps.
  */
 @SuppressWarnings("serial")
 public class NewsSiteMapDetectorBolt extends SiteMapParserBolt {
 
-    private static final org.slf4j.Logger LOG = LoggerFactory
-            .getLogger(NewsSiteMapDetectorBolt.class);
+    private static final org.slf4j.Logger LOG =
+            LoggerFactory.getLogger(NewsSiteMapDetectorBolt.class);
 
     protected static final int maxOffsetContentGuess = 1024;
-    private static ContentDetector contentDetector = new ContentDetector(
-            NewsSiteMapParserBolt.contentClues, maxOffsetContentGuess);
+    private static ContentDetector contentDetector =
+            new ContentDetector(NewsSiteMapParserBolt.contentClues, maxOffsetContentGuess);
 
     private ParseFilter parseFilters;
 
-
     @Override
     public void execute(Tuple tuple) {
         Metadata metadata = (Metadata) tuple.getValueByField("metadata");
@@ -57,10 +53,9 @@ public void execute(Tuple tuple) {
         byte[] content = tuple.getBinaryByField("content");
         String url = tuple.getStringByField("url");
 
-        boolean isSitemap = Boolean.valueOf(
-                metadata.getFirstValue(SiteMapParserBolt.isSitemapKey));
-        boolean isNewsSitemap = Boolean.valueOf(
-                metadata.getFirstValue(NewsSiteMapParserBolt.isSitemapNewsKey));
+        boolean isSitemap = Boolean.valueOf(metadata.getFirstValue(SiteMapParserBolt.isSitemapKey));
+        boolean isNewsSitemap =
+                Boolean.valueOf(metadata.getFirstValue(NewsSiteMapParserBolt.isSitemapNewsKey));
 
         if (!isNewsSitemap || !isSitemap) {
             int match = contentDetector.getFirstMatch(content);
@@ -70,10 +65,8 @@ public void execute(Tuple tuple) {
                 metadata.setValue(SiteMapParserBolt.isSitemapKey, "true");
                 if (match <= NewsSiteMapParserBolt.contentCluesSitemapNewsMatchUpTo) {
                     isNewsSitemap = true;
-                    LOG.info("{} detected as news sitemap based on content",
-                            url);
-                    metadata.setValue(NewsSiteMapParserBolt.isSitemapNewsKey,
-                            "true");
+                    LOG.info("{} detected as news sitemap based on content", url);
+                    metadata.setValue(NewsSiteMapParserBolt.isSitemapNewsKey, "true");
                 }
             }
         }
@@ -85,8 +78,8 @@ public void execute(Tuple tuple) {
             parseData.setMetadata(metadata);
             parseFilters.filter(url, content, null, parse);
             // emit status
-            collector.emit(Constants.StatusStreamName, tuple,
-                    new Values(url, metadata, Status.FETCHED));
+            collector.emit(
+                    Constants.StatusStreamName, tuple, new Values(url, metadata, Status.FETCHED));
         } else {
             // pass on
             collector.emit(tuple, tuple.getValues());
@@ -95,11 +88,9 @@ public void execute(Tuple tuple) {
     }
 
     @Override
-    @SuppressWarnings({ "rawtypes", "unchecked" })
-    public void prepare(Map stormConf, TopologyContext context,
-            OutputCollector collect) {
+    @SuppressWarnings({"rawtypes", "unchecked"})
+    public void prepare(Map stormConf, TopologyContext context, OutputCollector collect) {
         super.prepare(stormConf, context, collect);
         parseFilters = ParseFilters.fromConf(stormConf);
     }
-
 }
diff --git a/src/test/java/org/commoncrawl/stormcrawler/FastURLFilterTest.java b/src/test/java/org/commoncrawl/stormcrawler/FastURLFilterTest.java
index 64df0ba..58aaa04 100644
--- a/src/test/java/org/commoncrawl/stormcrawler/FastURLFilterTest.java
+++ b/src/test/java/org/commoncrawl/stormcrawler/FastURLFilterTest.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to DigitalPebble Ltd under one or more contributor license agreements. See the NOTICE
  * file distributed with this work for additional information regarding copyright ownership.
  * DigitalPebble licenses this file to You under the Apache License, Version 2.0 (the "License");
@@ -14,74 +14,72 @@
  */
 package org.commoncrawl.stormcrawler;
 
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.HashMap;
 import java.util.Map;
-
+import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.filtering.URLFilter;
 import org.commoncrawl.stormcrawler.filter.FastURLFilter;
 import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
-import org.apache.stormcrawler.Metadata;
-import org.apache.stormcrawler.filtering.URLFilter;
-import com.fasterxml.jackson.databind.node.JsonNodeFactory;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-
 public class FastURLFilterTest {
 
     protected static URLFilter filter;
 
     @BeforeClass
     public static void init() {
-	filter = createFilter("fast-urlfilter.txt");
+        filter = createFilter("fast-urlfilter.txt");
     }
 
     public static FastURLFilter createFilter(String fileName) {
-	ObjectNode filterParams = new ObjectNode(JsonNodeFactory.instance);
-	filterParams.put("file", fileName);
-	FastURLFilter filter = new FastURLFilter();
-	Map conf = new HashMap<>();
-	conf.put("fast.urlfilter.refresh", 10);
-	filter.configure(conf, filterParams);
-	return filter;
+        ObjectNode filterParams = new ObjectNode(JsonNodeFactory.instance);
+        filterParams.put("file", fileName);
+        FastURLFilter filter = new FastURLFilter();
+        Map conf = new HashMap<>();
+        conf.put("fast.urlfilter.refresh", 10);
+        filter.configure(conf, filterParams);
+        return filter;
     }
 
     @Test
     public void testHostFilter() throws MalformedURLException {
-	URL url = new URL("http://may.go.com/image.jpg");
-	Metadata metadata = new Metadata();
-	String filterResult = filter.filter(url, metadata, url.toExternalForm());
-	Assert.assertEquals(url.toString(), filterResult);
-	
-	url = new URL("http://no.go.com/");
-	filterResult = filter.filter(url, metadata, url.toExternalForm());
-	Assert.assertEquals(null, filterResult);
+        URL url = new URL("http://may.go.com/image.jpg");
+        Metadata metadata = new Metadata();
+        String filterResult = filter.filter(url, metadata, url.toExternalForm());
+        Assert.assertEquals(url.toString(), filterResult);
+
+        url = new URL("http://no.go.com/");
+        filterResult = filter.filter(url, metadata, url.toExternalForm());
+        Assert.assertEquals(null, filterResult);
     }
 
     @Test
     public void testDomainNotAllowed() throws MalformedURLException {
-	URL url = new URL("http://domainnotallowed.com/forum/search.php");
-	Metadata metadata = new Metadata();
-	String filterResult = filter.filter(url, metadata, url.toExternalForm());
-	Assert.assertEquals(null, filterResult);
-	
-	url = new URL("http://domainnotallowed.com/");
-	filterResult = filter.filter(url, metadata, url.toExternalForm());
-	Assert.assertEquals(null, filterResult);
-	
-	url = new URL("http://partiallyallowed.com/");
-	filterResult = filter.filter(url, metadata, url.toExternalForm());
-	Assert.assertEquals(url.toString(), filterResult);
-	
-	url = new URL("http://partiallyallowed.com/verbotten");
-	filterResult = filter.filter(url, metadata, url.toExternalForm());
-	Assert.assertEquals(null, filterResult);
+        URL url = new URL("http://domainnotallowed.com/forum/search.php");
+        Metadata metadata = new Metadata();
+        String filterResult = filter.filter(url, metadata, url.toExternalForm());
+        Assert.assertEquals(null, filterResult);
+
+        url = new URL("http://domainnotallowed.com/");
+        filterResult = filter.filter(url, metadata, url.toExternalForm());
+        Assert.assertEquals(null, filterResult);
+
+        url = new URL("http://partiallyallowed.com/");
+        filterResult = filter.filter(url, metadata, url.toExternalForm());
+        Assert.assertEquals(url.toString(), filterResult);
+
+        url = new URL("http://partiallyallowed.com/verbotten");
+        filterResult = filter.filter(url, metadata, url.toExternalForm());
+        Assert.assertEquals(null, filterResult);
 
-	// allowed
-	url = new URL("http://digitalpebble.com/");
-	filterResult = filter.filter(url, metadata, url.toExternalForm());
-	Assert.assertEquals(url.toString(), filterResult);
+        // allowed
+        url = new URL("http://digitalpebble.com/");
+        filterResult = filter.filter(url, metadata, url.toExternalForm());
+        Assert.assertEquals(url.toString(), filterResult);
     }
 }
diff --git a/src/test/java/org/commoncrawl/stormcrawler/news/NewsSiteMapParserTest.java b/src/test/java/org/commoncrawl/stormcrawler/news/NewsSiteMapParserTest.java
index db73d67..b0a0d5a 100644
--- a/src/test/java/org/commoncrawl/stormcrawler/news/NewsSiteMapParserTest.java
+++ b/src/test/java/org/commoncrawl/stormcrawler/news/NewsSiteMapParserTest.java
@@ -16,6 +16,7 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotEquals;
 
+import crawlercommons.sitemaps.UnknownFormatException;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
@@ -25,75 +26,83 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-
 import org.apache.commons.io.IOUtils;
-import org.commoncrawl.stormcrawler.news.NewsSiteMapParserBolt.SitemapType;
-import org.junit.Before;
-import org.junit.Test;
-
 import org.apache.stormcrawler.Metadata;
 import org.apache.stormcrawler.parse.Outlink;
 import org.apache.stormcrawler.parse.ParsingTester;
-
-import crawlercommons.sitemaps.UnknownFormatException;
+import org.commoncrawl.stormcrawler.news.NewsSiteMapParserBolt.SitemapType;
+import org.junit.Before;
+import org.junit.Test;
 
 public class NewsSiteMapParserTest extends ParsingTester {
 
     @Before
     public void setupParserBolt() {
-	setupParserBolt(new NewsSiteMapParserBolt());
-	Map config = new HashMap<>();
-	config.put("sitemap.sniffContent", true);
-	// allow items published during the last week
-	config.put("sitemap.filter.hours.since.modified", 168);
-	prepareParserBolt("test.parsefilters.json", config);
+        setupParserBolt(new NewsSiteMapParserBolt());
+        Map config = new HashMap<>();
+        config.put("sitemap.sniffContent", true);
+        // allow items published during the last week
+        config.put("sitemap.filter.hours.since.modified", 168);
+        prepareParserBolt("test.parsefilters.json", config);
     }
 
     @Test
     public void testSiteMapParser() throws IOException, UnknownFormatException {
-	String url = "https://example.org/sitemap-news.xml";
-	byte[] content = readContent("sitemap-news.xml");
-	String contentType = "";
-	Metadata parentMetadata = new Metadata();
-	List links = new ArrayList<>();
-
-	SitemapType type = ((NewsSiteMapParserBolt) bolt).detectContent(url, content);
-	assertEquals(SitemapType.NEWS, type);
-
-	((NewsSiteMapParserBolt) bolt).parseSiteMap(url, content, contentType, parentMetadata, links);
-
-	// unmodified sitemap:
-	// - publication date is far in the past, link should be skipped
-	// 2008-12-23
-	assertEquals("Outdated link not skipped", 0, links.size());
-
-	// now set the publication date to yesterday
-	LocalDateTime yesterday = LocalDateTime.now().minusDays(1);
-	content = (new String(content, StandardCharsets.UTF_8))
-		.replace("2008-12-23", ""
-			+ yesterday.format(DateTimeFormatter.ofPattern("yyyy-MM-dd")) + "")
-		.getBytes(StandardCharsets.UTF_8);
-	((NewsSiteMapParserBolt) bolt).parseSiteMap(url, content, contentType, parentMetadata, links);
-
-	assertEquals("Expected one  and one additional  link - image links are ignored", 2,
-		links.size());
+        String url = "https://example.org/sitemap-news.xml";
+        byte[] content = readContent("sitemap-news.xml");
+        String contentType = "";
+        Metadata parentMetadata = new Metadata();
+        List links = new ArrayList<>();
+
+        SitemapType type = ((NewsSiteMapParserBolt) bolt).detectContent(url, content);
+        assertEquals(SitemapType.NEWS, type);
+
+        ((NewsSiteMapParserBolt) bolt)
+                .parseSiteMap(url, content, contentType, parentMetadata, links);
+
+        // unmodified sitemap:
+        // - publication date is far in the past, link should be skipped
+        // 2008-12-23
+        assertEquals("Outdated link not skipped", 0, links.size());
+
+        // now set the publication date to yesterday
+        LocalDateTime yesterday = LocalDateTime.now().minusDays(1);
+        content =
+                (new String(content, StandardCharsets.UTF_8))
+                        .replace(
+                                "2008-12-23",
+                                ""
+                                        + yesterday.format(
+                                                DateTimeFormatter.ofPattern("yyyy-MM-dd"))
+                                        + "")
+                        .getBytes(StandardCharsets.UTF_8);
+        ((NewsSiteMapParserBolt) bolt)
+                .parseSiteMap(url, content, contentType, parentMetadata, links);
+
+        assertEquals(
+                "Expected one  and one additional  link - image links are ignored",
+                2,
+                links.size());
     }
 
     protected byte[] readContent(String filename) throws IOException {
-	ByteArrayOutputStream baos = new ByteArrayOutputStream();
-	IOUtils.copy(getClass().getClassLoader().getResourceAsStream(filename), baos);
-	return baos.toByteArray();
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        IOUtils.copy(getClass().getClassLoader().getResourceAsStream(filename), baos);
+        return baos.toByteArray();
     }
 
-	@Test
-     public void testFeedWithSitemapNamespace() throws IOException, UnknownFormatException {
-         String url = "https://example.org/feed.xml";
-		byte[] content = readContent("feed-with-sitemap-namespace.xml");
-         SitemapType type = ((NewsSiteMapParserBolt) bolt).detectContent(url, content);
-         assertNotEquals("RSS feed with sitemap namespace should not be detected as sitemap",
-                 SitemapType.NEWS, type);
-         assertNotEquals("RSS feed with sitemap namespace should not be detected as sitemap",
-                 SitemapType.SITEMAP, type);
-     }
-
+    @Test
+    public void testFeedWithSitemapNamespace() throws IOException, UnknownFormatException {
+        String url = "https://example.org/feed.xml";
+        byte[] content = readContent("feed-with-sitemap-namespace.xml");
+        SitemapType type = ((NewsSiteMapParserBolt) bolt).detectContent(url, content);
+        assertNotEquals(
+                "RSS feed with sitemap namespace should not be detected as sitemap",
+                SitemapType.NEWS,
+                type);
+        assertNotEquals(
+                "RSS feed with sitemap namespace should not be detected as sitemap",
+                SitemapType.SITEMAP,
+                type);
+    }
 }