cachedCounters;
+
+ /**
+ * Creates a new ErrorTracker for the specified counter group.
+ *
+ * This constructor creates an ErrorTracker without cached counters.
+ * Call {@link #initCounters(TaskInputOutputContext)} in setup() to cache
+ * counter references for better performance.
+ *
+ * @param group the Hadoop counter group name (e.g., NutchMetrics.GROUP_FETCHER)
+ */
+ public ErrorTracker(String group) {
+ this.group = group;
+ this.counts = new EnumMap<>(ErrorType.class);
+ this.cachedCounters = new EnumMap<>(ErrorType.class);
+ this.totalCount = new AtomicLong(0);
+
+ // Initialize all counts to 0
+ for (ErrorType type : ErrorType.values()) {
+ counts.put(type, new AtomicLong(0));
+ }
+ }
+
+ /**
+ * Creates a new ErrorTracker with cached counter references.
+ *
+ *
This constructor caches all counter references at creation time,
+ * avoiding repeated counter lookups in hot paths.
+ *
+ * @param group the Hadoop counter group name
+ * @param context the Hadoop task context for caching counters
+ */
+ public ErrorTracker(String group, TaskInputOutputContext context) {
+ this(group);
+ initCounters(context);
+ }
+
+ /**
+ * Initializes cached counter references from the Hadoop context.
+ *
+ *
Call this method in the mapper/reducer setup() method to cache
+ * counter references and avoid repeated lookups during processing.
+ *
+ * @param context the Hadoop task context
+ */
+ public void initCounters(TaskInputOutputContext context) {
+ cachedTotalCounter = context.getCounter(group, NutchMetrics.ERROR_TOTAL);
+ for (ErrorType type : ErrorType.values()) {
+ cachedCounters.put(type, context.getCounter(group, getCounterName(type)));
+ }
+ }
+
+ /**
+ * Records an error with automatic categorization based on the throwable type.
+ *
+ * @param t the throwable to categorize and record
+ */
+ public void recordError(Throwable t) {
+ recordError(categorize(t));
+ }
+
+ /**
+ * Records an error with explicit category.
+ *
+ * @param type the error type category
+ */
+ public void recordError(ErrorType type) {
+ counts.get(type).incrementAndGet();
+ totalCount.incrementAndGet();
+ }
+
+ /**
+ * Returns the count for a specific error type.
+ *
+ * @param type the error type
+ * @return the count for that error type
+ */
+ public long getCount(ErrorType type) {
+ return counts.get(type).get();
+ }
+
+ /**
+ * Returns the total count of all errors.
+ *
+ * @return the total error count
+ */
+ public long getTotalCount() {
+ return totalCount.get();
+ }
+
+ /**
+ * Emits all error counters to the Hadoop context.
+ *
+ *
Should be called once during cleanup to emit aggregated metrics.
+ * Only emits counters for error types that have non-zero counts.
+ *
+ *
If counters were cached via {@link #initCounters(TaskInputOutputContext)},
+ * uses the cached references for better performance.
+ *
+ * @param context the Hadoop task context
+ */
+ public void emitCounters(TaskInputOutputContext context) {
+ // Use cached counters if available, otherwise look up
+ if (cachedTotalCounter != null) {
+ cachedTotalCounter.increment(totalCount.get());
+ for (ErrorType type : ErrorType.values()) {
+ long count = counts.get(type).get();
+ if (count > 0) {
+ cachedCounters.get(type).increment(count);
+ }
+ }
+ } else {
+ // Fallback to direct lookup
+ context.getCounter(group, NutchMetrics.ERROR_TOTAL).increment(totalCount.get());
+ for (ErrorType type : ErrorType.values()) {
+ long count = counts.get(type).get();
+ if (count > 0) {
+ context.getCounter(group, getCounterName(type)).increment(count);
+ }
+ }
+ }
+ }
+
+ /**
+ * Directly increments cached error counters without local accumulation.
+ *
+ *
Use this method when you want to immediately update Hadoop counters
+ * rather than accumulating locally and emitting in cleanup.
+ * Requires {@link #initCounters(TaskInputOutputContext)} to have been called.
+ *
+ * @param t the throwable to categorize and count
+ * @throws IllegalStateException if counters have not been initialized
+ */
+ public void incrementCounters(Throwable t) {
+ incrementCounters(categorize(t));
+ }
+
+ /**
+ * Directly increments cached error counters without local accumulation.
+ *
+ *
Use this method when you want to immediately update Hadoop counters
+ * rather than accumulating locally and emitting in cleanup.
+ * Requires {@link #initCounters(TaskInputOutputContext)} to have been called.
+ *
+ * @param type the error type to count
+ * @throws IllegalStateException if counters have not been initialized
+ */
+ public void incrementCounters(ErrorType type) {
+ if (cachedTotalCounter == null) {
+ throw new IllegalStateException(
+ "Counters not initialized. Call initCounters() first.");
+ }
+ cachedTotalCounter.increment(1);
+ cachedCounters.get(type).increment(1);
+ }
+
+ /**
+ * Categorizes a throwable into an error type.
+ *
+ *
The categorization checks the exception class hierarchy to determine
+ * the most appropriate category. Timeout exceptions are checked first as
+ * they are a subclass of IOException.
+ *
+ * @param t the throwable to categorize
+ * @return the appropriate ErrorType for the throwable
+ */
+ public static ErrorType categorize(Throwable t) {
+ if (t == null) {
+ return ErrorType.OTHER;
+ }
+
+ String className = t.getClass().getName();
+
+ // Check for timeout first (before general IOException)
+ if (t instanceof SocketTimeoutException
+ || className.contains("TimeoutException")
+ || className.contains("Timeout")) {
+ return ErrorType.TIMEOUT;
+ }
+
+ // Network errors
+ if (t instanceof SocketException
+ || t instanceof UnknownHostException
+ || className.contains("ConnectException")
+ || className.contains("NoRouteToHostException")
+ || className.contains("ConnectionRefusedException")) {
+ return ErrorType.NETWORK;
+ }
+
+ // URL errors (check before general IOException since MalformedURLException extends IOException)
+ if (t instanceof MalformedURLException
+ || className.contains("URLFilterException")
+ || className.contains("URISyntaxException")) {
+ return ErrorType.URL;
+ }
+
+ // General IOException (but not the specific subtypes above)
+ if (t instanceof IOException) {
+ return ErrorType.NETWORK;
+ }
+
+ // Protocol errors
+ if (className.contains("ProtocolException")
+ || className.contains("ProtocolNotFound")) {
+ return ErrorType.PROTOCOL;
+ }
+
+ // Parsing errors
+ if (className.contains("ParseException")
+ || className.contains("ParserNotFound")
+ || className.contains("SAXException")
+ || className.contains("ParserConfigurationException")) {
+ return ErrorType.PARSING;
+ }
+
+ // Scoring errors
+ if (className.contains("ScoringFilterException")) {
+ return ErrorType.SCORING;
+ }
+
+ // Indexing errors
+ if (className.contains("IndexingException")) {
+ return ErrorType.INDEXING;
+ }
+
+ // Check cause chain for more specific categorization
+ Throwable cause = t.getCause();
+ if (cause != null && cause != t) {
+ ErrorType causeType = categorize(cause);
+ if (causeType != ErrorType.OTHER) {
+ return causeType;
+ }
+ }
+
+ return ErrorType.OTHER;
+ }
+
+ /**
+ * Gets the counter name constant for a given error type.
+ *
+ * @param type the error type
+ * @return the counter name constant from NutchMetrics
+ */
+ public static String getCounterName(ErrorType type) {
+ switch (type) {
+ case NETWORK:
+ return NutchMetrics.ERROR_NETWORK_TOTAL;
+ case PROTOCOL:
+ return NutchMetrics.ERROR_PROTOCOL_TOTAL;
+ case PARSING:
+ return NutchMetrics.ERROR_PARSING_TOTAL;
+ case URL:
+ return NutchMetrics.ERROR_URL_TOTAL;
+ case SCORING:
+ return NutchMetrics.ERROR_SCORING_TOTAL;
+ case INDEXING:
+ return NutchMetrics.ERROR_INDEXING_TOTAL;
+ case TIMEOUT:
+ return NutchMetrics.ERROR_TIMEOUT_TOTAL;
+ case OTHER:
+ default:
+ return NutchMetrics.ERROR_OTHER_TOTAL;
+ }
+ }
+
+ /**
+ * Gets the counter name for a throwable based on its categorization.
+ *
+ *
This is a convenience method for direct use in catch blocks:
+ *
+ * } catch (Exception e) {
+ * context.getCounter(group, ErrorTracker.getCounterName(e)).increment(1);
+ * }
+ *
+ *
+ * @param t the throwable to get the counter name for
+ * @return the counter name constant from NutchMetrics
+ */
+ public static String getCounterName(Throwable t) {
+ return getCounterName(categorize(t));
+ }
+}
diff --git a/src/java/org/apache/nutch/metrics/LatencyTracker.java b/src/java/org/apache/nutch/metrics/LatencyTracker.java
new file mode 100644
index 0000000000..3777bb29e3
--- /dev/null
+++ b/src/java/org/apache/nutch/metrics/LatencyTracker.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.metrics;
+
+import org.apache.hadoop.mapreduce.TaskInputOutputContext;
+
+import com.tdunning.math.stats.TDigest;
+
+/**
+ * A utility class for tracking latency metrics using TDigest for percentile
+ * calculation.
+ *
+ * This class wraps a TDigest data structure to collect latency samples and
+ * emit Hadoop counters with count, sum, and percentile values (p50, p95, p99).
+ *
+ *
Usage:
+ *
+ * // In mapper/reducer setup
+ * latencyTracker = new LatencyTracker(NutchMetrics.GROUP_FETCHER, NutchMetrics.FETCHER_LATENCY);
+ *
+ * // During processing
+ * long start = System.currentTimeMillis();
+ * // ... operation ...
+ * latencyTracker.record(System.currentTimeMillis() - start);
+ *
+ * // In cleanup
+ * latencyTracker.emitCounters(context);
+ *
+ *
+ * Emits the following counters:
+ *
+ * - {prefix}_count_total - total number of samples
+ * - {prefix}_sum_ms - sum of all latencies in milliseconds
+ * - {prefix}_p50_ms - 50th percentile (median) latency
+ * - {prefix}_p95_ms - 95th percentile latency
+ * - {prefix}_p99_ms - 99th percentile latency
+ *
+ *
+ * @since 1.22
+ */
+public class LatencyTracker {
+
+ /** Default compression factor for TDigest (controls accuracy vs memory). */
+ private static final double DEFAULT_COMPRESSION = 100.0;
+
+ private final TDigest digest;
+ private final String group;
+ private final String prefix;
+ private long count = 0;
+ private long sum = 0;
+
+ /**
+ * Creates a new LatencyTracker.
+ *
+ * @param group the Hadoop counter group name
+ * @param prefix the prefix for counter names (e.g., "fetch_latency")
+ */
+ public LatencyTracker(String group, String prefix) {
+ this.digest = TDigest.createDigest(DEFAULT_COMPRESSION);
+ this.group = group;
+ this.prefix = prefix;
+ }
+
+ /**
+ * Records a latency sample.
+ *
+ * @param latencyMs the latency in milliseconds
+ */
+ public void record(long latencyMs) {
+ digest.add(latencyMs);
+ count++;
+ sum += latencyMs;
+ }
+
+ /**
+ * Returns the number of recorded samples.
+ *
+ * @return the count of recorded latency samples
+ */
+ public long getCount() {
+ return count;
+ }
+
+ /**
+ * Returns the sum of all recorded latencies.
+ *
+ * @return the sum of latencies in milliseconds
+ */
+ public long getSum() {
+ return sum;
+ }
+
+ /**
+ * Returns the percentile value for the given quantile.
+ *
+ * @param quantile the quantile (0.0 to 1.0)
+ * @return the percentile value in milliseconds
+ */
+ public long getPercentile(double quantile) {
+ if (count == 0) {
+ return 0;
+ }
+ return (long) digest.quantile(quantile);
+ }
+
+ /**
+ * Emits all latency counters to the Hadoop context.
+ *
+ * Should be called once during cleanup to emit aggregated metrics.
+ *
+ * @param context the Hadoop task context
+ */
+ public void emitCounters(TaskInputOutputContext context) {
+ context.getCounter(group, prefix + "_count_total").setValue(count);
+ context.getCounter(group, prefix + "_sum_ms").setValue(sum);
+
+ if (count > 0) {
+ context.getCounter(group, prefix + "_p50_ms").setValue((long) digest.quantile(0.50));
+ context.getCounter(group, prefix + "_p95_ms").setValue((long) digest.quantile(0.95));
+ context.getCounter(group, prefix + "_p99_ms").setValue((long) digest.quantile(0.99));
+ } else {
+ // Set to 0 if no samples recorded
+ context.getCounter(group, prefix + "_p50_ms").setValue(0);
+ context.getCounter(group, prefix + "_p95_ms").setValue(0);
+ context.getCounter(group, prefix + "_p99_ms").setValue(0);
+ }
+ }
+}
+
+
diff --git a/src/java/org/apache/nutch/metrics/NutchMetrics.java b/src/java/org/apache/nutch/metrics/NutchMetrics.java
new file mode 100644
index 0000000000..14979803a1
--- /dev/null
+++ b/src/java/org/apache/nutch/metrics/NutchMetrics.java
@@ -0,0 +1,720 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.metrics;
+
+import org.apache.hadoop.io.Text;
+import org.apache.nutch.crawl.CrawlDatum;
+import org.apache.nutch.crawl.FetchSchedule;
+
+/**
+ * Centralized constants for Hadoop metrics counter groups and names.
+ *
+ *
Follows Prometheus
+ * naming conventions:
+ *
+ * - Counter groups use the {@code nutch_} prefix namespace
+ * - Counter names use snake_case
+ * - Accumulating counters use {@code _total} suffix
+ * - Units are included in counter names where applicable (e.g., {@code _bytes})
+ *
+ *
+ * @since 1.22
+ */
+public final class NutchMetrics {
+
+ private NutchMetrics() {
+ // Utility class - prevent instantiation
+ }
+
+ // =========================================================================
+ // Counter Groups (Prometheus namespace style with nutch_ prefix)
+ // =========================================================================
+
+ /** Counter group for fetcher operations. */
+ public static final String GROUP_FETCHER = "nutch_fetcher";
+
+ /** Counter group for fetcher outlink processing. */
+ public static final String GROUP_FETCHER_OUTLINKS = "nutch_fetcher_outlinks";
+
+ /** Counter group for generator operations. */
+ public static final String GROUP_GENERATOR = "nutch_generator";
+
+ /** Counter group for indexer operations. */
+ public static final String GROUP_INDEXER = "nutch_indexer";
+
+ /** Counter group for CrawlDb operations. */
+ public static final String GROUP_CRAWLDB = "nutch_crawldb";
+
+ /** Counter group for CrawlDb filter operations. */
+ public static final String GROUP_CRAWLDB_FILTER = "nutch_crawldb_filter";
+
+ /** Counter group for injector operations. */
+ public static final String GROUP_INJECTOR = "nutch_injector";
+
+ /** Counter group for HostDb operations. */
+ public static final String GROUP_HOSTDB = "nutch_hostdb";
+
+ /** Counter group for parser operations. */
+ public static final String GROUP_PARSER = "nutch_parser";
+
+ /** Counter group for deduplication operations. */
+ public static final String GROUP_DEDUP = "nutch_dedup";
+
+ /** Counter group for cleaning job operations. */
+ public static final String GROUP_CLEANING = "nutch_cleaning";
+
+ /** Counter group for WebGraph operations. */
+ public static final String GROUP_WEBGRAPH = "nutch_webgraph";
+
+ /** Counter group for sitemap processing operations. */
+ public static final String GROUP_SITEMAP = "nutch_sitemap";
+
+ /** Counter group for WARC export operations. */
+ public static final String GROUP_WARC_EXPORTER = "nutch_warc_exporter";
+
+ /** Counter group for Common Crawl data dumper tool. */
+ public static final String GROUP_COMMONCRAWL_DUMPER = "nutch_commoncrawl_dumper";
+
+ /** Counter group for domain statistics operations. */
+ public static final String GROUP_DOMAIN_STATS = "nutch_domain_stats";
+
+ // =========================================================================
+ // Fetcher Counters
+ // =========================================================================
+
+ /** Total bytes downloaded by fetcher. */
+ public static final String FETCHER_BYTES_DOWNLOADED_TOTAL = "bytes_downloaded_total";
+
+ /** URLs denied by robots.txt. */
+ public static final String FETCHER_ROBOTS_DENIED_TOTAL = "robots_denied_total";
+
+ /** URLs denied due to crawl delay exceeding maximum. */
+ public static final String FETCHER_ROBOTS_DENIED_MAXCRAWLDELAY_TOTAL = "robots_denied_maxcrawldelay_total";
+
+ /** URLs dropped due to robots.txt deferred visits. */
+ public static final String FETCHER_ROBOTS_DEFER_VISITS_DROPPED_TOTAL = "robots_defer_visits_dropped_total";
+
+ /** Redirects that exceeded maximum redirect count. */
+ public static final String FETCHER_REDIRECT_COUNT_EXCEEDED_TOTAL = "redirect_count_exceeded_total";
+
+ /** Redirects deduplicated (already seen). */
+ public static final String FETCHER_REDIRECT_DEDUPLICATED_TOTAL = "redirect_deduplicated_total";
+
+ /** FetchItems not created for redirects. */
+ public static final String FETCHER_REDIRECT_NOT_CREATED_TOTAL = "redirect_not_created_total";
+
+ /** URLs hit by time limit. */
+ public static final String FETCHER_HIT_BY_TIMELIMIT_TOTAL = "hit_by_timelimit_total";
+
+ /** URLs hit by timeout. */
+ public static final String FETCHER_HIT_BY_TIMEOUT_TOTAL = "hit_by_timeout_total";
+
+ /** URLs hit by throughput threshold. */
+ public static final String FETCHER_HIT_BY_THROUGHPUT_THRESHOLD_TOTAL = "hit_by_throughput_threshold_total";
+
+ /** Threads that hung during fetching. */
+ public static final String FETCHER_HUNG_THREADS_TOTAL = "hung_threads_total";
+
+ /** URLs filtered during fetching. */
+ public static final String FETCHER_FILTERED_TOTAL = "filtered_total";
+
+ /** URLs dropped due to exception threshold in queue. */
+ public static final String FETCHER_ABOVE_EXCEPTION_THRESHOLD_TOTAL = "above_exception_threshold_total";
+
+ // =========================================================================
+ // Fetcher Outlinks Counters
+ // =========================================================================
+
+ /** Outlinks detected during parsing. */
+ public static final String FETCHER_OUTLINKS_DETECTED_TOTAL = "outlinks_detected_total";
+
+ /** Outlinks being followed. */
+ public static final String FETCHER_OUTLINKS_FOLLOWING_TOTAL = "outlinks_following_total";
+
+ // =========================================================================
+ // Fetcher Common Crawl extensions
+ // =========================================================================
+
+ /** HTTP protocol version group with dynamic counters. */
+ public static final String FETCHER_HTTP_PROTOCOL_VERSION_GROUP = "http_protocol_version";
+
+ public static final String FETCHER_HTTP_PROTOCOL_UNKNOWN = "unknown";
+
+ /** SSL/TLS protocol version group with dynamic counters. */
+ public static final String FETCHER_TLS_PROTOCOL_VERSION_GROUP = "tls_protocol_version";
+
+ /** IP address version group with two counters: ipv4 and ipv6. */
+ public static final String FETCHER_IP_ADDRESS_VERSION_GROUP = "ip_address_version";
+
+ /** Number of fetches over IPv4. */
+ public static final String FETCHER_IPV4_TOTAL = "ipv4";
+
+ /** Number of fetches over IPv6. */
+ public static final String FETCHER_IPV6_TOTAL = "ipv6";
+
+ /** Archiving of robots.txt captures. */
+ public static final String FETCHER_ROBOTSTXT_ARCHIVING_GROUP = "robotstxt_archiving";
+
+ /** Robots.txt not archived: URL rejected by URL filters. */
+ public static final String FETCHER_ROBOTSTXT_ARCHIVING_FILTERED_TOTAL = "filtered";
+
+ /** Robots.txt not archived: MIME type rejected. */
+ public static final String FETCHER_ROBOTSTXT_ARCHIVING_FILTERED_MIME_TOTAL = "filtered_mime";
+
+ /**
+ * Robots.txt not archived: URL path not /robots.txt and
+ * disallowed by robots.txt.
+ */
+ public static final String FETCHER_ROBOTSTXT_ARCHIVING_ROBOTS_DENIED_TOTAL = "robots_denied";
+
+ // =========================================================================
+ // Common Crawl's WarcWriter
+ // =========================================================================
+
+ /** Counter group for Common Crawl's WARC writer. */
+ public static final String GROUP_WARC_WRITER = "warc_writer";
+
+ /** Skipped records because no content (and protocol status) is available. */
+ public static final String WARC_WRITER_SKIPPED_NO_CONTENT_TOTAL = "skipped_no_content";
+
+ /** Fixed records: invalid URI normalized. */
+ public static final String WARC_WRITER_URI_NORMALIZED_TOTAL = "fixed_uri";
+
+ /** Skipped records because URL is not a valid URI (no WARC-Target-URI). */
+ public static final String WARC_WRITER_SKIPPED_INVALID_URI_TOTAL = "skipped_invalid_uri";
+
+ /** Skipped records by content type / MIME type. */
+ public static final String WARC_WRITER_SKIPPED_BY_CONTENT_TYPE_TOTAL = "skipped_by_content_type";
+
+ /** Skipped duplicate records. */
+ public static final String WARC_WRITER_SKIPPED_DUPLICATE_TOTAL = "skipped_duplicate";
+
+ /** Skipped records: no protocol status. */
+ public static final String WARC_WRITER_SKIPPED_NO_PROTOCOL_STATUS_TOTAL = "skipped_no_protocol_status";
+
+ /** Skipped records: unknown protocol status. */
+ public static final String WARC_WRITER_SKIPPED_UNKNOWN_PROTOCOL_STATUS_TOTAL = "skipped_unknown_protocol_status";
+
+ /** Prefix for error status of language identification (LID), returned by CLD2 Java bindings. */
+ public static final String WARC_WRITER_LID_ERROR_PREFIX = "lid_error: ";
+
+ /** Language identification (LID): no result. */
+ public static final String WARC_WRITER_LID_NO_RESULT_TOTAL = "lid_no_result";
+
+ /** Language identification (LID): result is reliable. */
+ public static final String WARC_WRITER_LID_RESULT_RELIABLE_TOTAL = "lid_reliable";
+
+ /** Language identification (LID): result is not reliable. */
+ public static final String WARC_WRITER_LID_RESULT_NOT_RELIABLE_TOTAL = "lid_not_reliable";
+
+ // =========================================================================
+ // Generator Counters
+ // =========================================================================
+
+ /** URLs rejected by URL filters. */
+ public static final String GENERATOR_URL_FILTERS_REJECTED_TOTAL = "url_filters_rejected_total";
+
+ /** URLs rejected by fetch schedule. */
+ public static final String GENERATOR_SCHEDULE_REJECTED_TOTAL = "schedule_rejected_total";
+
+ /** URLs waiting for CrawlDb update. */
+ public static final String GENERATOR_WAIT_FOR_UPDATE_TOTAL = "wait_for_update_total";
+
+ /** URLs rejected by JEXL expression. */
+ public static final String GENERATOR_EXPR_REJECTED_TOTAL = "expr_rejected_total";
+
+ /** URLs rejected due to status restriction. */
+ public static final String GENERATOR_STATUS_REJECTED_TOTAL = "status_rejected_total";
+
+ /** URLs rejected due to score below threshold. */
+ public static final String GENERATOR_SCORE_TOO_LOW_TOTAL = "score_too_low_total";
+
+ /** URLs rejected due to fetch interval exceeding threshold. */
+ public static final String GENERATOR_INTERVAL_REJECTED_TOTAL = "interval_rejected_total";
+
+ /** URLs skipped due to per-host overflow. */
+ public static final String GENERATOR_URLS_SKIPPED_PER_HOST_OVERFLOW_TOTAL = "urls_skipped_per_host_overflow_total";
+
+ /** Hosts affected by per-host overflow. */
+ public static final String GENERATOR_HOSTS_AFFECTED_PER_HOST_OVERFLOW_TOTAL = "hosts_affected_per_host_overflow_total";
+
+ // =========================================================================
+ // Generator2-specific Counters
+ // =========================================================================
+
+ /** Domains affected by per-domain overflow. All remaining URLs of this domain have been skipped, but were not counted. */
+ public static final String GENERATOR_DOMAINS_AFFECTED_PER_DOMAIN_OVERFLOW_TOTAL = "domains_affected_per_domain_overflow_total";
+
+ /** Domains affected by max. number of hosts per domain overflow. URLs from further hosts below this domain have been skipped. */
+ public static final String GENERATOR_DOMAINS_AFFECTED_PER_MAX_NUM_HOSTS_OVERFLOW_TOTAL = "domains_affected_num_hosts_overflow_total";
+
+ /** URLs skipped due to the max. number of hosts per domain overflow. */
+ public static final String GENERATOR_URLS_SKIPPED_PER_MAX_NUM_HOSTS_OVERFLOW_TOTAL = "urls_skipped_per_max_num_host_overflow_total";
+
+ /** URLs skipped due to per-segment overflow. */
+ public static final String GENERATOR_URLS_SKIPPED_PER_SEGMENT_OVERFLOW_TOTAL = "urls_skipped_per_segment_overflow_total";
+
+ /**
+ * Counter group for items by status, rejected by the fetch schedule. See
+ * {@link FetchSchedule#shouldFetch(Text, CrawlDatum, long)}.
+ */
+ public static final String GROUP_GENERATOR_SCHEDULE_REJECTED_BY_STATUS = "schedule_rejected_by_status";
+
+ /**
+ * Counter group for items by status, rejected because the generator score is
+ * lower than the minimum score defined per generate.min.score.
+ */
+ public static final String GROUP_GENERATOR_SCORE_REJECTED_BY_STATUS = "score_rejected_by_status";
+
+ /** Counter group for items by status, selected for fetch. */
+ public static final String GROUP_GENERATOR_SELECTED_BY_STATUS = "selected_by_status";
+
+ // =========================================================================
+ // Indexer Counters
+ // =========================================================================
+
+ /** Documents deleted due to robots noindex. */
+ public static final String INDEXER_DELETED_ROBOTS_NOINDEX_TOTAL = "deleted_robots_noindex_total";
+
+ /** Documents deleted because they are gone. */
+ public static final String INDEXER_DELETED_GONE_TOTAL = "deleted_gone_total";
+
+ /** Documents deleted due to redirects. */
+ public static final String INDEXER_DELETED_REDIRECTS_TOTAL = "deleted_redirects_total";
+
+ /** Documents deleted as duplicates. */
+ public static final String INDEXER_DELETED_DUPLICATES_TOTAL = "deleted_duplicates_total";
+
+ /** Documents deleted by indexing filter. */
+ public static final String INDEXER_DELETED_BY_INDEXING_FILTER_TOTAL = "deleted_by_indexing_filter_total";
+
+ /** Documents skipped (not modified). */
+ public static final String INDEXER_SKIPPED_NOT_MODIFIED_TOTAL = "skipped_not_modified_total";
+
+ /** Documents skipped by indexing filter. */
+ public static final String INDEXER_SKIPPED_BY_INDEXING_FILTER_TOTAL = "skipped_by_indexing_filter_total";
+
+ /** Documents indexed (added or updated). */
+ public static final String INDEXER_INDEXED_TOTAL = "indexed_total";
+
+ // =========================================================================
+ // CrawlDb Counters
+ // =========================================================================
+
+ /** URLs filtered during CrawlDb operations. */
+ public static final String CRAWLDB_URLS_FILTERED_TOTAL = "urls_filtered_total";
+
+ /** Gone (404) records removed during CrawlDb operations. */
+ public static final String CRAWLDB_GONE_RECORDS_REMOVED_TOTAL = "gone_records_removed_total";
+
+ /** Orphan records removed during CrawlDb operations. */
+ public static final String CRAWLDB_ORPHAN_RECORDS_REMOVED_TOTAL = "orphan_records_removed_total";
+
+ // =========================================================================
+ // Injector Counters
+ // =========================================================================
+
+ /** URLs filtered during injection. */
+ public static final String INJECTOR_URLS_FILTERED_TOTAL = "urls_filtered_total";
+
+ /** URLs injected. */
+ public static final String INJECTOR_URLS_INJECTED_TOTAL = "urls_injected_total";
+
+ /** Unique URLs injected. */
+ public static final String INJECTOR_URLS_INJECTED_UNIQUE_TOTAL = "urls_injected_unique_total";
+
+ /** URLs merged with existing CrawlDb entries. */
+ public static final String INJECTOR_URLS_MERGED_TOTAL = "urls_merged_total";
+
+ /** URLs purged due to 404 status. */
+ public static final String INJECTOR_URLS_PURGED_404_TOTAL = "urls_purged_404_total";
+
+ /** URLs purged by filter. */
+ public static final String INJECTOR_URLS_PURGED_FILTER_TOTAL = "urls_purged_filter_total";
+
+ // =========================================================================
+ // HostDb Counters
+ // =========================================================================
+
+ /** Records filtered in HostDb. */
+ public static final String HOSTDB_FILTERED_RECORDS_TOTAL = "filtered_records_total";
+
+ /** Total hosts processed. */
+ public static final String HOSTDB_TOTAL_HOSTS_TOTAL = "total_hosts_total";
+
+ /** Hosts skipped (not eligible). */
+ public static final String HOSTDB_SKIPPED_NOT_ELIGIBLE_TOTAL = "skipped_not_eligible_total";
+
+ /** Hosts where URL limit was not reached. */
+ public static final String HOSTDB_URL_LIMIT_NOT_REACHED_TOTAL = "url_limit_not_reached_total";
+
+ /** New known hosts discovered. */
+ public static final String HOSTDB_NEW_KNOWN_HOST_TOTAL = "new_known_host_total";
+
+ /** Rediscovered hosts. */
+ public static final String HOSTDB_REDISCOVERED_HOST_TOTAL = "rediscovered_host_total";
+
+ /** Existing known hosts. */
+ public static final String HOSTDB_EXISTING_KNOWN_HOST_TOTAL = "existing_known_host_total";
+
+ /** New unknown hosts. */
+ public static final String HOSTDB_NEW_UNKNOWN_HOST_TOTAL = "new_unknown_host_total";
+
+ /** Existing unknown hosts. */
+ public static final String HOSTDB_EXISTING_UNKNOWN_HOST_TOTAL = "existing_unknown_host_total";
+
+ /** Purged unknown hosts. */
+ public static final String HOSTDB_PURGED_UNKNOWN_HOST_TOTAL = "purged_unknown_host_total";
+
+ /** Hosts checked. */
+ public static final String HOSTDB_CHECKED_HOSTS_TOTAL = "checked_hosts_total";
+
+ // =========================================================================
+ // Deduplication Counters
+ // =========================================================================
+
+ /** Documents marked as duplicate. */
+ public static final String DEDUP_DOCUMENTS_MARKED_DUPLICATE_TOTAL = "documents_marked_duplicate_total";
+
+ // =========================================================================
+ // Redirect Deduplication Counters
+ // =========================================================================
+
+ /** Redirects kept as non-duplicates. */
+ public static final String DEDUP_REDIRECTS_NOT_DUPLICATES_TOTAL = "redirects_marked_not_duplicate_total";
+
+ /** Redirects in CrawlDb. */
+ public static final String DEDUP_REDIRECTS_IN_CRAWLDB_TOTAL = "redirects_in_crawldb_total";
+
+ /** Self-referential redirects in CrawlDb. */
+ public static final String DEDUP_REDIRECTS_SELF_REFERENTIAL_TOTAL = "redirects_self_referential_total";
+
+ /** Self-referential redirects kept as non-duplicates. */
+ public static final String DEDUP_REDIRECTS_SELF_REFERENTIAL_NOT_DUPLICATES_TOTAL = "redirects_self_referential_marked_not_duplicate_total";
+
+ // =========================================================================
+ // Cleaning Job Counters
+ // =========================================================================
+
+ /** Documents deleted during cleaning. */
+ public static final String CLEANING_DELETED_DOCUMENTS_TOTAL = "deleted_documents_total";
+
+ // =========================================================================
+ // WebGraph Counters
+ // =========================================================================
+
+ /** Links added to WebGraph. */
+ public static final String WEBGRAPH_ADDED_LINKS_TOTAL = "added_links_total";
+
+ /** Links removed from WebGraph. */
+ public static final String WEBGRAPH_REMOVED_LINKS_TOTAL = "removed_links_total";
+
+ // =========================================================================
+ // Sitemap Counters
+ // =========================================================================
+
+ /** Filtered records in sitemap processing. */
+ public static final String SITEMAP_FILTERED_RECORDS_TOTAL = "filtered_records_total";
+
+ /** Seeds extracted from sitemaps. */
+ public static final String SITEMAP_SEEDS_TOTAL = "sitemap_seeds_total";
+
+ /** Sitemaps discovered from hostname. */
+ public static final String SITEMAP_FROM_HOSTNAME_TOTAL = "sitemaps_from_hostname_total";
+
+ /** Sitemaps filtered from hostname. */
+ public static final String SITEMAP_FILTERED_FROM_HOSTNAME_TOTAL = "filtered_sitemaps_from_hostname_total";
+
+ /** Failed sitemap fetches. */
+ public static final String SITEMAP_FAILED_FETCHES_TOTAL = "failed_fetches_total";
+
+ /** Existing sitemap entries. */
+ public static final String SITEMAP_EXISTING_ENTRIES_TOTAL = "existing_sitemap_entries_total";
+
+ /** New sitemap entries. */
+ public static final String SITEMAP_NEW_ENTRIES_TOTAL = "new_sitemap_entries_total";
+
+ // =========================================================================
+ // SitemapInjector Counters
+ // =========================================================================
+
+ /** SitemapInjector counter group. */
+ public static final String GROUP_SITEMAP_INJECTOR = "sitemap_injector";
+
+ /** Failed to fetch sitemap content, disallowed per robots.txt. */
+ public static final String SITEMAP_ROBOTSTXT_DISALLOW_TOTAL = "sitemap_robotstxt_disallow";
+
+ /** Sitemap failed to parse. */
+ public static final String SITEMAP_FAILED_TO_PARSE_TOTAL = "sitemaps_failed_to_parse";
+
+ /** Prefix for sitemap type counter. */
+ public static final String SITEMAP_TYPE_PREFIX = "sitemap_type_";
+
+ /** Sitemaps processed total. */
+ public static final String SITEMAP_PROCESSED_TOTAL = "sitemaps_processed";
+
+ /** Sitemap index: affected by URL limit. */
+ public static final String SITEMAP_INDEX_AFFECTED_BY_URL_LIMIT_TOTAL = "sitemap_index_url_limit";
+
+ /** Sitemap index: affected by depth limit. */
+ public static final String SITEMAP_INDEX_AFFECTED_BY_DEPTH_LIMIT_TOTAL = "sitemap_index_depth_limit";
+
+ /** Sitemap index: affected by time limit. */
+ public static final String SITEMAP_INDEX_AFFECTED_BY_TIME_LIMIT_TOTAL = "sitemap_index_time_limit";
+
+ /** Sitemap index: skipped because no URLs found after 50% of time limit. */
+ public static final String SITEMAP_INDEX_NO_URLS_AFTER_50_PERCENT_OF_TIME_LIMIT_TOTAL = "sitemap_index_no_urls_after_50_percent_of_time_limit";
+
+ /** Sitemap index: skipped because of too many fetch failures. */
+ public static final String SITEMAP_INDEX_TOO_MANY_FAILURES_TOTAL = "sitemap_index_too_many_failures";
+
+ /** Sitemap index: processed sitemaps. */
+ public static final String SITEMAP_INDEX_PROCESSED_SITEMAPS_TOTAL = "sitemap_index_processed_sitemaps";
+
+ /** Skipped duplicated or recursive sitemap URLs. */
+ public static final String SITEMAP_SKIPPED_DUPLICATE_OR_RECURSIVE_URL_TOTAL = "sitemap_skipped_duplicate_or_recursive_sitemap_url";
+
+ /** Sitemap index: affected by max. number of sitemaps in index. */
+ public static final String SITEMAP_INDEX_MAX_SITEMAPS_LIMIT_TOTAL = "sitemap_index_max_sitemaps_limit";
+
+ /** Sitemap failed to fetch. */
+ public static final String SITEMAP_FAILED_TO_FETCH_TOTAL = "sitemap_failed_to_fetch";
+
+ /** Sitemap skipped because of overlong URL. */
+ public static final String SITEMAP_SKIPPED_OVERLONG_URL_TOTAL = "sitemap_skipped_overlong_url";
+
+ /** Sitemap rejected by URL filters */
+ public static final String SITEMAP_REJECTED_BY_URL_FILTERS_TOTAL = "sitemap_rejected_by_url_filters";
+
+ /** Sitemap skipped, too many failures per host. */
+ public static final String SITEMAP_SKIPPED_TOO_MANY_FAILURES_PER_HOST_TOTAL = "sitemap_skipped_too_many_failures_per_host";
+
+ /** Could not fetch sitemap content, protocol not supported. */
+ public static final String SITEMAP_PROTOCOL_NOT_SUPPORTED_TOTAL = "sitemap_protocol_not_supported";
+
+ /** Failed to fetch sitemap content because of timeout. */
+ public static final String SITEMAP_FAILED_TO_FETCH_TIMEOUT_TOTAL = "sitemap_failed_to_fetch_timeout";
+
+ /** Failed to fetch sitemap content because of exception. */
+ public static final String SITEMAP_FAILED_TO_FETCH_EXCEPTION_TOTAL = "sitemap_failed_to_fetch_exception";
+
+ /** Sitemap redirect. */
+ public static final String SITEMAP_REDIRECT_TOTAL = "sitemap_redirect";
+
+ /** Sitemap redirect target rejected by URL filters */
+ public static final String SITEMAP_REDIRECT_TARGET_REJECTED_BY_URL_FILTERS_TOTAL = "sitemap_redirect_target_rejected_by_url_filters";
+
+ /** Sitemap redirect limit exceeded (max. number of redirects followed). */
+ public static final String SITEMAP_REDIRECT_LIMIT_EXCEEDED_TOTAL = "sitemap_redirect_limit_exceeded";
+
+ /** Failed to fetch sitemap content, HTTP status != 200. */
+ public static final String SITEMAP_FAILED_TO_FETCH_CONTENT_HTTP_STATUS_CODE_NOT_200_TOTAL = "sitemap_failed_to_fetch_http_status_code_not_200";
+
+ /** Failed to fetch sitemap content, empty content. */
+ public static final String SITEMAP_EMPTY_CONTENT_TOTAL = "sitemap_empty_content";
+
+ /** Empty sitemap. */
+ public static final String SITEMAP_EMPTY_TOTAL = "sitemap_empty";
+
+ /** Sitemap URL limit reached. */
+ public static final String SITEMAP_URL_LIMIT_REACHED_TOTAL = "sitemap_url_limit_reached";
+
+ /** URLs randomly skipped. */
+ public static final String SITEMAP_RANDOM_SKIP_TOTAL = "urls_random_skip";
+
+ /** URLs from sitemaps rejected, host limit reached. */
+ public static final String SITEMAP_URLS_SKIPPED_HOST_LIMIT_REACHED_TOTAL = "urls_skipped_host_limit_reached";
+
+ /** URLs from sitemaps rejected, target not allowed by cross-submit. */
+ public static final String SITEMAP_URLS_SKIPPED_NOT_ALLOWED_BY_CROSS_SUBMITS_TOTAL = "urls_skipped_not_allowed_by_cross_submits";
+
+ /** URLs from sitemaps rejected by URL filters. */
+ public static final String SITEMAP_URLS_FROM_REJECTED_BY_URL_FILTERS = "urls_from_sitemaps_rejected_by_url_filters";
+
+ /** URLs from sitemaps injected. */
+ public static final String SITEMAP_URLS_INJECTED = "urls_from_sitemaps_injected";
+
+ // =========================================================================
+ // WARC Exporter Counters
+ // =========================================================================
+
+ /** Missing content in WARC export. */
+ public static final String WARC_MISSING_CONTENT_TOTAL = "missing_content_total";
+
+ /** Missing metadata in WARC export. */
+ public static final String WARC_MISSING_METADATA_TOTAL = "missing_metadata_total";
+
+ /** Omitted empty responses in WARC export. */
+ public static final String WARC_OMITTED_EMPTY_RESPONSE_TOTAL = "omitted_empty_response_total";
+
+ /** WARC records generated. */
+ public static final String WARC_RECORDS_GENERATED_TOTAL = "records_generated_total";
+
+ // =========================================================================
+ // Domain Statistics Counters (enum-based, kept for compatibility)
+ // =========================================================================
+
+ /** Fetched URLs in domain statistics. */
+ public static final String DOMAIN_STATS_FETCHED_TOTAL = "fetched_total";
+
+ /** Not fetched URLs in domain statistics. */
+ public static final String DOMAIN_STATS_NOT_FETCHED_TOTAL = "not_fetched_total";
+
+ /** Empty results in domain statistics. */
+ public static final String DOMAIN_STATS_EMPTY_RESULT_TOTAL = "empty_result_total";
+
+ // =========================================================================
+ // UrlCleaner
+ // =========================================================================
+
+ public static final String GROUP_URLCLEANER = "urlcleaner";
+
+ public static final String URLCLEANER_REJECTED_TOTAL = "urls_rejected";
+
+ public static final String URLCLEANER_REJECTED_INVALID_DOMAIN_TOTAL = "urls_rejected_invalid_domain";
+
+ public static final String URLCLEANER_ACCEPTED_UNCHANGED_TOTAL = "urls_accepted_unchanged";
+
+ public static final String URLCLEANER_ACCEPTED_NORMALIZED_TOTAL = "urls_accepted_normalized";
+
+ // =========================================================================
+ // UrlSampler and UrlSamplerHost
+ // =========================================================================
+
+ public static final String GROUP_URLSAMPLER = "urlsampler";
+
+ public static final String GROUP_URLSAMPLER_HOST = "urlsamplerhost";
+
+ public static final String URLSAMPLER_MALFORMED_URL_TOTAL = "malformed_url";
+
+ public static final String URLSAMPLER_SKIPPED_MAX_URLS_TOTAL = "skipped_max_urls";
+
+ public static final String URLSAMPLER_SKIPPED_MAX_URLS_PER_HOST_TOTAL = "skipped_max_urls_per_host";
+
+ public static final String URLSAMPLER_SKIPPED_MAX_HOSTS_TOTAL = "skipped_max_hosts";
+
+ public static final String URLSAMPLER_HOSTS = "hosts";
+
+ public static final String URLSAMPLER_URLS = "urls";
+
+ public static final String URLSAMPLER_HOSTS_WITH_LIMIT = "hosts_with_limit";
+
+ public static final String URLSAMPLER_URLS_HOST_WITH_LIMIT = "urls_host_with_limit";
+
+ public static final String URLSAMPLER_HOSTS_WITHOUT_LIMIT = "hosts_without_limit";
+
+ public static final String URLSAMPLER_URLS_HOST_WITHOUT_LIMIT = "urls_host_without_limit";
+
+ public static final String URLSAMPLER_URLS_SAMPLED = "urls_sampled";
+
+ public static final String URLSAMPLER_HOSTS_SAMPLED = "hosts_sampled";
+
+ public static final String URLSAMPLER_HOSTS_WITH_LIMIT_SAMPLED = "hosts_with_limit_sampled";
+
+ public static final String URLSAMPLER_URLS_HOST_WITH_LIMIT_SAMPLED = "urls_host_with_limit_sampled";
+
+ public static final String URLSAMPLER_HOSTS_WITHOUT_LIMIT_SAMPLED = "hosts_without_limit_sampled";
+
+ public static final String URLSAMPLER_URLS_HOST_WITHOUT_LIMIT_SAMPLED = "urls_host_without_limit_sampled";
+
+ public static final String URLSAMPLER_SKIPPED_MAX_URLS_PER_HOST = "skipped_max_urls_per_host";
+
+ public static final String URLSAMPLER_SKIPPED_RANDOM = "skipped_random";
+
+ // =========================================================================
+ // Latency Metric Prefixes (used with LatencyTracker)
+ // =========================================================================
+
+ /**
+ * Prefix for fetch latency metrics.
+ * Used with {@link LatencyTracker} to emit fetch timing counters.
+ */
+ public static final String FETCHER_LATENCY = "fetch_latency";
+
+ /**
+ * Prefix for parse latency metrics.
+ * Used with {@link LatencyTracker} to emit parse timing counters.
+ */
+ public static final String PARSER_LATENCY = "parse_latency";
+
+ /**
+ * Prefix for indexer latency metrics.
+ * Used with {@link LatencyTracker} to emit indexing timing counters.
+ */
+ public static final String INDEXER_LATENCY = "index_latency";
+
+ // =========================================================================
+ // Common Error Counter Names (used with component-specific groups)
+ // These constants are shared across all components for consistent error
+ // categorization. Use with ErrorTracker for automatic classification.
+ // =========================================================================
+
+ /**
+ * Total errors across all categories.
+ * This is incremented alongside any category-specific error counter.
+ */
+ public static final String ERROR_TOTAL = "errors_total";
+
+ /**
+ * Network-related errors.
+ * Includes: IOException, SocketException, ConnectException, UnknownHostException
+ */
+ public static final String ERROR_NETWORK_TOTAL = "errors_network_total";
+
+ /**
+ * Protocol errors.
+ * Includes: ProtocolException, ProtocolNotFound
+ */
+ public static final String ERROR_PROTOCOL_TOTAL = "errors_protocol_total";
+
+ /**
+ * Parsing errors.
+ * Includes: ParseException, ParserNotFound
+ */
+ public static final String ERROR_PARSING_TOTAL = "errors_parsing_total";
+
+ /**
+ * URL-related errors.
+ * Includes: MalformedURLException, URLFilterException
+ */
+ public static final String ERROR_URL_TOTAL = "errors_url_total";
+
+ /**
+ * Scoring filter errors.
+ * Includes: ScoringFilterException
+ */
+ public static final String ERROR_SCORING_TOTAL = "errors_scoring_total";
+
+ /**
+ * Indexing filter errors.
+ * Includes: IndexingException
+ */
+ public static final String ERROR_INDEXING_TOTAL = "errors_indexing_total";
+
+ /**
+ * Timeout errors.
+ * Includes: SocketTimeoutException, connection timeouts
+ */
+ public static final String ERROR_TIMEOUT_TOTAL = "errors_timeout_total";
+
+ /**
+ * Other uncategorized errors.
+ * Used as fallback for exceptions not matching any specific category.
+ */
+ public static final String ERROR_OTHER_TOTAL = "errors_other_total";
+}
+
diff --git a/src/java/org/apache/nutch/metrics/package-info.java b/src/java/org/apache/nutch/metrics/package-info.java
new file mode 100644
index 0000000000..376605d043
--- /dev/null
+++ b/src/java/org/apache/nutch/metrics/package-info.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Metrics infrastructure for Apache Nutch.
+ *
+ * This package provides centralized constants and utilities for Hadoop
+ * MapReduce metrics/counters following
+ * Prometheus naming
+ * conventions.
+ *
+ *
The main class is {@link org.apache.nutch.metrics.NutchMetrics} which
+ * defines all counter group names and counter names as constants.
+ *
+ * @since 1.22
+ */
+package org.apache.nutch.metrics;
+
diff --git a/src/java/org/apache/nutch/parse/ParseSegment.java b/src/java/org/apache/nutch/parse/ParseSegment.java
index 6b2fb5cee7..0b2a6f2290 100644
--- a/src/java/org/apache/nutch/parse/ParseSegment.java
+++ b/src/java/org/apache/nutch/parse/ParseSegment.java
@@ -37,6 +37,9 @@
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.nutch.metadata.Metadata;
import org.apache.nutch.metadata.Nutch;
+import org.apache.nutch.metrics.ErrorTracker;
+import org.apache.nutch.metrics.LatencyTracker;
+import org.apache.nutch.metrics.NutchMetrics;
import org.apache.nutch.net.protocols.Response;
import org.apache.nutch.protocol.Content;
import org.apache.nutch.scoring.ScoringFilterException;
@@ -80,12 +83,25 @@ public static class ParseSegmentMapper extends
private Text newKey = new Text();
private ScoringFilters scfilters;
private boolean skipTruncated;
+ private LatencyTracker parseLatencyTracker;
+ private ErrorTracker errorTracker;
@Override
public void setup(Mapper, Content, Text, ParseImpl>.Context context) {
Configuration conf = context.getConfiguration();
scfilters = new ScoringFilters(conf);
skipTruncated = conf.getBoolean(SKIP_TRUNCATED, true);
+ parseLatencyTracker = new LatencyTracker(
+ NutchMetrics.GROUP_PARSER, NutchMetrics.PARSER_LATENCY);
+ // Initialize error tracker with cached counters
+ errorTracker = new ErrorTracker(NutchMetrics.GROUP_PARSER, context);
+ }
+
+ @Override
+ public void cleanup(Mapper, Content, Text, ParseImpl>.Context context)
+ throws IOException, InterruptedException {
+ // Emit parse latency metrics
+ parseLatencyTracker.emitCounters(context);
}
@Override
@@ -121,6 +137,7 @@ public void map(WritableComparable key, Content content,
parseResult = parseUtil.parse(content);
} catch (Exception e) {
LOG.warn("Error parsing: {}: {}", key, StringUtils.stringifyException(e));
+ errorTracker.incrementCounters(e);
return;
}
@@ -129,7 +146,8 @@ public void map(WritableComparable key, Content content,
Parse parse = entry.getValue();
ParseStatus parseStatus = parse.getData().getStatus();
- context.getCounter("ParserStatus",
+ // Dynamic counter based on parse status
+ context.getCounter(NutchMetrics.GROUP_PARSER,
ParseStatus.majorCodes[parseStatus.getMajorCode()]).increment(1);
if (!parseStatus.isSuccess()) {
@@ -151,10 +169,13 @@ public void map(WritableComparable key, Content content,
scfilters.passScoreAfterParsing(url, content, parse);
} catch (ScoringFilterException e) {
LOG.warn("Error passing score: {}: {}", url, e.getMessage());
+ errorTracker.incrementCounters(ErrorTracker.ErrorType.SCORING);
}
long end = System.currentTimeMillis();
- LOG.info("Parsed ({}ms): {}", (end - start), url);
+ long parseTime = end - start;
+ parseLatencyTracker.record(parseTime);
+ LOG.info("Parsed ({}ms): {}", parseTime, url);
context.write(
url,
diff --git a/src/java/org/apache/nutch/parse/ParseStatus.java b/src/java/org/apache/nutch/parse/ParseStatus.java
index 052a342247..25b8ae1b47 100644
--- a/src/java/org/apache/nutch/parse/ParseStatus.java
+++ b/src/java/org/apache/nutch/parse/ParseStatus.java
@@ -56,7 +56,7 @@ public class ParseStatus implements Writable {
// Secondary failure codes go here:
/**
- * Parsing failed. An Exception occured (which may be retrieved from the
+ * Parsing failed. An Exception occurred (which may be retrieved from the
* arguments).
*/
public static final short FAILED_EXCEPTION = 200;
diff --git a/src/java/org/apache/nutch/plugin/PluginManifestParser.java b/src/java/org/apache/nutch/plugin/PluginManifestParser.java
index 10ce4fdb7b..95208fa433 100644
--- a/src/java/org/apache/nutch/plugin/PluginManifestParser.java
+++ b/src/java/org/apache/nutch/plugin/PluginManifestParser.java
@@ -18,11 +18,11 @@
import java.io.File;
import java.io.IOException;
-import java.io.UnsupportedEncodingException;
import java.lang.invoke.MethodHandles;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLDecoder;
+import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Map;
@@ -124,10 +124,7 @@ public File getPluginFolder(String name) {
String path = url.getPath();
if (WINDOWS && path.startsWith("/")) // patch a windows bug
path = path.substring(1);
- try {
- path = URLDecoder.decode(path, "UTF-8"); // decode the url path
- } catch (UnsupportedEncodingException e) {
- }
+ path = URLDecoder.decode(path, StandardCharsets.UTF_8); // decode the url path
directory = new File(path);
} else if (!directory.exists()) {
LOG.warn("Plugins: directory not found: {}", name);
diff --git a/src/java/org/apache/nutch/protocol/Protocol.java b/src/java/org/apache/nutch/protocol/Protocol.java
index ab4162c87f..2514eae33e 100644
--- a/src/java/org/apache/nutch/protocol/Protocol.java
+++ b/src/java/org/apache/nutch/protocol/Protocol.java
@@ -16,6 +16,7 @@
*/
package org.apache.nutch.protocol;
+import java.net.URL;
import java.util.List;
import org.apache.hadoop.conf.Configurable;
@@ -57,4 +58,24 @@ public interface Protocol extends Pluggable, Configurable {
BaseRobotRules getRobotRules(Text url, CrawlDatum datum,
List robotsTxtContent);
+ /**
+ * Retrieve robot rules applicable for this URL.
+ *
+ * @param url
+ * URL to check
+ * @param datum
+ * page datum
+ * @param robotsTxtContent
+ * container to store responses when fetching the robots.txt file for
+ * debugging or archival purposes. Instead of a robots.txt file, it
+ * may include redirects or an error page (404, etc.). Response
+ * {@link Content} is appended to the passed list. If null is passed
+ * nothing is stored.
+ * @return robot rules (specific for this URL or default), never null
+ */
+ default BaseRobotRules getRobotRules(URL url, CrawlDatum datum,
+ List robotsTxtContent) {
+ return getRobotRules(new Text(url.toString()), datum, robotsTxtContent);
+ }
+
}
diff --git a/src/java/org/apache/nutch/scoring/webgraph/WebGraph.java b/src/java/org/apache/nutch/scoring/webgraph/WebGraph.java
index 4daefcd8f3..fee0921d0a 100644
--- a/src/java/org/apache/nutch/scoring/webgraph/WebGraph.java
+++ b/src/java/org/apache/nutch/scoring/webgraph/WebGraph.java
@@ -48,6 +48,7 @@
import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.Counter;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.output.MapFileOutputFormat;
import org.apache.hadoop.mapreduce.Mapper;
@@ -58,6 +59,7 @@
import org.apache.nutch.crawl.NutchWritable;
import org.apache.nutch.crawl.CrawlDatum;
import org.apache.nutch.metadata.Nutch;
+import org.apache.nutch.metrics.NutchMetrics;
import org.apache.nutch.net.URLFilters;
import org.apache.nutch.net.URLNormalizers;
import org.apache.nutch.parse.Outlink;
@@ -327,6 +329,10 @@ public static class OutlinkDbReducer extends
// url normalizers, filters and job configuration
private Configuration conf;
+ // Cached counter references for performance
+ private Counter addedLinksCounter;
+ private Counter removedLinksCounter;
+
/**
* Configures the OutlinkDb job reducer. Sets up internal links and link limiting.
*/
@@ -339,6 +345,18 @@ public void setup(Reducer.Context context)
limitPages = conf.getBoolean("link.ignore.limit.page", true);
limitDomains = conf.getBoolean("link.ignore.limit.domain", true);
+ // Initialize cached counter references
+ initCounters(context);
+ }
+
+ /**
+ * Initialize cached counter references to avoid repeated lookups in hot paths.
+ */
+ private void initCounters(Context context) {
+ addedLinksCounter = context.getCounter(
+ NutchMetrics.GROUP_WEBGRAPH, NutchMetrics.WEBGRAPH_ADDED_LINKS_TOTAL);
+ removedLinksCounter = context.getCounter(
+ NutchMetrics.GROUP_WEBGRAPH, NutchMetrics.WEBGRAPH_REMOVED_LINKS_TOTAL);
}
@Override
@@ -361,14 +379,14 @@ public void reduce(Text key, Iterable values,
mostRecent = timestamp;
}
outlinkList.add(WritableUtils.clone(next, conf));
- context.getCounter("WebGraph.outlinks", "added links").increment(1);
+ addedLinksCounter.increment(1);
} else if (value instanceof BooleanWritable) {
BooleanWritable delete = (BooleanWritable) value;
// Actually, delete is always true, otherwise we don't emit it in the
// mapper in the first place
if (delete.get() == true) {
// This page is gone, do not emit it's outlinks
- context.getCounter("WebGraph.outlinks", "removed links").increment(1);
+ removedLinksCounter.increment(1);
return;
}
}
diff --git a/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java b/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java
index d5d5035e89..8e37c21fcf 100644
--- a/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java
+++ b/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java
@@ -67,6 +67,8 @@
import org.apache.nutch.crawl.LinkDbReader;
import org.apache.nutch.metadata.Metadata;
import org.apache.nutch.metadata.Nutch;
+import org.apache.nutch.metrics.ErrorTracker;
+import org.apache.nutch.metrics.NutchMetrics;
import org.apache.nutch.protocol.Content;
import org.apache.nutch.util.DumpFileUtil;
import org.apache.nutch.util.NutchConfiguration;
@@ -188,6 +190,7 @@ public class CommonCrawlDataDumper extends NutchTool implements Tool {
private GzipCompressorOutputStream gzipOutput = null;
private TarArchiveOutputStream tarOutput = null;
private ArrayList fileList = null;
+ private ErrorTracker errorTracker;
/**
* Main method for invoking this tool
@@ -210,6 +213,7 @@ public static void main(String[] args) throws Exception {
* @param config A populated {@link CommonCrawlConfig}
*/
public CommonCrawlDataDumper(CommonCrawlConfig config) {
+ this();
this.config = config;
}
@@ -217,6 +221,7 @@ public CommonCrawlDataDumper(CommonCrawlConfig config) {
* Constructor
*/
public CommonCrawlDataDumper() {
+ this.errorTracker = new ErrorTracker(NutchMetrics.GROUP_COMMONCRAWL_DUMPER);
}
/**
@@ -274,7 +279,8 @@ public void dump(File outputDir, File segmentRootDir, File linkdb, boolean gzip,
if (parts == null || parts.size() == 0) {
LOG.error( "No segment directories found in {} ",
segmentRootDir.getAbsolutePath());
- System.exit(1);
+ this.errorTracker.recordError(ErrorTracker.ErrorType.OTHER);
+ return;
}
LOG.info("Found {} segment parts", parts.size());
if (gzip && !warc) {
diff --git a/src/java/org/apache/nutch/tools/warc/WARCExporter.java b/src/java/org/apache/nutch/tools/warc/WARCExporter.java
index bf824f9b3f..14b59ac85c 100644
--- a/src/java/org/apache/nutch/tools/warc/WARCExporter.java
+++ b/src/java/org/apache/nutch/tools/warc/WARCExporter.java
@@ -41,6 +41,7 @@
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.Counter;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.Job;
@@ -57,6 +58,8 @@
import org.apache.nutch.parse.ParseText;
import org.apache.nutch.protocol.Content;
import org.apache.nutch.tools.WARCUtils;
+import org.apache.nutch.metrics.ErrorTracker;
+import org.apache.nutch.metrics.NutchMetrics;
import org.apache.nutch.util.HadoopFSUtil;
import org.apache.nutch.util.NutchConfiguration;
import org.apache.nutch.util.NutchJob;
@@ -111,6 +114,35 @@ public static class WARCReducer
// Metadata to JSON
Gson gson = new Gson();
+ // Cached counter references to avoid repeated lookups in hot paths
+ private Counter missingContentCounter;
+ private Counter missingMetadataCounter;
+ private Counter omittedEmptyResponseCounter;
+ private Counter recordsGeneratedCounter;
+ private ErrorTracker errorTracker;
+
+ @Override
+ public void setup(Context context) {
+ // Initialize cached counter references
+ initCounters(context);
+ // Initialize error tracker with cached counters
+ errorTracker = new ErrorTracker(NutchMetrics.GROUP_WARC_EXPORTER, context);
+ }
+
+ /**
+ * Initialize cached counter references to avoid repeated lookups in hot paths.
+ */
+ private void initCounters(Context context) {
+ missingContentCounter = context.getCounter(
+ NutchMetrics.GROUP_WARC_EXPORTER, NutchMetrics.WARC_MISSING_CONTENT_TOTAL);
+ missingMetadataCounter = context.getCounter(
+ NutchMetrics.GROUP_WARC_EXPORTER, NutchMetrics.WARC_MISSING_METADATA_TOTAL);
+ omittedEmptyResponseCounter = context.getCounter(
+ NutchMetrics.GROUP_WARC_EXPORTER, NutchMetrics.WARC_OMITTED_EMPTY_RESPONSE_TOTAL);
+ recordsGeneratedCounter = context.getCounter(
+ NutchMetrics.GROUP_WARC_EXPORTER, NutchMetrics.WARC_RECORDS_GENERATED_TOTAL);
+ }
+
@Override
public void reduce(Text key, Iterable values,
Context context) throws IOException, InterruptedException {
@@ -147,13 +179,13 @@ public void reduce(Text key, Iterable values,
// check that we have everything we need
if (content == null) {
LOG.info("Missing content for {}", key);
- context.getCounter("WARCExporter", "missing content").increment(1);
+ missingContentCounter.increment(1);
return;
}
if (cd == null) {
LOG.info("Missing fetch datum for {}", key);
- context.getCounter("WARCExporter", "missing metadata").increment(1);
+ missingMetadataCounter.increment(1);
return;
}
@@ -161,8 +193,7 @@ public void reduce(Text key, Iterable values,
// Empty responses is everything that was not a regular response
if (!(cd.getStatus() == CrawlDatum.STATUS_FETCH_SUCCESS
|| cd.getStatus() == CrawlDatum.STATUS_FETCH_NOTMODIFIED)) {
- context.getCounter("WARCExporter", "omitted empty response")
- .increment(1);
+ omittedEmptyResponseCounter.increment(1);
return;
}
}
@@ -237,7 +268,7 @@ public void reduce(Text key, Iterable values,
.append(uri.toASCIIString()).append(CRLF);
} catch (Exception e) {
LOG.error("Invalid URI {} ", key);
- context.getCounter("WARCExporter", "invalid URI").increment(1);
+ errorTracker.incrementCounters(e);
return;
}
@@ -269,12 +300,12 @@ public void reduce(Text key, Iterable values,
new ByteArrayInputStream(bos.toByteArray()));
WARCRecord record = new WARCRecord(in);
context.write(NullWritable.get(), new WARCWritable(record));
- context.getCounter("WARCExporter", "records generated").increment(1);
+ recordsGeneratedCounter.increment(1);
} catch (IOException | IllegalStateException exception) {
LOG.error(
"Exception when generating WARC resource record for {} : {}", key,
exception.getMessage());
- context.getCounter("WARCExporter", "exception").increment(1);
+ errorTracker.incrementCounters(exception);
}
// Do we need to emit a metadata record too?
@@ -316,7 +347,7 @@ public void reduce(Text key, Iterable values,
.append(uri.toASCIIString()).append(CRLF);
} catch (Exception e) {
LOG.error("Invalid URI {} ", key);
- context.getCounter("WARCExporter", "invalid URI").increment(1);
+ errorTracker.incrementCounters(e);
return;
}
@@ -332,13 +363,12 @@ public void reduce(Text key, Iterable values,
new ByteArrayInputStream(bos.toByteArray()));
WARCRecord record = new WARCRecord(in);
context.write(NullWritable.get(), new WARCWritable(record));
- context.getCounter("WARCExporter", "records generated")
- .increment(1);
+ recordsGeneratedCounter.increment(1);
} catch (IOException | IllegalStateException exception) {
LOG.error(
"Exception when generating WARC metadata record for {} : {}",
key, exception.getMessage(), exception);
- context.getCounter("WARCExporter", "exception").increment(1);
+ errorTracker.incrementCounters(exception);
}
}
@@ -376,7 +406,7 @@ public void reduce(Text key, Iterable values,
.append(uri.toASCIIString()).append(CRLF);
} catch (Exception e) {
LOG.error("Invalid URI {} ", key);
- context.getCounter("WARCExporter", "invalid URI").increment(1);
+ errorTracker.incrementCounters(e);
return;
}
@@ -392,13 +422,12 @@ public void reduce(Text key, Iterable values,
new ByteArrayInputStream(bos.toByteArray()));
WARCRecord record = new WARCRecord(in);
context.write(NullWritable.get(), new WARCWritable(record));
- context.getCounter("WARCExporter", "records generated")
- .increment(1);
+ recordsGeneratedCounter.increment(1);
} catch (IOException | IllegalStateException exception) {
LOG.error(
"Exception when generating WARC metadata record for {} : {}",
key, exception.getMessage(), exception);
- context.getCounter("WARCExporter", "exception").increment(1);
+ errorTracker.incrementCounters(exception);
}
}
}
diff --git a/src/java/org/apache/nutch/util/DomainStatistics.java b/src/java/org/apache/nutch/util/DomainStatistics.java
index 5ee09c846a..4057795d52 100644
--- a/src/java/org/apache/nutch/util/DomainStatistics.java
+++ b/src/java/org/apache/nutch/util/DomainStatistics.java
@@ -28,6 +28,7 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Counter;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
@@ -38,6 +39,7 @@
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.apache.nutch.crawl.CrawlDatum;
+import org.apache.nutch.metrics.NutchMetrics;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -52,10 +54,6 @@ public class DomainStatistics extends Configured implements Tool {
private static final Text FETCHED_TEXT = new Text("FETCHED");
private static final Text NOT_FETCHED_TEXT = new Text("NOT_FETCHED");
- public static enum MyCounter {
- FETCHED, NOT_FETCHED, EMPTY_RESULT
- };
-
private static final int MODE_HOST = 1;
private static final int MODE_DOMAIN = 2;
private static final int MODE_SUFFIX = 3;
@@ -158,10 +156,29 @@ static class DomainStatisticsMapper extends
Mapper {
int mode = 0;
+ // Cached counter references for performance
+ private Counter fetchedCounter;
+ private Counter notFetchedCounter;
+ private Counter emptyResultCounter;
+
@Override
public void setup(Context context) {
mode = context.getConfiguration().getInt("domain.statistics.mode",
MODE_DOMAIN);
+ // Initialize cached counter references
+ initCounters(context);
+ }
+
+ /**
+ * Initialize cached counter references to avoid repeated lookups in hot paths.
+ */
+ private void initCounters(Context context) {
+ fetchedCounter = context.getCounter(
+ NutchMetrics.GROUP_DOMAIN_STATS, NutchMetrics.DOMAIN_STATS_FETCHED_TOTAL);
+ notFetchedCounter = context.getCounter(
+ NutchMetrics.GROUP_DOMAIN_STATS, NutchMetrics.DOMAIN_STATS_NOT_FETCHED_TOTAL);
+ emptyResultCounter = context.getCounter(
+ NutchMetrics.GROUP_DOMAIN_STATS, NutchMetrics.DOMAIN_STATS_EMPTY_RESULT_TOTAL);
}
@Override
@@ -197,17 +214,17 @@ public void map(Text urlText, CrawlDatum datum, Context context)
}
if (out.trim().equals("")) {
LOG.info("url : {}", url);
- context.getCounter(MyCounter.EMPTY_RESULT).increment(1);
+ emptyResultCounter.increment(1);
}
context.write(new Text(out), new LongWritable(1));
} catch (Exception ex) {
}
- context.getCounter(MyCounter.FETCHED).increment(1);
+ fetchedCounter.increment(1);
context.write(FETCHED_TEXT, new LongWritable(1));
} else {
- context.getCounter(MyCounter.NOT_FETCHED).increment(1);
+ notFetchedCounter.increment(1);
context.write(NOT_FETCHED_TEXT, new LongWritable(1));
}
}
diff --git a/src/java/org/apache/nutch/util/JexlUtil.java b/src/java/org/apache/nutch/util/JexlUtil.java
index 549aebc419..29e8a4f204 100644
--- a/src/java/org/apache/nutch/util/JexlUtil.java
+++ b/src/java/org/apache/nutch/util/JexlUtil.java
@@ -23,33 +23,159 @@
import org.apache.commons.jexl3.JexlBuilder;
import org.apache.commons.jexl3.JexlEngine;
+import org.apache.commons.jexl3.JexlFeatures;
import org.apache.commons.jexl3.JexlScript;
+import org.apache.commons.jexl3.introspection.JexlSandbox;
import org.apache.commons.lang3.time.DateUtils;
+import org.apache.hadoop.conf.Configuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * Utility methods for handling JEXL expressions
+ * Utility methods for handling JEXL expressions used in crawl and index
+ * pipelines. Expressions are evaluated under a {@link JexlSandbox} with
+ * {@link JexlFeatures#newInstance(boolean)} disabled so arbitrary classes cannot
+ * be instantiated from user-supplied configuration.
*/
public class JexlUtil {
private static final Logger LOG = LoggerFactory
.getLogger(MethodHandles.lookup().lookupClass());
+ /**
+ * When {@code true}, JEXL parsing skips the sandbox (unsafe). For trusted
+ * environments only; not recommended.
+ */
+ public static final String DISABLE_SANDBOX_KEY = "nutch.jexl.disable.sandbox";
+
/** Supported format for date parsing yyyy-MM-ddTHH:mm:ssZ */
- private static final Pattern DATE_PATTERN = Pattern.compile("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z");
+ private static final Pattern DATE_PATTERN = Pattern
+ .compile("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z");
+
+ /**
+ * Classes and interfaces that may be introspected when evaluating Nutch JEXL
+ * scripts. Default-deny sandbox: anything not listed is blocked.
+ */
+ private static final String[] SANDBOX_ALLOW_CLASSES = {
+ "java.lang.String",
+ "java.lang.Boolean",
+ "java.lang.Byte",
+ "java.lang.Character",
+ "java.lang.Short",
+ "java.lang.Integer",
+ "java.lang.Long",
+ "java.lang.Float",
+ "java.lang.Double",
+ "java.lang.Number",
+ "java.lang.Math",
+ "java.lang.Comparable",
+ "java.lang.CharSequence",
+ "java.util.Map",
+ "java.util.List",
+ "java.util.Collection",
+ "java.util.Set",
+ "java.util.SortedMap",
+ "java.util.SortedSet",
+ "java.util.Iterator",
+ "java.lang.Iterable",
+ "java.util.AbstractList",
+ "java.util.AbstractCollection",
+ "java.util.AbstractMap",
+ "java.util.AbstractSet",
+ "java.util.ArrayList",
+ "java.util.LinkedList",
+ "java.util.HashMap",
+ "java.util.LinkedHashMap",
+ "java.util.HashSet",
+ "java.util.LinkedHashSet",
+ "java.util.TreeMap",
+ "java.util.TreeSet",
+ "java.util.Collections",
+ "java.util.Arrays",
+ "java.util.regex.Pattern",
+ "java.util.regex.Matcher",
+ "org.apache.commons.jexl3.MapContext",
+ "org.apache.nutch.indexer.NutchDocument",
+ "org.apache.nutch.indexer.NutchField",
+ };
+
+ private static volatile JexlEngine sandboxedEngine;
+ private static volatile JexlEngine legacyEngine;
+
+ private JexlUtil() {
+ }
+
+ private static JexlSandbox createSandbox() {
+ JexlSandbox sandbox = new JexlSandbox(false);
+ for (String name : SANDBOX_ALLOW_CLASSES) {
+ sandbox.allow(name);
+ }
+ return sandbox;
+ }
+
+ private static JexlFeatures createFeatures() {
+ return new JexlFeatures(JexlFeatures.createDefault()).newInstance(false);
+ }
+
+ private static JexlEngine getSandboxedEngine() {
+ if (sandboxedEngine == null) {
+ synchronized (JexlUtil.class) {
+ if (sandboxedEngine == null) {
+ sandboxedEngine = new JexlBuilder().silent(true).strict(true)
+ .sandbox(createSandbox()).features(createFeatures()).create();
+ }
+ }
+ }
+ return sandboxedEngine;
+ }
+
+ private static JexlEngine getLegacyEngine() {
+ if (legacyEngine == null) {
+ synchronized (JexlUtil.class) {
+ if (legacyEngine == null) {
+ legacyEngine = new JexlBuilder().silent(true).strict(true).create();
+ }
+ }
+ }
+ return legacyEngine;
+ }
+
+ private static JexlEngine engineFor(Configuration conf) {
+ if (conf != null && conf.getBoolean(DISABLE_SANDBOX_KEY, false)) {
+ LOG.warn("{}=true: JEXL sandbox is disabled; only use in fully trusted environments.",
+ DISABLE_SANDBOX_KEY);
+ return getLegacyEngine();
+ }
+ return getSandboxedEngine();
+ }
/**
- * Parses the given expression to a JEXL expression. This supports
- * date parsing.
+ * Parses a JEXL expression using the default (sandboxed) engine. Use
+ * {@link #parseExpression(Configuration, String)} when a {@link Configuration}
+ * is available so {@link #DISABLE_SANDBOX_KEY} can be honored.
*
* @param expr string JEXL expression
* @return parsed JEXL expression or null in case of parse error
*/
public static JexlScript parseExpression(String expr) {
- if (expr == null) return null;
-
+ return parseExpression(null, expr);
+ }
+
+ /**
+ * Parses a JEXL expression. Unless {@link #DISABLE_SANDBOX_KEY} is set to
+ * {@code true} in {@code conf}, the expression is parsed for execution under
+ * a restrictive sandbox.
+ *
+ * @param conf Hadoop configuration, or null to always use the sandbox
+ * @param expr string JEXL expression
+ * @return parsed JEXL expression or null in case of parse error
+ */
+ public static JexlScript parseExpression(Configuration conf, String expr) {
+ if (expr == null) {
+ return null;
+ }
+
try {
// Translate any date object into a long. Dates must be in the DATE_PATTERN
// format. For example: 2016-03-20T00:00:00Z
@@ -57,22 +183,21 @@ public static JexlScript parseExpression(String expr) {
if (matcher.find()) {
String date = matcher.group();
-
+
// parse the matched substring and get the epoch
- Date parsedDate = DateUtils.parseDateStrictly(date, new String[] {"yyyy-MM-dd'T'HH:mm:ss'Z'"});
+ Date parsedDate = DateUtils.parseDateStrictly(date,
+ new String[] { "yyyy-MM-dd'T'HH:mm:ss'Z'" });
long time = parsedDate.getTime();
-
+
// replace the original string date with the numeric value
expr = expr.replace(date, Long.toString(time));
}
- JexlEngine jexl = new JexlBuilder().silent(true).strict(true).create();
-
- return jexl.createScript(expr);
+ return engineFor(conf).createScript(expr);
} catch (Exception e) {
LOG.error(e.getMessage());
}
-
+
return null;
}
}
diff --git a/src/java/org/apache/nutch/util/SitemapProcessor.java b/src/java/org/apache/nutch/util/SitemapProcessor.java
index d83a6e358c..21362223cd 100644
--- a/src/java/org/apache/nutch/util/SitemapProcessor.java
+++ b/src/java/org/apache/nutch/util/SitemapProcessor.java
@@ -31,6 +31,7 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.Counter;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
@@ -45,6 +46,8 @@
import org.apache.hadoop.util.ToolRunner;
import org.apache.nutch.crawl.CrawlDatum;
import org.apache.nutch.hostdb.HostDatum;
+import org.apache.nutch.metrics.ErrorTracker;
+import org.apache.nutch.metrics.NutchMetrics;
import org.apache.nutch.net.URLFilters;
import org.apache.nutch.net.URLNormalizers;
import org.apache.nutch.protocol.Content;
@@ -113,6 +116,14 @@ private static class SitemapMapper extends Mapper values, Context context)
originalDatum.setModifiedTime(sitemapDatum.getModifiedTime());
}
- context.getCounter("Sitemap", "existing_sitemap_entries").increment(1);
+ existingEntriesCounter.increment(1);
context.write(key, originalDatum);
}
else if(sitemapDatum != null) {
// For the newly discovered links via sitemap, set the status as unfetched and emit
- context.getCounter("Sitemap", "new_sitemap_entries").increment(1);
+ newEntriesCounter.increment(1);
sitemapDatum.setStatus(CrawlDatum.STATUS_DB_UNFETCHED);
context.write(key, sitemapDatum);
}
@@ -457,11 +507,11 @@ public void sitemap(Path crawldb, Path hostdb, Path sitemapUrlDir, boolean stric
FSUtils.replace(fs, current, tempCrawlDb, true);
LockUtil.removeLockFile(fs, lock);
- long filteredRecords = job.getCounters().findCounter("Sitemap", "filtered_records").getValue();
- long fromHostname = job.getCounters().findCounter("Sitemap", "sitemaps_from_hostname").getValue();
- long fromSeeds = job.getCounters().findCounter("Sitemap", "sitemap_seeds").getValue();
- long failedFetches = job.getCounters().findCounter("Sitemap", "failed_fetches").getValue();
- long newSitemapEntries = job.getCounters().findCounter("Sitemap", "new_sitemap_entries").getValue();
+ long filteredRecords = job.getCounters().findCounter(NutchMetrics.GROUP_SITEMAP, NutchMetrics.SITEMAP_FILTERED_RECORDS_TOTAL).getValue();
+ long fromHostname = job.getCounters().findCounter(NutchMetrics.GROUP_SITEMAP, NutchMetrics.SITEMAP_FROM_HOSTNAME_TOTAL).getValue();
+ long fromSeeds = job.getCounters().findCounter(NutchMetrics.GROUP_SITEMAP, NutchMetrics.SITEMAP_SEEDS_TOTAL).getValue();
+ long failedFetches = job.getCounters().findCounter(NutchMetrics.GROUP_SITEMAP, NutchMetrics.SITEMAP_FAILED_FETCHES_TOTAL).getValue();
+ long newSitemapEntries = job.getCounters().findCounter(NutchMetrics.GROUP_SITEMAP, NutchMetrics.SITEMAP_NEW_ENTRIES_TOTAL).getValue();
LOG.info("SitemapProcessor: Total records rejected by filters: {}", filteredRecords);
LOG.info("SitemapProcessor: Total sitemaps from host name: {}", fromHostname);
diff --git a/src/java/org/apache/nutch/util/URLUtil.java b/src/java/org/apache/nutch/util/URLUtil.java
index 0cfce1c650..fd036480a6 100644
--- a/src/java/org/apache/nutch/util/URLUtil.java
+++ b/src/java/org/apache/nutch/util/URLUtil.java
@@ -16,6 +16,7 @@
*/
package org.apache.nutch.util;
+import java.lang.invoke.MethodHandles;
import java.net.IDN;
import java.net.MalformedURLException;
import java.net.URI;
@@ -23,11 +24,22 @@
import java.util.Locale;
import java.util.regex.Pattern;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.ibm.icu.text.IDNA;
+
import crawlercommons.domains.EffectiveTldFinder;
/** Utility class for URL analysis */
public class URLUtil {
+ private static final Logger LOG = LoggerFactory
+ .getLogger(MethodHandles.lookup().lookupClass());
+
+ private static final IDNA idna = IDNA.getUTS46Instance(
+ IDNA.NONTRANSITIONAL_TO_ASCII | IDNA.NONTRANSITIONAL_TO_UNICODE);
+
/**
* Resolve relative URL-s and fix a java.net.URL error in handling of URLs
* with pure query targets.
@@ -103,7 +115,7 @@ static URL fixPureQueryTargets(URL base, String target)
* https://publicsuffix.org/list/public_suffix_list.dat and are compared
* using
+ * "https://crawler-commons.github.io/crawler-commons/1.6/crawlercommons/domains/EffectiveTldFinder.html">
* crawler-commons' EffectiveTldFinder. Only ICANN domain suffixes are
* used. Because EffectiveTldFinder loads the public suffix list as file
* "effective_tld_names.dat" from the Java classpath, it's possible to use the
@@ -262,7 +274,7 @@ public static String getDomainSuffix(URL url) {
EffectiveTldFinder.EffectiveTLD suffix = EffectiveTldFinder.getEffectiveTLD(host, true);
if (suffix != null) {
- return suffix.getDomain();
+ return suffix.getSuffix();
}
return null;
@@ -520,17 +532,39 @@ public static String getProtocol(URL url) {
return url.getProtocol();
}
+ public static boolean isAscii(String str) {
+ char[] chars = str.toCharArray();
+ for (char c : chars) {
+ if (c > 127) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Convert URL with IDN host/domain name into the ASCII representation.
+ *
+ * @param url
+ * URL string to convert
+ * @return URL string with ASCII host/domain name or null if conversion fails.
+ */
public static String toASCII(String url) {
try {
URL u = new URL(url);
String host = u.getHost();
- if (host == null || host.isEmpty()) {
- // no host name => no punycoded domain name
- // also do not add additional slashes for file: URLs (NUTCH-1880)
+ String hostLowerCase = host.toLowerCase(Locale.ROOT);
+ if (host == null || host.isEmpty()
+ || (isAscii(host) && host.equals(hostLowerCase))) {
+ // - no host name => no punycoded domain name
+ // - also do not add additional slashes for file: URLs (NUTCH-1880)
+ // - do nothing if host is already ASCII-only
+ // - not already in lowercase => conversion also lowercases host name
return url;
}
- URI p = new URI(u.getProtocol(), u.getUserInfo(), IDN.toASCII(host),
- u.getPort(), u.getPath(), u.getQuery(), u.getRef());
+ URI p = new URI(u.getProtocol(), u.getUserInfo(),
+ convertIDNA2008(hostLowerCase, true), u.getPort(), u.getPath(),
+ u.getQuery(), u.getRef());
return p.toString();
} catch (Exception e) {
@@ -538,13 +572,25 @@ public static String toASCII(String url) {
}
}
+ /**
+ * Convert URL with IDN host/domain name to the Unicode representation.
+ *
+ * @param url
+ * URL string to convert
+ * @return URL string with Unicode host/domain name or null if conversion
+ * fails.
+ */
public static String toUNICODE(String url) {
try {
URL u = new URL(url);
String host = u.getHost();
- if (host == null || host.isEmpty()) {
- // no host name => no punycoded domain name
- // also do not add additional slashes for file: URLs (NUTCH-1880)
+ String hostLowerCase = host.toLowerCase(Locale.ROOT);
+ if (host == null || host.isEmpty()
+ || (!hostLowerCase.contains("xn--") && host.equals(hostLowerCase))) {
+ // - no host name => no punycoded domain name
+ // - also do not add additional slashes for file: URLs (NUTCH-1880)
+ // - contains 'xn--' => needs conversion
+ // - not already in lowercase => conversion also lowercases host name
return url;
}
StringBuilder sb = new StringBuilder();
@@ -554,7 +600,7 @@ public static String toUNICODE(String url) {
sb.append(u.getUserInfo());
sb.append('@');
}
- sb.append(IDN.toUnicode(host));
+ sb.append(convertIDNA2008(hostLowerCase, false));
if (u.getPort() != -1) {
sb.append(':');
sb.append(u.getPort());
@@ -572,22 +618,83 @@ public static String toUNICODE(String url) {
}
/**
- * For testing
- * @param args print with no args to get help
+ * Convert IDN host to ASCII or Unicode using Java's built-in {@link IDN}
+ * class.
+ *
+ * The conversion supports only IDNA2003, it does not support IDNA2008.
+ * However, unless the parameter strictIDNA2003 is true, the
+ * methods {@link IDN#toASCII(String, int)} resp.
+ * {@link IDN#toUnicode(String, int)} are called passing the flag
+ * {@link IDN#ALLOW_UNASSIGNED} to avoid that the conversion fails on
+ * characters not in the repertoire of Unicode 3.2.
+ *
+ * @param host
+ * host name to be converted (lowercase expected)
+ * @param toAscii
+ * if true convert to ASCII, otherwise to Unicode
+ * @param strictIDNA2003
+ * if true, do
+ * @return converted host name
+ * @throws MalformedURLException
+ * if the conversion fails
*/
- public static void main(String[] args) {
-
- if (args.length != 1) {
- System.err.println("Usage : URLUtil ");
- return;
+ public static String convertIDNA2003(String host, boolean toAscii,
+ boolean strictIDNA2003) throws MalformedURLException {
+ try {
+ if (toAscii) {
+ return IDN.toASCII(host, strictIDNA2003 ? 0 : IDN.ALLOW_UNASSIGNED);
+ } else {
+ return IDN.toUnicode(host, strictIDNA2003 ? 0 : IDN.ALLOW_UNASSIGNED);
+ }
+ } catch (IllegalArgumentException | IndexOutOfBoundsException e) {
+ // IllegalArgumentException: thrown if the input string contains
+ // non-convertible Unicode codepoints
+ // IndexOutOfBoundsException: thrown (undocumented) if one "label"
+ // (non-ASCII dot-separated segment) is longer than 256 characters,
+ // cf. https://bugs.openjdk.java.net/browse/JDK-6806873
+ LOG.debug("Failed to convert IDN host {}: ", host, e);
+ throw (MalformedURLException) new MalformedURLException(
+ "Invalid IDN " + host + ": " + e.getMessage()).initCause(e);
}
+ }
- String url = args[0];
- try {
- System.out.println(URLUtil.getDomainName(new URL(url)));
- } catch (MalformedURLException ex) {
- ex.printStackTrace();
+ /**
+ * Convert IDN host to ASCII or Unicode using ICU's {@link IDNA} class.
+ *
+ * The conversion supports IDNA2008 names.
+ *
+ * @param host
+ * host name to be converted (lowercase expected)
+ * @param toAscii
+ * if true convert to ASCII, otherwise to Unicode
+ * @return converted host name
+ * @throws MalformedURLException
+ * if the conversion fails
+ */
+ public static String convertIDNA2008(String host, boolean toAscii)
+ throws MalformedURLException {
+ final IDNA.Info idnaInfo = new IDNA.Info();
+ final StringBuilder hostConverted = new StringBuilder();
+ if (toAscii) {
+ idna.nameToASCII(host, hostConverted, idnaInfo);
+ } else {
+ idna.nameToUnicode(host, hostConverted, idnaInfo);
}
+ if (idnaInfo.hasErrors()) {
+ StringBuilder msg = new StringBuilder();
+ for (IDNA.Error error : idnaInfo.getErrors()) {
+ if (msg.length() == 0) {
+ msg.append("Invalid IDNA2008 host").append(host).append(": ");
+ } else {
+ msg.append(", ");
+ }
+ msg.append(error.name());
+ }
+ String errorMsg = msg.toString();
+ LOG.debug("Failed to convert IDN host {}: {}", host, errorMsg);
+ throw new MalformedURLException(errorMsg);
+ }
+ return hostConverted.toString();
}
/**
@@ -610,4 +717,24 @@ public static boolean isHomePageOf(URL url, String hostName) {
&& url.getRef() == null //
&& url.getUserInfo() == null;
}
+
+ /**
+ * For testing
+ * @param args print with no args to get help
+ */
+ public static void main(String[] args) {
+
+ if (args.length != 1) {
+ System.err.println("Usage : URLUtil ");
+ System.err.println("\nExtract and print pay-level domain names for the input URL");
+ return;
+ }
+
+ String url = args[0];
+ try {
+ System.out.println(URLUtil.getDomainName(new URL(url)));
+ } catch (MalformedURLException ex) {
+ ex.printStackTrace();
+ }
+ }
}
diff --git a/src/java/org/commoncrawl/tools/UrlCleaner.java b/src/java/org/commoncrawl/tools/UrlCleaner.java
new file mode 100644
index 0000000000..c4d92ca669
--- /dev/null
+++ b/src/java/org/commoncrawl/tools/UrlCleaner.java
@@ -0,0 +1,381 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.commoncrawl.tools;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Pattern;
+
+import org.apache.commons.lang3.time.StopWatch;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.nutch.metrics.NutchMetrics;
+import org.apache.nutch.net.URLFilterException;
+import org.apache.nutch.net.URLFilters;
+import org.apache.nutch.net.URLNormalizers;
+import org.apache.nutch.util.NutchConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import crawlercommons.domains.EffectiveTldFinder;
+
+public class UrlCleaner extends Configured implements Tool {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(MethodHandles.lookup().lookupClass());
+
+ public static enum OutputKeyType {
+ URL,
+ HOST,
+ DOMAIN,
+ HOST_REVERSED,
+ DOMAIN_REVERSED,
+ HOST_REVERSED_TRAILING_DOT,
+ DOMAIN_REVERSED_TRAILING_DOT;
+
+ public static OutputKeyType get(String name) {
+ for (OutputKeyType t : OutputKeyType.values()) {
+ if (name.equalsIgnoreCase(t.toString())) {
+ return t;
+ }
+ }
+ return URL;
+ }
+ }
+
+ private static final String CHECK_DOMAIN = "urlcleaner.check.domain";
+ private static final String OUTPUT_KEY_TYPE = "urlcleaner.output.key.type";
+ private static final String SUM_VALUES = "urlcleaner.sum.values";
+
+ private Configuration config;
+
+
+ public static class UrlCleanerMapper extends Mapper {
+
+ public static final String URL_NORMALIZING_SCOPE = "crawldb.url.normalizers.scope";
+
+ private static Pattern SPLIT_HOST_PATTERN = Pattern.compile("\\.");
+
+ private URLNormalizers urlNormalizers;
+ private URLFilters filters;
+ private String scope;
+ private boolean checkDomain;
+ private boolean needDomain;
+ private boolean needHost;
+ private boolean sumValues;
+ private OutputKeyType outputType;
+
+ @Override
+ public void setup(Context context) {
+ Configuration conf = context.getConfiguration();
+ scope = conf.get(URL_NORMALIZING_SCOPE, URLNormalizers.SCOPE_INJECT);
+ urlNormalizers = new URLNormalizers(conf, scope);
+ filters = new URLFilters(conf);
+ checkDomain = conf.getBoolean(CHECK_DOMAIN, false);
+ sumValues = conf.getBoolean(SUM_VALUES, false);
+ outputType = OutputKeyType.get(conf.get(OUTPUT_KEY_TYPE));
+ LOG.info("check domain names: {}", checkDomain);
+ LOG.info("output type: {}", outputType);
+ needDomain = checkDomain || outputType == OutputKeyType.DOMAIN
+ || outputType == OutputKeyType.DOMAIN_REVERSED
+ || outputType == OutputKeyType.DOMAIN_REVERSED_TRAILING_DOT;
+ needHost = needDomain || outputType == OutputKeyType.HOST
+ || outputType == OutputKeyType.HOST_REVERSED
+ || outputType == OutputKeyType.HOST_REVERSED_TRAILING_DOT;
+ }
+
+ public static String[] reverseHost(String hostName) {
+ String[] rev = SPLIT_HOST_PATTERN.split(hostName);
+ for (int i = 0; i < (rev.length/2); i++) {
+ String temp = rev[i];
+ rev[i] = rev[rev.length - i - 1];
+ rev[rev.length - i - 1] = temp;
+ }
+ return rev;
+ }
+
+ @Override
+ public void map(Text key, Text value, Context context)
+ throws IOException, InterruptedException {
+
+ String urlOrig = key.toString();
+ String url = urlOrig.trim();
+
+ try {
+ url = urlNormalizers.normalize(url, scope);
+ } catch (MalformedURLException e) {
+ context.getCounter(NutchMetrics.GROUP_URLCLEANER,
+ NutchMetrics.URLCLEANER_REJECTED_TOTAL).increment(1);
+ return;
+ }
+ try {
+ url = filters.filter(url);
+ } catch (URLFilterException e) {
+ context.getCounter(NutchMetrics.GROUP_URLCLEANER,
+ NutchMetrics.URLCLEANER_REJECTED_TOTAL).increment(1);
+ return;
+ }
+
+ if (url == null) {
+ context.getCounter(NutchMetrics.GROUP_URLCLEANER,
+ NutchMetrics.URLCLEANER_REJECTED_TOTAL).increment(1);
+ return;
+ }
+
+ String host = null, domain = null;
+ if (needHost) {
+ try {
+ URL u = new URL(url);
+ host = u.getHost();
+ if (needDomain) {
+ domain = EffectiveTldFinder.getAssignedDomain(host, true, true);
+ if (checkDomain && domain == null) {
+ context
+ .getCounter(NutchMetrics.GROUP_URLCLEANER,
+ NutchMetrics.URLCLEANER_REJECTED_INVALID_DOMAIN_TOTAL)
+ .increment(1);
+ return;
+ }
+ }
+ } catch (MalformedURLException e) {
+ context.getCounter(NutchMetrics.GROUP_URLCLEANER,
+ NutchMetrics.URLCLEANER_REJECTED_TOTAL).increment(1);
+ return;
+ }
+ }
+
+ if (url.equals(urlOrig)) {
+ context.getCounter(NutchMetrics.GROUP_URLCLEANER,
+ NutchMetrics.URLCLEANER_ACCEPTED_UNCHANGED_TOTAL).increment(1);
+ } else {
+ context.getCounter(NutchMetrics.GROUP_URLCLEANER,
+ NutchMetrics.URLCLEANER_ACCEPTED_NORMALIZED_TOTAL).increment(1);
+ key.set(url);
+ }
+
+ String keyVal = null;
+ String addVal = url;
+ switch (outputType) {
+ case HOST:
+ keyVal = (host != null) ? host : "";
+ break;
+ case DOMAIN:
+ keyVal = (domain != null) ? domain : "";
+ addVal = host + "\t" + url;
+ break;
+ case HOST_REVERSED:
+ keyVal = (host != null) ? host : "";
+ keyVal = String.join(".", reverseHost(keyVal));
+ break;
+ case DOMAIN_REVERSED:
+ keyVal = (domain != null) ? domain : "";
+ keyVal = String.join(".", reverseHost(keyVal));
+ addVal = String.join(".", reverseHost(host)) + "\t" + url;
+ break;
+ case HOST_REVERSED_TRAILING_DOT:
+ keyVal = (host != null) ? host : "";
+ List parts = new ArrayList<>();
+ parts.addAll(Arrays.asList(reverseHost(keyVal)));
+ parts.add("");
+ keyVal = String.join(".", parts);
+ break;
+ case DOMAIN_REVERSED_TRAILING_DOT:
+ keyVal = (domain != null) ? domain : "";
+ parts = new ArrayList<>();
+ parts.addAll(Arrays.asList(reverseHost(keyVal)));
+ parts.add("");
+ keyVal = String.join(".", parts);
+ parts = new ArrayList<>();
+ parts.addAll(Arrays.asList(reverseHost(host)));
+ parts.add("");
+ addVal = String.join(".", parts) + "\t" + url;
+ break;
+ default:
+ break;
+ }
+
+ if (sumValues) {
+ if (outputType != OutputKeyType.URL) {
+ key.set(keyVal + "\t" + addVal);
+ }
+ context.write(key, value);
+ } else {
+ if (outputType != OutputKeyType.URL) {
+ key.set(keyVal);
+ value.set(addVal + "\t" + value.toString());
+ }
+ context.write(key, value);
+ }
+ }
+
+ }
+
+ public static class UrlCleanerTextSumReducer
+ extends Reducer {
+
+ private Text result = new Text();
+
+ @Override
+ public void reduce(Text key, Iterable values, Context context)
+ throws IOException, InterruptedException {
+ long sum = 0;
+ for (Text val : values) {
+ try {
+ long v = Long.parseLong(val.toString());
+ sum += v;
+ } catch (NumberFormatException e) {
+ LOG.error("Value is not a long integer (key: {} value: {})", key, val);
+ sum += 1;
+ }
+ }
+ result.set(Long.toString(sum));
+ context.write(key, result);
+ }
+ }
+
+ @Override
+ public Configuration getConf() {
+ return config;
+ }
+
+ @Override
+ public void setConf(Configuration conf) {
+ config = conf;
+ }
+
+ public void clean(Path[] inputs, Path output, boolean checkDomain,
+ OutputKeyType outputKeyType, boolean sumValues) throws Exception {
+
+ StopWatch stopWatch = new StopWatch();
+ stopWatch.start();
+ LOG.info("UrlCleaner: starting");
+
+ Configuration conf = getConf();
+ conf.setBoolean(CHECK_DOMAIN, checkDomain);
+ conf.set(OUTPUT_KEY_TYPE, outputKeyType.toString());
+ conf.setBoolean(SUM_VALUES, sumValues);
+
+ Job job = Job.getInstance(conf, UrlCleaner.class.getName());
+ job.setJarByClass(UrlCleaner.class);
+ job.setMapperClass(UrlCleanerMapper.class);
+ if (sumValues) {
+ job.setReducerClass(UrlCleanerTextSumReducer.class);
+ } else {
+ job.setReducerClass(Reducer.class);
+ }
+ job.setInputFormatClass(KeyValueTextInputFormat.class);
+ job.setOutputFormatClass(TextOutputFormat.class);
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(Text.class);
+
+ for (Path input : inputs) {
+ KeyValueTextInputFormat.addInputPath(job, input);
+ }
+ FileOutputFormat.setOutputPath(job, output);
+
+ try {
+ // run the job
+ boolean success = job.waitForCompletion(true);
+ if (!success) {
+ String message = "UrlCleaner job did not succeed, job status: "
+ + job.getStatus().getState() + ", reason: "
+ + job.getStatus().getFailureInfo();
+ LOG.error(message);
+ // throw exception so that calling routine can exit with error
+ throw new RuntimeException(message);
+ }
+
+ } catch (IOException | InterruptedException | ClassNotFoundException | NullPointerException e) {
+ LOG.error("UrlCleaner job failed: {}", e.getMessage());
+ throw e;
+ }
+
+ stopWatch.stop();
+ LOG.info("UrlCleaner: finished, elapsed: {} ms",
+ stopWatch.getTime(TimeUnit.MILLISECONDS));
+ }
+
+ public void usage() {
+ System.err.println(
+ "Usage: UrlCleaner [-D...] [-checkDomain] [-sumValues] [-outputKey <...>] ... \n");
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+
+ boolean checkDomain = false;
+ OutputKeyType outputKeyType = OutputKeyType.URL;
+ boolean sumValues = false;
+
+ int i = 0;
+ for (; i < (args.length - 2); i++) {
+ if (args[i].equals("-checkDomain")) {
+ checkDomain = true;
+ } else if (args[i].equals("-outputKey")) {
+ String key = args[++i];
+ outputKeyType = OutputKeyType.get(key);
+ } else if (args[i].equals("-sumValues")) {
+ sumValues = true;
+ } else {
+ break;
+ }
+ }
+
+ if ((args.length - i) < 2) {
+ usage();
+ return -1;
+ }
+
+ Path[] inputs = new Path[args.length - i - 1];
+ for (int j = 0; j < (args.length - i - 1); j++) {
+ inputs[j] = new Path(args[i + j]);
+ }
+ Path output = new Path(args[args.length - 1]);
+
+ try {
+ clean(inputs, output, checkDomain, outputKeyType, sumValues);
+ } catch (Exception e) {
+ LOG.error("UrlCleaner: ", e);
+ return -1;
+ }
+ return 0;
+ }
+
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(NutchConfiguration.create(), new UrlCleaner(),
+ args);
+ System.exit(res);
+ }
+
+}
diff --git a/src/java/org/commoncrawl/tools/UrlSampler.java b/src/java/org/commoncrawl/tools/UrlSampler.java
new file mode 100644
index 0000000000..e2060e1f47
--- /dev/null
+++ b/src/java/org/commoncrawl/tools/UrlSampler.java
@@ -0,0 +1,376 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.commoncrawl.tools;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Pattern;
+
+import org.apache.commons.lang3.time.StopWatch;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.nutch.crawl.Generator2;
+import org.apache.nutch.crawl.Generator2.DomainScorePair;
+import org.apache.nutch.crawl.URLPartitioner;
+import org.apache.nutch.metrics.NutchMetrics;
+import org.apache.nutch.util.NutchConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Sample URLs used as Nutch seeds following per-domain limits.
+ *
+ * Input:
+ *
+ * <url> \t nutch.score=<score>
+ *
+ *
+ * The score is calculated as:
+ *
+ *
+ * 0.001 * default_score * log10(1 + inlink_count)
+ *
+ */
+public class UrlSampler extends Configured implements Tool {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(MethodHandles.lookup().lookupClass());
+
+ private static final Pattern URL_PATTERN = Pattern.compile("^https?://",
+ Pattern.CASE_INSENSITIVE);
+
+ public static class TextCountPair implements Writable {
+
+ private Text text = new Text();
+ private LongWritable count = new LongWritable();
+
+ public void set(Text text, long count) {
+ this.text.set(text);
+ this.count.set(count);
+ }
+
+ public Text getText() {
+ return text;
+ }
+
+ public LongWritable getCount() {
+ return count;
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ text.readFields(in);
+ count.readFields(in);
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ text.write(out);
+ count.write(out);
+ }
+
+ }
+
+ public static class SampleMapper
+ extends Mapper {
+
+ private DomainScorePair outputKey = new DomainScorePair();
+ private TextCountPair outputValue = new TextCountPair();
+
+ @Override
+ public void map(Text key, Text value, Context context)
+ throws IOException, InterruptedException {
+
+ if (!URL_PATTERN.matcher(key.toString()).find()) {
+ // got
+ outputKey.set(key, Float.MAX_VALUE);
+ outputValue.set(value, -1);
+ context.write(outputKey, outputValue);
+ return;
+ }
+
+ // got
+ String url = key.toString();
+ String domain;
+ try {
+ URL u = new URL(url);
+ domain = URLPartitioner.getDomainName(u.getHost());
+ } catch (Exception e) {
+ LOG.warn("Malformed URL: '{}', skipping ({})", url, e.getMessage());
+ context.getCounter(NutchMetrics.GROUP_URLSAMPLER,
+ NutchMetrics.URLSAMPLER_MALFORMED_URL_TOTAL).increment(1);
+ return;
+ }
+
+ long count = 0;
+ try {
+ count = Long.parseLong(value.toString());
+ } catch (NumberFormatException e) {
+ LOG.error("Value is not a long integer (url: {} value: {})", url,
+ value);
+ }
+ outputKey.set(domain, (float) (Math.random() * count));
+ outputValue.set(key, count);
+ context.write(outputKey, outputValue);
+ }
+ }
+
+ public static class SampleReducer
+ extends Reducer {
+
+ private int maxUrlsPerDomain = 40;
+ private int maxHostsPerDomain = 2;
+ private int maxUrlsPerHost = 20;
+ private float defaultScore = .001f;
+ private Text meta = new Text();
+
+ @Override
+ public void setup(Context context) {
+ Configuration conf = context.getConfiguration();
+ maxUrlsPerDomain = conf.getInt("urlsample.urls.per.domain",
+ maxUrlsPerDomain);
+ maxUrlsPerHost = conf.getInt("urlsample.urls.per.host", maxUrlsPerHost);
+ maxHostsPerDomain = conf.getInt("urlsample.hosts.per.domain",
+ maxHostsPerDomain);
+ defaultScore = conf.getFloat("urlsample.default.score", defaultScore);
+ }
+
+ @Override
+ public void reduce(DomainScorePair key, Iterable values,
+ Context context) throws IOException, InterruptedException {
+ int maxUrls = maxUrlsPerDomain;
+ int maxHosts = maxHostsPerDomain;
+ int maxPerHost = maxUrlsPerHost;
+ float domainScore = defaultScore;
+ int nUrls = 0;
+ int nUrlsSampled = 0;
+ int skippedMaxUrls = 0;
+ int skippedMaxHosts = 0;
+ int skippedMaxUrlsPerHost = 0;
+ double sumScores = .0;
+ String domain = null;
+ Text limits = null;
+ Map hosts = null;
+
+ for (TextCountPair val : values) {
+ Text text = val.getText();
+ long count = val.getCount().get();
+ if (limits == null && count == -1) {
+ limits = new Text(text);
+ // store limits but parse later lazily (if there are URLs to sample)
+ continue;
+ } else if (limits != null) {
+ String[] l = limits.toString().split("\t");
+ if (l.length >= 5) {
+ try {
+ // long rank = l[0]
+ maxUrls = Integer.parseInt(l[1]);
+ maxHosts = Integer.parseInt(l[2]);
+ maxPerHost = Integer.parseInt(l[3]);
+ domainScore = Float.parseFloat(l[4]);
+ } catch (NumberFormatException e) {
+ LOG.warn("Invalid domain limits: {}", limits, e);
+ }
+ }
+ limits = null;
+ }
+ if (domain == null) {
+ // processing first URL in values
+ domain = key.getDomain().toString();
+ hosts = new HashMap<>();
+ }
+ String host = null;
+ try {
+ host = new URL(text.toString()).getHost().toLowerCase(Locale.ROOT);
+ if (host.endsWith(domain)) {
+ // clip common domain name suffix to save storage space in map keys
+ host = host.substring(0, host.length() - domain.length());
+ } else {
+ LOG.warn("Host {} does not have domain {} as suffix!", host,
+ domain);
+ }
+ } catch (MalformedURLException e) {
+ context.getCounter(NutchMetrics.GROUP_URLSAMPLER,
+ NutchMetrics.URLSAMPLER_MALFORMED_URL_TOTAL).increment(1);
+ continue;
+ }
+ nUrls++;
+ if (nUrlsSampled > maxUrls) {
+ skippedMaxUrls++;
+ continue;
+ }
+ int[] nUrlsPerHost = hosts.get(host);
+ if (nUrlsPerHost != null) {
+ if (nUrlsPerHost[0]++ > maxPerHost) {
+ skippedMaxUrlsPerHost++;
+ continue;
+ }
+ } else {
+ if (hosts.size() >= maxHosts) {
+ skippedMaxHosts++;
+ continue;
+ }
+ hosts.put(host, new int[] { 0 });
+ }
+ nUrlsSampled++;
+ double score = .001d * domainScore * Math.log10(1 + count);
+ sumScores += score;
+ meta.set(String.format(Locale.ROOT, "nutch.score=%.12f", score));
+ context.write(text, meta);
+ }
+ if (nUrls == 0)
+ return;
+ context.getCounter(NutchMetrics.GROUP_URLSAMPLER,
+ NutchMetrics.URLSAMPLER_SKIPPED_MAX_URLS_TOTAL).increment(skippedMaxUrls);
+ context
+ .getCounter(NutchMetrics.GROUP_URLSAMPLER,
+ NutchMetrics.URLSAMPLER_SKIPPED_MAX_URLS_PER_HOST_TOTAL)
+ .increment(skippedMaxUrlsPerHost);
+ context.getCounter(NutchMetrics.GROUP_URLSAMPLER,
+ NutchMetrics.URLSAMPLER_SKIPPED_MAX_HOSTS_TOTAL).increment(skippedMaxHosts);
+ LOG.info(
+ "Sampled for domain {} : {} hosts, {} URLs ({} skipped: {} max. URLs, {} max. per host, {} max. hosts), sum of scores = {}",
+ domain, hosts.size(), nUrlsSampled, (nUrls - nUrlsSampled),
+ skippedMaxUrls, skippedMaxUrlsPerHost, skippedMaxHosts, sumScores);
+ }
+ }
+
+ private void sample(Path[] inputs, Path output) throws Exception {
+
+ StopWatch stopWatch = new StopWatch();
+ stopWatch.start();
+ LOG.info("UrlSampler: starting");
+
+ Configuration conf = getConf();
+ conf.setInt("partition.url.seed", new Random().nextInt());
+
+ Job job = Job.getInstance(conf, UrlSampler.class.getName());
+ job.setJarByClass(UrlSampler.class);
+ job.setMapperClass(SampleMapper.class);
+ job.setPartitionerClass(Generator2.Selector.class);
+ job.setSortComparatorClass(Generator2.ScoreComparator.class);
+ job.setGroupingComparatorClass(Generator2.DomainComparator.class);
+ job.setReducerClass(SampleReducer.class);
+ job.setInputFormatClass(KeyValueTextInputFormat.class);
+ job.setOutputFormatClass(TextOutputFormat.class);
+ job.setMapOutputKeyClass(Generator2.DomainScorePair.class);
+ job.setMapOutputValueClass(TextCountPair.class);
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(Text.class);
+
+ for (Path input : inputs) {
+ KeyValueTextInputFormat.addInputPath(job, input);
+ }
+ FileOutputFormat.setOutputPath(job, output);
+
+ try {
+ // run the job
+ boolean success = job.waitForCompletion(true);
+ if (!success) {
+ String message = "UrlSampler job did not succeed, job status: "
+ + job.getStatus().getState() + ", reason: "
+ + job.getStatus().getFailureInfo();
+ LOG.error(message);
+ // throw exception so that calling routine can exit with error
+ throw new RuntimeException(message);
+ }
+
+ } catch (IOException | InterruptedException | ClassNotFoundException
+ | NullPointerException e) {
+ LOG.error("UrlSampler job failed: {}", e.getMessage());
+ throw e;
+ }
+
+ stopWatch.stop();
+ LOG.info("UrlSampler: finished}, elapsed: {} ms",
+ stopWatch.getTime(TimeUnit.MILLISECONDS));
+ }
+
+ public void usage() {
+ System.err.println(
+ "Usage: UrlSampler [-D...] ... \n");
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ if (args.length < 2) {
+ usage();
+ return -1;
+ }
+
+ Path[] inputs = new Path[args.length - 1];
+ for (int i = 0; i < (args.length - 1); i++) {
+ inputs[i] = new Path(args[i]);
+ }
+ Path output = new Path(args[args.length - 1]);
+
+ try {
+ sample(inputs, output);
+ } catch (Exception e) {
+ LOG.error("UrlSampler: ", e);
+ return -1;
+ }
+ return 0;
+ }
+
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(NutchConfiguration.create(), new UrlSampler(),
+ args);
+ System.exit(res);
+ }
+
+}
diff --git a/src/java/org/commoncrawl/tools/UrlSamplerHost.java b/src/java/org/commoncrawl/tools/UrlSamplerHost.java
new file mode 100644
index 0000000000..bce68ad50f
--- /dev/null
+++ b/src/java/org/commoncrawl/tools/UrlSamplerHost.java
@@ -0,0 +1,437 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.commoncrawl.tools;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.net.URL;
+import java.util.Locale;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Pattern;
+
+import org.apache.commons.lang3.time.StopWatch;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.nutch.crawl.Generator2;
+import org.apache.nutch.crawl.Generator2.DomainScorePair;
+import org.apache.nutch.metrics.NutchMetrics;
+import org.apache.nutch.util.NutchConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Sample URLs used as Nutch seeds following per-host limits.
+ *
+ * Input:
+ *
+ * <url> \t nutch.score=<score>
+ *
+ *
+ * The score is calculated as:
+ *
+ *
+ * 0.001 * default_score * log10(1 + inlink_count)
+ *
+ */
+public class UrlSamplerHost extends Configured implements Tool {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(MethodHandles.lookup().lookupClass());
+
+ private static final Pattern URL_PATTERN = Pattern.compile("^https?://",
+ Pattern.CASE_INSENSITIVE);
+
+ public static class TextCountPair implements Writable {
+
+ private Text text = new Text();
+ private LongWritable count = new LongWritable();
+
+ public void set(Text text, long count) {
+ this.text.set(text);
+ this.count.set(count);
+ }
+
+ public Text getText() {
+ return text;
+ }
+
+ public LongWritable getCount() {
+ return count;
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ text.readFields(in);
+ count.readFields(in);
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ text.write(out);
+ count.write(out);
+ }
+
+ }
+
+ public static class SampleMapper
+ extends Mapper {
+
+ private boolean hostStripWWW = false;
+
+ private DomainScorePair outputKey = new DomainScorePair();
+ private TextCountPair outputValue = new TextCountPair();
+
+ /**
+ * Strip leading www. from host name.
+ *
+ * But do not strip if the host name is "e;www.tld"e; (e.g.,
+ * www.com).
+ *
+ * Stripping is required for per-host limit configurations before 2026,
+ * based on Common Crawl web graphs where the leading www. was
+ * stripped.
+ *
+ * @param host
+ * name
+ * @return host name with leading www. stripped
+ */
+ private static String hostStripWWW(String host) {
+ // min. length: 4 + 3 + 1 = 8 (www. + 1-letter-domain + .2-letter-tld)
+ if (host.length() >= 8 && host.startsWith("www.")
+ && host.indexOf('.', 4) != -1) {
+ host = host.substring(4);
+ }
+ return host;
+ }
+
+ @Override
+ public void setup(Context context) {
+ Configuration conf = context.getConfiguration();
+ hostStripWWW = conf.getBoolean("urlsample.host.strip.www", false);
+ }
+
+ @Override
+ public void map(Text key, Text value, Context context)
+ throws IOException, InterruptedException {
+
+ if (!URL_PATTERN.matcher(key.toString()).find()) {
+ // got
+ outputKey.set(key, Float.MAX_VALUE);
+ outputValue.set(value, -1);
+ context.write(outputKey, outputValue);
+ return;
+ }
+
+ // got
+ String url = key.toString();
+ String host;
+ try {
+ URL u = new URL(url);
+ host = u.getHost();
+ if (hostStripWWW) {
+ host = hostStripWWW(host);
+ }
+ } catch (Exception e) {
+ LOG.warn("Malformed URL: '{}', skipping ({})", url, e.getMessage());
+ context.getCounter(NutchMetrics.GROUP_URLSAMPLER,
+ NutchMetrics.URLSAMPLER_MALFORMED_URL_TOTAL).increment(1);
+ return;
+ }
+
+ long count = 0;
+ try {
+ count = Long.parseLong(value.toString());
+ } catch (NumberFormatException e) {
+ LOG.error("Value is not a long integer (url: {} value: {})", url,
+ value);
+ }
+ outputKey.set(host, (float) (Math.random() * count));
+ outputValue.set(key, count);
+ context.write(outputKey, outputValue);
+ }
+ }
+
+ public static class SampleReducer
+ extends Reducer {
+
+ private int maxUrlsPerHost = -1; // -1 : sample randomly
+ private float defaultScore = .001f;
+ private Text meta = new Text();
+
+ @Override
+ public void setup(Context context) {
+ Configuration conf = context.getConfiguration();
+ maxUrlsPerHost = conf.getInt("urlsample.urls.per.host", maxUrlsPerHost);
+ defaultScore = conf.getFloat("urlsample.default.score", defaultScore);
+ }
+
+ @Override
+ public void reduce(DomainScorePair key, Iterable values,
+ Context context) throws IOException, InterruptedException {
+ int maxUrls = maxUrlsPerHost;
+ float hostScore = defaultScore;
+ int nUrls = 0;
+ int nUrlsSampled = 0;
+ int skippedMaxUrlsPerHost = 0;
+ int skippedRandom = 0;
+ double sumScores = .0;
+ String host = null;
+ Text limits = null;
+
+ for (TextCountPair val : values) {
+ Text text = val.getText();
+ long count = val.getCount().get();
+ if (limits == null && count == -1) {
+ limits = new Text(text);
+ // store limits but parse later lazily (if there are URLs to sample)
+ continue;
+ } else if (limits != null) {
+ String[] l = limits.toString().split("\t");
+ if (l.length >= 3) {
+ try {
+ // long rank = l[0]
+ maxUrls = Integer.parseInt(l[1]);
+ hostScore = Float.parseFloat(l[2]);
+ } catch (NumberFormatException e) {
+ LOG.warn("Invalid host limits: {}", limits, e);
+ }
+ }
+ limits = null;
+ }
+ if (host == null) {
+ // processing first URL in values
+ host = key.getDomain().toString();
+ }
+ nUrls++;
+ if (nUrlsSampled > maxUrls) {
+ if (maxUrls > -1) {
+ skippedMaxUrlsPerHost++;
+ continue;
+ } else {
+ // no limits, sample randomly with low probability
+ double sampleProb = (.1d / (1 + nUrlsSampled))
+ * Math.log10(1 + count);
+ if (sampleProb < Math.random()) {
+ skippedRandom++;
+ continue;
+ }
+ }
+ }
+ nUrlsSampled++;
+ double score = .001d * hostScore * Math.log10(1 + count);
+ sumScores += score;
+ meta.set(String.format(Locale.ROOT, "nutch.score=%.12f", score));
+ context.write(text, meta);
+ }
+ // hosts == reduce input groups
+ context.getCounter(NutchMetrics.GROUP_URLSAMPLER_HOST,
+ NutchMetrics.URLSAMPLER_HOSTS).increment(1);
+ // URLs == map output records, reduce input records
+ context.getCounter(NutchMetrics.GROUP_URLSAMPLER_HOST,
+ NutchMetrics.URLSAMPLER_URLS).increment(nUrls);
+ if (nUrls > 0) {
+ if (maxUrls > -1) {
+ context.getCounter(NutchMetrics.GROUP_URLSAMPLER_HOST,
+ NutchMetrics.URLSAMPLER_HOSTS_WITH_LIMIT).increment(1);
+ context
+ .getCounter(NutchMetrics.GROUP_URLSAMPLER_HOST,
+ NutchMetrics.URLSAMPLER_URLS_HOST_WITH_LIMIT)
+ .increment(nUrls);
+ } else {
+ context.getCounter(NutchMetrics.GROUP_URLSAMPLER_HOST,
+ NutchMetrics.URLSAMPLER_HOSTS_WITHOUT_LIMIT).increment(1);
+ context
+ .getCounter(NutchMetrics.GROUP_URLSAMPLER_HOST,
+ NutchMetrics.URLSAMPLER_URLS_HOST_WITHOUT_LIMIT)
+ .increment(nUrls);
+ }
+ if (nUrlsSampled > 0) {
+ context.getCounter(NutchMetrics.GROUP_URLSAMPLER_HOST,
+ NutchMetrics.URLSAMPLER_URLS_SAMPLED).increment(nUrlsSampled);
+ context.getCounter(NutchMetrics.GROUP_URLSAMPLER_HOST,
+ NutchMetrics.URLSAMPLER_HOSTS_SAMPLED).increment(1);
+ if (maxUrls > -1) {
+ context
+ .getCounter(NutchMetrics.GROUP_URLSAMPLER_HOST,
+ NutchMetrics.URLSAMPLER_HOSTS_WITH_LIMIT_SAMPLED)
+ .increment(1);
+ context
+ .getCounter(NutchMetrics.GROUP_URLSAMPLER_HOST,
+ NutchMetrics.URLSAMPLER_URLS_HOST_WITH_LIMIT_SAMPLED)
+ .increment(nUrlsSampled);
+ } else {
+ context
+ .getCounter(NutchMetrics.GROUP_URLSAMPLER_HOST,
+ NutchMetrics.URLSAMPLER_HOSTS_WITHOUT_LIMIT_SAMPLED)
+ .increment(1);
+ context
+ .getCounter(NutchMetrics.GROUP_URLSAMPLER_HOST,
+ NutchMetrics.URLSAMPLER_URLS_HOST_WITHOUT_LIMIT_SAMPLED)
+ .increment(nUrlsSampled);
+ }
+ }
+ context
+ .getCounter(NutchMetrics.GROUP_URLSAMPLER_HOST,
+ NutchMetrics.URLSAMPLER_SKIPPED_MAX_URLS_PER_HOST)
+ .increment(skippedMaxUrlsPerHost);
+ context
+ .getCounter(NutchMetrics.GROUP_URLSAMPLER_HOST,
+ NutchMetrics.URLSAMPLER_SKIPPED_RANDOM)
+ .increment(skippedRandom);
+ LOG.info(
+ "Sampled for host {} : {} URLs ({} skipped: {} max. per host, {} random), sum of scores = {}",
+ host, nUrlsSampled, (nUrls - nUrlsSampled), skippedMaxUrlsPerHost,
+ skippedRandom, sumScores);
+ }
+ }
+ }
+
+ private void sample(Path[] inputs, Path output) throws Exception {
+
+ StopWatch stopWatch = new StopWatch();
+ stopWatch.start();
+ LOG.info("UrlSamplerHost: starting");
+
+ Configuration conf = getConf();
+ conf.setInt("partition.url.seed", new Random().nextInt());
+
+ Job job = Job.getInstance(conf, UrlSamplerHost.class.getName());
+ job.setJarByClass(UrlSamplerHost.class);
+ job.setMapperClass(SampleMapper.class);
+ job.setPartitionerClass(Generator2.Selector.class);
+ job.setSortComparatorClass(Generator2.ScoreComparator.class);
+ job.setGroupingComparatorClass(Generator2.DomainComparator.class);
+ job.setReducerClass(SampleReducer.class);
+ job.setInputFormatClass(KeyValueTextInputFormat.class);
+ job.setOutputFormatClass(TextOutputFormat.class);
+ job.setMapOutputKeyClass(Generator2.DomainScorePair.class);
+ job.setMapOutputValueClass(TextCountPair.class);
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(Text.class);
+
+ for (Path input : inputs) {
+ KeyValueTextInputFormat.addInputPath(job, input);
+ }
+ FileOutputFormat.setOutputPath(job, output);
+
+ try {
+ // run the job
+ boolean success = job.waitForCompletion(true);
+ if (!success) {
+ String message = "UrlSampler job did not succeed, job status: "
+ + job.getStatus().getState() + ", reason: "
+ + job.getStatus().getFailureInfo();
+ LOG.error(message);
+ // throw exception so that calling routine can exit with error
+ throw new RuntimeException(message);
+ }
+
+ } catch (IOException | InterruptedException | ClassNotFoundException
+ | NullPointerException e) {
+ LOG.error("UrlSampler job failed: {}", e.getMessage());
+ throw e;
+ }
+
+ stopWatch.stop();
+ LOG.info("UrlSamplerHost: finished, elapsed: {} ms",
+ stopWatch.getTime(TimeUnit.MILLISECONDS));
+ }
+
+ public void usage() {
+ System.err.println(
+ "Usage: UrlSamplerHost [-D...] ... \n");
+ System.err.println(
+ "\nThe host_limits file defines the maximum number of URLs to sample per host.");
+ System.err.println("\nProperties:");
+ System.err.println(
+ "\t-Durlsample.host.strip.www=(true|false)\tstrip leading www. from host names");
+ System.err.println(
+ "\t\t\t(depending on whether the limits file uses stripped host names)");
+ System.err.println(
+ "Properties to configure defaults, if host is not in the limits file:");
+ System.err.println(
+ "\t-Durlsample.urls.per.host\tmax. number of URLs to sample per host");
+ System.err
+ .println("\t\t\t-1 : sample randomly with low probability (default)");
+ System.err.println(
+ "\t-Durlsample.default.score\tdefault score for sampled URLs (default: 0.001)");
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ if (args.length < 2) {
+ usage();
+ return -1;
+ }
+
+ Path[] inputs = new Path[args.length - 1];
+ for (int i = 0; i < (args.length - 1); i++) {
+ inputs[i] = new Path(args[i]);
+ }
+ Path output = new Path(args[args.length - 1]);
+
+ try {
+ sample(inputs, output);
+ } catch (Exception e) {
+ LOG.error("UrlSamplerHost: ", e);
+ return -1;
+ }
+ return 0;
+ }
+
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(NutchConfiguration.create(), new UrlSamplerHost(),
+ args);
+ System.exit(res);
+ }
+
+}
diff --git a/src/java/org/commoncrawl/tools/WarcExport.java b/src/java/org/commoncrawl/tools/WarcExport.java
new file mode 100644
index 0000000000..ab4a9aeafc
--- /dev/null
+++ b/src/java/org/commoncrawl/tools/WarcExport.java
@@ -0,0 +1,252 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.commoncrawl.tools;
+
+import org.apache.commons.lang3.time.StopWatch;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.lib.input.MultipleInputs;
+import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.nutch.crawl.CrawlDatum;
+import org.apache.nutch.crawl.NutchWritable;
+import org.apache.nutch.parse.ParseData;
+import org.apache.nutch.protocol.Content;
+import org.apache.nutch.util.HadoopFSUtil;
+import org.apache.nutch.util.NutchConfiguration;
+import org.commoncrawl.util.WarcCapture;
+import org.commoncrawl.util.WarcOutputFormat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+public class WarcExport extends Configured implements Tool {
+ public static Logger LOG = LoggerFactory.getLogger(WarcExport.class);
+
+ static {
+ Configuration.addDefaultResource("nutch-default.xml");
+ Configuration.addDefaultResource("nutch-site.xml");
+ }
+
+ public static class ExportMapper
+ extends Mapper {
+ @Override
+ public void map(Text key, Writable value, Context context)
+ throws IOException, InterruptedException {
+ if (key.getLength() == 0) {
+ return;
+ }
+ context.write(key, new NutchWritable(value));
+ }
+ }
+
+ public static class ExportReducer
+ extends Reducer {
+
+ private boolean generateCrawlDiagnostics = false;
+ private boolean generateRobotsTxt = false;
+
+ @Override
+ public void setup(Context context) {
+ Configuration conf = context.getConfiguration();
+ generateCrawlDiagnostics = conf.getBoolean("warc.export.crawldiagnostics",
+ false);
+ generateRobotsTxt = conf.getBoolean("warc.export.robotstxt", false);
+ }
+
+ @Override
+ public void reduce(Text key, Iterable values,
+ Context context) throws IOException, InterruptedException {
+ CrawlDatum datum = null;
+ Content content = null;
+
+ for (NutchWritable nutchValue : values) {
+ final Writable value = nutchValue.get(); // unwrap
+ if (value instanceof CrawlDatum) {
+ datum = (CrawlDatum) value;
+ } else if (value instanceof ParseData) {
+ ParseData parseData = (ParseData) value;
+ // Get the robots meta data
+ String robotsMeta = parseData.getMeta("robots");
+
+ // Has it a noindex for this url?
+ if (robotsMeta != null
+ && robotsMeta.toLowerCase().contains("noindex")) {
+ return;
+ }
+ } else if (value instanceof Content) {
+ content = (Content) value;
+ }
+ }
+
+ if (content == null) {
+ return;
+ }
+
+ if (datum == null) {
+ if (!generateRobotsTxt)
+ return;
+ } else if (datum.getStatus() != CrawlDatum.STATUS_FETCH_SUCCESS) {
+ if (!generateCrawlDiagnostics)
+ return;
+ }
+
+ WarcCapture completeData = new WarcCapture(key, datum, content);
+
+ context.write(key, completeData);
+ }
+ }
+
+ public void export(Path outputDir, List segments,
+ boolean generateCrawlDiagnostics, boolean generateRobotsTxt, Path cdxPath)
+ throws IOException, InterruptedException, ClassNotFoundException {
+ Configuration conf = getConf();
+
+ // We compress ourselves, so this isn't necessary
+ conf.setBoolean(
+ org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.COMPRESS,
+ false);
+
+ conf.setBoolean("warc.export.crawldiagnostics", generateCrawlDiagnostics);
+ conf.setBoolean("warc.export.robotstxt", generateRobotsTxt);
+ if (cdxPath != null) {
+ conf.setBoolean("warc.export.cdx", true);
+ conf.set("warc.export.cdx.path", cdxPath.toString());
+ }
+
+ Job job = Job.getInstance(conf);
+ job.setJobName("WarcExport: " + outputDir.toString());
+ job.setJarByClass(WarcExport.class);
+
+ FileOutputFormat.setOutputPath(job, new Path("out"));
+
+ for (final Path segment : segments) {
+ LOG.info("ExporterMapReduces: adding segment: {}", segment);
+ FileSystem fs = segment.getFileSystem(getConf());
+
+ MultipleInputs.addInputPath(job,
+ new Path(segment, CrawlDatum.FETCH_DIR_NAME),
+ SequenceFileInputFormat.class);
+
+ Path parseDataPath = new Path(segment, ParseData.DIR_NAME);
+ if (fs.exists(parseDataPath)) {
+ MultipleInputs.addInputPath(job, parseDataPath,
+ SequenceFileInputFormat.class);
+ }
+
+ MultipleInputs.addInputPath(job, new Path(segment, Content.DIR_NAME),
+ SequenceFileInputFormat.class);
+ }
+
+ job.setMapperClass(ExportMapper.class);
+ job.setReducerClass(ExportReducer.class);
+
+ job.setMapOutputValueClass(NutchWritable.class);
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(WarcCapture.class);
+
+ job.setOutputFormatClass(WarcOutputFormat.class);
+ WarcOutputFormat.setOutputPath(job, outputDir);
+
+ StopWatch stopWatch = new StopWatch();
+ stopWatch.start();
+ LOG.info("WarcExport: starting");
+
+ try {
+ boolean success = job.waitForCompletion(true);
+ if (!success) {
+ String message = "WarcExport: job did not succeed, job status: "
+ + job.getStatus().getState() + ", reason: "
+ + job.getStatus().getFailureInfo();
+ LOG.error(message);
+ throw new RuntimeException(message);
+ }
+ } catch (IOException | InterruptedException | ClassNotFoundException e) {
+ LOG.error("WarcExport job failed: {}", e.getMessage());
+ throw e;
+ }
+ stopWatch.stop();
+ LOG.info("WarcExport: finished, elapsed: {} ms",
+ stopWatch.getTime(TimeUnit.MILLISECONDS));
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ if (args.length < 2) {
+ System.err.println(
+ "Usage: WarcExport ( ... | -dir ) [-crawldiagnostics] [-robotstxt] [-cdx path]");
+ return -1;
+ }
+
+ final Path outputDir = new Path(args[0]);
+
+ final List segments = new ArrayList();
+ boolean generateCrawlDiagnostics = false;
+ boolean generateRobotsTxt = false;
+ Path cdxPath = null;
+
+ for (int i = 1; i < args.length; i++) {
+ if (args[i].equals("-dir")) {
+ Path dir = new Path(args[++i]);
+ FileSystem fs = dir.getFileSystem(getConf());
+ FileStatus[] fstats = fs.listStatus(dir,
+ HadoopFSUtil.getPassDirectoriesFilter(fs));
+ Path[] files = HadoopFSUtil.getPaths(fstats);
+ for (Path p : files) {
+ segments.add(p);
+ }
+ } else if (args[i].equals("-crawldiagnostics")) {
+ generateCrawlDiagnostics = true;
+ } else if (args[i].equals("-robotstxt")) {
+ generateRobotsTxt = true;
+ } else if (args[i].equals("-cdx")) {
+ cdxPath = new Path(args[++i]);
+ } else {
+ segments.add(new Path(args[i]));
+ }
+ }
+
+ try {
+ export(outputDir, segments, generateCrawlDiagnostics, generateRobotsTxt,
+ cdxPath);
+ return 0;
+ } catch (final Exception e) {
+ LOG.error("WARC Exporter:", e);
+ return -1;
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+ final int res = ToolRunner.run(NutchConfiguration.create(),
+ new WarcExport(), args);
+ System.exit(res);
+ }
+}
diff --git a/src/java/org/commoncrawl/util/ByteArrayCharSequence.java b/src/java/org/commoncrawl/util/ByteArrayCharSequence.java
new file mode 100644
index 0000000000..9946933a30
--- /dev/null
+++ b/src/java/org/commoncrawl/util/ByteArrayCharSequence.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.commoncrawl.util;
+
+import java.nio.charset.StandardCharsets;
+
+/**
+ * Wrap a byte array as a {@link CharSequence} in
+ * {@link StandardCharsets#ISO_8859_1} encoding.
+ *
+ * For regular expression matching on ASCII characters only, the wrapper should
+ * be faster than creating a {@link String} from the byte array or a
+ * subsequence, because no bytes are converted to chars and no memory is
+ * allocated for a new String.
+ *
+ * Similar wrappers are part of
+ * extJWNL,
+ * BUbiNG, and other Java
+ * libraries.
+ */
+public class ByteArrayCharSequence implements CharSequence {
+
+ private final byte[] data;
+ private final int length;
+ private final int offset;
+
+ public ByteArrayCharSequence() {
+ this(new byte[0], 0, 0);
+ }
+
+ public ByteArrayCharSequence(final byte[] data) {
+ this(data, 0, data.length);
+ }
+
+ public ByteArrayCharSequence(final byte[] data, int length) {
+ this(data, 0, length);
+ }
+
+ public ByteArrayCharSequence(final byte[] data, int offset, int length) {
+ this.data = data;
+ if (offset < 0) {
+ throw new ArrayIndexOutOfBoundsException("Negative offset: " + offset);
+ }
+ if (length < 0) {
+ throw new IllegalArgumentException("Negative length:" + length);
+ }
+ if ((offset + length) > data.length) {
+ throw new ArrayIndexOutOfBoundsException(
+ "(Offset + length) > array_length");
+ }
+ this.length = length;
+ this.offset = offset;
+ }
+
+ @Override
+ public int length() {
+ return this.length;
+ }
+
+ @Override
+ public char charAt(int index) {
+ if (index >= length) {
+ throw new IndexOutOfBoundsException("" + index);
+ }
+ return (char) (data[offset + index] & 0xff);
+ }
+
+ @Override
+ public CharSequence subSequence(int start, int end) {
+ return new ByteArrayCharSequence(data, offset + start, end - start);
+ }
+
+ @Override
+ public String toString() {
+ return new String(data, offset, length, StandardCharsets.ISO_8859_1);
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/commoncrawl/util/CanonicalLinkDetector.java b/src/java/org/commoncrawl/util/CanonicalLinkDetector.java
new file mode 100644
index 0000000000..8e40777013
--- /dev/null
+++ b/src/java/org/commoncrawl/util/CanonicalLinkDetector.java
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.commoncrawl.util;
+
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.http.HeaderElement;
+import org.apache.http.NameValuePair;
+import org.apache.http.ParseException;
+import org.apache.http.message.BasicHeaderValueParser;
+import org.apache.nutch.protocol.Content;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class CanonicalLinkDetector {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(MethodHandles.lookup().lookupClass());
+
+ protected static Set SUPPORTED_CONTENT_TYPES = new HashSet<>();
+ static {
+ SUPPORTED_CONTENT_TYPES.add("text/html");
+ SUPPORTED_CONTENT_TYPES.add("application/xhtml+xml");
+ }
+
+ /**
+ * Pattern to match canonical link elements in HTML. The length of the
+ * canonical link URL inside the element is limited to max. 2048 characters.
+ */
+ private static Pattern canonicalLinkPattern = Pattern.compile(
+ "]{0,2054}rel=(?:'canonical'|\"canonical\"|canonical\\b)[^>]{0,2054}>",
+ Pattern.CASE_INSENSITIVE | Pattern.MULTILINE);
+ private static Pattern hrefPattern = Pattern
+ .compile("href=['\"]?([^'\"\\s]{0,2048})", Pattern.CASE_INSENSITIVE);
+
+ private static Pattern canonicalRelValuePattern = Pattern
+ .compile("\\bcanonical\\b", Pattern.CASE_INSENSITIVE);
+ private static final Pattern linkInParentheses = Pattern
+ .compile("^\\s*<\\s*(.*?)\\s*>\\s*$");
+
+ private static final List EMPTY_RESULT = List.of();
+
+ /** top-N bytes of HTML to look for canonical link */
+ private static int CHUNK_SIZE = 65536;
+
+ /** max. number canonical links to detect */
+ private static int MAX_LINKS = 1;
+
+ /**
+ * Extract canonical link from HTTP header.
+ *
+ * The extraction is delegated to {@link BasicHeaderValueParser} because
+ * parsing multi-valued link attributes is far from trivial, e.g.
+ *
+ *
+ Link: ; rel="canonical",; rel="shortlink",; rel="shortcut icon"
+ *
+ *
+ * @param "Link"
+ * header values
+ * @return the canonical links found, or an empty list if no canonical link is
+ * found
+ */
+ protected static List detectCanonicalLinksHttpHeader(
+ String[] linkHeaders, int maxResults) {
+ List result = EMPTY_RESULT;
+ for (String httpHeaderLink : linkHeaders) {
+ HeaderElement elem;
+ try {
+ elem = BasicHeaderValueParser.parseHeaderElement(httpHeaderLink,
+ BasicHeaderValueParser.INSTANCE);
+ } catch (ParseException e) {
+ LOG.error("Failed to parse Link HTTP header: {}", httpHeaderLink, e);
+ continue;
+ }
+ for (NameValuePair param : elem.getParameters()) {
+ if ("rel".equalsIgnoreCase(param.getName())
+ && canonicalRelValuePattern.matcher(param.getValue()).find()) {
+ String link = elem.getName();
+ // match inside < ... >
+ Matcher urlMatcher = linkInParentheses.matcher(link);
+ if (urlMatcher.matches()) {
+ link = urlMatcher.group(1);
+ if (result == EMPTY_RESULT) {
+ result = new ArrayList(1);
+ }
+ result.add(link);
+ if (result.size() >= maxResults) {
+ break;
+ }
+ }
+ }
+ }
+ }
+ return result;
+ }
+
+ public static boolean isEligibleContentType(String contentType) {
+ return SUPPORTED_CONTENT_TYPES.contains(contentType);
+ }
+
+ /**
+ * Extract canonical link from HTTP header.
+ *
+ * The extraction is delegated to {@link BasicHeaderValueParser} because
+ * parsing multi-valued link attributes is far from trivial, e.g.
+ *
+ *
+ Link: ; rel="canonical",; rel="shortlink",; rel="shortcut icon"
+ *
+ *
+ * @param "Link"
+ * header values
+ * @return the canonical links found, or an empty list if no canonical link is
+ * found
+ */
+ public static List detectCanonicalLinksHTML(byte[] content, int chunkSize,
+ int maxResults) {
+ List result = EMPTY_RESULT;
+ int length = content.length < chunkSize ? content.length : chunkSize;
+ CharSequence cs;
+ cs = new ByteArrayCharSequence(content, length);
+ Matcher clMatcher = canonicalLinkPattern.matcher(cs);
+ while (clMatcher.find()) {
+ CharSequence cls;
+ cls = cs.subSequence(clMatcher.start(), clMatcher.end());
+ Matcher hrefMatcher = hrefPattern.matcher(cls);
+ if (hrefMatcher.find(5)) {
+ String cl = hrefMatcher.group(1);
+ if (result == EMPTY_RESULT) {
+ result = new ArrayList(1);
+ }
+ result.add(cl);
+ if (result.size() >= maxResults) {
+ break;
+ }
+ }
+ }
+ return result;
+ }
+
+ public static List detectCanonicalLinks(Content content,
+ int chunkSize, int maxLinks) {
+
+ /*
+ * Note: the HTTP header look-up is case-insensitive if
+ * CaseInsensitiveMetadata or SpellCheckedMetadata is used.
+ */
+ String[] linkHeaders = content.getMetadata().getValues("Link");
+ List canonicalLinks = detectCanonicalLinksHttpHeader(linkHeaders,
+ maxLinks);
+
+ if (canonicalLinks.size() < maxLinks
+ && isEligibleContentType(content.getContentType())) {
+ List linksHtml = detectCanonicalLinksHTML(content.getContent(), chunkSize, maxLinks);
+ if (linksHtml.size() > 0) {
+ if (canonicalLinks == EMPTY_RESULT) {
+ canonicalLinks = linksHtml;
+ } else {
+ canonicalLinks.addAll(linksHtml);
+ }
+ }
+ }
+ return canonicalLinks;
+ }
+
+ public static List detectCanonicalLinks(Content content) {
+ return detectCanonicalLinks(content, CHUNK_SIZE, MAX_LINKS);
+ }
+
+}
diff --git a/src/java/org/commoncrawl/util/LanguageDetector.java b/src/java/org/commoncrawl/util/LanguageDetector.java
new file mode 100644
index 0000000000..092e534aaa
--- /dev/null
+++ b/src/java/org/commoncrawl/util/LanguageDetector.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.commoncrawl.util;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.net.URI;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.nutch.net.protocols.Response;
+import org.apache.nutch.protocol.Content;
+import org.apache.tika.detect.AutoDetectReader;
+import org.apache.tika.detect.CompositeEncodingDetector;
+import org.apache.tika.detect.EncodingDetector;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.html.HtmlEncodingDetector;
+import org.apache.tika.parser.txt.Icu4jEncodingDetector;
+import org.commoncrawl.langdetect.cld2.CLDHints;
+import org.commoncrawl.langdetect.cld2.Cld2;
+import org.commoncrawl.langdetect.cld2.Flags;
+import org.commoncrawl.util.LanguageDetector.Result.Status;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Utf8;
+
+public class LanguageDetector {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(MethodHandles.lookup().lookupClass());
+
+ protected static Set SUPPORTED_CONTENT_TYPES = new HashSet<>();
+ static {
+ SUPPORTED_CONTENT_TYPES.add("text/html");
+ SUPPORTED_CONTENT_TYPES.add("application/xhtml+xml");
+ }
+
+ protected Flags flags = new Flags();
+
+ protected EncodingDetector charsetDetector = new CompositeEncodingDetector(
+ Arrays.asList(new HtmlEncodingDetector(), new Icu4jEncodingDetector()));
+
+ public void setBestEffort(boolean bestEffort) {
+ flags.setBestEffort(bestEffort);
+ }
+
+ public static class Result {
+ Charset charset;
+ org.commoncrawl.langdetect.cld2.Result languages;
+ String errorReason;
+ enum Status {
+ EMPTY_CONTENT("empty content"),
+ CHARSET_DETECTION_FAILED("failed to detect charset"),
+ UNSUPPORTED_MIME_TYPE("MIME type not supported");
+ String name;
+ Status(String name) {
+ this.name = name;
+ }
+ };
+ Status errorStatus;
+ }
+
+ protected Result detectLanguage(URI uri, Content content) {
+
+ LanguageDetector.Result result = new Result();
+
+ if (content.getContent().length == 0) {
+ // empty content, nothing to detect
+ LOG.debug("Skipping empty document for language and charset detection");
+ result.errorStatus = Status.EMPTY_CONTENT;
+ return result;
+ }
+
+ String detectedContentType = content.getContentType();
+ boolean isPlainText = false;
+ if (!SUPPORTED_CONTENT_TYPES.contains(detectedContentType)) {
+ // TODO: as an improvement, parse documents of non-HTML content types and
+ // do the language detection on extracted text, for now skip them
+ LOG.debug("Skipping document of Content-Type {} for language detection",
+ detectedContentType);
+ result.errorReason = "Content-Type " + detectedContentType + " not supported";
+ result.errorStatus = Status.UNSUPPORTED_MIME_TYPE;
+ return result;
+ }
+
+ String httpContentLanguage = content.getMetadata()
+ .get(Response.CONTENT_LANGUAGE);
+ String httpContentType = WarcWriter.getMeta(content.getMetadata(),
+ Response.CONTENT_TYPE);
+
+ Metadata metadata = new Metadata();
+ if (httpContentType != null) {
+ LOG.debug(" Content-Type: {}", httpContentType);
+ metadata.add(Metadata.CONTENT_TYPE, httpContentType);
+ }
+ String text;
+ byte[] bytes = content.getContent();
+
+ try (AutoDetectReader charsetDetectReader = new AutoDetectReader(
+ new ByteArrayInputStream(bytes), metadata, charsetDetector)) {
+ result.charset = charsetDetectReader.getCharset();
+ boolean isValidUtf8 = false;
+ if (result.charset.equals(StandardCharsets.UTF_8)) {
+ // need to validate UTF-8 because CLD2 may segfault on invalid UTF-8
+ LOG.debug("Validating UTF-8 for: {}", uri);
+ if (Utf8.isWellFormed(bytes)) {
+ isValidUtf8 = true;
+ }
+ }
+ if (isValidUtf8) {
+ // CLD2 requires that the input byte[] includes a trailing zero byte
+ bytes = Cld2.bytesToNative(bytes);
+ } else {
+ LOG.debug("Recoding from {}: {}", result.charset, uri);
+ text = new String(bytes, result.charset);
+ bytes = Cld2.encodeNative(text);
+ }
+ } catch (IOException | TikaException e) {
+ LOG.error("Failed to convert charset:", e);
+ result.errorReason = "Failed to convert charset " + e.getMessage();
+ result.errorStatus = Status.CHARSET_DETECTION_FAILED;
+ return result;
+ }
+ CLDHints hints = new CLDHints();
+ hints.setEncodingHint(result.charset);
+ hints.setTopLevelDomainHint(uri);
+ if (httpContentLanguage != null) {
+ hints.setContentLanguageHint(httpContentLanguage);
+ }
+ result.languages = Cld2.detect(bytes, hints, flags, isPlainText);
+ result.languages.configurePruning(10, 2, 0.0);
+
+ return result;
+ }
+
+}
diff --git a/src/java/org/commoncrawl/util/NullOutputCommitter.java b/src/java/org/commoncrawl/util/NullOutputCommitter.java
new file mode 100644
index 0000000000..8975e5f655
--- /dev/null
+++ b/src/java/org/commoncrawl/util/NullOutputCommitter.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.commoncrawl.util;
+
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.OutputCommitter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+import java.io.IOException;
+
+public class NullOutputCommitter extends OutputCommitter {
+
+ @Override
+ public void setupJob(JobContext jobContext) throws IOException {
+ }
+
+ @Override
+ public void setupTask(TaskAttemptContext taskContext) throws IOException {
+ }
+
+ @Override
+ public boolean needsTaskCommit(TaskAttemptContext taskContext)
+ throws IOException {
+ return false;
+ }
+
+ @Override
+ public void commitTask(TaskAttemptContext taskContext) throws IOException {
+ }
+
+ @Override
+ public void abortTask(TaskAttemptContext taskContext) throws IOException {
+ }
+
+}
\ No newline at end of file
diff --git a/src/java/org/commoncrawl/util/S3FileOutputFormat.java b/src/java/org/commoncrawl/util/S3FileOutputFormat.java
new file mode 100644
index 0000000000..cd2c625d8a
--- /dev/null
+++ b/src/java/org/commoncrawl/util/S3FileOutputFormat.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.commoncrawl.util;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.OutputCommitter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+
+/**
+ * {@link FileOutputFormat} using {@link NullOutputCommitter} when writing to
+ * AWS S3.
+ *
+ * Because default FileOutputCommitter is slow on S3, the
+ * {@link NullOutputCommitter} is used until a better solution is available, cf.
+ * Hadoop
+ * AWS committers and Hadoop
+ * AWS committer architecture.
+ */
+public abstract class S3FileOutputFormat extends FileOutputFormat {
+
+ private OutputCommitter committer;
+
+ @Override
+ public synchronized OutputCommitter getOutputCommitter(
+ TaskAttemptContext context) throws java.io.IOException {
+ if (committer == null) {
+ Path output = getOutputPath(context);
+
+ String scheme = output.getFileSystem(context.getConfiguration())
+ .getScheme();
+ if (scheme.startsWith("s3")) {
+ committer = new NullOutputCommitter();
+ } else {
+ committer = super.getOutputCommitter(context);
+ }
+ }
+ return committer;
+ }
+
+}
\ No newline at end of file
diff --git a/src/java/org/commoncrawl/util/WarcCapture.java b/src/java/org/commoncrawl/util/WarcCapture.java
new file mode 100644
index 0000000000..f2ad2df497
--- /dev/null
+++ b/src/java/org/commoncrawl/util/WarcCapture.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.commoncrawl.util;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.nutch.crawl.CrawlDatum;
+import org.apache.nutch.protocol.Content;
+
+/**
+ * Container to hold all Nutch objects related to a single page capture and
+ * necessary to write the WARC request and response records.
+ */
+public class WarcCapture implements Writable {
+ public Text url;
+ public CrawlDatum datum;
+ public Content content;
+
+ public WarcCapture() {
+ url = new Text();
+ datum = new CrawlDatum();
+ content = new Content();
+ }
+
+ public WarcCapture(Text url, CrawlDatum datum, Content content) {
+ this.url = url;
+ this.datum = datum;
+ this.content = content;
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ url.readFields(in);
+ if (in.readBoolean()) {
+ datum.readFields(in);
+ } else {
+ datum = null;
+ }
+ if (in.readBoolean()) {
+ content.readFields(in);
+ } else {
+ content = null;
+ }
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ url.write(out);
+ if (datum != null) {
+ out.writeBoolean(true);
+ datum.write(out);
+ } else {
+ out.writeBoolean(false);
+ }
+ if (content != null) {
+ out.writeBoolean(true);
+ content.write(out);
+ } else {
+ out.writeBoolean(false);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "url=" + url.toString() + ", datum=" + datum.toString();
+ }
+}
\ No newline at end of file
diff --git a/src/java/org/commoncrawl/util/WarcCdxWriter.java b/src/java/org/commoncrawl/util/WarcCdxWriter.java
new file mode 100644
index 0000000000..9c9c756da0
--- /dev/null
+++ b/src/java/org/commoncrawl/util/WarcCdxWriter.java
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.commoncrawl.util;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.LinkedHashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.TimeZone;
+
+import org.apache.commons.io.output.CountingOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.nutch.metadata.Metadata;
+import org.apache.nutch.net.URLNormalizers;
+import org.apache.nutch.net.protocols.Response;
+import org.apache.nutch.protocol.Content;
+import org.archive.url.WaybackURLKeyMaker;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.core.JsonGenerationException;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.util.MinimalPrettyPrinter;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectWriter;
+
+public class WarcCdxWriter extends WarcWriter {
+
+ public static Logger LOG = LoggerFactory.getLogger(WarcCdxWriter.class);
+
+ private static final Charset UTF_8 = StandardCharsets.UTF_8;
+
+ protected CountingOutputStream countingOut;
+ protected OutputStream cdxOut;
+ protected String warcFilename;
+
+ private SimpleDateFormat timestampFormat;
+ private ObjectWriter jsonWriter;
+ private WaybackURLKeyMaker surtKeyMaker = new WaybackURLKeyMaker(true);
+ private URLNormalizers urlNormalizersRedirect;
+
+ /**
+ * JSON indentation same as by Python WayBack
+ * (https://github.com/ikreymer/pywb)
+ */
+ @SuppressWarnings("serial")
+ public static class JsonIndenter extends MinimalPrettyPrinter {
+
+ // @Override
+ @Override
+ public void writeObjectFieldValueSeparator(JsonGenerator jg)
+ throws IOException, JsonGenerationException {
+ jg.writeRaw(": ");
+ }
+
+ // @Override
+ @Override
+ public void writeObjectEntrySeparator(JsonGenerator jg)
+ throws IOException, JsonGenerationException {
+ jg.writeRaw(", ");
+ }
+ }
+
+ public WarcCdxWriter(OutputStream warcOut, OutputStream cdxOut,
+ Path warcFilePath, URLNormalizers redirectNormalizers) {
+ super(new CountingOutputStream(warcOut));
+ countingOut = (CountingOutputStream) this.out;
+ this.cdxOut = cdxOut;
+ timestampFormat = new SimpleDateFormat("yyyyMMddHHmmss", Locale.ROOT);
+ timestampFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ warcFilename = warcFilePath.toUri().getPath().replaceFirst("^/", "");
+ ObjectMapper jsonMapper = new ObjectMapper();
+ jsonMapper.getFactory().configure(JsonGenerator.Feature.ESCAPE_NON_ASCII,
+ true);
+ jsonWriter = jsonMapper.writer(new JsonIndenter());
+ urlNormalizersRedirect = redirectNormalizers;
+ }
+
+ @Override
+ public URI writeWarcRevisitRecord(final URI targetUri, final String ip,
+ final int httpStatusCode, final Date date, final URI warcinfoId,
+ final URI relatedId, final String warcProfile, final Date refersToDate,
+ final String payloadDigest, final String blockDigest,
+ String[] protocolVersions, String[] cipherSuites, byte[] block,
+ Content content) throws IOException {
+ long offset = countingOut.getByteCount();
+ URI recordId = super.writeWarcRevisitRecord(targetUri, ip, httpStatusCode,
+ date, warcinfoId, relatedId, warcProfile, refersToDate, payloadDigest,
+ blockDigest, protocolVersions, cipherSuites, block, content);
+ long length = (countingOut.getByteCount() - offset);
+ writeCdxLine(targetUri, date, offset, length, payloadDigest, content, true,
+ null, null);
+ return recordId;
+ }
+
+ @Override
+ public URI writeWarcResponseRecord(final URI targetUri, final String ip,
+ final int httpStatusCode, final Date date, final URI warcinfoId,
+ final URI relatedId, final String payloadDigest, final String blockDigest,
+ final String truncated, String[] protocolVersions, String[] cipherSuites,
+ final byte[] block, Content content) throws IOException {
+ long offset = countingOut.getByteCount();
+ URI recordId = super.writeWarcResponseRecord(targetUri, ip, httpStatusCode,
+ date, warcinfoId, relatedId, payloadDigest, blockDigest, truncated,
+ protocolVersions, cipherSuites, block, content);
+ long length = (countingOut.getByteCount() - offset);
+ String redirectLocation = null;
+ if (isRedirect(httpStatusCode)) {
+ redirectLocation = getMeta(content.getMetadata(), "Location");
+ if (redirectLocation != null) {
+ try {
+ // convert redirects from relative to absolute URLs
+ redirectLocation = new URL(targetUri.toURL(), redirectLocation).toString();
+ if (urlNormalizersRedirect != null) {
+ // normalize the redirect target URL
+ redirectLocation = urlNormalizersRedirect.normalize(redirectLocation,
+ URLNormalizers.SCOPE_FETCHER);
+ }
+ } catch (MalformedURLException e) {
+ redirectLocation = null;
+ }
+ }
+ }
+ writeCdxLine(targetUri, date, offset, length, payloadDigest, content, false,
+ redirectLocation, truncated);
+ return recordId;
+ }
+
+ public void writeCdxLine(final URI targetUri, final Date date, long offset,
+ long length, String payloadDigest, Content content, boolean revisit,
+ String redirectLocation, String truncated) throws IOException {
+ String url = targetUri.toASCIIString();
+ String surt = url;
+ Metadata meta = content.getMetadata();
+ try {
+ surt = surtKeyMaker.makeKey(url);
+ } catch (URISyntaxException e) {
+ LOG.error("Failed to make SURT for {}: {}", url,
+ StringUtils.stringifyException(e));
+ return;
+ }
+ if (payloadDigest == null) {
+ // no content, e.g., revisit record
+ } else if (payloadDigest.startsWith("sha1:")) {
+ payloadDigest = payloadDigest.substring(5);
+ }
+ cdxOut.write(surt.getBytes(UTF_8));
+ cdxOut.write(' ');
+ cdxOut.write(timestampFormat.format(date).getBytes(UTF_8));
+ cdxOut.write(' ');
+ Map data = new LinkedHashMap();
+ data.put("url", url);
+ if (revisit) {
+ data.put("mime", "warc/revisit");
+ } else {
+ data.put("mime", cleanMimeType(getMeta(meta, Response.CONTENT_TYPE)));
+ data.put("mime-detected", content.getContentType());
+ }
+ data.put("status", meta.get(WarcWriter.HTTP_STATUS_CODE));
+ if (payloadDigest != null) {
+ data.put("digest", payloadDigest);
+ }
+ data.put("length", String.format("%d", length));
+ data.put("offset", String.format("%d", offset));
+ data.put("filename", warcFilename);
+ String val = meta.get(WarcWriter.DETECTED_CHARSET);
+ if (val != null) {
+ data.put("charset", val);
+ }
+ val = meta.get(WarcWriter.DETECTED_LANGUAGE);
+ if (val != null) {
+ data.put("languages", val);
+ }
+ if (truncated != null) {
+ data.put("truncated", truncated);
+ }
+ if (redirectLocation != null) {
+ data.put("redirect", redirectLocation);
+ }
+ cdxOut.write(jsonWriter.writeValueAsBytes(data));
+ cdxOut.write('\n');
+ }
+
+ protected static String cleanMimeType(String mime) {
+ if (mime == null)
+ return "unk";
+ final char[] delimiters = { ';', ' ' };
+ for (char delim : delimiters) {
+ int pos = mime.indexOf(delim);
+ if (pos > -1)
+ mime = mime.substring(0, pos);
+ }
+ if (mime.isEmpty())
+ return "unk";
+ return mime;
+ }
+
+ protected static boolean isRedirect(int httpStatusCode) {
+ return httpStatusCode >= 300 && httpStatusCode < 400
+ && httpStatusCode != 304;
+ }
+}
diff --git a/src/java/org/commoncrawl/util/WarcOutputFormat.java b/src/java/org/commoncrawl/util/WarcOutputFormat.java
new file mode 100644
index 0000000000..0a993f09a5
--- /dev/null
+++ b/src/java/org/commoncrawl/util/WarcOutputFormat.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.commoncrawl.util;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.InvalidJobConfException;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.OutputCommitter;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskID;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.security.TokenCache;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class WarcOutputFormat extends FileOutputFormat {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(MethodHandles.lookup().lookupClass());
+
+ private OutputCommitter committer;
+
+ @Override
+ public RecordWriter getRecordWriter(
+ TaskAttemptContext context) throws IOException {
+
+ TaskID taskid = context.getTaskAttemptID().getTaskID();
+ int partition = taskid.getId();
+ LOG.info("Partition: {}", partition);
+
+ Configuration conf = context.getConfiguration();
+ Path outputPath = getOutputPath(context);
+
+ String warcOutputPath = conf.get("warc.export.path");
+ if (warcOutputPath != null) {
+ LOG.info("Writing WARC output to {} as configured by warc.export.path",
+ warcOutputPath);
+ outputPath = new Path(warcOutputPath);
+ }
+
+ return new WarcRecordWriter(conf, outputPath, partition, context);
+ }
+
+ @Override
+ public synchronized OutputCommitter getOutputCommitter(
+ TaskAttemptContext context) throws java.io.IOException {
+ if (committer == null) {
+ Path output = getOutputPath(context);
+
+ String scheme = output.getFileSystem(context.getConfiguration()).getScheme();
+ if (scheme.startsWith("s3")) {
+ /*
+ * The default FileOutputCommitter is slow on S3, use
+ * NullOutputCommitter until a better solution is available, cf.
+ * https://hadoop.apache.org/docs/r3.1.0/hadoop-aws/tools/hadoop-aws/
+ * committers.html and
+ * https://hadoop.apache.org/docs/r3.1.0/hadoop-aws/tools/hadoop-aws/
+ * committer_architecture.html
+ */
+ committer = new NullOutputCommitter();
+ } else {
+ committer = super.getOutputCommitter(context);
+ }
+ }
+ return committer;
+ }
+
+ @Override
+ public void checkOutputSpecs(JobContext job)
+ throws FileAlreadyExistsException, IOException {
+ // Ensure that the output directory is set and not already there
+ Path outDir = getOutputPath(job);
+ if (outDir == null) {
+ throw new InvalidJobConfException("Output directory not set.");
+ }
+
+ // get delegation token for outDir's file system
+ TokenCache.obtainTokensForNamenodes(job.getCredentials(),
+ new Path[] { outDir }, job.getConfiguration());
+ }
+
+}
\ No newline at end of file
diff --git a/src/java/org/commoncrawl/util/WarcRecordWriter.java b/src/java/org/commoncrawl/util/WarcRecordWriter.java
new file mode 100644
index 0000000000..05f2a304f6
--- /dev/null
+++ b/src/java/org/commoncrawl/util/WarcRecordWriter.java
@@ -0,0 +1,1005 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.commoncrawl.util;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.net.InetAddress;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.UnknownHostException;
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.text.NumberFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.TimeZone;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.regex.Pattern;
+import java.util.zip.GZIPOutputStream;
+
+import org.apache.commons.codec.binary.Base32;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.nutch.crawl.CrawlDatum;
+import org.apache.nutch.metadata.Nutch;
+import org.apache.nutch.metrics.NutchMetrics;
+import org.apache.nutch.net.URLNormalizers;
+import org.apache.nutch.net.protocols.HttpDateFormat;
+import org.apache.nutch.net.protocols.Response;
+import org.apache.nutch.protocol.ProtocolStatus;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.net.InetAddresses;
+
+class WarcRecordWriter extends RecordWriter {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(MethodHandles.lookup().lookupClass());
+
+ /**
+ * Holds duration of fetch in metadata, see
+ * {@link org.apache.nutch.protocol.http.api.HttpBase#RESPONSE_TIME}
+ */
+ protected static final Text FETCH_DURATION = new Text("_rs_");
+ public static final String CRLF = "\r\n";
+ public static final String COLONSP = ": ";
+ protected static final Pattern PROBLEMATIC_HEADERS = Pattern
+ .compile("(?i)(?:Content-(?:Encoding|Length)|Transfer-Encoding)");
+ protected static final String X_HIDE_HEADER = "X-Crawler-";
+
+ protected static final Pattern STATUS_LINE_PATTERN = Pattern
+ .compile("^HTTP/1\\.[01] [0-9]{3}(?: .*)?$");
+ protected static final Pattern WS_PATTERN = Pattern.compile("\\s+");
+ protected static final Pattern HTTP_VERSION_PATTERN = Pattern
+ .compile("^HTTP/1\\.[01]$");
+ protected static final Pattern HTTP_STATUS_CODE_PATTERN = Pattern
+ .compile("^[0-9]{3}$");
+ protected static final String HTTP_VERSION_FALLBACK = "HTTP/1.1";
+
+ private TaskAttemptContext context;
+ private DataOutputStream warcOut;
+ private WarcWriter warcWriter;
+ private DataOutputStream crawlDiagnosticsWarcOut;
+ private WarcWriter crawlDiagnosticsWarcWriter;
+ private DataOutputStream robotsTxtWarcOut;
+ private WarcWriter robotsTxtWarcWriter;
+ private DataOutputStream cdxOut;
+ private DataOutputStream crawlDiagnosticsCdxOut;
+ private DataOutputStream robotsTxtCdxOut;
+ private URI warcinfoId;
+ private URI crawlDiagnosticsWarcinfoId;
+ private URI robotsTxtWarcinfoId;
+ private MessageDigest sha1 = null;
+ private Base32 base32 = new Base32();
+ private LanguageDetector langDetect;
+ private boolean generateCrawlDiagnostics;
+ private boolean generateRobotsTxt;
+ private boolean generateCdx;
+ private boolean deduplicate;
+ private boolean detectLanguage;
+ private boolean skipByContent;
+ Pattern mimetypeSkipPattern;
+ float mimetypeSkipFactor = .0f;
+ float truncatedSkipFactor = .0f;
+ int maxContent = Integer.MAX_VALUE;
+ private String precedingURL = ""; // for deduplication
+ private URLNormalizers urlNormalizers;
+ private URLNormalizers urlNormalizersRedirect;
+
+ private SimpleDateFormat isoDate;
+
+ public WarcRecordWriter(Configuration conf, Path outputPath, int partition,
+ TaskAttemptContext context) throws IOException {
+
+ this.context = context;
+
+ FileSystem fs = outputPath.getFileSystem(conf);
+
+ SimpleDateFormat fileDate = new SimpleDateFormat("yyyyMMddHHmmss",
+ Locale.ROOT);
+ fileDate.setTimeZone(TimeZone.getTimeZone("UTC"));
+
+ isoDate = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ROOT);
+ isoDate.setTimeZone(TimeZone.getTimeZone("UTC"));
+
+ String prefix = conf.get("warc.export.prefix", "NUTCH-CRAWL");
+
+ /*
+ * WARC-Date : "The timestamp shall represent the instant that data capture
+ * for record creation began."
+ * (http://iipc.github.io/warc-specifications/specifications/warc-format/
+ * warc-1.1/#warc-date-mandatory)
+ */
+ String date = conf.get("warc.export.date", fileDate.format(new Date()));
+ String endDate = conf.get("warc.export.date.end", date);
+ Date captureStartDate = new Date();
+ try {
+ captureStartDate = fileDate.parse(date);
+ } catch (ParseException e) {
+ LOG.error("Failed to parse warc.export.date {}: {}", date,
+ e.getMessage());
+ }
+
+ String hostname = conf.get("warc.export.hostname", getHostname());
+ String filename = getFileName(prefix, date, endDate, hostname, partition);
+
+ String publisher = conf.get("warc.export.publisher", null);
+ String operator = conf.get("warc.export.operator", null);
+ String software = conf.get("warc.export.software", "Apache Nutch");
+ String isPartOf = conf.get("warc.export.isPartOf", null);
+ String description = conf.get("warc.export.description", null);
+ generateCrawlDiagnostics = conf.getBoolean("warc.export.crawldiagnostics",
+ false);
+ generateRobotsTxt = conf.getBoolean("warc.export.robotstxt", false);
+ generateCdx = conf.getBoolean("warc.export.cdx", false);
+ deduplicate = conf.getBoolean("warc.deduplicate", false);
+ detectLanguage = conf.getBoolean("warc.detect.language", false);
+ mimetypeSkipPattern = conf.getPattern("warc.skip.mimetype.pattern", null);
+ mimetypeSkipFactor = conf.getFloat("warc.skip.mimetype.factor", .0f);
+ truncatedSkipFactor = conf.getFloat("warc.skip.mimetype.truncated.factor", .0f);
+ maxContent = conf.getInt("http.content.limit", Integer.MAX_VALUE);
+ if ((mimetypeSkipPattern != null && mimetypeSkipFactor > .0f) || truncatedSkipFactor > .0f) {
+ skipByContent = true;
+ }
+ urlNormalizers = new URLNormalizers(conf, URLNormalizers.SCOPE_INDEXER);
+ if (generateCdx) {
+ // URL normalizers to normalize the redirect targets/locations put into
+ // the CDX index
+ urlNormalizersRedirect = new URLNormalizers(conf,
+ URLNormalizers.SCOPE_FETCHER);
+ }
+
+ Path warcPath = new Path(new Path(outputPath, "warc"), filename);
+ warcOut = fs.create(warcPath);
+ Path cdxPath = null;
+ if (generateCdx) {
+ cdxPath = new Path(
+ conf.get("warc.export.cdx.path", outputPath.toString()));
+ cdxOut = openCdxOutputStream(new Path(cdxPath, "warc"), filename, conf);
+ }
+ warcWriter = openWarcWriter(warcPath, warcOut, cdxOut,
+ urlNormalizersRedirect);
+ warcinfoId = warcWriter.writeWarcinfoRecord(filename, hostname, publisher,
+ operator, software, isPartOf, description, captureStartDate);
+
+ if (generateCrawlDiagnostics) {
+ Path crawlDiagnosticsWarcPath = new Path(
+ new Path(outputPath, "crawldiagnostics"), filename);
+ crawlDiagnosticsWarcOut = fs.create(crawlDiagnosticsWarcPath);
+ if (generateCdx) {
+ crawlDiagnosticsCdxOut = openCdxOutputStream(
+ new Path(cdxPath, "crawldiagnostics"), filename, conf);
+ }
+ crawlDiagnosticsWarcWriter = openWarcWriter(crawlDiagnosticsWarcPath,
+ crawlDiagnosticsWarcOut, crawlDiagnosticsCdxOut,
+ urlNormalizersRedirect);
+ crawlDiagnosticsWarcinfoId = crawlDiagnosticsWarcWriter
+ .writeWarcinfoRecord(filename, hostname, publisher, operator,
+ software, isPartOf, description, captureStartDate);
+ }
+
+ if (generateRobotsTxt) {
+ Path robotsTxtWarcPath = new Path(new Path(outputPath, "robotstxt"),
+ filename);
+ robotsTxtWarcOut = fs.create(robotsTxtWarcPath);
+ if (generateCdx) {
+ robotsTxtCdxOut = openCdxOutputStream(new Path(cdxPath, "robotstxt"),
+ filename, conf);
+ }
+ robotsTxtWarcWriter = openWarcWriter(robotsTxtWarcPath, robotsTxtWarcOut,
+ robotsTxtCdxOut, urlNormalizersRedirect);
+ robotsTxtWarcinfoId = robotsTxtWarcWriter.writeWarcinfoRecord(filename,
+ hostname, publisher, operator, software, isPartOf, description,
+ captureStartDate);
+ }
+
+ try {
+ sha1 = MessageDigest.getInstance("SHA1");
+ } catch (NoSuchAlgorithmException e) {
+ LOG.error("Unable to instantiate SHA1 MessageDigest object");
+ throw new RuntimeException(e);
+ }
+
+ if (detectLanguage) {
+ boolean bestEffort = conf
+ .getBoolean("warc.detect.language.cld2.besteffort", false);
+ langDetect = new LanguageDetector();
+ langDetect.setBestEffort(bestEffort);
+ }
+ }
+
+ /**
+ * Compose a unique WARC file name.
+ *
+ * The WARC specification recommends:
+ * Prefix-Timestamp-Serial-Crawlhost.warc.gz (WARC
+ * 1.1, Annex C)
+ *
+ * @param prefix
+ * WARC file name prefix
+ * @param startDate
+ * capture start date
+ * @param endDate
+ * capture end date
+ * @param hostname
+ * name of the crawling host
+ * @param partition
+ * MapReduce partition
+ * @return (unique) WARC file name
+ */
+ protected String getFileName(String prefix, String startDate, String endDate,
+ String host, int partition) {
+ NumberFormat numberFormat = NumberFormat.getInstance();
+ numberFormat.setMinimumIntegerDigits(5);
+ numberFormat.setGroupingUsed(false);
+ return prefix + "-" + startDate + "-" + endDate + "-"
+ + numberFormat.format(partition) + ".warc.gz";
+ }
+
+ protected String getSha1DigestWithAlg(byte[] bytes) {
+ sha1.reset();
+ return "sha1:" + base32.encodeAsString(sha1.digest(bytes));
+ }
+
+ protected static String getStatusLine(String httpHeader) {
+ int eol = httpHeader.indexOf('\n');
+ if (eol == -1) {
+ return httpHeader;
+ }
+ if (eol > 0 && httpHeader.charAt(eol - 1) == '\r') {
+ eol--;
+ }
+ return httpHeader.substring(0, eol);
+ }
+
+ protected static int getStatusCode(String statusLine) {
+ int start = statusLine.indexOf(" ");
+ int end = statusLine.indexOf(" ", start + 1);
+ if (end == -1)
+ end = statusLine.length();
+ int code = 200;
+ try {
+ code = Integer.parseInt(statusLine.substring(start + 1, end));
+ } catch (NumberFormatException e) {
+ }
+ return code;
+ }
+
+ /** Format status line and pair-wise list of headers as string */
+ public static String formatHttpHeaders(String statusLine, List headers) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(statusLine).append(CRLF);
+ Iterator it = headers.iterator();
+ while (it.hasNext()) {
+ String name = it.next();
+ if (!it.hasNext()) {
+ // no value for name
+ break;
+ }
+ String value = it.next();
+ sb.append(name).append(COLONSP).append(value).append(CRLF);
+ }
+ sb.append(CRLF);
+ return sb.toString();
+ }
+
+ /**
+ * Fix the HTTP version in the status line - replace HTTP/2
+ * by HTTP/1.1 ({@link this#HTTP_VERSION_FALLBACK}}.
+ *
+ * See also {@link #fixHttpHeaders(String, int)}
+ *
+ * @param headers
+ * HTTP 1.1 or 1.0 request header string, CR-LF-separated lines,
+ * first line is the status line
+ * @return safe HTTP request header
+ */
+ public static String fixHttpRequestHeaders(String headers) {
+ String http2version = " HTTP/2\r\n";
+ int pos = headers.indexOf(http2version);
+ if (pos >= 0) {
+ StringBuilder replacement = new StringBuilder();
+ String statusLinePrefix = headers.substring(0, pos);
+ if (statusLinePrefix.indexOf(CRLF) > 0) {
+ // match in subsequent header lines (should not or rarely happen)
+ return headers;
+ }
+ replacement.append(statusLinePrefix);
+ replacement.append(' ');
+ replacement.append(HTTP_VERSION_FALLBACK);
+ replacement.append(CRLF);
+ replacement.append(headers.substring(pos + http2version.length()));
+ return replacement.toString();
+ }
+ return headers;
+ }
+
+ /**
+ * Modify verbatim HTTP response headers: fix, remove or replace headers
+ * Content-Length, Content-Encoding and
+ * Transfer-Encoding which may confuse WARC readers. Ensure that
+ * returned header end with a single empty line (\r\n\r\n).
+ *
+ * If the HTTP version in the status line is HTTP/2, replace it
+ * by HTTP/1.1 ({@link this#HTTP_VERSION_FALLBACK}}.
+ *
+ * @param headers
+ * HTTP 1.1 or 1.0 response header string, CR-LF-separated lines,
+ * first line is the status line
+ * @return safe HTTP response header
+ */
+ public static String fixHttpHeaders(String headers, int contentLength) {
+ int start = 0, lineEnd = 0, last = 0, trailingCrLf= 0;
+ boolean hasContentLength = false;
+ StringBuilder replacement = new StringBuilder();
+ while (start < headers.length()) {
+ lineEnd = headers.indexOf(CRLF, start);
+ trailingCrLf = 1;
+ if (lineEnd == -1) {
+ lineEnd = headers.length();
+ trailingCrLf = 0;
+ }
+ int colonPos = -1;
+ for (int i = start; i < lineEnd; i++) {
+ if (headers.charAt(i) == ':') {
+ colonPos = i;
+ break;
+ }
+ }
+ if (colonPos == -1) {
+ boolean valid = true;
+ if (start == 0) {
+ // status line (without colon)
+ final String statusLine = headers.substring(0, lineEnd);
+ if (!STATUS_LINE_PATTERN.matcher(statusLine).matches()) {
+ final String[] parts = WS_PATTERN
+ .split(headers.substring(0, lineEnd), 3);
+ if (parts.length < 2
+ || !HTTP_STATUS_CODE_PATTERN.matcher(parts[1]).matches()) {
+ // nothing we can do here, leave status line as is
+ LOG.warn(
+ "WARC parsers may fail on non-standard HTTP 1.0 / 1.1 response status line: {}",
+ statusLine);
+ } else {
+ if (HTTP_VERSION_PATTERN.matcher(parts[0]).matches()) {
+ replacement.append(parts[0]);
+ } else {
+ replacement.append(HTTP_VERSION_FALLBACK);
+ }
+ replacement.append(' ');
+ replacement.append(parts[1]); // status code
+ replacement.append(' ');
+ if (parts.length == 3) {
+ replacement.append(parts[2]); // message
+ }
+ replacement.append(CRLF);
+ last = lineEnd + 2 * trailingCrLf;
+ }
+ }
+ } else if ((lineEnd + 4) == headers.length()
+ && headers.endsWith(CRLF + CRLF)) {
+ // ok, trailing empty line
+ trailingCrLf = 2;
+ } else if (start == lineEnd) {
+ // skip/remove empty line
+ LOG.debug("Skipping empty header line");
+ valid = false;
+ } else {
+ LOG.warn("Invalid header line: {}",
+ headers.substring(start, lineEnd));
+ valid = false;
+ }
+ if (!valid) {
+ if (last < start) {
+ replacement.append(headers.substring(last, start));
+ }
+ last = lineEnd + 2 * trailingCrLf;
+ }
+ start = lineEnd + 2 * trailingCrLf;
+ /*
+ * skip over invalid header line, no further check for problematic
+ * headers required
+ */
+ continue;
+ }
+ String name = headers.substring(start, colonPos);
+ if (PROBLEMATIC_HEADERS.matcher(name).matches()) {
+ boolean needsFix = true;
+ if (name.equalsIgnoreCase("content-length")) {
+ hasContentLength = true;
+ String value = headers.substring(colonPos + 1, lineEnd).trim();
+ try {
+ int l = Integer.parseInt(value);
+ if (l == contentLength) {
+ needsFix = false;
+ }
+ } catch (NumberFormatException e) {
+ // needs to be fixed
+ }
+ }
+ if (needsFix) {
+ if (last < start) {
+ replacement.append(headers.substring(last, start));
+ }
+ last = lineEnd + 2 * trailingCrLf;
+ replacement.append(X_HIDE_HEADER)
+ .append(headers.substring(start, lineEnd + 2 * trailingCrLf));
+ if (trailingCrLf == 0) {
+ replacement.append(CRLF);
+ trailingCrLf = 1;
+ }
+ if (name.equalsIgnoreCase("content-length")) {
+ // add effective uncompressed and unchunked length of content
+ replacement.append("Content-Length").append(COLONSP)
+ .append(contentLength).append(CRLF);
+ }
+ }
+ }
+ start = lineEnd + 2 * trailingCrLf;
+ }
+ if (last > 0 || trailingCrLf != 2 || !hasContentLength) {
+ if (last < headers.length()) {
+ // append trailing headers
+ replacement.append(headers.substring(last));
+ }
+ if (!hasContentLength) {
+ replacement.append("Content-Length").append(COLONSP).append(contentLength)
+ .append(CRLF);
+ }
+ while (trailingCrLf < 2) {
+ replacement.append(CRLF);
+ trailingCrLf++;
+ }
+ return replacement.toString();
+ }
+ return headers;
+ }
+
+ protected static String getHostname() {
+ try {
+ return InetAddress.getLocalHost().getHostName();
+ } catch (UnknownHostException e) {
+ LOG.warn("Failed to get hostname: {}", e.getMessage());
+ }
+ return "localhost";
+ }
+
+ /**
+ * Canonicalize IPv6 address strings.
+ *
+ * @param ip
+ * IP address string representation
+ * @return the canonical IP address
+ */
+ protected static String canonicalizeIP(String ip) {
+ if (ip.indexOf(':') > -1) {
+ return InetAddresses.toAddrString(InetAddresses.forString(ip));
+ }
+ return ip;
+ }
+
+ private WarcWriter openWarcWriter(Path warcPath, DataOutputStream warcOut,
+ DataOutputStream cdxOut, URLNormalizers redirectNormalizers) {
+ if (cdxOut != null) {
+ return new WarcCdxWriter(warcOut, cdxOut, warcPath, redirectNormalizers);
+ }
+ return new WarcWriter(warcOut);
+ }
+
+ protected static DataOutputStream openCdxOutputStream(Path cdxPath,
+ String warcFilename, Configuration conf) throws IOException {
+ String cdxFilename = warcFilename.replaceFirst("\\.warc\\.gz$", ".cdx.gz");
+ Path cdxFile = new Path(cdxPath, cdxFilename);
+ FileSystem fs = cdxPath.getFileSystem(conf);
+ return new DataOutputStream(new GZIPOutputStream(fs.create(cdxFile)));
+ }
+
+ @Override
+ public synchronized void write(Text key, WarcCapture value)
+ throws IOException {
+
+ if (value.content == null) {
+ ProtocolStatus pstatus = null;
+ if (value.datum != null) {
+ pstatus = (ProtocolStatus) value.datum.getMetaData()
+ .get(Nutch.WRITABLE_PROTO_STATUS_KEY);
+ }
+ if (pstatus != null) {
+ LOG.warn(
+ "Cannot write WARC record, no content for {}, protocol status: {} - {}",
+ value.url, pstatus.getName(), pstatus.getMessage());
+ } else {
+ LOG.warn(
+ "Cannot write WARC record, no content and protocol status for {}",
+ value.url);
+ }
+ context.getCounter(NutchMetrics.GROUP_WARC_WRITER,
+ NutchMetrics.WARC_WRITER_SKIPPED_NO_CONTENT_TOTAL).increment(1);
+ return;
+ }
+
+ URI targetUri = null;
+ String url = value.url.toString();
+ try {
+ targetUri = new URI(url);
+ } catch (URISyntaxException e) {
+ if (value.datum != null
+ && value.datum.getStatus() == CrawlDatum.STATUS_FETCH_SUCCESS) {
+ // if a successful capture, try to normalize the URL
+ String urlNorm = null;
+ try {
+ urlNorm = urlNormalizers.normalize(url, URLNormalizers.SCOPE_INDEXER);
+ } catch (MalformedURLException ee) {
+ // ignore, log exception observed on original URL
+ }
+ if (urlNorm != null && !url.equals(urlNorm)) {
+ try {
+ targetUri = new URI(urlNorm);
+ LOG.info("Normalized URL to valid URI: {} -> {}", url, urlNorm);
+ context.getCounter(NutchMetrics.GROUP_WARC_WRITER,
+ NutchMetrics.WARC_WRITER_URI_NORMALIZED_TOTAL).increment(1);
+ } catch (URISyntaxException ee) {
+ // ignore, log exception observed on original URL
+ }
+ }
+ }
+ if (targetUri == null) {
+ LOG.error("Cannot write WARC record, invalid URI: {}", url);
+ context
+ .getCounter(NutchMetrics.GROUP_WARC_WRITER,
+ NutchMetrics.WARC_WRITER_SKIPPED_INVALID_URI_TOTAL)
+ .increment(1);
+ return;
+ }
+ }
+
+ if (skipByContent) {
+ float factor = .0f;
+ if (mimetypeSkipFactor > .0f && mimetypeSkipPattern.matcher(value.content.getContentType()).find()) {
+ factor += mimetypeSkipFactor;
+ }
+ if (factor > .0f) {
+ factor *= value.content.getContent().length / (1.0 * maxContent);
+ String truncated = value.content.getMetadata().get(Response.TRUNCATED_CONTENT_REASON);
+ if (truncatedSkipFactor > .0f && truncated != null) {
+ factor += truncatedSkipFactor;
+ }
+ if (ThreadLocalRandom.current().nextFloat() < factor) {
+ LOG.info(
+ "Skipped record by content (truncated: {}, content-type: {}, length: {}): {}",
+ (truncated != null ? truncated : "-"),
+ value.content.getContentType(), value.content.getContent().length,
+ value.url);
+ context
+ .getCounter(NutchMetrics.GROUP_WARC_WRITER,
+ NutchMetrics.WARC_WRITER_SKIPPED_BY_CONTENT_TYPE_TOTAL)
+ .increment(1);
+ return;
+ }
+ }
+ }
+
+ if (deduplicate) {
+ // given that the reducer input is sorted, a comparison with the preceding
+ // URL is sufficient
+ if (precedingURL.equals(url)) {
+ // LOG.info("Skipping duplicate record: {}", value.url);
+ try {
+ String status = "?";
+ if (value.content.getMetadata()
+ .get(Response.RESPONSE_HEADERS) != null) {
+ status = Integer.toString(getStatusCode(getStatusLine(
+ value.content.getMetadata().get(Response.RESPONSE_HEADERS))));
+ } else if (value.datum != null) {
+ ProtocolStatus pstatus = (ProtocolStatus) value.datum.getMetaData()
+ .get(Nutch.WRITABLE_PROTO_STATUS_KEY);
+ status = pstatus.getName();
+ }
+ Instant date = null;
+ if (value.datum != null) {
+ date = Instant.ofEpochMilli(value.datum.getFetchTime());
+ } else {
+ String fetchTime = value.content.getMetadata()
+ .get(Nutch.FETCH_TIME_KEY);
+ if (fetchTime != null) {
+ try {
+ date = Instant.ofEpochMilli(Long.parseLong(fetchTime));
+ } catch (NumberFormatException e) {
+ LOG.error("Invalid fetch time '{}' in content metadata of {}",
+ fetchTime, value.url.toString());
+ }
+ }
+ }
+ LOG.info("Skipping duplicate record: {} ({}, status: {}, size: {})",
+ value.url, date, status, value.content.getContent().length);
+ } catch (Throwable t) {
+ LOG.error(t.getMessage());
+ }
+ context.getCounter(NutchMetrics.GROUP_WARC_WRITER,
+ NutchMetrics.WARC_WRITER_SKIPPED_DUPLICATE_TOTAL).increment(1);
+ return;
+ }
+ precedingURL = url;
+ }
+
+ String ip = "0.0.0.0";
+ Date date = null;
+ boolean notModified = false;
+ Date lastModifiedDate = null;
+ String verbatimResponseHeaders = null;
+ String verbatimRequestHeaders = null;
+ List headers = new ArrayList<>();
+ String responseHeaders = null;
+ String statusLine = "";
+ int httpStatusCode = 200;
+ String fetchDuration = null;
+ String truncatedReason = null;
+ String[] protocolVersions = null;
+ String[] cipherSuites = null;
+
+ if (value.datum != null) {
+ date = new Date(value.datum.getFetchTime());
+ // This is for older crawl dbs that don't include the verbatim status
+ // line in the metadata
+ ProtocolStatus pstatus = (ProtocolStatus) value.datum.getMetaData()
+ .get(Nutch.WRITABLE_PROTO_STATUS_KEY);
+ if (pstatus == null) {
+ LOG.warn("Cannot write WARC record, no protocol status for {}",
+ value.url);
+ context
+ .getCounter(NutchMetrics.GROUP_WARC_WRITER,
+ NutchMetrics.WARC_WRITER_SKIPPED_NO_PROTOCOL_STATUS_TOTAL)
+ .increment(1);
+ return;
+ }
+ switch (pstatus.getCode()) {
+ case ProtocolStatus.SUCCESS:
+ statusLine = "HTTP/1.1 200 OK";
+ httpStatusCode = 200;
+ break;
+ case ProtocolStatus.TEMP_MOVED:
+ statusLine = "HTTP/1.1 302 Found";
+ httpStatusCode = 302;
+ break;
+ case ProtocolStatus.MOVED:
+ statusLine = "HTTP/1.1 301 Moved Permanently";
+ httpStatusCode = 301;
+ break;
+ case ProtocolStatus.NOTMODIFIED:
+ statusLine = "HTTP/1.1 304 Not Modified";
+ httpStatusCode = 304;
+ notModified = true;
+ long modifiedTime = value.datum.getModifiedTime();
+ if (modifiedTime > 0) {
+ lastModifiedDate = new Date(modifiedTime);
+ }
+ break;
+ default:
+ if (value.content.getMetadata()
+ .get(Response.RESPONSE_HEADERS) == null) {
+ LOG.warn("Unknown or ambiguous protocol status: {}", pstatus);
+ context.getCounter(NutchMetrics.GROUP_WARC_WRITER,
+ NutchMetrics.WARC_WRITER_SKIPPED_UNKNOWN_PROTOCOL_STATUS_TOTAL)
+ .increment(1);
+ return;
+ }
+ }
+ String httpStatusCodeVal = value.datum.getMetaData()
+ .get(Nutch.PROTOCOL_STATUS_CODE_KEY).toString();
+ if (httpStatusCodeVal != null) {
+ try {
+ httpStatusCode = Integer.parseInt(httpStatusCodeVal);
+ } catch (NumberFormatException e) {
+ }
+ }
+ if (value.datum.getMetaData().get(FETCH_DURATION) != null) {
+ fetchDuration = value.datum.getMetaData().get(FETCH_DURATION)
+ .toString();
+ }
+ } else {
+ // robots.txt, no CrawlDatum available
+ String fetchTime = value.content.getMetadata().get(Nutch.FETCH_TIME_KEY);
+ if (fetchTime != null) {
+ try {
+ date = new Date(Long.parseLong(fetchTime));
+ } catch (NumberFormatException e) {
+ LOG.error("Invalid fetch time '{}' in content metadata of {}",
+ fetchTime, value.url.toString());
+ }
+ }
+ if (date == null) {
+ String httpDate = value.content.getMetadata().get("Date");
+ if (httpDate != null) {
+ try {
+ date = HttpDateFormat.toDate(httpDate);
+ } catch (ParseException e) {
+ LOG.warn("Failed to parse HTTP Date {} for {}", httpDate,
+ targetUri);
+ date = new Date();
+ }
+ } else {
+ LOG.warn("No HTTP Date for {}", targetUri);
+ date = new Date();
+ }
+ }
+ // status is taken from header
+ }
+
+ boolean useVerbatimResponseHeaders = false;
+
+ for (String name : value.content.getMetadata().names()) {
+ String val = value.content.getMetadata().get(name);
+ switch (name) {
+ case Response.IP_ADDRESS:
+ ip = canonicalizeIP(val);
+ break;
+ case Response.REQUEST:
+ verbatimRequestHeaders = val;
+ break;
+ case Response.RESPONSE_HEADERS:
+ verbatimResponseHeaders = val;
+ if (verbatimResponseHeaders.contains(CRLF)) {
+ useVerbatimResponseHeaders = true;
+ }
+ statusLine = getStatusLine(verbatimResponseHeaders);
+ httpStatusCode = getStatusCode(statusLine);
+ break;
+ case Response.TRUNCATED_CONTENT_REASON:
+ truncatedReason = val;
+ break;
+ case Response.PROTOCOL_VERSIONS:
+ protocolVersions = val.split(",");
+ break;
+ case Response.CIPHER_SUITES:
+ cipherSuites = val.split(",");
+ break;
+ case Nutch.SEGMENT_NAME_KEY:
+ case Nutch.FETCH_STATUS_KEY:
+ case Nutch.SCORE_KEY:
+ case Nutch.SIGNATURE_KEY:
+ case Response.FETCH_TIME:
+ break; // ignore, not required for WARC record
+ default:
+ // We have to fix up a few headers because we don't have the raw
+ // responses to avoid that WARC readers try to read the content
+ // as chunked or gzip-compressed.
+ if (name.equalsIgnoreCase(Response.CONTENT_LENGTH)) {
+ int origContentLength = -1;
+ try {
+ origContentLength = Integer.parseInt(val);
+ } catch (NumberFormatException e) {
+ // ignore
+ }
+ headers.add(Response.CONTENT_LENGTH);
+ if (origContentLength != value.content.getContent().length) {
+ headers.add("" + value.content.getContent().length);
+ headers.add(X_HIDE_HEADER + Response.CONTENT_LENGTH);
+ }
+ } else if (name.equalsIgnoreCase(Response.CONTENT_ENCODING)) {
+ if (val.equalsIgnoreCase("identity")) {
+ headers.add(name);
+ } else {
+ headers.add(X_HIDE_HEADER + Response.CONTENT_ENCODING);
+ }
+ } else if (name.equalsIgnoreCase(Response.TRANSFER_ENCODING)) {
+ if (val.equalsIgnoreCase("identity")) {
+ headers.add(name);
+ } else {
+ headers.add(X_HIDE_HEADER + Response.TRANSFER_ENCODING);
+ }
+ } else {
+ headers.add(name);
+ }
+ headers.add(val);
+ }
+ }
+
+ if (verbatimRequestHeaders == null) {
+ LOG.error("No request headers for {}", url);
+ }
+
+ if (useVerbatimResponseHeaders && verbatimResponseHeaders != null) {
+ responseHeaders = fixHttpHeaders(verbatimResponseHeaders, value.content.getContent().length);
+ } else {
+ responseHeaders = formatHttpHeaders(statusLine, headers);
+ }
+
+ WarcWriter writer = warcWriter;
+ URI infoId = this.warcinfoId;
+ if (value.datum == null) {
+ // no CrawlDatum: must be a robots.txt
+ if (!generateRobotsTxt)
+ return;
+ writer = robotsTxtWarcWriter;
+ infoId = robotsTxtWarcinfoId;
+ } else if (value.datum.getStatus() != CrawlDatum.STATUS_FETCH_SUCCESS) {
+ if (!generateCrawlDiagnostics)
+ return;
+ writer = crawlDiagnosticsWarcWriter;
+ infoId = crawlDiagnosticsWarcinfoId;
+ }
+
+ LOG.info("WARC {} record {} ({}, status: {}, size: {})",
+ (notModified ? "revisit" : "response"), targetUri, isoDate.format(date),
+ httpStatusCode, value.content.getContent().length);
+
+ URI requestId = null;
+ if (verbatimRequestHeaders != null) {
+ requestId = writer.writeWarcRequestRecord(targetUri, ip, date, infoId,
+ protocolVersions, cipherSuites,
+ fixHttpRequestHeaders(verbatimRequestHeaders)
+ .getBytes(StandardCharsets.UTF_8));
+ }
+
+ if (generateCdx) {
+ value.content.getMetadata().add(WarcWriter.HTTP_STATUS_CODE,
+ String.format("%d", httpStatusCode));
+ }
+
+ LanguageDetector.Result ldres = null;
+ if (detectLanguage && writer == warcWriter) {
+ // detect language only for successfully fetched primary documents
+ ldres = langDetect.detectLanguage(targetUri, value.content);
+ if (ldres.errorReason != null) {
+ context.getCounter(NutchMetrics.GROUP_WARC_WRITER,
+ NutchMetrics.WARC_WRITER_LID_ERROR_PREFIX + ldres.errorStatus.name)
+ .increment(1);
+ } else if (ldres.languages == null) {
+ context.getCounter(NutchMetrics.GROUP_WARC_WRITER,
+ NutchMetrics.WARC_WRITER_LID_NO_RESULT_TOTAL).increment(1);
+ } else if (ldres.languages.isReliable()) {
+ context
+ .getCounter(NutchMetrics.GROUP_WARC_WRITER,
+ NutchMetrics.WARC_WRITER_LID_RESULT_RELIABLE_TOTAL)
+ .increment(1);
+ } else {
+ context
+ .getCounter(NutchMetrics.GROUP_WARC_WRITER,
+ NutchMetrics.WARC_WRITER_LID_RESULT_NOT_RELIABLE_TOTAL)
+ .increment(1);
+ }
+ if (generateCdx) {
+ if (ldres.charset != null) {
+ value.content.getMetadata().add(WarcWriter.DETECTED_CHARSET,
+ ldres.charset.name());
+ }
+ org.commoncrawl.langdetect.cld2.Result lr = ldres.languages;
+ if (lr != null) {
+ String codes = lr.getLanguageCodesISO639_3(",", true);
+ if (codes != null && !codes.isEmpty()) {
+ value.content.getMetadata().add(WarcWriter.DETECTED_LANGUAGE,
+ codes);
+ }
+ }
+ }
+ }
+
+ URI responseId = null;
+ if (notModified) {
+ /*
+ * revisit record of profile WarcWriter.PROFILE_REVISIT_NOT_MODIFIED
+ *
+ * Note: "revisits" identified by signature comparison
+ * (WarcWriter.PROFILE_REVISIT_IDENTICAL_DIGEST) are stored as response
+ * records.
+ *
+ * The modified date of the CrawlDatum is the date of the last successful
+ * fetch with content (status 200). It is used for the
+ * WARC-Refers-To-Date.
+ */
+ byte[] responseHeaderBytes = responseHeaders
+ .getBytes(StandardCharsets.UTF_8);
+ String blockDigest = getSha1DigestWithAlg(responseHeaderBytes);
+ /*
+ * HTTP 304 not-modified responses do not have a payload, should not add a
+ * digest for it according to the WARC specification:
+ * "The WARC-Payload-Digest field ... shall not be used on records without
+ * a well-defined payload."
+ */
+ String payloadDigest = null;
+ responseId = writer.writeWarcRevisitRecord(targetUri, ip, httpStatusCode, date, infoId,
+ requestId, WarcWriter.PROFILE_REVISIT_NOT_MODIFIED, lastModifiedDate,
+ payloadDigest, blockDigest, protocolVersions, cipherSuites,
+ responseHeaderBytes, value.content);
+ } else {
+ StringBuilder responsesb = new StringBuilder(4096);
+ responsesb.append(responseHeaders);
+
+ byte[] responseHeaderBytes = responsesb.toString()
+ .getBytes(StandardCharsets.UTF_8);
+ byte[] responseBytes = new byte[responseHeaderBytes.length
+ + value.content.getContent().length];
+ System.arraycopy(responseHeaderBytes, 0, responseBytes, 0,
+ responseHeaderBytes.length);
+ System.arraycopy(value.content.getContent(), 0, responseBytes,
+ responseHeaderBytes.length, value.content.getContent().length);
+
+ String payloadDigest = getSha1DigestWithAlg(value.content.getContent());
+ String blockDigest = getSha1DigestWithAlg(responseBytes);
+ responseId = writer.writeWarcResponseRecord(targetUri, ip,
+ httpStatusCode, date, infoId, requestId, payloadDigest, blockDigest,
+ truncatedReason, protocolVersions, cipherSuites, responseBytes, value.content);
+ }
+
+ // Write metadata record
+ StringBuilder metadatasb = new StringBuilder(4096);
+ Map metadata = new LinkedHashMap();
+
+ if (fetchDuration != null) {
+ metadata.put("fetchTimeMs", fetchDuration);
+ }
+ if (ldres != null) {
+ if (ldres.charset != null) {
+ metadata.put("charset-detected", ldres.charset.name());
+ }
+ if (ldres.languages != null) {
+ metadata.put("languages-cld2", ldres.languages.toJSON());
+ }
+ }
+ if (metadata.size() > 0) {
+ for (Map.Entry entry : metadata.entrySet()) {
+ metadatasb.append(entry.getKey()).append(COLONSP)
+ .append(entry.getValue()).append(CRLF);
+ }
+ metadatasb.append(CRLF);
+
+ writer.writeWarcMetadataRecord(targetUri, date, infoId, responseId, null,
+ metadatasb.toString().getBytes(StandardCharsets.UTF_8));
+ }
+ }
+
+ @Override
+ public synchronized void close(TaskAttemptContext context)
+ throws IOException {
+ context.setStatus("closing WARC output writers");
+ warcOut.close();
+ if (generateCrawlDiagnostics) {
+ crawlDiagnosticsWarcOut.close();
+ }
+ if (generateRobotsTxt) {
+ robotsTxtWarcOut.close();
+ }
+ if (generateCdx) {
+ cdxOut.close();
+ if (generateCrawlDiagnostics) {
+ crawlDiagnosticsCdxOut.close();
+ }
+ if (generateRobotsTxt) {
+ robotsTxtCdxOut.close();
+ }
+ }
+ }
+}
diff --git a/src/java/org/commoncrawl/util/WarcWriter.java b/src/java/org/commoncrawl/util/WarcWriter.java
new file mode 100644
index 0000000000..aa9b20ba3b
--- /dev/null
+++ b/src/java/org/commoncrawl/util/WarcWriter.java
@@ -0,0 +1,485 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.commoncrawl.util;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.lang.invoke.MethodHandles;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.TimeZone;
+import java.util.UUID;
+import java.util.regex.Pattern;
+import java.util.zip.GZIPOutputStream;
+
+import org.apache.nutch.metadata.Metadata;
+import org.apache.nutch.protocol.Content;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.LinkedListMultimap;
+import com.google.common.collect.Multimap;
+
+public class WarcWriter {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(MethodHandles.lookup().lookupClass());
+
+ protected OutputStream out = null;
+ protected OutputStream origOut = null;
+
+ private static final String WARC_VERSION = "WARC/1.0";
+
+ // Record types
+ private static final String WARC_INFO = "warcinfo";
+ private static final String WARC_RESPONSE = "response";
+ private static final String WARC_REQUEST = "request";
+ private static final String WARC_REVISIT = "revisit";
+ private static final String WARC_CONVERSION = "conversion";
+ private static final String WARC_METADATA = "metadata";
+
+ // Defined fields
+ private static final String WARC_TYPE = "WARC-Type";
+ private static final String WARC_DATE = "WARC-Date";
+ private static final String WARC_RECORD_ID = "WARC-Record-ID";
+ private static final String CONTENT_LENGTH = "Content-Length";
+ private static final String CONTENT_TYPE = "Content-Type";
+ private static final String WARC_IP_ADDRESS = "WARC-IP-Address";
+ private static final String WARC_WARCINFO_ID = "WARC-Warcinfo-ID";
+ private static final String WARC_TARGET_URI = "WARC-Target-URI";
+ private static final String WARC_CONCURRENT_TO = "WARC-Concurrent-To";
+ private static final String WARC_REFERS_TO = "WARC-Refers-To";
+ private static final String WARC_REFERS_TO_TARGET_URI = "WARC-Refers-To-Target-URI";
+ private static final String WARC_REFERS_TO_DATE = "WARC-Refers-To-Date";
+ private static final String WARC_BLOCK_DIGEST = "WARC-Block-Digest";
+ private static final String WARC_PAYLOAD_DIGEST = "WARC-Payload-Digest";
+ private static final String WARC_TRUNCATED = "WARC-Truncated";
+ private static final String WARC_IDENTIFIED_PAYLOAD_TYPE = "WARC-Identified-Payload-Type";
+ private static final String WARC_PROFILE = "WARC-Profile";
+ private static final String WARC_FILENAME = "WARC-Filename";
+ /** WARC-Protocol, see https://github.com/iipc/warc-specifications/issues/42 */
+ private static final String WARC_PROTOCOL = "WARC-Protocol";
+ /** WARC-Cipher-Suite, see https://github.com/iipc/warc-specifications/issues/94 */
+ private static final String WARC_CIPHER_SUITE = "WARC-Cipher-Suite";
+
+ public static final String PROFILE_REVISIT_IDENTICAL_DIGEST = "http://netpreserve.org/warc/1.1/revisit/identical-payload-digest";
+ public static final String PROFILE_REVISIT_NOT_MODIFIED = "http://netpreserve.org/warc/1.1/revisit/server-not-modified";
+
+ private static final String CRLF = "\r\n";
+ private static final String COLONSP = ": ";
+
+ /* Metadata names to pass from WARC to CDX */
+ protected static final String HTTP_STATUS_CODE = "HTTP-Status-Code";
+ protected static final String DETECTED_CHARSET = "Detected-Charset";
+ protected static final String DETECTED_LANGUAGE = "Detected-Language";
+
+ public static final String CONTENT_TYPE_RESPONSE = "application/http; msgtype=response";
+ public static final String CONTENT_TYPE_METADATA = "application/warc-fields";
+
+ private SimpleDateFormat isoDate;
+
+ public static class CompressedOutputStream extends GZIPOutputStream {
+ public CompressedOutputStream(OutputStream out) throws IOException {
+ super(out);
+ }
+
+ public void end() {
+ def.end();
+ }
+ }
+
+ public WarcWriter(final OutputStream out) {
+ this.origOut = this.out = out;
+ isoDate = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ROOT);
+ isoDate.setTimeZone(TimeZone.getTimeZone("UTC"));
+ }
+
+ /**
+ * Class to hold HTTP and SSL/TLS protocol versions to fill the
+ * WARC-Protocol field. Protocol names require normalization, see
+ * https://github.com/iipc/warc-specifications/issues/42
+ */
+ public static class WarcProtocol {
+ public static Set protocols = Set.of("dns", "ftp", "gemini",
+ "gopher", "http/0.9", "http/1.0", "http/1.1", "h2", "h2c", "spdy/1",
+ "spdy/2", "spdy/3", "ssl/2", "ssl/3", "tls/1.0", "tls/1.1", "tls/1.2",
+ "tls/1.3");
+ public static Pattern vPattern = Pattern.compile("^(?:ssl|tls)v[0-9]",
+ Pattern.CASE_INSENSITIVE);
+ private String name;
+
+ public WarcProtocol(final String protocol) {
+ name = protocol.toLowerCase(Locale.ROOT);
+ if (vPattern.matcher(name).find()) {
+ name = name.substring(0, 3) + '/' + name.substring(4);
+ }
+ if (!protocols.contains(name)) {
+ LOG.warn("Unknown protocol name or version: {}", name);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+ }
+
+ /**
+ *
+ * @return record id for the warcinfo record
+ * @throws IOException
+ */
+ public URI writeWarcinfoRecord(String filename, String hostname,
+ String publisher, String operator, String software, String isPartOf,
+ String description, Date date)
+ throws IOException {
+ Multimap extra = LinkedListMultimap.create();
+ extra.put(WARC_FILENAME, filename);
+
+ StringBuilder sb = new StringBuilder();
+ Multimap settings = LinkedListMultimap.create();
+
+ if (isPartOf != null) {
+ settings.put("isPartOf", isPartOf);
+ }
+
+ if (publisher != null) {
+ settings.put("publisher", publisher);
+ }
+
+ if (description != null) {
+ settings.put("description", description);
+ }
+
+ if (operator != null) {
+ settings.put("operator", operator);
+ }
+
+ if (hostname != null) {
+ settings.put("hostname", hostname);
+ }
+
+ if (software != null) {
+ settings.put("software", software);
+ }
+
+ String robotsTxtParser = String.format(Locale.ROOT,
+ "checked via crawler-commons %s (https://github.com/crawler-commons/crawler-commons)",
+ crawlercommons.CrawlerCommons.getVersion());
+ settings.put("robots", robotsTxtParser);
+
+ settings.put("format", "WARC File Format 1.1");
+ settings.put("conformsTo",
+ "https://iipc.github.io/warc-specifications/specifications/warc-format/warc-1.1/");
+
+ writeWarcKeyValue(sb, settings);
+
+ byte[] ba = sb.toString().getBytes(StandardCharsets.UTF_8);
+ URI recordId = getRecordId();
+
+ writeRecord(WARC_INFO, date, CONTENT_TYPE_METADATA, recordId, extra,
+ new ByteArrayInputStream(ba), ba.length);
+ return recordId;
+ }
+
+ public URI writeWarcRequestRecord(final URI targetUri, final String ip,
+ final Date date, final URI warcinfoId, String[] protocolVersions,
+ String[] cipherSuites, final byte[] block) throws IOException {
+ Multimap extra = LinkedListMultimap.create();
+ extra.put(WARC_WARCINFO_ID, "<" + warcinfoId.toString() + ">");
+ extra.put(WARC_IP_ADDRESS, ip);
+ extra.put(WARC_TARGET_URI, targetUri.toASCIIString());
+ if (protocolVersions != null) {
+ for (String pVersion : protocolVersions) {
+ extra.put(WARC_PROTOCOL, new WarcProtocol(pVersion).toString());
+ }
+ }
+ if (cipherSuites != null) {
+ for (String cipher : cipherSuites) {
+ extra.put(WARC_CIPHER_SUITE, cipher);
+ }
+ }
+
+ URI recordId = getRecordId();
+ writeRecord(WARC_REQUEST, date, "application/http; msgtype=request",
+ recordId, extra, block);
+ return recordId;
+ }
+
+ public URI writeWarcResponseRecord(final URI targetUri, final String ip,
+ final int httpStatusCode, final Date date, final URI warcinfoId,
+ final URI relatedId, final String payloadDigest, final String blockDigest,
+ final String truncated, String[] protocolVersions, String[] cipherSuites,
+ final byte[] block, Content content) throws IOException {
+ Multimap extra = LinkedListMultimap.create();
+ extra.put(WARC_WARCINFO_ID, "<" + warcinfoId.toString() + ">");
+ if (relatedId != null) {
+ extra.put(WARC_CONCURRENT_TO, "<" + relatedId.toString() + ">");
+ }
+ extra.put(WARC_IP_ADDRESS, ip);
+ extra.put(WARC_TARGET_URI, targetUri.toASCIIString());
+ if (protocolVersions != null) {
+ for (String pVersion : protocolVersions) {
+ extra.put(WARC_PROTOCOL, new WarcProtocol(pVersion).toString());
+ }
+ }
+ if (cipherSuites != null) {
+ for (String cipher : cipherSuites) {
+ extra.put(WARC_CIPHER_SUITE, cipher);
+ }
+ }
+
+ if (payloadDigest != null) {
+ extra.put(WARC_PAYLOAD_DIGEST, payloadDigest);
+ }
+
+ if (blockDigest != null) {
+ extra.put(WARC_BLOCK_DIGEST, blockDigest);
+ }
+
+ if (truncated != null) {
+ extra.put(WARC_TRUNCATED, truncated);
+ }
+
+ extra.put(WARC_IDENTIFIED_PAYLOAD_TYPE, content.getContentType());
+
+ URI recordId = getRecordId();
+ writeRecord(WARC_RESPONSE, date, CONTENT_TYPE_RESPONSE, recordId, extra, block);
+ return recordId;
+ }
+
+ public URI writeWarcRevisitRecord(final URI targetUri, final String ip,
+ final int httpStatusCode, final Date date, final URI warcinfoId,
+ final URI relatedId, final String warcProfile, final Date refersToDate,
+ final String payloadDigest, final String blockDigest,
+ String[] protocolVersions, String[] cipherSuites, byte[] block,
+ Content content) throws IOException {
+ Multimap extra = LinkedListMultimap.create();
+ extra.put(WARC_WARCINFO_ID, "<" + warcinfoId.toString() + ">");
+ extra.put(WARC_REFERS_TO, "<" + relatedId.toString() + ">");
+ extra.put(WARC_IP_ADDRESS, ip);
+ extra.put(WARC_TARGET_URI, targetUri.toASCIIString());
+ if (protocolVersions != null) {
+ for (String pVersion : protocolVersions) {
+ extra.put(WARC_PROTOCOL, new WarcProtocol(pVersion).toString());
+ }
+ }
+ if (cipherSuites != null) {
+ for (String cipher : cipherSuites) {
+ extra.put(WARC_CIPHER_SUITE, cipher);
+ }
+ }
+ // WARC-Refers-To-Target-URI only useful for revisit by digest
+ extra.put(WARC_REFERS_TO_TARGET_URI, targetUri.toASCIIString());
+ if (refersToDate != null) {
+ extra.put(WARC_REFERS_TO_DATE, isoDate.format(refersToDate));
+ }
+ extra.put(WARC_PROFILE, warcProfile);
+
+ if (payloadDigest != null) {
+ extra.put(WARC_PAYLOAD_DIGEST, payloadDigest);
+ }
+ if (blockDigest != null) {
+ extra.put(WARC_BLOCK_DIGEST, blockDigest);
+ }
+
+ URI recordId = getRecordId();
+ writeRecord(WARC_REVISIT, date, CONTENT_TYPE_RESPONSE, recordId, extra, block);
+ return recordId;
+ }
+
+ public URI writeWarcMetadataRecord(final URI targetUri, final Date date,
+ final URI warcinfoId, final URI relatedId, final String blockDigest,
+ final byte[] block) throws IOException {
+ Multimap extra = LinkedListMultimap.create();
+ extra.put(WARC_WARCINFO_ID, "<" + warcinfoId.toString() + ">");
+ extra.put(WARC_CONCURRENT_TO, "<" + relatedId.toString() + ">");
+ extra.put(WARC_TARGET_URI, targetUri.toASCIIString());
+
+ if (blockDigest != null) {
+ extra.put(WARC_BLOCK_DIGEST, blockDigest);
+ }
+
+ URI recordId = getRecordId();
+ writeRecord(WARC_METADATA, date, CONTENT_TYPE_METADATA, recordId, extra, block);
+ return recordId;
+ }
+
+ public URI writeWarcConversionRecord(final URI targetUri, final Date date,
+ final URI warcinfoId, final URI relatedId, final String blockDigest,
+ final String contentType, final byte[] block) throws IOException {
+ Multimap extra = LinkedListMultimap.create();
+ extra.put(WARC_WARCINFO_ID, "<" + warcinfoId.toString() + ">");
+ extra.put(WARC_REFERS_TO, "<" + relatedId.toString() + ">");
+ extra.put(WARC_TARGET_URI, targetUri.toASCIIString());
+
+ if (blockDigest != null) {
+ extra.put(WARC_BLOCK_DIGEST, blockDigest);
+ }
+
+ URI recordId = getRecordId();
+ writeRecord(WARC_CONVERSION, date, contentType, recordId, extra, block);
+ return recordId;
+ }
+
+ protected void writeRecord(final String type, final Date date,
+ final String contentType, final URI recordId,
+ Multimap extra, final InputStream content,
+ final long contentLength) throws IOException {
+ StringBuilder sb = new StringBuilder(4096);
+
+ sb.append(WARC_VERSION).append(CRLF);
+
+ Multimap header = LinkedListMultimap.create();
+ header.put(WARC_TYPE, type);
+ header.put(WARC_DATE, isoDate.format(date));
+ header.put(WARC_RECORD_ID, "<" + recordId.toString() + ">");
+ header.put(CONTENT_LENGTH, Long.toString(contentLength));
+ header.put(CONTENT_TYPE, contentType);
+
+ writeWarcKeyValue(sb, header);
+ writeWarcKeyValue(sb, extra);
+
+ sb.append(CRLF);
+
+ startRecord();
+ out.write(sb.toString().getBytes(StandardCharsets.UTF_8));
+ if (contentLength != 0 && content != null) {
+ copyStream(content, out, contentLength);
+ }
+
+ out.write(CRLF.getBytes());
+ out.write(CRLF.getBytes());
+ endRecord();
+ }
+
+ protected void writeRecord(final String type, final Date date,
+ final String contentType, final URI recordId,
+ Multimap
-For the latest information about Nutch, please visit our website at:
+For the latest information about Nutch, please visit the Nutch website at:
https://nutch.apache.org/
diff --git a/build.xml b/build.xml
index 278c19e257..8e68ebd9a0 100644
--- a/build.xml
+++ b/build.xml
@@ -48,6 +48,8 @@