cachedCounters;
+
+ /**
+ * Creates a new ErrorTracker for the specified counter group.
+ *
+ * This constructor creates an ErrorTracker without cached counters.
+ * Call {@link #initCounters(TaskInputOutputContext)} in setup() to cache
+ * counter references for better performance.
+ *
+ * @param group the Hadoop counter group name (e.g., NutchMetrics.GROUP_FETCHER)
+ */
+ public ErrorTracker(String group) {
+ this.group = group;
+ this.counts = new EnumMap<>(ErrorType.class);
+ this.cachedCounters = new EnumMap<>(ErrorType.class);
+ this.totalCount = new AtomicLong(0);
+
+ // Initialize all counts to 0
+ for (ErrorType type : ErrorType.values()) {
+ counts.put(type, new AtomicLong(0));
+ }
+ }
+
+ /**
+ * Creates a new ErrorTracker with cached counter references.
+ *
+ *
This constructor caches all counter references at creation time,
+ * avoiding repeated counter lookups in hot paths.
+ *
+ * @param group the Hadoop counter group name
+ * @param context the Hadoop task context for caching counters
+ */
+ public ErrorTracker(String group, TaskInputOutputContext context) {
+ this(group);
+ initCounters(context);
+ }
+
+ /**
+ * Initializes cached counter references from the Hadoop context.
+ *
+ *
Call this method in the mapper/reducer setup() method to cache
+ * counter references and avoid repeated lookups during processing.
+ *
+ * @param context the Hadoop task context
+ */
+ public void initCounters(TaskInputOutputContext context) {
+ cachedTotalCounter = context.getCounter(group, NutchMetrics.ERROR_TOTAL);
+ for (ErrorType type : ErrorType.values()) {
+ cachedCounters.put(type, context.getCounter(group, getCounterName(type)));
+ }
+ }
+
+ /**
+ * Records an error with automatic categorization based on the throwable type.
+ *
+ * @param t the throwable to categorize and record
+ */
+ public void recordError(Throwable t) {
+ recordError(categorize(t));
+ }
+
+ /**
+ * Records an error with explicit category.
+ *
+ * @param type the error type category
+ */
+ public void recordError(ErrorType type) {
+ counts.get(type).incrementAndGet();
+ totalCount.incrementAndGet();
+ }
+
+ /**
+ * Returns the count for a specific error type.
+ *
+ * @param type the error type
+ * @return the count for that error type
+ */
+ public long getCount(ErrorType type) {
+ return counts.get(type).get();
+ }
+
+ /**
+ * Returns the total count of all errors.
+ *
+ * @return the total error count
+ */
+ public long getTotalCount() {
+ return totalCount.get();
+ }
+
+ /**
+ * Emits all error counters to the Hadoop context.
+ *
+ *
Should be called once during cleanup to emit aggregated metrics.
+ * Only emits counters for error types that have non-zero counts.
+ *
+ *
If counters were cached via {@link #initCounters(TaskInputOutputContext)},
+ * uses the cached references for better performance.
+ *
+ * @param context the Hadoop task context
+ */
+ public void emitCounters(TaskInputOutputContext context) {
+ // Use cached counters if available, otherwise look up
+ if (cachedTotalCounter != null) {
+ cachedTotalCounter.increment(totalCount.get());
+ for (ErrorType type : ErrorType.values()) {
+ long count = counts.get(type).get();
+ if (count > 0) {
+ cachedCounters.get(type).increment(count);
+ }
+ }
+ } else {
+ // Fallback to direct lookup
+ context.getCounter(group, NutchMetrics.ERROR_TOTAL).increment(totalCount.get());
+ for (ErrorType type : ErrorType.values()) {
+ long count = counts.get(type).get();
+ if (count > 0) {
+ context.getCounter(group, getCounterName(type)).increment(count);
+ }
+ }
+ }
+ }
+
+ /**
+ * Directly increments cached error counters without local accumulation.
+ *
+ *
Use this method when you want to immediately update Hadoop counters
+ * rather than accumulating locally and emitting in cleanup.
+ * Requires {@link #initCounters(TaskInputOutputContext)} to have been called.
+ *
+ * @param t the throwable to categorize and count
+ * @throws IllegalStateException if counters have not been initialized
+ */
+ public void incrementCounters(Throwable t) {
+ incrementCounters(categorize(t));
+ }
+
+ /**
+ * Directly increments cached error counters without local accumulation.
+ *
+ *
Use this method when you want to immediately update Hadoop counters
+ * rather than accumulating locally and emitting in cleanup.
+ * Requires {@link #initCounters(TaskInputOutputContext)} to have been called.
+ *
+ * @param type the error type to count
+ * @throws IllegalStateException if counters have not been initialized
+ */
+ public void incrementCounters(ErrorType type) {
+ if (cachedTotalCounter == null) {
+ throw new IllegalStateException(
+ "Counters not initialized. Call initCounters() first.");
+ }
+ cachedTotalCounter.increment(1);
+ cachedCounters.get(type).increment(1);
+ }
+
+ /**
+ * Categorizes a throwable into an error type.
+ *
+ *
The categorization checks the exception class hierarchy to determine
+ * the most appropriate category. Timeout exceptions are checked first as
+ * they are a subclass of IOException.
+ *
+ * @param t the throwable to categorize
+ * @return the appropriate ErrorType for the throwable
+ */
+ public static ErrorType categorize(Throwable t) {
+ if (t == null) {
+ return ErrorType.OTHER;
+ }
+
+ String className = t.getClass().getName();
+
+ // Check for timeout first (before general IOException)
+ if (t instanceof SocketTimeoutException
+ || className.contains("TimeoutException")
+ || className.contains("Timeout")) {
+ return ErrorType.TIMEOUT;
+ }
+
+ // Network errors
+ if (t instanceof SocketException
+ || t instanceof UnknownHostException
+ || className.contains("ConnectException")
+ || className.contains("NoRouteToHostException")
+ || className.contains("ConnectionRefusedException")) {
+ return ErrorType.NETWORK;
+ }
+
+ // URL errors (check before general IOException since MalformedURLException extends IOException)
+ if (t instanceof MalformedURLException
+ || className.contains("URLFilterException")
+ || className.contains("URISyntaxException")) {
+ return ErrorType.URL;
+ }
+
+ // General IOException (but not the specific subtypes above)
+ if (t instanceof IOException) {
+ return ErrorType.NETWORK;
+ }
+
+ // Protocol errors
+ if (className.contains("ProtocolException")
+ || className.contains("ProtocolNotFound")) {
+ return ErrorType.PROTOCOL;
+ }
+
+ // Parsing errors
+ if (className.contains("ParseException")
+ || className.contains("ParserNotFound")
+ || className.contains("SAXException")
+ || className.contains("ParserConfigurationException")) {
+ return ErrorType.PARSING;
+ }
+
+ // Scoring errors
+ if (className.contains("ScoringFilterException")) {
+ return ErrorType.SCORING;
+ }
+
+ // Indexing errors
+ if (className.contains("IndexingException")) {
+ return ErrorType.INDEXING;
+ }
+
+ // Check cause chain for more specific categorization
+ Throwable cause = t.getCause();
+ if (cause != null && cause != t) {
+ ErrorType causeType = categorize(cause);
+ if (causeType != ErrorType.OTHER) {
+ return causeType;
+ }
+ }
+
+ return ErrorType.OTHER;
+ }
+
+ /**
+ * Gets the counter name constant for a given error type.
+ *
+ * @param type the error type
+ * @return the counter name constant from NutchMetrics
+ */
+ public static String getCounterName(ErrorType type) {
+ switch (type) {
+ case NETWORK:
+ return NutchMetrics.ERROR_NETWORK_TOTAL;
+ case PROTOCOL:
+ return NutchMetrics.ERROR_PROTOCOL_TOTAL;
+ case PARSING:
+ return NutchMetrics.ERROR_PARSING_TOTAL;
+ case URL:
+ return NutchMetrics.ERROR_URL_TOTAL;
+ case SCORING:
+ return NutchMetrics.ERROR_SCORING_TOTAL;
+ case INDEXING:
+ return NutchMetrics.ERROR_INDEXING_TOTAL;
+ case TIMEOUT:
+ return NutchMetrics.ERROR_TIMEOUT_TOTAL;
+ case OTHER:
+ default:
+ return NutchMetrics.ERROR_OTHER_TOTAL;
+ }
+ }
+
+ /**
+ * Gets the counter name for a throwable based on its categorization.
+ *
+ *
This is a convenience method for direct use in catch blocks:
+ *
+ * } catch (Exception e) {
+ * context.getCounter(group, ErrorTracker.getCounterName(e)).increment(1);
+ * }
+ *
+ *
+ * @param t the throwable to get the counter name for
+ * @return the counter name constant from NutchMetrics
+ */
+ public static String getCounterName(Throwable t) {
+ return getCounterName(categorize(t));
+ }
+}
diff --git a/src/java/org/apache/nutch/metrics/LatencyTracker.java b/src/java/org/apache/nutch/metrics/LatencyTracker.java
new file mode 100644
index 0000000000..3777bb29e3
--- /dev/null
+++ b/src/java/org/apache/nutch/metrics/LatencyTracker.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.metrics;
+
+import org.apache.hadoop.mapreduce.TaskInputOutputContext;
+
+import com.tdunning.math.stats.TDigest;
+
+/**
+ * A utility class for tracking latency metrics using TDigest for percentile
+ * calculation.
+ *
+ * This class wraps a TDigest data structure to collect latency samples and
+ * emit Hadoop counters with count, sum, and percentile values (p50, p95, p99).
+ *
+ *
Usage:
+ *
+ * // In mapper/reducer setup
+ * latencyTracker = new LatencyTracker(NutchMetrics.GROUP_FETCHER, NutchMetrics.FETCHER_LATENCY);
+ *
+ * // During processing
+ * long start = System.currentTimeMillis();
+ * // ... operation ...
+ * latencyTracker.record(System.currentTimeMillis() - start);
+ *
+ * // In cleanup
+ * latencyTracker.emitCounters(context);
+ *
+ *
+ * Emits the following counters:
+ *
+ * - {prefix}_count_total - total number of samples
+ * - {prefix}_sum_ms - sum of all latencies in milliseconds
+ * - {prefix}_p50_ms - 50th percentile (median) latency
+ * - {prefix}_p95_ms - 95th percentile latency
+ * - {prefix}_p99_ms - 99th percentile latency
+ *
+ *
+ * @since 1.22
+ */
+public class LatencyTracker {
+
+ /** Default compression factor for TDigest (controls accuracy vs memory). */
+ private static final double DEFAULT_COMPRESSION = 100.0;
+
+ private final TDigest digest;
+ private final String group;
+ private final String prefix;
+ private long count = 0;
+ private long sum = 0;
+
+ /**
+ * Creates a new LatencyTracker.
+ *
+ * @param group the Hadoop counter group name
+ * @param prefix the prefix for counter names (e.g., "fetch_latency")
+ */
+ public LatencyTracker(String group, String prefix) {
+ this.digest = TDigest.createDigest(DEFAULT_COMPRESSION);
+ this.group = group;
+ this.prefix = prefix;
+ }
+
+ /**
+ * Records a latency sample.
+ *
+ * @param latencyMs the latency in milliseconds
+ */
+ public void record(long latencyMs) {
+ digest.add(latencyMs);
+ count++;
+ sum += latencyMs;
+ }
+
+ /**
+ * Returns the number of recorded samples.
+ *
+ * @return the count of recorded latency samples
+ */
+ public long getCount() {
+ return count;
+ }
+
+ /**
+ * Returns the sum of all recorded latencies.
+ *
+ * @return the sum of latencies in milliseconds
+ */
+ public long getSum() {
+ return sum;
+ }
+
+ /**
+ * Returns the percentile value for the given quantile.
+ *
+ * @param quantile the quantile (0.0 to 1.0)
+ * @return the percentile value in milliseconds
+ */
+ public long getPercentile(double quantile) {
+ if (count == 0) {
+ return 0;
+ }
+ return (long) digest.quantile(quantile);
+ }
+
+ /**
+ * Emits all latency counters to the Hadoop context.
+ *
+ * Should be called once during cleanup to emit aggregated metrics.
+ *
+ * @param context the Hadoop task context
+ */
+ public void emitCounters(TaskInputOutputContext context) {
+ context.getCounter(group, prefix + "_count_total").setValue(count);
+ context.getCounter(group, prefix + "_sum_ms").setValue(sum);
+
+ if (count > 0) {
+ context.getCounter(group, prefix + "_p50_ms").setValue((long) digest.quantile(0.50));
+ context.getCounter(group, prefix + "_p95_ms").setValue((long) digest.quantile(0.95));
+ context.getCounter(group, prefix + "_p99_ms").setValue((long) digest.quantile(0.99));
+ } else {
+ // Set to 0 if no samples recorded
+ context.getCounter(group, prefix + "_p50_ms").setValue(0);
+ context.getCounter(group, prefix + "_p95_ms").setValue(0);
+ context.getCounter(group, prefix + "_p99_ms").setValue(0);
+ }
+ }
+}
+
+
diff --git a/src/java/org/apache/nutch/metrics/NutchMetrics.java b/src/java/org/apache/nutch/metrics/NutchMetrics.java
new file mode 100644
index 0000000000..ccb2d70ed3
--- /dev/null
+++ b/src/java/org/apache/nutch/metrics/NutchMetrics.java
@@ -0,0 +1,717 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.metrics;
+
+import org.apache.hadoop.io.Text;
+import org.apache.nutch.crawl.CrawlDatum;
+import org.apache.nutch.crawl.FetchSchedule;
+
+/**
+ * Centralized constants for Hadoop metrics counter groups and names.
+ *
+ *
Follows Prometheus
+ * naming conventions:
+ *
+ * - Counter groups use the {@code nutch_} prefix namespace
+ * - Counter names use snake_case
+ * - Accumulating counters use {@code _total} suffix
+ * - Units are included in counter names where applicable (e.g., {@code _bytes})
+ *
+ *
+ * @since 1.22
+ */
+public final class NutchMetrics {
+
+ private NutchMetrics() {
+ // Utility class - prevent instantiation
+ }
+
+ // =========================================================================
+ // Counter Groups (Prometheus namespace style with nutch_ prefix)
+ // =========================================================================
+
+ /** Counter group for fetcher operations. */
+ public static final String GROUP_FETCHER = "nutch_fetcher";
+
+ /** Counter group for fetcher outlink processing. */
+ public static final String GROUP_FETCHER_OUTLINKS = "nutch_fetcher_outlinks";
+
+ /** Counter group for generator operations. */
+ public static final String GROUP_GENERATOR = "nutch_generator";
+
+ /** Counter group for indexer operations. */
+ public static final String GROUP_INDEXER = "nutch_indexer";
+
+ /** Counter group for CrawlDb operations. */
+ public static final String GROUP_CRAWLDB = "nutch_crawldb";
+
+ /** Counter group for CrawlDb filter operations. */
+ public static final String GROUP_CRAWLDB_FILTER = "nutch_crawldb_filter";
+
+ /** Counter group for injector operations. */
+ public static final String GROUP_INJECTOR = "nutch_injector";
+
+ /** Counter group for HostDb operations. */
+ public static final String GROUP_HOSTDB = "nutch_hostdb";
+
+ /** Counter group for parser operations. */
+ public static final String GROUP_PARSER = "nutch_parser";
+
+ /** Counter group for deduplication operations. */
+ public static final String GROUP_DEDUP = "nutch_dedup";
+
+ /** Counter group for cleaning job operations. */
+ public static final String GROUP_CLEANING = "nutch_cleaning";
+
+ /** Counter group for WebGraph operations. */
+ public static final String GROUP_WEBGRAPH = "nutch_webgraph";
+
+ /** Counter group for sitemap processing operations. */
+ public static final String GROUP_SITEMAP = "nutch_sitemap";
+
+ /** Counter group for WARC export operations. */
+ public static final String GROUP_WARC_EXPORTER = "nutch_warc_exporter";
+
+ /** Counter group for domain statistics operations. */
+ public static final String GROUP_DOMAIN_STATS = "nutch_domain_stats";
+
+ // =========================================================================
+ // Fetcher Counters
+ // =========================================================================
+
+ /** Total bytes downloaded by fetcher. */
+ public static final String FETCHER_BYTES_DOWNLOADED_TOTAL = "bytes_downloaded_total";
+
+ /** URLs denied by robots.txt. */
+ public static final String FETCHER_ROBOTS_DENIED_TOTAL = "robots_denied_total";
+
+ /** URLs denied due to crawl delay exceeding maximum. */
+ public static final String FETCHER_ROBOTS_DENIED_MAXCRAWLDELAY_TOTAL = "robots_denied_maxcrawldelay_total";
+
+ /** URLs dropped due to robots.txt deferred visits. */
+ public static final String FETCHER_ROBOTS_DEFER_VISITS_DROPPED_TOTAL = "robots_defer_visits_dropped_total";
+
+ /** Redirects that exceeded maximum redirect count. */
+ public static final String FETCHER_REDIRECT_COUNT_EXCEEDED_TOTAL = "redirect_count_exceeded_total";
+
+ /** Redirects deduplicated (already seen). */
+ public static final String FETCHER_REDIRECT_DEDUPLICATED_TOTAL = "redirect_deduplicated_total";
+
+ /** FetchItems not created for redirects. */
+ public static final String FETCHER_REDIRECT_NOT_CREATED_TOTAL = "redirect_not_created_total";
+
+ /** URLs hit by time limit. */
+ public static final String FETCHER_HIT_BY_TIMELIMIT_TOTAL = "hit_by_timelimit_total";
+
+ /** URLs hit by timeout. */
+ public static final String FETCHER_HIT_BY_TIMEOUT_TOTAL = "hit_by_timeout_total";
+
+ /** URLs hit by throughput threshold. */
+ public static final String FETCHER_HIT_BY_THROUGHPUT_THRESHOLD_TOTAL = "hit_by_throughput_threshold_total";
+
+ /** Threads that hung during fetching. */
+ public static final String FETCHER_HUNG_THREADS_TOTAL = "hung_threads_total";
+
+ /** URLs filtered during fetching. */
+ public static final String FETCHER_FILTERED_TOTAL = "filtered_total";
+
+ /** URLs dropped due to exception threshold in queue. */
+ public static final String FETCHER_ABOVE_EXCEPTION_THRESHOLD_TOTAL = "above_exception_threshold_total";
+
+ // =========================================================================
+ // Fetcher Outlinks Counters
+ // =========================================================================
+
+ /** Outlinks detected during parsing. */
+ public static final String FETCHER_OUTLINKS_DETECTED_TOTAL = "outlinks_detected_total";
+
+ /** Outlinks being followed. */
+ public static final String FETCHER_OUTLINKS_FOLLOWING_TOTAL = "outlinks_following_total";
+
+ // =========================================================================
+ // Fetcher Common Crawl extensions
+ // =========================================================================
+
+ /** HTTP protocol version group with dynamic counters. */
+ public static final String FETCHER_HTTP_PROTOCOL_VERSION_GROUP = "http_protocol_version";
+
+ public static final String FETCHER_HTTP_PROTOCOL_UNKNOWN = "unknown";
+
+ /** SSL/TLS protocol version group with dynamic counters. */
+ public static final String FETCHER_TLS_PROTOCOL_VERSION_GROUP = "tls_protocol_version";
+
+ /** IP address version group with two counters: ipv4 and ipv6. */
+ public static final String FETCHER_IP_ADDRESS_VERSION_GROUP = "ip_address_version";
+
+ /** Number of fetches over IPv4. */
+ public static final String FETCHER_IPV4_TOTAL = "ipv4";
+
+ /** Number of fetches over IPv6. */
+ public static final String FETCHER_IPV6_TOTAL = "ipv6";
+
+ /** Archiving of robots.txt captures. */
+ public static final String FETCHER_ROBOTSTXT_ARCHIVING_GROUP = "robotstxt_archiving";
+
+ /** Robots.txt not archived: URL rejected by URL filters. */
+ public static final String FETCHER_ROBOTSTXT_ARCHIVING_FILTERED_TOTAL = "filtered";
+
+ /** Robots.txt not archived: MIME type rejected. */
+ public static final String FETCHER_ROBOTSTXT_ARCHIVING_FILTERED_MIME_TOTAL = "filtered_mime";
+
+ /**
+ * Robots.txt not archived: URL path not /robots.txt and
+ * disallowed by robots.txt.
+ */
+ public static final String FETCHER_ROBOTSTXT_ARCHIVING_ROBOTS_DENIED_TOTAL = "robots_denied";
+
+ // =========================================================================
+ // Common Crawl's WarcWriter
+ // =========================================================================
+
+ /** Counter group for Common Crawl's WARC writer. */
+ public static final String GROUP_WARC_WRITER = "warc_writer";
+
+ /** Skipped records because no content (and protocol status) is available. */
+ public static final String WARC_WRITER_SKIPPED_NO_CONTENT_TOTAL = "skipped_no_content";
+
+ /** Fixed records: invalid URI normalized. */
+ public static final String WARC_WRITER_URI_NORMALIZED_TOTAL = "fixed_uri";
+
+ /** Skipped records because URL is not a valid URI (no WARC-Target-URI). */
+ public static final String WARC_WRITER_SKIPPED_INVALID_URI_TOTAL = "skipped_invalid_uri";
+
+ /** Skipped records by content type / MIME type. */
+ public static final String WARC_WRITER_SKIPPED_BY_CONTENT_TYPE_TOTAL = "skipped_by_content_type";
+
+ /** Skipped duplicate records. */
+ public static final String WARC_WRITER_SKIPPED_DUPLICATE_TOTAL = "skipped_duplicate";
+
+ /** Skipped records: no protocol status. */
+ public static final String WARC_WRITER_SKIPPED_NO_PROTOCOL_STATUS_TOTAL = "skipped_no_protocol_status";
+
+ /** Skipped records: unknown protocol status. */
+ public static final String WARC_WRITER_SKIPPED_UNKNOWN_PROTOCOL_STATUS_TOTAL = "skipped_unknown_protocol_status";
+
+ /** Prefix for error status of language identification (LID), returned by CLD2 Java bindings. */
+ public static final String WARC_WRITER_LID_ERROR_PREFIX = "lid_error: ";
+
+ /** Language identification (LID): no result. */
+ public static final String WARC_WRITER_LID_NO_RESULT_TOTAL = "lid_no_result";
+
+ /** Language identification (LID): result is reliable. */
+ public static final String WARC_WRITER_LID_RESULT_RELIABLE_TOTAL = "lid_reliable";
+
+ /** Language identification (LID): result is not reliable. */
+ public static final String WARC_WRITER_LID_RESULT_NOT_RELIABLE_TOTAL = "lid_not_reliable";
+
+ // =========================================================================
+ // Generator Counters
+ // =========================================================================
+
+ /** URLs rejected by URL filters. */
+ public static final String GENERATOR_URL_FILTERS_REJECTED_TOTAL = "url_filters_rejected_total";
+
+ /** URLs rejected by fetch schedule. */
+ public static final String GENERATOR_SCHEDULE_REJECTED_TOTAL = "schedule_rejected_total";
+
+ /** URLs waiting for CrawlDb update. */
+ public static final String GENERATOR_WAIT_FOR_UPDATE_TOTAL = "wait_for_update_total";
+
+ /** URLs rejected by JEXL expression. */
+ public static final String GENERATOR_EXPR_REJECTED_TOTAL = "expr_rejected_total";
+
+ /** URLs rejected due to status restriction. */
+ public static final String GENERATOR_STATUS_REJECTED_TOTAL = "status_rejected_total";
+
+ /** URLs rejected due to score below threshold. */
+ public static final String GENERATOR_SCORE_TOO_LOW_TOTAL = "score_too_low_total";
+
+ /** URLs rejected due to fetch interval exceeding threshold. */
+ public static final String GENERATOR_INTERVAL_REJECTED_TOTAL = "interval_rejected_total";
+
+ /** URLs skipped due to per-host overflow. */
+ public static final String GENERATOR_URLS_SKIPPED_PER_HOST_OVERFLOW_TOTAL = "urls_skipped_per_host_overflow_total";
+
+ /** Hosts affected by per-host overflow. */
+ public static final String GENERATOR_HOSTS_AFFECTED_PER_HOST_OVERFLOW_TOTAL = "hosts_affected_per_host_overflow_total";
+
+ // =========================================================================
+ // Generator2-specific Counters
+ // =========================================================================
+
+ /** Domains affected by per-domain overflow. All remaining URLs of this domain have been skipped, but were not counted. */
+ public static final String GENERATOR_DOMAINS_AFFECTED_PER_DOMAIN_OVERFLOW_TOTAL = "domains_affected_per_domain_overflow_total";
+
+ /** Domains affected by max. number of hosts per domain overflow. URLs from further hosts below this domain have been skipped. */
+ public static final String GENERATOR_DOMAINS_AFFECTED_PER_MAX_NUM_HOSTS_OVERFLOW_TOTAL = "domains_affected_num_hosts_overflow_total";
+
+ /** URLs skipped due to the max. number of hosts per domain overflow. */
+ public static final String GENERATOR_URLS_SKIPPED_PER_MAX_NUM_HOSTS_OVERFLOW_TOTAL = "urls_skipped_per_max_num_host_overflow_total";
+
+ /** URLs skipped due to per-segment overflow. */
+ public static final String GENERATOR_URLS_SKIPPED_PER_SEGMENT_OVERFLOW_TOTAL = "urls_skipped_per_segment_overflow_total";
+
+ /**
+ * Counter group for items by status, rejected by the fetch schedule. See
+ * {@link FetchSchedule#shouldFetch(Text, CrawlDatum, long)}.
+ */
+ public static final String GROUP_GENERATOR_SCHEDULE_REJECTED_BY_STATUS = "schedule_rejected_by_status";
+
+ /**
+ * Counter group for items by status, rejected because the generator score is
+ * lower than the minimum score defined per generate.min.score.
+ */
+ public static final String GROUP_GENERATOR_SCORE_REJECTED_BY_STATUS = "score_rejected_by_status";
+
+ /** Counter group for items by status, selected for fetch. */
+ public static final String GROUP_GENERATOR_SELECTED_BY_STATUS = "selected_by_status";
+
+ // =========================================================================
+ // Indexer Counters
+ // =========================================================================
+
+ /** Documents deleted due to robots noindex. */
+ public static final String INDEXER_DELETED_ROBOTS_NOINDEX_TOTAL = "deleted_robots_noindex_total";
+
+ /** Documents deleted because they are gone. */
+ public static final String INDEXER_DELETED_GONE_TOTAL = "deleted_gone_total";
+
+ /** Documents deleted due to redirects. */
+ public static final String INDEXER_DELETED_REDIRECTS_TOTAL = "deleted_redirects_total";
+
+ /** Documents deleted as duplicates. */
+ public static final String INDEXER_DELETED_DUPLICATES_TOTAL = "deleted_duplicates_total";
+
+ /** Documents deleted by indexing filter. */
+ public static final String INDEXER_DELETED_BY_INDEXING_FILTER_TOTAL = "deleted_by_indexing_filter_total";
+
+ /** Documents skipped (not modified). */
+ public static final String INDEXER_SKIPPED_NOT_MODIFIED_TOTAL = "skipped_not_modified_total";
+
+ /** Documents skipped by indexing filter. */
+ public static final String INDEXER_SKIPPED_BY_INDEXING_FILTER_TOTAL = "skipped_by_indexing_filter_total";
+
+ /** Documents indexed (added or updated). */
+ public static final String INDEXER_INDEXED_TOTAL = "indexed_total";
+
+ // =========================================================================
+ // CrawlDb Counters
+ // =========================================================================
+
+ /** URLs filtered during CrawlDb operations. */
+ public static final String CRAWLDB_URLS_FILTERED_TOTAL = "urls_filtered_total";
+
+ /** Gone (404) records removed during CrawlDb operations. */
+ public static final String CRAWLDB_GONE_RECORDS_REMOVED_TOTAL = "gone_records_removed_total";
+
+ /** Orphan records removed during CrawlDb operations. */
+ public static final String CRAWLDB_ORPHAN_RECORDS_REMOVED_TOTAL = "orphan_records_removed_total";
+
+ // =========================================================================
+ // Injector Counters
+ // =========================================================================
+
+ /** URLs filtered during injection. */
+ public static final String INJECTOR_URLS_FILTERED_TOTAL = "urls_filtered_total";
+
+ /** URLs injected. */
+ public static final String INJECTOR_URLS_INJECTED_TOTAL = "urls_injected_total";
+
+ /** Unique URLs injected. */
+ public static final String INJECTOR_URLS_INJECTED_UNIQUE_TOTAL = "urls_injected_unique_total";
+
+ /** URLs merged with existing CrawlDb entries. */
+ public static final String INJECTOR_URLS_MERGED_TOTAL = "urls_merged_total";
+
+ /** URLs purged due to 404 status. */
+ public static final String INJECTOR_URLS_PURGED_404_TOTAL = "urls_purged_404_total";
+
+ /** URLs purged by filter. */
+ public static final String INJECTOR_URLS_PURGED_FILTER_TOTAL = "urls_purged_filter_total";
+
+ // =========================================================================
+ // HostDb Counters
+ // =========================================================================
+
+ /** Records filtered in HostDb. */
+ public static final String HOSTDB_FILTERED_RECORDS_TOTAL = "filtered_records_total";
+
+ /** Total hosts processed. */
+ public static final String HOSTDB_TOTAL_HOSTS_TOTAL = "total_hosts_total";
+
+ /** Hosts skipped (not eligible). */
+ public static final String HOSTDB_SKIPPED_NOT_ELIGIBLE_TOTAL = "skipped_not_eligible_total";
+
+ /** Hosts where URL limit was not reached. */
+ public static final String HOSTDB_URL_LIMIT_NOT_REACHED_TOTAL = "url_limit_not_reached_total";
+
+ /** New known hosts discovered. */
+ public static final String HOSTDB_NEW_KNOWN_HOST_TOTAL = "new_known_host_total";
+
+ /** Rediscovered hosts. */
+ public static final String HOSTDB_REDISCOVERED_HOST_TOTAL = "rediscovered_host_total";
+
+ /** Existing known hosts. */
+ public static final String HOSTDB_EXISTING_KNOWN_HOST_TOTAL = "existing_known_host_total";
+
+ /** New unknown hosts. */
+ public static final String HOSTDB_NEW_UNKNOWN_HOST_TOTAL = "new_unknown_host_total";
+
+ /** Existing unknown hosts. */
+ public static final String HOSTDB_EXISTING_UNKNOWN_HOST_TOTAL = "existing_unknown_host_total";
+
+ /** Purged unknown hosts. */
+ public static final String HOSTDB_PURGED_UNKNOWN_HOST_TOTAL = "purged_unknown_host_total";
+
+ /** Hosts checked. */
+ public static final String HOSTDB_CHECKED_HOSTS_TOTAL = "checked_hosts_total";
+
+ // =========================================================================
+ // Deduplication Counters
+ // =========================================================================
+
+ /** Documents marked as duplicate. */
+ public static final String DEDUP_DOCUMENTS_MARKED_DUPLICATE_TOTAL = "documents_marked_duplicate_total";
+
+ // =========================================================================
+ // Redirect Deduplication Counters
+ // =========================================================================
+
+ /** Redirects kept as non-duplicates. */
+ public static final String DEDUP_REDIRECTS_NOT_DUPLICATES_TOTAL = "redirects_marked_not_duplicate_total";
+
+ /** Redirects in CrawlDb. */
+ public static final String DEDUP_REDIRECTS_IN_CRAWLDB_TOTAL = "redirects_in_crawldb_total";
+
+ /** Self-referential redirects in CrawlDb. */
+ public static final String DEDUP_REDIRECTS_SELF_REFERENTIAL_TOTAL = "redirects_self_referential_total";
+
+ /** Self-referential redirects kept as non-duplicates. */
+ public static final String DEDUP_REDIRECTS_SELF_REFERENTIAL_NOT_DUPLICATES_TOTAL = "redirects_self_referential_marked_not_duplicate_total";
+
+ // =========================================================================
+ // Cleaning Job Counters
+ // =========================================================================
+
+ /** Documents deleted during cleaning. */
+ public static final String CLEANING_DELETED_DOCUMENTS_TOTAL = "deleted_documents_total";
+
+ // =========================================================================
+ // WebGraph Counters
+ // =========================================================================
+
+ /** Links added to WebGraph. */
+ public static final String WEBGRAPH_ADDED_LINKS_TOTAL = "added_links_total";
+
+ /** Links removed from WebGraph. */
+ public static final String WEBGRAPH_REMOVED_LINKS_TOTAL = "removed_links_total";
+
+ // =========================================================================
+ // Sitemap Counters
+ // =========================================================================
+
+ /** Filtered records in sitemap processing. */
+ public static final String SITEMAP_FILTERED_RECORDS_TOTAL = "filtered_records_total";
+
+ /** Seeds extracted from sitemaps. */
+ public static final String SITEMAP_SEEDS_TOTAL = "sitemap_seeds_total";
+
+ /** Sitemaps discovered from hostname. */
+ public static final String SITEMAP_FROM_HOSTNAME_TOTAL = "sitemaps_from_hostname_total";
+
+ /** Sitemaps filtered from hostname. */
+ public static final String SITEMAP_FILTERED_FROM_HOSTNAME_TOTAL = "filtered_sitemaps_from_hostname_total";
+
+ /** Failed sitemap fetches. */
+ public static final String SITEMAP_FAILED_FETCHES_TOTAL = "failed_fetches_total";
+
+ /** Existing sitemap entries. */
+ public static final String SITEMAP_EXISTING_ENTRIES_TOTAL = "existing_sitemap_entries_total";
+
+ /** New sitemap entries. */
+ public static final String SITEMAP_NEW_ENTRIES_TOTAL = "new_sitemap_entries_total";
+
+ // =========================================================================
+ // SitemapInjector Counters
+ // =========================================================================
+
+ /** SitemapInjector counter group. */
+ public static final String GROUP_SITEMAP_INJECTOR = "sitemap_injector";
+
+ /** Failed to fetch sitemap content, disallowed per robots.txt. */
+ public static final String SITEMAP_ROBOTSTXT_DISALLOW_TOTAL = "sitemap_robotstxt_disallow";
+
+ /** Sitemap failed to parse. */
+ public static final String SITEMAP_FAILED_TO_PARSE_TOTAL = "sitemaps_failed_to_parse";
+
+ /** Prefix for sitemap type counter. */
+ public static final String SITEMAP_TYPE_PREFIX = "sitemap_type_";
+
+ /** Sitemaps processed total. */
+ public static final String SITEMAP_PROCESSED_TOTAL = "sitemaps_processed";
+
+ /** Sitemap index: affected by URL limit. */
+ public static final String SITEMAP_INDEX_AFFECTED_BY_URL_LIMIT_TOTAL = "sitemap_index_url_limit";
+
+ /** Sitemap index: affected by depth limit. */
+ public static final String SITEMAP_INDEX_AFFECTED_BY_DEPTH_LIMIT_TOTAL = "sitemap_index_depth_limit";
+
+ /** Sitemap index: affected by time limit. */
+ public static final String SITEMAP_INDEX_AFFECTED_BY_TIME_LIMIT_TOTAL = "sitemap_index_time_limit";
+
+ /** Sitemap index: skipped because no URLs found after 50% of time limit. */
+ public static final String SITEMAP_INDEX_NO_URLS_AFTER_50_PERCENT_OF_TIME_LIMIT_TOTAL = "sitemap_index_no_urls_after_50_percent_of_time_limit";
+
+ /** Sitemap index: skipped because of too many fetch failures. */
+ public static final String SITEMAP_INDEX_TOO_MANY_FAILURES_TOTAL = "sitemap_index_too_many_failures";
+
+ /** Sitemap index: processed sitemaps. */
+ public static final String SITEMAP_INDEX_PROCESSED_SITEMAPS_TOTAL = "sitemap_index_processed_sitemaps";
+
+ /** Skipped duplicated or recursive sitemap URLs. */
+ public static final String SITEMAP_SKIPPED_DUPLICATE_OR_RECURSIVE_URL_TOTAL = "sitemap_skipped_duplicate_or_recursive_sitemap_url";
+
+ /** Sitemap index: affected by max. number of sitemaps in index. */
+ public static final String SITEMAP_INDEX_MAX_SITEMAPS_LIMIT_TOTAL = "sitemap_index_max_sitemaps_limit";
+
+ /** Sitemap failed to fetch. */
+ public static final String SITEMAP_FAILED_TO_FETCH_TOTAL = "sitemap_failed_to_fetch";
+
+ /** Sitemap skipped because of overlong URL. */
+ public static final String SITEMAP_SKIPPED_OVERLONG_URL_TOTAL = "sitemap_skipped_overlong_url";
+
+ /** Sitemap rejected by URL filters */
+ public static final String SITEMAP_REJECTED_BY_URL_FILTERS_TOTAL = "sitemap_rejected_by_url_filters";
+
+ /** Sitemap skipped, too many failures per host. */
+ public static final String SITEMAP_SKIPPED_TOO_MANY_FAILURES_PER_HOST_TOTAL = "sitemap_skipped_too_many_failures_per_host";
+
+ /** Could not fetch sitemap content, protocol not supported. */
+ public static final String SITEMAP_PROTOCOL_NOT_SUPPORTED_TOTAL = "sitemap_protocol_not_supported";
+
+ /** Failed to fetch sitemap content because of timeout. */
+ public static final String SITEMAP_FAILED_TO_FETCH_TIMEOUT_TOTAL = "sitemap_failed_to_fetch_timeout";
+
+ /** Failed to fetch sitemap content because of exception. */
+ public static final String SITEMAP_FAILED_TO_FETCH_EXCEPTION_TOTAL = "sitemap_failed_to_fetch_exception";
+
+ /** Sitemap redirect. */
+ public static final String SITEMAP_REDIRECT_TOTAL = "sitemap_redirect";
+
+ /** Sitemap redirect target rejected by URL filters */
+ public static final String SITEMAP_REDIRECT_TARGET_REJECTED_BY_URL_FILTERS_TOTAL = "sitemap_redirect_target_rejected_by_url_filters";
+
+ /** Sitemap redirect limit exceeded (max. number of redirects followed). */
+ public static final String SITEMAP_REDIRECT_LIMIT_EXCEEDED_TOTAL = "sitemap_redirect_limit_exceeded";
+
+ /** Failed to fetch sitemap content, HTTP status != 200. */
+ public static final String SITEMAP_FAILED_TO_FETCH_CONTENT_HTTP_STATUS_CODE_NOT_200_TOTAL = "sitemap_failed_to_fetch_http_status_code_not_200";
+
+ /** Failed to fetch sitemap content, empty content. */
+ public static final String SITEMAP_EMPTY_CONTENT_TOTAL = "sitemap_empty_content";
+
+ /** Empty sitemap. */
+ public static final String SITEMAP_EMPTY_TOTAL = "sitemap_empty";
+
+ /** Sitemap URL limit reached. */
+ public static final String SITEMAP_URL_LIMIT_REACHED_TOTAL = "sitemap_url_limit_reached";
+
+ /** URLs randomly skipped. */
+ public static final String SITEMAP_RANDOM_SKIP_TOTAL = "urls_random_skip";
+
+ /** URLs from sitemaps rejected, host limit reached. */
+ public static final String SITEMAP_URLS_SKIPPED_HOST_LIMIT_REACHED_TOTAL = "urls_skipped_host_limit_reached";
+
+ /** URLs from sitemaps rejected, target not allowed by cross-submit. */
+ public static final String SITEMAP_URLS_SKIPPED_NOT_ALLOWED_BY_CROSS_SUBMITS_TOTAL = "urls_skipped_not_allowed_by_cross_submits";
+
+ /** URLs from sitemaps rejected by URL filters. */
+ public static final String SITEMAP_URLS_FROM_REJECTED_BY_URL_FILTERS = "urls_from_sitemaps_rejected_by_url_filters";
+
+ /** URLs from sitemaps injected. */
+ public static final String SITEMAP_URLS_INJECTED = "urls_from_sitemaps_injected";
+
+ // =========================================================================
+ // WARC Exporter Counters
+ // =========================================================================
+
+ /** Missing content in WARC export. */
+ public static final String WARC_MISSING_CONTENT_TOTAL = "missing_content_total";
+
+ /** Missing metadata in WARC export. */
+ public static final String WARC_MISSING_METADATA_TOTAL = "missing_metadata_total";
+
+ /** Omitted empty responses in WARC export. */
+ public static final String WARC_OMITTED_EMPTY_RESPONSE_TOTAL = "omitted_empty_response_total";
+
+ /** WARC records generated. */
+ public static final String WARC_RECORDS_GENERATED_TOTAL = "records_generated_total";
+
+ // =========================================================================
+ // Domain Statistics Counters (enum-based, kept for compatibility)
+ // =========================================================================
+
+ /** Fetched URLs in domain statistics. */
+ public static final String DOMAIN_STATS_FETCHED_TOTAL = "fetched_total";
+
+ /** Not fetched URLs in domain statistics. */
+ public static final String DOMAIN_STATS_NOT_FETCHED_TOTAL = "not_fetched_total";
+
+ /** Empty results in domain statistics. */
+ public static final String DOMAIN_STATS_EMPTY_RESULT_TOTAL = "empty_result_total";
+
+ // =========================================================================
+ // UrlCleaner
+ // =========================================================================
+
+ public static final String GROUP_URLCLEANER = "urlcleaner";
+
+ public static final String URLCLEANER_REJECTED_TOTAL = "urls_rejected";
+
+ public static final String URLCLEANER_REJECTED_INVALID_DOMAIN_TOTAL = "urls_rejected_invalid_domain";
+
+ public static final String URLCLEANER_ACCEPTED_UNCHANGED_TOTAL = "urls_accepted_unchanged";
+
+ public static final String URLCLEANER_ACCEPTED_NORMALIZED_TOTAL = "urls_accepted_normalized";
+
+ // =========================================================================
+ // UrlSampler and UrlSamplerHost
+ // =========================================================================
+
+ public static final String GROUP_URLSAMPLER = "urlsampler";
+
+ public static final String GROUP_URLSAMPLER_HOST = "urlsamplerhost";
+
+ public static final String URLSAMPLER_MALFORMED_URL_TOTAL = "malformed_url";
+
+ public static final String URLSAMPLER_SKIPPED_MAX_URLS_TOTAL = "skipped_max_urls";
+
+ public static final String URLSAMPLER_SKIPPED_MAX_URLS_PER_HOST_TOTAL = "skipped_max_urls_per_host";
+
+ public static final String URLSAMPLER_SKIPPED_MAX_HOSTS_TOTAL = "skipped_max_hosts";
+
+ public static final String URLSAMPLER_HOSTS = "hosts";
+
+ public static final String URLSAMPLER_URLS = "urls";
+
+ public static final String URLSAMPLER_HOSTS_WITH_LIMIT = "hosts_with_limit";
+
+ public static final String URLSAMPLER_URLS_HOST_WITH_LIMIT = "urls_host_with_limit";
+
+ public static final String URLSAMPLER_HOSTS_WITHOUT_LIMIT = "hosts_without_limit";
+
+ public static final String URLSAMPLER_URLS_HOST_WITHOUT_LIMIT = "urls_host_without_limit";
+
+ public static final String URLSAMPLER_URLS_SAMPLED = "urls_sampled";
+
+ public static final String URLSAMPLER_HOSTS_SAMPLED = "hosts_sampled";
+
+ public static final String URLSAMPLER_HOSTS_WITH_LIMIT_SAMPLED = "hosts_with_limit_sampled";
+
+ public static final String URLSAMPLER_URLS_HOST_WITH_LIMIT_SAMPLED = "urls_host_with_limit_sampled";
+
+ public static final String URLSAMPLER_HOSTS_WITHOUT_LIMIT_SAMPLED = "hosts_without_limit_sampled";
+
+ public static final String URLSAMPLER_URLS_HOST_WITHOUT_LIMIT_SAMPLED = "urls_host_without_limit_sampled";
+
+ public static final String URLSAMPLER_SKIPPED_MAX_URLS_PER_HOST = "skipped_max_urls_per_host";
+
+ public static final String URLSAMPLER_SKIPPED_RANDOM = "skipped_random";
+
+ // =========================================================================
+ // Latency Metric Prefixes (used with LatencyTracker)
+ // =========================================================================
+
+ /**
+ * Prefix for fetch latency metrics.
+ * Used with {@link LatencyTracker} to emit fetch timing counters.
+ */
+ public static final String FETCHER_LATENCY = "fetch_latency";
+
+ /**
+ * Prefix for parse latency metrics.
+ * Used with {@link LatencyTracker} to emit parse timing counters.
+ */
+ public static final String PARSER_LATENCY = "parse_latency";
+
+ /**
+ * Prefix for indexer latency metrics.
+ * Used with {@link LatencyTracker} to emit indexing timing counters.
+ */
+ public static final String INDEXER_LATENCY = "index_latency";
+
+ // =========================================================================
+ // Common Error Counter Names (used with component-specific groups)
+ // These constants are shared across all components for consistent error
+ // categorization. Use with ErrorTracker for automatic classification.
+ // =========================================================================
+
+ /**
+ * Total errors across all categories.
+ * This is incremented alongside any category-specific error counter.
+ */
+ public static final String ERROR_TOTAL = "errors_total";
+
+ /**
+ * Network-related errors.
+ * Includes: IOException, SocketException, ConnectException, UnknownHostException
+ */
+ public static final String ERROR_NETWORK_TOTAL = "errors_network_total";
+
+ /**
+ * Protocol errors.
+ * Includes: ProtocolException, ProtocolNotFound
+ */
+ public static final String ERROR_PROTOCOL_TOTAL = "errors_protocol_total";
+
+ /**
+ * Parsing errors.
+ * Includes: ParseException, ParserNotFound
+ */
+ public static final String ERROR_PARSING_TOTAL = "errors_parsing_total";
+
+ /**
+ * URL-related errors.
+ * Includes: MalformedURLException, URLFilterException
+ */
+ public static final String ERROR_URL_TOTAL = "errors_url_total";
+
+ /**
+ * Scoring filter errors.
+ * Includes: ScoringFilterException
+ */
+ public static final String ERROR_SCORING_TOTAL = "errors_scoring_total";
+
+ /**
+ * Indexing filter errors.
+ * Includes: IndexingException
+ */
+ public static final String ERROR_INDEXING_TOTAL = "errors_indexing_total";
+
+ /**
+ * Timeout errors.
+ * Includes: SocketTimeoutException, connection timeouts
+ */
+ public static final String ERROR_TIMEOUT_TOTAL = "errors_timeout_total";
+
+ /**
+ * Other uncategorized errors.
+ * Used as fallback for exceptions not matching any specific category.
+ */
+ public static final String ERROR_OTHER_TOTAL = "errors_other_total";
+}
+
diff --git a/src/java/org/apache/nutch/metrics/package-info.java b/src/java/org/apache/nutch/metrics/package-info.java
new file mode 100644
index 0000000000..376605d043
--- /dev/null
+++ b/src/java/org/apache/nutch/metrics/package-info.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Metrics infrastructure for Apache Nutch.
+ *
+ * This package provides centralized constants and utilities for Hadoop
+ * MapReduce metrics/counters following
+ * Prometheus naming
+ * conventions.
+ *
+ *
The main class is {@link org.apache.nutch.metrics.NutchMetrics} which
+ * defines all counter group names and counter names as constants.
+ *
+ * @since 1.22
+ */
+package org.apache.nutch.metrics;
+
diff --git a/src/java/org/apache/nutch/parse/ParseSegment.java b/src/java/org/apache/nutch/parse/ParseSegment.java
index 6b2fb5cee7..0b2a6f2290 100644
--- a/src/java/org/apache/nutch/parse/ParseSegment.java
+++ b/src/java/org/apache/nutch/parse/ParseSegment.java
@@ -37,6 +37,9 @@
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.nutch.metadata.Metadata;
import org.apache.nutch.metadata.Nutch;
+import org.apache.nutch.metrics.ErrorTracker;
+import org.apache.nutch.metrics.LatencyTracker;
+import org.apache.nutch.metrics.NutchMetrics;
import org.apache.nutch.net.protocols.Response;
import org.apache.nutch.protocol.Content;
import org.apache.nutch.scoring.ScoringFilterException;
@@ -80,12 +83,25 @@ public static class ParseSegmentMapper extends
private Text newKey = new Text();
private ScoringFilters scfilters;
private boolean skipTruncated;
+ private LatencyTracker parseLatencyTracker;
+ private ErrorTracker errorTracker;
@Override
public void setup(Mapper, Content, Text, ParseImpl>.Context context) {
Configuration conf = context.getConfiguration();
scfilters = new ScoringFilters(conf);
skipTruncated = conf.getBoolean(SKIP_TRUNCATED, true);
+ parseLatencyTracker = new LatencyTracker(
+ NutchMetrics.GROUP_PARSER, NutchMetrics.PARSER_LATENCY);
+ // Initialize error tracker with cached counters
+ errorTracker = new ErrorTracker(NutchMetrics.GROUP_PARSER, context);
+ }
+
+ @Override
+ public void cleanup(Mapper, Content, Text, ParseImpl>.Context context)
+ throws IOException, InterruptedException {
+ // Emit parse latency metrics
+ parseLatencyTracker.emitCounters(context);
}
@Override
@@ -121,6 +137,7 @@ public void map(WritableComparable key, Content content,
parseResult = parseUtil.parse(content);
} catch (Exception e) {
LOG.warn("Error parsing: {}: {}", key, StringUtils.stringifyException(e));
+ errorTracker.incrementCounters(e);
return;
}
@@ -129,7 +146,8 @@ public void map(WritableComparable key, Content content,
Parse parse = entry.getValue();
ParseStatus parseStatus = parse.getData().getStatus();
- context.getCounter("ParserStatus",
+ // Dynamic counter based on parse status
+ context.getCounter(NutchMetrics.GROUP_PARSER,
ParseStatus.majorCodes[parseStatus.getMajorCode()]).increment(1);
if (!parseStatus.isSuccess()) {
@@ -151,10 +169,13 @@ public void map(WritableComparable key, Content content,
scfilters.passScoreAfterParsing(url, content, parse);
} catch (ScoringFilterException e) {
LOG.warn("Error passing score: {}: {}", url, e.getMessage());
+ errorTracker.incrementCounters(ErrorTracker.ErrorType.SCORING);
}
long end = System.currentTimeMillis();
- LOG.info("Parsed ({}ms): {}", (end - start), url);
+ long parseTime = end - start;
+ parseLatencyTracker.record(parseTime);
+ LOG.info("Parsed ({}ms): {}", parseTime, url);
context.write(
url,
diff --git a/src/java/org/apache/nutch/protocol/Protocol.java b/src/java/org/apache/nutch/protocol/Protocol.java
index ab4162c87f..2514eae33e 100644
--- a/src/java/org/apache/nutch/protocol/Protocol.java
+++ b/src/java/org/apache/nutch/protocol/Protocol.java
@@ -16,6 +16,7 @@
*/
package org.apache.nutch.protocol;
+import java.net.URL;
import java.util.List;
import org.apache.hadoop.conf.Configurable;
@@ -57,4 +58,24 @@ public interface Protocol extends Pluggable, Configurable {
BaseRobotRules getRobotRules(Text url, CrawlDatum datum,
List robotsTxtContent);
+ /**
+ * Retrieve robot rules applicable for this URL.
+ *
+ * @param url
+ * URL to check
+ * @param datum
+ * page datum
+ * @param robotsTxtContent
+ * container to store responses when fetching the robots.txt file for
+ * debugging or archival purposes. Instead of a robots.txt file, it
+ * may include redirects or an error page (404, etc.). Response
+ * {@link Content} is appended to the passed list. If null is passed
+ * nothing is stored.
+ * @return robot rules (specific for this URL or default), never null
+ */
+ default BaseRobotRules getRobotRules(URL url, CrawlDatum datum,
+ List robotsTxtContent) {
+ return getRobotRules(new Text(url.toString()), datum, robotsTxtContent);
+ }
+
}
diff --git a/src/java/org/apache/nutch/scoring/webgraph/WebGraph.java b/src/java/org/apache/nutch/scoring/webgraph/WebGraph.java
index 4daefcd8f3..fee0921d0a 100644
--- a/src/java/org/apache/nutch/scoring/webgraph/WebGraph.java
+++ b/src/java/org/apache/nutch/scoring/webgraph/WebGraph.java
@@ -48,6 +48,7 @@
import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.Counter;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.output.MapFileOutputFormat;
import org.apache.hadoop.mapreduce.Mapper;
@@ -58,6 +59,7 @@
import org.apache.nutch.crawl.NutchWritable;
import org.apache.nutch.crawl.CrawlDatum;
import org.apache.nutch.metadata.Nutch;
+import org.apache.nutch.metrics.NutchMetrics;
import org.apache.nutch.net.URLFilters;
import org.apache.nutch.net.URLNormalizers;
import org.apache.nutch.parse.Outlink;
@@ -327,6 +329,10 @@ public static class OutlinkDbReducer extends
// url normalizers, filters and job configuration
private Configuration conf;
+ // Cached counter references for performance
+ private Counter addedLinksCounter;
+ private Counter removedLinksCounter;
+
/**
* Configures the OutlinkDb job reducer. Sets up internal links and link limiting.
*/
@@ -339,6 +345,18 @@ public void setup(Reducer.Context context)
limitPages = conf.getBoolean("link.ignore.limit.page", true);
limitDomains = conf.getBoolean("link.ignore.limit.domain", true);
+ // Initialize cached counter references
+ initCounters(context);
+ }
+
+ /**
+ * Initialize cached counter references to avoid repeated lookups in hot paths.
+ */
+ private void initCounters(Context context) {
+ addedLinksCounter = context.getCounter(
+ NutchMetrics.GROUP_WEBGRAPH, NutchMetrics.WEBGRAPH_ADDED_LINKS_TOTAL);
+ removedLinksCounter = context.getCounter(
+ NutchMetrics.GROUP_WEBGRAPH, NutchMetrics.WEBGRAPH_REMOVED_LINKS_TOTAL);
}
@Override
@@ -361,14 +379,14 @@ public void reduce(Text key, Iterable values,
mostRecent = timestamp;
}
outlinkList.add(WritableUtils.clone(next, conf));
- context.getCounter("WebGraph.outlinks", "added links").increment(1);
+ addedLinksCounter.increment(1);
} else if (value instanceof BooleanWritable) {
BooleanWritable delete = (BooleanWritable) value;
// Actually, delete is always true, otherwise we don't emit it in the
// mapper in the first place
if (delete.get() == true) {
// This page is gone, do not emit it's outlinks
- context.getCounter("WebGraph.outlinks", "removed links").increment(1);
+ removedLinksCounter.increment(1);
return;
}
}
diff --git a/src/java/org/apache/nutch/tools/warc/WARCExporter.java b/src/java/org/apache/nutch/tools/warc/WARCExporter.java
index bf824f9b3f..14b59ac85c 100644
--- a/src/java/org/apache/nutch/tools/warc/WARCExporter.java
+++ b/src/java/org/apache/nutch/tools/warc/WARCExporter.java
@@ -41,6 +41,7 @@
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.Counter;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.Job;
@@ -57,6 +58,8 @@
import org.apache.nutch.parse.ParseText;
import org.apache.nutch.protocol.Content;
import org.apache.nutch.tools.WARCUtils;
+import org.apache.nutch.metrics.ErrorTracker;
+import org.apache.nutch.metrics.NutchMetrics;
import org.apache.nutch.util.HadoopFSUtil;
import org.apache.nutch.util.NutchConfiguration;
import org.apache.nutch.util.NutchJob;
@@ -111,6 +114,35 @@ public static class WARCReducer
// Metadata to JSON
Gson gson = new Gson();
+ // Cached counter references to avoid repeated lookups in hot paths
+ private Counter missingContentCounter;
+ private Counter missingMetadataCounter;
+ private Counter omittedEmptyResponseCounter;
+ private Counter recordsGeneratedCounter;
+ private ErrorTracker errorTracker;
+
+ @Override
+ public void setup(Context context) {
+ // Initialize cached counter references
+ initCounters(context);
+ // Initialize error tracker with cached counters
+ errorTracker = new ErrorTracker(NutchMetrics.GROUP_WARC_EXPORTER, context);
+ }
+
+ /**
+ * Initialize cached counter references to avoid repeated lookups in hot paths.
+ */
+ private void initCounters(Context context) {
+ missingContentCounter = context.getCounter(
+ NutchMetrics.GROUP_WARC_EXPORTER, NutchMetrics.WARC_MISSING_CONTENT_TOTAL);
+ missingMetadataCounter = context.getCounter(
+ NutchMetrics.GROUP_WARC_EXPORTER, NutchMetrics.WARC_MISSING_METADATA_TOTAL);
+ omittedEmptyResponseCounter = context.getCounter(
+ NutchMetrics.GROUP_WARC_EXPORTER, NutchMetrics.WARC_OMITTED_EMPTY_RESPONSE_TOTAL);
+ recordsGeneratedCounter = context.getCounter(
+ NutchMetrics.GROUP_WARC_EXPORTER, NutchMetrics.WARC_RECORDS_GENERATED_TOTAL);
+ }
+
@Override
public void reduce(Text key, Iterable values,
Context context) throws IOException, InterruptedException {
@@ -147,13 +179,13 @@ public void reduce(Text key, Iterable values,
// check that we have everything we need
if (content == null) {
LOG.info("Missing content for {}", key);
- context.getCounter("WARCExporter", "missing content").increment(1);
+ missingContentCounter.increment(1);
return;
}
if (cd == null) {
LOG.info("Missing fetch datum for {}", key);
- context.getCounter("WARCExporter", "missing metadata").increment(1);
+ missingMetadataCounter.increment(1);
return;
}
@@ -161,8 +193,7 @@ public void reduce(Text key, Iterable values,
// Empty responses is everything that was not a regular response
if (!(cd.getStatus() == CrawlDatum.STATUS_FETCH_SUCCESS
|| cd.getStatus() == CrawlDatum.STATUS_FETCH_NOTMODIFIED)) {
- context.getCounter("WARCExporter", "omitted empty response")
- .increment(1);
+ omittedEmptyResponseCounter.increment(1);
return;
}
}
@@ -237,7 +268,7 @@ public void reduce(Text key, Iterable values,
.append(uri.toASCIIString()).append(CRLF);
} catch (Exception e) {
LOG.error("Invalid URI {} ", key);
- context.getCounter("WARCExporter", "invalid URI").increment(1);
+ errorTracker.incrementCounters(e);
return;
}
@@ -269,12 +300,12 @@ public void reduce(Text key, Iterable values,
new ByteArrayInputStream(bos.toByteArray()));
WARCRecord record = new WARCRecord(in);
context.write(NullWritable.get(), new WARCWritable(record));
- context.getCounter("WARCExporter", "records generated").increment(1);
+ recordsGeneratedCounter.increment(1);
} catch (IOException | IllegalStateException exception) {
LOG.error(
"Exception when generating WARC resource record for {} : {}", key,
exception.getMessage());
- context.getCounter("WARCExporter", "exception").increment(1);
+ errorTracker.incrementCounters(exception);
}
// Do we need to emit a metadata record too?
@@ -316,7 +347,7 @@ public void reduce(Text key, Iterable values,
.append(uri.toASCIIString()).append(CRLF);
} catch (Exception e) {
LOG.error("Invalid URI {} ", key);
- context.getCounter("WARCExporter", "invalid URI").increment(1);
+ errorTracker.incrementCounters(e);
return;
}
@@ -332,13 +363,12 @@ public void reduce(Text key, Iterable values,
new ByteArrayInputStream(bos.toByteArray()));
WARCRecord record = new WARCRecord(in);
context.write(NullWritable.get(), new WARCWritable(record));
- context.getCounter("WARCExporter", "records generated")
- .increment(1);
+ recordsGeneratedCounter.increment(1);
} catch (IOException | IllegalStateException exception) {
LOG.error(
"Exception when generating WARC metadata record for {} : {}",
key, exception.getMessage(), exception);
- context.getCounter("WARCExporter", "exception").increment(1);
+ errorTracker.incrementCounters(exception);
}
}
@@ -376,7 +406,7 @@ public void reduce(Text key, Iterable values,
.append(uri.toASCIIString()).append(CRLF);
} catch (Exception e) {
LOG.error("Invalid URI {} ", key);
- context.getCounter("WARCExporter", "invalid URI").increment(1);
+ errorTracker.incrementCounters(e);
return;
}
@@ -392,13 +422,12 @@ public void reduce(Text key, Iterable values,
new ByteArrayInputStream(bos.toByteArray()));
WARCRecord record = new WARCRecord(in);
context.write(NullWritable.get(), new WARCWritable(record));
- context.getCounter("WARCExporter", "records generated")
- .increment(1);
+ recordsGeneratedCounter.increment(1);
} catch (IOException | IllegalStateException exception) {
LOG.error(
"Exception when generating WARC metadata record for {} : {}",
key, exception.getMessage(), exception);
- context.getCounter("WARCExporter", "exception").increment(1);
+ errorTracker.incrementCounters(exception);
}
}
}
diff --git a/src/java/org/apache/nutch/util/DomainStatistics.java b/src/java/org/apache/nutch/util/DomainStatistics.java
index 5ee09c846a..4057795d52 100644
--- a/src/java/org/apache/nutch/util/DomainStatistics.java
+++ b/src/java/org/apache/nutch/util/DomainStatistics.java
@@ -28,6 +28,7 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Counter;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
@@ -38,6 +39,7 @@
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.apache.nutch.crawl.CrawlDatum;
+import org.apache.nutch.metrics.NutchMetrics;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -52,10 +54,6 @@ public class DomainStatistics extends Configured implements Tool {
private static final Text FETCHED_TEXT = new Text("FETCHED");
private static final Text NOT_FETCHED_TEXT = new Text("NOT_FETCHED");
- public static enum MyCounter {
- FETCHED, NOT_FETCHED, EMPTY_RESULT
- };
-
private static final int MODE_HOST = 1;
private static final int MODE_DOMAIN = 2;
private static final int MODE_SUFFIX = 3;
@@ -158,10 +156,29 @@ static class DomainStatisticsMapper extends
Mapper {
int mode = 0;
+ // Cached counter references for performance
+ private Counter fetchedCounter;
+ private Counter notFetchedCounter;
+ private Counter emptyResultCounter;
+
@Override
public void setup(Context context) {
mode = context.getConfiguration().getInt("domain.statistics.mode",
MODE_DOMAIN);
+ // Initialize cached counter references
+ initCounters(context);
+ }
+
+ /**
+ * Initialize cached counter references to avoid repeated lookups in hot paths.
+ */
+ private void initCounters(Context context) {
+ fetchedCounter = context.getCounter(
+ NutchMetrics.GROUP_DOMAIN_STATS, NutchMetrics.DOMAIN_STATS_FETCHED_TOTAL);
+ notFetchedCounter = context.getCounter(
+ NutchMetrics.GROUP_DOMAIN_STATS, NutchMetrics.DOMAIN_STATS_NOT_FETCHED_TOTAL);
+ emptyResultCounter = context.getCounter(
+ NutchMetrics.GROUP_DOMAIN_STATS, NutchMetrics.DOMAIN_STATS_EMPTY_RESULT_TOTAL);
}
@Override
@@ -197,17 +214,17 @@ public void map(Text urlText, CrawlDatum datum, Context context)
}
if (out.trim().equals("")) {
LOG.info("url : {}", url);
- context.getCounter(MyCounter.EMPTY_RESULT).increment(1);
+ emptyResultCounter.increment(1);
}
context.write(new Text(out), new LongWritable(1));
} catch (Exception ex) {
}
- context.getCounter(MyCounter.FETCHED).increment(1);
+ fetchedCounter.increment(1);
context.write(FETCHED_TEXT, new LongWritable(1));
} else {
- context.getCounter(MyCounter.NOT_FETCHED).increment(1);
+ notFetchedCounter.increment(1);
context.write(NOT_FETCHED_TEXT, new LongWritable(1));
}
}
diff --git a/src/java/org/apache/nutch/util/SitemapProcessor.java b/src/java/org/apache/nutch/util/SitemapProcessor.java
index d83a6e358c..21362223cd 100644
--- a/src/java/org/apache/nutch/util/SitemapProcessor.java
+++ b/src/java/org/apache/nutch/util/SitemapProcessor.java
@@ -31,6 +31,7 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.Counter;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
@@ -45,6 +46,8 @@
import org.apache.hadoop.util.ToolRunner;
import org.apache.nutch.crawl.CrawlDatum;
import org.apache.nutch.hostdb.HostDatum;
+import org.apache.nutch.metrics.ErrorTracker;
+import org.apache.nutch.metrics.NutchMetrics;
import org.apache.nutch.net.URLFilters;
import org.apache.nutch.net.URLNormalizers;
import org.apache.nutch.protocol.Content;
@@ -113,6 +116,14 @@ private static class SitemapMapper extends Mapper values, Context context)
originalDatum.setModifiedTime(sitemapDatum.getModifiedTime());
}
- context.getCounter("Sitemap", "existing_sitemap_entries").increment(1);
+ existingEntriesCounter.increment(1);
context.write(key, originalDatum);
}
else if(sitemapDatum != null) {
// For the newly discovered links via sitemap, set the status as unfetched and emit
- context.getCounter("Sitemap", "new_sitemap_entries").increment(1);
+ newEntriesCounter.increment(1);
sitemapDatum.setStatus(CrawlDatum.STATUS_DB_UNFETCHED);
context.write(key, sitemapDatum);
}
@@ -457,11 +507,11 @@ public void sitemap(Path crawldb, Path hostdb, Path sitemapUrlDir, boolean stric
FSUtils.replace(fs, current, tempCrawlDb, true);
LockUtil.removeLockFile(fs, lock);
- long filteredRecords = job.getCounters().findCounter("Sitemap", "filtered_records").getValue();
- long fromHostname = job.getCounters().findCounter("Sitemap", "sitemaps_from_hostname").getValue();
- long fromSeeds = job.getCounters().findCounter("Sitemap", "sitemap_seeds").getValue();
- long failedFetches = job.getCounters().findCounter("Sitemap", "failed_fetches").getValue();
- long newSitemapEntries = job.getCounters().findCounter("Sitemap", "new_sitemap_entries").getValue();
+ long filteredRecords = job.getCounters().findCounter(NutchMetrics.GROUP_SITEMAP, NutchMetrics.SITEMAP_FILTERED_RECORDS_TOTAL).getValue();
+ long fromHostname = job.getCounters().findCounter(NutchMetrics.GROUP_SITEMAP, NutchMetrics.SITEMAP_FROM_HOSTNAME_TOTAL).getValue();
+ long fromSeeds = job.getCounters().findCounter(NutchMetrics.GROUP_SITEMAP, NutchMetrics.SITEMAP_SEEDS_TOTAL).getValue();
+ long failedFetches = job.getCounters().findCounter(NutchMetrics.GROUP_SITEMAP, NutchMetrics.SITEMAP_FAILED_FETCHES_TOTAL).getValue();
+ long newSitemapEntries = job.getCounters().findCounter(NutchMetrics.GROUP_SITEMAP, NutchMetrics.SITEMAP_NEW_ENTRIES_TOTAL).getValue();
LOG.info("SitemapProcessor: Total records rejected by filters: {}", filteredRecords);
LOG.info("SitemapProcessor: Total sitemaps from host name: {}", fromHostname);
diff --git a/src/java/org/apache/nutch/util/URLUtil.java b/src/java/org/apache/nutch/util/URLUtil.java
index 158125999e..44c6309d2a 100644
--- a/src/java/org/apache/nutch/util/URLUtil.java
+++ b/src/java/org/apache/nutch/util/URLUtil.java
@@ -103,7 +103,7 @@ static URL fixPureQueryTargets(URL base, String target)
* https://publicsuffix.org/list/public_suffix_list.dat and are compared
* using
+ * "https://crawler-commons.github.io/crawler-commons/1.6/crawlercommons/domains/EffectiveTldFinder.html">
* crawler-commons' EffectiveTldFinder. Only ICANN domain suffixes are
* used. Because EffectiveTldFinder loads the public suffix list as file
* "effective_tld_names.dat" from the Java classpath, it's possible to use the
diff --git a/src/java/org/commoncrawl/tools/UrlCleaner.java b/src/java/org/commoncrawl/tools/UrlCleaner.java
index a3f26b126b..c4d92ca669 100644
--- a/src/java/org/commoncrawl/tools/UrlCleaner.java
+++ b/src/java/org/commoncrawl/tools/UrlCleaner.java
@@ -40,6 +40,7 @@
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.nutch.metrics.NutchMetrics;
import org.apache.nutch.net.URLFilterException;
import org.apache.nutch.net.URLFilters;
import org.apache.nutch.net.URLNormalizers;
@@ -134,18 +135,21 @@ public void map(Text key, Text value, Context context)
try {
url = urlNormalizers.normalize(url, scope);
} catch (MalformedURLException e) {
- context.getCounter("urlcleaner", "urls_rejected").increment(1);
- return;
+ context.getCounter(NutchMetrics.GROUP_URLCLEANER,
+ NutchMetrics.URLCLEANER_REJECTED_TOTAL).increment(1);
+ return;
}
try {
url = filters.filter(url);
} catch (URLFilterException e) {
- context.getCounter("urlcleaner", "urls_rejected").increment(1);
+ context.getCounter(NutchMetrics.GROUP_URLCLEANER,
+ NutchMetrics.URLCLEANER_REJECTED_TOTAL).increment(1);
return;
}
if (url == null) {
- context.getCounter("urlcleaner", "urls_rejected").increment(1);
+ context.getCounter(NutchMetrics.GROUP_URLCLEANER,
+ NutchMetrics.URLCLEANER_REJECTED_TOTAL).increment(1);
return;
}
@@ -157,21 +161,26 @@ public void map(Text key, Text value, Context context)
if (needDomain) {
domain = EffectiveTldFinder.getAssignedDomain(host, true, true);
if (checkDomain && domain == null) {
- context.getCounter("urlcleaner", "urls_rejected_invalid_domain")
+ context
+ .getCounter(NutchMetrics.GROUP_URLCLEANER,
+ NutchMetrics.URLCLEANER_REJECTED_INVALID_DOMAIN_TOTAL)
.increment(1);
return;
}
}
} catch (MalformedURLException e) {
- context.getCounter("urlcleaner", "urls_rejected").increment(1);
+ context.getCounter(NutchMetrics.GROUP_URLCLEANER,
+ NutchMetrics.URLCLEANER_REJECTED_TOTAL).increment(1);
return;
}
}
if (url.equals(urlOrig)) {
- context.getCounter("urlcleaner", "urls_accepted_unchanged").increment(1);
+ context.getCounter(NutchMetrics.GROUP_URLCLEANER,
+ NutchMetrics.URLCLEANER_ACCEPTED_UNCHANGED_TOTAL).increment(1);
} else {
- context.getCounter("urlcleaner", "urls_accepted_normalized").increment(1);
+ context.getCounter(NutchMetrics.GROUP_URLCLEANER,
+ NutchMetrics.URLCLEANER_ACCEPTED_NORMALIZED_TOTAL).increment(1);
key.set(url);
}
diff --git a/src/java/org/commoncrawl/tools/UrlSampler.java b/src/java/org/commoncrawl/tools/UrlSampler.java
index f28447a4cf..e2060e1f47 100644
--- a/src/java/org/commoncrawl/tools/UrlSampler.java
+++ b/src/java/org/commoncrawl/tools/UrlSampler.java
@@ -48,6 +48,7 @@
import org.apache.nutch.crawl.Generator2;
import org.apache.nutch.crawl.Generator2.DomainScorePair;
import org.apache.nutch.crawl.URLPartitioner;
+import org.apache.nutch.metrics.NutchMetrics;
import org.apache.nutch.util.NutchConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -150,7 +151,8 @@ public void map(Text key, Text value, Context context)
domain = URLPartitioner.getDomainName(u.getHost());
} catch (Exception e) {
LOG.warn("Malformed URL: '{}', skipping ({})", url, e.getMessage());
- context.getCounter("UrlSampler", "MALFORMED_URL").increment(1);
+ context.getCounter(NutchMetrics.GROUP_URLSAMPLER,
+ NutchMetrics.URLSAMPLER_MALFORMED_URL_TOTAL).increment(1);
return;
}
@@ -242,7 +244,8 @@ public void reduce(DomainScorePair key, Iterable values,
domain);
}
} catch (MalformedURLException e) {
- context.getCounter("UrlSampler", "MALFORMED_URL").increment(1);
+ context.getCounter(NutchMetrics.GROUP_URLSAMPLER,
+ NutchMetrics.URLSAMPLER_MALFORMED_URL_TOTAL).increment(1);
continue;
}
nUrls++;
@@ -271,12 +274,14 @@ public void reduce(DomainScorePair key, Iterable values,
}
if (nUrls == 0)
return;
- context.getCounter("UrlSampler", "SKIPPED_MAX_URLS")
- .increment(skippedMaxUrls);
- context.getCounter("UrlSampler", "SKIPPED_MAX_URLS_PER_HOST")
+ context.getCounter(NutchMetrics.GROUP_URLSAMPLER,
+ NutchMetrics.URLSAMPLER_SKIPPED_MAX_URLS_TOTAL).increment(skippedMaxUrls);
+ context
+ .getCounter(NutchMetrics.GROUP_URLSAMPLER,
+ NutchMetrics.URLSAMPLER_SKIPPED_MAX_URLS_PER_HOST_TOTAL)
.increment(skippedMaxUrlsPerHost);
- context.getCounter("UrlSampler", "SKIPPED_MAX_HOSTS")
- .increment(skippedMaxHosts);
+ context.getCounter(NutchMetrics.GROUP_URLSAMPLER,
+ NutchMetrics.URLSAMPLER_SKIPPED_MAX_HOSTS_TOTAL).increment(skippedMaxHosts);
LOG.info(
"Sampled for domain {} : {} hosts, {} URLs ({} skipped: {} max. URLs, {} max. per host, {} max. hosts), sum of scores = {}",
domain, hosts.size(), nUrlsSampled, (nUrls - nUrlsSampled),
@@ -336,8 +341,8 @@ private void sample(Path[] inputs, Path output) throws Exception {
}
public void usage() {
- System.err
- .println("Usage: UrlSampler [-D...] ... \n");
+ System.err.println(
+ "Usage: UrlSampler [-D...] ... \n");
}
@Override
diff --git a/src/java/org/commoncrawl/tools/UrlSamplerHost.java b/src/java/org/commoncrawl/tools/UrlSamplerHost.java
index e296ffa90b..bce68ad50f 100644
--- a/src/java/org/commoncrawl/tools/UrlSamplerHost.java
+++ b/src/java/org/commoncrawl/tools/UrlSamplerHost.java
@@ -44,6 +44,7 @@
import org.apache.hadoop.util.ToolRunner;
import org.apache.nutch.crawl.Generator2;
import org.apache.nutch.crawl.Generator2.DomainScorePair;
+import org.apache.nutch.metrics.NutchMetrics;
import org.apache.nutch.util.NutchConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -60,7 +61,8 @@
*
*
*
- * host name (leading www. may be stripped), limits and default score
+ * host name (leading www. may be stripped), limits and default
+ * score
*
*
* <host_name> \t <rank> \t <max_urls> \t <default_score>
@@ -180,7 +182,8 @@ public void map(Text key, Text value, Context context)
}
} catch (Exception e) {
LOG.warn("Malformed URL: '{}', skipping ({})", url, e.getMessage());
- context.getCounter("UrlSampler", "MALFORMED_URL").increment(1);
+ context.getCounter(NutchMetrics.GROUP_URLSAMPLER,
+ NutchMetrics.URLSAMPLER_MALFORMED_URL_TOTAL).increment(1);
return;
}
@@ -270,40 +273,59 @@ public void reduce(DomainScorePair key, Iterable values,
context.write(text, meta);
}
// hosts == reduce input groups
- context.getCounter("UrlSamplerHost", "HOSTS").increment(1);
+ context.getCounter(NutchMetrics.GROUP_URLSAMPLER_HOST,
+ NutchMetrics.URLSAMPLER_HOSTS).increment(1);
// URLs == map output records, reduce input records
- context.getCounter("UrlSamplerHost", "URLS").increment(nUrls);
+ context.getCounter(NutchMetrics.GROUP_URLSAMPLER_HOST,
+ NutchMetrics.URLSAMPLER_URLS).increment(nUrls);
if (nUrls > 0) {
if (maxUrls > -1) {
- context.getCounter("UrlSamplerHost", "HOSTS_WITH_LIMIT").increment(1);
- context.getCounter("UrlSamplerHost", "URLS_HOST_WITH_LIMIT")
+ context.getCounter(NutchMetrics.GROUP_URLSAMPLER_HOST,
+ NutchMetrics.URLSAMPLER_HOSTS_WITH_LIMIT).increment(1);
+ context
+ .getCounter(NutchMetrics.GROUP_URLSAMPLER_HOST,
+ NutchMetrics.URLSAMPLER_URLS_HOST_WITH_LIMIT)
.increment(nUrls);
} else {
- context.getCounter("UrlSamplerHost", "HOSTS_WITHOUT_LIMIT")
- .increment(1);
- context.getCounter("UrlSamplerHost", "URLS_HOST_WITHOUT_LIMIT")
+ context.getCounter(NutchMetrics.GROUP_URLSAMPLER_HOST,
+ NutchMetrics.URLSAMPLER_HOSTS_WITHOUT_LIMIT).increment(1);
+ context
+ .getCounter(NutchMetrics.GROUP_URLSAMPLER_HOST,
+ NutchMetrics.URLSAMPLER_URLS_HOST_WITHOUT_LIMIT)
.increment(nUrls);
}
if (nUrlsSampled > 0) {
- context.getCounter("UrlSamplerHost", "URLS_SAMPLED")
- .increment(nUrlsSampled);
- context.getCounter("UrlSamplerHost", "HOSTS_SAMPLED").increment(1);
+ context.getCounter(NutchMetrics.GROUP_URLSAMPLER_HOST,
+ NutchMetrics.URLSAMPLER_URLS_SAMPLED).increment(nUrlsSampled);
+ context.getCounter(NutchMetrics.GROUP_URLSAMPLER_HOST,
+ NutchMetrics.URLSAMPLER_HOSTS_SAMPLED).increment(1);
if (maxUrls > -1) {
- context.getCounter("UrlSamplerHost", "HOSTS_WITH_LIMIT_SAMPLED")
+ context
+ .getCounter(NutchMetrics.GROUP_URLSAMPLER_HOST,
+ NutchMetrics.URLSAMPLER_HOSTS_WITH_LIMIT_SAMPLED)
.increment(1);
- context.getCounter("UrlSamplerHost", "URLS_HOST_WITH_LIMIT_SAMPLED")
+ context
+ .getCounter(NutchMetrics.GROUP_URLSAMPLER_HOST,
+ NutchMetrics.URLSAMPLER_URLS_HOST_WITH_LIMIT_SAMPLED)
.increment(nUrlsSampled);
} else {
- context.getCounter("UrlSamplerHost", "HOSTS_WITHOUT_LIMIT_SAMPLED")
+ context
+ .getCounter(NutchMetrics.GROUP_URLSAMPLER_HOST,
+ NutchMetrics.URLSAMPLER_HOSTS_WITHOUT_LIMIT_SAMPLED)
.increment(1);
context
- .getCounter("UrlSamplerHost", "URLS_HOST_WITHOUT_LIMIT_SAMPLED")
+ .getCounter(NutchMetrics.GROUP_URLSAMPLER_HOST,
+ NutchMetrics.URLSAMPLER_URLS_HOST_WITHOUT_LIMIT_SAMPLED)
.increment(nUrlsSampled);
}
}
- context.getCounter("UrlSamplerHost", "SKIPPED_MAX_URLS_PER_HOST")
+ context
+ .getCounter(NutchMetrics.GROUP_URLSAMPLER_HOST,
+ NutchMetrics.URLSAMPLER_SKIPPED_MAX_URLS_PER_HOST)
.increment(skippedMaxUrlsPerHost);
- context.getCounter("UrlSamplerHost", "SKIPPED_RANDOM")
+ context
+ .getCounter(NutchMetrics.GROUP_URLSAMPLER_HOST,
+ NutchMetrics.URLSAMPLER_SKIPPED_RANDOM)
.increment(skippedRandom);
LOG.info(
"Sampled for host {} : {} URLs ({} skipped: {} max. per host, {} random), sum of scores = {}",
@@ -365,8 +387,8 @@ private void sample(Path[] inputs, Path output) throws Exception {
}
public void usage() {
- System.err
- .println("Usage: UrlSamplerHost [-D...] ... \n");
+ System.err.println(
+ "Usage: UrlSamplerHost [-D...] ... \n");
System.err.println(
"\nThe host_limits file defines the maximum number of URLs to sample per host.");
System.err.println("\nProperties:");
@@ -374,11 +396,12 @@ public void usage() {
"\t-Durlsample.host.strip.www=(true|false)\tstrip leading www. from host names");
System.err.println(
"\t\t\t(depending on whether the limits file uses stripped host names)");
- System.err.println("Properties to configure defaults, if host is not in the limits file:");
System.err.println(
- "\t-Durlsample.urls.per.host\tmax. number of URLs to sample per host");
+ "Properties to configure defaults, if host is not in the limits file:");
System.err.println(
- "\t\t\t-1 : sample randomly with low probability (default)");
+ "\t-Durlsample.urls.per.host\tmax. number of URLs to sample per host");
+ System.err
+ .println("\t\t\t-1 : sample randomly with low probability (default)");
System.err.println(
"\t-Durlsample.default.score\tdefault score for sampled URLs (default: 0.001)");
}
diff --git a/src/java/org/commoncrawl/util/WarcRecordWriter.java b/src/java/org/commoncrawl/util/WarcRecordWriter.java
index 5656c2b3a3..05f2a304f6 100644
--- a/src/java/org/commoncrawl/util/WarcRecordWriter.java
+++ b/src/java/org/commoncrawl/util/WarcRecordWriter.java
@@ -52,6 +52,7 @@
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.nutch.crawl.CrawlDatum;
import org.apache.nutch.metadata.Nutch;
+import org.apache.nutch.metrics.NutchMetrics;
import org.apache.nutch.net.URLNormalizers;
import org.apache.nutch.net.protocols.HttpDateFormat;
import org.apache.nutch.net.protocols.Response;
@@ -76,7 +77,6 @@ class WarcRecordWriter extends RecordWriter {
protected static final Pattern PROBLEMATIC_HEADERS = Pattern
.compile("(?i)(?:Content-(?:Encoding|Length)|Transfer-Encoding)");
protected static final String X_HIDE_HEADER = "X-Crawler-";
- public static final String WARC_WRITER_COUNTER_GROUP = "WARC-Writer";
protected static final Pattern STATUS_LINE_PATTERN = Pattern
.compile("^HTTP/1\\.[01] [0-9]{3}(?: .*)?$");
@@ -117,6 +117,8 @@ class WarcRecordWriter extends RecordWriter {
private URLNormalizers urlNormalizers;
private URLNormalizers urlNormalizersRedirect;
+ private SimpleDateFormat isoDate;
+
public WarcRecordWriter(Configuration conf, Path outputPath, int partition,
TaskAttemptContext context) throws IOException {
@@ -128,6 +130,9 @@ public WarcRecordWriter(Configuration conf, Path outputPath, int partition,
Locale.ROOT);
fileDate.setTimeZone(TimeZone.getTimeZone("UTC"));
+ isoDate = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ROOT);
+ isoDate.setTimeZone(TimeZone.getTimeZone("UTC"));
+
String prefix = conf.get("warc.export.prefix", "NUTCH-CRAWL");
/*
@@ -527,18 +532,22 @@ public synchronized void write(Text key, WarcCapture value)
throws IOException {
if (value.content == null) {
- String reason = "";
+ ProtocolStatus pstatus = null;
if (value.datum != null) {
- ProtocolStatus pstatus = (ProtocolStatus) value.datum.getMetaData()
+ pstatus = (ProtocolStatus) value.datum.getMetaData()
.get(Nutch.WRITABLE_PROTO_STATUS_KEY);
- if (pstatus != null) {
- reason = ": " + pstatus.getName() + " - " + pstatus.getMessage();
- }
}
- LOG.warn("Cannot write WARC record, no content for {}{}", value.url,
- reason);
- context.getCounter(WARC_WRITER_COUNTER_GROUP,
- "skipped records (no content)").increment(1);
+ if (pstatus != null) {
+ LOG.warn(
+ "Cannot write WARC record, no content for {}, protocol status: {} - {}",
+ value.url, pstatus.getName(), pstatus.getMessage());
+ } else {
+ LOG.warn(
+ "Cannot write WARC record, no content and protocol status for {}",
+ value.url);
+ }
+ context.getCounter(NutchMetrics.GROUP_WARC_WRITER,
+ NutchMetrics.WARC_WRITER_SKIPPED_NO_CONTENT_TOTAL).increment(1);
return;
}
@@ -560,10 +569,8 @@ public synchronized void write(Text key, WarcCapture value)
try {
targetUri = new URI(urlNorm);
LOG.info("Normalized URL to valid URI: {} -> {}", url, urlNorm);
- context
- .getCounter(WARC_WRITER_COUNTER_GROUP,
- "fixed records (invalid URI successfully normalized)")
- .increment(1);
+ context.getCounter(NutchMetrics.GROUP_WARC_WRITER,
+ NutchMetrics.WARC_WRITER_URI_NORMALIZED_TOTAL).increment(1);
} catch (URISyntaxException ee) {
// ignore, log exception observed on original URL
}
@@ -571,8 +578,10 @@ public synchronized void write(Text key, WarcCapture value)
}
if (targetUri == null) {
LOG.error("Cannot write WARC record, invalid URI: {}", url);
- context.getCounter(WARC_WRITER_COUNTER_GROUP,
- "skipped records (invalid URI)").increment(1);
+ context
+ .getCounter(NutchMetrics.GROUP_WARC_WRITER,
+ NutchMetrics.WARC_WRITER_SKIPPED_INVALID_URI_TOTAL)
+ .increment(1);
return;
}
}
@@ -594,8 +603,10 @@ public synchronized void write(Text key, WarcCapture value)
(truncated != null ? truncated : "-"),
value.content.getContentType(), value.content.getContent().length,
value.url);
- context.getCounter(WARC_WRITER_COUNTER_GROUP,
- "skipped records (by content)").increment(1);
+ context
+ .getCounter(NutchMetrics.GROUP_WARC_WRITER,
+ NutchMetrics.WARC_WRITER_SKIPPED_BY_CONTENT_TYPE_TOTAL)
+ .increment(1);
return;
}
}
@@ -637,8 +648,8 @@ public synchronized void write(Text key, WarcCapture value)
} catch (Throwable t) {
LOG.error(t.getMessage());
}
- context.getCounter(WARC_WRITER_COUNTER_GROUP,
- "skipped records (duplicate)").increment(1);
+ context.getCounter(NutchMetrics.GROUP_WARC_WRITER,
+ NutchMetrics.WARC_WRITER_SKIPPED_DUPLICATE_TOTAL).increment(1);
return;
}
precedingURL = url;
@@ -668,8 +679,10 @@ public synchronized void write(Text key, WarcCapture value)
if (pstatus == null) {
LOG.warn("Cannot write WARC record, no protocol status for {}",
value.url);
- context.getCounter(WARC_WRITER_COUNTER_GROUP,
- "skipped records (no protocol status)").increment(1);
+ context
+ .getCounter(NutchMetrics.GROUP_WARC_WRITER,
+ NutchMetrics.WARC_WRITER_SKIPPED_NO_PROTOCOL_STATUS_TOTAL)
+ .increment(1);
return;
}
switch (pstatus.getCode()) {
@@ -698,8 +711,9 @@ public synchronized void write(Text key, WarcCapture value)
if (value.content.getMetadata()
.get(Response.RESPONSE_HEADERS) == null) {
LOG.warn("Unknown or ambiguous protocol status: {}", pstatus);
- context.getCounter(WARC_WRITER_COUNTER_GROUP,
- "skipped records (unknown protocol status)").increment(1);
+ context.getCounter(NutchMetrics.GROUP_WARC_WRITER,
+ NutchMetrics.WARC_WRITER_SKIPPED_UNKNOWN_PROTOCOL_STATUS_TOTAL)
+ .increment(1);
return;
}
}
@@ -839,8 +853,8 @@ public synchronized void write(Text key, WarcCapture value)
}
LOG.info("WARC {} record {} ({}, status: {}, size: {})",
- (notModified ? "revisit" : "response"), targetUri, date, httpStatusCode,
- value.content.getContent().length);
+ (notModified ? "revisit" : "response"), targetUri, isoDate.format(date),
+ httpStatusCode, value.content.getContent().length);
URI requestId = null;
if (verbatimRequestHeaders != null) {
@@ -860,17 +874,22 @@ public synchronized void write(Text key, WarcCapture value)
// detect language only for successfully fetched primary documents
ldres = langDetect.detectLanguage(targetUri, value.content);
if (ldres.errorReason != null) {
- context.getCounter(WARC_WRITER_COUNTER_GROUP,
- "language detection: " + ldres.errorStatus.name).increment(1);
+ context.getCounter(NutchMetrics.GROUP_WARC_WRITER,
+ NutchMetrics.WARC_WRITER_LID_ERROR_PREFIX + ldres.errorStatus.name)
+ .increment(1);
} else if (ldres.languages == null) {
- context.getCounter(WARC_WRITER_COUNTER_GROUP,
- "language detection: no result").increment(1);
+ context.getCounter(NutchMetrics.GROUP_WARC_WRITER,
+ NutchMetrics.WARC_WRITER_LID_NO_RESULT_TOTAL).increment(1);
} else if (ldres.languages.isReliable()) {
- context.getCounter(WARC_WRITER_COUNTER_GROUP,
- "language detection: reliable").increment(1);
+ context
+ .getCounter(NutchMetrics.GROUP_WARC_WRITER,
+ NutchMetrics.WARC_WRITER_LID_RESULT_RELIABLE_TOTAL)
+ .increment(1);
} else {
- context.getCounter(WARC_WRITER_COUNTER_GROUP,
- "language detection: not reliable").increment(1);
+ context
+ .getCounter(NutchMetrics.GROUP_WARC_WRITER,
+ NutchMetrics.WARC_WRITER_LID_RESULT_NOT_RELIABLE_TOTAL)
+ .increment(1);
}
if (generateCdx) {
if (ldres.charset != null) {
diff --git a/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java b/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
index 79b45882eb..caa3f861ea 100755
--- a/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
+++ b/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
@@ -721,6 +721,12 @@ public BaseRobotRules getRobotRules(Text url, CrawlDatum datum,
return this.robots.getRobotRulesSet(this, url, robotsTxtContent);
}
+ @Override
+ public BaseRobotRules getRobotRules(URL url, CrawlDatum datum,
+ List robotsTxtContent) {
+ return this.robots.getRobotRulesSet(this, url, robotsTxtContent);
+ }
+
/**
* Transforming a String[] into a HashMap for faster searching
*
diff --git a/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java b/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java
index e4d2010696..877873b64b 100644
--- a/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java
+++ b/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java
@@ -232,4 +232,14 @@ public BaseRobotRules getRobotRules(Text url, CrawlDatum datum,
return RobotRulesParser.EMPTY_RULES;
}
+ /**
+ * No robots parsing is done for file protocol. So this returns a set of empty
+ * rules which will allow every url.
+ */
+ @Override
+ public BaseRobotRules getRobotRules(URL url, CrawlDatum datum,
+ List robotsTxtContent) {
+ return RobotRulesParser.EMPTY_RULES;
+ }
+
}
diff --git a/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java b/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java
index 2a47b63d61..8cf58f75e7 100644
--- a/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java
+++ b/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java
@@ -304,6 +304,15 @@ public BaseRobotRules getRobotRules(Text url, CrawlDatum datum,
return robots.getRobotRulesSet(this, url, robotsTxtContent);
}
+ /**
+ * Get the robots rules for a given url
+ */
+ @Override
+ public BaseRobotRules getRobotRules(URL url, CrawlDatum datum,
+ List robotsTxtContent) {
+ return robots.getRobotRulesSet(this, url, robotsTxtContent);
+ }
+
public int getBufferSize() {
return BUFFER_SIZE;
}
diff --git a/src/plugin/protocol-okhttp/ivy.xml b/src/plugin/protocol-okhttp/ivy.xml
index 0768def785..28f355d7b9 100644
--- a/src/plugin/protocol-okhttp/ivy.xml
+++ b/src/plugin/protocol-okhttp/ivy.xml
@@ -37,8 +37,9 @@
-
-
+
+
+
-
+
diff --git a/src/plugin/protocol-okhttp/plugin.xml b/src/plugin/protocol-okhttp/plugin.xml
index e2183d2b50..51f65f5d25 100755
--- a/src/plugin/protocol-okhttp/plugin.xml
+++ b/src/plugin/protocol-okhttp/plugin.xml
@@ -28,13 +28,15 @@
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
diff --git a/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/OkHttp.java b/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/OkHttp.java
index 954c3f6df1..a9d2b14d42 100644
--- a/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/OkHttp.java
+++ b/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/OkHttp.java
@@ -52,15 +52,19 @@
import org.slf4j.LoggerFactory;
import okhttp3.Authenticator;
+import okhttp3.CompressionInterceptor;
import okhttp3.Connection;
import okhttp3.ConnectionPool;
+import okhttp3.Gzip;
import okhttp3.Handshake;
import okhttp3.Headers;
import okhttp3.Interceptor;
import okhttp3.OkHttpClient;
import okhttp3.Protocol;
import okhttp3.Request;
-import okhttp3.brotli.BrotliInterceptor;
+import okhttp3.brotli.Brotli;
+import okhttp3.zstd.Zstd;
+
public class OkHttp extends HttpBase {
@@ -156,13 +160,11 @@ public boolean verify(String hostname, SSLSession session) {
String proxyUsername = conf.get("http.proxy.username");
if (proxyUsername == null) {
ProxySelector selector = new ProxySelector() {
- @SuppressWarnings("serial")
private final List noProxyList = new ArrayList() {
{
add(Proxy.NO_PROXY);
}
};
- @SuppressWarnings("serial")
private final List proxyList = new ArrayList() {
{
add(proxy);
@@ -224,8 +226,9 @@ public Request authenticate(okhttp3.Route route,
builder.addNetworkInterceptor(new HTTPHeadersInterceptor());
}
- // enable support for Brotli compression (Content-Encoding)
- builder.addInterceptor(BrotliInterceptor.INSTANCE);
+ // enable support for Zstd, Brotli, Gzip Content-Encoding
+ builder.addInterceptor(new CompressionInterceptor(Zstd.INSTANCE,
+ Brotli.INSTANCE, Gzip.INSTANCE));
// instantiate connection pool(s), cf.
// https://square.github.io/okhttp/3.x/okhttp/okhttp3/ConnectionPool.html
diff --git a/src/test/org/apache/nutch/crawl/TestAdaptiveFetchSchedule.java b/src/test/org/apache/nutch/crawl/TestAdaptiveFetchSchedule.java
index 377d49ec81..c06ae30076 100644
--- a/src/test/org/apache/nutch/crawl/TestAdaptiveFetchSchedule.java
+++ b/src/test/org/apache/nutch/crawl/TestAdaptiveFetchSchedule.java
@@ -24,6 +24,12 @@
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Date;
+import java.util.Properties;
/**
* Test cases for AdaptiveFetchSchedule.
@@ -117,5 +123,91 @@ private void validateFetchInterval(int changed, int getInterval) {
}
}
+
+ /**
+ * Test https://issues.apache.org/jira/browse/NUTCH-1564
+ */
+ @Test
+ public void testSetFetchSchedule1() {
+ // db.fetch.schedule.adaptive.sync_delta_rate = 0.3 (default)
+ // db.fetch.interval.default = 172800 (2 days)
+ // db.fetch.schedule.adaptive.min_interval = 86400 (1 day)
+ // db.fetch.schedule.adaptive.max_interval = 604800 (7 days)
+ // db.fetch.interval.max = 604800 (7 days)
+ // 3-days cycle
+ // 30 days since last modified
+ doTestSetFetchSchedule(0.3, 2, 1, 7, 7, 3, 30);
+ }
+
+ @Test
+ public void testSetFetchSchedule2() {
+ // db.fetch.schedule.adaptive.sync_delta_rate = 0.3 (default)
+ // db.fetch.interval.default = 86400 (1 day)
+ // db.fetch.schedule.adaptive.min_interval = 86400 (1 day)
+ // db.fetch.schedule.adaptive.max_interval = 172800 (2 days)
+ // db.fetch.interval.max = 604800 (7 days)
+ // 1-day cycle
+ // 10 days since last modified
+ doTestSetFetchSchedule(0.3, 1, 1, 2, 7, 1, 10);
+ }
+
+ @Test
+ public void testSetFetchSchedule3() {
+ // db.fetch.schedule.adaptive.sync_delta_rate = 0.3 (default)
+ // db.fetch.interval.default = 172800 (2 days)
+ // db.fetch.schedule.adaptive.min_interval = 86400 (1 day)
+ // db.fetch.schedule.adaptive.max_interval = 864000 (10 days)
+ // db.fetch.interval.max = 864000 (10 days)
+ // 3-days cycle
+ // 180 days since last modified
+ doTestSetFetchSchedule(0.3, 2, 1, 10, 10, 3, 180);
+ }
+
+ private void doTestSetFetchSchedule(double deltaRate, int intervalDefaultDays,
+ int minIntervalDays, int maxIntervalDays, int intervalMaxDays,
+ int previousFetchTimeDays, int modifiedTimeDays) {
+ // need to properly override defaults
+ Properties props = new Properties();
+ props.setProperty("db.fetch.schedule.class", "org.apache.nutch.crawl.AdaptiveFetchSchedule");
+ props.setProperty("db.fetch.schedule.adaptive.sync_delta", "true"); // default
+ props.setProperty("db.fetch.schedule.adaptive.sync_delta_rate", String.valueOf(deltaRate));
+ props.setProperty("db.fetch.interval.default", String.valueOf(FetchSchedule.SECONDS_PER_DAY * intervalDefaultDays));
+ props.setProperty("db.fetch.schedule.adaptive.min_interval", String.valueOf(FetchSchedule.SECONDS_PER_DAY * minIntervalDays));
+ props.setProperty("db.fetch.schedule.adaptive.max_interval", String.valueOf(FetchSchedule.SECONDS_PER_DAY * maxIntervalDays));
+ props.setProperty("db.fetch.interval.max", String.valueOf(FetchSchedule.SECONDS_PER_DAY * intervalMaxDays));
+
+ conf = NutchConfiguration.create(true, props);
+ inc_rate = conf.getFloat("db.fetch.schedule.adaptive.inc_rate", 0.2f); // default
+ dec_rate = conf.getFloat("db.fetch.schedule.adaptive.dec_rate", 0.2f); // default
+
+ // ignore adaptive-host-specific-intervals.txt
+ Text url = new Text("http://www.example2.com");
+
+ AdaptiveFetchSchedule fs = new AdaptiveFetchSchedule();
+ fs.setConf(conf);
+
+ CrawlDatum datum = prepareCrawlDatum();
+ Date fetchTime = Date.from(Instant.now());
+ Date previousFetchTime = Date.from(Instant.now().minus(Duration.ofDays(previousFetchTimeDays)));
+ Date modifiedTime = Date.from(Instant.now().minus(Duration.ofDays(modifiedTimeDays)));
+ datum.setStatus(CrawlDatum.STATUS_FETCH_SUCCESS);
+ datum.setRetriesSinceFetch(0);
+ datum.setModifiedTime(modifiedTime.getTime());
+ datum.setFetchTime(fetchTime.getTime());
+
+ System.out.println("CrawlDatum fetchTime: " + fetchTime + "; modifiedTime: " + modifiedTime);
+
+ fs.setFetchSchedule(url, datum, previousFetchTime.getTime(), modifiedTime.getTime(),
+ fetchTime.getTime(), modifiedTime.getTime(), CrawlDatum.STATUS_DB_NOTMODIFIED);
+
+ Date nextFetchTime = new Date(datum.getFetchTime());
+ System.out.println("CrawlDatum next fetchTime: " + nextFetchTime);
+
+ assertTrue(nextFetchTime.after(fetchTime));
+ // adapt milliseconds to seconds
+ long fetchTimeDiff = (nextFetchTime.getTime() - fetchTime.getTime()) / 1000L ;
+ assertTrue(fetchTimeDiff >= FetchSchedule.SECONDS_PER_DAY * minIntervalDays);
+ assertTrue(fetchTimeDiff <= FetchSchedule.SECONDS_PER_DAY * maxIntervalDays);
+ }
}
diff --git a/src/test/org/apache/nutch/crawl/TODOTestCrawlDbStates.java b/src/test/org/apache/nutch/crawl/TestCrawlDbStatesExtended.java
similarity index 99%
rename from src/test/org/apache/nutch/crawl/TODOTestCrawlDbStates.java
rename to src/test/org/apache/nutch/crawl/TestCrawlDbStatesExtended.java
index dfad393512..2e6ea55af1 100644
--- a/src/test/org/apache/nutch/crawl/TODOTestCrawlDbStates.java
+++ b/src/test/org/apache/nutch/crawl/TestCrawlDbStatesExtended.java
@@ -29,7 +29,7 @@
import static org.apache.nutch.crawl.CrawlDatum.*;
import static org.junit.jupiter.api.Assertions.fail;
-public class TODOTestCrawlDbStates extends TestCrawlDbStates {
+public class TestCrawlDbStatesExtended extends TestCrawlDbStates {
private static final Logger LOG = LoggerFactory
.getLogger(MethodHandles.lookup().lookupClass());
diff --git a/src/test/org/apache/nutch/metrics/TestErrorTracker.java b/src/test/org/apache/nutch/metrics/TestErrorTracker.java
new file mode 100644
index 0000000000..5caa3e3a71
--- /dev/null
+++ b/src/test/org/apache/nutch/metrics/TestErrorTracker.java
@@ -0,0 +1,514 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.metrics;
+
+import java.io.IOException;
+import java.net.ConnectException;
+import java.net.MalformedURLException;
+import java.net.SocketException;
+import java.net.SocketTimeoutException;
+import java.net.URISyntaxException;
+import java.net.UnknownHostException;
+
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.TaskInputOutputContext;
+import org.apache.nutch.indexer.IndexingException;
+import org.apache.nutch.net.URLFilterException;
+import org.apache.nutch.parse.ParseException;
+import org.apache.nutch.parse.ParserNotFound;
+import org.apache.nutch.protocol.ProtocolException;
+import org.apache.nutch.protocol.ProtocolNotFound;
+import org.apache.nutch.scoring.ScoringFilterException;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.xml.sax.SAXException;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.*;
+
+import org.apache.nutch.metrics.ErrorTracker.ErrorType;
+
+/**
+ * Unit tests for {@link ErrorTracker} categorization, counting, and Hadoop
+ * counter integration.
+ */
+@ExtendWith(MockitoExtension.class)
+public class TestErrorTracker {
+
+ @Mock
+ private TaskInputOutputContext mockContext;
+
+ @Mock
+ private Counter mockCounter;
+
+ @BeforeEach
+ void setUp() {
+ // Configure mock context to return mock counter for any counter request
+ lenient().when(mockContext.getCounter(anyString(), anyString())).thenReturn(mockCounter);
+ }
+
+ // =========================================================================
+ // Network Error Categorization Tests
+ // =========================================================================
+
+ @Test
+ public void testCategorizeNetworkErrors() {
+ // Test IOException
+ assertEquals(ErrorType.NETWORK,
+ ErrorTracker.categorize(new IOException("Connection failed")));
+
+ // Test SocketException
+ assertEquals(ErrorType.NETWORK,
+ ErrorTracker.categorize(new SocketException("Socket closed")));
+
+ // Test UnknownHostException
+ assertEquals(ErrorType.NETWORK,
+ ErrorTracker.categorize(new UnknownHostException("example.com")));
+
+ // Test ConnectException
+ assertEquals(ErrorType.NETWORK,
+ ErrorTracker.categorize(new ConnectException("Connection refused")));
+ }
+
+ // =========================================================================
+ // Timeout Error Categorization Tests
+ // =========================================================================
+
+ @Test
+ public void testCategorizeTimeoutErrors() {
+ // Test SocketTimeoutException
+ assertEquals(ErrorType.TIMEOUT,
+ ErrorTracker.categorize(new SocketTimeoutException("Read timed out")));
+ }
+
+ @Test
+ public void testCategorizeTimeoutByClassName() {
+ // Test custom exception with "Timeout" in class name
+ // The categorize method checks className.contains("Timeout")
+ Exception customTimeout = new CustomTimeoutException("Custom timeout");
+ assertEquals(ErrorType.TIMEOUT, ErrorTracker.categorize(customTimeout));
+ }
+
+ // Custom exception class for testing class name-based detection
+ private static class CustomTimeoutException extends Exception {
+ CustomTimeoutException(String message) {
+ super(message);
+ }
+ }
+
+ // =========================================================================
+ // URL Error Categorization Tests
+ // =========================================================================
+
+ @Test
+ public void testCategorizeUrlErrors() {
+ // Test MalformedURLException
+ assertEquals(ErrorType.URL,
+ ErrorTracker.categorize(new MalformedURLException("Invalid URL")));
+
+ // Test URISyntaxException
+ assertEquals(ErrorType.URL,
+ ErrorTracker.categorize(new URISyntaxException("bad uri", "Invalid syntax")));
+ }
+
+ @Test
+ public void testCategorizeUrlFilterException() {
+ // Test URLFilterException (Nutch-specific)
+ assertEquals(ErrorType.URL,
+ ErrorTracker.categorize(new URLFilterException("URL filtered")));
+ }
+
+ // =========================================================================
+ // Protocol Error Categorization Tests
+ // =========================================================================
+
+ @Test
+ public void testCategorizeProtocolErrors() {
+ // Test ProtocolException (Nutch-specific)
+ assertEquals(ErrorType.PROTOCOL,
+ ErrorTracker.categorize(new ProtocolException("Protocol error")));
+
+ // Test ProtocolNotFound (Nutch-specific)
+ assertEquals(ErrorType.PROTOCOL,
+ ErrorTracker.categorize(new ProtocolNotFound("ftp")));
+ }
+
+ // =========================================================================
+ // Parsing Error Categorization Tests
+ // =========================================================================
+
+ @Test
+ public void testCategorizeParsingErrors() {
+ // Test ParseException (Nutch-specific)
+ assertEquals(ErrorType.PARSING,
+ ErrorTracker.categorize(new ParseException("Parse failed")));
+
+ // Test ParserNotFound (Nutch-specific)
+ assertEquals(ErrorType.PARSING,
+ ErrorTracker.categorize(new ParserNotFound("text/unknown")));
+
+ // Test SAXException
+ assertEquals(ErrorType.PARSING,
+ ErrorTracker.categorize(new SAXException("XML parse error")));
+ }
+
+ // =========================================================================
+ // Scoring Error Categorization Tests
+ // =========================================================================
+
+ @Test
+ public void testCategorizeScoringErrors() {
+ // Test ScoringFilterException (Nutch-specific)
+ assertEquals(ErrorType.SCORING,
+ ErrorTracker.categorize(new ScoringFilterException("Scoring failed")));
+ }
+
+ // =========================================================================
+ // Indexing Error Categorization Tests
+ // =========================================================================
+
+ @Test
+ public void testCategorizeIndexingErrors() {
+ // Test IndexingException (Nutch-specific)
+ assertEquals(ErrorType.INDEXING,
+ ErrorTracker.categorize(new IndexingException("Indexing failed")));
+ }
+
+ // =========================================================================
+ // Other/Fallback Categorization Tests
+ // =========================================================================
+
+ @Test
+ public void testCategorizeNullThrowable() {
+ // Null should return OTHER
+ assertEquals(ErrorType.OTHER, ErrorTracker.categorize(null));
+ }
+
+ @Test
+ public void testCategorizeGenericException() {
+ // Generic Exception should return OTHER
+ assertEquals(ErrorType.OTHER,
+ ErrorTracker.categorize(new Exception("Generic error")));
+
+ // RuntimeException should return OTHER
+ assertEquals(ErrorType.OTHER,
+ ErrorTracker.categorize(new RuntimeException("Runtime error")));
+ }
+
+ // =========================================================================
+ // Cause Chain Categorization Tests
+ // =========================================================================
+
+ @Test
+ public void testCategorizeCauseChain() {
+ // Exception with a network cause should be categorized as NETWORK
+ IOException cause = new IOException("Root cause");
+ Exception wrapper = new Exception("Wrapper", cause);
+ assertEquals(ErrorType.NETWORK, ErrorTracker.categorize(wrapper));
+
+ // Exception with a timeout cause should be categorized as TIMEOUT
+ SocketTimeoutException timeoutCause = new SocketTimeoutException("Timeout");
+ Exception timeoutWrapper = new Exception("Wrapper", timeoutCause);
+ assertEquals(ErrorType.TIMEOUT, ErrorTracker.categorize(timeoutWrapper));
+ }
+
+ @Test
+ public void testCategorizeNestedCauseChain() {
+ // Deep nested cause chain: RuntimeException -> Exception -> IOException
+ IOException rootCause = new IOException("Root cause");
+ Exception middleWrapper = new Exception("Middle", rootCause);
+ RuntimeException outerWrapper = new RuntimeException("Outer", middleWrapper);
+ assertEquals(ErrorType.NETWORK, ErrorTracker.categorize(outerWrapper));
+
+ // Deep nested with Nutch-specific exception
+ ScoringFilterException scoringCause = new ScoringFilterException("Scoring error");
+ Exception wrapper1 = new Exception("Wrapper 1", scoringCause);
+ Exception wrapper2 = new Exception("Wrapper 2", wrapper1);
+ assertEquals(ErrorType.SCORING, ErrorTracker.categorize(wrapper2));
+ }
+
+ // =========================================================================
+ // Record Error Tests (Local Accumulation)
+ // =========================================================================
+
+ @Test
+ public void testRecordErrorByType() {
+ ErrorTracker tracker = new ErrorTracker(NutchMetrics.GROUP_FETCHER);
+
+ // Initially all counts should be 0
+ assertEquals(0, tracker.getTotalCount());
+ assertEquals(0, tracker.getCount(ErrorType.NETWORK));
+
+ // Record a NETWORK error
+ tracker.recordError(ErrorType.NETWORK);
+ assertEquals(1, tracker.getTotalCount());
+ assertEquals(1, tracker.getCount(ErrorType.NETWORK));
+ assertEquals(0, tracker.getCount(ErrorType.TIMEOUT));
+
+ // Record another NETWORK error
+ tracker.recordError(ErrorType.NETWORK);
+ assertEquals(2, tracker.getTotalCount());
+ assertEquals(2, tracker.getCount(ErrorType.NETWORK));
+
+ // Record a TIMEOUT error
+ tracker.recordError(ErrorType.TIMEOUT);
+ assertEquals(3, tracker.getTotalCount());
+ assertEquals(2, tracker.getCount(ErrorType.NETWORK));
+ assertEquals(1, tracker.getCount(ErrorType.TIMEOUT));
+ }
+
+ @Test
+ public void testRecordErrorByThrowable() {
+ ErrorTracker tracker = new ErrorTracker(NutchMetrics.GROUP_FETCHER);
+
+ // Record an IOException (should be categorized as NETWORK)
+ tracker.recordError(new IOException("Test"));
+ assertEquals(1, tracker.getTotalCount());
+ assertEquals(1, tracker.getCount(ErrorType.NETWORK));
+
+ // Record a SocketTimeoutException (should be categorized as TIMEOUT)
+ tracker.recordError(new SocketTimeoutException("Test"));
+ assertEquals(2, tracker.getTotalCount());
+ assertEquals(1, tracker.getCount(ErrorType.TIMEOUT));
+
+ // Record a MalformedURLException (should be categorized as URL)
+ tracker.recordError(new MalformedURLException("Test"));
+ assertEquals(3, tracker.getTotalCount());
+ assertEquals(1, tracker.getCount(ErrorType.URL));
+ }
+
+ // =========================================================================
+ // Counter Name Mapping Tests
+ // =========================================================================
+
+ @Test
+ public void testGetCounterName() {
+ // Test counter name mapping
+ assertEquals(NutchMetrics.ERROR_NETWORK_TOTAL,
+ ErrorTracker.getCounterName(ErrorType.NETWORK));
+ assertEquals(NutchMetrics.ERROR_PROTOCOL_TOTAL,
+ ErrorTracker.getCounterName(ErrorType.PROTOCOL));
+ assertEquals(NutchMetrics.ERROR_PARSING_TOTAL,
+ ErrorTracker.getCounterName(ErrorType.PARSING));
+ assertEquals(NutchMetrics.ERROR_URL_TOTAL,
+ ErrorTracker.getCounterName(ErrorType.URL));
+ assertEquals(NutchMetrics.ERROR_SCORING_TOTAL,
+ ErrorTracker.getCounterName(ErrorType.SCORING));
+ assertEquals(NutchMetrics.ERROR_INDEXING_TOTAL,
+ ErrorTracker.getCounterName(ErrorType.INDEXING));
+ assertEquals(NutchMetrics.ERROR_TIMEOUT_TOTAL,
+ ErrorTracker.getCounterName(ErrorType.TIMEOUT));
+ assertEquals(NutchMetrics.ERROR_OTHER_TOTAL,
+ ErrorTracker.getCounterName(ErrorType.OTHER));
+ }
+
+ @Test
+ public void testGetCounterNameForThrowable() {
+ // Test getting counter name directly from throwable
+ assertEquals(NutchMetrics.ERROR_NETWORK_TOTAL,
+ ErrorTracker.getCounterName(new IOException("Test")));
+ assertEquals(NutchMetrics.ERROR_TIMEOUT_TOTAL,
+ ErrorTracker.getCounterName(new SocketTimeoutException("Test")));
+ assertEquals(NutchMetrics.ERROR_URL_TOTAL,
+ ErrorTracker.getCounterName(new MalformedURLException("Test")));
+ assertEquals(NutchMetrics.ERROR_OTHER_TOTAL,
+ ErrorTracker.getCounterName(new RuntimeException("Test")));
+
+ // Test Nutch-specific exceptions
+ assertEquals(NutchMetrics.ERROR_PROTOCOL_TOTAL,
+ ErrorTracker.getCounterName(new ProtocolException("Test")));
+ assertEquals(NutchMetrics.ERROR_PARSING_TOTAL,
+ ErrorTracker.getCounterName(new ParseException("Test")));
+ assertEquals(NutchMetrics.ERROR_SCORING_TOTAL,
+ ErrorTracker.getCounterName(new ScoringFilterException("Test")));
+ assertEquals(NutchMetrics.ERROR_INDEXING_TOTAL,
+ ErrorTracker.getCounterName(new IndexingException("Test")));
+ }
+
+ // =========================================================================
+ // Hadoop Context Integration Tests (Using Mocks)
+ // =========================================================================
+
+ @Test
+ public void testConstructorWithContext() {
+ // Create ErrorTracker with context - should initialize counters
+ ErrorTracker tracker = new ErrorTracker(NutchMetrics.GROUP_FETCHER, mockContext);
+
+ // Verify counters were requested from context
+ // Total counter + 8 error type counters = 9 calls
+ verify(mockContext, atLeast(9)).getCounter(anyString(), anyString());
+ }
+
+ @Test
+ public void testInitCounters() {
+ ErrorTracker tracker = new ErrorTracker(NutchMetrics.GROUP_FETCHER);
+
+ // Initialize counters
+ tracker.initCounters(mockContext);
+
+ // Verify counters were requested
+ verify(mockContext).getCounter(NutchMetrics.GROUP_FETCHER, NutchMetrics.ERROR_TOTAL);
+ verify(mockContext).getCounter(NutchMetrics.GROUP_FETCHER, NutchMetrics.ERROR_NETWORK_TOTAL);
+ verify(mockContext).getCounter(NutchMetrics.GROUP_FETCHER, NutchMetrics.ERROR_TIMEOUT_TOTAL);
+ }
+
+ @Test
+ public void testIncrementCountersWithType() {
+ ErrorTracker tracker = new ErrorTracker(NutchMetrics.GROUP_FETCHER, mockContext);
+
+ // Increment counters directly
+ tracker.incrementCounters(ErrorType.NETWORK);
+
+ // Verify counter was incremented (total + specific type)
+ verify(mockCounter, times(2)).increment(1);
+ }
+
+ @Test
+ public void testIncrementCountersWithThrowable() {
+ ErrorTracker tracker = new ErrorTracker(NutchMetrics.GROUP_FETCHER, mockContext);
+
+ // Increment counters with throwable
+ tracker.incrementCounters(new IOException("Test"));
+
+ // Verify counter was incremented (total + NETWORK type)
+ verify(mockCounter, times(2)).increment(1);
+ }
+
+ @Test
+ public void testIncrementCountersWithoutInit() {
+ ErrorTracker tracker = new ErrorTracker(NutchMetrics.GROUP_FETCHER);
+
+ // Should throw IllegalStateException when counters not initialized
+ assertThrows(IllegalStateException.class, () -> {
+ tracker.incrementCounters(ErrorType.NETWORK);
+ });
+ }
+
+ @Test
+ public void testEmitCounters() {
+ ErrorTracker tracker = new ErrorTracker(NutchMetrics.GROUP_FETCHER);
+
+ // Record some errors locally
+ tracker.recordError(ErrorType.NETWORK);
+ tracker.recordError(ErrorType.NETWORK);
+ tracker.recordError(ErrorType.TIMEOUT);
+
+ // Emit counters (without cached counters - uses fallback)
+ tracker.emitCounters(mockContext);
+
+ // Verify counters were requested and incremented
+ verify(mockContext).getCounter(NutchMetrics.GROUP_FETCHER, NutchMetrics.ERROR_TOTAL);
+ verify(mockContext).getCounter(NutchMetrics.GROUP_FETCHER, NutchMetrics.ERROR_NETWORK_TOTAL);
+ verify(mockContext).getCounter(NutchMetrics.GROUP_FETCHER, NutchMetrics.ERROR_TIMEOUT_TOTAL);
+ }
+
+ @Test
+ public void testEmitCountersWithCachedCounters() {
+ ErrorTracker tracker = new ErrorTracker(NutchMetrics.GROUP_FETCHER, mockContext);
+
+ // Reset mock to clear constructor calls
+ reset(mockCounter);
+
+ // Record some errors locally
+ tracker.recordError(ErrorType.NETWORK);
+ tracker.recordError(ErrorType.NETWORK);
+ tracker.recordError(ErrorType.TIMEOUT);
+
+ // Emit counters (with cached counters)
+ tracker.emitCounters(mockContext);
+
+ // Verify cached counters were used (increment called with accumulated values)
+ verify(mockCounter).increment(3L); // total count
+ verify(mockCounter).increment(2L); // NETWORK count
+ verify(mockCounter).increment(1L); // TIMEOUT count
+ }
+
+ // =========================================================================
+ // Thread Safety Tests
+ // =========================================================================
+
+ @Test
+ public void testThreadSafety() throws InterruptedException {
+ ErrorTracker tracker = new ErrorTracker(NutchMetrics.GROUP_FETCHER);
+
+ // Create multiple threads that record errors concurrently
+ Thread[] threads = new Thread[10];
+ for (int i = 0; i < threads.length; i++) {
+ threads[i] = new Thread(() -> {
+ for (int j = 0; j < 100; j++) {
+ tracker.recordError(ErrorType.NETWORK);
+ }
+ });
+ }
+
+ // Start all threads
+ for (Thread thread : threads) {
+ thread.start();
+ }
+
+ // Wait for all threads to complete
+ for (Thread thread : threads) {
+ thread.join();
+ }
+
+ // Verify counts
+ assertEquals(1000, tracker.getTotalCount());
+ assertEquals(1000, tracker.getCount(ErrorType.NETWORK));
+ }
+
+ @Test
+ public void testThreadSafetyMixedErrorTypes() throws InterruptedException {
+ ErrorTracker tracker = new ErrorTracker(NutchMetrics.GROUP_FETCHER);
+
+ // Create threads that record different error types concurrently
+ Thread networkThread = new Thread(() -> {
+ for (int i = 0; i < 500; i++) {
+ tracker.recordError(ErrorType.NETWORK);
+ }
+ });
+
+ Thread timeoutThread = new Thread(() -> {
+ for (int i = 0; i < 300; i++) {
+ tracker.recordError(ErrorType.TIMEOUT);
+ }
+ });
+
+ Thread urlThread = new Thread(() -> {
+ for (int i = 0; i < 200; i++) {
+ tracker.recordError(ErrorType.URL);
+ }
+ });
+
+ networkThread.start();
+ timeoutThread.start();
+ urlThread.start();
+
+ networkThread.join();
+ timeoutThread.join();
+ urlThread.join();
+
+ // Verify counts
+ assertEquals(1000, tracker.getTotalCount());
+ assertEquals(500, tracker.getCount(ErrorType.NETWORK));
+ assertEquals(300, tracker.getCount(ErrorType.TIMEOUT));
+ assertEquals(200, tracker.getCount(ErrorType.URL));
+ }
+}
diff --git a/src/test/org/apache/nutch/util/TestURLUtil.java b/src/test/org/apache/nutch/util/TestURLUtil.java
index b14b55af09..4d8ae07971 100644
--- a/src/test/org/apache/nutch/util/TestURLUtil.java
+++ b/src/test/org/apache/nutch/util/TestURLUtil.java
@@ -146,7 +146,9 @@ public void testGetDomainSuffix() throws Exception {
url = new URL("http://www.example.2000.hu");
assertEquals("2000.hu", URLUtil.getDomainSuffix(url));
- // test non-ascii
+ // test non-ASCII
+ url = new URL("https://www.taiuru.māori.nz/");
+ assertEquals("xn--mori-qsa.nz", URLUtil.getDomainSuffix(url));
url = new URL("http://www.example.flå.no");
assertEquals("xn--fl-zia.no", URLUtil.getDomainSuffix(url));
url = new URL("http://www.example.栃木.jp");