FTP passive mode with Testcontainers requires that the PASV response IP + * matches the host-visible address of the container, which is not reliable + * across Docker Desktop (macOS/Windows) and Linux Docker environments. An + * in-process {@link FakeFtpServer} from MockFtpServer avoids this constraint + * while still testing the Nutch FTP client against a real FTP protocol + * implementation. + */ +public class FtpProtocolIT implements ProtocolPluginIntegrationTest { + + private static final String FTP_USER = "testuser"; + private static final String FTP_PASS = "testpass"; + private static final String FTP_HOME = "/home/testuser"; + private static final String TEST_FILE = "test.txt"; + private static final String TEST_CONTENT = "FTP integration test content"; + + private static FakeFtpServer fakeFtpServer; + private Ftp protocol; + + @BeforeAll + static void startFtpServer() { + fakeFtpServer = new FakeFtpServer(); + fakeFtpServer.setServerControlPort(0); // bind to a random free port + + UserAccount userAccount = new UserAccount(FTP_USER, FTP_PASS, FTP_HOME); + fakeFtpServer.addUserAccount(userAccount); + + UnixFakeFileSystem fileSystem = new UnixFakeFileSystem(); + fileSystem.add(new DirectoryEntry(FTP_HOME)); + fileSystem.add(new FileEntry(FTP_HOME + "/" + TEST_FILE, TEST_CONTENT)); + fakeFtpServer.setFileSystem(fileSystem); + + fakeFtpServer.start(); + } + + @AfterAll + static void stopFtpServer() { + if (fakeFtpServer != null) { + fakeFtpServer.stop(); + } + } + + @BeforeEach + @Override + public void setUpProtocol() throws Exception { + Configuration conf = NutchConfiguration.create(); + conf.set("plugin.includes", "protocol-ftp|nutch-extensionpoints"); + conf.set("http.agent.name", "NutchFtpProtocolIT"); + conf.set("ftp.username", FTP_USER); + conf.set("ftp.password", FTP_PASS); + conf.setInt("ftp.timeout", 10000); + protocol = new Ftp(); + protocol.setConf(conf); + } + + @AfterEach + @Override + public void tearDownProtocol() { + protocol = null; + } + + @Override + public Protocol getProtocol() { + return protocol; + } + + @Override + public String getTestUrl() { + return "ftp://localhost:" + fakeFtpServer.getServerControlPort() + + FTP_HOME + "/" + TEST_FILE; + } + + @Test + void testFtpFileDownload() throws Exception { + CrawlDatum datum = new CrawlDatum(); + ProtocolOutput output = protocol.getProtocolOutput(new Text(getTestUrl()), datum); + + assertNotNull(output, "ProtocolOutput must not be null"); + int code = Integer.parseInt( + datum.getMetaData().get(Nutch.PROTOCOL_STATUS_CODE_KEY).toString()); + assertEquals(200, code, "Expected FTP 200 for file download"); + + assertNotNull(output.getContent(), "Content must not be null"); + String body = new String(output.getContent().getContent(), StandardCharsets.UTF_8); + assertTrue(body.contains(TEST_CONTENT), + "Downloaded content must match the file on the FTP server"); + } + + @Test + void testFtpDirectoryListing() throws Exception { + String dirUrl = "ftp://localhost:" + fakeFtpServer.getServerControlPort() + + FTP_HOME + "/"; + CrawlDatum datum = new CrawlDatum(); + ProtocolOutput output = protocol.getProtocolOutput(new Text(dirUrl), datum); + + assertNotNull(output, "ProtocolOutput for directory listing must not be null"); + int code = Integer.parseInt( + datum.getMetaData().get(Nutch.PROTOCOL_STATUS_CODE_KEY).toString()); + assertEquals(200, code, "Expected FTP 200 for directory listing"); + } + + @Test + void testFtpMissingFileReturnsError() throws Exception { + String missingUrl = "ftp://localhost:" + fakeFtpServer.getServerControlPort() + + FTP_HOME + "/nonexistent.txt"; + CrawlDatum datum = new CrawlDatum(); + ProtocolOutput output = protocol.getProtocolOutput(new Text(missingUrl), datum); + assertNotNull(output, "ProtocolOutput must not be null even for missing files"); + // FTP 550 "No such file" maps to a non-200 Nutch status + int code = Integer.parseInt( + datum.getMetaData().get(Nutch.PROTOCOL_STATUS_CODE_KEY).toString()); + assertTrue(code != 200, "Expected non-200 code for missing FTP file, got: " + code); + } +} diff --git a/src/plugin/protocol-htmlunit/src/test/org/apache/nutch/protocol/htmlunit/HtmlUnitProtocolIT.java b/src/plugin/protocol-htmlunit/src/test/org/apache/nutch/protocol/htmlunit/HtmlUnitProtocolIT.java new file mode 100644 index 0000000000..42c551b72f --- /dev/null +++ b/src/plugin/protocol-htmlunit/src/test/org/apache/nutch/protocol/htmlunit/HtmlUnitProtocolIT.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nutch.protocol.htmlunit; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import org.apache.hadoop.conf.Configuration; +import org.apache.nutch.crawl.CrawlDatum; +import org.apache.nutch.protocol.AbstractProtocolPluginIT; +import org.apache.nutch.protocol.Protocol; +import org.apache.nutch.protocol.ProtocolOutput; +import org.apache.nutch.util.NutchConfiguration; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +/** + * Integration tests for protocol-htmlunit using a real nginx container. + */ +@Testcontainers(disabledWithoutDocker = true) +public class HtmlUnitProtocolIT extends AbstractProtocolPluginIT { + + @Container + private static final GenericContainer nginx = + new GenericContainer<>("nginx:alpine").withExposedPorts(80); + + private Http protocol; + + @Override + public void setUpProtocol() throws Exception { + Configuration conf = NutchConfiguration.create(); + conf.set("plugin.includes", + "protocol-htmlunit|lib-htmlunit|lib-http|nutch-extensionpoints"); + conf.set("http.agent.name", "Nutch-Test"); + conf.setInt("http.timeout", 10000); + conf.setBoolean("store.http.headers", true); + protocol = new Http(); + protocol.setConf(conf); + } + + @Override + public void tearDownProtocol() { + protocol = null; + } + + @Override + public Protocol getProtocol() { + return protocol; + } + + @Override + public String getTestUrl() { + return "http://" + nginx.getHost() + ":" + nginx.getMappedPort(80) + "/"; + } + + @Test + void testFetchReturnsContent() throws Exception { + CrawlDatum datum = new CrawlDatum(); + ProtocolOutput output = protocol.getProtocolOutput( + new org.apache.hadoop.io.Text(getTestUrl()), datum); + assertNotNull(output.getContent(), + "protocol-htmlunit must return non-null content for a live nginx page"); + } +} diff --git a/src/plugin/protocol-http/src/test/org/apache/nutch/protocol/http/HttpProtocolIT.java b/src/plugin/protocol-http/src/test/org/apache/nutch/protocol/http/HttpProtocolIT.java new file mode 100644 index 0000000000..87db32335b --- /dev/null +++ b/src/plugin/protocol-http/src/test/org/apache/nutch/protocol/http/HttpProtocolIT.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nutch.protocol.http; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Text; +import org.apache.nutch.crawl.CrawlDatum; +import org.apache.nutch.protocol.AbstractProtocolPluginIT; +import org.apache.nutch.protocol.Protocol; +import org.apache.nutch.protocol.ProtocolOutput; +import org.apache.nutch.util.NutchConfiguration; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +/** + * Integration tests for protocol-http using a real nginx container. + */ +@Testcontainers(disabledWithoutDocker = true) +public class HttpProtocolIT extends AbstractProtocolPluginIT { + + @Container + private static final GenericContainer nginx = + new GenericContainer<>("nginx:alpine").withExposedPorts(80); + + private Http protocol; + + @Override + public void setUpProtocol() throws Exception { + Configuration conf = NutchConfiguration.create(); + conf.set("plugin.includes", "protocol-http|lib-http|nutch-extensionpoints"); + conf.set("http.agent.name", "Nutch-Test"); + conf.setInt("http.timeout", 10000); + conf.setBoolean("store.http.headers", true); + protocol = new Http(); + protocol.setConf(conf); + } + + @Override + public void tearDownProtocol() { + protocol = null; + } + + @Override + public Protocol getProtocol() { + return protocol; + } + + @Override + public String getTestUrl() { + return "http://" + nginx.getHost() + ":" + nginx.getMappedPort(80) + "/"; + } + + @Test + void testFetchRedirect301() throws Exception { + // nginx returns 301 for directory URLs without trailing slash when autoindex + // is off; test a manual redirect via the default nginx welcome page path + String redirectUrl = + "http://" + nginx.getHost() + ":" + nginx.getMappedPort(80) + "/index.html"; + CrawlDatum datum = new CrawlDatum(); + protocol.getProtocolOutput(new Text(redirectUrl), datum); + int code = getHttpStatusCode(datum); + // nginx serves index.html directly with 200; the base test covers 200/404 + assertEquals(200, code, "Expected 200 for index.html from nginx"); + } +} diff --git a/src/plugin/protocol-httpclient/ivy.xml b/src/plugin/protocol-httpclient/ivy.xml index 0b3ce0af73..e5987074b8 100644 --- a/src/plugin/protocol-httpclient/ivy.xml +++ b/src/plugin/protocol-httpclient/ivy.xml @@ -38,6 +38,7 @@ + diff --git a/src/plugin/protocol-httpclient/src/test/org/apache/nutch/protocol/httpclient/HttpClientProtocolIT.java b/src/plugin/protocol-httpclient/src/test/org/apache/nutch/protocol/httpclient/HttpClientProtocolIT.java new file mode 100644 index 0000000000..7345e4b029 --- /dev/null +++ b/src/plugin/protocol-httpclient/src/test/org/apache/nutch/protocol/httpclient/HttpClientProtocolIT.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nutch.protocol.httpclient; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.core.WireMockConfiguration; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Text; +import org.apache.nutch.crawl.CrawlDatum; +import org.apache.nutch.metadata.Nutch; +import org.apache.nutch.protocol.Protocol; +import org.apache.nutch.protocol.ProtocolOutput; +import org.apache.nutch.protocol.ProtocolPluginIntegrationTest; +import org.apache.nutch.util.NutchConfiguration; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Integration tests for protocol-httpclient using an in-process WireMock + * server. + * + *

WireMock runs in the test JVM so no Docker container is required. The + * Nutch httpclient plugin connects to it over a real TCP socket, exercising + * the full HTTP client stack including header handling and Basic-auth + * challenge/response. + */ +public class HttpClientProtocolIT implements ProtocolPluginIntegrationTest { + + private static WireMockServer wireMock; + private Http protocol; + + @BeforeAll + static void startWireMock() { + wireMock = new WireMockServer(WireMockConfiguration.options().dynamicPort()); + wireMock.start(); + + wireMock.stubFor(get(urlEqualTo("/")) + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Type", "text/html") + .withBody("Integration test"))); + + wireMock.stubFor(get(urlEqualTo("/notfound")) + .willReturn(aResponse().withStatus(404))); + + wireMock.stubFor(get(urlEqualTo("/secure")) + .withBasicAuth("testuser", "testpass") + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Type", "text/html") + .withBody("Authenticated"))); + + wireMock.stubFor(get(urlEqualTo("/secure")) + .willReturn(aResponse() + .withStatus(401) + .withHeader("WWW-Authenticate", "Basic realm=\"Test\"") + .withBody("Unauthorized"))); + } + + @AfterAll + static void stopWireMock() { + if (wireMock != null) { + wireMock.stop(); + } + } + + @BeforeEach + @Override + public void setUpProtocol() throws Exception { + Configuration conf = NutchConfiguration.create(); + conf.set("plugin.includes", + "protocol-httpclient|lib-http|nutch-extensionpoints"); + conf.set("http.agent.name", "Nutch-Test"); + conf.setInt("http.timeout", 10000); + conf.setBoolean("store.http.headers", true); + protocol = new Http(); + protocol.setConf(conf); + } + + @AfterEach + @Override + public void tearDownProtocol() { + protocol = null; + } + + @Override + public Protocol getProtocol() { + return protocol; + } + + @Override + public String getTestUrl() { + return "http://localhost:" + wireMock.port() + "/"; + } + + @Test + void testFetch200() throws Exception { + CrawlDatum datum = new CrawlDatum(); + ProtocolOutput output = protocol.getProtocolOutput(new Text(getTestUrl()), datum); + assertNotNull(output, "ProtocolOutput must not be null"); + int code = Integer.parseInt( + datum.getMetaData().get(Nutch.PROTOCOL_STATUS_CODE_KEY).toString()); + assertEquals(200, code, "Expected HTTP 200 from WireMock stub"); + } + + @Test + void testFetch404() throws Exception { + String url = "http://localhost:" + wireMock.port() + "/notfound"; + CrawlDatum datum = new CrawlDatum(); + protocol.getProtocolOutput(new Text(url), datum); + int code = Integer.parseInt( + datum.getMetaData().get(Nutch.PROTOCOL_STATUS_CODE_KEY).toString()); + assertEquals(404, code, "Expected HTTP 404 for /notfound stub"); + } + + @Test + void testUnauthenticatedRequestReturns401() throws Exception { + String secureUrl = "http://localhost:" + wireMock.port() + "/secure"; + CrawlDatum datum = new CrawlDatum(); + protocol.getProtocolOutput(new Text(secureUrl), datum); + int code = Integer.parseInt( + datum.getMetaData().get(Nutch.PROTOCOL_STATUS_CODE_KEY).toString()); + assertEquals(401, code, + "Unauthenticated request to /secure should return 401"); + } +} diff --git a/src/plugin/protocol-okhttp/src/test/org/apache/nutch/protocol/okhttp/OkHttpProtocolIT.java b/src/plugin/protocol-okhttp/src/test/org/apache/nutch/protocol/okhttp/OkHttpProtocolIT.java new file mode 100644 index 0000000000..d5342d8309 --- /dev/null +++ b/src/plugin/protocol-okhttp/src/test/org/apache/nutch/protocol/okhttp/OkHttpProtocolIT.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nutch.protocol.okhttp; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import org.apache.hadoop.conf.Configuration; +import org.apache.nutch.crawl.CrawlDatum; +import org.apache.nutch.protocol.AbstractProtocolPluginIT; +import org.apache.nutch.protocol.Protocol; +import org.apache.nutch.protocol.ProtocolOutput; +import org.apache.nutch.util.NutchConfiguration; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +/** + * Integration tests for protocol-okhttp using a real nginx container. + */ +@Testcontainers(disabledWithoutDocker = true) +public class OkHttpProtocolIT extends AbstractProtocolPluginIT { + + @Container + private static final GenericContainer nginx = + new GenericContainer<>("nginx:alpine").withExposedPorts(80); + + private OkHttp protocol; + + @Override + public void setUpProtocol() throws Exception { + Configuration conf = NutchConfiguration.create(); + conf.set("plugin.includes", "protocol-okhttp|lib-http|nutch-extensionpoints"); + conf.set("http.agent.name", "Nutch-Test"); + conf.setInt("http.timeout", 10000); + conf.setBoolean("store.http.headers", true); + protocol = new OkHttp(); + protocol.setConf(conf); + } + + @Override + public void tearDownProtocol() { + protocol = null; + } + + @Override + public Protocol getProtocol() { + return protocol; + } + + @Override + public String getTestUrl() { + return "http://" + nginx.getHost() + ":" + nginx.getMappedPort(80) + "/"; + } + + /** OkHttp transparently decompresses gzip; verify content is returned. */ + @Test + void testFetchWithAcceptEncoding() throws Exception { + CrawlDatum datum = new CrawlDatum(); + ProtocolOutput output = protocol.getProtocolOutput( + new org.apache.hadoop.io.Text(getTestUrl()), datum); + assertNotNull(output.getContent(), + "Content must be present even when server uses compression"); + } +} diff --git a/src/plugin/protocol-selenium/src/test/org/apache/nutch/protocol/selenium/SeleniumProtocolIT.java b/src/plugin/protocol-selenium/src/test/org/apache/nutch/protocol/selenium/SeleniumProtocolIT.java new file mode 100644 index 0000000000..ec928df64f --- /dev/null +++ b/src/plugin/protocol-selenium/src/test/org/apache/nutch/protocol/selenium/SeleniumProtocolIT.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nutch.protocol.selenium; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import org.apache.hadoop.conf.Configuration; +import org.apache.nutch.crawl.CrawlDatum; +import org.apache.nutch.protocol.AbstractProtocolPluginIT; +import org.apache.nutch.protocol.Protocol; +import org.apache.nutch.protocol.ProtocolOutput; +import org.apache.nutch.util.NutchConfiguration; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +/** + * Integration tests for protocol-selenium using a real nginx container. + * + *

Note: protocol-selenium uses raw HTTP sockets (the same underlying + * transport as protocol-http) rather than a Selenium WebDriver. Tests here + * validate that the plugin connects to and fetches content from a live HTTP + * server. Browser-based rendering is covered by protocol-interactiveselenium + * which is excluded from automated integration tests due to its stateful + * handler requirements. + */ +@Testcontainers(disabledWithoutDocker = true) +public class SeleniumProtocolIT extends AbstractProtocolPluginIT { + + @Container + private static final GenericContainer nginx = + new GenericContainer<>("nginx:alpine").withExposedPorts(80); + + private Http protocol; + + @Override + public void setUpProtocol() throws Exception { + Configuration conf = NutchConfiguration.create(); + conf.set("plugin.includes", + "protocol-selenium|lib-http|lib-selenium|nutch-extensionpoints"); + conf.set("http.agent.name", "Nutch-Test"); + conf.setInt("http.timeout", 10000); + conf.setBoolean("store.http.headers", true); + protocol = new Http(); + protocol.setConf(conf); + } + + @Override + public void tearDownProtocol() { + protocol = null; + } + + @Override + public Protocol getProtocol() { + return protocol; + } + + @Override + public String getTestUrl() { + return "http://" + nginx.getHost() + ":" + nginx.getMappedPort(80) + "/"; + } + + @Test + void testFetchReturnsContent() throws Exception { + CrawlDatum datum = new CrawlDatum(); + ProtocolOutput output = protocol.getProtocolOutput( + new org.apache.hadoop.io.Text(getTestUrl()), datum); + assertNotNull(output.getContent(), + "protocol-selenium must return non-null content for a live nginx page"); + } +} diff --git a/src/test/org/apache/nutch/protocol/AbstractProtocolPluginIT.java b/src/test/org/apache/nutch/protocol/AbstractProtocolPluginIT.java new file mode 100644 index 0000000000..9469b168fb --- /dev/null +++ b/src/test/org/apache/nutch/protocol/AbstractProtocolPluginIT.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nutch.protocol; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import org.apache.hadoop.io.Text; +import org.apache.nutch.crawl.CrawlDatum; +import org.apache.nutch.metadata.Nutch; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.junit.jupiter.Testcontainers; + +/** + * Abstract base for Protocol plugin integration tests using Testcontainers. + * Provides common test logic for fetching URLs and verifying status codes. + * + *

Subclasses declare a static {@code @Container} field for the server + * container, implement {@link ProtocolPluginIntegrationTest}, and may add + * protocol-specific tests (e.g., redirect handling, authentication). + */ +@Testcontainers(disabledWithoutDocker = true) +public abstract class AbstractProtocolPluginIT implements ProtocolPluginIntegrationTest { + + @BeforeEach + void setUp() throws Exception { + setUpProtocol(); + } + + @AfterEach + void tearDown() throws Exception { + tearDownProtocol(); + } + + /** Fetch the test URL and assert an HTTP 200 response. */ + @Test + void testFetch200() throws Exception { + CrawlDatum datum = new CrawlDatum(); + ProtocolOutput output = getProtocol() + .getProtocolOutput(new Text(getTestUrl()), datum); + assertNotNull(output, "ProtocolOutput must not be null"); + assertEquals(200, getHttpStatusCode(datum), + "Expected HTTP 200 for " + getTestUrl()); + verifyFetchedContent(output, datum); + } + + /** Fetch a non-existent path and assert an HTTP 404 response. */ + @Test + void testFetch404() throws Exception { + String url = get404Url(); + CrawlDatum datum = new CrawlDatum(); + getProtocol().getProtocolOutput(new Text(url), datum); + assertEquals(404, getHttpStatusCode(datum), + "Expected HTTP 404 for " + url); + } + + /** + * Returns a URL expected to produce a 404. Default appends a random path + * segment to {@link #getTestUrl()}; override if the server needs a specific + * path. + */ + protected String get404Url() { + String base = getTestUrl(); + if (base.endsWith("/")) { + return base + "nonexistent-path-xyz"; + } + return base + "/nonexistent-path-xyz"; + } + + /** + * Reads the HTTP status code stored in the CrawlDatum metadata by Nutch + * protocol plugins. Returns -1 if no status code was stored. + */ + protected static int getHttpStatusCode(CrawlDatum datum) { + if (datum.getMetaData().containsKey(Nutch.PROTOCOL_STATUS_CODE_KEY)) { + return Integer.parseInt( + datum.getMetaData().get(Nutch.PROTOCOL_STATUS_CODE_KEY).toString()); + } + return -1; + } +} diff --git a/src/test/org/apache/nutch/protocol/ProtocolPluginIntegrationTest.java b/src/test/org/apache/nutch/protocol/ProtocolPluginIntegrationTest.java new file mode 100644 index 0000000000..b3778077d9 --- /dev/null +++ b/src/test/org/apache/nutch/protocol/ProtocolPluginIntegrationTest.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nutch.protocol; + +import org.apache.nutch.crawl.CrawlDatum; + +/** + * Contract for Protocol plugin integration tests. Implementations run against + * real server backends (via Testcontainers or embedded servers). + */ +public interface ProtocolPluginIntegrationTest { + + /** Set up the protocol plugin and its backing server before tests. */ + void setUpProtocol() throws Exception; + + /** Shut down the protocol plugin after tests. */ + void tearDownProtocol() throws Exception; + + /** The Protocol under test. */ + Protocol getProtocol(); + + /** + * A URL that the backing server will serve with a 200/success response. + * Must point into the container or embedded server started by this test. + */ + String getTestUrl(); + + /** + * Optional extra verification after a successful fetch. + * Default is a no-op; override to inspect content, headers, etc. + */ + default void verifyFetchedContent(ProtocolOutput output, CrawlDatum datum) + throws Exception { + // no-op + } +} From 5cc412c38c48dda065ce7a77833dfc92cf300a92 Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Fri, 15 May 2026 19:13:12 +0200 Subject: [PATCH 6/8] Integrate NUTCH-3175 into build of CCF fork --- .github/workflows/cc-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cc-build.yml b/.github/workflows/cc-build.yml index 1e8f23a691..2ef2fb6fca 100644 --- a/.github/workflows/cc-build.yml +++ b/.github/workflows/cc-build.yml @@ -61,4 +61,4 @@ jobs: restore-keys: | ${{ runner.os }}-ivy- - name: Test - run: ant clean test -buildfile build.xml + run: ant clean test test-protocol-integration -buildfile build.xml From 98704c5c8a301d2e700b1d706e81de708522bdaf Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Sun, 17 May 2026 14:49:22 +0200 Subject: [PATCH 7/8] NUTCH-3176 URLUtil and urlnormalizer-basic: add support for IDNA2008 - URLUtil: - make IDNA2008 the default for the methods toASCII and toUNICODE - provide methods to convert host names both for IDNA2003 and IDNA2008 - also convert host to lowercase it (if not already lowercased) - urlnormalizer-basic: - convert host names using IDNA2008 if the property urlnormalizer.basic.host.idna2008 is true - refactor to share methods between URLUtil and urlnormalizer-basic - refactor calls of URLDecoder and pass Charset instead of String (since Java 10) --- conf/nutch-default.xml | 11 ++ .../apache/nutch/crawl/DeduplicationJob.java | 9 +- .../nutch/plugin/PluginManifestParser.java | 7 +- src/java/org/apache/nutch/util/URLUtil.java | 169 +++++++++++++++--- .../nutch/protocol/file/FileResponse.java | 17 +- .../nutch/protocol/ftp/FtpResponse.java | 3 +- .../urlnormalizer/ajax/AjaxURLNormalizer.java | 17 +- .../basic/BasicURLNormalizer.java | 59 +++--- .../basic/TestBasicURLNormalizer.java | 66 +++++++ .../org/apache/nutch/util/TestURLUtil.java | 126 ++++++++++++- 10 files changed, 397 insertions(+), 87 deletions(-) diff --git a/conf/nutch-default.xml b/conf/nutch-default.xml index 7a8a73f81b..8ae225735d 100644 --- a/conf/nutch-default.xml +++ b/conf/nutch-default.xml @@ -1657,6 +1657,17 @@ + + urlnormalizer.basic.host.idna2008 + false + If true, let urlnormalizer-basic + normalize Internationalized Domain Names (IDNs) using the + standard IDNA2008 (RFC 5890). If false, use IDNA2003 (RFC 3490). + Note that urlnormalizer.basic.host.idn must be set, otherwise + this property has no effect. + + + urlnormalizer.basic.host.trim-trailing-dot false diff --git a/src/java/org/apache/nutch/crawl/DeduplicationJob.java b/src/java/org/apache/nutch/crawl/DeduplicationJob.java index 50aa4cd7bd..52bf422308 100644 --- a/src/java/org/apache/nutch/crawl/DeduplicationJob.java +++ b/src/java/org/apache/nutch/crawl/DeduplicationJob.java @@ -17,9 +17,9 @@ package org.apache.nutch.crawl; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.lang.invoke.MethodHandles; import java.net.URLDecoder; +import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; @@ -34,7 +34,6 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapreduce.Counter; -import org.apache.hadoop.mapreduce.CounterGroup; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; @@ -70,7 +69,7 @@ public class DeduplicationJob extends NutchTool implements Tool { protected final static Text urlKey = new Text("_URLTEMPKEY_"); protected final static String DEDUPLICATION_GROUP_MODE = "deduplication.group.mode"; protected final static String DEDUPLICATION_COMPARE_ORDER = "deduplication.compare.order"; - protected final static String UTF_8 = StandardCharsets.UTF_8.toString(); + protected final static Charset UTF_8 = StandardCharsets.UTF_8; public static class DBFilter extends Mapper { @@ -224,13 +223,13 @@ protected CrawlDatum getDuplicate(CrawlDatum existingDoc, CrawlDatum newDoc) { String urlnewDoc = newDoc.getMetaData().get(urlKey).toString(); try { urlExisting = URLDecoder.decode(urlExisting, UTF_8); - } catch (UnsupportedEncodingException | IllegalArgumentException e) { + } catch (IllegalArgumentException e) { LOG.error("Error decoding: {}", urlExisting, e); // use the encoded URL } try { urlnewDoc = URLDecoder.decode(urlnewDoc, UTF_8); - } catch (UnsupportedEncodingException | IllegalArgumentException e) { + } catch (IllegalArgumentException e) { LOG.error("Error decoding: {}", urlnewDoc, e); // use the encoded URL } diff --git a/src/java/org/apache/nutch/plugin/PluginManifestParser.java b/src/java/org/apache/nutch/plugin/PluginManifestParser.java index 10ce4fdb7b..95208fa433 100644 --- a/src/java/org/apache/nutch/plugin/PluginManifestParser.java +++ b/src/java/org/apache/nutch/plugin/PluginManifestParser.java @@ -18,11 +18,11 @@ import java.io.File; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.lang.invoke.MethodHandles; import java.net.MalformedURLException; import java.net.URL; import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; @@ -124,10 +124,7 @@ public File getPluginFolder(String name) { String path = url.getPath(); if (WINDOWS && path.startsWith("/")) // patch a windows bug path = path.substring(1); - try { - path = URLDecoder.decode(path, "UTF-8"); // decode the url path - } catch (UnsupportedEncodingException e) { - } + path = URLDecoder.decode(path, StandardCharsets.UTF_8); // decode the url path directory = new File(path); } else if (!directory.exists()) { LOG.warn("Plugins: directory not found: {}", name); diff --git a/src/java/org/apache/nutch/util/URLUtil.java b/src/java/org/apache/nutch/util/URLUtil.java index 44c6309d2a..fd036480a6 100644 --- a/src/java/org/apache/nutch/util/URLUtil.java +++ b/src/java/org/apache/nutch/util/URLUtil.java @@ -16,6 +16,7 @@ */ package org.apache.nutch.util; +import java.lang.invoke.MethodHandles; import java.net.IDN; import java.net.MalformedURLException; import java.net.URI; @@ -23,11 +24,22 @@ import java.util.Locale; import java.util.regex.Pattern; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.ibm.icu.text.IDNA; + import crawlercommons.domains.EffectiveTldFinder; /** Utility class for URL analysis */ public class URLUtil { + private static final Logger LOG = LoggerFactory + .getLogger(MethodHandles.lookup().lookupClass()); + + private static final IDNA idna = IDNA.getUTS46Instance( + IDNA.NONTRANSITIONAL_TO_ASCII | IDNA.NONTRANSITIONAL_TO_UNICODE); + /** * Resolve relative URL-s and fix a java.net.URL error in handling of URLs * with pure query targets. @@ -520,17 +532,39 @@ public static String getProtocol(URL url) { return url.getProtocol(); } + public static boolean isAscii(String str) { + char[] chars = str.toCharArray(); + for (char c : chars) { + if (c > 127) { + return false; + } + } + return true; + } + + /** + * Convert URL with IDN host/domain name into the ASCII representation. + * + * @param url + * URL string to convert + * @return URL string with ASCII host/domain name or null if conversion fails. + */ public static String toASCII(String url) { try { URL u = new URL(url); String host = u.getHost(); - if (host == null || host.isEmpty()) { - // no host name => no punycoded domain name - // also do not add additional slashes for file: URLs (NUTCH-1880) + String hostLowerCase = host.toLowerCase(Locale.ROOT); + if (host == null || host.isEmpty() + || (isAscii(host) && host.equals(hostLowerCase))) { + // - no host name => no punycoded domain name + // - also do not add additional slashes for file: URLs (NUTCH-1880) + // - do nothing if host is already ASCII-only + // - not already in lowercase => conversion also lowercases host name return url; } - URI p = new URI(u.getProtocol(), u.getUserInfo(), IDN.toASCII(host), - u.getPort(), u.getPath(), u.getQuery(), u.getRef()); + URI p = new URI(u.getProtocol(), u.getUserInfo(), + convertIDNA2008(hostLowerCase, true), u.getPort(), u.getPath(), + u.getQuery(), u.getRef()); return p.toString(); } catch (Exception e) { @@ -538,13 +572,25 @@ public static String toASCII(String url) { } } + /** + * Convert URL with IDN host/domain name to the Unicode representation. + * + * @param url + * URL string to convert + * @return URL string with Unicode host/domain name or null if conversion + * fails. + */ public static String toUNICODE(String url) { try { URL u = new URL(url); String host = u.getHost(); - if (host == null || host.isEmpty()) { - // no host name => no punycoded domain name - // also do not add additional slashes for file: URLs (NUTCH-1880) + String hostLowerCase = host.toLowerCase(Locale.ROOT); + if (host == null || host.isEmpty() + || (!hostLowerCase.contains("xn--") && host.equals(hostLowerCase))) { + // - no host name => no punycoded domain name + // - also do not add additional slashes for file: URLs (NUTCH-1880) + // - contains 'xn--' => needs conversion + // - not already in lowercase => conversion also lowercases host name return url; } StringBuilder sb = new StringBuilder(); @@ -554,7 +600,7 @@ public static String toUNICODE(String url) { sb.append(u.getUserInfo()); sb.append('@'); } - sb.append(IDN.toUnicode(host)); + sb.append(convertIDNA2008(hostLowerCase, false)); if (u.getPort() != -1) { sb.append(':'); sb.append(u.getPort()); @@ -572,22 +618,83 @@ public static String toUNICODE(String url) { } /** - * For testing - * @param args print with no args to get help + * Convert IDN host to ASCII or Unicode using Java's built-in {@link IDN} + * class. + * + * The conversion supports only IDNA2003, it does not support IDNA2008. + * However, unless the parameter strictIDNA2003 is true, the + * methods {@link IDN#toASCII(String, int)} resp. + * {@link IDN#toUnicode(String, int)} are called passing the flag + * {@link IDN#ALLOW_UNASSIGNED} to avoid that the conversion fails on + * characters not in the repertoire of Unicode 3.2. + * + * @param host + * host name to be converted (lowercase expected) + * @param toAscii + * if true convert to ASCII, otherwise to Unicode + * @param strictIDNA2003 + * if true, do + * @return converted host name + * @throws MalformedURLException + * if the conversion fails */ - public static void main(String[] args) { - - if (args.length != 1) { - System.err.println("Usage : URLUtil "); - return; + public static String convertIDNA2003(String host, boolean toAscii, + boolean strictIDNA2003) throws MalformedURLException { + try { + if (toAscii) { + return IDN.toASCII(host, strictIDNA2003 ? 0 : IDN.ALLOW_UNASSIGNED); + } else { + return IDN.toUnicode(host, strictIDNA2003 ? 0 : IDN.ALLOW_UNASSIGNED); + } + } catch (IllegalArgumentException | IndexOutOfBoundsException e) { + // IllegalArgumentException: thrown if the input string contains + // non-convertible Unicode codepoints + // IndexOutOfBoundsException: thrown (undocumented) if one "label" + // (non-ASCII dot-separated segment) is longer than 256 characters, + // cf. https://bugs.openjdk.java.net/browse/JDK-6806873 + LOG.debug("Failed to convert IDN host {}: ", host, e); + throw (MalformedURLException) new MalformedURLException( + "Invalid IDN " + host + ": " + e.getMessage()).initCause(e); } + } - String url = args[0]; - try { - System.out.println(URLUtil.getDomainName(new URL(url))); - } catch (MalformedURLException ex) { - ex.printStackTrace(); + /** + * Convert IDN host to ASCII or Unicode using ICU's {@link IDNA} class. + * + * The conversion supports IDNA2008 names. + * + * @param host + * host name to be converted (lowercase expected) + * @param toAscii + * if true convert to ASCII, otherwise to Unicode + * @return converted host name + * @throws MalformedURLException + * if the conversion fails + */ + public static String convertIDNA2008(String host, boolean toAscii) + throws MalformedURLException { + final IDNA.Info idnaInfo = new IDNA.Info(); + final StringBuilder hostConverted = new StringBuilder(); + if (toAscii) { + idna.nameToASCII(host, hostConverted, idnaInfo); + } else { + idna.nameToUnicode(host, hostConverted, idnaInfo); } + if (idnaInfo.hasErrors()) { + StringBuilder msg = new StringBuilder(); + for (IDNA.Error error : idnaInfo.getErrors()) { + if (msg.length() == 0) { + msg.append("Invalid IDNA2008 host").append(host).append(": "); + } else { + msg.append(", "); + } + msg.append(error.name()); + } + String errorMsg = msg.toString(); + LOG.debug("Failed to convert IDN host {}: {}", host, errorMsg); + throw new MalformedURLException(errorMsg); + } + return hostConverted.toString(); } /** @@ -610,4 +717,24 @@ public static boolean isHomePageOf(URL url, String hostName) { && url.getRef() == null // && url.getUserInfo() == null; } + + /** + * For testing + * @param args print with no args to get help + */ + public static void main(String[] args) { + + if (args.length != 1) { + System.err.println("Usage : URLUtil "); + System.err.println("\nExtract and print pay-level domain names for the input URL"); + return; + } + + String url = args[0]; + try { + System.out.println(URLUtil.getDomainName(new URL(url))); + } catch (MalformedURLException ex) { + ex.printStackTrace(); + } + } } diff --git a/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java b/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java index c50988c2dd..64186b9035 100644 --- a/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java +++ b/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java @@ -16,20 +16,18 @@ */ package org.apache.nutch.protocol.file; -import java.net.URL; import java.io.IOException; -import java.io.UnsupportedEncodingException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import org.apache.hadoop.conf.Configuration; import org.apache.nutch.crawl.CrawlDatum; -import org.apache.nutch.protocol.Content; import org.apache.nutch.metadata.Metadata; import org.apache.nutch.net.protocols.HttpDateFormat; import org.apache.nutch.net.protocols.Response; - +import org.apache.nutch.protocol.Content; import org.apache.tika.Tika; -import org.apache.hadoop.conf.Configuration; - /** * FileResponse.java mimics file replies as http response. It tries its best to * follow http's way for headers, response codes as well as exceptions. @@ -125,11 +123,8 @@ public FileResponse(URL url, CrawlDatum datum, File file, Configuration conf) String path = url.getPath().isEmpty() ? "/" : url.getPath(); - try { - // specify the encoding via the config later? - path = java.net.URLDecoder.decode(path, "UTF-8"); - } catch (UnsupportedEncodingException ex) { - } + // specify the encoding via the config later? + path = java.net.URLDecoder.decode(path, StandardCharsets.UTF_8); try { diff --git a/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpResponse.java b/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpResponse.java index 8796cfc0b3..0d7ad1b289 100644 --- a/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpResponse.java +++ b/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpResponse.java @@ -30,6 +30,7 @@ import java.net.InetAddress; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.LinkedList; import java.io.ByteArrayOutputStream; @@ -245,7 +246,7 @@ public FtpResponse(URL url, CrawlDatum datum, Ftp ftp, Configuration conf) this.content = null; - path = java.net.URLDecoder.decode(path, "UTF-8"); + path = java.net.URLDecoder.decode(path, StandardCharsets.UTF_8); if (path.endsWith("/")) { getDirAsHttpResponse(path, datum.getModifiedTime()); diff --git a/src/plugin/urlnormalizer-ajax/src/java/org/apache/nutch/net/urlnormalizer/ajax/AjaxURLNormalizer.java b/src/plugin/urlnormalizer-ajax/src/java/org/apache/nutch/net/urlnormalizer/ajax/AjaxURLNormalizer.java index 2342ced68f..5518e39544 100644 --- a/src/plugin/urlnormalizer-ajax/src/java/org/apache/nutch/net/urlnormalizer/ajax/AjaxURLNormalizer.java +++ b/src/plugin/urlnormalizer-ajax/src/java/org/apache/nutch/net/urlnormalizer/ajax/AjaxURLNormalizer.java @@ -16,19 +16,18 @@ */ package org.apache.nutch.net.urlnormalizer.ajax; +import static java.nio.charset.StandardCharsets.UTF_8; + import java.lang.invoke.MethodHandles; +import java.net.MalformedURLException; import java.net.URL; import java.net.URLDecoder; -import java.net.MalformedURLException; -import java.nio.charset.Charset; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.nutch.net.URLNormalizer; import org.apache.nutch.net.URLNormalizers; -import org.apache.hadoop.conf.Configuration; - +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * URLNormalizer capable of dealing with AJAX URL's. * @@ -43,13 +42,11 @@ public class AjaxURLNormalizer implements URLNormalizer { public static String ESCAPED_URL_PART = "_escaped_fragment_="; private Configuration conf; - private Charset utf8; /** * Default constructor. */ public AjaxURLNormalizer() { - utf8 = Charset.forName("UTF-8"); } /** @@ -195,7 +192,7 @@ protected String escape(String fragmentPart) { String hex = null; StringBuilder sb = new StringBuilder(fragmentPart.length()); - for (byte b : fragmentPart.getBytes(utf8)) { + for (byte b : fragmentPart.getBytes(UTF_8)) { if (b < 33) { sb.append('%'); diff --git a/src/plugin/urlnormalizer-basic/src/java/org/apache/nutch/net/urlnormalizer/basic/BasicURLNormalizer.java b/src/plugin/urlnormalizer-basic/src/java/org/apache/nutch/net/urlnormalizer/basic/BasicURLNormalizer.java index 2123d8fa9f..4ff9fc64a6 100644 --- a/src/plugin/urlnormalizer-basic/src/java/org/apache/nutch/net/urlnormalizer/basic/BasicURLNormalizer.java +++ b/src/plugin/urlnormalizer-basic/src/java/org/apache/nutch/net/urlnormalizer/basic/BasicURLNormalizer.java @@ -16,12 +16,10 @@ */ package org.apache.nutch.net.urlnormalizer.basic; -import java.lang.invoke.MethodHandles; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; -import java.io.UnsupportedEncodingException; -import java.net.IDN; +import java.lang.invoke.MethodHandles; import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; @@ -36,6 +34,7 @@ import org.apache.nutch.net.URLNormalizer; import org.apache.nutch.net.URLNormalizers; import org.apache.nutch.util.NutchConfiguration; +import org.apache.nutch.util.URLUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,6 +46,12 @@ *