FTP passive mode with Testcontainers requires that the PASV response IP + * matches the host-visible address of the container, which is not reliable + * across Docker Desktop (macOS/Windows) and Linux Docker environments. An + * in-process {@link FakeFtpServer} from MockFtpServer avoids this constraint + * while still testing the Nutch FTP client against a real FTP protocol + * implementation. + */ +public class FtpProtocolIT implements ProtocolPluginIntegrationTest { + + private static final String FTP_USER = "testuser"; + private static final String FTP_PASS = "testpass"; + private static final String FTP_HOME = "/home/testuser"; + private static final String TEST_FILE = "test.txt"; + private static final String TEST_CONTENT = "FTP integration test content"; + + private static FakeFtpServer fakeFtpServer; + private Ftp protocol; + + @BeforeAll + static void startFtpServer() { + fakeFtpServer = new FakeFtpServer(); + fakeFtpServer.setServerControlPort(0); // bind to a random free port + + UserAccount userAccount = new UserAccount(FTP_USER, FTP_PASS, FTP_HOME); + fakeFtpServer.addUserAccount(userAccount); + + UnixFakeFileSystem fileSystem = new UnixFakeFileSystem(); + fileSystem.add(new DirectoryEntry(FTP_HOME)); + fileSystem.add(new FileEntry(FTP_HOME + "/" + TEST_FILE, TEST_CONTENT)); + fakeFtpServer.setFileSystem(fileSystem); + + fakeFtpServer.start(); + } + + @AfterAll + static void stopFtpServer() { + if (fakeFtpServer != null) { + fakeFtpServer.stop(); + } + } + + @BeforeEach + @Override + public void setUpProtocol() throws Exception { + Configuration conf = NutchConfiguration.create(); + conf.set("plugin.includes", "protocol-ftp|nutch-extensionpoints"); + conf.set("http.agent.name", "NutchFtpProtocolIT"); + conf.set("ftp.username", FTP_USER); + conf.set("ftp.password", FTP_PASS); + conf.setInt("ftp.timeout", 10000); + protocol = new Ftp(); + protocol.setConf(conf); + } + + @AfterEach + @Override + public void tearDownProtocol() { + protocol = null; + } + + @Override + public Protocol getProtocol() { + return protocol; + } + + @Override + public String getTestUrl() { + return "ftp://localhost:" + fakeFtpServer.getServerControlPort() + + FTP_HOME + "/" + TEST_FILE; + } + + @Test + void testFtpFileDownload() throws Exception { + CrawlDatum datum = new CrawlDatum(); + ProtocolOutput output = protocol.getProtocolOutput(new Text(getTestUrl()), datum); + + assertNotNull(output, "ProtocolOutput must not be null"); + int code = Integer.parseInt( + datum.getMetaData().get(Nutch.PROTOCOL_STATUS_CODE_KEY).toString()); + assertEquals(200, code, "Expected FTP 200 for file download"); + + assertNotNull(output.getContent(), "Content must not be null"); + String body = new String(output.getContent().getContent(), StandardCharsets.UTF_8); + assertTrue(body.contains(TEST_CONTENT), + "Downloaded content must match the file on the FTP server"); + } + + @Test + void testFtpDirectoryListing() throws Exception { + String dirUrl = "ftp://localhost:" + fakeFtpServer.getServerControlPort() + + FTP_HOME + "/"; + CrawlDatum datum = new CrawlDatum(); + ProtocolOutput output = protocol.getProtocolOutput(new Text(dirUrl), datum); + + assertNotNull(output, "ProtocolOutput for directory listing must not be null"); + int code = Integer.parseInt( + datum.getMetaData().get(Nutch.PROTOCOL_STATUS_CODE_KEY).toString()); + assertEquals(200, code, "Expected FTP 200 for directory listing"); + } + + @Test + void testFtpMissingFileReturnsError() throws Exception { + String missingUrl = "ftp://localhost:" + fakeFtpServer.getServerControlPort() + + FTP_HOME + "/nonexistent.txt"; + CrawlDatum datum = new CrawlDatum(); + ProtocolOutput output = protocol.getProtocolOutput(new Text(missingUrl), datum); + assertNotNull(output, "ProtocolOutput must not be null even for missing files"); + // FTP 550 "No such file" maps to a non-200 Nutch status + int code = Integer.parseInt( + datum.getMetaData().get(Nutch.PROTOCOL_STATUS_CODE_KEY).toString()); + assertTrue(code != 200, "Expected non-200 code for missing FTP file, got: " + code); + } +} diff --git a/src/plugin/protocol-htmlunit/src/test/org/apache/nutch/protocol/htmlunit/HtmlUnitProtocolIT.java b/src/plugin/protocol-htmlunit/src/test/org/apache/nutch/protocol/htmlunit/HtmlUnitProtocolIT.java new file mode 100644 index 0000000000..42c551b72f --- /dev/null +++ b/src/plugin/protocol-htmlunit/src/test/org/apache/nutch/protocol/htmlunit/HtmlUnitProtocolIT.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nutch.protocol.htmlunit; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import org.apache.hadoop.conf.Configuration; +import org.apache.nutch.crawl.CrawlDatum; +import org.apache.nutch.protocol.AbstractProtocolPluginIT; +import org.apache.nutch.protocol.Protocol; +import org.apache.nutch.protocol.ProtocolOutput; +import org.apache.nutch.util.NutchConfiguration; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +/** + * Integration tests for protocol-htmlunit using a real nginx container. + */ +@Testcontainers(disabledWithoutDocker = true) +public class HtmlUnitProtocolIT extends AbstractProtocolPluginIT { + + @Container + private static final GenericContainer nginx = + new GenericContainer<>("nginx:alpine").withExposedPorts(80); + + private Http protocol; + + @Override + public void setUpProtocol() throws Exception { + Configuration conf = NutchConfiguration.create(); + conf.set("plugin.includes", + "protocol-htmlunit|lib-htmlunit|lib-http|nutch-extensionpoints"); + conf.set("http.agent.name", "Nutch-Test"); + conf.setInt("http.timeout", 10000); + conf.setBoolean("store.http.headers", true); + protocol = new Http(); + protocol.setConf(conf); + } + + @Override + public void tearDownProtocol() { + protocol = null; + } + + @Override + public Protocol getProtocol() { + return protocol; + } + + @Override + public String getTestUrl() { + return "http://" + nginx.getHost() + ":" + nginx.getMappedPort(80) + "/"; + } + + @Test + void testFetchReturnsContent() throws Exception { + CrawlDatum datum = new CrawlDatum(); + ProtocolOutput output = protocol.getProtocolOutput( + new org.apache.hadoop.io.Text(getTestUrl()), datum); + assertNotNull(output.getContent(), + "protocol-htmlunit must return non-null content for a live nginx page"); + } +} diff --git a/src/plugin/protocol-http/src/test/org/apache/nutch/protocol/http/HttpProtocolIT.java b/src/plugin/protocol-http/src/test/org/apache/nutch/protocol/http/HttpProtocolIT.java new file mode 100644 index 0000000000..87db32335b --- /dev/null +++ b/src/plugin/protocol-http/src/test/org/apache/nutch/protocol/http/HttpProtocolIT.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nutch.protocol.http; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Text; +import org.apache.nutch.crawl.CrawlDatum; +import org.apache.nutch.protocol.AbstractProtocolPluginIT; +import org.apache.nutch.protocol.Protocol; +import org.apache.nutch.protocol.ProtocolOutput; +import org.apache.nutch.util.NutchConfiguration; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +/** + * Integration tests for protocol-http using a real nginx container. + */ +@Testcontainers(disabledWithoutDocker = true) +public class HttpProtocolIT extends AbstractProtocolPluginIT { + + @Container + private static final GenericContainer nginx = + new GenericContainer<>("nginx:alpine").withExposedPorts(80); + + private Http protocol; + + @Override + public void setUpProtocol() throws Exception { + Configuration conf = NutchConfiguration.create(); + conf.set("plugin.includes", "protocol-http|lib-http|nutch-extensionpoints"); + conf.set("http.agent.name", "Nutch-Test"); + conf.setInt("http.timeout", 10000); + conf.setBoolean("store.http.headers", true); + protocol = new Http(); + protocol.setConf(conf); + } + + @Override + public void tearDownProtocol() { + protocol = null; + } + + @Override + public Protocol getProtocol() { + return protocol; + } + + @Override + public String getTestUrl() { + return "http://" + nginx.getHost() + ":" + nginx.getMappedPort(80) + "/"; + } + + @Test + void testFetchRedirect301() throws Exception { + // nginx returns 301 for directory URLs without trailing slash when autoindex + // is off; test a manual redirect via the default nginx welcome page path + String redirectUrl = + "http://" + nginx.getHost() + ":" + nginx.getMappedPort(80) + "/index.html"; + CrawlDatum datum = new CrawlDatum(); + protocol.getProtocolOutput(new Text(redirectUrl), datum); + int code = getHttpStatusCode(datum); + // nginx serves index.html directly with 200; the base test covers 200/404 + assertEquals(200, code, "Expected 200 for index.html from nginx"); + } +} diff --git a/src/plugin/protocol-httpclient/ivy.xml b/src/plugin/protocol-httpclient/ivy.xml index 0b3ce0af73..e5987074b8 100644 --- a/src/plugin/protocol-httpclient/ivy.xml +++ b/src/plugin/protocol-httpclient/ivy.xml @@ -38,6 +38,7 @@ + diff --git a/src/plugin/protocol-httpclient/src/test/org/apache/nutch/protocol/httpclient/HttpClientProtocolIT.java b/src/plugin/protocol-httpclient/src/test/org/apache/nutch/protocol/httpclient/HttpClientProtocolIT.java new file mode 100644 index 0000000000..7345e4b029 --- /dev/null +++ b/src/plugin/protocol-httpclient/src/test/org/apache/nutch/protocol/httpclient/HttpClientProtocolIT.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nutch.protocol.httpclient; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.core.WireMockConfiguration; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Text; +import org.apache.nutch.crawl.CrawlDatum; +import org.apache.nutch.metadata.Nutch; +import org.apache.nutch.protocol.Protocol; +import org.apache.nutch.protocol.ProtocolOutput; +import org.apache.nutch.protocol.ProtocolPluginIntegrationTest; +import org.apache.nutch.util.NutchConfiguration; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Integration tests for protocol-httpclient using an in-process WireMock + * server. + * + *

WireMock runs in the test JVM so no Docker container is required. The + * Nutch httpclient plugin connects to it over a real TCP socket, exercising + * the full HTTP client stack including header handling and Basic-auth + * challenge/response. + */ +public class HttpClientProtocolIT implements ProtocolPluginIntegrationTest { + + private static WireMockServer wireMock; + private Http protocol; + + @BeforeAll + static void startWireMock() { + wireMock = new WireMockServer(WireMockConfiguration.options().dynamicPort()); + wireMock.start(); + + wireMock.stubFor(get(urlEqualTo("/")) + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Type", "text/html") + .withBody("Integration test"))); + + wireMock.stubFor(get(urlEqualTo("/notfound")) + .willReturn(aResponse().withStatus(404))); + + wireMock.stubFor(get(urlEqualTo("/secure")) + .withBasicAuth("testuser", "testpass") + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Type", "text/html") + .withBody("Authenticated"))); + + wireMock.stubFor(get(urlEqualTo("/secure")) + .willReturn(aResponse() + .withStatus(401) + .withHeader("WWW-Authenticate", "Basic realm=\"Test\"") + .withBody("Unauthorized"))); + } + + @AfterAll + static void stopWireMock() { + if (wireMock != null) { + wireMock.stop(); + } + } + + @BeforeEach + @Override + public void setUpProtocol() throws Exception { + Configuration conf = NutchConfiguration.create(); + conf.set("plugin.includes", + "protocol-httpclient|lib-http|nutch-extensionpoints"); + conf.set("http.agent.name", "Nutch-Test"); + conf.setInt("http.timeout", 10000); + conf.setBoolean("store.http.headers", true); + protocol = new Http(); + protocol.setConf(conf); + } + + @AfterEach + @Override + public void tearDownProtocol() { + protocol = null; + } + + @Override + public Protocol getProtocol() { + return protocol; + } + + @Override + public String getTestUrl() { + return "http://localhost:" + wireMock.port() + "/"; + } + + @Test + void testFetch200() throws Exception { + CrawlDatum datum = new CrawlDatum(); + ProtocolOutput output = protocol.getProtocolOutput(new Text(getTestUrl()), datum); + assertNotNull(output, "ProtocolOutput must not be null"); + int code = Integer.parseInt( + datum.getMetaData().get(Nutch.PROTOCOL_STATUS_CODE_KEY).toString()); + assertEquals(200, code, "Expected HTTP 200 from WireMock stub"); + } + + @Test + void testFetch404() throws Exception { + String url = "http://localhost:" + wireMock.port() + "/notfound"; + CrawlDatum datum = new CrawlDatum(); + protocol.getProtocolOutput(new Text(url), datum); + int code = Integer.parseInt( + datum.getMetaData().get(Nutch.PROTOCOL_STATUS_CODE_KEY).toString()); + assertEquals(404, code, "Expected HTTP 404 for /notfound stub"); + } + + @Test + void testUnauthenticatedRequestReturns401() throws Exception { + String secureUrl = "http://localhost:" + wireMock.port() + "/secure"; + CrawlDatum datum = new CrawlDatum(); + protocol.getProtocolOutput(new Text(secureUrl), datum); + int code = Integer.parseInt( + datum.getMetaData().get(Nutch.PROTOCOL_STATUS_CODE_KEY).toString()); + assertEquals(401, code, + "Unauthenticated request to /secure should return 401"); + } +} diff --git a/src/plugin/protocol-okhttp/src/test/org/apache/nutch/protocol/okhttp/OkHttpProtocolIT.java b/src/plugin/protocol-okhttp/src/test/org/apache/nutch/protocol/okhttp/OkHttpProtocolIT.java new file mode 100644 index 0000000000..d5342d8309 --- /dev/null +++ b/src/plugin/protocol-okhttp/src/test/org/apache/nutch/protocol/okhttp/OkHttpProtocolIT.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nutch.protocol.okhttp; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import org.apache.hadoop.conf.Configuration; +import org.apache.nutch.crawl.CrawlDatum; +import org.apache.nutch.protocol.AbstractProtocolPluginIT; +import org.apache.nutch.protocol.Protocol; +import org.apache.nutch.protocol.ProtocolOutput; +import org.apache.nutch.util.NutchConfiguration; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +/** + * Integration tests for protocol-okhttp using a real nginx container. + */ +@Testcontainers(disabledWithoutDocker = true) +public class OkHttpProtocolIT extends AbstractProtocolPluginIT { + + @Container + private static final GenericContainer nginx = + new GenericContainer<>("nginx:alpine").withExposedPorts(80); + + private OkHttp protocol; + + @Override + public void setUpProtocol() throws Exception { + Configuration conf = NutchConfiguration.create(); + conf.set("plugin.includes", "protocol-okhttp|lib-http|nutch-extensionpoints"); + conf.set("http.agent.name", "Nutch-Test"); + conf.setInt("http.timeout", 10000); + conf.setBoolean("store.http.headers", true); + protocol = new OkHttp(); + protocol.setConf(conf); + } + + @Override + public void tearDownProtocol() { + protocol = null; + } + + @Override + public Protocol getProtocol() { + return protocol; + } + + @Override + public String getTestUrl() { + return "http://" + nginx.getHost() + ":" + nginx.getMappedPort(80) + "/"; + } + + /** OkHttp transparently decompresses gzip; verify content is returned. */ + @Test + void testFetchWithAcceptEncoding() throws Exception { + CrawlDatum datum = new CrawlDatum(); + ProtocolOutput output = protocol.getProtocolOutput( + new org.apache.hadoop.io.Text(getTestUrl()), datum); + assertNotNull(output.getContent(), + "Content must be present even when server uses compression"); + } +} diff --git a/src/plugin/protocol-selenium/src/test/org/apache/nutch/protocol/selenium/SeleniumProtocolIT.java b/src/plugin/protocol-selenium/src/test/org/apache/nutch/protocol/selenium/SeleniumProtocolIT.java new file mode 100644 index 0000000000..ec928df64f --- /dev/null +++ b/src/plugin/protocol-selenium/src/test/org/apache/nutch/protocol/selenium/SeleniumProtocolIT.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nutch.protocol.selenium; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import org.apache.hadoop.conf.Configuration; +import org.apache.nutch.crawl.CrawlDatum; +import org.apache.nutch.protocol.AbstractProtocolPluginIT; +import org.apache.nutch.protocol.Protocol; +import org.apache.nutch.protocol.ProtocolOutput; +import org.apache.nutch.util.NutchConfiguration; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +/** + * Integration tests for protocol-selenium using a real nginx container. + * + *

Note: protocol-selenium uses raw HTTP sockets (the same underlying + * transport as protocol-http) rather than a Selenium WebDriver. Tests here + * validate that the plugin connects to and fetches content from a live HTTP + * server. Browser-based rendering is covered by protocol-interactiveselenium + * which is excluded from automated integration tests due to its stateful + * handler requirements. + */ +@Testcontainers(disabledWithoutDocker = true) +public class SeleniumProtocolIT extends AbstractProtocolPluginIT { + + @Container + private static final GenericContainer nginx = + new GenericContainer<>("nginx:alpine").withExposedPorts(80); + + private Http protocol; + + @Override + public void setUpProtocol() throws Exception { + Configuration conf = NutchConfiguration.create(); + conf.set("plugin.includes", + "protocol-selenium|lib-http|lib-selenium|nutch-extensionpoints"); + conf.set("http.agent.name", "Nutch-Test"); + conf.setInt("http.timeout", 10000); + conf.setBoolean("store.http.headers", true); + protocol = new Http(); + protocol.setConf(conf); + } + + @Override + public void tearDownProtocol() { + protocol = null; + } + + @Override + public Protocol getProtocol() { + return protocol; + } + + @Override + public String getTestUrl() { + return "http://" + nginx.getHost() + ":" + nginx.getMappedPort(80) + "/"; + } + + @Test + void testFetchReturnsContent() throws Exception { + CrawlDatum datum = new CrawlDatum(); + ProtocolOutput output = protocol.getProtocolOutput( + new org.apache.hadoop.io.Text(getTestUrl()), datum); + assertNotNull(output.getContent(), + "protocol-selenium must return non-null content for a live nginx page"); + } +} diff --git a/src/plugin/urlnormalizer-ajax/src/java/org/apache/nutch/net/urlnormalizer/ajax/AjaxURLNormalizer.java b/src/plugin/urlnormalizer-ajax/src/java/org/apache/nutch/net/urlnormalizer/ajax/AjaxURLNormalizer.java index 2342ced68f..5518e39544 100644 --- a/src/plugin/urlnormalizer-ajax/src/java/org/apache/nutch/net/urlnormalizer/ajax/AjaxURLNormalizer.java +++ b/src/plugin/urlnormalizer-ajax/src/java/org/apache/nutch/net/urlnormalizer/ajax/AjaxURLNormalizer.java @@ -16,19 +16,18 @@ */ package org.apache.nutch.net.urlnormalizer.ajax; +import static java.nio.charset.StandardCharsets.UTF_8; + import java.lang.invoke.MethodHandles; +import java.net.MalformedURLException; import java.net.URL; import java.net.URLDecoder; -import java.net.MalformedURLException; -import java.nio.charset.Charset; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.nutch.net.URLNormalizer; import org.apache.nutch.net.URLNormalizers; -import org.apache.hadoop.conf.Configuration; - +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * URLNormalizer capable of dealing with AJAX URL's. * @@ -43,13 +42,11 @@ public class AjaxURLNormalizer implements URLNormalizer { public static String ESCAPED_URL_PART = "_escaped_fragment_="; private Configuration conf; - private Charset utf8; /** * Default constructor. */ public AjaxURLNormalizer() { - utf8 = Charset.forName("UTF-8"); } /** @@ -195,7 +192,7 @@ protected String escape(String fragmentPart) { String hex = null; StringBuilder sb = new StringBuilder(fragmentPart.length()); - for (byte b : fragmentPart.getBytes(utf8)) { + for (byte b : fragmentPart.getBytes(UTF_8)) { if (b < 33) { sb.append('%'); diff --git a/src/plugin/urlnormalizer-basic/src/java/org/apache/nutch/net/urlnormalizer/basic/BasicURLNormalizer.java b/src/plugin/urlnormalizer-basic/src/java/org/apache/nutch/net/urlnormalizer/basic/BasicURLNormalizer.java index 2123d8fa9f..4ff9fc64a6 100644 --- a/src/plugin/urlnormalizer-basic/src/java/org/apache/nutch/net/urlnormalizer/basic/BasicURLNormalizer.java +++ b/src/plugin/urlnormalizer-basic/src/java/org/apache/nutch/net/urlnormalizer/basic/BasicURLNormalizer.java @@ -16,12 +16,10 @@ */ package org.apache.nutch.net.urlnormalizer.basic; -import java.lang.invoke.MethodHandles; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; -import java.io.UnsupportedEncodingException; -import java.net.IDN; +import java.lang.invoke.MethodHandles; import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; @@ -36,6 +34,7 @@ import org.apache.nutch.net.URLNormalizer; import org.apache.nutch.net.URLNormalizers; import org.apache.nutch.util.NutchConfiguration; +import org.apache.nutch.util.URLUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,6 +46,12 @@ *

Subclasses declare a static {@code @Container} field for the server + * container, implement {@link ProtocolPluginIntegrationTest}, and may add + * protocol-specific tests (e.g., redirect handling, authentication). + */ +@Testcontainers(disabledWithoutDocker = true) +public abstract class AbstractProtocolPluginIT implements ProtocolPluginIntegrationTest { + + @BeforeEach + void setUp() throws Exception { + setUpProtocol(); + } + + @AfterEach + void tearDown() throws Exception { + tearDownProtocol(); + } + + /** Fetch the test URL and assert an HTTP 200 response. */ + @Test + void testFetch200() throws Exception { + CrawlDatum datum = new CrawlDatum(); + ProtocolOutput output = getProtocol() + .getProtocolOutput(new Text(getTestUrl()), datum); + assertNotNull(output, "ProtocolOutput must not be null"); + assertEquals(200, getHttpStatusCode(datum), + "Expected HTTP 200 for " + getTestUrl()); + verifyFetchedContent(output, datum); + } + + /** Fetch a non-existent path and assert an HTTP 404 response. */ + @Test + void testFetch404() throws Exception { + String url = get404Url(); + CrawlDatum datum = new CrawlDatum(); + getProtocol().getProtocolOutput(new Text(url), datum); + assertEquals(404, getHttpStatusCode(datum), + "Expected HTTP 404 for " + url); + } + + /** + * Returns a URL expected to produce a 404. Default appends a random path + * segment to {@link #getTestUrl()}; override if the server needs a specific + * path. + */ + protected String get404Url() { + String base = getTestUrl(); + if (base.endsWith("/")) { + return base + "nonexistent-path-xyz"; + } + return base + "/nonexistent-path-xyz"; + } + + /** + * Reads the HTTP status code stored in the CrawlDatum metadata by Nutch + * protocol plugins. Returns -1 if no status code was stored. + */ + protected static int getHttpStatusCode(CrawlDatum datum) { + if (datum.getMetaData().containsKey(Nutch.PROTOCOL_STATUS_CODE_KEY)) { + return Integer.parseInt( + datum.getMetaData().get(Nutch.PROTOCOL_STATUS_CODE_KEY).toString()); + } + return -1; + } +} diff --git a/src/test/org/apache/nutch/protocol/ProtocolPluginIntegrationTest.java b/src/test/org/apache/nutch/protocol/ProtocolPluginIntegrationTest.java new file mode 100644 index 0000000000..b3778077d9 --- /dev/null +++ b/src/test/org/apache/nutch/protocol/ProtocolPluginIntegrationTest.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nutch.protocol; + +import org.apache.nutch.crawl.CrawlDatum; + +/** + * Contract for Protocol plugin integration tests. Implementations run against + * real server backends (via Testcontainers or embedded servers). + */ +public interface ProtocolPluginIntegrationTest { + + /** Set up the protocol plugin and its backing server before tests. */ + void setUpProtocol() throws Exception; + + /** Shut down the protocol plugin after tests. */ + void tearDownProtocol() throws Exception; + + /** The Protocol under test. */ + Protocol getProtocol(); + + /** + * A URL that the backing server will serve with a 200/success response. + * Must point into the container or embedded server started by this test. + */ + String getTestUrl(); + + /** + * Optional extra verification after a successful fetch. + * Default is a no-op; override to inspect content, headers, etc. + */ + default void verifyFetchedContent(ProtocolOutput output, CrawlDatum datum) + throws Exception { + // no-op + } +} diff --git a/src/test/org/apache/nutch/tools/TestCommonCrawlDataDumper.java b/src/test/org/apache/nutch/tools/TestCommonCrawlDataDumper.java index fee72b65a5..8124fe20b1 100644 --- a/src/test/org/apache/nutch/tools/TestCommonCrawlDataDumper.java +++ b/src/test/org/apache/nutch/tools/TestCommonCrawlDataDumper.java @@ -17,14 +17,19 @@ package org.apache.nutch.tools; import java.io.File; +import java.lang.reflect.Field; import java.nio.file.Files; import java.util.Collection; import org.apache.commons.io.FileUtils; import org.apache.commons.io.filefilter.FileFilterUtils; +import org.apache.nutch.metrics.ErrorTracker; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; /** * @@ -106,6 +111,22 @@ public void testDump() throws Exception { } + @Test + public void testDumpWithNoSegmentDirectoriesRecordsOtherError() throws Exception { + File emptySegmentDir = Files.createTempDirectory("empty-segments").toFile(); + File outputDir = Files.createTempDirectory("dump-output").toFile(); + + ErrorTracker mockErrorTracker = mock(ErrorTracker.class); + CommonCrawlDataDumper dumper = new CommonCrawlDataDumper(); + Field errorTrackerField = CommonCrawlDataDumper.class.getDeclaredField("errorTracker"); + errorTrackerField.setAccessible(true); + errorTrackerField.set(dumper, mockErrorTracker); + + dumper.dump(outputDir, emptySegmentDir, null, false, null, false, "", false); + + verify(mockErrorTracker, times(1)).recordError(ErrorTracker.ErrorType.OTHER); + } + private boolean hasFile(String fileName, Collection files) { for (File f : files) { if (f.getName().equals(fileName)) { diff --git a/src/test/org/apache/nutch/util/TestJexlUtil.java b/src/test/org/apache/nutch/util/TestJexlUtil.java new file mode 100644 index 0000000000..221fffea22 --- /dev/null +++ b/src/test/org/apache/nutch/util/TestJexlUtil.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nutch.util; + +import org.apache.commons.jexl3.JexlScript; +import org.apache.commons.jexl3.MapContext; +import org.apache.hadoop.conf.Configuration; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Unit tests for {@link JexlUtil} sandboxing. + */ +public class TestJexlUtil { + + @Test + public void testSandboxAllowsDocFieldCompare() throws Exception { + JexlScript script = JexlUtil.parseExpression("doc.lang == 'en'"); + assertNotNull(script); + MapContext doc = new MapContext(); + doc.set("lang", "en"); + MapContext root = new MapContext(); + root.set("doc", doc); + assertTrue(Boolean.TRUE.equals(script.execute(root))); + } + + @Test + public void testSandboxAllowsScoreCompare() throws Exception { + JexlScript script = JexlUtil.parseExpression("score > 0.5"); + assertNotNull(script); + MapContext root = new MapContext(); + root.set("score", 0.9f); + assertTrue(Boolean.TRUE.equals(script.execute(root))); + } + + @Test + public void testNewInstanceIoBlocked() { + assertNull(JexlUtil.parseExpression("new java.io.File('/')")); + } + + @Test + public void testNewInstanceFileOutputStreamBlocked() { + assertNull(JexlUtil.parseExpression( + "new java.io.FileOutputStream('/tmp/nutch-jexl-poc')")); + } + + @Test + public void testDisableSandboxAllowsNewExpressionParse() { + Configuration conf = new Configuration(); + conf.setBoolean(JexlUtil.DISABLE_SANDBOX_KEY, true); + JexlScript script = JexlUtil.parseExpression(conf, + "new java.io.File('/')"); + assertNotNull(script); + } + + @Test + public void testArithmeticAllowed() throws Exception { + JexlScript script = JexlUtil.parseExpression("2 * 3 + 1 == 7"); + assertNotNull(script); + assertTrue(Boolean.TRUE.equals(script.execute(new MapContext()))); + } + + @Test + public void testStringMethodsAllowed() throws Exception { + JexlScript script = JexlUtil.parseExpression( + "url.startsWith('http://')"); + assertNotNull(script); + MapContext root = new MapContext(); + root.set("url", "http://example.org/"); + assertTrue(Boolean.TRUE.equals(script.execute(root))); + } + + @Test + public void testDateRewriteStillParses() { + JexlScript script = JexlUtil.parseExpression( + "fetchTime > 2016-03-20T00:00:00Z"); + assertNotNull(script); + } + + @Test + public void testNullExpression() { + assertNull(JexlUtil.parseExpression(null)); + assertNull(JexlUtil.parseExpression(new Configuration(), null)); + } + + @Test + public void testInvalidSyntaxReturnsNull() { + assertNull(JexlUtil.parseExpression("doc.lang=<>:='en'")); + } + + @Test + public void testListSize() throws Exception { + JexlScript script = JexlUtil.parseExpression("doc.tags.size() == 2"); + assertNotNull(script); + MapContext doc = new MapContext(); + java.util.List tags = new java.util.ArrayList<>(); + tags.add("a"); + tags.add("b"); + doc.set("tags", tags); + MapContext root = new MapContext(); + root.set("doc", doc); + assertTrue(Boolean.TRUE.equals(script.execute(root))); + } + + @Test + public void testGeneratorStyleMetadata() throws Exception { + JexlScript script = JexlUtil.parseExpression( + "warc_import_time > 0 && score > 0"); + assertNotNull(script); + MapContext root = new MapContext(); + root.set("warc_import_time", 1); + root.set("score", 1.0f); + assertTrue(Boolean.TRUE.equals(script.execute(root))); + } + + @Test + public void testEqualsIgnoreCase() throws Exception { + JexlScript script = JexlUtil.parseExpression( + "status.equalsIgnoreCase('FETCHED')"); + assertNotNull(script); + MapContext root = new MapContext(); + root.set("status", "fetched"); + assertTrue(Boolean.TRUE.equals(script.execute(root))); + } + + @Test + public void testRegex() throws Exception { + JexlScript script = JexlUtil.parseExpression( + "url =~ 'https?://.*\\.example\\.org/.*'"); + assertNotNull(script); + MapContext root = new MapContext(); + root.set("url", "http://foo.example.org/bar"); + assertTrue(Boolean.TRUE.equals(script.execute(root))); + } + + @Test + public void testTernary() throws Exception { + JexlScript script = JexlUtil.parseExpression("true ? 1 : 0"); + assertNotNull(script); + assertEquals(1, script.execute(new MapContext())); + } +} diff --git a/src/test/org/apache/nutch/util/TestURLUtil.java b/src/test/org/apache/nutch/util/TestURLUtil.java index 4d8ae07971..573af66430 100644 --- a/src/test/org/apache/nutch/util/TestURLUtil.java +++ b/src/test/org/apache/nutch/util/TestURLUtil.java @@ -16,11 +16,18 @@ */ package org.apache.nutch.util; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.net.MalformedURLException; import java.net.URL; import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.*; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; /** Test class for URLUtil */ public class TestURLUtil { @@ -312,7 +319,20 @@ public void testToUNICODE() throws Exception { assertEquals("http://www.medizin.uni-tübingen.de:8080/search.php?q=abc#p1", URLUtil .toUNICODE("http://www.medizin.xn--uni-tbingen-xhb.de:8080/search.php?q=abc#p1")); - + // do not fail on characters not in Unicode 3.2 + assertEquals("https://example.ᬩᬮᬶ.id/", + URLUtil.toUNICODE("https://example.xn--9tfky.id/")); + // IDNA2008 + assertEquals("http://straße.de/", + URLUtil.toUNICODE("http://xn--strae-oqa.de/")); + // host names with uppercase characters + assertEquals("https://googie.com/", + URLUtil.toUNICODE("https://googIe.com/")); + assertEquals("https://googie.com/", URLUtil.toASCII("https://googIe.com/")); + assertEquals("https://xn--90ax2c.xn--p1ai/", + URLUtil.toASCII("https://нЭб.РФ/")); + assertEquals("https://нэб.рф/", + URLUtil.toUNICODE("https://Xn--90Ax2c.xN--P1ai/")); } @Test @@ -324,6 +344,106 @@ public void testToASCII() throws Exception { assertEquals("http://www.medizin.xn--uni-tbingen-xhb.de:8080/search.php?q=abc#p1", URLUtil .toASCII("http://www.medizin.uni-tübingen.de:8080/search.php?q=abc#p1")); + // IDNA2003 + // assertEquals("http://strasse.de/", + // URLUtil.toASCII("http://straße.de/")); + // do not fail on characters not in Unicode 3.2 + assertEquals("https://example.xn--9tfky.id/", + URLUtil.toASCII("https://example.ᬩᬮᬶ.id/")); + // IDNA2008 + assertEquals("http://xn--strae-oqa.de/", + URLUtil.toASCII("http://straße.de/")); + } + + @ParameterizedTest + @CsvSource({ // + "www.xn--evir-zoa.com,www.çevir.com,IDNA2003,true", // + "xn--uni-tbingen-xhb.de,uni-tübingen.de,IDNA2003,true", // + "example.xn--9tfky.id,example.ᬩᬮᬶ.id,IDNA2008,true", // + // Test examples from whatwg-url + "xn--53h.example,☕.example,IDNA2008,true", // + "xn--0ca.xn--ssa73l,à.א̈,IDNA2008,true", // + "xn--mgba3gch31f060k.com,\u0646\u0627\u0645\u0647\u200c\u0627\u06cc.com,IDNA2008,true", // + /* Note: IDNA2008 and IDNA2003 deviate for the following examples, + * cf. https://www.unicode.org/reports/tr46/#IDNA2003-Section */ + "xn--strae-oqa.de,straße.de,IDNA2008,true", // + "strasse.de,straße.de,IDNA2003,false", // + "strasse.de,strasse.de,IDNA2003,true", // + "xn--fa-hia.de,faß.de,IDNA2008,true", // + "fass.de,faß.de,IDNA2003,false", // + "fass.de,fass.de,IDNA2003,true", // + "xn--nxasmm1c.com,βόλος.com,IDNA2008,true", // + "xn--nxasmq6b.com,βόλος.com,IDNA2003,false", // + "xn--nxasmq6b.com,βόλοσ.com,IDNA2003,true", // + "xn--10cl1a0b660p.com,ශ්‍රී.com,IDNA2008,true", // + "xn--10cl1a0b.com,ශ්‍රී.com,IDNA2003,false", // + "xn--10cl1a0b.com,ශ්රී.com,IDNA2003,true", // + "xn--mgba3gch31f060k.com,نامه‌ای.com,IDNA2008,true", // + "xn--mgba3gch31f.com,نامه‌ای.com,IDNA2003,false", // + "xn--mgba3gch31f.com,نامهای.com,IDNA2003,true", // + // mixed lowercase/uppercase: no round trip conversion + "xn--bb-eka.at,ÖBB.at,IDNA2003,false", // + "xn--bb-eka.at,öbb.at,IDNA2003,true", // + // mixed encoding (Punycode and Unicode) + "xn--p1ai.xn--p1ai,рф.xn--p1ai,IDNA2003,false", // + "xn--p1ai.xn--p1ai,xn--p1ai.рф,IDNA2003,false", // + // percent-encoding is not supported + // "xn--p1ai.xn--p1ai,xn--p1ai.%D1%80%D1%84,IDNA2003,false", // + }) + public final void testConvertHost(String ascii, String unicode, String type, + boolean roundTrip) throws Exception { + System.out.println(ascii + " <> " + unicode); + if ("IDNA2008".equals(type)) { + assertEquals(ascii, URLUtil.convertIDNA2008(unicode, true)); + assertEquals(unicode, URLUtil.convertIDNA2008(ascii, false)); + try { + assertNotNull(URLUtil.convertIDNA2003(unicode, true, false)); + } catch (MalformedURLException e) { + /* + * Ok. A IDNA2008 input may raise an exception when using the IDNA2003 + * method + */ + } + } else if ("IDNA2003".equals(type)) { + assertEquals(ascii, URLUtil.convertIDNA2003(unicode, true, true)); + assertEquals(ascii, URLUtil.convertIDNA2003(unicode, true, false)); + if (roundTrip) { + assertEquals(unicode, URLUtil.convertIDNA2003(ascii, false, true)); + assertEquals(unicode, URLUtil.convertIDNA2003(ascii, false, false)); + } + } + } + + @Test + public final void testConvertHostInvalid() { + // broken Punycode + assertDoesNotThrow(() -> assertEquals("xn--xn--bss-7z6ccid.com", + URLUtil.convertIDNA2003("xn--xn--bss-7z6ccid.com", false, true))); + + // invalid Punycode + assertThrows(MalformedURLException.class, + () -> URLUtil.convertIDNA2008("xn--0.pt", false)); + + // IDNA2003 not allowing characters not in Unicode 3.2 + assertThrows(MalformedURLException.class, + () -> URLUtil.convertIDNA2003("☕.example", true, true)); + assertDoesNotThrow(() -> assertEquals("xn--53h.example", + URLUtil.convertIDNA2003("xn--53h.example", false, true))); + + // IDNA2008 invalid, + // cf. https://www.unicode.org/reports/tr46/#Implementation_Notes + // cf. https://www.unicode.org/Public/17.0.0/idna/IdnaTestV2.txt + // disallowed character: ⒈ (U+2488 - DIGIT ONE FULL STOP) + assertThrows(MalformedURLException.class, + () -> URLUtil.convertIDNA2008("\u2488com", true)); + assertThrows(MalformedURLException.class, + () -> URLUtil.convertIDNA2008("xn--acom-0w1b", false)); + assertThrows(MalformedURLException.class, + () -> URLUtil.convertIDNA2008("xn--xn--a--gua.pt", false)); + assertThrows(MalformedURLException.class, + () -> URLUtil.convertIDNA2008("xn--a-ä.pt", false)); + assertThrows(MalformedURLException.class, + () -> URLUtil.convertIDNA2008("xn--a-ä.pt", true)); } @Test