This class will write until we hit >= maxSize. The check is done at + *

This class will write until we hit >= maxSize. The check is done at * record boundary. Records do not span ARC files. We will then close current * file and open another and then continue writing. * @@ -95,9 +95,9 @@ * alexa * ARC c-tools: *

- * % av_procarc hx20040109230030-0.arc.gz | av_ziparc > \
+ * % av_procarc hx20040109230030-0.arc.gz | av_ziparc > \
  *     /tmp/hx20040109230030-0.dat.gz
- * % av_ripdat /tmp/hx20040109230030-0.dat.gz > /tmp/hx20040109230030-0.cdx
+ * % av_ripdat /tmp/hx20040109230030-0.dat.gz > /tmp/hx20040109230030-0.cdx
  * 

* This involves converting it to the largest unit - * (of B, KiB, MiB, GiB, TiB) for which the amount will be > 1. + * (of B, KiB, MiB, GiB, TiB) for which the amount will be > 1. *

* Additionally, at least 2 significant digits are always displayed. *

From 0d881e967daf2a023006032dd0d015b714821b11 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Tue, 15 Oct 2024 17:42:23 +0900 Subject: [PATCH 026/174] [maven-release-plugin] prepare release webarchive-commons-1.1.10 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f0c6ac73..2dd9223b 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ org.netpreserve.commons webarchive-commons - 1.1.10-SNAPSHOT + 1.1.10 jar webarchive-commons From 76d95ccd75ddc31c5b8c3e9136f9e422ab528898 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Tue, 15 Oct 2024 17:42:28 +0900 Subject: [PATCH 027/174] [maven-release-plugin] prepare for next development iteration --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2dd9223b..dc3088f0 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ org.netpreserve.commons webarchive-commons - 1.1.10 + 1.1.11-SNAPSHOT jar webarchive-commons From 835f4e115b2cd288bed3f703136a7325c81fa751 Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Sat, 9 Nov 2024 20:27:47 +0100 Subject: [PATCH 028/174] Make MetaData multi-valued to preserve values of repeating WARC and HTTP headers - code cleanup: fix indentation, remove unneeded return statements --- src/main/java/org/archive/resource/MetaData.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/archive/resource/MetaData.java b/src/main/java/org/archive/resource/MetaData.java index 30ce849b..fb3b24a4 100755 --- a/src/main/java/org/archive/resource/MetaData.java +++ b/src/main/java/org/archive/resource/MetaData.java @@ -83,7 +83,6 @@ public int optInt(String key, int defaultValue) { return super.getInt(key); } catch(JSONException e) { LOG.severe(e.getMessage()); - return defaultValue; } } return defaultValue; @@ -106,7 +105,6 @@ public long optLong(String key, long defaultValue) { return super.getLong(key); } catch(JSONException e) { LOG.severe(e.getMessage()); - return defaultValue; } } return defaultValue; @@ -167,10 +165,10 @@ public JSONObject put(String key, Object value) { ((JSONArray) super.get(key)).put(value); return this; } else { - JSONArray array = new JSONArray(); - array.put(super.get(key)); - array.put(value); - super.put(key, array); + JSONArray array = new JSONArray(); + array.put(super.get(key)); + array.put(value); + super.put(key, array); } return super.accumulate(key, value); } From a4748d9e79abb972a6571f5f4d46951be6049b1a Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Wed, 27 Nov 2024 13:24:17 +0100 Subject: [PATCH 029/174] URLParser and WaybackURLKeyMaker fail on URLs with IPv6 address hostname --- src/main/java/org/archive/url/URLParser.java | 11 ++++++++++- .../java/org/archive/url/URLRegexTransformer.java | 4 ++++ src/test/java/org/archive/url/URLParserTest.java | 3 +++ .../java/org/archive/url/WaybackURLKeyMakerTest.java | 3 +++ 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/archive/url/URLParser.java b/src/main/java/org/archive/url/URLParser.java index a7860b02..bcd0b7fb 100644 --- a/src/main/java/org/archive/url/URLParser.java +++ b/src/main/java/org/archive/url/URLParser.java @@ -226,7 +226,16 @@ public static HandyURL parse(String urlString) throws URISyntaxException { String colonPort = null; int atIndex = uriAuthority.indexOf(COMMERCIAL_AT); - int portColonIndex = uriAuthority.indexOf(COLON,(atIndex<0)?0:atIndex); + int portColonIndex = -1; + int startColonIndex = 0; + if (atIndex > -1) { + startColonIndex = atIndex; + } + if (uriAuthority.charAt(startColonIndex) == '[') { + // IPv6 address + startColonIndex = uriAuthority.indexOf(']', (startColonIndex + 1)); + } + portColonIndex = uriAuthority.indexOf(COLON, startColonIndex); if(atIndex<0 && portColonIndex<0) { // most common case: neither userinfo nor port diff --git a/src/main/java/org/archive/url/URLRegexTransformer.java b/src/main/java/org/archive/url/URLRegexTransformer.java index 617e0225..5f31c81c 100644 --- a/src/main/java/org/archive/url/URLRegexTransformer.java +++ b/src/main/java/org/archive/url/URLRegexTransformer.java @@ -121,6 +121,10 @@ public static String hostToSURT(String host) { // TODO: ensure we DONT reverse IP addresses! String parts[] = host.split("\\.",-1); if(parts.length == 1) { + // strip enclosing "[" and "]" from IPv6 hosts + if (host.charAt(0) == '[' && host.charAt(host.length() - 1) == ']') { + return host.substring(1, host.length() - 1); + } return host; } StringBuilder sb = new StringBuilder(host.length()); diff --git a/src/test/java/org/archive/url/URLParserTest.java b/src/test/java/org/archive/url/URLParserTest.java index b060ffa7..68dfcd23 100644 --- a/src/test/java/org/archive/url/URLParserTest.java +++ b/src/test/java/org/archive/url/URLParserTest.java @@ -86,6 +86,9 @@ public void testParse() throws UnsupportedEncodingException, URISyntaxException checkParse(" \n http://:****@www.archive.org:8080/inde\rx.html?query#foo \r\n \t ", null, "http", "", "****", "www.archive.org", 8080, "/index.html", "query", "foo", "http://:****@www.archive.org:8080/index.html?query#foo", "/index.html?query"); + checkParse("https://[2600:1f18:200d:fb00:2b74:867c:ab0c:150a]/robots.txt", null, "https", null, null, + "[2600:1f18:200d:fb00:2b74:867c:ab0c:150a]", -1, "/robots.txt", null, null, + "https://[2600:1f18:200d:fb00:2b74:867c:ab0c:150a]/robots.txt", "/robots.txt"); } private void checkParse(String s, String opaque, String scheme, String authUser, diff --git a/src/test/java/org/archive/url/WaybackURLKeyMakerTest.java b/src/test/java/org/archive/url/WaybackURLKeyMakerTest.java index 26161456..1a1403ee 100644 --- a/src/test/java/org/archive/url/WaybackURLKeyMakerTest.java +++ b/src/test/java/org/archive/url/WaybackURLKeyMakerTest.java @@ -23,6 +23,9 @@ public void testMakeKey() throws URISyntaxException { assertEquals("org,archive)/goo?a&b", km.makeKey("http://archive.org/goo/?b&a")); assertEquals("org,archive)/goo?a=1&a=2&b", km.makeKey("http://archive.org/goo/?a=2&b&a=1")); assertEquals("org,archive)/", km.makeKey("http://archive.org:/")); + assertEquals("192,211,203,34)/robots.txt", km.makeKey("https://34.203.211.192/robots.txt")); + assertEquals("2600:1f18:200d:fb00:2b74:867c:ab0c:150a)/robots.txt", + km.makeKey("https://[2600:1f18:200d:fb00:2b74:867c:ab0c:150a]/robots.txt")); } } From 8e89847d79ea2882bc55e2d00939fd8a2ca21865 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Wed, 27 Nov 2024 21:51:58 +0900 Subject: [PATCH 030/174] Update release plugins --- pom.xml | 110 ++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 67 insertions(+), 43 deletions(-) diff --git a/pom.xml b/pom.xml index dc3088f0..048787a5 100644 --- a/pom.xml +++ b/pom.xml @@ -1,12 +1,6 @@ 4.0.0 - - org.sonatype.oss - oss-parent - 7 - - org.netpreserve.commons webarchive-commons 1.1.11-SNAPSHOT @@ -45,19 +39,13 @@ scm:git:git@github.com:iipc/webarchive-commons.git scm:git:git@github.com:iipc/webarchive-commons.git - git@github.com:iipc/webarchive-commons.git + https://github.com/iipc/webarchive-commons UTF-8 ${maven.build.timestamp} yyyyMMddhhmmss - - - sonatype-nexus-staging - https://oss.sonatype.org/service/local/staging/deploy/maven2/ - sonatype-nexus-snapshots - https://oss.sonatype.org/content/repositories/snapshots/ @@ -201,24 +189,6 @@ 8 - - maven-assembly-plugin - 2.4 - - - jar-with-dependencies - - webarchive-commons - - - - package - - single - - - - org.apache.maven.plugins maven-enforcer-plugin @@ -251,17 +221,71 @@ - + + + release + + + ossrh + https://oss.sonatype.org/content/repositories/snapshots + + + + + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.6.7 + true + + ossrh + https://oss.sonatype.org/ + true + + + + org.apache.maven.plugins + maven-source-plugin + 2.2.1 + + + attach-sources + + jar-no-fork + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 2.9.1 + + + attach-javadocs + + jar + + + + + + org.apache.maven.plugins + maven-gpg-plugin + 1.5 + + + sign-artifacts + verify + + sign + + + + + + + + From 829566b1385a8dae6bc9774cd1299469f37e78c3 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Wed, 27 Nov 2024 21:53:22 +0900 Subject: [PATCH 031/174] [maven-release-plugin] prepare release webarchive-commons-1.1.11 --- pom.xml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 048787a5..28bd9145 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ org.netpreserve.commons webarchive-commons - 1.1.11-SNAPSHOT + 1.1.11 jar webarchive-commons @@ -40,7 +40,8 @@ scm:git:git@github.com:iipc/webarchive-commons.git scm:git:git@github.com:iipc/webarchive-commons.git https://github.com/iipc/webarchive-commons - + webarchive-commons-1.1.11 + UTF-8 From 9b0bbcfdeea7a9c2ac9a28b245bce2f8e9df5dce Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Wed, 27 Nov 2024 21:53:27 +0900 Subject: [PATCH 032/174] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 28bd9145..c86add9f 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ org.netpreserve.commons webarchive-commons - 1.1.11 + 1.1.12-SNAPSHOT jar webarchive-commons @@ -40,7 +40,7 @@ scm:git:git@github.com:iipc/webarchive-commons.git scm:git:git@github.com:iipc/webarchive-commons.git https://github.com/iipc/webarchive-commons - webarchive-commons-1.1.11 + HEAD From 9e4723b313a542320a4f09f4b4e2dbccdc0f58ac Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Wed, 27 Nov 2024 21:58:55 +0900 Subject: [PATCH 033/174] Update CHANGES.md --- CHANGES.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 6fe7c4bd..579b659f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,7 +1,14 @@ +1.1.11 +------ + +#### Bug fixes + +* Fixed URLParser and WaybackURLKeyMaker failing on URLs with IPv6 address hostnames [#100](https://github.com/iipc/webarchive-commons/pull/100) + 1.1.10 ------ -#### Fixes +#### Bug fixes * [WAT extractor: do not fail on missing WARC-Filename in warcinfo record](https://github.com/iipc/webarchive-commons/pull/89) * [ExtractingParseObserver: extract rel, hreflang and type attributes](https://github.com/iipc/webarchive-commons/pull/86) From cd2da63f1f56d41705e014e2c3290635fcc99099 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Wed, 27 Nov 2024 22:00:18 +0900 Subject: [PATCH 034/174] Add description to pom.xml (now mandatory for central) --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index c86add9f..18aca329 100644 --- a/pom.xml +++ b/pom.xml @@ -8,6 +8,7 @@ webarchive-commons https://github.com/iipc/webarchive-commons + Common web archive utility code The International Internet Preservation Consortium From 7b6df0c619899ae70e350fb0d955c00b59ba68e5 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Wed, 27 Nov 2024 22:02:31 +0900 Subject: [PATCH 035/174] [maven-release-plugin] prepare release webarchive-commons-1.1.11 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 18aca329..a57230d9 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ org.netpreserve.commons webarchive-commons - 1.1.12-SNAPSHOT + 1.1.11 jar webarchive-commons @@ -41,7 +41,7 @@ scm:git:git@github.com:iipc/webarchive-commons.git scm:git:git@github.com:iipc/webarchive-commons.git https://github.com/iipc/webarchive-commons - HEAD + webarchive-commons-1.1.11 From a70f23e8b654d3a661877641f2fa7e51d696ceeb Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Wed, 27 Nov 2024 22:02:36 +0900 Subject: [PATCH 036/174] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index a57230d9..18aca329 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ org.netpreserve.commons webarchive-commons - 1.1.11 + 1.1.12-SNAPSHOT jar webarchive-commons @@ -41,7 +41,7 @@ scm:git:git@github.com:iipc/webarchive-commons.git scm:git:git@github.com:iipc/webarchive-commons.git https://github.com/iipc/webarchive-commons - webarchive-commons-1.1.11 + HEAD From 0514b2387decaf5e40e24bcda0f7c70b438d0997 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Wed, 27 Nov 2024 22:08:04 +0900 Subject: [PATCH 037/174] Add Maven Central and Javadoc shields to README --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 72858a52..55be6e68 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ IIPC Web Archive Commons ======================== - -[](https://travis-ci.org/iipc/webarchive-commons/) +[](https://maven-badges.herokuapp.com/maven-central/org.netpreserve.commons/webarchive-commons) [](https://www.javadoc.io/doc/org.netpreserve.commons/webarchive-commons) This repository contains common utility code for [OpenWayback][1] and other projects. From c6095082fdecadd6882456a51c5f91b8a3d4faa5 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Fri, 29 Nov 2024 15:42:05 +0900 Subject: [PATCH 038/174] Bump guava from 33.3.0-jre to 33.3.1-jre --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 18aca329..0ac11df9 100644 --- a/pom.xml +++ b/pom.xml @@ -60,7 +60,7 @@ com.google.guava guava - 33.3.0-jre + 33.3.1-jre From 23c8887c2a3eb4d4d5b0bac0cf805c71fcaeabaf Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Fri, 29 Nov 2024 15:42:41 +0900 Subject: [PATCH 039/174] Bump commons-io from 2.14.0 to 2.18.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0ac11df9..84822a4f 100644 --- a/pom.xml +++ b/pom.xml @@ -140,7 +140,7 @@ commons-io commons-io - 2.14.0 + 2.18.0 From f13c7b2a3b254a83827ad5a1c27131c6980c79eb Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Fri, 29 Nov 2024 15:47:28 +0900 Subject: [PATCH 040/174] Bump commons-lang from 2.5 to 2.6 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 84822a4f..3d5f995f 100644 --- a/pom.xml +++ b/pom.xml @@ -134,7 +134,7 @@ commons-lang commons-lang - 2.5 + 2.6 From 5528afc05f77189b7ef59dbb9cdcce2bd35656e7 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Fri, 29 Nov 2024 16:24:04 +0900 Subject: [PATCH 041/174] Bump junit from 4.13.1 to 4.13.2 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3d5f995f..46f26766 100644 --- a/pom.xml +++ b/pom.xml @@ -54,7 +54,7 @@ junit junit - 4.13.1 + 4.13.2 From 7426c563310f73a0820a9af729b5f3621cea57f4 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Fri, 29 Nov 2024 16:24:52 +0900 Subject: [PATCH 042/174] Bump hadoop from 3.4.0 to 3.4.1 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 46f26766..c1bc7798 100644 --- a/pom.xml +++ b/pom.xml @@ -95,7 +95,7 @@ org.apache.hadoop hadoop-common - 3.4.0 + 3.4.1 true @@ -108,7 +108,7 @@ org.apache.hadoop hadoop-mapreduce-client-core - 3.4.0 + 3.4.1 true From 88607b2ed67c8c73e8b199adf85ac1ddf2fcdddb Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Fri, 29 Nov 2024 16:26:16 +0900 Subject: [PATCH 043/174] Bump httpcore from 4.3 to 4.4.16 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c1bc7798..a993945e 100644 --- a/pom.xml +++ b/pom.xml @@ -176,7 +176,7 @@ org.apache.httpcomponents httpcore - 4.3 + 4.4.16 From 0256ae6131e80c49e1ed4a16e5631ccff0d74e36 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Fri, 29 Nov 2024 16:27:04 +0900 Subject: [PATCH 044/174] Bump htmlparser from 1.6 to 2.1 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a993945e..ce0a2aec 100644 --- a/pom.xml +++ b/pom.xml @@ -71,7 +71,7 @@ org.htmlparser htmlparser - 1.6 + 2.1 From e1d458a86a2203ca1cd5cab967fb17f268994082 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Fri, 29 Nov 2024 16:27:31 +0900 Subject: [PATCH 045/174] Bump json from 20231013 to 20240303 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ce0a2aec..1023560c 100644 --- a/pom.xml +++ b/pom.xml @@ -66,7 +66,7 @@ org.json json - 20231013 + 20240303 org.htmlparser From c839700d472bac5b4625ea4fe10ef47ee02a5a31 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Fri, 29 Nov 2024 16:35:12 +0900 Subject: [PATCH 046/174] Update CHANGES.md --- CHANGES.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 579b659f..e3afd137 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,21 @@ +1.2.0 +----- + +#### New features + +* MetaData is now multivalued to support repeated WARC and HTTP headers. [#98](https://github.com/iipc/webarchive-commons/pull/98/files) + +#### Dependency upgrades + +* commons-io 2.18.0 +* commons-lang 2.6 +* guava 33.3.1-jre +* hadoop 3.4.1 +* htmlparser 2.1 +* httpcore 4.4.16 +* json 20240303 +* junit 4.13.2 + 1.1.11 ------ From 91c01ddb0561798d204c957fefafa782c0b53921 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Fri, 29 Nov 2024 16:37:15 +0900 Subject: [PATCH 047/174] [maven-release-plugin] prepare release webarchive-commons-1.2.0 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 1023560c..12dfae9f 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ org.netpreserve.commons webarchive-commons - 1.1.12-SNAPSHOT + 1.2.0 jar webarchive-commons @@ -41,7 +41,7 @@ scm:git:git@github.com:iipc/webarchive-commons.git scm:git:git@github.com:iipc/webarchive-commons.git https://github.com/iipc/webarchive-commons - HEAD + webarchive-commons-1.2.0 From f37418d08d8fa7fd4ccad4fbb919cc0fc371f2f2 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Fri, 29 Nov 2024 16:37:20 +0900 Subject: [PATCH 048/174] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 12dfae9f..0d84b0d2 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ org.netpreserve.commons webarchive-commons - 1.2.0 + 1.2.1-SNAPSHOT jar webarchive-commons @@ -41,7 +41,7 @@ scm:git:git@github.com:iipc/webarchive-commons.git scm:git:git@github.com:iipc/webarchive-commons.git https://github.com/iipc/webarchive-commons - webarchive-commons-1.2.0 + HEAD From 3ae5720ad43e2e80b5ab853078e891ee53641a3c Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Tue, 3 Dec 2024 20:22:10 +0900 Subject: [PATCH 049/174] Remove dependency on dsiutils --- pom.xml | 16 ++-------------- .../java/org/archive/url/UsableURIFactory.java | 5 ++--- 2 files changed, 4 insertions(+), 17 deletions(-) diff --git a/pom.xml b/pom.xml index 0d84b0d2..da2e14da 100644 --- a/pom.xml +++ b/pom.xml @@ -150,20 +150,8 @@ it.unimi.dsi - dsiutils - 2.2.8 - compile - - - ch.qos.logback - logback-classic - - - - commons-collections - commons-collections - - + fastutil + 7.0.10 diff --git a/src/main/java/org/archive/url/UsableURIFactory.java b/src/main/java/org/archive/url/UsableURIFactory.java index d44b5c84..3dfc33a7 100644 --- a/src/main/java/org/archive/url/UsableURIFactory.java +++ b/src/main/java/org/archive/url/UsableURIFactory.java @@ -20,7 +20,6 @@ import gnu.inet.encoding.IDNA; import gnu.inet.encoding.IDNAException; -import it.unimi.dsi.lang.MutableString; import java.io.UnsupportedEncodingException; import java.util.BitSet; @@ -485,7 +484,7 @@ private String fixup(String uri, final URI base, final String charset) // Preallocate. The '1's and '2's in below are space for ':', // '//', etc. URI characters. - MutableString s = new MutableString( + StringBuilder s = new StringBuilder( ((uriScheme != null)? uriScheme.length(): 0) + 1 // ';' + ((uriAuthority != null)? uriAuthority.length(): 0) @@ -707,7 +706,7 @@ private String checkPort(String uriAuthority) * @param substr Suffix or prefix to use if str is not null. * @param suffix True if substr is a suffix. */ - private void appendNonNull(MutableString b, String str, String substr, + private void appendNonNull(StringBuilder b, String str, String substr, boolean suffix) { if (str != null && str.length() > 0) { if (!suffix) { From 33556bf741eaa10421b9214bbbd69f40618d27d1 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Tue, 3 Dec 2024 20:38:46 +0900 Subject: [PATCH 050/174] Remove pom-cdh4.xml --- pom-cdh4.xml | 229 --------------------------------------------------- 1 file changed, 229 deletions(-) delete mode 100644 pom-cdh4.xml diff --git a/pom-cdh4.xml b/pom-cdh4.xml deleted file mode 100644 index de19d8d0..00000000 --- a/pom-cdh4.xml +++ /dev/null @@ -1,229 +0,0 @@ - - 4.0.0 - - org.archive - ia-web-commons - 1.0-SNAPSHOT - jar - - ia-web-commons - http://maven.apache.org - - - UTF-8 - ${maven.build.timestamp} - yyyyMMddhhmmss - - - - - junit - junit - 3.8.1 - test - - - - com.google.guava - guava - 14.0.1 - - - - org.json - json - 20090211 - - - org.htmlparser - htmlparser - 1.6 - - - - org.mozilla - juniversalchardet - 1.0.3 - - - - commons-httpclient - commons-httpclient - 3.1 - - - - org.apache.hadoop - hadoop-core - 2.0.0-mr1-cdh4.2.0 - - - commons-httpclient - commons-httpclient - - - javax.servlet - servlet-api - - - javax.servlet.jsp - jsp-api - - - org.mortbay.jetty - jetty - - - org.mortbay.jetty - jetty-util - - - tomcat - jasper-runtime - - - tomcat - jasper-compiler - - - - - org.apache.hadoop - hadoop-common - 2.0.0-cdh4.2.0 - - - org.apache.hadoop - hadoop-mapreduce-client-common - 2.0.0-cdh4.2.0 - - - org.apache.hadoop - hadoop-mapreduce-client-core - 2.0.0-cdh4.2.0 - - - - org.apache.pig - pig - 0.11.1 - provided - - - - commons-lang - commons-lang - 2.5 - - - - commons-io - commons-io - 2.4 - - - - org.gnu.inet - libidn - 1.15 - - - it.unimi.dsi - mg4j - 1.0.1 - compile - - - org.apache.httpcomponents - httpcore - 4.3 - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 2.3.2 - - 1.6 - 1.6 - - - - maven-assembly-plugin - 2.4 - - - jar-with-dependencies - - ia-web-commons - - - - package - - single - - - - - - - - src/main/resources - true - - - - - - - internetarchive - Internet Archive Maven Repository - http://builds.archive.org:8080/maven2 - default - - - true - daily - warn - - - true - daily - warn - - - - - cloudera - Cloudera Hadoop - https://repository.cloudera.com/artifactory/cloudera-repos/ - default - - - true - daily - warn - - - true - daily - warn - - - - - - - - repository - - ${repository.url} - - - - From 4bb03baec41d90795e312e4a2865abb0395670f3 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Tue, 3 Dec 2024 20:42:29 +0900 Subject: [PATCH 051/174] Use Files.createLink instead of shelling out to ln --- .../io/ObjectPlusFilesOutputStream.java | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/archive/io/ObjectPlusFilesOutputStream.java b/src/main/java/org/archive/io/ObjectPlusFilesOutputStream.java index 224f24e7..bd5c1eea 100644 --- a/src/main/java/org/archive/io/ObjectPlusFilesOutputStream.java +++ b/src/main/java/org/archive/io/ObjectPlusFilesOutputStream.java @@ -18,10 +18,8 @@ */ package org.archive.io; -import java.io.File; -import java.io.IOException; -import java.io.ObjectOutputStream; -import java.io.OutputStream; +import java.io.*; +import java.nio.file.Files; import java.util.LinkedList; import org.archive.util.FileUtils; @@ -116,19 +114,10 @@ public void snapshotAppendOnlyFile(File file) throws IOException { * @throws IOException */ private void hardlinkOrCopy(File file, File destination) throws IOException { - // For Linux/UNIX, try a hard link first. - Process link = Runtime.getRuntime().exec("ln "+file.getAbsolutePath()+" "+destination.getAbsolutePath()); - // TODO NTFS also supports hard links; add appropriate try try { - link.waitFor(); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - if(link.exitValue()!=0) { - // hard link failed + Files.createLink(destination.toPath(), file.toPath()); + } catch (UnsupportedEncodingException e) { FileUtils.copyFile(file,destination); } } - } From 328aef2788313a2abc6123c385f9c31b863d6f1b Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Wed, 4 Dec 2024 15:07:23 +0900 Subject: [PATCH 052/174] Remove dependency on fastutil Fastutil is our largest dependency and consumes a third of the overall Heritrix distribution size. If we update to the latest version it will be even larger. But we're only using two tiny classes from it: the trivial RepositionableStream interface and the unsynchronized FastBufferedOutputStream. Some downstream users (e.g. lockss-core) actually implement RepositionableStream, so to preserve API compatiblity this change includes a copy of just that interface while keeping the same package name. Regarding FastBufferedOutputStream, for WARC writing the outer GZIPOutputStream is synchronized anyway. And RecordingOutputStream will typically be doing moderately large writes copying from the network. So in both usages it seems unlikely that there's much practical benefit in using it here over the standard BufferedOutputStream. The JVM JIT has a lot of optimizations for synchronized these days too. --- pom.xml | 5 --- .../dsi/fastutil/io/RepositionableStream.java | 42 +++++++++++++++++++ .../org/archive/io/RecordingOutputStream.java | 5 +-- .../java/org/archive/io/WriterPoolMember.java | 5 +-- 4 files changed, 46 insertions(+), 11 deletions(-) create mode 100644 src/main/java/it/unimi/dsi/fastutil/io/RepositionableStream.java diff --git a/pom.xml b/pom.xml index da2e14da..5e5fa419 100644 --- a/pom.xml +++ b/pom.xml @@ -148,11 +148,6 @@ libidn 1.15 - - it.unimi.dsi - fastutil - 7.0.10 - diff --git a/src/main/java/it/unimi/dsi/fastutil/io/RepositionableStream.java b/src/main/java/it/unimi/dsi/fastutil/io/RepositionableStream.java new file mode 100644 index 00000000..a81645f0 --- /dev/null +++ b/src/main/java/it/unimi/dsi/fastutil/io/RepositionableStream.java @@ -0,0 +1,42 @@ +// copied from fastutil, keeping the original package name to avoid breaking +// compatibility with existing user code that implements this interface +package it.unimi.dsi.fastutil.io; + +/* + * Copyright (C) 2005-2015 Sebastiano Vigna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +/** A basic interface specifying positioning methods for a byte stream. + * + * @author Sebastiano Vigna + * @since 4.4 + */ + +public interface RepositionableStream { + + /** Sets the current stream position. + * + * @param newPosition the new stream position. + */ + void position( long newPosition ) throws java.io.IOException; + + /** Returns the current stream position. + * + * @return the current stream position. + */ + long position() throws java.io.IOException; + +} diff --git a/src/main/java/org/archive/io/RecordingOutputStream.java b/src/main/java/org/archive/io/RecordingOutputStream.java index 7d2ff212..6c77997b 100644 --- a/src/main/java/org/archive/io/RecordingOutputStream.java +++ b/src/main/java/org/archive/io/RecordingOutputStream.java @@ -19,8 +19,7 @@ package org.archive.io; -import it.unimi.dsi.fastutil.io.FastBufferedOutputStream; - +import java.io.BufferedOutputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; @@ -207,7 +206,7 @@ public void open(OutputStream wrappedStream) throws IOException { protected OutputStream ensureDiskStream() throws FileNotFoundException { if (this.diskStream == null) { FileOutputStream fis = new FileOutputStream(this.backingFilename); - this.diskStream = new FastBufferedOutputStream(fis); + this.diskStream = new BufferedOutputStream(fis); } return this.diskStream; } diff --git a/src/main/java/org/archive/io/WriterPoolMember.java b/src/main/java/org/archive/io/WriterPoolMember.java index 893007ec..e10d443b 100644 --- a/src/main/java/org/archive/io/WriterPoolMember.java +++ b/src/main/java/org/archive/io/WriterPoolMember.java @@ -19,8 +19,7 @@ package org.archive.io; -import it.unimi.dsi.fastutil.io.FastBufferedOutputStream; - +import java.io.BufferedOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -200,7 +199,7 @@ protected String createFile(final File file) throws IOException { close(); this.f = file; FileOutputStream fos = new FileOutputStream(this.f); - this.countOut = new MiserOutputStream(new FastBufferedOutputStream(fos),settings.getFrequentFlushes()); + this.countOut = new MiserOutputStream(new BufferedOutputStream(fos),settings.getFrequentFlushes()); this.out = this.countOut; logger.fine("Opened " + this.f.getAbsolutePath()); return this.f.getName(); From 8988fbbc3528afcc7f792bcc967189311e8a1286 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Wed, 4 Dec 2024 16:54:03 +0900 Subject: [PATCH 053/174] Deprecate some classes specific to HttpClient 3 These are intended to be removed in webarchive-commons 2. #78 --- .../java/org/archive/httpclient/HttpRecorderGetMethod.java | 2 ++ src/main/java/org/archive/httpclient/HttpRecorderMethod.java | 2 ++ .../java/org/archive/httpclient/HttpRecorderPostMethod.java | 2 ++ .../org/archive/httpclient/SingleHttpConnectionManager.java | 2 ++ .../archive/httpclient/ThreadLocalHttpConnectionManager.java | 4 +++- .../util/binsearch/impl/HTTPSeekableLineReaderFactory.java | 1 + .../archive/util/binsearch/impl/http/ApacheHttp31SLR.java | 4 ++++ .../util/binsearch/impl/http/ApacheHttp31SLRFactory.java | 5 +++++ 8 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/archive/httpclient/HttpRecorderGetMethod.java b/src/main/java/org/archive/httpclient/HttpRecorderGetMethod.java index ef241b48..1a94af1f 100644 --- a/src/main/java/org/archive/httpclient/HttpRecorderGetMethod.java +++ b/src/main/java/org/archive/httpclient/HttpRecorderGetMethod.java @@ -70,7 +70,9 @@ * * @author stack * @version $Revision$, $Date$ + * @deprecated Commons HttpClient 3 is end of life, this will be removed in webarchive-commons 2.0 */ +@Deprecated public class HttpRecorderGetMethod extends GetMethod { protected static Logger logger = diff --git a/src/main/java/org/archive/httpclient/HttpRecorderMethod.java b/src/main/java/org/archive/httpclient/HttpRecorderMethod.java index 932e7e98..b08bc0bd 100644 --- a/src/main/java/org/archive/httpclient/HttpRecorderMethod.java +++ b/src/main/java/org/archive/httpclient/HttpRecorderMethod.java @@ -34,7 +34,9 @@ * * @author stack * @version $Revision$, $Date$ + * @deprecated Commons HttpClient 3 is end of life, this will be removed in webarchive-commons 2.0 */ +@Deprecated public class HttpRecorderMethod { protected static Logger logger = Logger.getLogger(HttpRecorderMethod.class.getName()); diff --git a/src/main/java/org/archive/httpclient/HttpRecorderPostMethod.java b/src/main/java/org/archive/httpclient/HttpRecorderPostMethod.java index 20f1bfd1..d55d816a 100644 --- a/src/main/java/org/archive/httpclient/HttpRecorderPostMethod.java +++ b/src/main/java/org/archive/httpclient/HttpRecorderPostMethod.java @@ -36,7 +36,9 @@ * * @author stack * @version $Date$ $Revision$ + * @deprecated Commons HttpClient 3 is end of life, this will be removed in webarchive-commons 2.0 */ +@Deprecated public class HttpRecorderPostMethod extends PostMethod { /** * Instance of http recorder method. diff --git a/src/main/java/org/archive/httpclient/SingleHttpConnectionManager.java b/src/main/java/org/archive/httpclient/SingleHttpConnectionManager.java index 4ba6a837..d6cf27ab 100644 --- a/src/main/java/org/archive/httpclient/SingleHttpConnectionManager.java +++ b/src/main/java/org/archive/httpclient/SingleHttpConnectionManager.java @@ -32,7 +32,9 @@ * with external mechanisms. * * @author gojomo + * @deprecated Commons HttpClient 3 is end of life, this will be removed in webarchive-commons 2.0 */ +@Deprecated public class SingleHttpConnectionManager extends SimpleHttpConnectionManager { public SingleHttpConnectionManager() { diff --git a/src/main/java/org/archive/httpclient/ThreadLocalHttpConnectionManager.java b/src/main/java/org/archive/httpclient/ThreadLocalHttpConnectionManager.java index 91e850ea..16821b36 100644 --- a/src/main/java/org/archive/httpclient/ThreadLocalHttpConnectionManager.java +++ b/src/main/java/org/archive/httpclient/ThreadLocalHttpConnectionManager.java @@ -36,8 +36,10 @@ * * Java >= 1.4 is recommended. * - * @author Christian Kohlschuetter + * @author Christian Kohlschuetter + * @deprecated Commons HttpClient 3 is end of life, this will be removed in webarchive-commons 2.0 */ +@Deprecated public final class ThreadLocalHttpConnectionManager implements HttpConnectionManager { diff --git a/src/main/java/org/archive/util/binsearch/impl/HTTPSeekableLineReaderFactory.java b/src/main/java/org/archive/util/binsearch/impl/HTTPSeekableLineReaderFactory.java index b4a23db0..68ee6551 100644 --- a/src/main/java/org/archive/util/binsearch/impl/HTTPSeekableLineReaderFactory.java +++ b/src/main/java/org/archive/util/binsearch/impl/HTTPSeekableLineReaderFactory.java @@ -20,6 +20,7 @@ protected HTTPSeekableLineReaderFactory() public enum HttpLibs { + @Deprecated APACHE_31, APACHE_43, URLCONN, diff --git a/src/main/java/org/archive/util/binsearch/impl/http/ApacheHttp31SLR.java b/src/main/java/org/archive/util/binsearch/impl/http/ApacheHttp31SLR.java index c4fdbba8..124d3d03 100644 --- a/src/main/java/org/archive/util/binsearch/impl/http/ApacheHttp31SLR.java +++ b/src/main/java/org/archive/util/binsearch/impl/http/ApacheHttp31SLR.java @@ -14,6 +14,10 @@ import org.apache.commons.io.input.CountingInputStream; import org.archive.util.binsearch.impl.HTTPSeekableLineReader; +/** + * @deprecated Commons HttpClient 3 is end of life, this will be removed in webarchive-commons 2.0 + */ +@Deprecated public class ApacheHttp31SLR extends HTTPSeekableLineReader { private HttpClient http; diff --git a/src/main/java/org/archive/util/binsearch/impl/http/ApacheHttp31SLRFactory.java b/src/main/java/org/archive/util/binsearch/impl/http/ApacheHttp31SLRFactory.java index bc5b83f4..2af03dab 100644 --- a/src/main/java/org/archive/util/binsearch/impl/http/ApacheHttp31SLRFactory.java +++ b/src/main/java/org/archive/util/binsearch/impl/http/ApacheHttp31SLRFactory.java @@ -15,6 +15,11 @@ import org.archive.util.binsearch.impl.HTTPSeekableLineReader; import org.archive.util.binsearch.impl.HTTPSeekableLineReaderFactory; +/** + * + * @deprecated Commons HttpClient 3 is end of life, this will be removed in webarchive-commons 2.0 + */ +@Deprecated public class ApacheHttp31SLRFactory extends HTTPSeekableLineReaderFactory { private final static Logger LOGGER = Logger.getLogger(ApacheHttp31SLRFactory.class.getName()); From b8a91bb3b7e8a36b2162251314ff52b42a379221 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Thu, 5 Dec 2024 07:49:10 +0900 Subject: [PATCH 054/174] Remove unused dependency on commons-collections --- pom.xml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/pom.xml b/pom.xml index 5e5fa419..6dec154c 100644 --- a/pom.xml +++ b/pom.xml @@ -149,13 +149,6 @@ 1.15 - - - commons-collections - commons-collections - 3.2.2 - - org.apache.httpcomponents httpcore From a80b98dfe4b1c2a7556e7df2574c16426849f6d9 Mon Sep 17 00:00:00 2001 From: Tom Morris Date: Sat, 26 Aug 2023 20:05:34 -0400 Subject: [PATCH 055/174] Add failing test from Sebastian's issue --- src/test/java/org/archive/url/BasicURLCanonicalizerTest.java | 3 +++ src/test/java/org/archive/url/WaybackURLKeyMakerTest.java | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/src/test/java/org/archive/url/BasicURLCanonicalizerTest.java b/src/test/java/org/archive/url/BasicURLCanonicalizerTest.java index c21bcbe8..cc100e4c 100644 --- a/src/test/java/org/archive/url/BasicURLCanonicalizerTest.java +++ b/src/test/java/org/archive/url/BasicURLCanonicalizerTest.java @@ -143,6 +143,9 @@ public void testUnescapeRepeatedly() { assertEquals("%",guc.unescapeRepeatedly("%25%32%35")); assertEquals("168.188.99.26",guc.unescapeRepeatedly("%31%36%38%2e%31%38%38%2e%39%39%2e%32%36")); + + assertEquals("tag=%E4%EE%F8%EA%EE%EB%FC%ED%EE%E5", + guc.unescapeRepeatedly("tag=%E4%EE%F8%EA%EE%EB%FC%ED%EE%E5")); } public void testAttemptIPFormats() throws URIException { diff --git a/src/test/java/org/archive/url/WaybackURLKeyMakerTest.java b/src/test/java/org/archive/url/WaybackURLKeyMakerTest.java index 1a1403ee..86250972 100644 --- a/src/test/java/org/archive/url/WaybackURLKeyMakerTest.java +++ b/src/test/java/org/archive/url/WaybackURLKeyMakerTest.java @@ -26,6 +26,10 @@ public void testMakeKey() throws URISyntaxException { assertEquals("192,211,203,34)/robots.txt", km.makeKey("https://34.203.211.192/robots.txt")); assertEquals("2600:1f18:200d:fb00:2b74:867c:ab0c:150a)/robots.txt", km.makeKey("https://[2600:1f18:200d:fb00:2b74:867c:ab0c:150a]/robots.txt")); + assertEquals("ua,1kr)/newslist.html?tag=%e4%ee%f8%ea%ee%eb%fc%ed%ee%e5", + km.makeKey("http://1kr.ua/newslist.html?tag=%E4%EE%F8%EA%EE%EB%FC%ED%EE%E5")); + assertEquals("com,aluroba)/tags/%c3%ce%ca%c7%d1%e5%c7.htm", + km.makeKey("http://www.aluroba.com/tags/%C3%CE%CA%C7%D1%E5%C7.htm")); } } From 5161306d9ec993d1986f0d092c056f33ba3abdfe Mon Sep 17 00:00:00 2001 From: Tom Morris Date: Sun, 27 Aug 2023 13:01:19 -0400 Subject: [PATCH 056/174] Add non-UTF-8 encoded test from mailing list --- src/test/java/org/archive/url/WaybackURLKeyMakerTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/org/archive/url/WaybackURLKeyMakerTest.java b/src/test/java/org/archive/url/WaybackURLKeyMakerTest.java index 86250972..26371ba8 100644 --- a/src/test/java/org/archive/url/WaybackURLKeyMakerTest.java +++ b/src/test/java/org/archive/url/WaybackURLKeyMakerTest.java @@ -30,6 +30,8 @@ public void testMakeKey() throws URISyntaxException { km.makeKey("http://1kr.ua/newslist.html?tag=%E4%EE%F8%EA%EE%EB%FC%ED%EE%E5")); assertEquals("com,aluroba)/tags/%c3%ce%ca%c7%d1%e5%c7.htm", km.makeKey("http://www.aluroba.com/tags/%C3%CE%CA%C7%D1%E5%C7.htm")); + assertEquals("ac,insbase)/xoops2/modules/xpwiki?%a4%d5%a4%af%a4%aa%a4%ab%b8%a9%a4%aa%a4%aa%a4%ce%a4%b8%a4%e7%a4%a6%bb%d4", + km.makeKey("https://www.insbase.ac/xoops2/modules/xpwiki/?%A4%D5%A4%AF%A4%AA%A4%AB%B8%A9%A4%AA%A4%AA%A4%CE%A4%B8%A4%E7%A4%A6%BB%D4")); } } From f7be47bc523c4d06cc7960dc2d3b1b58f9580906 Mon Sep 17 00:00:00 2001 From: Tom Morris Date: Sun, 27 Aug 2023 13:11:30 -0400 Subject: [PATCH 057/174] Handle non-UTF-8 encoded characters. Fixes #6 --- .../archive/url/BasicURLCanonicalizer.java | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/archive/url/BasicURLCanonicalizer.java b/src/main/java/org/archive/url/BasicURLCanonicalizer.java index c09ad6e6..37b448c1 100644 --- a/src/main/java/org/archive/url/BasicURLCanonicalizer.java +++ b/src/main/java/org/archive/url/BasicURLCanonicalizer.java @@ -15,18 +15,18 @@ /** * Canonicalizer that does more or less basic fixup. Based initially on rules * specified at https://developers.google.com/safe-browsing/developers_guide_v2# - * Canonicalization. These rules are designed for clients of google's + * Canonicalization. These rules are designed for clients of Google's * "experimental" Safe Browsing API to "check URLs against Google's * constantly-updated blacklists of suspected phishing and malware pages". * *

- * This class differs from google in treatment of non-ascii input. Google's + * This class differs from Google in treatment of non-ascii input. Google's * rules don't really address this except with one example test case, which * seems to suggest taking raw input bytes and pct-encoding them byte for byte. * Since the input to this class consists of java strings, not raw bytes, that - * wouldn't be possible, even if deemed preferable. Instead + * wouldn't be possible, even if deemed preferable. Instead, * BasicURLCanonicalizer expresses non-ascii characters pct-encoded UTF-8. */ public class BasicURLCanonicalizer implements URLCanonicalizer { @@ -212,6 +212,10 @@ protected static Charset UTF8() { return _UTF8; } + /** + * @param input String to be percent-encoded. Assumed to be fully unescaped. + * @return percent-encoded string + */ public String escapeOnce(String input) { if (input == null) { return null; @@ -243,6 +247,19 @@ public String escapeOnce(String input) { */ sb = new StringBuilder(input.substring(0, i)); } + if (b == '%' && i < utf8bytes.length - 2) { + // Any hex escapes left at this point represent non-UTF-8 encoded characters + // Unescape them, so they don't get double escaped + int hex1 = getHex(utf8bytes[i + 1]); + if (hex1 >= 0) { + int hex2 = getHex(utf8bytes[i + 2]); + if (hex2 >= 0) { + i = i+2; + b = hex1 * 16 + hex2; + } + } + + } sb.append("%"); String hex = Integer.toHexString(b).toUpperCase(); if (hex.length() == 1) { @@ -337,7 +354,7 @@ public String decode(String input) { * Decodes bytes in bbuf as utf-8 and appends decoded characters to sb. If * decoding of any portion fails, appends the un-decodable %xx%xx sequence * extracted from inputStr instead of decoded characters. See "bad unicode" - * tests in GoogleCanonicalizerTest#testDecode(). Variables only make sense + * tests in BasicURLCanonicalizerTest#testDecode(). Variables only make sense * within context of {@link #decode(String)}. * * @param sb From 6a3cf1b317c87305d05faee73d2c3ee3f5ec08b0 Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Wed, 11 Dec 2024 21:14:06 +0100 Subject: [PATCH 058/174] WAT: Duplicated payload metadata values for "Actual-Content-Length" and "Trailing-Slop-Length" --- .../org/archive/resource/arc/ARCResource.java | 2 + .../http/HTTPHeadersResourceFactory.java | 11 +++-- .../archive/resource/warc/WARCResource.java | 14 ++++-- .../record/WARCMetaDataResourceFactory.java | 10 +++- .../archive/resource/arc/ARCResourceTest.java | 48 +++++++++++++++++++ .../resource/warc/WARCResourceTest.java | 46 ++++++++++++++++++ 6 files changed, 123 insertions(+), 8 deletions(-) create mode 100644 src/test/java/org/archive/resource/arc/ARCResourceTest.java create mode 100644 src/test/java/org/archive/resource/warc/WARCResourceTest.java diff --git a/src/main/java/org/archive/resource/arc/ARCResource.java b/src/main/java/org/archive/resource/arc/ARCResource.java index b6e0a1c1..b0195f08 100644 --- a/src/main/java/org/archive/resource/arc/ARCResource.java +++ b/src/main/java/org/archive/resource/arc/ARCResource.java @@ -64,10 +64,12 @@ public ARCResource(MetaData metaData, ResourceContainer container, } } + @Override public InputStream getInputStream() { return new EOFNotifyingInputStream(digIS, this); } + @Override public void notifyEOF() throws IOException { metaData.putLong(PAYLOAD_LENGTH, countingIS.getCount()); String digString = Base32.encode(digIS.getMessageDigest().digest()); diff --git a/src/main/java/org/archive/resource/http/HTTPHeadersResourceFactory.java b/src/main/java/org/archive/resource/http/HTTPHeadersResourceFactory.java index 79805090..eb25d821 100644 --- a/src/main/java/org/archive/resource/http/HTTPHeadersResourceFactory.java +++ b/src/main/java/org/archive/resource/http/HTTPHeadersResourceFactory.java @@ -31,6 +31,7 @@ public HTTPHeadersResourceFactory(String name, String type) { parser = new HttpHeaderParser(); } + @Override public Resource getResource(InputStream is, MetaData parentMetaData, ResourceContainer container) throws ResourceParseException, IOException { @@ -40,9 +41,13 @@ public Resource getResource(InputStream is, MetaData parentMetaData, if(headers.isCorrupt()) { parentMetaData.putBoolean(HTTP_HEADERS_CORRUPT, true); } - parentMetaData.putLong(PAYLOAD_LENGTH, bytes); - - parentMetaData.putLong(PAYLOAD_SLOP_BYTES, StreamCopy.readToEOF(is)); + if (!parentMetaData.has(PAYLOAD_LENGTH) || bytes != parentMetaData.getLong(PAYLOAD_LENGTH)) { + parentMetaData.putLong(PAYLOAD_LENGTH, bytes); + } + long trailingSlopBytes = StreamCopy.readToEOF(is); + if (!parentMetaData.has(PAYLOAD_SLOP_BYTES) || trailingSlopBytes > 0) { + parentMetaData.putLong(PAYLOAD_SLOP_BYTES, trailingSlopBytes); + } if(type != null) { parentMetaData.putString(PAYLOAD_CONTENT_TYPE, type); } diff --git a/src/main/java/org/archive/resource/warc/WARCResource.java b/src/main/java/org/archive/resource/warc/WARCResource.java index d538a25d..a9c3fcc3 100644 --- a/src/main/java/org/archive/resource/warc/WARCResource.java +++ b/src/main/java/org/archive/resource/warc/WARCResource.java @@ -53,7 +53,7 @@ public WARCResource(MetaData metaData, ResourceContainer container, countingIS = new CountingInputStream( ByteStreams.limit(response, length)); } else { - throw new ResourceParseException(null); + throw new ResourceParseException(new Exception("Zero or negative length: " + length)); } try { digIS = new DigestInputStream(countingIS, @@ -63,14 +63,18 @@ public WARCResource(MetaData metaData, ResourceContainer container, } } + @Override public InputStream getInputStream() { return new EOFNotifyingInputStream(digIS, this); } + @Override public void notifyEOF() throws IOException { String digString = Base32.encode(digIS.getMessageDigest().digest()); if(container.isCompressed()) { - metaData.putLong(PAYLOAD_LENGTH, countingIS.getCount()); + if (!metaData.has(PAYLOAD_LENGTH) || countingIS.getCount() != metaData.getLong(PAYLOAD_LENGTH)) { + metaData.putLong(PAYLOAD_LENGTH, countingIS.getCount()); + } metaData.putLong(PAYLOAD_SLOP_BYTES, StreamCopy.readToEOF(response)); metaData.putString(PAYLOAD_DIGEST, "sha1:"+digString); } else { @@ -81,13 +85,17 @@ public void notifyEOF() throws IOException { (PushBackOneByteInputStream) raw; long numNewlines = StreamCopy.skipChars(pb1bis, CR_NL_CHARS); if(numNewlines > 0) { - metaData.putLong(PAYLOAD_LENGTH, countingIS.getCount()); + long payloadLength = countingIS.getCount(); + if (!metaData.has(PAYLOAD_LENGTH) || payloadLength != metaData.getLong(PAYLOAD_LENGTH)) { + metaData.putLong(PAYLOAD_LENGTH, payloadLength); + } metaData.putLong(PAYLOAD_SLOP_BYTES, numNewlines); metaData.putString(PAYLOAD_DIGEST, "sha1:"+digString); } } } } + public MetaData getEnvelopeMetaData() { return envelope; } diff --git a/src/main/java/org/archive/resource/warc/record/WARCMetaDataResourceFactory.java b/src/main/java/org/archive/resource/warc/record/WARCMetaDataResourceFactory.java index 0dfb2834..ba8a35da 100644 --- a/src/main/java/org/archive/resource/warc/record/WARCMetaDataResourceFactory.java +++ b/src/main/java/org/archive/resource/warc/record/WARCMetaDataResourceFactory.java @@ -21,6 +21,7 @@ public WARCMetaDataResourceFactory() { parser = new HttpHeaderParser(); } + @Override public Resource getResource(InputStream is, MetaData parentMetaData, ResourceContainer container) throws ResourceParseException, IOException { @@ -33,8 +34,13 @@ public Resource getResource(InputStream is, MetaData parentMetaData, if(headers.isCorrupt()) { md.putBoolean(WARC_META_FIELDS_CORRUPT, true); } - parentMetaData.putLong(PAYLOAD_SLOP_BYTES, StreamCopy.readToEOF(is)); - parentMetaData.putLong(PAYLOAD_LENGTH, bytes); + long trailingSlopBytes = StreamCopy.readToEOF(is); + if (!parentMetaData.has(PAYLOAD_SLOP_BYTES) || trailingSlopBytes > 0) { + parentMetaData.putLong(PAYLOAD_SLOP_BYTES, trailingSlopBytes); + } + if (!parentMetaData.has(PAYLOAD_LENGTH) || bytes != parentMetaData.getLong(PAYLOAD_LENGTH)) { + parentMetaData.putLong(PAYLOAD_LENGTH, bytes); + } return new WARCMetaDataResource(md,container, headers); } catch (HttpParseException e) { diff --git a/src/test/java/org/archive/resource/arc/ARCResourceTest.java b/src/test/java/org/archive/resource/arc/ARCResourceTest.java new file mode 100644 index 00000000..43116af7 --- /dev/null +++ b/src/test/java/org/archive/resource/arc/ARCResourceTest.java @@ -0,0 +1,48 @@ +package org.archive.resource.arc; + + +import static org.archive.resource.ResourceConstants.PAYLOAD_LENGTH; +import static org.archive.resource.ResourceConstants.PAYLOAD_SLOP_BYTES; + +import java.io.IOException; + +import org.archive.extract.ExtractingResourceFactoryMapper; +import org.archive.extract.ExtractingResourceProducer; +import org.archive.extract.ProducerUtils; +import org.archive.extract.ResourceFactoryMapper; +import org.archive.resource.Resource; +import org.archive.resource.ResourceParseException; +import org.archive.resource.ResourceProducer; +import org.archive.util.StreamCopy; + +import org.json.JSONObject; + +import junit.framework.TestCase; + +public class ARCResourceTest extends TestCase { + + public void testARCResource() throws ResourceParseException, IOException { + String testFileName = "../../format/arc/IAH-20080430204825-00000-blackbook-truncated.arc"; + ResourceProducer producer = ProducerUtils.getProducer(getClass().getResource(testFileName).getPath()); + ResourceFactoryMapper mapper = new ExtractingResourceFactoryMapper(); + ExtractingResourceProducer extractor = new ExtractingResourceProducer(producer, mapper); + + Resource resource = extractor.getNext(); + + while (resource != null) { + JSONObject payloadMD = resource.getMetaData().getTopMetaData().getJSONObject("Envelope") + .getJSONObject("Payload-Metadata"); + System.err.println(payloadMD); + + if (payloadMD.has(PAYLOAD_LENGTH)) { + assertTrue(payloadMD.getLong(PAYLOAD_LENGTH) != -1); + } + if (payloadMD.has(PAYLOAD_SLOP_BYTES)) { + // does not occur with the tested ARC file + } + + StreamCopy.readToEOF(resource.getInputStream()); + resource = extractor.getNext(); + } + } +} diff --git a/src/test/java/org/archive/resource/warc/WARCResourceTest.java b/src/test/java/org/archive/resource/warc/WARCResourceTest.java new file mode 100644 index 00000000..1b935405 --- /dev/null +++ b/src/test/java/org/archive/resource/warc/WARCResourceTest.java @@ -0,0 +1,46 @@ +package org.archive.resource.warc; + +import static org.archive.resource.ResourceConstants.PAYLOAD_LENGTH; +import static org.archive.resource.ResourceConstants.PAYLOAD_SLOP_BYTES; + +import java.io.IOException; + +import org.archive.extract.ExtractingResourceFactoryMapper; +import org.archive.extract.ExtractingResourceProducer; +import org.archive.extract.ProducerUtils; +import org.archive.extract.ResourceFactoryMapper; +import org.archive.resource.Resource; +import org.archive.resource.ResourceParseException; +import org.archive.resource.ResourceProducer; +import org.archive.util.StreamCopy; + +import org.json.JSONObject; + +import junit.framework.TestCase; + +public class WARCResourceTest extends TestCase { + + public void testWARCResource() throws ResourceParseException, IOException { + String testFileName = "../../format/warc/IAH-urls-wget.warc"; + ResourceProducer producer = ProducerUtils.getProducer(getClass().getResource(testFileName).getPath()); + ResourceFactoryMapper mapper = new ExtractingResourceFactoryMapper(); + ExtractingResourceProducer extractor = new ExtractingResourceProducer(producer, mapper); + + Resource resource = extractor.getNext(); + + while (resource != null) { + JSONObject payloadMD = resource.getMetaData().getTopMetaData().getJSONObject("Envelope") + .getJSONObject("Payload-Metadata"); + + if (payloadMD.has(PAYLOAD_LENGTH)) { + assertTrue(payloadMD.getLong(PAYLOAD_LENGTH) != -1); + } + if (payloadMD.has(PAYLOAD_SLOP_BYTES)) { + assertEquals(4, payloadMD.getLong(PAYLOAD_SLOP_BYTES)); + } + + StreamCopy.readToEOF(resource.getInputStream()); + resource = extractor.getNext(); + } + } +} From c5b779128edd1f0fad2709d4ab1b797326c2cb6c Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Fri, 20 Dec 2024 14:10:44 +0900 Subject: [PATCH 059/174] Update CHANGES.md for 1.3.0 --- CHANGES.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index e3afd137..8a0a7d20 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,40 @@ +1.3.0 +----- + +#### URL Canonicalization Changed + +The output of WaybackURLKeyMaker and other canonicalizers based on BasicURLCanonicalizer has changed for URLs that +contain non UTF-8 percent encoded sequences. For example when a URL contains "%C3%23" it will now be normalised to +"%c3%23" whereas previous releases produced "%25c3%23". This change brings webarchive-commons more inline with pywb, +surt (Python), warcio.js and RFC 3986. While CDX file compatibility with these newer tools should improve, note that CDX +files generated by the new release which contain such URLs may not work correctly with existing versions of +OpenWayback that use the older webarchive-commons. [#102](https://github.com/iipc/webarchive-commons/pull/102) + +#### Bug fixes + +* WAT: Duplicated payload metadata values for "Actual-Content-Length" and "Trailing-Slop-Length" [#103](https://github.com/iipc/webarchive-commons/pull/103) +* ObjectPlusFilesOutputStream.hardlinkOrCopy now uses `Files.createLink()` instead of executing `ln`. This + prevents the potential for security vulnerabilities from command line option injection and improves portability. + +#### Dependency upgrades + +* fastutil removed +* dsiutils removed + +#### Deprecations + +The following classes and enum members have been marked deprecated as a step towards removal of the dependency on +Apache Commons HttpClient 3.1. + +* org.archive.httpclient.HttpRecorderGetMethod +* org.archive.httpclient.HttpRecorderMethod +* org.archive.httpclient.HttpRecorderPostMethod +* org.archive.httpclient.SingleHttpConnectionManager +* org.archive.httpclient.ThreadLocalHttpConnectionManager +* org.archive.util.binsearch.impl.http.ApacheHttp31SLR +* org.archive.util.binsearch.impl.http.ApacheHttp31SLRFactory +* org.archive.util.binsearch.impl.http.HTTPSeekableLineReaderFactory.HttpLibs.APACHE_31 + 1.2.0 ----- From eee48cc18017dde59b1d12f11654a2c752c63d45 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Fri, 20 Dec 2024 14:12:09 +0900 Subject: [PATCH 060/174] [maven-release-plugin] prepare release webarchive-commons-1.3.0 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 6dec154c..f489826c 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ org.netpreserve.commons webarchive-commons - 1.2.1-SNAPSHOT + 1.3.0 jar webarchive-commons @@ -41,7 +41,7 @@ scm:git:git@github.com:iipc/webarchive-commons.git scm:git:git@github.com:iipc/webarchive-commons.git https://github.com/iipc/webarchive-commons - HEAD + webarchive-commons-1.3.0 From a8fd8a74b83d3327bc074cf783f6315659fbc715 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Fri, 20 Dec 2024 14:12:13 +0900 Subject: [PATCH 061/174] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index f489826c..74a4bbe6 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ org.netpreserve.commons webarchive-commons - 1.3.0 + 1.3.1-SNAPSHOT jar webarchive-commons @@ -41,7 +41,7 @@ scm:git:git@github.com:iipc/webarchive-commons.git scm:git:git@github.com:iipc/webarchive-commons.git https://github.com/iipc/webarchive-commons - webarchive-commons-1.3.0 + HEAD From a3a39598fc7b6947e38161e9f27f6842eed95456 Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Tue, 11 Mar 2025 10:20:00 +0100 Subject: [PATCH 062/174] Upgrade GitHub workflow actions cache --- .github/workflows/maven.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 8bb55c4e..60fac096 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -24,7 +24,7 @@ jobs: distribution: 'temurin' cache: maven - name: Cache local Maven repository - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} From c427a12e82f3cebd6ba57152209d0bb5b9de2619 Mon Sep 17 00:00:00 2001 From: Alex Osborne Date: Sun, 18 May 2025 09:39:48 +0900 Subject: [PATCH 063/174] Upgrade to JUnit 5 --- CHANGES.md | 7 + pom.xml | 7 +- .../java/org/archive/util/TmpDirTestCase.java | 119 ---- .../extract/RealCDXExtractorOutputTest.java | 31 +- .../format/dns/DNSResponseParserTest.java | 10 +- .../format/gzip/GZIPMemberSeriesTest.java | 38 +- .../format/gzip/GZIPMemberWriterTest.java | 5 +- .../format/gzip/zipnum/ZipNumWriterTest.java | 16 +- .../http/HttpRequestMessageParserTest.java | 12 +- .../format/http/HttpResponseParserTest.java | 14 +- .../json/CompoundORJSONPathSpecTest.java | 5 +- .../format/json/JSONPathSpecFactoryTest.java | 5 +- .../org/archive/format/json/JSONViewTest.java | 9 +- .../format/json/SimpleJSONPathSpecTest.java | 5 +- .../format/text/html/CDATALexerTest.java | 14 +- .../archive/io/ArchiveReaderFactoryTest.java | 27 +- .../io/BufferedSeekInputStreamTest.java | 9 +- .../archive/io/HeaderedArchiveRecordTest.java | 22 +- .../archive/io/RecordingInputStreamTest.java | 39 +- .../archive/io/RecordingOutputStreamTest.java | 74 ++- .../archive/io/ReplayCharSequenceTest.java | 110 ++-- .../io/RepositionableInputStreamTest.java | 20 +- .../archive/io/arc/ARCReaderFactoryTest.java | 13 +- .../org/archive/io/arc/ARCWriterPoolTest.java | 41 +- .../org/archive/io/arc/ARCWriterTest.java | 121 ++-- .../io/warc/WARCReaderFactoryTest.java | 7 +- .../org/archive/io/warc/WARCWriterTest.java | 67 ++- .../org/archive/net/PublicSuffixesTest.java | 55 +- .../org/archive/resource/MetaDataTest.java | 21 +- .../archive/resource/arc/ARCResourceTest.java | 6 +- .../html/ExtractingParseObserverTest.java | 24 +- .../resource/html/HTMLMetaDataTest.java | 12 +- .../resource/warc/WARCResourceTest.java | 7 +- .../org/archive/uid/UUIDGeneratorTest.java | 7 +- .../url/AggressiveIAURLCanonicalizerTest.java | 9 +- .../url/BasicURLCanonicalizerTest.java | 39 +- .../java/org/archive/url/HandyURLTest.java | 13 +- .../archive/url/IAURLCanonicalizerTest.java | 13 +- .../url/OrdinaryIAURLCanonicalizerTest.java | 10 +- .../java/org/archive/url/URLParserTest.java | 11 +- .../archive/url/URLRegexTransformerTest.java | 45 +- .../org/archive/url/UsableURIFactoryTest.java | 564 +++++++++--------- .../java/org/archive/url/UsableURITest.java | 16 +- .../archive/url/WaybackURLKeyMakerTest.java | 7 +- .../org/archive/util/ArchiveUtilsTest.java | 231 ++++--- .../java/org/archive/util/ByteOpTest.java | 14 +- .../org/archive/util/CrossProductTest.java | 8 +- .../java/org/archive/util/FileUtilsTest.java | 69 ++- .../util/InterruptibleCharSequenceTest.java | 21 +- .../org/archive/util/MimetypeUtilsTest.java | 63 +- .../org/archive/util/PropertyUtilsTest.java | 11 +- .../util/StringFieldExtractorTest.java | 10 +- src/test/java/org/archive/util/TestUtils.java | 17 +- .../org/archive/util/anvl/ANVLRecordTest.java | 56 +- .../util/binsearch/SortedTextFileTest.java | 8 +- .../iterator/CachingStringFilterTest.java | 5 +- .../iterator/FilterStringIteratorTest.java | 25 +- .../iterator/SortedCompositeIteratorTest.java | 8 +- .../util/zip/GZIPMembersInputStreamTest.java | 157 ++--- 59 files changed, 1236 insertions(+), 1173 deletions(-) delete mode 100644 src/main/java/org/archive/util/TmpDirTestCase.java diff --git a/CHANGES.md b/CHANGES.md index 8a0a7d20..478238bf 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,10 @@ +Unreleased +---------- + +#### Dependency upgrades + +- **junit**: 4.13.2 → 5.12.2 + 1.3.0 ----- diff --git a/pom.xml b/pom.xml index 74a4bbe6..c70a2cd7 100644 --- a/pom.xml +++ b/pom.xml @@ -52,9 +52,10 @@ - junit - junit - 4.13.2 + org.junit.jupiter + junit-jupiter + 5.12.2 + test diff --git a/src/main/java/org/archive/util/TmpDirTestCase.java b/src/main/java/org/archive/util/TmpDirTestCase.java deleted file mode 100644 index 09ec345b..00000000 --- a/src/main/java/org/archive/util/TmpDirTestCase.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.archive.util; - -import java.io.File; -import java.io.IOException; - -import junit.framework.TestCase; - - -/** - * Base class for TestCases that want access to a tmp dir for the writing - * of files. - * - * @author stack - */ -public abstract class TmpDirTestCase extends TestCase -{ - /** - * Name of the system property that holds pointer to tmp directory into - * which we can safely write files. - */ - public static final String TEST_TMP_SYSTEM_PROPERTY_NAME = "testtmpdir"; - - /** - * Default test tmp. - */ - public static final String DEFAULT_TEST_TMP_DIR = File.separator + "tmp" + - File.separator + "heritrix-junit-tests"; - - /** - * Directory to write temporary files to. - */ - private File tmpDir = null; - - - public TmpDirTestCase() - { - super(); - } - - public TmpDirTestCase(String testName) - { - super(testName); - } - - /* - * @see TestCase#setUp() - */ - protected void setUp() throws Exception { - super.setUp(); - this.tmpDir = tmpDir(); - } - - /** - * @return Returns the tmpDir. - */ - public File getTmpDir() - { - return this.tmpDir; - } - - /** - * Delete any files left over from previous run. - * - * @param basename Base name of files we're to clean up. - */ - public void cleanUpOldFiles(String basename) { - cleanUpOldFiles(getTmpDir(), basename); - } - - /** - * Delete any files left over from previous run. - * - * @param prefix Base name of files we're to clean up. - * @param basedir Directory to start cleaning in. - */ - public void cleanUpOldFiles(File basedir, String prefix) { - File [] files = FileUtils.getFilesWithPrefix(basedir, prefix); - if (files != null) { - for (int i = 0; i < files.length; i++) { - org.apache.commons.io.FileUtils.deleteQuietly(files[i]); - } - } - } - - - public static File tmpDir() throws IOException { - String tmpDirStr = System.getProperty(TEST_TMP_SYSTEM_PROPERTY_NAME); - tmpDirStr = (tmpDirStr == null)? DEFAULT_TEST_TMP_DIR: tmpDirStr; - File tmpDir = new File(tmpDirStr); - FileUtils.ensureWriteableDirectory(tmpDir); - - if (!tmpDir.canWrite()) - { - throw new IOException(tmpDir.getAbsolutePath() + - " is unwriteable."); - } - - return tmpDir; - } -} diff --git a/src/test/java/org/archive/extract/RealCDXExtractorOutputTest.java b/src/test/java/org/archive/extract/RealCDXExtractorOutputTest.java index 14f8489d..a716df82 100644 --- a/src/test/java/org/archive/extract/RealCDXExtractorOutputTest.java +++ b/src/test/java/org/archive/extract/RealCDXExtractorOutputTest.java @@ -1,28 +1,29 @@ package org.archive.extract; -import java.net.MalformedURLException; import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.net.URLEncoder; -import junit.framework.TestCase; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; -public class RealCDXExtractorOutputTest extends TestCase { +public class RealCDXExtractorOutputTest { + + @Test public void testEscapeResolvedUrl() throws Exception { - String context ="http://www.uni-giessen.de/cms/studium/dateien/informationberatung/merkblattpdf"; - String spec = "http://fss.plone.uni-giessen.de/fß/studium/dateien/informationberatung/merkblattpdf/file/Mérkblatt zur Gestaltung von Nachteilsausgleichen.pdf?föo=bar#änchor"; - String escaped = RealCDXExtractorOutput.resolve(context, spec); - assertTrue(escaped.indexOf(" ") < 0); - URI parsed = new URI(escaped); - assertEquals("änchor", parsed.getFragment()); + String context = "http://www.uni-giessen.de/cms/studium/dateien/informationberatung/merkblattpdf"; + String spec = "http://fss.plone.uni-giessen.de/fß/studium/dateien/informationberatung/merkblattpdf/file/Mérkblatt zur Gestaltung von Nachteilsausgleichen.pdf?föo=bar#änchor"; + String escaped = RealCDXExtractorOutput.resolve(context, spec); + assertTrue(escaped.indexOf(" ") < 0); + URI parsed = new URI(escaped); + assertEquals("änchor", parsed.getFragment()); } + @Test public void testNoDoubleEscaping() throws Exception { - String spec = "https://www.google.com/search?q=java+escape+url+spaces&ie=utf-8&oe=utf-8"; - String resolved = RealCDXExtractorOutput.resolve(spec, spec); - assertTrue(spec.equals(resolved)); + String spec = "https://www.google.com/search?q=java+escape+url+spaces&ie=utf-8&oe=utf-8"; + String resolved = RealCDXExtractorOutput.resolve(spec, spec); + assertTrue(spec.equals(resolved)); } } diff --git a/src/test/java/org/archive/format/dns/DNSResponseParserTest.java b/src/test/java/org/archive/format/dns/DNSResponseParserTest.java index 27d0fdad..7ade0ad5 100644 --- a/src/test/java/org/archive/format/dns/DNSResponseParserTest.java +++ b/src/test/java/org/archive/format/dns/DNSResponseParserTest.java @@ -3,15 +3,13 @@ import java.io.ByteArrayInputStream; import java.io.IOException; -import org.archive.format.dns.DNSParseException; -import org.archive.format.dns.DNSRecord; -import org.archive.format.dns.DNSResponse; -import org.archive.format.dns.DNSResponseParser; +import org.junit.jupiter.api.Test; -import junit.framework.TestCase; +import static org.junit.jupiter.api.Assertions.assertEquals; -public class DNSResponseParserTest extends TestCase { +public class DNSResponseParserTest { DNSResponseParser parser = new DNSResponseParser(); + @Test public void testParse() throws DNSParseException, IOException { verifyResults("20110328212258\nfarm6.static.flickr.a06.yahoodns.net.\t300\tIN\tA\t98.136.170.121\n", "20110328212258",new String[][] {{"farm6.static.flickr.a06.yahoodns.net.","300","IN","A","98.136.170.121"}}); diff --git a/src/test/java/org/archive/format/gzip/GZIPMemberSeriesTest.java b/src/test/java/org/archive/format/gzip/GZIPMemberSeriesTest.java index 2eec46ec..6f218ebb 100644 --- a/src/test/java/org/archive/format/gzip/GZIPMemberSeriesTest.java +++ b/src/test/java/org/archive/format/gzip/GZIPMemberSeriesTest.java @@ -9,9 +9,6 @@ import org.archive.util.ByteOp; import org.archive.util.IAUtils; import org.archive.util.TestUtils; -import org.archive.format.gzip.GZIPFormatException; -import org.archive.format.gzip.GZIPMemberSeries; -import org.archive.format.gzip.GZIPSeriesMember; import org.archive.streamcontext.ByteArrayWrappedStream; import org.archive.streamcontext.SimpleStream; import org.archive.streamcontext.Stream; @@ -19,10 +16,13 @@ import com.google.common.io.ByteStreams; import com.google.common.primitives.Bytes; -import junit.framework.TestCase; +import org.junit.jupiter.api.Test; -public class GZIPMemberSeriesTest extends TestCase { +import static org.junit.jupiter.api.Assertions.*; +public class GZIPMemberSeriesTest { + + @Test public void testSingle() throws IndexOutOfBoundsException, FileNotFoundException, IOException { InputStream is = getClass().getResourceAsStream("abcd.gz"); @@ -38,6 +38,7 @@ public void testSingle() throws IndexOutOfBoundsException, FileNotFoundException assertNull(s.getNextMember()); } + @Test public void testSingleEmpty() throws IndexOutOfBoundsException, FileNotFoundException, IOException { InputStream is = getClass().getResourceAsStream("empty.gz"); @@ -59,6 +60,7 @@ public void testSingleEmpty() throws IndexOutOfBoundsException, FileNotFoundExce assertTrue(s.gotEOF()); } + @Test public void testDouble() throws IndexOutOfBoundsException, FileNotFoundException, IOException { InputStream is = getClass().getResourceAsStream("abcd.gz"); @@ -81,14 +83,14 @@ public void testDouble() throws IndexOutOfBoundsException, FileNotFoundException assertNull(s.getNextMember()); } - + @Test public void testSingleCRCStrict() throws IndexOutOfBoundsException, FileNotFoundException, IOException { InputStream is = getClass().getResourceAsStream("abcd.gz"); byte abcd[] = ByteStreams.toByteArray(is); byte oldb = abcd[abcd.length-1]; abcd[abcd.length-1] = (byte) (abcd[abcd.length-1] + 1); - assertFalse(oldb == abcd[abcd.length-1]); + assertNotEquals(oldb, abcd[abcd.length - 1]); ByteArrayInputStream bais = new ByteArrayInputStream(abcd); Stream stream = new SimpleStream(bais); @@ -117,14 +119,15 @@ public void testSingleCRCStrict() throws IndexOutOfBoundsException, FileNotFound } assertNotNull(e); } - + + @Test public void testSingleCRCLAX() throws IndexOutOfBoundsException, FileNotFoundException, IOException { InputStream is = getClass().getResourceAsStream("abcd.gz"); byte abcd[] = ByteStreams.toByteArray(is); byte oldb = abcd[abcd.length-1]; abcd[abcd.length-1] = (byte) (abcd[abcd.length-1] + 1); - assertFalse(oldb == abcd[abcd.length-1]); + assertNotEquals(oldb, abcd[abcd.length - 1]); ByteArrayInputStream bais = new ByteArrayInputStream(abcd); Stream stream = new SimpleStream(bais); @@ -154,7 +157,8 @@ public void testSingleCRCLAX() throws IndexOutOfBoundsException, FileNotFoundExc assertNull(e); assertNull(s.getNextMember()); } - + + @Test public void testDoubleCRC1LAX() throws IndexOutOfBoundsException, FileNotFoundException, IOException { InputStream is = getClass().getResourceAsStream("abcd.gz"); @@ -162,7 +166,7 @@ public void testDoubleCRC1LAX() throws IndexOutOfBoundsException, FileNotFoundEx byte abcdorig[] = ByteOp.copy(abcd); byte oldb = abcd[abcd.length-1]; abcd[abcd.length-1] = (byte) (abcd[abcd.length-1] + 1); - assertFalse(oldb == abcd[abcd.length-1]); + assertNotEquals(oldb, abcd[abcd.length - 1]); byte both[] = Bytes.concat(abcd,abcdorig); @@ -195,7 +199,8 @@ public void testDoubleCRC1LAX() throws IndexOutOfBoundsException, FileNotFoundEx assertNotNull(m); TestUtils.assertStreamEquals(m,"abcd".getBytes(IAUtils.UTF8)); } - + + @Test public void testSingleDeflateError() throws IndexOutOfBoundsException, IOException { InputStream is = getClass().getResourceAsStream("abcd.gz"); @@ -240,7 +245,7 @@ public void testSingleDeflateError() throws IndexOutOfBoundsException, IOExcepti assertNull(m); } - + @Test public void testDoubleDeflateError() throws IndexOutOfBoundsException, IOException { InputStream is = getClass().getResourceAsStream("abcd.gz"); @@ -290,7 +295,8 @@ public void testDoubleDeflateError() throws IndexOutOfBoundsException, IOExcepti assertFalse(s.gotIOError()); } - + + @Test public void testDoubleBiggerDeflateErrOnFirst() throws IOException { String resource = "double-single-inflate-error.gz"; InputStream is = getClass().getResourceAsStream(resource); @@ -333,7 +339,8 @@ public void testDoubleBiggerDeflateErrOnFirst() throws IOException { } - + + @Test public void testAutoSkip() throws IOException { InputStream is = getClass().getResourceAsStream("abcd.gz"); byte abcd[] = ByteStreams.toByteArray(is); @@ -375,6 +382,7 @@ public void testAutoSkip() throws IOException { assertTrue(s.gotEOF()); } + @Test public void testWgetProblem() throws IndexOutOfBoundsException, FileNotFoundException, IOException { InputStream is = getClass().getResourceAsStream("IAH-urls-wget.warc.gz"); new GZIPDecoder().parseHeader(is); diff --git a/src/test/java/org/archive/format/gzip/GZIPMemberWriterTest.java b/src/test/java/org/archive/format/gzip/GZIPMemberWriterTest.java index 483d2baf..45bc18e4 100644 --- a/src/test/java/org/archive/format/gzip/GZIPMemberWriterTest.java +++ b/src/test/java/org/archive/format/gzip/GZIPMemberWriterTest.java @@ -7,10 +7,11 @@ import org.archive.util.IAUtils; -import junit.framework.TestCase; +import org.junit.jupiter.api.Test; -public class GZIPMemberWriterTest extends TestCase { +public class GZIPMemberWriterTest { + @Test public void testWrite() throws IOException { File outFile = File.createTempFile("tmp", ".gz"); GZIPMemberWriter gzw = new GZIPMemberWriter(new FileOutputStream(outFile)); diff --git a/src/test/java/org/archive/format/gzip/zipnum/ZipNumWriterTest.java b/src/test/java/org/archive/format/gzip/zipnum/ZipNumWriterTest.java index cfadbd79..25a5eaa7 100644 --- a/src/test/java/org/archive/format/gzip/zipnum/ZipNumWriterTest.java +++ b/src/test/java/org/archive/format/gzip/zipnum/ZipNumWriterTest.java @@ -10,19 +10,21 @@ import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; -import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import org.archive.format.gzip.GZIPMemberSeries; import org.archive.format.gzip.GZIPSeriesMember; import org.archive.streamcontext.SimpleStream; -import junit.framework.TestCase; +import org.junit.jupiter.api.Test; -public class ZipNumWriterTest extends TestCase { +import static org.junit.jupiter.api.Assertions.assertEquals; +public class ZipNumWriterTest { + + @Test public void testAddRecord() throws IOException { - Charset UTF8 = Charset.forName("UTF-8"); - File main = File.createTempFile("test-znw",".main"); + File main = File.createTempFile("test-znw",".main"); File summ = File.createTempFile("test-znw",".summ"); main.deleteOnExit(); summ.deleteOnExit(); @@ -31,11 +33,11 @@ public void testAddRecord() throws IOException { ZipNumWriter znw = new ZipNumWriter(new FileOutputStream(main,false), new FileOutputStream(summ,false), limit); for(int i = 0; i < 1000; i++) { - znw.addRecord(String.format("%06d\n",i).getBytes(UTF8)); + znw.addRecord(String.format("%06d\n",i).getBytes(StandardCharsets.UTF_8)); } znw.close(); InputStreamReader isr = - new InputStreamReader(new FileInputStream(summ),UTF8); + new InputStreamReader(new FileInputStream(summ), StandardCharsets.UTF_8); BufferedReader br = new BufferedReader(isr); String line = null; int count = 0; diff --git a/src/test/java/org/archive/format/http/HttpRequestMessageParserTest.java b/src/test/java/org/archive/format/http/HttpRequestMessageParserTest.java index 50df9dde..9a5d69af 100644 --- a/src/test/java/org/archive/format/http/HttpRequestMessageParserTest.java +++ b/src/test/java/org/archive/format/http/HttpRequestMessageParserTest.java @@ -3,16 +3,16 @@ import java.io.ByteArrayInputStream; import java.io.IOException; -import org.archive.format.http.HttpConstants; -import org.archive.format.http.HttpParseException; -import org.archive.format.http.HttpRequestMessage; -import org.archive.format.http.HttpRequestMessageParser; import org.archive.util.IAUtils; -import junit.framework.TestCase; +import org.junit.jupiter.api.Test; -public class HttpRequestMessageParserTest extends TestCase implements HttpConstants { +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class HttpRequestMessageParserTest implements HttpConstants { HttpRequestMessageParser parser = new HttpRequestMessageParser(); + + @Test public void testParse() throws IOException { assertParse("GET / HTTP/1.0\r\n", METHOD_GET, "/", VERSION_0); assertParse("GET / HTTP/1.1\r\n", METHOD_GET, "/", VERSION_1); diff --git a/src/test/java/org/archive/format/http/HttpResponseParserTest.java b/src/test/java/org/archive/format/http/HttpResponseParserTest.java index ea076a69..631d67c7 100644 --- a/src/test/java/org/archive/format/http/HttpResponseParserTest.java +++ b/src/test/java/org/archive/format/http/HttpResponseParserTest.java @@ -5,16 +5,14 @@ import org.archive.util.IAUtils; import org.archive.util.TestUtils; -import org.archive.format.http.HttpHeader; -import org.archive.format.http.HttpHeaders; -import org.archive.format.http.HttpParseException; -import org.archive.format.http.HttpResponse; -import org.archive.format.http.HttpResponseParser; -import junit.framework.TestCase; +import org.junit.jupiter.api.Test; -public class HttpResponseParserTest extends TestCase { +import static org.junit.jupiter.api.Assertions.*; +public class HttpResponseParserTest { + + @Test public void testParse() throws IOException { HttpResponseParser parser = new HttpResponseParser(); @@ -38,6 +36,7 @@ public void testParse() throws IOException { } + @Test public void testParseWithLf() throws IOException { HttpResponseParser parser = new HttpResponseParser(); @@ -57,6 +56,7 @@ public void testParseWithLf() throws IOException { } + @Test public void testParseEmptyHeaderField() throws IOException { HttpResponseParser parser = new HttpResponseParser(); diff --git a/src/test/java/org/archive/format/json/CompoundORJSONPathSpecTest.java b/src/test/java/org/archive/format/json/CompoundORJSONPathSpecTest.java index 57c21965..ef8c2fa0 100644 --- a/src/test/java/org/archive/format/json/CompoundORJSONPathSpecTest.java +++ b/src/test/java/org/archive/format/json/CompoundORJSONPathSpecTest.java @@ -6,11 +6,12 @@ import org.json.JSONException; import org.json.JSONObject; -import junit.framework.TestCase; +import org.junit.jupiter.api.Test; -public class CompoundORJSONPathSpecTest extends TestCase { +public class CompoundORJSONPathSpecTest { String json1S = "{\"a\":\"A\"}"; String json2S = "{\"b\":\"B\"}"; + @Test public void testExtract() throws JSONException { JSONObject json1 = new JSONObject(json1S); JSONObject json2 = new JSONObject(json2S); diff --git a/src/test/java/org/archive/format/json/JSONPathSpecFactoryTest.java b/src/test/java/org/archive/format/json/JSONPathSpecFactoryTest.java index ab999dca..257cb112 100644 --- a/src/test/java/org/archive/format/json/JSONPathSpecFactoryTest.java +++ b/src/test/java/org/archive/format/json/JSONPathSpecFactoryTest.java @@ -4,9 +4,9 @@ import org.json.JSONException; import org.json.JSONObject; -import junit.framework.TestCase; +import org.junit.jupiter.api.Test; -public class JSONPathSpecFactoryTest extends TestCase { +public class JSONPathSpecFactoryTest { String json1S = "{\"a\":\"A\"}"; String json2S = "{\"b\":\"B\"}"; @@ -14,6 +14,7 @@ public class JSONPathSpecFactoryTest extends TestCase { String json4S = "{\"b\":[{\"x\":\"x1\", \"y\":\"y1\"},{\"x\":\"x2\", \"y\":\"y2\"}]}"; + @Test public void testGet() throws JSONException { JSONObject json1 = new JSONObject(json1S); JSONObject json2 = new JSONObject(json2S); diff --git a/src/test/java/org/archive/format/json/JSONViewTest.java b/src/test/java/org/archive/format/json/JSONViewTest.java index 20bd4fe6..aabbe7df 100644 --- a/src/test/java/org/archive/format/json/JSONViewTest.java +++ b/src/test/java/org/archive/format/json/JSONViewTest.java @@ -4,14 +4,15 @@ import org.json.JSONException; import org.json.JSONObject; -import junit.framework.TestCase; +import org.junit.jupiter.api.Test; -public class JSONViewTest extends TestCase { +public class JSONViewTest { public int getInt(byte b[]) { return b[0] & 0xff; } - + + @Test public void testBytes() throws JSONException { JSONObject o = new JSONObject(); o.append("name1", "val\\rue1"); @@ -28,6 +29,8 @@ public void testBytes() throws JSONException { System.out.format("I(%d) gi(%d)\n",i,gi); } } + + @Test public void testApply() throws JSONException { String json1S = "{\"url\":\"a\",\"link\":[{\"zz\":\"1\",\"qq\":\"qa\"},{\"zz2\":\"2\",\"qq\":\"qb\"},{\"zz\":\"3\",\"qq\":\"qc\"},{\"zz\":\"4\"}]}"; JSONObject json1 = new JSONObject(json1S); diff --git a/src/test/java/org/archive/format/json/SimpleJSONPathSpecTest.java b/src/test/java/org/archive/format/json/SimpleJSONPathSpecTest.java index a703b49a..640a5a80 100644 --- a/src/test/java/org/archive/format/json/SimpleJSONPathSpecTest.java +++ b/src/test/java/org/archive/format/json/SimpleJSONPathSpecTest.java @@ -4,15 +4,16 @@ import org.json.JSONException; import org.json.JSONObject; -import junit.framework.TestCase; +import org.junit.jupiter.api.Test; -public class SimpleJSONPathSpecTest extends TestCase { +public class SimpleJSONPathSpecTest { String json1 = "{\"a\": { \"b\": \"Foo\" }}"; String json2 = "{\"a\": { \"b\": [{\"a\":\"1\"},{\"a\":\"2\"}] }}"; String json3 = "{\"a\": { \"b\": {\"A\":\"11\",\"B\":\"22\"} }}"; String json4 = "{\"a\": { \"b\": [{\"A\":\"11\",\"B\":\"22\"},{\"A\":\"33\",\"B\":\"44\"}] }}"; + @Test public void testExtract() throws JSONException { JSONObject json = new JSONObject(json1); JSONPathSpec spec = new SimpleJSONPathSpec("a.b"); diff --git a/src/test/java/org/archive/format/text/html/CDATALexerTest.java b/src/test/java/org/archive/format/text/html/CDATALexerTest.java index 481a3eda..856576ba 100644 --- a/src/test/java/org/archive/format/text/html/CDATALexerTest.java +++ b/src/test/java/org/archive/format/text/html/CDATALexerTest.java @@ -1,17 +1,16 @@ package org.archive.format.text.html; -import org.archive.format.text.html.CDATALexer; -import org.archive.format.text.html.NodeUtils; import org.htmlparser.Node; import org.htmlparser.lexer.Page; -//import org.htmlparser.nodes.RemarkNode; import org.htmlparser.nodes.TagNode; import org.htmlparser.nodes.TextNode; import org.htmlparser.util.ParserException; -import junit.framework.TestCase; +import org.junit.jupiter.api.Test; -public class CDATALexerTest extends TestCase { +import static org.junit.jupiter.api.Assertions.*; + +public class CDATALexerTest { CDATALexer l; Node n; private CDATALexer makeLexer(String html) { @@ -19,7 +18,8 @@ private CDATALexer makeLexer(String html) { t.setPage(new Page(html)); return t; } - + + @Test public void testNextNode() throws ParserException { l = makeLexer("blem"); n = l.nextNode(); @@ -35,6 +35,7 @@ public void testNextNode() throws ParserException { assertNull(l.nextNode()); } + @Test public void testInJS() throws ParserException { l = makeLexer(""); assertFalse(l.inCSS()); @@ -54,6 +55,7 @@ public void testInJS() throws ParserException { assertTrue(NodeUtils.isCloseTagNodeNamed(n, "SCRIPT")); } + @Test public void testInCSS() throws ParserException { l = makeLexer(""); assertFalse(l.inCSS()); diff --git a/src/test/java/org/archive/io/ArchiveReaderFactoryTest.java b/src/test/java/org/archive/io/ArchiveReaderFactoryTest.java index 2313868c..f7ad75d2 100644 --- a/src/test/java/org/archive/io/ArchiveReaderFactoryTest.java +++ b/src/test/java/org/archive/io/ArchiveReaderFactoryTest.java @@ -21,29 +21,34 @@ import java.io.File; import java.io.IOException; -import java.net.MalformedURLException; import java.net.URL; import java.util.Iterator; import org.apache.commons.lang.StringUtils; -import org.archive.io.ArchiveRecord; import org.archive.io.arc.ARCWriterTest; -import org.archive.util.TmpDirTestCase; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class ArchiveReaderFactoryTest { + @TempDir + File tempDir; -public class ArchiveReaderFactoryTest extends TmpDirTestCase { /** * Test local file as URL * @throws IOException */ + @Test public void testGetFileURL() throws IOException { - File arc = ARCWriterTest.createARCFile(getTmpDir(), true); + File arc = ARCWriterTest.createARCFile(tempDir, true); ArchiveReader reader = null; try { reader = ArchiveReaderFactory. get(new URL("file:////" + arc.getAbsolutePath())); for (Iterator i = reader.iterator(); i.hasNext();) { ArchiveRecord r = (ArchiveRecord)i.next(); - assertTrue("mime unread",StringUtils.isNotBlank(r.getHeader().getMimetype())); + assertTrue(StringUtils.isNotBlank(r.getHeader().getMimetype()),"mime unread"); } } finally { if (reader != null) { @@ -56,14 +61,15 @@ public void testGetFileURL() throws IOException { * Test local file as File * @throws IOException */ + @Test public void testGetFile() throws IOException { - File arc = ARCWriterTest.createARCFile(getTmpDir(), true); + File arc = ARCWriterTest.createARCFile(tempDir, true); ArchiveReader reader = null; try { reader = ArchiveReaderFactory.get(arc.getAbsoluteFile()); for (Iterator i = reader.iterator(); i.hasNext();) { ArchiveRecord r = (ArchiveRecord)i.next(); - assertTrue("mime unread",StringUtils.isNotBlank(r.getHeader().getMimetype())); + assertTrue(StringUtils.isNotBlank(r.getHeader().getMimetype()),"mime unread"); } } finally { if (reader != null) { @@ -76,14 +82,15 @@ public void testGetFile() throws IOException { * Test local file as String path * @throws IOException */ + @Test public void testGetPath() throws IOException { - File arc = ARCWriterTest.createARCFile(getTmpDir(), true); + File arc = ARCWriterTest.createARCFile(tempDir, true); ArchiveReader reader = null; try { reader = ArchiveReaderFactory.get(arc.getAbsoluteFile().getAbsolutePath()); for (Iterator i = reader.iterator(); i.hasNext();) { ArchiveRecord r = (ArchiveRecord)i.next(); - assertTrue("mime unread",StringUtils.isNotBlank(r.getHeader().getMimetype())); + assertTrue(StringUtils.isNotBlank(r.getHeader().getMimetype()),"mime unread"); } } finally { if (reader != null) { diff --git a/src/test/java/org/archive/io/BufferedSeekInputStreamTest.java b/src/test/java/org/archive/io/BufferedSeekInputStreamTest.java index 270e45e0..f7e8e0b2 100644 --- a/src/test/java/org/archive/io/BufferedSeekInputStreamTest.java +++ b/src/test/java/org/archive/io/BufferedSeekInputStreamTest.java @@ -18,9 +18,11 @@ */ package org.archive.io; +import org.junit.jupiter.api.Test; + import java.util.Random; -import junit.framework.TestCase; +import static org.junit.jupiter.api.Assertions.assertEquals; /** @@ -29,11 +31,12 @@ * * @author pjack */ -public class BufferedSeekInputStreamTest extends TestCase { +public class BufferedSeekInputStreamTest { private static byte[] TEST_DATA = makeTestData(); - + + @Test public void testPosition() throws Exception { Random random = new Random(); ArraySeekInputStream asis = new ArraySeekInputStream(TEST_DATA); diff --git a/src/test/java/org/archive/io/HeaderedArchiveRecordTest.java b/src/test/java/org/archive/io/HeaderedArchiveRecordTest.java index 9f7e2a15..7988cb2b 100644 --- a/src/test/java/org/archive/io/HeaderedArchiveRecordTest.java +++ b/src/test/java/org/archive/io/HeaderedArchiveRecordTest.java @@ -26,13 +26,15 @@ import java.util.Map; import java.util.Set; -import junit.framework.TestCase; - import org.apache.commons.httpclient.Header; import org.archive.io.arc.ARCRecord; import org.archive.io.warc.WARCRecord; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; -public class HeaderedArchiveRecordTest extends TestCase { +public class HeaderedArchiveRecordTest { private static final String HTTPHEADER = "HTTP/1.1 200 OK\r\n" + "Last-Modified: Sun, 28 Aug 2005 14:10:55 GMT\r\n" + "Content-Length: 108\r\n" + "Connection: close\r\n" @@ -41,6 +43,7 @@ public class HeaderedArchiveRecordTest extends TestCase { + " \r\n" + " \r\n" + " \r\n" + " \r\n" + ""; + @Test public void testParseHttpHeadersInWARC() throws IOException { final String url = "http://foo.maths.uq.edu.au/index.html"; // final String warcHeader = "WARC/0.10 000000000486 response " + @@ -76,8 +79,8 @@ public void testParseHttpHeadersInWARC() throws IOException { String bodyRead = new String(b); assertEquals(BODY, bodyRead); assertHeaderCorrectlyParsed(har.getContentHeaders()); - assertEquals("failed to retrieve Url from metadata", har.getHeader() - .getUrl(), url); + assertEquals(har.getHeader().getUrl(), url, + "failed to retrieve Url from metadata"); } public void testParseHttpHeadersInARC() throws IOException { @@ -165,6 +168,7 @@ public String getVersion() { assertHeaderCorrectlyParsed(har.getContentHeaders()); } + @Test public void testEasierParseHttpHeadersInARC() throws IOException { final String url = "http://www.archive.org/index.htm"; final String arcHeader = url @@ -181,14 +185,13 @@ public void testEasierParseHttpHeadersInARC() throws IOException { String bodyRead = new String(b); assertEquals(BODY, bodyRead); assertHeaderCorrectlyParsed(har.getContentHeaders()); - assertEquals("failed to retrieve Url from metadata", har.getHeader() - .getUrl(), url); + assertEquals(har.getHeader().getUrl(), url, "failed to retrieve Url from metadata"); } private void assertHeaderCorrectlyParsed(Header[] headers) { final List orgHeaders = Arrays.asList(HTTPHEADER.split("\r\n")); - assertEquals("not all HTTP header entries have been retrieved", - orgHeaders.size(), headers.length + 1); + assertEquals(orgHeaders.size(), headers.length + 1, + "not all HTTP header entries have been retrieved"); for (Header header : headers) { assertTrue(orgHeaders.contains(header.getName() + ": " @@ -196,6 +199,7 @@ private void assertHeaderCorrectlyParsed(Header[] headers) { } } + @Test public void testNoheaderWARC() throws IOException { String b = "hello world"; String c = "WARC/0.12\r\nContent-Type: text/plain\r\n" diff --git a/src/test/java/org/archive/io/RecordingInputStreamTest.java b/src/test/java/org/archive/io/RecordingInputStreamTest.java index 20a8b8b3..9ddc7457 100644 --- a/src/test/java/org/archive/io/RecordingInputStreamTest.java +++ b/src/test/java/org/archive/io/RecordingInputStreamTest.java @@ -25,7 +25,11 @@ import java.io.PipedInputStream; import java.io.PipedOutputStream; -import org.archive.util.TmpDirTestCase; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -33,17 +37,10 @@ * * @author gojomo */ -public class RecordingInputStreamTest extends TmpDirTestCase -{ - +public class RecordingInputStreamTest { + @TempDir + File tempDir; - /* - * @see TmpDirTestCase#setUp() - */ - protected void setUp() throws Exception - { - super.setUp(); - } /** * Test readFullyOrUntil soft (no exception) and hard (exception) @@ -53,10 +50,11 @@ protected void setUp() throws Exception * @throws InterruptedException * @throws RecorderTimeoutException */ + @Test public void testReadFullyOrUntil() throws RecorderTimeoutException, IOException, InterruptedException { RecordingInputStream ris = new RecordingInputStream(16384, (new File( - getTmpDir(), "testReadFullyOrUntil").getAbsolutePath())); + tempDir, "testReadFullyOrUntil").getAbsolutePath())); ByteArrayInputStream bais = new ByteArrayInputStream( "abcdefghijklmnopqrstuvwxyz".getBytes()); // test soft max @@ -67,7 +65,7 @@ public void testReadFullyOrUntil() throws RecorderTimeoutException, IOException, ReplayInputStream res = ris.getReplayInputStream(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); res.readFullyTo(baos); - assertEquals("soft max cutoff","abcdefg",new String(baos.toByteArray())); + assertEquals("abcdefg",new String(baos.toByteArray()),"soft max cutoff"); // test hard max bais.reset(); baos.reset(); @@ -79,25 +77,26 @@ public void testReadFullyOrUntil() throws RecorderTimeoutException, IOException, } catch (RecorderLengthExceededException ex) { exceptionThrown = true; } - assertTrue("hard max exception",exceptionThrown); + assertTrue(exceptionThrown,"hard max exception"); ris.close(); res = ris.getReplayInputStream(); res.readFullyTo(baos); - assertEquals("hard max cutoff","abcdefghijk", - new String(baos.toByteArray())); + assertEquals("abcdefghijk",new String(baos.toByteArray()), + "hard max cutoff"); // test timeout PipedInputStream pin = new PipedInputStream(); PipedOutputStream pout = new PipedOutputStream(pin); ris.open(pin); exceptionThrown = false; trickle("abcdefghijklmnopqrstuvwxyz".getBytes(),pout); + int timeout = 200; try { - ris.setLimits(0,5000,0); + ris.setLimits(0, timeout,0); ris.readFullyOrUntil(0); } catch (RecorderTimeoutException ex) { exceptionThrown = true; } - assertTrue("timeout exception",exceptionThrown); + assertTrue(exceptionThrown,"timeout exception"); ris.close(); // test rate limit bais = new ByteArrayInputStream(new byte[1024*2*5]); @@ -107,7 +106,7 @@ public void testReadFullyOrUntil() throws RecorderTimeoutException, IOException, ris.readFullyOrUntil(0); long endTime = System.currentTimeMillis(); long duration = endTime - startTime; - assertTrue("read too fast: "+duration,duration>=5000); + assertTrue(duration>= timeout,"read too fast: "+duration); ris.close(); } @@ -116,7 +115,7 @@ protected void trickle(final byte[] bytes, final PipedOutputStream pout) { public void run() { try { for (int i = 0; i < bytes.length; i++) { - Thread.sleep(1000); + Thread.sleep(200); pout.write(bytes[i]); } pout.close(); diff --git a/src/test/java/org/archive/io/RecordingOutputStreamTest.java b/src/test/java/org/archive/io/RecordingOutputStreamTest.java index f697ff31..c94f8245 100644 --- a/src/test/java/org/archive/io/RecordingOutputStreamTest.java +++ b/src/test/java/org/archive/io/RecordingOutputStreamTest.java @@ -25,7 +25,11 @@ import java.io.IOException; import org.archive.util.Base32; -import org.archive.util.TmpDirTestCase; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; /** @@ -33,8 +37,7 @@ * * @author stack */ -public class RecordingOutputStreamTest extends TmpDirTestCase -{ +public class RecordingOutputStreamTest { /** * Size of buffer used in tests. */ @@ -45,14 +48,9 @@ public class RecordingOutputStreamTest extends TmpDirTestCase */ private static final int WRITE_TOTAL = 10; + @TempDir + File tempDir; - /* - * @see TmpDirTestCase#setUp() - */ - protected void setUp() throws Exception - { - super.setUp(); - } /** * Test reusing instance of RecordingOutputStream. @@ -60,13 +58,13 @@ protected void setUp() throws Exception * @throws IOException Failed open of backing file or opening of * input streams verifying recording. */ + @Test public void testReuse() throws IOException { final String BASENAME = "testReuse"; - cleanUpOldFiles(BASENAME); RecordingOutputStream ros = new RecordingOutputStream(BUFFER_SIZE, - (new File(getTmpDir(), BASENAME + "Bkg.txt")).getAbsolutePath()); + (new File(tempDir, BASENAME + "Bkg.txt")).getAbsolutePath()); for (int i = 0; i < 3; i++) { reuse(BASENAME, ros, i); @@ -92,13 +90,13 @@ private void reuse(String baseName, RecordingOutputStream ros, int index) * @throws IOException Failed open of backing file or opening of * input streams verifying recording. */ + @Test public void testWriteint() throws IOException { final String BASENAME = "testWriteint"; - cleanUpOldFiles(BASENAME); RecordingOutputStream ros = new RecordingOutputStream(BUFFER_SIZE, - (new File(getTmpDir(), BASENAME + "Backing.txt")).getAbsolutePath()); + (new File(tempDir, BASENAME + "Backing.txt")).getAbsolutePath()); File f = writeIntRecordedFile(ros, BASENAME, WRITE_TOTAL); verifyRecording(ros, f, WRITE_TOTAL); // Do again to test that I can get a new ReplayInputStream on same @@ -114,13 +112,13 @@ public void testWriteint() * @throws IOException Failed open of backing file or opening of * input streams verifying recording. */ + @Test public void testWritebytearray() throws IOException { final String BASENAME = "testWritebytearray"; - cleanUpOldFiles(BASENAME); RecordingOutputStream ros = new RecordingOutputStream(BUFFER_SIZE, - (new File(getTmpDir(), BASENAME + "Backing.txt")).getAbsolutePath()); + (new File(tempDir, BASENAME + "Backing.txt")).getAbsolutePath()); File f = writeByteRecordedFile(ros, BASENAME, WRITE_TOTAL); verifyRecording(ros, f, WRITE_TOTAL); // Do again to test that I can get a new ReplayInputStream on same @@ -132,12 +130,12 @@ public void testWritebytearray() * Test mark and reset. * @throws IOException */ + @Test public void testMarkReset() throws IOException { final String BASENAME = "testMarkReset"; - cleanUpOldFiles(BASENAME); RecordingOutputStream ros = new RecordingOutputStream(BUFFER_SIZE, - (new File(getTmpDir(), BASENAME + "Backing.txt")).getAbsolutePath()); + (new File(tempDir, BASENAME + "Backing.txt")).getAbsolutePath()); File f = writeByteRecordedFile(ros, BASENAME, WRITE_TOTAL); verifyRecording(ros, f, WRITE_TOTAL); ReplayInputStream ris = ros.getReplayInputStream(); @@ -148,15 +146,15 @@ public void testMarkReset() throws IOException ris.read(); // Reset it. It should be back at zero. ris.reset(); - assertEquals("Reset to zero", ris.read(), 0); - assertEquals("Reset to zero char 1", ris.read(), 1); - assertEquals("Reset to zero char 2", ris.read(), 2); + assertEquals(0, ris.read(), "Reset to zero"); + assertEquals(1, ris.read(), "Reset to zero char 1"); + assertEquals(2, ris.read(), "Reset to zero char 2"); // Mark stream. Here. Next character should be '3'. ris.mark(10 /* Arbitrary value*/); ris.read(); ris.read(); ris.reset(); - assertEquals("Reset to zero char 3", ris.read(), 3); + assertEquals(3, ris.read(), "Reset to zero char 3"); } /** @@ -179,7 +177,7 @@ private File writeIntRecordedFile(RecordingOutputStream ros, String basename, int size) throws IOException { - File f = new File(getTmpDir(), basename + ".txt"); + File f = new File(tempDir, basename + ".txt"); FileOutputStream fos = new FileOutputStream(f); ros.open(fos); for (int i = 0; i < WRITE_TOTAL; i++) @@ -188,8 +186,8 @@ private File writeIntRecordedFile(RecordingOutputStream ros, } ros.close(); fos.close(); - assertEquals("Content-Length test", size, - ros.getResponseContentLength()); + assertEquals(size, ros.getResponseContentLength(), + "Content-Length test"); return f; } @@ -213,7 +211,7 @@ private File writeByteRecordedFile(RecordingOutputStream ros, String basename, int size) throws IOException { - File f = new File(getTmpDir(), basename + ".txt"); + File f = new File(tempDir, basename + ".txt"); FileOutputStream fos = new FileOutputStream(f); ros.open(fos); byte [] b = new byte[size]; @@ -224,8 +222,8 @@ private File writeByteRecordedFile(RecordingOutputStream ros, ros.write(b); ros.close(); fos.close(); - assertEquals("Content-Length test", size, - ros.getResponseContentLength()); + assertEquals(size, ros.getResponseContentLength(), + "Content-Length test"); return f; } @@ -243,28 +241,28 @@ private File writeByteRecordedFile(RecordingOutputStream ros, private void verifyRecording(RecordingOutputStream ros, File f, int size) throws IOException { - assertEquals("Recorded file size.", size, f.length()); + assertEquals(size, f.length(), "Recorded file size."); FileInputStream fis = new FileInputStream(f); - assertNotNull("FileInputStream not null", fis); + assertNotNull(fis, "FileInputStream not null"); ReplayInputStream ris = ros.getReplayInputStream(); - assertNotNull("ReplayInputStream not null", ris); + assertNotNull(ris, "ReplayInputStream not null"); for (int i = 0; i < size; i++) { - assertEquals("ReplayInputStream content verification", i, - ris.read()); - assertEquals("Recorded file content verification", i, - fis.read()); + assertEquals(i, ris.read(), + "ReplayInputStream content verification"); + assertEquals(i, fis.read(), + "Recorded file content verification"); } - assertEquals("ReplayInputStream at EOF", -1, ris.read()); + assertEquals(-1, ris.read(), "ReplayInputStream at EOF"); fis.close(); ris.close(); } + @Test public void testMessageBodyBegin() throws IOException { final String BASENAME = "testMessageBodyBegin"; - cleanUpOldFiles(BASENAME); RecordingOutputStream ros = new RecordingOutputStream(BUFFER_SIZE, - (new File(getTmpDir(), BASENAME + "Backing.txt")).getAbsolutePath()); + (new File(tempDir, BASENAME + "Backing.txt")).getAbsolutePath()); ros.setSha1Digest(); ros.open(new ByteArrayOutputStream()); diff --git a/src/test/java/org/archive/io/ReplayCharSequenceTest.java b/src/test/java/org/archive/io/ReplayCharSequenceTest.java index 9208594a..3234259c 100644 --- a/src/test/java/org/archive/io/ReplayCharSequenceTest.java +++ b/src/test/java/org/archive/io/ReplayCharSequenceTest.java @@ -19,17 +19,24 @@ package org.archive.io; +import java.io.File; import java.io.IOException; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.text.NumberFormat; import java.util.Date; import java.util.Random; import java.util.logging.Logger; import org.archive.util.FileUtils; -import org.archive.util.TmpDirTestCase; import com.google.common.base.Charsets; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.*; /** * Test ReplayCharSequences. @@ -37,8 +44,7 @@ * @author stack, gojomo * @version $Revision$, $Date$ */ -public class ReplayCharSequenceTest extends TmpDirTestCase -{ +public class ReplayCharSequenceTest { /** * Logger. */ @@ -56,16 +62,17 @@ public class ReplayCharSequenceTest extends TmpDirTestCase */ private byte [] regularBuffer = null; - /* - * @see TestCase#setUp() - */ + @TempDir + File tempDir; + + @BeforeEach protected void setUp() throws Exception { - super.setUp(); this.regularBuffer = fillBufferWithRegularContent(new byte [BUFFER_SIZE]); } - + + @Test public void testShiftjis() throws IOException { // Here's the bytes for the JIS encoding of the Japanese form of Nihongo @@ -86,19 +93,18 @@ public void testShiftjis() throws IOException { // Now check that start of the rcs comes back in as nihongo string. String rcsStr = rcs.subSequence(0, nihongo.length()).toString(); - assertTrue("Nihongo " + nihongo + " does not equal converted string" + - " from rcs " + rcsStr, - nihongo.equals(rcsStr)); + assertEquals(nihongo, rcsStr, "Nihongo " + nihongo + " does not equal converted string" + + " from rcs " + rcsStr); // And assert next string is also properly nihongo. if (rcs.length() >= (nihongo.length() * 2)) { rcsStr = rcs.subSequence(nihongo.length(), nihongo.length() + nihongo.length()).toString(); - assertTrue("Nihongo " + nihongo + " does not equal converted " + - " string from rcs (2nd time)" + rcsStr, - nihongo.equals(rcsStr)); + assertEquals(nihongo, rcsStr, "Nihongo " + nihongo + " does not equal converted " + + " string from rcs (2nd time)" + rcsStr); } } + @Test public void testGetReplayCharSequenceByteZeroOffset() throws IOException { RecordingOutputStream ros = writeTestStream( @@ -120,7 +126,7 @@ private ReplayCharSequence getReplayCharSequence(RecordingOutputStream ros, Char ros.getBufferLength()/2, ros.backingFilename, charset); } - + @Test public void testGetReplayCharSequenceMultiByteZeroOffset() throws IOException { @@ -133,7 +139,8 @@ public void testGetReplayCharSequenceMultiByteZeroOffset() accessingCharacters(rcs); } } - + + @Test public void testReplayCharSequenceByteToString() throws IOException { String fileContent = "Some file content"; byte [] buffer = fileContent.getBytes(); @@ -142,7 +149,7 @@ public void testReplayCharSequenceByteToString() throws IOException { "testReplayCharSequenceByteToString.txt",0); ReplayCharSequence rcs = getReplayCharSequence(ros); String result = rcs.toString(); - assertEquals("Strings don't match",result,fileContent); + assertEquals(fileContent, result,"Strings don't match"); } private String toHexString(String str) @@ -160,7 +167,8 @@ private String toHexString(String str) else return "null"; } - + + @Test public void testSingleByteEncodings() throws IOException { byte[] bytes = { (byte) 0x61, (byte) 0x62, (byte) 0x63, (byte) 0x64, @@ -175,7 +183,7 @@ public void testSingleByteEncodings() throws IOException { String result = rcs.toString(); logger.fine("latin1[0] " + toHexString(latin1String)); logger.fine("latin1[1] " + toHexString(result)); - assertEquals("latin1 strings don't match", result, latin1String); + assertEquals(result, latin1String, "latin1 strings don't match"); String w1252String = new String(bytes, "windows-1252"); ros = writeTestStream( @@ -184,18 +192,19 @@ public void testSingleByteEncodings() throws IOException { result = rcs.toString(); logger.fine("windows-1252[0] " + toHexString(w1252String)); logger.fine("windows-1252[1] " + toHexString(result)); - assertEquals("windows-1252 strings don't match", result, w1252String); + assertEquals(result, w1252String, "windows-1252 strings don't match"); - String asciiString = new String(bytes, "ascii"); + String asciiString = new String(bytes, StandardCharsets.US_ASCII); ros = writeTestStream( bytes, 1, "testSingleByteEncodings-ascii.txt", 0); - rcs = getReplayCharSequence(ros,Charset.forName("ascii")); + rcs = getReplayCharSequence(ros, StandardCharsets.US_ASCII); result = rcs.toString(); logger.fine("ascii[0] " + toHexString(asciiString)); logger.fine("ascii[1] " + toHexString(result)); - assertEquals("ascii strings don't match", result, asciiString); + assertEquals(result, asciiString, "ascii strings don't match"); } - + + @Test public void testReplayCharSequenceByteToStringOverflow() throws IOException { String fileContent = "Some file content. "; // ascii byte [] buffer = fileContent.getBytes(); @@ -212,15 +221,16 @@ public void testReplayCharSequenceByteToStringOverflow() throws IOException { ReplayCharSequence rcs1252 = getReplayCharSequence(ros,Charset.forName("windows-1252")); String result = rcsUtf8.toString(); - assertEquals("Strings don't match", expectedContent, result); + assertEquals(expectedContent, result, "Strings don't match"); result = rcs1252.toString(); - assertEquals("Strings don't match", expectedContent, result); + assertEquals(expectedContent, result, "Strings don't match"); } - + + @Test public void testReplayCharSequenceByteToStringMulti() throws IOException { String fileContent = "Some file content"; - byte [] buffer = fileContent.getBytes("UTF-8"); + byte [] buffer = fileContent.getBytes(StandardCharsets.UTF_8); final int MULTIPLICAND = 10; StringBuilder sb = new StringBuilder(MULTIPLICAND * fileContent.length()); @@ -232,15 +242,17 @@ public void testReplayCharSequenceByteToStringMulti() throws IOException { buffer,1, "testReplayCharSequenceByteToStringMulti.txt",MULTIPLICAND-1); for (int i = 0; i < 3; i++) { - ReplayCharSequence rcs = getReplayCharSequence(ros,Charsets.UTF_8); + ReplayCharSequence rcs = getReplayCharSequence(ros,StandardCharsets.UTF_8); String result = rcs.toString(); - assertEquals("Strings don't match", result, expectedResult); + assertEquals(result, expectedResult, "Strings don't match"); rcs.close(); System.gc(); System.runFinalization(); } } - + + @Test + @Disabled public void xestHugeReplayCharSequence() throws IOException { String fileContent = "01234567890123456789"; String characterEncoding = "ascii"; @@ -255,14 +267,13 @@ public void xestHugeReplayCharSequence() throws IOException { ReplayCharSequence rcs = getReplayCharSequence(ros,Charset.forName(characterEncoding)); if (reps * fileContent.length() > (long) Integer.MAX_VALUE) { - assertTrue("ReplayCharSequence has wrong length (length()=" - + rcs.length() + ") (should be " + Integer.MAX_VALUE + ")", - rcs.length() == Integer.MAX_VALUE); + assertEquals(Integer.MAX_VALUE, rcs.length(), "ReplayCharSequence has wrong length (length()=" + + rcs.length() + ") (should be " + Integer.MAX_VALUE + ")"); } else { - assertEquals("ReplayCharSequence has wrong length (length()=" + assertEquals(rcs.length(), reps * (long) fileContent.length(), + "ReplayCharSequence has wrong length (length()=" + rcs.length() + ") (should be " - + (reps * fileContent.length()) + ")", (long) rcs.length(), - reps * (long) fileContent.length()); + + (reps * fileContent.length()) + ")"); } // boundary cases or something @@ -270,10 +281,9 @@ public void xestHugeReplayCharSequence() throws IOException { rcs.length() - 1, rcs.length() / 4 }) { // logger.info("testing char at index=" + // NumberFormat.getInstance().format(index)); - assertEquals("Characters don't match (index=" - + NumberFormat.getInstance().format(index) + ")", - fileContent.charAt(index % fileContent.length()), rcs - .charAt(index)); + assertEquals(fileContent.charAt(index % fileContent.length()), + rcs.charAt(index), "Characters don't match (index=" + + NumberFormat.getInstance().format(index) + ")"); } // check that out of bounds indices throw exception @@ -295,10 +305,9 @@ public void xestHugeReplayCharSequence() throws IOException { int index = rand.nextInt(rcs.length()); // logger.info(i + ". testing char at index=" + // NumberFormat.getInstance().format(index)); - assertEquals("Characters don't match (index=" - + NumberFormat.getInstance().format(index) + ")", - fileContent.charAt(index % fileContent.length()), rcs - .charAt(index)); + assertEquals(fileContent.charAt(index % fileContent.length()), + rcs.charAt(index), "Characters don't match (index=" + + NumberFormat.getInstance().format(index) + ")"); } } @@ -338,8 +347,8 @@ private void accessingCharacters(CharSequence rcs) { */ private void checkCharacter(CharSequence rcs, int i) { int c = rcs.charAt(i); - assertTrue("Character " + Integer.toString(c) + " at offset " + i + - " unexpected.", (c % SEQUENCE_LENGTH) == (i % SEQUENCE_LENGTH)); + assertEquals((c % SEQUENCE_LENGTH), (i % SEQUENCE_LENGTH), "Character " + Integer.toString(c) + " at offset " + i + + " unexpected."); } /** @@ -349,7 +358,7 @@ private void checkCharacter(CharSequence rcs, int i) { */ private RecordingOutputStream writeTestStream(byte[] content, int memReps, String baseName, long fileReps) throws IOException { - String backingFilename = FileUtils.maybeRelative(getTmpDir(),baseName).getAbsolutePath(); + String backingFilename = FileUtils.maybeRelative(tempDir,baseName).getAbsolutePath(); RecordingOutputStream ros = new RecordingOutputStream( content.length * memReps, backingFilename); @@ -383,9 +392,4 @@ private RecordingOutputStream writeTestStream(byte[] content, } return buffer; } - - public void testCheckParameters() - { - // TODO. - } } diff --git a/src/test/java/org/archive/io/RepositionableInputStreamTest.java b/src/test/java/org/archive/io/RepositionableInputStreamTest.java index 1c7cc74c..228c9042 100644 --- a/src/test/java/org/archive/io/RepositionableInputStreamTest.java +++ b/src/test/java/org/archive/io/RepositionableInputStreamTest.java @@ -23,23 +23,29 @@ import java.io.FileOutputStream; import java.io.PrintWriter; -import org.archive.util.TmpDirTestCase; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; -public class RepositionableInputStreamTest extends TmpDirTestCase { +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class RepositionableInputStreamTest { private File testFile; private static final String LINE = "0123456789abcdefghijklmnopqrstuv"; + @TempDir + File tempDir; + + @BeforeEach protected void setUp() throws Exception { - super.setUp(); - this.testFile = new File(getTmpDir(), this.getClass().getName()); + this.testFile = new File(tempDir, this.getClass().getName()); PrintWriter pw = new PrintWriter(new FileOutputStream(testFile)); for (int i = 0; i < 100; i++) { pw.print(LINE); } pw.close(); } - protected void tearDown() throws Exception { - super.tearDown(); - } + + @Test public void testname() throws Exception { // Make buffer awkward size so we run into buffers spanning issues. RepositionableInputStream ris = diff --git a/src/test/java/org/archive/io/arc/ARCReaderFactoryTest.java b/src/test/java/org/archive/io/arc/ARCReaderFactoryTest.java index 0721f795..25d5218e 100644 --- a/src/test/java/org/archive/io/arc/ARCReaderFactoryTest.java +++ b/src/test/java/org/archive/io/arc/ARCReaderFactoryTest.java @@ -2,14 +2,16 @@ import java.io.File; import java.io.FileInputStream; -import java.io.FileNotFoundException; import java.io.InputStream; import java.io.RandomAccessFile; import org.archive.io.ArchiveReader; import org.archive.io.ArchiveRecord; -import junit.framework.TestCase; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * @@ -18,7 +20,7 @@ * @author csr@statsbiblioteket.dk (Colin Rosenthal) * */ -public class ARCReaderFactoryTest extends TestCase { +public class ARCReaderFactoryTest { private File testfile1 = new File("src/test/resources/org/archive/format/arc/IAH-20080430204825-00000-blackbook-truncated.arc"); @@ -27,6 +29,7 @@ public class ARCReaderFactoryTest extends TestCase { * https://github.com/iipc/openwayback/issues/101 * @throws Exception */ + @Test public void testGetResource() throws Exception { this.offsetResourceTest(testfile1, 1515, "http://www.archive.org/robots.txt" ); this.offsetResourceTest(testfile1, 36420, "http://www.archive.org/services/collection-rss.php" ); @@ -43,11 +46,11 @@ private void offsetResourceTest( File testfile, long offset, String uri ) throws ArchiveRecord record = reader.get(); final String url = record.getHeader().getUrl(); - assertEquals("URL of record is not as expected.", uri, url); + assertEquals(uri, url, "URL of record is not as expected."); final long position = record.getPosition(); final long recordLength = record.getHeader().getLength(); - assertTrue("Position " + position + " is after end of record " + recordLength, position <= recordLength); + assertTrue(position <= recordLength, "Position " + position + " is after end of record " + recordLength); // Clean up: if( raf != null ) diff --git a/src/test/java/org/archive/io/arc/ARCWriterPoolTest.java b/src/test/java/org/archive/io/arc/ARCWriterPoolTest.java index f0be6506..07548b4c 100644 --- a/src/test/java/org/archive/io/arc/ARCWriterPoolTest.java +++ b/src/test/java/org/archive/io/arc/ARCWriterPoolTest.java @@ -21,26 +21,31 @@ import java.io.ByteArrayOutputStream; import java.io.File; +import java.nio.file.Path; import java.util.Arrays; import org.archive.io.WriterPool; import org.archive.io.WriterPoolMember; import org.archive.io.WriterPoolSettings; -import org.archive.util.TmpDirTestCase; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.assertEquals; /** * Test ARCWriterPool */ @SuppressWarnings("deprecation") -public class ARCWriterPoolTest extends TmpDirTestCase { - private static final String PREFIX = "TEST"; - +public class ARCWriterPoolTest { + @TempDir + Path tempDir; + + @Test public void testARCWriterPool() throws Exception { final int MAX_ACTIVE = 3; final int MAX_WAIT_MILLISECONDS = 100; - cleanUpOldFiles(PREFIX); WriterPool pool = new ARCWriterPool(getSettings(true), MAX_ACTIVE, MAX_WAIT_MILLISECONDS); WriterPoolMember [] writers = new WriterPoolMember[MAX_ACTIVE]; @@ -49,7 +54,7 @@ public void testARCWriterPool() baos.write(CONTENT.getBytes()); for (int i = 0; i < MAX_ACTIVE; i++) { writers[i] = pool.borrowFile(); - assertEquals("Number active", i + 1, pool.getNumActive()); + assertEquals(i + 1, pool.getNumActive(), "Number active"); ((ARCWriter)writers[i]).write("http://one.two.three", "no-type", "0.0.0.0", 1234567890, CONTENT.length(), baos); } @@ -60,17 +65,17 @@ public void testARCWriterPool() for (int i = (MAX_ACTIVE - 1); i >= 0; i--) { pool.returnFile(writers[i]); - assertEquals("Number active", i, pool.getNumActive()); - assertEquals("Number idle", MAX_ACTIVE - pool.getNumActive(), - pool.getNumIdle()); + assertEquals(i, pool.getNumActive(), "Number active"); + assertEquals(MAX_ACTIVE - pool.getNumActive(), pool.getNumIdle(), + "Number idle"); } pool.close(); } - + + @Test public void testInvalidate() throws Exception { final int MAX_ACTIVE = 3; final int MAX_WAIT_MILLISECONDS = 100; - cleanUpOldFiles(PREFIX); WriterPool pool = new ARCWriterPool(getSettings(true), MAX_ACTIVE, MAX_WAIT_MILLISECONDS); WriterPoolMember [] writers = new WriterPoolMember[MAX_ACTIVE]; @@ -79,7 +84,7 @@ public void testInvalidate() throws Exception { baos.write(CONTENT.getBytes()); for (int i = 0; i < MAX_ACTIVE; i++) { writers[i] = pool.borrowFile(); - assertEquals("Number active", i + 1, pool.getNumActive()); + assertEquals(i + 1, pool.getNumActive(), "Number active"); ((ARCWriter)writers[i]).write("http://one.two.three", "no-type", "0.0.0.0", 1234567890, CONTENT.length(), baos); } @@ -96,23 +101,23 @@ public void testInvalidate() throws Exception { for (int i = 0; i < MAX_ACTIVE; i++) { writers[i] = pool.borrowFile(); - assertEquals("Number active", i + 1, pool.getNumActive()); + assertEquals(i + 1, pool.getNumActive(), "Number active"); ((ARCWriter)writers[i]).write("http://one.two.three", "no-type", "0.0.0.0", 1234567890, CONTENT.length(), baos); } for (int i = (MAX_ACTIVE - 1); i >= 0; i--) { pool.returnFile(writers[i]); - assertEquals("Number active", i, pool.getNumActive()); - assertEquals("Number idle", MAX_ACTIVE - pool.getNumActive(), - pool.getNumIdle()); + assertEquals(i, pool.getNumActive(), "Number active"); + assertEquals(MAX_ACTIVE - pool.getNumActive(), pool.getNumIdle(), + "Number idle"); } pool.close(); } private WriterPoolSettings getSettings(final boolean isCompressed) { - File [] files = {getTmpDir()}; + File [] files = {tempDir.toFile()}; return new WriterPoolSettingsData( - PREFIX, + "TEST", "${prefix}-${timestamp17}-${serialno}-${heritrix.hostname}", ARCConstants.DEFAULT_MAX_ARC_FILE_SIZE, isCompressed, diff --git a/src/test/java/org/archive/io/arc/ARCWriterTest.java b/src/test/java/org/archive/io/arc/ARCWriterTest.java index f6e2bf6a..84539391 100644 --- a/src/test/java/org/archive/io/arc/ARCWriterTest.java +++ b/src/test/java/org/archive/io/arc/ARCWriterTest.java @@ -42,9 +42,12 @@ import org.archive.io.WriterPoolMember; import org.archive.io.WriterPoolSettings; import org.archive.util.ArchiveUtils; -import org.archive.util.TmpDirTestCase; import com.google.common.io.Closeables; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.*; /** @@ -55,8 +58,7 @@ * * @author stack */ -public class ARCWriterTest -extends TmpDirTestCase implements ARCConstants { +public class ARCWriterTest implements ARCConstants { /** * Utility class for writing bad ARCs (with trailing junk) */ @@ -90,20 +92,13 @@ public void setEndJunk(byte[] b) throws IOException { private static final AtomicInteger SERIAL_NO = new AtomicInteger(); - /* - * @see TestCase#setUp() - */ - protected void setUp() throws Exception { - super.setUp(); - } + @TempDir + File tempDir; - /* - * @see TestCase#tearDown() - */ - protected void tearDown() throws Exception { - super.tearDown(); + protected static String getContent(int index) { + return getContent(Integer.toString(index)); } - + protected static String getContent() { return getContent(null); } @@ -140,8 +135,7 @@ protected int writeRandomHTTPRecord(ARCWriter arcWriter, int index) private File writeRecords(String baseName, boolean compress, long maxSize, int recordCount) throws IOException { - cleanUpOldFiles(baseName); - File [] files = {getTmpDir()}; + File [] files = {tempDir}; ARCWriter arcWriter = new ARCWriter( SERIAL_NO, @@ -157,9 +151,9 @@ private File writeRecords(String baseName, boolean compress, writeRandomHTTPRecord(arcWriter, i); } arcWriter.close(); - assertTrue("Doesn't exist: " + - arcWriter.getFile().getAbsolutePath(), - arcWriter.getFile().exists()); + assertTrue(arcWriter.getFile().exists(), + "Doesn't exist: " + + arcWriter.getFile().getAbsolutePath()); return arcWriter.getFile(); } @@ -183,34 +177,38 @@ private void validate(File arcFile, int recordCount) ARCRecordMetaData meta = (ARCRecordMetaData)metaDatas.get(i); ArchiveRecord r = reader.get(meta.getOffset()); String mimeType = r.getHeader().getMimetype(); - assertTrue("Record is bogus", - mimeType != null && mimeType.length() > 0); + assertTrue(mimeType != null && mimeType.length() > 0, + "Record is bogus"); reader.close(); } - assertEquals("Metadata count not as expected",recordCount, metaDatas.size()); + assertEquals(recordCount,metaDatas.size(), "Metadata count not as expected"); for (Iterator i = metaDatas.iterator(); i.hasNext();) { ARCRecordMetaData r = (ARCRecordMetaData)i.next(); - assertTrue("Record is empty", r.getLength() > 0); + assertTrue(r.getLength() > 0, "Record is empty"); } } + @Test public void testCheckARCFileSize() throws IOException { runCheckARCFileSizeTest("checkARCFileSize", false); } + @Test public void testCheckARCFileSizeCompressed() throws IOException { runCheckARCFileSizeTest("checkARCFileSize", true); } + @Test public void testWriteRecord() throws IOException { final int recordCount = 2; File arcFile = writeRecords("writeRecord", false, DEFAULT_MAX_ARC_FILE_SIZE, recordCount); validate(arcFile, recordCount + 1); // Header record. } - + + @Test public void testRandomAccess() throws IOException { final int recordCount = 3; File arcFile = writeRecords("writeRecord", true, @@ -252,6 +250,7 @@ public void testRandomAccess() throws IOException { assertEquals(totalRecords - 1, count); } + @Test public void testWriteRecordCompressed() throws IOException { final int recordCount = 2; File arcFile = writeRecords("writeRecordCompressed", true, @@ -291,7 +290,7 @@ private void runCheckARCFileSizeTest(String baseName, boolean compress) } protected CorruptibleARCWriter createARCWriter(String name, boolean compress) { - File [] files = {getTmpDir()}; + File [] files = {tempDir}; return new CorruptibleARCWriter( SERIAL_NO, new WriterPoolSettingsData( @@ -326,8 +325,8 @@ protected int iterateRecords(ARCReader r) ARCRecord rec = (ARCRecord)i.next(); rec.close(); if (count != 0) { - assertTrue("Unexpected URL " + rec.getMetaData().getUrl(), - rec.getMetaData().getUrl().startsWith(SOME_URL)); + assertTrue(rec.getMetaData().getUrl().startsWith(SOME_URL), + "Unexpected URL " + rec.getMetaData().getUrl()); } count++; } @@ -343,7 +342,8 @@ protected CorruptibleARCWriter createArcWithOneRecord(String name, content.length(), getBais(content)); return writer; } - + + @Test public void testSpaceInURL() { String eMessage = null; try { @@ -351,10 +351,11 @@ public void testSpaceInURL() { } catch (IOException e) { eMessage = e.getMessage(); } - assertTrue("Didn't get expected exception: " + eMessage, - eMessage.startsWith("Metadata line doesn't match")); + assertTrue(eMessage.startsWith("Metadata line doesn't match"), + "Didn't get expected exception: " + eMessage); } + @Test public void testTabInURL() { String eMessage = null; try { @@ -362,8 +363,8 @@ public void testTabInURL() { } catch (IOException e) { eMessage = e.getMessage(); } - assertTrue("Didn't get expected exception: " + eMessage, - eMessage.startsWith("Metadata line doesn't match")); + assertTrue(eMessage.startsWith("Metadata line doesn't match"), + "Didn't get expected exception: " + eMessage); } protected void holeyUrl(String name, boolean compress, String urlInsert) @@ -385,11 +386,13 @@ protected void holeyUrl(String name, boolean compress, String urlInsert) // public void testLengthTooShort() throws IOException { // lengthTooShort("testLengthTooShort-" + PREFIX, false); // } - + + @Test public void testLengthTooShortCompressed() throws IOException { lengthTooShort("testLengthTooShortCompressed", true, false); } - + + @Test public void testLengthTooShortCompressedStrict() throws IOException { String eMessage = null; @@ -399,8 +402,8 @@ public void testLengthTooShortCompressedStrict() } catch (RuntimeException e) { eMessage = e.getMessage(); } - assertTrue("Didn't get expected exception: " + eMessage, - eMessage.startsWith("java.io.IOException: Record STARTING at")); + assertTrue(eMessage.startsWith("java.io.IOException: Record STARTING at"), + "Didn't get expected exception: " + eMessage); } protected void lengthTooShort(String name, boolean compress, boolean strict) @@ -430,13 +433,13 @@ protected void lengthTooShort(String name, boolean compress, boolean strict) r = ARCReaderFactory.get(writer.getFile()); r.setStrict(strict); int count = iterateRecords(r); - assertTrue("Count wrong " + count, count == 4); + assertTrue(count == 4, "Count wrong " + count); // Make sure we get the warning string which complains about the // trailing bytes. String err = os.toString(); - assertTrue("No message " + err, err.startsWith("WARNING") && - (err.indexOf("Record STARTING at") > 0)); + assertTrue(err.startsWith("WARNING") && + (err.indexOf("Record STARTING at") > 0), "No message " + err); r.close(); } finally { Closeables.close(r, true); @@ -451,13 +454,15 @@ protected void lengthTooShort(String name, boolean compress, boolean strict) // lengthTooLong("testLengthTooLongCompressed-" + PREFIX, // false, false); // } - + + @Test public void testLengthTooLongCompressed() throws IOException { lengthTooLong("testLengthTooLongCompressed", true, false); } - + + @Test public void testLengthTooLongCompressedStrict() { String eMessage = null; try { @@ -466,8 +471,8 @@ public void testLengthTooLongCompressedStrict() { } catch (IOException e) { eMessage = e.getMessage(); } - assertTrue("Didn't get expected exception: " + eMessage, - eMessage.startsWith("Premature EOF before end-of-record")); + assertTrue(eMessage.startsWith("Premature EOF before end-of-record"), + "Didn't get expected exception: " + eMessage); } protected void lengthTooLong(String name, boolean compress, @@ -493,19 +498,20 @@ protected void lengthTooLong(String name, boolean compress, r = ARCReaderFactory.get(writer.getFile()); r.setStrict(strict); int count = iterateRecords(r); - assertTrue("Count wrong " + count, count == 4); + assertTrue(count == 4, "Count wrong " + count); // Make sure we get the warning string which complains about the // trailing bytes. String err = os.toString(); - assertTrue("No message " + err, - err.startsWith("WARNING Premature EOF before end-of-record")); + assertTrue(err.startsWith("WARNING Premature EOF before end-of-record"), + "No message " + err); } finally { Closeables.close(r, true); System.setErr(origErr); } } - + + @Test public void testGapError() throws IOException { ARCWriter writer = createArcWithOneRecord("testGapError", true); String content = getContent(); @@ -527,9 +533,9 @@ public long remaining() { IOUtils.closeQuietly(ris); } writer.close(); - assertTrue("No gap when should be", - message != null && - message.indexOf("Gap between expected and actual") >= 0); + assertTrue(message != null && + message.indexOf("Gap between expected and actual") >= 0, + "No gap when should be"); } /** @@ -570,8 +576,8 @@ public static File createARCFile(File arcdir, boolean compress) // writer.close(); // logger.info("Finished speed write test."); // } - - + + @Test public void testValidateMetaLine() throws Exception { final String line = "http://www.aandw.net/images/walden2.png " + "128.197.34.86 20060111174224 image/png 2160"; @@ -584,7 +590,8 @@ public void testValidateMetaLine() throws Exception { w.close(); } } - + + @Test public void testArcRecordOffsetReads() throws Exception { ARCReader r = getSingleRecordReader("testArcRecordInBufferStream"); ARCRecord ar = getSingleRecord(r); @@ -603,6 +610,7 @@ public void testArcRecordOffsetReads() throws Exception { } // available should always be >= 0; extra read()s should all give EOF + @Test public void testArchiveRecordAvailableConsistent() throws Exception { // first test reading byte-at-a-time via no-param read() ARCReader r = getSingleRecordReader("testArchiveRecordAvailableConsistent"); @@ -613,13 +621,14 @@ public void testArchiveRecordAvailableConsistent() throws Exception { } // consecutive reads after EOR should always give -1, still show zero available() for (int i=0; i<5; i++) { - assertTrue("available negative:"+record.available(), record.available()>=0); + assertTrue(record.available()>=0, "available negative:"+record.available()); assertEquals(-1, record.read()); } r.close(); } // should always give -1 on repeated reads past EOR + @Test public void testArchiveRecordEORConsistent() throws Exception { ARCReader r = getSingleRecordReader("testArchiveRecordEORConsistent"); ARCRecord record = getSingleRecord(r); @@ -633,6 +642,7 @@ public void testArchiveRecordEORConsistent() throws Exception { // should not throw premature EOF when wrapped with BufferedInputStream // [HER-1450] showed this was the case using Apache Tika + @Test public void testArchiveRecordMarkSupport() throws Exception { ARCReader r = getSingleRecordReader("testArchiveRecordMarkSupport"); ARCRecord record = getSingleRecord(r); @@ -657,6 +667,7 @@ public void testArchiveRecordMarkSupport() throws Exception { * * @throws IOException */ + @Test public void testReadIterator() throws IOException { final int recordCount = 3; File arcFile = writeRecords("writeRecord", true, diff --git a/src/test/java/org/archive/io/warc/WARCReaderFactoryTest.java b/src/test/java/org/archive/io/warc/WARCReaderFactoryTest.java index 25028797..c6617559 100644 --- a/src/test/java/org/archive/io/warc/WARCReaderFactoryTest.java +++ b/src/test/java/org/archive/io/warc/WARCReaderFactoryTest.java @@ -8,9 +8,11 @@ import org.archive.io.ArchiveReader; import org.archive.io.ArchiveRecord; -import junit.framework.TestCase; +import org.junit.jupiter.api.Test; -public class WARCReaderFactoryTest extends TestCase { +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class WARCReaderFactoryTest { // Test files: String[] files = new String[] { @@ -18,6 +20,7 @@ public class WARCReaderFactoryTest extends TestCase { "src/test/resources/org/archive/format/warc/IAH-urls-wget.warc" }; + @Test public void testGetStringInputstreamBoolean() throws IOException { // Check the test files can be opened: for( String file : files ) { diff --git a/src/test/java/org/archive/io/warc/WARCWriterTest.java b/src/test/java/org/archive/io/warc/WARCWriterTest.java index 35c68714..1039119e 100644 --- a/src/test/java/org/archive/io/warc/WARCWriterTest.java +++ b/src/test/java/org/archive/io/warc/WARCWriterTest.java @@ -38,16 +38,18 @@ import org.archive.uid.RecordIDGenerator; import org.archive.uid.UUIDGenerator; import org.archive.util.ArchiveUtils; -import org.archive.util.TmpDirTestCase; import org.archive.util.anvl.ANVLRecord; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.*; /** * Test Writer and Reader. * @author stack * @version $Date: 2006-08-29 19:35:48 -0700 (Tue, 29 Aug 2006) $ $Version$ */ -public class WARCWriterTest -extends TmpDirTestCase implements WARCConstants { +public class WARCWriterTest implements WARCConstants { private static final AtomicInteger SERIAL_NO = new AtomicInteger(); @@ -59,8 +61,12 @@ public class WARCWriterTest private static final String SUFFIX = "JUNIT"; private static final String SOME_URL = "http://www.archive.org/test/"; - + + @TempDir + File tempDir; + @SuppressWarnings("unchecked") + @Test public void testCheckHeaderLineValue() throws Exception { WARCWriter writer = new WARCWriter( SERIAL_NO, @@ -85,21 +91,23 @@ public void testCheckHeaderLineValue() throws Exception { } @SuppressWarnings("unchecked") + @Test public void testMimetypes() throws IOException { WARCWriter writer = new WARCWriter(SERIAL_NO, new WARCWriterPoolSettingsData( "m","testM",1,false,Collections.EMPTY_LIST,Collections.EMPTY_LIST,generator)); writer.checkHeaderLineMimetypeParameter("text/xml"); writer.checkHeaderLineMimetypeParameter("text/xml+rdf"); - assertEquals(writer.checkHeaderLineMimetypeParameter( - "text/plain; charset=SHIFT-JIS"), "text/plain; charset=SHIFT-JIS"); - assertEquals(writer.checkHeaderLineMimetypeParameter( - "multipart/mixed; \r\n boundary=\"simple boundary\""), - "multipart/mixed; boundary=\"simple boundary\""); + assertEquals("text/plain; charset=SHIFT-JIS", writer.checkHeaderLineMimetypeParameter( + "text/plain; charset=SHIFT-JIS")); + assertEquals("multipart/mixed; boundary=\"simple boundary\"", + writer.checkHeaderLineMimetypeParameter( + "multipart/mixed; \r\n boundary=\"simple boundary\"")); } - + + @Test public void testWriteRecord() throws IOException { - File [] files = {getTmpDir()}; + File [] files = {tempDir}; // Write uncompressed. WARCWriter writer = @@ -245,8 +253,7 @@ protected int writeRandomHTTPRecord(WARCWriter w, int index) private File writeRecords(String baseName, boolean compress, int maxSize, int recordCount) throws IOException { - cleanUpOldFiles(baseName); - File [] files = {getTmpDir()}; + File [] files = {tempDir}; WARCWriter w = new WARCWriter(SERIAL_NO, new WARCWriterPoolSettingsData( baseName + '-' + SUFFIX, "${prefix}", maxSize, compress, Arrays.asList(files), null, generator)); @@ -255,8 +262,8 @@ private File writeRecords(String baseName, boolean compress, writeRandomHTTPRecord(w, i); } w.close(); - assertTrue("Doesn't exist: " + w.getFile().getAbsolutePath(), - w.getFile().exists()); + assertTrue(w.getFile().exists(), + "Doesn't exist: " + w.getFile().getAbsolutePath()); return w.getFile(); } @@ -288,18 +295,19 @@ private void validate(File f, int recordCount) ArchiveRecordHeader h = (ArchiveRecordHeader)headers.get(i); ArchiveRecord r = reader.get(h.getOffset()); String mimeType = r.getHeader().getMimetype(); - assertTrue("Record is bogus", - mimeType != null && mimeType.length() > 0); + assertTrue(mimeType != null && mimeType.length() > 0, + "Record is bogus"); reader.close(); } - assertTrue("Metadatas not equal", headers.size() == recordCount); + assertTrue(headers.size() == recordCount, "Metadatas not equal"); for (Iterator i = headers.iterator(); i.hasNext();) { ArchiveRecordHeader r = (ArchiveRecordHeader)i.next(); - assertTrue("Record is empty", r.getLength() > 0); + assertTrue(r.getLength() > 0, "Record is empty"); } } + @Test public void testWriteRecords() throws IOException { final int recordCount = 2; File f = writeRecords("writeRecords", false, DEFAULT_MAX_WARC_FILE_SIZE, @@ -307,6 +315,7 @@ public void testWriteRecords() throws IOException { validate(f, recordCount + 1); // Header record. } + @Test public void testRandomAccess() throws IOException { final int recordCount = 3; File f = writeRecords("randomAccess", true, DEFAULT_MAX_WARC_FILE_SIZE, @@ -348,7 +357,8 @@ public void testRandomAccess() throws IOException { reader.close(); assertEquals(totalRecords - 1, count); } - + + @Test public void testWriteRecordCompressed() throws IOException { final int recordCount = 2; File arcFile = writeRecords("writeRecordCompressed", true, @@ -358,7 +368,7 @@ public void testWriteRecordCompressed() throws IOException { protected WARCWriter createWARCWriter(String name, boolean compress) { - File [] files = {getTmpDir()}; + File [] files = {tempDir}; return new WARCWriter(SERIAL_NO, new WARCWriterPoolSettingsData( name, @@ -401,8 +411,8 @@ protected int iterateRecords(WARCReader r) ArchiveRecord ar = i.next(); ar.close(); if (count != 0) { - assertTrue("Unexpected URL " + ar.getHeader().getUrl(), - ar.getHeader().getUrl().equals(SOME_URL)); + assertTrue(ar.getHeader().getUrl().equals(SOME_URL), + "Unexpected URL " + ar.getHeader().getUrl()); } count++; } @@ -418,15 +428,17 @@ protected WARCWriter createWithOneRecord(String name, content.length(), getBaos(content)); return writer; } - + + @Test public void testSpaceInURL() throws IOException { long bytesWritten = holeyUrl("testSpaceInURL", false, " "); - assertEquals("Unexpected successful writing occurred",0,bytesWritten); + assertEquals(0,bytesWritten,"Unexpected successful writing occurred"); } + @Test public void testTabInURL() throws IOException { long bytesWritten = holeyUrl("testTabInURL", false, "\t"); - assertEquals("Unexpected successful writing occurred",0,bytesWritten); + assertEquals(0,bytesWritten,"Unexpected successful writing occurred"); } protected long holeyUrl(String name, boolean compress, String urlInsert) @@ -483,7 +495,8 @@ public static File createWARCFile(File arcdir, boolean compress) // writer.close(); // logger.info("Finished speed write test."); // } - + + @Test public void testArcRecordOffsetReads() throws Exception { // Get an ARC with one record. WriterPoolMember w = diff --git a/src/test/java/org/archive/net/PublicSuffixesTest.java b/src/test/java/org/archive/net/PublicSuffixesTest.java index ca6e6408..758d7f46 100644 --- a/src/test/java/org/archive/net/PublicSuffixesTest.java +++ b/src/test/java/org/archive/net/PublicSuffixesTest.java @@ -24,9 +24,10 @@ import java.util.ArrayList; import java.util.regex.Matcher; -import junit.framework.TestCase; - import org.archive.net.PublicSuffixes.Node; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; /** * Test cases for PublicSuffixes utility. Confirm expected matches/nonmatches @@ -34,10 +35,11 @@ * * @author gojomo */ -public class PublicSuffixesTest extends TestCase { +public class PublicSuffixesTest { // test of low level implementation private final String NL = System.getProperty("line.separator"); - + + @Test public void testCompare() { Node n = new Node("hoge"); assertTrue(n.compareTo('a') > 0); @@ -75,6 +77,8 @@ protected String dump(Node alt) { PublicSuffixes.dump(alt, 0, new PrintWriter(w)); return w.toString(); } + + @Test public void testTrie1() { Node alt = new Node(null, new ArrayList()); alt.addBranch("ac,"); @@ -92,6 +96,8 @@ public void testTrie1() { " \"edu,\"" + NL + " \"\"" + NL, dump(alt)); } + + @Test public void testTrie2() { Node alt = new Node(null, new ArrayList()); alt.addBranch("ac,"); @@ -101,6 +107,7 @@ public void testTrie2() { " \"*,\"" + NL, dump(alt)); } + @Test public void testTrie3() { Node alt = new Node(null, new ArrayList()); alt.addBranch("ac,"); @@ -119,6 +126,7 @@ public void testTrie3() { Matcher m = PublicSuffixes.getTopmostAssignedSurtPrefixPattern() .matcher(""); + @Test public void testBasics() { matchPrefix("com,example,www,", "com,example,"); matchPrefix("com,example,", "com,example,"); @@ -137,27 +145,32 @@ public void testBasics() { matchPrefix("jp,yokohama,public,assigned,", "jp,yokohama,public,assigned,"); } + @Test public void testDomainWithDash() { matchPrefix("de,bad-site,www", "de,bad-site,"); } - + + @Test public void testDomainWithNumbers() { matchPrefix("de,archive4u,www", "de,archive4u,"); } - + + @Test public void testIPV4() { - assertEquals("unexpected reduction", - "1.2.3.4", - PublicSuffixes.reduceSurtToAssignmentLevel("1.2.3.4")); + assertEquals("1.2.3.4", + PublicSuffixes.reduceSurtToAssignmentLevel("1.2.3.4"), + "unexpected reduction"); } - + + @Test public void testIPV6() { - assertEquals("unexpected reduction", - "[2001:0db8:85a3:08d3:1319:8a2e:0370:7344]", + assertEquals("[2001:0db8:85a3:08d3:1319:8a2e:0370:7344]", PublicSuffixes.reduceSurtToAssignmentLevel( - "[2001:0db8:85a3:08d3:1319:8a2e:0370:7344]")); + "[2001:0db8:85a3:08d3:1319:8a2e:0370:7344]"), + "unexpected reduction"); } - + + @Test public void testExceptions() { matchPrefix("uk,bl,www,", "uk,bl,"); matchPrefix("uk,bl,", "uk,bl,"); @@ -165,6 +178,7 @@ public void testExceptions() { matchPrefix("jp,tokyo,city,", "jp,tokyo,city,"); } + @Test public void testFakeTLD() { // we assume any new/unknonwn TLD should be assumed as 2-level; // this is preferable for our grouping purpose but might not be @@ -172,22 +186,23 @@ public void testFakeTLD() { matchPrefix("zzz,example,www,", "zzz,example,"); } + @Test public void testUnsegmentedHostname() { m.reset("example"); - assertFalse("unexpected match found in 'example'", m.find()); + assertFalse(m.find(), "unexpected match found in 'example'"); } + @Test public void testTopmostAssignedCaching() { - assertSame("topmostAssignedSurtPrefixPattern not cached",PublicSuffixes.getTopmostAssignedSurtPrefixPattern(),PublicSuffixes.getTopmostAssignedSurtPrefixPattern()); - assertSame("topmostAssignedSurtPrefixRegex not cached",PublicSuffixes.getTopmostAssignedSurtPrefixRegex(),PublicSuffixes.getTopmostAssignedSurtPrefixRegex()); + assertSame(PublicSuffixes.getTopmostAssignedSurtPrefixPattern(),PublicSuffixes.getTopmostAssignedSurtPrefixPattern(),"topmostAssignedSurtPrefixPattern not cached"); + assertSame(PublicSuffixes.getTopmostAssignedSurtPrefixRegex(),PublicSuffixes.getTopmostAssignedSurtPrefixRegex(),"topmostAssignedSurtPrefixRegex not cached"); } // TODO: test UTF domains? protected void matchPrefix(String surtDomain, String expectedAssignedPrefix) { m.reset(surtDomain); - assertTrue("expected match not found in '" + surtDomain, m.find()); - assertEquals("expected match not found", expectedAssignedPrefix, m - .group()); + assertTrue(m.find(), "expected match not found in '" + surtDomain); + assertEquals(expectedAssignedPrefix, m.group(), "expected match not found"); } } diff --git a/src/test/java/org/archive/resource/MetaDataTest.java b/src/test/java/org/archive/resource/MetaDataTest.java index ea66135a..88b8cd10 100644 --- a/src/test/java/org/archive/resource/MetaDataTest.java +++ b/src/test/java/org/archive/resource/MetaDataTest.java @@ -10,9 +10,11 @@ import org.json.JSONArray; import org.json.JSONObject; -import junit.framework.TestCase; +import org.junit.jupiter.api.Test; -public class MetaDataTest extends TestCase { +import static org.junit.jupiter.api.Assertions.*; + +public class MetaDataTest { private static String[] testFilePaths = { "src/test/resources/org/archive/format/warc/IAH-urls-wget.warc", @@ -59,13 +61,13 @@ private MetaData putMetaData(MetaData m) { private void verifyMultiValuedMetaData(MetaData m) { // boolean assertEquals(JSONArray.class, m.get("boolean-1").getClass()); - assertEquals(false, ((JSONArray) m.get("boolean-1")).getBoolean(0)); - assertEquals(true, ((JSONArray) m.get("boolean-1")).getBoolean(1)); - assertEquals(true, m.getBoolean("boolean-2")); - assertEquals(true, m.getBoolean("boolean-3")); + assertFalse(((JSONArray) m.get("boolean-1")).getBoolean(0)); + assertTrue(((JSONArray) m.get("boolean-1")).getBoolean(1)); + assertTrue(m.getBoolean("boolean-2")); + assertTrue(m.getBoolean("boolean-3")); assertEquals(Boolean.class, m.get("boolean-3").getClass()); - assertEquals(true, m.optBoolean("boolean-3", false)); - assertEquals(false, m.optBoolean("boolean-99", false)); + assertTrue(m.optBoolean("boolean-3", false)); + assertFalse(m.optBoolean("boolean-99", false)); // double assertEquals(JSONArray.class, m.get("double-1").getClass()); @@ -121,6 +123,7 @@ private void verifyMultiValuedMetaData(MetaData m) { assertEquals("world", ((JSONObject) m.get("obj-2")).get("hello")); } + @Test public void testMultiValued() { MetaData m = new MetaData(); m = putMetaData(m); @@ -151,6 +154,7 @@ private MetaData readNextWARCResponseAsMetaData(String filePath) throws IOExcept * Verify that in the legacy test file all WARC and HTTP headers are * single-valued, i.e. {@linkplain String}s. */ + @Test public void testSingleHeaders() throws IOException, ResourceParseException { MetaData m = readNextWARCResponseAsMetaData(testFilePaths[0]); @@ -166,6 +170,7 @@ public void testSingleHeaders() throws IOException, ResourceParseException { } } + @Test public void testMultipleHeaders() throws IOException, ResourceParseException { MetaData m = readNextWARCResponseAsMetaData(testFilePaths[1]); diff --git a/src/test/java/org/archive/resource/arc/ARCResourceTest.java b/src/test/java/org/archive/resource/arc/ARCResourceTest.java index 43116af7..e92d07be 100644 --- a/src/test/java/org/archive/resource/arc/ARCResourceTest.java +++ b/src/test/java/org/archive/resource/arc/ARCResourceTest.java @@ -3,6 +3,7 @@ import static org.archive.resource.ResourceConstants.PAYLOAD_LENGTH; import static org.archive.resource.ResourceConstants.PAYLOAD_SLOP_BYTES; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; @@ -17,10 +18,11 @@ import org.json.JSONObject; -import junit.framework.TestCase; +import org.junit.jupiter.api.Test; -public class ARCResourceTest extends TestCase { +public class ARCResourceTest { + @Test public void testARCResource() throws ResourceParseException, IOException { String testFileName = "../../format/arc/IAH-20080430204825-00000-blackbook-truncated.arc"; ResourceProducer producer = ProducerUtils.getProducer(getClass().getResource(testFileName).getPath()); diff --git a/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java b/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java index 4828ad64..157499ff 100644 --- a/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java +++ b/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java @@ -21,13 +21,16 @@ import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Multimap; -import junit.framework.TestCase; +import org.junit.jupiter.api.Test; -public class ExtractingParseObserverTest extends TestCase { +import static org.junit.jupiter.api.Assertions.*; + +public class ExtractingParseObserverTest { private static final Logger LOG = Logger.getLogger(ExtractingParseObserverTest.class.getName()); + @Test public void testHandleStyleNodeExceptions() throws Exception { String[] tests = { "some css", @@ -58,6 +61,7 @@ public void testHandleStyleNodeExceptions() throws Exception { } } + @Test public void testHandleStyleNode() throws Exception { String[][] tests = { {""}, @@ -80,6 +84,7 @@ public void testHandleStyleNode() throws Exception { * Test whether the pattern matcher does extract nothing and also does not * not hang-up if an overlong CSS link is truncated. */ + @Test public void testHandleStyleNodeNoHangupTruncated() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("url("); @@ -113,22 +118,22 @@ private void checkExtract(String[] data) throws JSONException { assertTrue(o instanceof JSONObject); JSONObject jo = (JSONObject) o; - assertEquals("CSS link extraction failed for <" + css + ">", - data[i], jo.getString("href")); + assertEquals(data[i], jo.getString("href"), + "CSS link extraction failed for <" + css + ">"); } } else { - assertNull("Expected no extracted link for <" + css + ">", a); + assertNull(a, "Expected no extracted link for <" + css + ">"); } } private void checkLink(Multimap links, String url, String path) { - assertTrue("Link with URL " + url + " not found", links.containsKey(url)); - assertTrue("Wrong path " + path + " for " + url, links.get(url).contains(path)); + assertTrue(links.containsKey(url), "Link with URL " + url + " not found"); + assertTrue(links.get(url).contains(path), "Wrong path " + path + " for " + url); } private void checkLinks(Resource resource, String[][] expectedLinks) { assertNotNull(resource); - assertTrue("Wrong instance type of Resource: " + resource.getClass(), resource instanceof HTMLResource); + assertInstanceOf(HTMLResource.class, resource, "Wrong instance type of Resource: " + resource.getClass()); MetaData md = resource.getMetaData(); LOG.info(md.toString()); Multimap links = ArrayListMultimap.create(); @@ -178,12 +183,13 @@ private void checkLinks(Resource resource, String[][] expectedLinks) { } } } - assertEquals("Unexpected number of links", expectedLinks.length, links.size()); + assertEquals(expectedLinks.length, links.size(), "Unexpected number of links"); for (String[] l : expectedLinks) { checkLink(links, l[0], l[1]); } } + @Test public void testLinkExtraction() throws ResourceParseException, IOException { String testFileName = "link-extraction-test.warc"; ResourceProducer producer = ProducerUtils.getProducer(getClass().getResource(testFileName).getPath()); diff --git a/src/test/java/org/archive/resource/html/HTMLMetaDataTest.java b/src/test/java/org/archive/resource/html/HTMLMetaDataTest.java index fb255d3c..3b4193b9 100644 --- a/src/test/java/org/archive/resource/html/HTMLMetaDataTest.java +++ b/src/test/java/org/archive/resource/html/HTMLMetaDataTest.java @@ -4,13 +4,11 @@ import org.json.JSONException; import org.json.JSONObject; -import junit.framework.TestCase; +import org.junit.jupiter.api.Test; -public class HTMLMetaDataTest extends TestCase { +public class HTMLMetaDataTest { - public void testHTMLParseData() { -// fail("Not yet implemented"); - } + @Test public void testJSON() throws JSONException { JSONObject data = new JSONObject(); JSONObject links = new JSONObject(); @@ -30,6 +28,8 @@ public void testJSON() throws JSONException { System.out.println(data.toString()); } + + @Test public void testJSON2() throws JSONException { String sa[][] = {{"one","1"},{"two","2"},{"three","3"}}; JSONObject jo = new JSONObject(); @@ -37,6 +37,8 @@ public void testJSON2() throws JSONException { appendStrArr(jo,sa); System.out.println(jo.toString(1)); } + + @Test public void testJSON3() throws JSONException { JSONObject jo = new JSONObject(); appendStrArr2(jo,"k",new String[] {"1","2","3","4"}); diff --git a/src/test/java/org/archive/resource/warc/WARCResourceTest.java b/src/test/java/org/archive/resource/warc/WARCResourceTest.java index 1b935405..71c2a4ee 100644 --- a/src/test/java/org/archive/resource/warc/WARCResourceTest.java +++ b/src/test/java/org/archive/resource/warc/WARCResourceTest.java @@ -2,6 +2,8 @@ import static org.archive.resource.ResourceConstants.PAYLOAD_LENGTH; import static org.archive.resource.ResourceConstants.PAYLOAD_SLOP_BYTES; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; @@ -16,10 +18,11 @@ import org.json.JSONObject; -import junit.framework.TestCase; +import org.junit.jupiter.api.Test; -public class WARCResourceTest extends TestCase { +public class WARCResourceTest { + @Test public void testWARCResource() throws ResourceParseException, IOException { String testFileName = "../../format/warc/IAH-urls-wget.warc"; ResourceProducer producer = ProducerUtils.getProducer(getClass().getResource(testFileName).getPath()); diff --git a/src/test/java/org/archive/uid/UUIDGeneratorTest.java b/src/test/java/org/archive/uid/UUIDGeneratorTest.java index 79e98fb6..66fbf7a8 100644 --- a/src/test/java/org/archive/uid/UUIDGeneratorTest.java +++ b/src/test/java/org/archive/uid/UUIDGeneratorTest.java @@ -23,13 +23,16 @@ import java.util.HashMap; import java.util.Map; -import junit.framework.TestCase; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertNotSame; /** * @author stack * @version $Revision$ $Date$ */ -public class UUIDGeneratorTest extends TestCase { +public class UUIDGeneratorTest { + @Test public void testQualifyRecordID() throws URISyntaxException { RecordIDGenerator g = new UUIDGenerator(); URI uri = g.getRecordID(); diff --git a/src/test/java/org/archive/url/AggressiveIAURLCanonicalizerTest.java b/src/test/java/org/archive/url/AggressiveIAURLCanonicalizerTest.java index 711dbede..fff1ea1f 100644 --- a/src/test/java/org/archive/url/AggressiveIAURLCanonicalizerTest.java +++ b/src/test/java/org/archive/url/AggressiveIAURLCanonicalizerTest.java @@ -2,10 +2,13 @@ import java.net.URISyntaxException; -import junit.framework.TestCase; +import org.junit.jupiter.api.Test; -public class AggressiveIAURLCanonicalizerTest extends TestCase { +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class AggressiveIAURLCanonicalizerTest { static AggressiveIAURLCanonicalizer ia = new AggressiveIAURLCanonicalizer(); + @Test public void testCanonicalize() throws URISyntaxException { // FULL end-to-end tests: check("http://www.alexa.com/","http://alexa.com/"); @@ -26,6 +29,6 @@ private static void check(String orig, String want) throws URISyntaxException { HandyURL u2 = URLParser.parse(got); ia.canonicalize(u2); String got2 = u2.getURLString(); - assertEquals("Second passs changed!",got,got2); + assertEquals(got,got2,"Second passs changed!"); } } diff --git a/src/test/java/org/archive/url/BasicURLCanonicalizerTest.java b/src/test/java/org/archive/url/BasicURLCanonicalizerTest.java index cc100e4c..dc000265 100644 --- a/src/test/java/org/archive/url/BasicURLCanonicalizerTest.java +++ b/src/test/java/org/archive/url/BasicURLCanonicalizerTest.java @@ -4,11 +4,15 @@ import org.apache.commons.httpclient.URIException; -import junit.framework.TestCase; +import org.junit.jupiter.api.Test; -public class BasicURLCanonicalizerTest extends TestCase { +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +public class BasicURLCanonicalizerTest { BasicURLCanonicalizer guc = new BasicURLCanonicalizer(); - + + @Test public void testGetHex() { assertEquals(0,guc.getHex('0')); assertEquals(1,guc.getHex('1')); @@ -37,7 +41,8 @@ public void testGetHex() { assertEquals(-1,guc.getHex('q')); assertEquals(-1,guc.getHex(' ')); } - + + @Test public void testDecode() { assertEquals("A",guc.decode("A")); assertEquals("AA",guc.decode("AA")); @@ -131,7 +136,7 @@ public void testDecode() { assertEquals("\u2691%E2%9A!\u2691%E2%9A", guc.decode("%E2%9A%91%E2%9A%21%E2%9A%91%E2%9A")); } - + @Test public void testUnescapeRepeatedly() { assertEquals("%!A!!%",guc.unescapeRepeatedly("%!A%21%21%25")); assertEquals("%",guc.unescapeRepeatedly("%")); @@ -147,10 +152,11 @@ public void testUnescapeRepeatedly() { assertEquals("tag=%E4%EE%F8%EA%EE%EB%FC%ED%EE%E5", guc.unescapeRepeatedly("tag=%E4%EE%F8%EA%EE%EB%FC%ED%EE%E5")); } - + + @Test public void testAttemptIPFormats() throws URIException { - assertEquals(null,guc.attemptIPFormats(null)); - assertEquals(null,guc.attemptIPFormats("www.foo.com")); + assertNull(guc.attemptIPFormats(null)); + assertNull(guc.attemptIPFormats("www.foo.com")); assertEquals("127.0.0.1",guc.attemptIPFormats("127.0.0.1")); assertEquals("15.0.0.1",guc.attemptIPFormats("017.0.0.1")); assertEquals("168.188.99.26",guc.attemptIPFormats("168.188.99.26")); @@ -190,11 +196,12 @@ In specifying the inet_addr() API, the POSIX standard [IEEE-1003.1] * For now, we'll enforce some strictness: */ - assertEquals(null,guc.attemptIPFormats("10.0.258")); - assertEquals(null,guc.attemptIPFormats("1.2.3.256")); + assertNull(guc.attemptIPFormats("10.0.258")); + assertNull(guc.attemptIPFormats("1.2.3.256")); } - + + @Test public void testFoo() { String path = "/a/b/c/"; String[] paths = path.split("/",-1); @@ -212,6 +219,7 @@ public void testFoo() { /* * Tests copied from https://developers.google.com/safe-browsing/developers_guide_v2#Canonicalization */ + @Test public void testGoogleExamples() throws URISyntaxException { checkCanonicalization("http://host/%25%32%35", "http://host/%25"); checkCanonicalization("http://host/%25%32%35%25%32%35", "http://host/%25%25"); @@ -249,19 +257,22 @@ public void testGoogleExamples() throws URISyntaxException { checkCanonicalization("http://host.com/ab%23cd", "http://host.com/ab%23cd"); checkCanonicalization("http://host.com//twoslashes?more//slashes", "http://host.com/twoslashes?more//slashes"); } - + + @Test public void testStraySpacing() throws URISyntaxException { checkCanonicalization("http://example.org/\u2028", "http://example.org/"); checkCanonicalization("\nhttp://examp\rle.org/", "http://example.org/"); checkCanonicalization("\nhttp://examp\u2029\t\rle.org/ ", "http://example.org/"); } - + + @Test public void testSchemeCapitalsPreserved() throws URISyntaxException { checkCanonicalization("Http://example.com", "Http://example.com/"); checkCanonicalization("HTTP://example.com", "HTTP://example.com/"); checkCanonicalization("ftP://example.com", "ftP://example.com/"); } - + + @Test public void testUnicodeEscaping() throws URISyntaxException { checkCanonicalization("http://example.org/\u2691", "http://example.org/%E2%9A%91"); checkCanonicalization("http://example.org/%e2%9a%91", "http://example.org/%E2%9A%91"); diff --git a/src/test/java/org/archive/url/HandyURLTest.java b/src/test/java/org/archive/url/HandyURLTest.java index 28edff77..ad108db5 100644 --- a/src/test/java/org/archive/url/HandyURLTest.java +++ b/src/test/java/org/archive/url/HandyURLTest.java @@ -1,9 +1,12 @@ package org.archive.url; -import junit.framework.TestCase; +import org.junit.jupiter.api.Test; -public class HandyURLTest extends TestCase { +import static org.junit.jupiter.api.Assertions.assertEquals; +public class HandyURLTest { + + @Test public void testGetPublicSuffix() { HandyURL h = new HandyURL(); h.setHost("www.fool.com"); @@ -23,10 +26,4 @@ public void testGetPublicSuffix() { assertEquals("funky-images",h.getPublicPrefix()); } - - public void testGetPublicPrefix() { -// -// fail("Not yet implemented"); - } - } diff --git a/src/test/java/org/archive/url/IAURLCanonicalizerTest.java b/src/test/java/org/archive/url/IAURLCanonicalizerTest.java index e2c46258..974bdd22 100644 --- a/src/test/java/org/archive/url/IAURLCanonicalizerTest.java +++ b/src/test/java/org/archive/url/IAURLCanonicalizerTest.java @@ -2,10 +2,13 @@ import java.net.URISyntaxException; -import junit.framework.TestCase; +import org.junit.jupiter.api.Test; -public class IAURLCanonicalizerTest extends TestCase { +import static org.junit.jupiter.api.Assertions.assertEquals; +public class IAURLCanonicalizerTest { + + @Test public void testFull() throws URISyntaxException { IAURLCanonicalizer iaC = new IAURLCanonicalizer(new DefaultIACanonicalizerRules()); compCan(iaC,"http://www.archive.org:80/","http://archive.org/"); @@ -26,6 +29,7 @@ private void compCan(URLCanonicalizer c, String orig, String want) throws URISyn assertEquals(want,got); } + @Test public void testAlphaReorderQuery() { assertEquals(null,IAURLCanonicalizer.alphaReorderQuery(null)); assertEquals("",IAURLCanonicalizer.alphaReorderQuery("")); @@ -41,6 +45,7 @@ public void testAlphaReorderQuery() { assertEquals("a=a&a=b&b=a&b=b",IAURLCanonicalizer.alphaReorderQuery("b=b&a=b&b=a&a=a")); } + @Test public void testMassageHost() { assertEquals("foo.com",IAURLCanonicalizer.massageHost("foo.com")); assertEquals("foo.com",IAURLCanonicalizer.massageHost("www.foo.com")); @@ -49,12 +54,14 @@ public void testMassageHost() { assertEquals("www2foo.com",IAURLCanonicalizer.massageHost("www2.www2foo.com")); } + @Test public void testGetDefaultPort() { assertEquals(0,IAURLCanonicalizer.getDefaultPort("foo")); assertEquals(80,IAURLCanonicalizer.getDefaultPort("http")); assertEquals(443,IAURLCanonicalizer.getDefaultPort("https")); } - + + @Test public void testStripSessionId() throws URISyntaxException { IAURLCanonicalizer iaC = new IAURLCanonicalizer(new DefaultIACanonicalizerRules()); compCan(iaC, diff --git a/src/test/java/org/archive/url/OrdinaryIAURLCanonicalizerTest.java b/src/test/java/org/archive/url/OrdinaryIAURLCanonicalizerTest.java index 3c131105..175491fd 100644 --- a/src/test/java/org/archive/url/OrdinaryIAURLCanonicalizerTest.java +++ b/src/test/java/org/archive/url/OrdinaryIAURLCanonicalizerTest.java @@ -2,11 +2,14 @@ import java.net.URISyntaxException; -import junit.framework.TestCase; +import org.junit.jupiter.api.Test; -public class OrdinaryIAURLCanonicalizerTest extends TestCase { +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class OrdinaryIAURLCanonicalizerTest { private OrdinaryIAURLCanonicalizer canon = new OrdinaryIAURLCanonicalizer(); - + + @Test public void testMisc() throws URISyntaxException { checkCanonicalization("http://...host..com..", "http://host.com/"); checkCanonicalization("http://example.org:80/", "http://example.org/"); @@ -17,6 +20,7 @@ public void testMisc() throws URISyntaxException { checkCanonicalization("http://example.org/foo/?", "http://example.org/foo/"); } + @Test public void testSchemeCapitals() throws URISyntaxException { checkCanonicalization("Http://example.com", "http://example.com/"); checkCanonicalization("HTTP://example.com", "http://example.com/"); diff --git a/src/test/java/org/archive/url/URLParserTest.java b/src/test/java/org/archive/url/URLParserTest.java index 68dfcd23..ff99fe38 100644 --- a/src/test/java/org/archive/url/URLParserTest.java +++ b/src/test/java/org/archive/url/URLParserTest.java @@ -4,13 +4,15 @@ import java.net.URISyntaxException; import java.net.URLDecoder; -import junit.framework.TestCase; - import org.apache.commons.httpclient.URIException; import com.google.common.net.InetAddresses; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; -public class URLParserTest extends TestCase { +public class URLParserTest { + @Test public void testGuava() throws URIException, UnsupportedEncodingException { Long l = Long.parseLong("3279880203"); int i2 = l.intValue(); @@ -18,6 +20,7 @@ public void testGuava() throws URIException, UnsupportedEncodingException { System.err.format("FromNum(%s)\n", InetAddresses.fromInteger(i2).getHostAddress()); } + @Test public void testAddDefaultSchemeIfNeeded() { assertEquals(null,URLParser.addDefaultSchemeIfNeeded(null)); assertEquals("http://",URLParser.addDefaultSchemeIfNeeded("")); @@ -27,7 +30,7 @@ public void testAddDefaultSchemeIfNeeded() { assertEquals("http://www.fool.com/",URLParser.addDefaultSchemeIfNeeded("www.fool.com/")); } - + @Test public void testParse() throws UnsupportedEncodingException, URISyntaxException { System.out.format("O(%s) E(%s)\n","%66",URLDecoder.decode("%66","UTF-8")); checkParse("http://www.archive.org/index.html#foo", diff --git a/src/test/java/org/archive/url/URLRegexTransformerTest.java b/src/test/java/org/archive/url/URLRegexTransformerTest.java index 71979d06..01e97aac 100644 --- a/src/test/java/org/archive/url/URLRegexTransformerTest.java +++ b/src/test/java/org/archive/url/URLRegexTransformerTest.java @@ -3,10 +3,13 @@ import org.apache.commons.httpclient.URIException; -import junit.framework.TestCase; +import org.junit.jupiter.api.Test; -public class URLRegexTransformerTest extends TestCase { +import static org.junit.jupiter.api.Assertions.assertEquals; +public class URLRegexTransformerTest { + + @Test public void testStripPathSessionID() { // strip jsessionid // String sid1 = "jsessionid=0123456789abcdefghijklemopqrstuv"; @@ -48,115 +51,117 @@ public void testStripPathSessionID() { private static void checkStripPathSessionID(String orig, String want) { String got = URLRegexTransformer.stripPathSessionID(orig); - assertTrue(String.format("FAIL Orig(%s) Got(%s) Want(%s)",orig,got,want),want.equals(got)); + assertEquals(want, got, String.format("FAIL Orig(%s) Got(%s) Want(%s)", orig, got, want)); } // private static final String BASE = "http://www.archive.org/index.html"; private static final String BASE = ""; + @Test public void testStripQuerySessionID() throws URIException { String str32id = "0123456789abcdefghijklemopqrstuv"; String url = BASE + "?jsessionid=" + str32id; String expectedResult = BASE + "?"; String result = URLRegexTransformer.stripQuerySessionID(url); - assertTrue("Failed " + result, expectedResult.equals(result)); + assertEquals(expectedResult, result, "Failed " + result); // Test that we don't strip if not 32 chars only. url = BASE + "?jsessionid=" + str32id + '0'; expectedResult = url; result = URLRegexTransformer.stripQuerySessionID(url); - assertTrue("Failed " + result, expectedResult.equals(result)); + assertEquals(expectedResult, result, "Failed " + result); // Test what happens when followed by another key/value pair. url = BASE + "?jsessionid=" + str32id + "&x=y"; expectedResult = BASE + "?x=y"; result = URLRegexTransformer.stripQuerySessionID(url); - assertTrue("Failed (" + result + ")", expectedResult.equals(result)); + assertEquals(expectedResult, result, "Failed (" + result + ")"); // Test what happens when followed by another key/value pair and // prefixed by a key/value pair. url = BASE + "?one=two&jsessionid=" + str32id + "&x=y"; expectedResult = BASE + "?one=two&x=y"; result = URLRegexTransformer.stripQuerySessionID(url); - assertTrue("Failed " + result, expectedResult.equals(result)); + assertEquals(expectedResult, result, "Failed " + result); // Test what happens when prefixed by a key/value pair. url = BASE + "?one=two&jsessionid=" + str32id; expectedResult = BASE + "?one=two&"; result = URLRegexTransformer.stripQuerySessionID(url); - assertTrue("Failed " + result, expectedResult.equals(result)); + assertEquals(expectedResult, result, "Failed " + result); // Test aspsession. url = BASE + "?aspsessionidABCDEFGH=" + "ABCDEFGHIJKLMNOPQRSTUVWX" + "&x=y"; expectedResult = BASE + "?x=y"; result = URLRegexTransformer.stripQuerySessionID(url); - assertTrue("Failed " + result, expectedResult.equals(result)); + assertEquals(expectedResult, result, "Failed " + result); // Test archive phpsession. url = BASE + "?phpsessid=" + str32id + "&x=y"; expectedResult = BASE + "?x=y"; result = URLRegexTransformer.stripQuerySessionID(url); - assertTrue("Failed " + result, expectedResult.equals(result)); + assertEquals(expectedResult, result, "Failed " + result); // With prefix too. url = BASE + "?one=two&phpsessid=" + str32id + "&x=y"; expectedResult = BASE + "?one=two&x=y"; result = URLRegexTransformer.stripQuerySessionID(url); - assertTrue("Failed " + result, expectedResult.equals(result)); + assertEquals(expectedResult, result, "Failed " + result); // With only prefix url = BASE + "?one=two&phpsessid=" + str32id; expectedResult = BASE + "?one=two&"; result = URLRegexTransformer.stripQuerySessionID(url); - assertTrue("Failed " + result, expectedResult.equals(result)); + assertEquals(expectedResult, result, "Failed " + result); // Test sid. url = BASE + "?" + "sid=9682993c8daa2c5497996114facdc805" + "&x=y"; expectedResult = BASE + "?x=y"; result = URLRegexTransformer.stripQuerySessionID(url); - assertTrue("Failed " + result, expectedResult.equals(result)); + assertEquals(expectedResult, result, "Failed " + result); // Igor test. url = BASE + "?" + "sid=9682993c8daa2c5497996114facdc805" + "&" + "jsessionid=" + str32id; expectedResult = BASE + "?"; result = URLRegexTransformer.stripQuerySessionID(url); - assertTrue("Failed " + result, expectedResult.equals(result)); + assertEquals(expectedResult, result, "Failed " + result); url = "?CFID=1169580&CFTOKEN=48630702&dtstamp=22%2F08%2F2006%7C06%3A58%3A11"; expectedResult = "?dtstamp=22%2F08%2F2006%7C06%3A58%3A11"; result = URLRegexTransformer.stripQuerySessionID(url); - assertTrue("Failed " + result, expectedResult.equals(result)); + assertEquals(expectedResult, result, "Failed " + result); url = "?CFID=12412453&CFTOKEN=15501799&dt=19_08_2006_22_39_28"; expectedResult = "?dt=19_08_2006_22_39_28"; result = URLRegexTransformer.stripQuerySessionID(url); - assertTrue("Failed " + result, expectedResult.equals(result)); + assertEquals(expectedResult, result, "Failed " + result); url = "?CFID=14475712&CFTOKEN=2D89F5AF-3048-2957-DA4EE4B6B13661AB&r=468710288378&m=forgotten"; expectedResult = "?r=468710288378&m=forgotten"; result = URLRegexTransformer.stripQuerySessionID(url); - assertTrue("Failed " + result, expectedResult.equals(result)); + assertEquals(expectedResult, result, "Failed " + result); url = "?CFID=16603925&CFTOKEN=2AE13EEE-3048-85B0-56CEDAAB0ACA44B8"; expectedResult = "?"; result = URLRegexTransformer.stripQuerySessionID(url); - assertTrue("Failed " + result, expectedResult.equals(result)); + assertEquals(expectedResult, result, "Failed " + result); url = "?CFID=4308017&CFTOKEN=63914124&requestID=200608200458360%2E39414378"; expectedResult = "?requestID=200608200458360%2E39414378"; result = URLRegexTransformer.stripQuerySessionID(url); - assertTrue("Failed " + result, expectedResult.equals(result)); + assertEquals(expectedResult, result, "Failed " + result); } - + + @Test public void testSURT() { assertEquals("org,archive,www",URLRegexTransformer.hostToSURT("www.archive.org")); } diff --git a/src/test/java/org/archive/url/UsableURIFactoryTest.java b/src/test/java/org/archive/url/UsableURIFactoryTest.java index 73f2b6db..368cc93d 100644 --- a/src/test/java/org/archive/url/UsableURIFactoryTest.java +++ b/src/test/java/org/archive/url/UsableURIFactoryTest.java @@ -19,15 +19,13 @@ package org.archive.url; -import java.util.Iterator; import java.util.TreeMap; -import junit.framework.TestCase; - import org.apache.commons.httpclient.URIException; import org.apache.commons.lang.SerializationUtils; -import org.archive.url.UsableURI; -import org.archive.url.UsableURIFactory; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; /** * Test UURIFactory for proper UURI creation across variety of @@ -37,8 +35,9 @@ * * @author igor stack gojomo */ -public class UsableURIFactoryTest extends TestCase { - +public class UsableURIFactoryTest { + + @Test public final void testEscaping() throws URIException { // Note: single quote is not being escaped by URI class. final String ESCAPED_URISTR = "http://archive.org/" + @@ -64,44 +63,46 @@ public final void testEscaping() throws URIException { UsableURI uuri = UsableURIFactory.getInstance(URISTR); final String uuriStr = uuri.toString(); - assertEquals("expected escaping", ESCAPED_URISTR, uuriStr); + assertEquals(ESCAPED_URISTR, uuriStr, "expected escaping"); } + @Test public final void testUnderscoreMakesPortParseFail() throws URIException { UsableURI uuri = UsableURIFactory.getInstance("http://one-two_three:8080/index.html"); int port = uuri.getPort(); - assertTrue("Failed find of port " + uuri, port == 8080); + assertEquals(8080, port, "Failed find of port " + uuri); } - + + @Test public final void testRelativeURIWithTwoSlashes() throws URIException { UsableURI base = UsableURIFactory.getInstance("http://www.archive.org"); UsableURI uuri = UsableURIFactory.getInstance(base, "one//index.html"); - assertTrue("Doesn't do right thing with two slashes " + uuri, - uuri.toString().equals( - "http://www.archive.org/one//index.html")); + assertEquals("http://www.archive.org/one//index.html", uuri.toString(), + "Doesn't do right thing with two slashes " + uuri); } - + + @Test public final void testSchemelessURI() throws URIException { UsableURI base = UsableURIFactory.getInstance("https://www.archive.org"); UsableURI uuri = UsableURIFactory.getInstance(base, "//example.com/monkey?this:uri:has:colons"); - assertTrue("Doesn't do right thing with a schemeless URI " + uuri, - uuri.toString().equals( - "https://example.com/monkey?this:uri:has:colons")); + assertEquals("https://example.com/monkey?this:uri:has:colons", uuri.toString(), + "Doesn't do right thing with a schemeless URI " + uuri); } - + + @Test public final void testTrailingEncodedSpace() throws URIException { UsableURI uuri = UsableURIFactory.getInstance("http://www.nps-shoes.co.uk%20"); - assertTrue("Doesn't strip trailing encoded space 1 " + uuri, - uuri.toString().equals("http://www.nps-shoes.co.uk/")); + assertEquals("http://www.nps-shoes.co.uk/", uuri.toString(), + "Doesn't strip trailing encoded space 1 " + uuri); uuri = UsableURIFactory.getInstance("http://www.nps-shoes.co.uk%20%20%20"); - assertTrue("Doesn't strip trailing encoded space 2 " + uuri, - uuri.toString().equals("http://www.nps-shoes.co.uk/")); + assertEquals("http://www.nps-shoes.co.uk/", uuri.toString(), + "Doesn't strip trailing encoded space 2 " + uuri); } - + + @Test public final void testPort0080is80() throws URIException { UsableURI uuri = UsableURIFactory.getInstance("http://archive.org:0080"); - assertTrue("Doesn't strip leading zeros " + uuri, - uuri.toString().equals("http://archive.org/")); + assertEquals("http://archive.org/", uuri.toString(), "Doesn't strip leading zeros " + uuri); } // DISABLING TEST AS PRECURSOR TO ELIMINATION @@ -127,13 +128,15 @@ public final void testPort0080is80() throws URIException { // } // assertNotNull("Didn't get expected exception.", message); // } - + + @Test public final void testEscapeEncoding() throws URIException { UsableURI uuri = UsableURIFactory.getInstance("http://www.y1y1.com/" + "albums/userpics/11111/normal_%E3%E4%EC%EC%EC.jpg", "windows-1256"); uuri.getPath(); } - + + @Test public final void testTooLongAfterEscaping() { StringBuffer buffer = new StringBuffer("http://www.archive.org/a/"); // Append bunch of spaces. When escaped, they'll triple in size. @@ -147,36 +150,36 @@ public final void testTooLongAfterEscaping() { } catch (URIException e) { message = e.getMessage(); } - assertTrue("Wrong or no exception: " + message, (message != null) && - message.startsWith("Created (escaped) uuri >")); + assertTrue((message != null) && message.startsWith("Created (escaped) uuri >"), + "Wrong or no exception: " + message); } - + + @Test public final void testFtpUris() throws URIException { final String FTP = "ftp"; final String AUTHORITY = "pfbuser:pfbuser@mprsrv.agri.gov.cn"; final String PATH = "/clzreceive/"; final String uri = FTP + "://" + AUTHORITY + PATH; UsableURI uuri = UsableURIFactory.getInstance(uri); - assertTrue("Failed to get matching scheme: " + uuri.getScheme(), - (uuri.getScheme()).equals(FTP)); - assertTrue("Failed to get matching authority: " + - uuri.getAuthority(), (uuri.getAuthority()).equals(AUTHORITY)); - assertTrue("Failed to get matching path: " + - uuri.getPath(), (uuri.getPath()).equals(PATH)); + assertEquals(FTP, (uuri.getScheme()), "Failed to get matching scheme: " + uuri.getScheme()); + assertEquals(AUTHORITY, (uuri.getAuthority()), "Failed to get matching authority: " + + uuri.getAuthority()); + assertEquals(PATH, (uuri.getPath()), "Failed to get matching path: " + + uuri.getPath()); } - + + @Test public final void testWhitespaceEscaped() throws URIException { // Test that we get all whitespace even if the uri is // already escaped. String uri = "http://archive.org/index%25 .html"; String tgtUri = "http://archive.org/index%25%20.html"; UsableURI uuri = UsableURIFactory.getInstance(uri); - assertTrue("Not equal " + uuri.toString(), - uuri.toString().equals(tgtUri)); + assertEquals(tgtUri, uuri.toString(), "Not equal " + uuri); uri = "http://archive.org/index%25\u001D.html"; tgtUri = "http://archive.org/index%25%1D.html"; uuri = UsableURIFactory.getInstance(uri); - assertEquals("whitespace escaping", tgtUri, uuri.toString()); + assertEquals(tgtUri, uuri.toString(), "whitespace escaping"); uri = "http://gemini.info.usaid.gov/directory/" + "pbResults.cfm?&urlNameLast=Rumplestiltskin"; tgtUri = "http://gemini.info.usaid.gov/directory/faxResults.cfm?" + @@ -184,13 +187,13 @@ public final void testWhitespaceEscaped() throws URIException { uuri = UsableURIFactory.getInstance(UsableURIFactory.getInstance(uri), "faxResults.cfm?name=Ebenezer +Rumplestiltskin,&location=" + "RRB%20%20%20%205%2E08%2D006"); - assertEquals("whitespace escaping", tgtUri, uuri.toString()); + assertEquals(tgtUri, uuri.toString(), "whitespace escaping"); // https://webarchive.jira.com/browse/HER-2089 uri = "http://archive.org/index%25\u3000.html"; tgtUri = "http://archive.org/index%25%E3%80%80.html"; uuri = UsableURIFactory.getInstance(uri); - assertEquals("U+3000 ideographic space escaping", tgtUri, uuri.toString()); + assertEquals(tgtUri, uuri.toString(), "U+3000 ideographic space escaping"); } // public final void testFailedGetPath() throws URIException { @@ -203,44 +206,48 @@ public final void testWhitespaceEscaped() throws URIException { // String foundPath = uuri.getPath(); // assertEquals("unexpected path", path, foundPath); // } - + + @Test public final void testDnsHost() throws URIException { String uri = "dns://ads.nandomedia.com:81/one.html"; UsableURI uuri = UsableURIFactory.getInstance(uri); String host = uuri.getReferencedHost(); - assertTrue("Host is wrong " + host, host.equals("ads.nandomedia.com")); + assertEquals("ads.nandomedia.com", host, "Host is wrong " + host); uri = "dns:ads.nandomedia.com"; uuri = UsableURIFactory.getInstance(uri); host = uuri.getReferencedHost(); - assertTrue("Host is wrong " + host, host.equals("ads.nandomedia.com")); + assertEquals("ads.nandomedia.com", host, "Host is wrong " + host); uri = "dns:ads.nandomedia.com?a=b"; uuri = UsableURIFactory.getInstance(uri); host = uuri.getReferencedHost(); - assertTrue("Host is wrong " + host, host.equals("ads.nandomedia.com")); + assertEquals("ads.nandomedia.com", host, "Host is wrong " + host); } - + + @Test public final void testPercentEscaping() throws URIException { final String uri = "http://archive.org/%a%%%%%.html"; // tests indicate firefox (1.0.6) does not encode '%' at all final String tgtUri = "http://archive.org/%a%%%%%.html"; UsableURI uuri = UsableURIFactory.getInstance(uri); - assertEquals("Not equal",tgtUri, uuri.toString()); + assertEquals(tgtUri,uuri.toString(), "Not equal"); } - + + @Test public final void testRelativeDblPathSlashes() throws URIException { UsableURI base = UsableURIFactory.getInstance("http://www.archive.org/index.html"); UsableURI uuri = UsableURIFactory.getInstance(base, "JIGOU//KYC//INDEX.HTM"); - assertTrue("Double slash not working " + uuri.toString(), - uuri.getPath().equals("/JIGOU//KYC//INDEX.HTM")); + assertEquals("/JIGOU//KYC//INDEX.HTM", uuri.getPath(), "Double slash not working " + uuri); } - + + @Test public final void testRelativeWithScheme() throws URIException { UsableURI base = UsableURIFactory.getInstance("http://www.example.com/some/page"); UsableURI uuri = UsableURIFactory.getInstance(base, "http:boo"); - assertTrue("Relative with scheme not working " + uuri.toString(), - uuri.toString().equals("http://www.example.com/some/boo")); + assertEquals("http://www.example.com/some/boo", uuri.toString(), + "Relative with scheme not working " + uuri); } - + + @Test public final void testBadBaseResolve() throws URIException { UsableURI base = UsableURIFactory.getInstance("http://license.joins.com/board/" + "etc_board_list.asp?board_name=new_main&b_type=&nPage=" + @@ -248,29 +255,29 @@ public final void testBadBaseResolve() throws URIException { "notice&gate=02"); UsableURIFactory.getInstance(base, "http://www.changeup.com/...[ 1010966 ] crawl.log has URIs with spaces in them. - * @throws URIException */ + @Test public final void testSpaceDoubleEncoding() throws URIException { final String uri = "http://www.brook.edu/i.html? %20taxonomy=Politics"; final String encodedUri = "http://www.brook.edu/i.html?%20%20taxonomy=Politics"; UsableURI uuri = UsableURIFactory.getInstance(uri, "ISO-8859-1"); - assertTrue("Not equal " + uuri.toString(), - uuri.toString().equals(encodedUri)); + assertEquals(encodedUri, uuri.toString(), "Not equal " + uuri.toString()); } /** * Test for doubly-encoded sequences. * See [ 966219 ] UURI doubly-encodes %XX sequences. - * @throws URIException */ + @Test public final void testDoubleEncoding() throws URIException { final char ae = '\u00E6'; final String uri = "http://archive.org/DIR WITH SPACES/home" + @@ -316,20 +322,20 @@ public final void testDoubleEncoding() throws URIException { final String encodedUri = "http://archive.org/DIR%20WITH%20SPACES/home%E6.html"; UsableURI uuri = UsableURIFactory.getInstance(uri, "ISO-8859-1"); - assertEquals("single encoding", encodedUri, uuri.toString()); + assertEquals(encodedUri, uuri.toString(), "single encoding"); // Dbl-encodes. uuri = UsableURIFactory.getInstance(uuri.toString(), "ISO-8859-1"); uuri = UsableURIFactory.getInstance(uuri.toString(), "ISO-8859-1"); - assertEquals("double encoding", encodedUri, uuri.toString()); + assertEquals(encodedUri, uuri.toString(), "double encoding"); // Do default utf-8 test. uuri = UsableURIFactory.getInstance(uri); final String encodedUtf8Uri = "http://archive.org/DIR%20WITH%20SPACES/home%C3%A6.html"; - assertEquals("Not equal utf8", encodedUtf8Uri, uuri.toString()); + assertEquals(encodedUtf8Uri, uuri.toString(), "Not equal utf8"); // Now dbl-encode. uuri = UsableURIFactory.getInstance(uuri.toString()); uuri = UsableURIFactory.getInstance(uuri.toString()); - assertEquals("Not equal (dbl-encoding) utf8", encodedUtf8Uri, uuri.toString()); + assertEquals(encodedUtf8Uri, uuri.toString(), "Not equal (dbl-encoding) utf8"); } /** @@ -337,26 +343,25 @@ public final void testDoubleEncoding() throws URIException { * @see [ 788219 ] URI Syntax Errors stop page parsing * @throws URIException */ + @Test public final void testThreeSlashes() throws URIException { UsableURI goodURI = UsableURIFactory. getInstance("http://lcweb.loc.gov/rr/goodtwo.html"); String uuri = "http:///lcweb.loc.gov/rr/goodtwo.html"; UsableURI rewrittenURI = UsableURIFactory.getInstance(uuri); - assertTrue("Not equal " + goodURI + ", " + uuri, - goodURI.toString().equals(rewrittenURI.toString())); + assertEquals(goodURI.toString(), rewrittenURI.toString(), "Not equal " + goodURI + ", " + uuri); uuri = "http:////lcweb.loc.gov/rr/goodtwo.html"; rewrittenURI = UsableURIFactory.getInstance(uuri); - assertTrue("Not equal " + goodURI + ", " + uuri, - goodURI.toString().equals(rewrittenURI.toString())); + assertEquals(goodURI.toString(), rewrittenURI.toString(), "Not equal " + goodURI + ", " + uuri); // Check https. goodURI = UsableURIFactory. getInstance("https://lcweb.loc.gov/rr/goodtwo.html"); uuri = "https:////lcweb.loc.gov/rr/goodtwo.html"; rewrittenURI = UsableURIFactory.getInstance(uuri); - assertTrue("Not equal " + goodURI + ", " + uuri, - goodURI.toString().equals(rewrittenURI.toString())); + assertEquals(goodURI.toString(), rewrittenURI.toString(), "Not equal " + goodURI + ", " + uuri); } - + + @Test public final void testNoScheme() { boolean expectedException = false; String uuri = "www.loc.gov/rr/european/egw/polishex.html"; @@ -366,10 +371,10 @@ public final void testNoScheme() { // Expected exception. expectedException = true; } - assertTrue("Didn't get expected exception: " + uuri, - expectedException); + assertTrue(expectedException, "Didn't get expected exception: " + uuri); } - + + @Test public final void testRelative() throws URIException { UsableURI uuriTgt = UsableURIFactory. getInstance("http://archive.org:83/home.html"); @@ -377,26 +382,25 @@ public final void testRelative() throws URIException { getInstance("http://archive.org:83/one/two/three.html"); UsableURI uuri = UsableURIFactory. getInstance(uri, "/home.html"); - assertTrue("Not equal", - uuriTgt.toString().equals(uuri.toString())); + assertEquals(uuriTgt.toString(), uuri.toString(), "Not equal"); } - + + @Test public void testSchemelessRelative() throws URIException { UsableURI base = UsableURIFactory.getInstance("http://www.itsnicethat.com/articles/laura-hobson"); UsableURI test1 = UsableURIFactory.getInstance(base, "//www.facebook.com/plugins/like.php"); - assertEquals("schemaless relative 1", "http://www.facebook.com/plugins/like.php", test1.toString()); + assertEquals("http://www.facebook.com/plugins/like.php", test1.toString(), "schemaless relative 1"); // reported by Erin Staniland UsableURI test2 = UsableURIFactory.getInstance(base, "//www.facebook.com/plugins/like.php?href=http://www.itsnicethat.com/articles/laura-hobson"); - assertEquals("schemeless relative 2", "http://www.facebook.com/plugins/like.php?href=http://www.itsnicethat.com/articles/laura-hobson", - test2.toString()); + assertEquals("http://www.facebook.com/plugins/like.php?href=http://www.itsnicethat.com/articles/laura-hobson", test2.toString(), + "schemeless relative 2"); } /** * Test that an empty uuri does the right thing -- that we get back the * base. - * - * @throws URIException */ + @Test public final void testRelativeEmpty() throws URIException { UsableURI uuriTgt = UsableURIFactory. getInstance("http://archive.org:83/one/two/three.html"); @@ -404,10 +408,10 @@ public final void testRelativeEmpty() throws URIException { getInstance("http://archive.org:83/one/two/three.html"); UsableURI uuri = UsableURIFactory. getInstance(uri, ""); - assertTrue("Empty length don't work", - uuriTgt.toString().equals(uuri.toString())); + assertEquals(uuriTgt.toString(), uuri.toString(), "Empty length don't work"); } - + + @Test public final void testAbsolute() throws URIException { UsableURI uuriTgt = UsableURIFactory. getInstance("http://archive.org:83/home.html"); @@ -415,14 +419,14 @@ public final void testAbsolute() throws URIException { getInstance("http://archive.org:83/one/two/three.html"); UsableURI uuri = UsableURIFactory. getInstance(uri, "http://archive.org:83/home.html"); - assertTrue("Not equal", - uuriTgt.toString().equals(uuri.toString())); + assertEquals(uuriTgt.toString(), uuri.toString(), "Not equal"); } /** * Test for [ 962892 ] UURI accepting/creating unUsable URIs (bad hosts). * @see [ 962892 ] UURI accepting/creating unUsable URIs (bad hosts) */ + @Test public final void testHostWithLessThan() { checkExceptionOnIllegalDomainlabel("http://www.betamobile.com[ 1012520 ] UURI.length() > 2k */ + @Test public final void test2kURI() throws URIException { final StringBuffer buffer = new StringBuffer("http://a.b"); final String subPath = "/123456789"; @@ -451,8 +455,7 @@ public final void test2kURI() throws URIException { } catch (URIException e) { gotException = true; } - assertTrue("No expected exception complaining about long URI", - gotException); + assertTrue(gotException, "No expected exception complaining about long URI"); } private void checkExceptionOnIllegalDomainlabel(String uuri) { @@ -463,23 +466,21 @@ private void checkExceptionOnIllegalDomainlabel(String uuri) { // Expected exception. expectedException = true; } - assertTrue("Didn't get expected exception: " + uuri, - expectedException); + assertTrue(expectedException, "Didn't get expected exception: " + uuri); } /** * Test for doing separate DNS lookup for same host * * @see [ 788277 ] Doing separate DNS lookup for same host - * @throws URIException */ + @Test public final void testHostWithPeriod() throws URIException { UsableURI uuri1 = UsableURIFactory. getInstance("http://www.loc.gov./index.html"); UsableURI uuri2 = UsableURIFactory. getInstance("http://www.loc.gov/index.html"); - assertEquals("Failed equating hosts with dot", - uuri1.getHost(), uuri2.getHost()); + assertEquals(uuri1.getHost(), uuri2.getHost(), "Failed equating hosts with dot"); } /** @@ -488,12 +489,12 @@ public final void testHostWithPeriod() throws URIException { * @see [ 874220 ] NPE in java.net.URI.encode * @throws URIException */ + @Test public final void testHostEncodedChars() throws URIException { String s = "http://g.msn.co.kr/0nwkokr0/00/19??" + "PS=10274&NC=10009&CE=42&CP=949&HL=" + "���?��"; - assertNotNull("Encoded chars " + s, - UsableURIFactory.getInstance(s)); + assertNotNull(UsableURIFactory.getInstance(s), "Encoded chars " + s); } /** @@ -501,6 +502,7 @@ public final void testHostEncodedChars() throws URIException { * * See [ 927940 ] java.net.URI parses %20 but getHost null */ + @Test public final void testSpaceInHost() { boolean expectedException = false; try { @@ -510,7 +512,7 @@ public final void testSpaceInHost() { } catch (URIException e) { expectedException = true; } - assertTrue("Did not fail with escaped space.", expectedException); + assertTrue(expectedException, "Did not fail with escaped space."); expectedException = false; try { @@ -520,26 +522,27 @@ public final void testSpaceInHost() { } catch (URIException e) { expectedException = true; } - assertTrue("Did not fail with real space.", expectedException); + assertTrue(expectedException, "Did not fail with real space."); } /** * Test for java.net.URI chokes on hosts_with_underscores. * * @see [ 808270 ] java.net.URI chokes on hosts_with_underscores - * @throws URIException - */ + */ + @Test public final void testHostWithUnderscores() throws URIException { UsableURI uuri = UsableURIFactory.getInstance( "http://x_underscore_underscore.2u.com.tw/nonexistent_page.html"); - assertEquals("Failed get of host with underscore", - "x_underscore_underscore.2u.com.tw", uuri.getHost()); + assertEquals("x_underscore_underscore.2u.com.tw", + uuri.getHost(), "Failed get of host with underscore"); } /** * Two dots for igor. */ + @Test public final void testTwoDots() { boolean expectedException = false; try { @@ -548,20 +551,19 @@ public final void testTwoDots() { } catch (URIException e) { expectedException = true; } - assertTrue("Two dots did not throw exception", expectedException); + assertTrue(expectedException, "Two dots did not throw exception"); } /** * Test for java.net.URI#getHost fails when leading digit. * * @see [ 910120 ] java.net.URI#getHost fails when leading digit. - * @throws URIException */ + @Test public final void testHostWithDigit() throws URIException { UsableURI uuri = UsableURIFactory. getInstance("http://0204chat.2u.com.tw/nonexistent_page.html"); - assertEquals("Failed get of host with digit", - "0204chat.2u.com.tw", uuri.getHost()); + assertEquals("0204chat.2u.com.tw", uuri.getHost(), "Failed get of host with digit"); } /** @@ -569,6 +571,7 @@ public final void testHostWithDigit() throws URIException { * * @see [ 949548 ] Constraining java URI class */ + @Test public final void testPort() { checkBadPort("http://www.tyopaikat.com:a/robots.txt"); checkBadPort("http://158.144.21.3:80808/robots.txt"); @@ -591,19 +594,18 @@ private void checkBadPort(String uri) { catch (URIException e) { exception = true; } - assertTrue("Didn't throw exception: " + uri, exception); + assertTrue(exception, "Didn't throw exception: " + uri); } /** * Preserve userinfo capitalization. - * @throws URIException */ + @Test public final void testUserinfo() throws URIException { final String authority = "stack:StAcK@www.tyopaikat.com"; final String uri = "http://" + authority + "/robots.txt"; UsableURI uuri = UsableURIFactory.getInstance(uri); - assertEquals("Authority not equal", uuri.getAuthority(), - authority); + assertEquals(authority, uuri.getAuthority(), "Authority not equal"); /* String tmp = uuri.toString(); assertTrue("URI not equal", tmp.equals(uri)); @@ -612,8 +614,8 @@ public final void testUserinfo() throws URIException { /** * Test user info + port - * @throws URIException */ + @Test public final void testUserinfoPlusPort() throws URIException { final String userInfo = "stack:StAcK"; final String authority = "www.tyopaikat.com"; @@ -621,14 +623,13 @@ public final void testUserinfoPlusPort() throws URIException { final String uri = "http://" + userInfo + "@" + authority + ":" + port + "/robots.txt"; UsableURI uuri = UsableURIFactory.getInstance(uri); - assertEquals("Host not equal", authority,uuri.getHost()); - assertEquals("Userinfo Not equal",userInfo,uuri.getUserinfo()); - assertEquals("Port not equal",port,uuri.getPort()); - assertEquals("Authority wrong","stack:StAcK@www.tyopaikat.com:8080", - uuri.getAuthority()); - assertEquals("AuthorityMinusUserinfo wrong","www.tyopaikat.com:8080", - uuri.getAuthorityMinusUserinfo()); - + assertEquals(authority, uuri.getHost(),"Host not equal"); + assertEquals(userInfo,uuri.getUserinfo(),"Userinfo Not equal"); + assertEquals(port,uuri.getPort(),"Port not equal"); + assertEquals("stack:StAcK@www.tyopaikat.com:8080",uuri.getAuthority(), + "Authority wrong"); + assertEquals("www.tyopaikat.com:8080",uuri.getAuthorityMinusUserinfo(), + "AuthorityMinusUserinfo wrong"); } public final void testRFC3986RelativeChange() throws URIException { @@ -664,9 +665,8 @@ public final void testRFC3986RelativeChange() throws URIException { * "../../" = "http://a/" * "../../g" = "http://a/g" * - * - * @throws URIException */ + @Test public final void testRFC3986Relative() throws URIException { UsableURI base = UsableURIFactory.getInstance("http://a/b/c/d;p?q"); tryRelative(base, "g:h", "g:h"); @@ -697,9 +697,8 @@ public final void testRFC3986Relative() throws URIException { protected void tryRelative(UsableURI base, String relative, String expected) throws URIException { UsableURI uuri = UsableURIFactory.getInstance(base, relative); - assertEquals("Derelativized " + relative + " gave " - + uuri + " not " + expected, - UsableURIFactory.getInstance(expected),uuri); + assertEquals(UsableURIFactory.getInstance(expected), uuri,"Derelativized " + relative + " gave " + + uuri + " not " + expected); } /** @@ -730,9 +729,8 @@ protected void tryRelative(UsableURI base, String relative, String expected) * ../../ = http://a/ * ../../g = http://a/g * - * - * @throws URIException */ + @Test public final void testRFC2396Relative() throws URIException { UsableURI base = UsableURIFactory. getInstance("http://a/b/c/d;p?q"); @@ -764,13 +762,11 @@ public final void testRFC2396Relative() throws URIException { m.put("/../../../../../../../../g", "http://a/g"); m.put("../../../../../../../../g", "http://a/g"); m.put("../G", "http://a/b/G"); - for (Iterator i = m.keySet().iterator(); i.hasNext();) { - String key = (String)i.next(); - String value = (String)m.get(key); - UsableURI uuri = UsableURIFactory.getInstance(base, key); - assertTrue("Unexpected " + key + " " + value + " " + uuri, - uuri.equals(UsableURIFactory.getInstance(value))); - } + for (String key : m.keySet()) { + String value = m.get(key); + UsableURI uuri = UsableURIFactory.getInstance(base, key); + assertEquals(uuri, UsableURIFactory.getInstance(value), "Unexpected " + key + " " + value + " " + uuri); + } } /** @@ -778,14 +774,13 @@ public final void testRFC2396Relative() throws URIException { * unused and irrelevant for network fetches. * * See [ 970666 ] #anchor links not trimmed, and thus recrawled - * - * @throws URIException */ + @Test public final void testAnchors() throws URIException { UsableURI uuri = UsableURIFactory. getInstance("http://www.example.com/path?query#anchor"); - assertEquals("Not equal", "http://www.example.com/path?query", - uuri.toString()); + assertEquals("http://www.example.com/path?query", uuri.toString(), + "Not equal"); } @@ -793,50 +788,47 @@ public final void testAnchors() throws URIException { * Ensure that URI strings beginning with a colon are treated * the same as browsers do (as relative, rather than as absolute * with zero-length scheme). - * - * @throws URIException */ + @Test public void testStartsWithColon() throws URIException { UsableURI base = UsableURIFactory.getInstance("http://www.example.com/path/page"); UsableURI uuri = UsableURIFactory.getInstance(base,":foo"); - assertEquals("derelativize starsWithColon", + assertEquals("http://www.example.com/path/:foo", uuri.getURI(), - "http://www.example.com/path/:foo"); + "derelativize starsWithColon"); } /** * Ensure that relative URIs with colons in late positions * aren't mistakenly interpreted as absolute URIs with long, * illegal schemes. - * - * @throws URIException */ + @Test public void testLateColon() throws URIException { UsableURI base = UsableURIFactory.getInstance("http://www.example.com/path/page"); UsableURI uuri1 = UsableURIFactory.getInstance(base,"example.html;jsessionid=deadbeef:deadbeed?parameter=this:value"); - assertEquals("derelativize lateColon", + assertEquals("http://www.example.com/path/example.html;jsessionid=deadbeef:deadbeed?parameter=this:value", uuri1.getURI(), - "http://www.example.com/path/example.html;jsessionid=deadbeef:deadbeed?parameter=this:value"); + "derelativize lateColon"); UsableURI uuri2 = UsableURIFactory.getInstance(base,"example.html?parameter=this:value"); - assertEquals("derelativize lateColon", + assertEquals("http://www.example.com/path/example.html?parameter=this:value", uuri2.getURI(), - "http://www.example.com/path/example.html?parameter=this:value"); + "derelativize lateColon"); } /** * Ensure that stray trailing '%' characters do not prevent * UURI instances from being created, and are reasonably * escaped when encountered. - * - * @throws URIException */ + @Test public void testTrailingPercents() throws URIException { String plainPath = "http://www.example.com/path%"; UsableURI plainPathUuri = UsableURIFactory.getInstance(plainPath); - assertEquals("plainPath getURI", plainPath, plainPathUuri.getURI()); - assertEquals("plainPath getEscapedURI", - "http://www.example.com/path%", // browsers don't escape '%' - plainPathUuri.getEscapedURI()); + assertEquals(plainPath, plainPathUuri.getURI(), "plainPath getURI"); + assertEquals("http://www.example.com/path%", + plainPathUuri.getEscapedURI(), // browsers don't escape '%' + "plainPath getEscapedURI"); String partiallyEscapedPath = "http://www.example.com/pa%20th%"; UsableURI partiallyEscapedPathUuri = UsableURIFactory.getInstance( @@ -845,9 +837,9 @@ public void testTrailingPercents() throws URIException { // "http://www.example.com/pa th%", // TODO: is this desirable? //// partiallyEscapedPath, // partiallyEscapedPathUuri.getURI()); - assertEquals("partiallyEscapedPath getEscapedURI", - "http://www.example.com/pa%20th%", - partiallyEscapedPathUuri.getEscapedURI()); + assertEquals("http://www.example.com/pa%20th%", + partiallyEscapedPathUuri.getEscapedURI(), + "partiallyEscapedPath getEscapedURI"); String plainQueryString = "http://www.example.com/path?q=foo%"; UsableURI plainQueryStringUuri = UsableURIFactory.getInstance( @@ -855,58 +847,58 @@ public void testTrailingPercents() throws URIException { // assertEquals("plainQueryString getURI", // plainQueryString, // plainQueryStringUuri.getURI()); - assertEquals("plainQueryString getEscapedURI", - "http://www.example.com/path?q=foo%", - plainQueryStringUuri.getEscapedURI()); + assertEquals("http://www.example.com/path?q=foo%", + plainQueryStringUuri.getEscapedURI(), + "plainQueryString getEscapedURI"); String partiallyEscapedQueryString = "http://www.example.com/pa%20th?q=foo%"; UsableURI partiallyEscapedQueryStringUuri = UsableURIFactory.getInstance( partiallyEscapedQueryString); - assertEquals("partiallyEscapedQueryString getURI", - "http://www.example.com/pa th?q=foo%", - partiallyEscapedQueryStringUuri.getURI()); - assertEquals("partiallyEscapedQueryString getEscapedURI", - "http://www.example.com/pa%20th?q=foo%", - partiallyEscapedQueryStringUuri.getEscapedURI()); + assertEquals("http://www.example.com/pa th?q=foo%", + partiallyEscapedQueryStringUuri.getURI(), + "partiallyEscapedQueryString getURI"); + assertEquals("http://www.example.com/pa%20th?q=foo%", + partiallyEscapedQueryStringUuri.getEscapedURI(), + "partiallyEscapedQueryString getEscapedURI"); } /** * Ensure that stray '%' characters do not prevent * UURI instances from being created, and are reasonably * escaped when encountered. - * - * @throws URIException */ + @Test public void testStrayPercents() throws URIException { String oneStray = "http://www.example.com/pa%th"; UsableURI oneStrayUuri = UsableURIFactory.getInstance(oneStray); - assertEquals("oneStray getURI", oneStray, oneStrayUuri.getURI()); - assertEquals("oneStray getEscapedURI", - "http://www.example.com/pa%th", // browsers don't escape '%' - oneStrayUuri.getEscapedURI()); + assertEquals(oneStray, oneStrayUuri.getURI(), "oneStray getURI"); + assertEquals("http://www.example.com/pa%th", + oneStrayUuri.getEscapedURI(), // browsers don't escape '%' + "oneStray getEscapedURI"); String precededByValidEscape = "http://www.example.com/pa%20th%way"; UsableURI precededByValidEscapeUuri = UsableURIFactory.getInstance( precededByValidEscape); - assertEquals("precededByValidEscape getURI", - "http://www.example.com/pa th%way", // getURI interprets escapes - precededByValidEscapeUuri.getURI()); - assertEquals("precededByValidEscape getEscapedURI", - "http://www.example.com/pa%20th%way", - precededByValidEscapeUuri.getEscapedURI()); + assertEquals("http://www.example.com/pa th%way", + precededByValidEscapeUuri.getURI(), // getURI interprets escapes + "precededByValidEscape getURI"); + assertEquals("http://www.example.com/pa%20th%way", + precededByValidEscapeUuri.getEscapedURI(), + "precededByValidEscape getEscapedURI"); String followedByValidEscape = "http://www.example.com/pa%th%20way"; UsableURI followedByValidEscapeUuri = UsableURIFactory.getInstance( followedByValidEscape); - assertEquals("followedByValidEscape getURI", - "http://www.example.com/pa%th way", // getURI interprets escapes - followedByValidEscapeUuri.getURI()); - assertEquals("followedByValidEscape getEscapedURI", - "http://www.example.com/pa%th%20way", - followedByValidEscapeUuri.getEscapedURI()); + assertEquals("http://www.example.com/pa%th way", + followedByValidEscapeUuri.getURI(), // getURI interprets escapes + "followedByValidEscape getURI"); + assertEquals("http://www.example.com/pa%th%20way", + followedByValidEscapeUuri.getEscapedURI(), + "followedByValidEscape getEscapedURI"); } - + + @Test public void testEscapingNotNecessary() throws URIException { String escapesUnnecessary = "http://www.example.com/misc;reserved:chars@that&don't=need" @@ -914,42 +906,46 @@ public void testEscapingNotNecessary() throws URIException { // expect everything but the #fragment String expected = escapesUnnecessary.substring(0, escapesUnnecessary .length() - 3); - assertEquals("escapes unnecessary", - expected, - UsableURIFactory.getInstance(escapesUnnecessary).toString()); + assertEquals(expected, + UsableURIFactory.getInstance(escapesUnnecessary).toString(), + "escapes unnecessary"); } - + + @Test public void testIdn() throws URIException { // See http://www.josefsson.org/idn.php. // http://räksmörgås.josefßon.org/ String idn1 = "http://r\u00e4ksm\u00f6rg\u00e5s.josef\u00dfon.org/"; String puny1 = "http://xn--rksmrgs-5wao1o.josefsson.org/"; - assertEquals("encoding of " + idn1, puny1, UsableURIFactory - .getInstance(idn1).toString()); + assertEquals(puny1, UsableURIFactory + .getInstance(idn1).toString(), "encoding of " + idn1); // http://www.pølse.dk/ String idn2 = "http://www.p\u00f8lse.dk/"; String puny2 = "http://www.xn--plse-gra.dk/"; - assertEquals("encoding of " + idn2, puny2, UsableURIFactory - .getInstance(idn2).toString()); + assertEquals(puny2, UsableURIFactory + .getInstance(idn2).toString(), "encoding of " + idn2); // http://例子.測試 String idn3 = "http://\u4F8B\u5B50.\u6E2C\u8A66"; String puny3 = "http://xn--fsqu00a.xn--g6w251d/"; - assertEquals("encoding of " + idn3, puny3, UsableURIFactory - .getInstance(idn3).toString()); + assertEquals(puny3, UsableURIFactory + .getInstance(idn3).toString(), "encoding of " + idn3); } - + + @Test public void testNewLineInURL() throws URIException { UsableURI uuri = UsableURIFactory.getInstance("http://www.ar\rchive\n." + "org/i\n\n\r\rndex.html"); assertEquals("http://www.archive.org/index.html", uuri.toString()); } - + + @Test public void testTabsInURL() throws URIException { UsableURI uuri = UsableURIFactory.getInstance("http://www.ar\tchive\t." + "org/i\t\r\n\tndex.html"); assertEquals("http://www.archive.org/index.html", uuri.toString()); } - + + @Test public void testQueryEscaping() throws URIException { UsableURI uuri = UsableURIFactory.getInstance( "http://www.yahoo.com/foo?somechars!@$%^&*()_-+={[}]|\'\";:/?.>,<"); @@ -963,50 +959,44 @@ public void testQueryEscaping() throws URIException { * Check that our 'normalization' does same as Nutch's * Below before-and-afters were taken from the nutch urlnormalizer-basic * TestBasicURLNormalizer class (December 2006, Nutch 0.9-dev). - * @throws URIException */ + @Test public void testSameAsNutchURLFilterBasic() throws URIException { - assertEquals(UsableURIFactory.getInstance(" http://foo.com/ ").toString(), - "http://foo.com/"); + assertEquals("http://foo.com/", + UsableURIFactory.getInstance(" http://foo.com/ ").toString()); // check that protocol is lower cased - assertEquals(UsableURIFactory.getInstance("HTTP://foo.com/").toString(), - "http://foo.com/"); + assertEquals("http://foo.com/", + UsableURIFactory.getInstance("HTTP://foo.com/").toString()); // check that host is lower cased - assertEquals(UsableURIFactory. - getInstance("http://Foo.Com/index.html").toString(), - "http://foo.com/index.html"); - assertEquals(UsableURIFactory. - getInstance("http://Foo.Com/index.html").toString(), - "http://foo.com/index.html"); + assertEquals("http://foo.com/index.html", + UsableURIFactory.getInstance("http://Foo.Com/index.html").toString()); + assertEquals("http://foo.com/index.html", + UsableURIFactory.getInstance("http://Foo.Com/index.html").toString()); // check that port number is normalized - assertEquals(UsableURIFactory. - getInstance("http://foo.com:80/index.html").toString(), - "http://foo.com/index.html"); - assertEquals(UsableURIFactory.getInstance("http://foo.com:81/").toString(), - "http://foo.com:81/"); + assertEquals("http://foo.com/index.html", + UsableURIFactory.getInstance("http://foo.com:80/index.html").toString()); + assertEquals("http://foo.com:81/", + UsableURIFactory.getInstance("http://foo.com:81/").toString()); // check that null path is normalized - assertEquals(UsableURIFactory.getInstance("http://foo.com").toString(), - "http://foo.com/"); + assertEquals("http://foo.com/", + UsableURIFactory.getInstance("http://foo.com").toString()); // check that references are removed - assertEquals(UsableURIFactory. - getInstance("http://foo.com/foo.html#ref").toString(), - "http://foo.com/foo.html"); + assertEquals("http://foo.com/foo.html", + UsableURIFactory.getInstance("http://foo.com/foo.html#ref").toString()); // // check that encoding is normalized // normalizeTest("http://foo.com/%66oo.html", "http://foo.com/foo.html"); // check that unnecessary "../" are removed - assertEquals(UsableURIFactory. - getInstance("http://foo.com/aa/../").toString(), - "http://foo.com/" ); - assertEquals(UsableURIFactory. - getInstance("http://foo.com/aa/bb/../").toString(), - "http://foo.com/aa/"); + assertEquals("http://foo.com/", + UsableURIFactory.getInstance("http://foo.com/aa/../").toString()); + assertEquals("http://foo.com/aa/", + UsableURIFactory.getInstance("http://foo.com/aa/bb/../").toString()); /* We fail this one. Here we produce: 'http://foo.com/'. assertEquals(UURIFactory. @@ -1014,45 +1004,33 @@ public void testSameAsNutchURLFilterBasic() throws URIException { "http://foo.com/aa/.."); */ - assertEquals(UsableURIFactory. - getInstance("http://foo.com/aa/bb/cc/../../foo.html").toString(), - "http://foo.com/aa/foo.html"); - assertEquals(UsableURIFactory. - getInstance("http://foo.com/aa/bb/../cc/dd/../ee/foo.html"). - toString(), - "http://foo.com/aa/cc/ee/foo.html"); - assertEquals(UsableURIFactory. - getInstance("http://foo.com/../foo.html").toString(), - "http://foo.com/foo.html" ); - assertEquals(UsableURIFactory. - getInstance("http://foo.com/../../foo.html").toString(), - "http://foo.com/foo.html" ); - assertEquals(UsableURIFactory. - getInstance("http://foo.com/../aa/../foo.html").toString(), - "http://foo.com/foo.html" ); - assertEquals(UsableURIFactory. - getInstance("http://foo.com/aa/../../foo.html").toString(), - "http://foo.com/foo.html" ); - assertEquals(UsableURIFactory. - getInstance("http://foo.com/aa/../bb/../foo.html/../../"). - toString(), - "http://foo.com/" ); - assertEquals(UsableURIFactory.getInstance("http://foo.com/../aa/foo.html"). - toString(), "http://foo.com/aa/foo.html" ); - assertEquals(UsableURIFactory. - getInstance("http://foo.com/../aa/../foo.html").toString(), - "http://foo.com/foo.html" ); - assertEquals(UsableURIFactory. - getInstance("http://foo.com/a..a/foo.html").toString(), - "http://foo.com/a..a/foo.html" ); - assertEquals(UsableURIFactory. - getInstance("http://foo.com/a..a/../foo.html").toString(), - "http://foo.com/foo.html" ); - assertEquals(UsableURIFactory. - getInstance("http://foo.com/foo.foo/../foo.html").toString(), - "http://foo.com/foo.html" ); + assertEquals("http://foo.com/aa/foo.html", + UsableURIFactory.getInstance("http://foo.com/aa/bb/cc/../../foo.html").toString()); + assertEquals("http://foo.com/aa/cc/ee/foo.html", + UsableURIFactory.getInstance("http://foo.com/aa/bb/../cc/dd/../ee/foo.html").toString()); + assertEquals("http://foo.com/foo.html", + UsableURIFactory.getInstance("http://foo.com/../foo.html").toString()); + assertEquals("http://foo.com/foo.html", + UsableURIFactory.getInstance("http://foo.com/../../foo.html").toString()); + assertEquals("http://foo.com/foo.html", + UsableURIFactory.getInstance("http://foo.com/../aa/../foo.html").toString()); + assertEquals("http://foo.com/foo.html", + UsableURIFactory.getInstance("http://foo.com/aa/../../foo.html").toString()); + assertEquals("http://foo.com/", + UsableURIFactory.getInstance("http://foo.com/aa/../bb/../foo.html/../../").toString()); + assertEquals("http://foo.com/aa/foo.html", + UsableURIFactory.getInstance("http://foo.com/../aa/foo.html").toString()); + assertEquals("http://foo.com/foo.html", + UsableURIFactory.getInstance("http://foo.com/../aa/../foo.html").toString()); + assertEquals("http://foo.com/a..a/foo.html", + UsableURIFactory.getInstance("http://foo.com/a..a/foo.html").toString()); + assertEquals("http://foo.com/foo.html", + UsableURIFactory.getInstance("http://foo.com/a..a/../foo.html").toString()); + assertEquals("http://foo.com/foo.html", + UsableURIFactory.getInstance("http://foo.com/foo.foo/../foo.html").toString()); } - + + @Test public void testHttpSchemeColonSlash() { boolean exception = false; try { @@ -1060,16 +1038,17 @@ public void testHttpSchemeColonSlash() { } catch (URIException e) { exception = true; } - assertTrue("Didn't throw exception when one expected", exception); + assertTrue(exception, "Didn't throw exception when one expected"); exception = false; try { UsableURIFactory.getInstance("http://"); } catch (URIException e) { exception = true; } - assertTrue("Didn't throw exception when one expected", exception); + assertTrue(exception, "Didn't throw exception when one expected"); } - + + @Test public void testNakedHttpsSchemeColon() { boolean exception = false; try { @@ -1077,7 +1056,7 @@ public void testNakedHttpsSchemeColon() { } catch (URIException e) { exception = true; } - assertTrue("Didn't throw exception when one expected", exception); + assertTrue(exception, "Didn't throw exception when one expected"); exception = false; try { UsableURI base = UsableURIFactory.getInstance("http://www.example.com"); @@ -1085,15 +1064,14 @@ public void testNakedHttpsSchemeColon() { } catch (URIException e) { exception = true; } - assertTrue("Didn't throw exception when one expected", exception); + assertTrue(exception, "Didn't throw exception when one expected"); } /** * Test motivated by [#HER-616] The UURI class may throw * NullPointerException in getReferencedHost() - * - * @throws URIException */ + @Test public void testMissingHttpColon() throws URIException { String suspectUri = "http//www.test.foo"; UsableURI base = UsableURIFactory.getInstance("http://www.example.com"); @@ -1105,7 +1083,7 @@ public void testMissingHttpColon() throws URIException { // should get relative-uri-no-base exception exceptionThrown = true; } finally { - assertTrue("expected exception not thrown",exceptionThrown); + assertTrue(exceptionThrown,"expected exception not thrown"); } UsableURI goodUuri = UsableURIFactory.getInstance(base,suspectUri); goodUuri.getReferencedHost(); @@ -1114,33 +1092,31 @@ public void testMissingHttpColon() throws URIException { /** * A UURI's string representation should be same after a * serialization roundtrip. - * - * @throws URIException */ + @Test public final void testSerializationRoundtrip() throws URIException { UsableURI uuri = UsableURIFactory. getInstance("http://www.example.com/path?query#anchor"); UsableURI uuri2 = (UsableURI) SerializationUtils.deserialize( SerializationUtils.serialize(uuri)); - assertEquals("Not equal", uuri.toString(), uuri2.toString()); + assertEquals(uuri.toString(), uuri2.toString(), "Not equal"); uuri = UsableURIFactory. getInstance("file://///boo_hoo/wwwroot/CMS/Images1/Banner.gif"); uuri2 = (UsableURI) SerializationUtils.deserialize( SerializationUtils.serialize(uuri)); - assertEquals("Not equal", uuri.toString(), uuri2.toString()); + assertEquals(uuri.toString(), uuri2.toString(), "Not equal"); } /** * A UURI's string representation should be same after a * toCustomString-getInstance roundtrip. - * - * @throws URIException */ + @Test public final void testToCustomStringRoundtrip() throws URIException { UsableURI uuri = UsableURIFactory. getInstance("http://www.example.com/path?query#anchor"); UsableURI uuri2 = UsableURIFactory.getInstance(uuri.toCustomString()); - assertEquals("Not equal", uuri.toString(), uuri2.toString()); + assertEquals(uuri.toString(), uuri2.toString(), "Not equal"); // TODO: fix // see [HER-1470] UURI String roundtrip (UURIFactory.getInstance(uuri.toString()) results in different URI for file: (and perhaps other) URIs // http://webteam.archive.org/jira/browse/HER-1470 @@ -1153,9 +1129,8 @@ public final void testToCustomStringRoundtrip() throws URIException { /** * A UURI's string representation should be same after a * toCustomString-getInstance roundtrip. - * - * @throws URIException */ + @Test public final void testHostnamePortRoundtrip() throws URIException { UsableURI base = UsableURIFactory. getInstance("http://www.example.com/path?query#anchor"); @@ -1163,13 +1138,14 @@ public final void testHostnamePortRoundtrip() throws URIException { System.out.println("scheme:"+test.getScheme()); System.out.println(test.toCustomString()); UsableURI roundtrip = UsableURIFactory.getInstance(test.toCustomString()); - assertEquals("Not equal", test.toString(), roundtrip.toString()); + assertEquals(test.toString(), roundtrip.toString(), "Not equal"); } /** * Test bad port throws URIException not NumberFormatException */ + @Test public void testExtremePort() { try { UsableURI uuri = UsableURIFactory.getInstance("http://Tel.:010101010101"); @@ -1183,9 +1159,8 @@ public void testExtremePort() { /** * Bars ('|') in path-segments aren't encoded by FF, preferred by some * RESTful-URI-ideas guides, so should work without error. - * - * @throws URIException */ + @Test public void testBarsInRelativePath() throws URIException { UsableURI base = UsableURIFactory.getInstance("http://www.example.com"); String relative = "foo/bar|baz|yorple"; @@ -1197,9 +1172,8 @@ public void testBarsInRelativePath() throws URIException { * To match IE behavior, backslashes in path-info (really, anywhere before * query string) assumed to be slashes, to match IE behavior. In * query-string, they are escaped to %5C. - * - * @throws URIException */ + @Test public void testBackslashes() throws URIException { UsableURI uuri = UsableURIFactory.getInstance("http:\\/www.example.com\\a/b\\c/d?q\\r\\|s/t\\v"); String expected = "http://www.example.com/a/b/c/d?q%5Cr%5C|s/t%5Cv"; diff --git a/src/test/java/org/archive/url/UsableURITest.java b/src/test/java/org/archive/url/UsableURITest.java index 2a2f41f5..9a4c1860 100644 --- a/src/test/java/org/archive/url/UsableURITest.java +++ b/src/test/java/org/archive/url/UsableURITest.java @@ -22,16 +22,20 @@ import org.apache.commons.httpclient.URIException; -import junit.framework.TestCase; +import org.junit.jupiter.api.Test; -public class UsableURITest extends TestCase { +import static org.junit.jupiter.api.Assertions.*; + +public class UsableURITest { + @Test public void testHasScheme() { assertTrue(UsableURI.hasScheme("http://www.archive.org")); assertTrue(UsableURI.hasScheme("http:")); assertFalse(UsableURI.hasScheme("ht/tp://www.archive.org")); assertFalse(UsableURI.hasScheme("/tmp")); } - + + @Test public void testGetFileName() throws URISyntaxException { final String filename = "x.arc.gz"; assertEquals(filename, @@ -43,11 +47,12 @@ public void testGetFileName() throws URISyntaxException { UsableURI.parseFilename("rsync://archive.org/tmp/one.two/" + filename)); } - + + @Test public void testSchemalessRelative() throws URIException { UsableURI base = new UsableURI("http://www.archive.org/a", true, "UTF-8"); UsableURI relative = new UsableURI("//www.facebook.com/?href=http://www.archive.org/a", true, "UTF-8"); - assertEquals(null, relative.getScheme()); + assertNull(relative.getScheme()); assertEquals("www.facebook.com", relative.getAuthority()); UsableURI test = new UsableURI(base, relative); assertEquals("http://www.facebook.com/?href=http://www.archive.org/a", test.toString()); @@ -56,6 +61,7 @@ public void testSchemalessRelative() throws URIException { /** * Test of toUnicodeHostString method, of class UsableURI. */ + @Test public void testToUnicodeHostString() throws URIException { assertEquals("http://øx.dk", new UsableURI("http://xn--x-4ga.dk", true, "UTF-8").toUnicodeHostString()); assertEquals("xn--x-4ga.dk", new UsableURI("xn--x-4ga.dk", true, "UTF-8").toUnicodeHostString()); diff --git a/src/test/java/org/archive/url/WaybackURLKeyMakerTest.java b/src/test/java/org/archive/url/WaybackURLKeyMakerTest.java index 26371ba8..28a00422 100644 --- a/src/test/java/org/archive/url/WaybackURLKeyMakerTest.java +++ b/src/test/java/org/archive/url/WaybackURLKeyMakerTest.java @@ -2,10 +2,13 @@ import java.net.URISyntaxException; -import junit.framework.TestCase; +import org.junit.jupiter.api.Test; -public class WaybackURLKeyMakerTest extends TestCase { +import static org.junit.jupiter.api.Assertions.assertEquals; +public class WaybackURLKeyMakerTest { + + @Test public void testMakeKey() throws URISyntaxException { WaybackURLKeyMaker km = new WaybackURLKeyMaker(); assertEquals("-", km.makeKey(null)); diff --git a/src/test/java/org/archive/util/ArchiveUtilsTest.java b/src/test/java/org/archive/util/ArchiveUtilsTest.java index 586a1821..18337498 100644 --- a/src/test/java/org/archive/util/ArchiveUtilsTest.java +++ b/src/test/java/org/archive/util/ArchiveUtilsTest.java @@ -19,15 +19,16 @@ package org.archive.util; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; + import java.text.ParseException; import java.util.Date; import java.util.HashSet; import java.util.concurrent.Semaphore; import java.util.concurrent.atomic.AtomicInteger; -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; +import static org.junit.jupiter.api.Assertions.*; /** * JUnit test suite for ArchiveUtils @@ -35,47 +36,21 @@ * @author James Casey * @version $Id$ */ -public class ArchiveUtilsTest extends TestCase { - - /** - * Create a new ArchiveUtilsTest object - * - * @param testName the name of the test - */ - public ArchiveUtilsTest(final String testName) { - super(testName); - } - - /** - * run all the tests for ArchiveUtilsTest - * - * @param argv the command line arguments - */ - public static void main(String argv[]) { - junit.textui.TestRunner.run(suite()); - } - - /** - * return the suite of tests for ArchiveUtilsTest - * - * @return the suite of test - */ - public static Test suite() { - return new TestSuite(ArchiveUtilsTest.class); - } +public class ArchiveUtilsTest { /** check the getXXDigitDate() methods produce valid dates*/ + @Test public void testGetXXDigitDate() { // TODO - we only really test the date lengths here. How to test // other stuff well ? final String date12 = ArchiveUtils.get12DigitDate(); - assertEquals("12 digits", 12, date12.length()); + assertEquals(12, (Object) date12.length(), "12 digits"); final String date14 = ArchiveUtils.get14DigitDate(); - assertEquals("14 digits", 14, date14.length()); + assertEquals(14, (Object) date14.length(), "14 digits"); final String date17 = ArchiveUtils.get17DigitDate(); - assertEquals("17 digits", 17, date17.length()); + assertEquals(17, (Object) date17.length(), "17 digits"); // now parse, and check they're all within 1 minute @@ -93,18 +68,19 @@ public void testGetXXDigitDate() { } /** check that getXXDigitDate(long) does the right thing */ + @Test public void testGetXXDigitDateLong() { final long now = System.currentTimeMillis(); final String date12 = ArchiveUtils.get12DigitDate(now); - assertEquals("12 digits", 12, date12.length()); + assertEquals(12, (Object) date12.length(), "12 digits"); final String date14 = ArchiveUtils.get14DigitDate(now); - assertEquals("14 digits", 14, date14.length()); - assertEquals("first twelve digits same as date12", date12, date14.substring(0, 12)); + assertEquals(14, (Object) date14.length(), "14 digits"); + assertEquals(date12, date14.substring(0, 12), "first twelve digits same as date12"); final String date17 = ArchiveUtils.get17DigitDate(now); - assertEquals("17 digits", 17, date17.length()); - assertEquals("first twelve digits same as date12", date12, date17.substring(0, 12)); - assertEquals("first fourteen digits same as date14", date14, date17.substring(0, 14)); + assertEquals(17, (Object) date17.length(), "17 digits"); + assertEquals(date12, date17.substring(0, 12), "first twelve digits same as date12"); + assertEquals(date14, date17.substring(0, 14), "first fourteen digits same as date14"); } /** @@ -112,6 +88,7 @@ public void testGetXXDigitDateLong() { * * @throws ParseException */ + @Test public void testParseXXDigitDate() throws ParseException { // given a date, check it get resolved properly // It's 02 Jan 2004, 12:40:02.111 @@ -128,7 +105,8 @@ public void testParseXXDigitDate() throws ParseException { fail("Could not parse a date : " + e.getMessage()); } } - + + @Test public void testTooShortParseDigitDate() throws ParseException { String d = "X"; boolean b = false; @@ -157,6 +135,7 @@ public void testTooShortParseDigitDate() throws ParseException { } /** check that parse12DigitDate doesn't accept a bad date */ + @Test public void testBad12Date() { // now try a badly formed dates assertBad12DigitDate("a-stringy-digit-date"); @@ -166,6 +145,7 @@ public void testBad12Date() { /** * check that parse14DigitDate doesn't accept a bad date */ + @Test public void testBad14Date() { // now try a badly formed dates assertBad14DigitDate("a-stringy-digit-date"); @@ -175,6 +155,7 @@ public void testBad14Date() { /** * check that parse12DigitDate doesn't accept a bad date */ + @Test public void testBad17Date() { // now try a badly formed dates assertBad17DigitDate("a-stringy-digit-date"); @@ -184,27 +165,30 @@ public void testBad17Date() { } /** check that padTo(String) works */ + @Test public void testPadToString() { - assertEquals("pad to one (smaller)", "foo", ArchiveUtils.padTo("foo", 1)); - assertEquals("pad to 0 (no sense)", "foo", ArchiveUtils.padTo("foo", 0)); - assertEquals("pad to neg (nonsense)", "foo", ArchiveUtils.padTo("foo", 0)); - assertEquals("pad to 4", " foo", ArchiveUtils.padTo("foo", 4)); - assertEquals("pad to 10", " foo", ArchiveUtils.padTo("foo", 10)); + assertEquals("foo", ArchiveUtils.padTo("foo", 1), "pad to one (smaller)"); + assertEquals("foo", ArchiveUtils.padTo("foo", 0), "pad to 0 (no sense)"); + assertEquals("foo", ArchiveUtils.padTo("foo", 0), "pad to neg (nonsense)"); + assertEquals(" foo", ArchiveUtils.padTo("foo", 4), "pad to 4"); + assertEquals(" foo", ArchiveUtils.padTo("foo", 10), "pad to 10"); } /** * check that padTo(int) works */ + @Test public void testPadToInt() { - assertEquals("pad to one (smaller)", "123", ArchiveUtils.padTo(123, 1)); - assertEquals("pad to 0 (no sense)", "123", ArchiveUtils.padTo(123, 0)); - assertEquals("pad to neg (nonsense)", "123", ArchiveUtils.padTo(123, 0)); - assertEquals("pad to 4", " 123", ArchiveUtils.padTo(123, 4)); - assertEquals("pad to 10", " 123", ArchiveUtils.padTo(123, 10)); - assertEquals("pad -123 to 10", " -123", ArchiveUtils.padTo(-123, 10)); + assertEquals("123", ArchiveUtils.padTo(123, 1), "pad to one (smaller)"); + assertEquals("123", ArchiveUtils.padTo(123, 0), "pad to 0 (no sense)"); + assertEquals("123", ArchiveUtils.padTo(123, 0), "pad to neg (nonsense)"); + assertEquals(" 123", ArchiveUtils.padTo(123, 4), "pad to 4"); + assertEquals(" 123", ArchiveUtils.padTo(123, 10), "pad to 10"); + assertEquals(" -123", ArchiveUtils.padTo(-123, 10), "pad -123 to 10"); } /** check that byteArrayEquals() works */ + @Test public void testByteArrayEquals() { // foo == foo2, foo != bar, foo != bar2 byte[] foo = new byte[10], bar = new byte[20]; @@ -214,62 +198,52 @@ public void testByteArrayEquals() { foo[i] = foo2[i] = bar[i] = i; bar2[i] = (byte)(01 + i); } - assertTrue("two nulls", ArchiveUtils.byteArrayEquals(null, null)); - assertFalse("lhs null", ArchiveUtils.byteArrayEquals(null, foo)); - assertFalse("rhs null", ArchiveUtils.byteArrayEquals(foo, null)); + assertTrue(ArchiveUtils.byteArrayEquals(null, null), "two nulls"); + assertFalse(ArchiveUtils.byteArrayEquals(null, foo), "lhs null"); + assertFalse(ArchiveUtils.byteArrayEquals(foo, null), "rhs null"); // now check with same length, with same (foo2) and different (bar2) // contents - assertFalse("different lengths", ArchiveUtils.byteArrayEquals(foo, bar)); + assertFalse(ArchiveUtils.byteArrayEquals(foo, bar), "different lengths"); - assertTrue("same to itself", ArchiveUtils.byteArrayEquals(foo, foo)); - assertTrue("same contents", ArchiveUtils.byteArrayEquals(foo, foo2)); - assertFalse("different contents", ArchiveUtils.byteArrayEquals(foo, bar2)); + assertTrue(ArchiveUtils.byteArrayEquals(foo, foo), "same to itself"); + assertTrue(ArchiveUtils.byteArrayEquals(foo, foo2), "same contents"); + assertFalse(ArchiveUtils.byteArrayEquals(foo, bar2), "different contents"); } /** test doubleToString() */ + @Test public void testDoubleToString(){ double test = 12.121d; - assertEquals( - "cecking zero precision", - "12", - ArchiveUtils.doubleToString(test, 0)); - assertEquals( - "cecking 2 character precision", - "12.12", - ArchiveUtils.doubleToString(test, 2)); - assertEquals( - "cecking precision higher then the double has", - "12.121", - ArchiveUtils.doubleToString(test, 65)); + assertEquals("12", ArchiveUtils.doubleToString(test, 0), "cecking zero precision"); + assertEquals("12.12", ArchiveUtils.doubleToString(test, 2), "cecking 2 character precision"); + assertEquals("12.121", ArchiveUtils.doubleToString(test, 65), "cecking precision higher then the double has"); } + @Test public void testFormatBytesForDisplayPrecise(){ - assertEquals("formating negative number", "0 B", ArchiveUtils - .formatBytesForDisplay(-1)); - assertEquals("0 bytes", "0 B", ArchiveUtils - .formatBytesForDisplay(0)); - assertEquals("1 B", ArchiveUtils.formatBytesForDisplay(1)); - assertEquals("9 B", ArchiveUtils.formatBytesForDisplay(9)); - assertEquals("512 B", ArchiveUtils.formatBytesForDisplay(512)); - assertEquals("1023 bytes", "1,023 B", ArchiveUtils - .formatBytesForDisplay(1023)); - assertEquals("1025 bytes", "1.0 KiB", ArchiveUtils - .formatBytesForDisplay(1025)); + assertEquals("0 B", ArchiveUtils + .formatBytesForDisplay(-1), "formating negative number"); + assertEquals("0 B", ArchiveUtils + .formatBytesForDisplay(0), "0 bytes"); + Object a2 = ArchiveUtils.formatBytesForDisplay(1); + assertEquals("1 B", a2); + Object a1 = ArchiveUtils.formatBytesForDisplay(9); + assertEquals("9 B", a1); + Object a = ArchiveUtils.formatBytesForDisplay(512); + assertEquals( "512 B", a); + assertEquals("1,023 B", ArchiveUtils + .formatBytesForDisplay(1023), "1023 bytes"); + assertEquals("1.0 KiB", ArchiveUtils + .formatBytesForDisplay(1025), "1025 bytes"); // expected display values taken from Google calculator - assertEquals("10,000 bytes", "9.8 KiB", - ArchiveUtils.formatBytesForDisplay(10000)); - assertEquals("1,000,000 bytes", "977 KiB", - ArchiveUtils.formatBytesForDisplay(1000000)); - assertEquals("100,000,000 bytes", "95 MiB", - ArchiveUtils.formatBytesForDisplay(100000000)); - assertEquals("100,000,000,000 bytes", "93 GiB", - ArchiveUtils.formatBytesForDisplay(100000000000L)); - assertEquals("100,000,000,000,000 bytes", "91 TiB", - ArchiveUtils.formatBytesForDisplay(100000000000000L)); - assertEquals("100,000,000,000,000,000 bytes", "90,949 TiB", - ArchiveUtils.formatBytesForDisplay(100000000000000000L)); + assertEquals("9.8 KiB", ArchiveUtils.formatBytesForDisplay(10000), "10,000 bytes"); + assertEquals("977 KiB", ArchiveUtils.formatBytesForDisplay(1000000), "1,000,000 bytes"); + assertEquals("95 MiB", ArchiveUtils.formatBytesForDisplay(100000000), "100,000,000 bytes"); + assertEquals("93 GiB", ArchiveUtils.formatBytesForDisplay(100000000000L), "100,000,000,000 bytes"); + assertEquals("91 TiB", ArchiveUtils.formatBytesForDisplay(100000000000000L), "100,000,000,000,000 bytes"); + assertEquals("90,949 TiB", ArchiveUtils.formatBytesForDisplay(100000000000000000L), "100,000,000,000,000,000 bytes"); } /* @@ -321,11 +295,12 @@ private void assertBad17DigitDate(final String date) { /** check that two longs are within a given delta */ private void assertClose(String desc, long date1, long date2, long delta) { - assertTrue(desc, date1 == date2 || + assertTrue(date1 == date2 || (date1 < date2 && date2 < (date1 + delta)) || - (date2 < date1 && date1 < (date2 + delta))); + (date2 < date1 && date1 < (date2 + delta)), desc); } - + + @Test public void testArrayToLong() { testOneArrayToLong(-1); testOneArrayToLong(1); @@ -339,19 +314,23 @@ private void testOneArrayToLong(final long testValue) { final long l = ArchiveUtils.byteArrayIntoLong(a, 0); assertEquals(testValue, l); } - + + @Test public void testSecondsSinceEpochCalculation() throws ParseException { - assertEquals(ArchiveUtils.secondsSinceEpoch("20010909014640"), - "1000000000"); - assertEquals(ArchiveUtils.secondsSinceEpoch("20010909014639"), - "0999999999"); - assertEquals(ArchiveUtils.secondsSinceEpoch("19700101"), - "0000000000"); - assertEquals(ArchiveUtils.secondsSinceEpoch("2005"), "1104537600"); - assertEquals(ArchiveUtils.secondsSinceEpoch("200501"), "1104537600"); - assertEquals(ArchiveUtils.secondsSinceEpoch("20050101"), "1104537600"); - assertEquals(ArchiveUtils.secondsSinceEpoch("2005010100"), - "1104537600"); + String m6 = ArchiveUtils.secondsSinceEpoch("20010909014640"); + assertEquals("1000000000", m6); + String m5 = ArchiveUtils.secondsSinceEpoch("20010909014639"); + assertEquals("0999999999", m5); + String m4 = ArchiveUtils.secondsSinceEpoch("19700101"); + assertEquals("0000000000", m4); + String m3 = ArchiveUtils.secondsSinceEpoch("2005"); + assertEquals("1104537600", m3); + String m2 = ArchiveUtils.secondsSinceEpoch("200501"); + assertEquals("1104537600", m2); + String m1 = ArchiveUtils.secondsSinceEpoch("20050101"); + assertEquals("1104537600", m1); + String m = ArchiveUtils.secondsSinceEpoch("2005010100"); + assertEquals("1104537600", m); boolean eThrown = false; try { ArchiveUtils.secondsSinceEpoch("20050"); @@ -360,10 +339,13 @@ public void testSecondsSinceEpochCalculation() throws ParseException { } assertTrue(eThrown); } - - public static void testZeroPadInteger() { - assertEquals(ArchiveUtils.zeroPadInteger(1), "0000000001"); - assertEquals(ArchiveUtils.zeroPadInteger(1000000000), "1000000000"); + + @Test + public void testZeroPadInteger() { + String m1 = ArchiveUtils.zeroPadInteger(1); + assertEquals("0000000001", m1); + String m = ArchiveUtils.zeroPadInteger(1000000000); + assertEquals("1000000000", m); } /** @@ -371,7 +353,9 @@ public static void testZeroPadInteger() { * * @throws InterruptedException */ - public static void testDateFormatConcurrency() throws InterruptedException { + @Test + @EnabledIfSystemProperty(named = "runSlowTests", matches = "true") + public void testDateFormatConcurrency() throws InterruptedException { final int COUNT = 1000; Thread [] ts = new Thread[COUNT]; final Semaphore allDone = new Semaphore(-COUNT+1); @@ -403,24 +387,29 @@ public void run() { while(!ts[i].isAlive()) /* Wait for thread to spin up*/; } allDone.acquire(); // wait for all threads to finish - assertEquals(failures.get()+" format mismatches",0,failures.get()); + String m = failures.get()+" format mismatches"; + assertEquals(0, (Object) failures.get(), m); } - + + @Test public void testIsTld() { - assertTrue("TLD test problem", ArchiveUtils.isTld("com")); - assertTrue("TLD test problem", ArchiveUtils.isTld("COM")); + assertTrue(ArchiveUtils.isTld("com"), "TLD test problem"); + assertTrue(ArchiveUtils.isTld("COM"), "TLD test problem"); } - + + @Test public void testUnique17() { HashSet uniqueTimestamps = new HashSet(); for(int i = 0; i<10; i++) { - assertTrue("timestamp17 repeated",uniqueTimestamps.add(ArchiveUtils.getUnique17DigitDate())); + assertTrue(uniqueTimestamps.add(ArchiveUtils.getUnique17DigitDate()),"timestamp17 repeated"); } } + + @Test public void testUnique14() { HashSet uniqueTimestamps = new HashSet(); for(int i = 0; i<10; i++) { - assertTrue("timestamp14 repeated",uniqueTimestamps.add(ArchiveUtils.getUnique14DigitDate())); + assertTrue(uniqueTimestamps.add(ArchiveUtils.getUnique14DigitDate()),"timestamp14 repeated"); } } } diff --git a/src/test/java/org/archive/util/ByteOpTest.java b/src/test/java/org/archive/util/ByteOpTest.java index de6a164f..49781c36 100644 --- a/src/test/java/org/archive/util/ByteOpTest.java +++ b/src/test/java/org/archive/util/ByteOpTest.java @@ -5,14 +5,15 @@ import java.io.DataInputStream; import java.io.IOException; -import org.archive.util.ByteOp; - import com.google.common.io.LittleEndianDataOutputStream; -import junit.framework.TestCase; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; -public class ByteOpTest extends TestCase { +public class ByteOpTest { + @Test public void testReadShort() throws IOException { byte a[] = new byte[]{0,1,2,3}; ByteArrayInputStream bais = new ByteArrayInputStream(a); @@ -31,6 +32,7 @@ public void testReadShort() throws IOException { } } + @Test public void testAppend() { byte a[] = new byte[]{1}; byte b[] = new byte[]{2}; @@ -48,8 +50,4 @@ public void testAppend() { assertEquals(5,n2[4]); } - - public void testReadInt() { - } - } diff --git a/src/test/java/org/archive/util/CrossProductTest.java b/src/test/java/org/archive/util/CrossProductTest.java index edadb859..211fa65e 100644 --- a/src/test/java/org/archive/util/CrossProductTest.java +++ b/src/test/java/org/archive/util/CrossProductTest.java @@ -3,11 +3,9 @@ import java.util.ArrayList; import java.util.List; -import org.archive.util.CrossProduct; +import org.junit.jupiter.api.Test; -import junit.framework.TestCase; - -public class CrossProductTest extends TestCase { +public class CrossProductTest { private void dumpC(List