Skip to content

Commit dcbb052

Browse files
atoato
authored
Merge pull request iipc#98 from sebastian-nagel/metadata-multivalued
Make MetaData multi-valued to preserve values of repeating WARC and HTTP headers
2 parents 0514b23 + 835f4e1 commit dcbb052

3 files changed

Lines changed: 309 additions & 3 deletions

File tree

‎src/main/java/org/archive/resource/MetaData.java‎

Lines changed: 76 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,15 @@
77
import org.json.JSONObject;
88
import org.json.JSONTokener;
99

10+
/**
11+
* A nested structure of {@linkplain JSONObject}s to hold the metadata of
12+
* content in nested containers, e.g. a HTML page as payload of a HTTP response
13+
* in a WARC record stored as gzip "member".
14+
*
15+
* MetaData is multi-valued: if a second value is added under the same "key"
16+
* ("name"), both values are stored in a {@linkplain JSONArray} as value. This
17+
* allows to hold all values of repeating WARC or HTTP headers.
18+
*/
1019
public class MetaData extends JSONObject {
1120

1221
private static final Logger LOG =
@@ -67,6 +76,18 @@ public int getInt(String key) {
6776
}
6877
}
6978

79+
@Override
80+
public int optInt(String key, int defaultValue) {
81+
if (has(key)) {
82+
try {
83+
return super.getInt(key);
84+
} catch(JSONException e) {
85+
LOG.severe(e.getMessage());
86+
}
87+
}
88+
return defaultValue;
89+
}
90+
7091
@Override
7192
public long getLong(String key) {
7293
try {
@@ -77,6 +98,18 @@ public long getLong(String key) {
7798
}
7899
}
79100

101+
@Override
102+
public long optLong(String key, long defaultValue) {
103+
if (has(key)) {
104+
try {
105+
return super.getLong(key);
106+
} catch(JSONException e) {
107+
LOG.severe(e.getMessage());
108+
}
109+
}
110+
return defaultValue;
111+
}
112+
80113
@Override
81114
public String getString(String key) {
82115
try {
@@ -102,9 +135,49 @@ public void setTopMetaData(MetaData topMetaData) {
102135
this.topMetaData = topMetaData;
103136
}
104137

138+
@Override
139+
public JSONObject put(String name, boolean value) throws JSONException {
140+
return super.accumulate(name, value);
141+
}
142+
143+
@Override
144+
public JSONObject put(String name, double value) throws JSONException {
145+
return super.accumulate(name, value);
146+
}
147+
148+
@Override
149+
public JSONObject put(String name, int value) throws JSONException {
150+
return super.accumulate(name, value);
151+
}
152+
153+
@Override
154+
public JSONObject put(String name, long value) throws JSONException {
155+
return super.accumulate(name, value);
156+
}
157+
158+
@Override
159+
public JSONObject put(String key, Object value) {
160+
if (value instanceof JSONArray) {
161+
super.remove(key);
162+
super.put(key, value);
163+
} else if (has(key)) {
164+
if (super.get(key) instanceof JSONArray) {
165+
((JSONArray) super.get(key)).put(value);
166+
return this;
167+
} else {
168+
JSONArray array = new JSONArray();
169+
array.put(super.get(key));
170+
array.put(value);
171+
super.put(key, array);
172+
}
173+
return super.accumulate(key, value);
174+
}
175+
return super.put(key, value);
176+
}
177+
105178
public JSONObject putString(String key, String val) {
106179
try {
107-
return super.put(key,val);
180+
return super.accumulate(key,val);
108181
} catch(JSONException e) {
109182
LOG.severe(e.getMessage());
110183
return null;
@@ -113,7 +186,7 @@ public JSONObject putString(String key, String val) {
113186

114187
public JSONObject putLong(String key, long val) {
115188
try {
116-
return super.put(key,String.valueOf(val));
189+
return super.accumulate(key,String.valueOf(val));
117190
} catch(JSONException e) {
118191
LOG.severe(e.getMessage());
119192
return null;
@@ -122,7 +195,7 @@ public JSONObject putLong(String key, long val) {
122195

123196
public JSONObject putBoolean(String key, boolean val) {
124197
try {
125-
return super.put(key,val);
198+
return super.accumulate(key,val);
126199
} catch(JSONException e) {
127200
LOG.severe(e.getMessage());
128201
return null;

‎src/test/java/org/archive/resource/MetaDataTest.java‎

Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
package org.archive.resource;
2+
3+
import java.io.IOException;
4+
5+
import org.archive.extract.ExtractingResourceFactoryMapper;
6+
import org.archive.extract.ExtractingResourceProducer;
7+
import org.archive.extract.ProducerUtils;
8+
import org.archive.extract.ResourceFactoryMapper;
9+
import org.archive.format.json.JSONUtils;
10+
import org.json.JSONArray;
11+
import org.json.JSONObject;
12+
13+
import junit.framework.TestCase;
14+
15+
public class MetaDataTest extends TestCase {
16+
17+
private static String[] testFilePaths = {
18+
"src/test/resources/org/archive/format/warc/IAH-urls-wget.warc",
19+
"src/test/resources/org/archive/format/warc/mutliple-headers.warc"
20+
};
21+
22+
private static JSONObject obj = new JSONObject("{\"foo\":\"bar\",\"hello\":\"world\"}");
23+
24+
private MetaData putMetaData(MetaData m) {
25+
m.putBoolean("boolean-1", false);
26+
m.putBoolean("boolean-2", true);
27+
m.put("boolean-3", true);
28+
m.put("boolean-1", true); // append
29+
30+
m.put("double-1", 0.5d);
31+
m.put("double-2", 2.5d);
32+
m.put("double-3", 3.5d);
33+
m.put("double-1", 1.5d); // append
34+
35+
m.put("int-1", 0);
36+
m.put("int-2", 2);
37+
m.put("int-3", 3);
38+
m.put("int-1", 1); // append
39+
40+
// choose JSON "numbers" which are forced into a Java long (too big for an integer)
41+
m.putLong("long-1", 0xffffffffL + 0L);
42+
m.putLong("long-2", 0xffffffffL + 2L);
43+
m.put("long-3", 0xffffffffL + 3L);
44+
m.put("long-1", 0xffffffffL + 1L); // append
45+
46+
m.putString("string-1", "0");
47+
m.putString("string-2", "2");
48+
m.put("string-3", "3");
49+
m.put("string-1", "1"); // append
50+
51+
m.putOpt("obj-1", obj);
52+
m.put("obj-1", obj); // append
53+
m.put("obj-2", obj);
54+
m.putOpt("obj-2", null); // do nothing because value is null
55+
56+
return m;
57+
}
58+
59+
private void verifyMultiValuedMetaData(MetaData m) {
60+
// boolean
61+
assertEquals(JSONArray.class, m.get("boolean-1").getClass());
62+
assertEquals(false, ((JSONArray) m.get("boolean-1")).getBoolean(0));
63+
assertEquals(true, ((JSONArray) m.get("boolean-1")).getBoolean(1));
64+
assertEquals(true, m.getBoolean("boolean-2"));
65+
assertEquals(true, m.getBoolean("boolean-3"));
66+
assertEquals(Boolean.class, m.get("boolean-3").getClass());
67+
assertEquals(true, m.optBoolean("boolean-3", false));
68+
assertEquals(false, m.optBoolean("boolean-99", false));
69+
70+
// double
71+
assertEquals(JSONArray.class, m.get("double-1").getClass());
72+
assertEquals(0.5d, ((JSONArray) m.get("double-1")).getDouble(0));
73+
assertEquals(1.5d, ((JSONArray) m.get("double-1")).getDouble(1));
74+
assertEquals(2.5d, m.getDouble("double-2"));
75+
assertEquals(3.5d, m.getDouble("double-3"));
76+
// could be Double or BigDecimal, depending on the Java version
77+
// assertEquals(Double.class, m.get("double-3").getClass());
78+
assertEquals(3.5d, m.optDouble("double-3"));
79+
assertEquals(99.5d, m.optDouble("double-99", 99.5d));
80+
81+
// int
82+
assertEquals(JSONArray.class, m.get("int-1").getClass());
83+
assertEquals(0, ((JSONArray) m.get("int-1")).getInt(0));
84+
assertEquals(1, ((JSONArray) m.get("int-1")).getInt(1));
85+
assertEquals(2, m.getInt("int-2"));
86+
assertEquals(3, m.getInt("int-3"));
87+
assertEquals(Integer.class, m.get("int-3").getClass());
88+
assertEquals(3, m.optInt("int-3"));
89+
assertEquals(99, m.optInt("int-99", 99));
90+
91+
// long
92+
assertEquals(JSONArray.class, m.get("long-1").getClass());
93+
assertEquals(0xffffffffL + 0L, ((JSONArray) m.get("long-1")).getLong(0));
94+
assertEquals(0xffffffffL + 1L, ((JSONArray) m.get("long-1")).getLong(1));
95+
assertEquals(0xffffffffL + 2L, m.getLong("long-2"));
96+
assertEquals(0xffffffffL + 3L, m.getLong("long-3"));
97+
assertEquals(Long.class, m.get("long-3").getClass());
98+
assertEquals(0xffffffffL + 3L, m.optLong("long-3"));
99+
assertEquals(0xffffffffL + 99L, m.optLong("long-99", 0xffffffffL + 99L));
100+
101+
// String
102+
assertEquals(JSONArray.class, m.get("string-1").getClass());
103+
assertEquals("0", ((JSONArray) m.get("string-1")).getString(0));
104+
assertEquals("1", ((JSONArray) m.get("string-1")).getString(1));
105+
assertEquals("2", m.getString("string-2"));
106+
assertEquals("3", m.getString("string-3"));
107+
assertEquals(String.class, m.get("string-3").getClass());
108+
assertEquals("3", m.optString("string-3"));
109+
assertEquals("99", m.optString("string-99", "99"));
110+
111+
// Object
112+
assertEquals(JSONArray.class, m.get("obj-1").getClass());
113+
assertEquals(JSONObject.class, ((JSONArray) m.get("obj-1")).get(0).getClass());
114+
assertEquals(JSONObject.class, ((JSONArray) m.get("obj-1")).get(1).getClass());
115+
assertEquals("bar", ((JSONObject) ((JSONArray) m.get("obj-1")).get(0)).get("foo"));
116+
assertEquals("world", ((JSONObject) ((JSONArray) m.get("obj-1")).get(0)).get("hello"));
117+
assertEquals("bar", ((JSONObject) ((JSONArray) m.get("obj-1")).get(1)).get("foo"));
118+
assertEquals("world", ((JSONObject) ((JSONArray) m.get("obj-1")).get(1)).get("hello"));
119+
assertEquals(JSONObject.class, m.get("obj-2").getClass());
120+
assertEquals("bar", ((JSONObject) m.get("obj-2")).get("foo"));
121+
assertEquals("world", ((JSONObject) m.get("obj-2")).get("hello"));
122+
}
123+
124+
public void testMultiValued() {
125+
MetaData m = new MetaData();
126+
m = putMetaData(m);
127+
verifyMultiValuedMetaData(m);
128+
129+
// test (de)serialization
130+
m = new MetaData(m.toString(2));
131+
verifyMultiValuedMetaData(m);
132+
}
133+
134+
private MetaData readNextWARCResponseAsMetaData(String filePath) throws IOException, ResourceParseException {
135+
ResourceProducer producer = ProducerUtils.getProducer(filePath);
136+
ResourceFactoryMapper mapper = new ExtractingResourceFactoryMapper();
137+
ExtractingResourceProducer exProducer = new ExtractingResourceProducer(producer, mapper);
138+
Resource r = exProducer.getNext();
139+
while (r != null) {
140+
MetaData top = r.getMetaData().getTopMetaData();
141+
JSONObject warcHeaders = JSONUtils.extractObject(top, "Envelope.WARC-Header-Metadata");
142+
if (warcHeaders.has("WARC-Type") && "response".equals(warcHeaders.getString("WARC-Type"))) {
143+
return top;
144+
}
145+
r = exProducer.getNext();
146+
}
147+
return null;
148+
}
149+
150+
/**
151+
* Verify that in the legacy test file all WARC and HTTP headers are
152+
* single-valued, i.e. {@linkplain String}s.
153+
*/
154+
public void testSingleHeaders() throws IOException, ResourceParseException {
155+
MetaData m = readNextWARCResponseAsMetaData(testFilePaths[0]);
156+
157+
JSONObject warcHeaders = JSONUtils.extractObject(m, "Envelope.WARC-Header-Metadata");
158+
JSONObject httpHeaders = JSONUtils.extractObject(m, "Envelope.Payload-Metadata.HTTP-Response-Metadata.Headers");
159+
160+
for (Object header : warcHeaders.keySet()) {
161+
assertEquals(String.class, warcHeaders.get(header.toString()).getClass());
162+
}
163+
164+
for (Object header : httpHeaders.keySet()) {
165+
assertEquals(String.class, httpHeaders.get(header.toString()).getClass());
166+
}
167+
}
168+
169+
public void testMultipleHeaders() throws IOException, ResourceParseException {
170+
MetaData m = readNextWARCResponseAsMetaData(testFilePaths[1]);
171+
172+
JSONObject warcHeaders = JSONUtils.extractObject(m, "Envelope.WARC-Header-Metadata");
173+
JSONObject httpHeaders = JSONUtils.extractObject(m, "Envelope.Payload-Metadata.HTTP-Response-Metadata.Headers");
174+
175+
assertEquals("https://www.example.com/index.html/", warcHeaders.getString("WARC-Target-URI"));
176+
assertEquals(JSONArray.class, warcHeaders.get("WARC-Protocol").getClass());
177+
assertEquals(2, ((JSONArray) warcHeaders.get("WARC-Protocol")).length());
178+
assertEquals("h2", ((JSONArray) warcHeaders.get("WARC-Protocol")).get(0));
179+
180+
assertEquals("108", httpHeaders.getString("Content-Length"));
181+
assertEquals(JSONArray.class, httpHeaders.get("x-powered-by").getClass());
182+
assertEquals(2, ((JSONArray) httpHeaders.get("x-powered-by")).length());
183+
assertEquals("PHP/8.3.11", ((JSONArray) httpHeaders.get("x-powered-by")).get(0));
184+
assertEquals("PleskLin", ((JSONArray) httpHeaders.get("x-powered-by")).get(1));
185+
}
186+
}

‎src/test/resources/org/archive/format/warc/mutliple-headers.warc‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
WARC/1.0
2+
WARC-Type: response
3+
WARC-Date: 2024-09-27T10:47:02Z
4+
WARC-Record-ID: <urn:uuid:7a10b628-4d3b-6f2e-8b73-c65d80646310>
5+
Content-Length: 971
6+
Content-Type: application/http; msgtype=response
7+
WARC-Warcinfo-ID: <urn:uuid:824d10d3-4f67-131a-9cbf-e40ecb5f0fa5>
8+
WARC-Concurrent-To: <urn:uuid:51776b84-429e-53cb-a335-b53cf855c57a>
9+
WARC-IP-Address: 172.67.184.105
10+
WARC-Target-URI: https://www.example.com/index.html/
11+
WARC-Protocol: h2
12+
WARC-Protocol: tls/1.3
13+
WARC-Cipher-Suite: TLS_AES_256_GCM_SHA384
14+
WARC-Payload-Digest: sha1:70FB81039DCE25916E0E0CB48CF6662E3F27FFFC
15+
WARC-Block-Digest: sha1:80573371A8271BE6B3AA26FD9DB72E9AD9F316D9
16+
WARC-Identified-Payload-Type: text/html
17+
18+
HTTP/1.1 200
19+
date: Fri, 27 Sep 2024 10:47:02 GMT
20+
content-type: text/html; charset=UTF-8
21+
x-powered-by: PHP/8.3.11
22+
x-powered-by: PleskLin
23+
x-pingback: https://www.example.com/xmlrpc.php
24+
link: <https://www.example.com/wp-json/>; rel="https://api.w.org/"
25+
link: <https://www.example.com/wp-json/wp/v2/posts/00000>; rel="alternate"; title="JSON"; type="application/json"
26+
link: <https://www.example.com/?p=00000>; rel=shortlink
27+
x-litespeed-cache: miss
28+
vary: Accept-Encoding
29+
x-turbo-charged-by: LiteSpeed
30+
cf-cache-status: DYNAMIC
31+
report-to: {"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v4?s=XXtestYY"}],"group":"cf-nel","max_age":604800}
32+
nel: {"success_fraction":0,"report_to":"cf-nel","max_age":604800}
33+
server: cloudflare
34+
cf-ray: 8bf61e4afb9e7f9e-IAD
35+
X-Crawler-content-encoding: br
36+
alt-svc: h3=":443"; ma=86400
37+
Content-Length: 108
38+
39+
<!DOCTYPE html>
40+
<html lang="en">
41+
<head>
42+
<meta charset="UTF-8">
43+
<title>Test</title>
44+
</head>
45+
<body/>
46+
</html>
47+

0 commit comments

Comments
 (0)