From dadf5514ace51f3921b83ba66175f9537fe02160 Mon Sep 17 00:00:00 2001 From: Jordan Mendelson Date: Mon, 25 Nov 2013 21:44:19 -0800 Subject: [PATCH 01/83] Write out WET files (warc encapsulated text) using the same parser that we have already run --- .../ExtractingResourceFactoryMapper.java | 7 +- .../org/archive/extract/ProducerUtils.java | 2 +- .../archive/extract/ResourceExtractor.java | 5 + .../archive/extract/WATExtractorOutput.java | 32 +++- .../archive/extract/WETExtractorOutput.java | 167 ++++++++++++++++++ .../org/archive/format/json/JSONUtils.java | 10 ++ .../archive/format/warc/WARCConstants.java | 4 +- .../archive/format/warc/WARCRecordWriter.java | 40 ++++- .../archive/hadoop/ResourceRecordReader.java | 18 +- .../archive/resource/ResourceConstants.java | 2 +- .../html/ExtractingParseObserver.java | 47 ++++- .../archive/resource/html/HTMLMetaData.java | 4 + .../http/HTTPResponseResourceFactory.java | 1 - .../resource/warc/WARCResourceFactory.java | 1 - 14 files changed, 312 insertions(+), 28 deletions(-) create mode 100644 src/main/java/org/archive/extract/WETExtractorOutput.java diff --git a/src/main/java/org/archive/extract/ExtractingResourceFactoryMapper.java b/src/main/java/org/archive/extract/ExtractingResourceFactoryMapper.java index ad10be40..99a93d50 100644 --- a/src/main/java/org/archive/extract/ExtractingResourceFactoryMapper.java +++ b/src/main/java/org/archive/extract/ExtractingResourceFactoryMapper.java @@ -152,8 +152,11 @@ private boolean isWARCInfoResource(MetaData envelope) { } private boolean isHTTPResponseWARCResource(MetaData envelope) { return childFieldEquals(envelope,WARC_HEADER_METADATA, - WARCConstants.CONTENT_TYPE, - WARCConstants.HTTP_RESPONSE_MIMETYPE); + WARCConstants.CONTENT_TYPE, + WARCConstants.HTTP_RESPONSE_MIMETYPE) || + childFieldEquals(envelope,WARC_HEADER_METADATA, + WARCConstants.CONTENT_TYPE, + WARCConstants.HTTP_RESPONSE_MIMETYPE_NS); } private boolean isWARCJSONResource(MetaData envelope) { return childFieldEquals(envelope,WARC_HEADER_METADATA, diff --git a/src/main/java/org/archive/extract/ProducerUtils.java b/src/main/java/org/archive/extract/ProducerUtils.java index b75d2f15..666b0714 100644 --- a/src/main/java/org/archive/extract/ProducerUtils.java +++ b/src/main/java/org/archive/extract/ProducerUtils.java @@ -29,7 +29,7 @@ public static ResourceProducer getProducer(String path, long offset) throws IOEx wf.setStrict(STRICT_GZ); File file = new File(path); - if(path.startsWith("hdfs://")) { + if(path.startsWith("hdfs://") || path.startsWith("s3a://")) { String name = file.getName(); Path fsPath = new Path(path); FileSystem fs = fsPath.getFileSystem(new Configuration()); diff --git a/src/main/java/org/archive/extract/ResourceExtractor.java b/src/main/java/org/archive/extract/ResourceExtractor.java index 7f4d6e7a..ff7d8c50 100644 --- a/src/main/java/org/archive/extract/ResourceExtractor.java +++ b/src/main/java/org/archive/extract/ResourceExtractor.java @@ -53,6 +53,8 @@ private static int USAGE(int exitCode) { System.err.println("\t\t\t (note that column 1 is NOT standard Wayback canonicalized)\n"); System.err.println("\t\t-wat\tembed JSON output in a compressed WARC" + "wrapper, for storage, or sharing."); + System.err.println("\t\t-wet\tembed text extracts in a compressed WARC" + + "wrapper, for storage, or sharing."); return exitCode; } @@ -101,6 +103,9 @@ public int run(String[] args) } else if(args[arg].equals("-wat")) { path = args[arg+1]; out = new WATExtractorOutput(os); + } else if(args[arg].equals("-wet")) { + path = args[arg+1]; + out = new WETExtractorOutput(os); } else { String filter = args[arg+1]; out = new JSONViewExtractorOutput(os, filter); diff --git a/src/main/java/org/archive/extract/WATExtractorOutput.java b/src/main/java/org/archive/extract/WATExtractorOutput.java index f4d27147..ee803672 100644 --- a/src/main/java/org/archive/extract/WATExtractorOutput.java +++ b/src/main/java/org/archive/extract/WATExtractorOutput.java @@ -29,11 +29,17 @@ public class WATExtractorOutput implements ExtractorOutput { private static int DEFAULT_BUFFER_RAM = 1024 * 1024; private int bufferRAM = DEFAULT_BUFFER_RAM; private final static Charset UTF8 = Charset.forName("UTF-8"); - - public WATExtractorOutput(OutputStream out) { + private String outFilename; + + public WATExtractorOutput(OutputStream out) { + this(out, null); + } + + public WATExtractorOutput(OutputStream out, String filename) { gzW = new GZIPMemberWriter(out); recW = new WARCRecordWriter(); wroteFirst = false; + outFilename = filename; } private CommitedOutputStream getOutput() { @@ -55,6 +61,11 @@ public void output(Resource resource) throws IOException { // hrm... throw new IOException("Missing Envelope.Format"); } + + // remove the text extracts if it exists + JSONUtils.removeObject(top, "Envelope.Payload-Metadata.HTTP-Response-Metadata.HTML-Metadata", "Text"); + + cos = getOutput(); if(envelopeFormat.equals("ARC")) { writeARC(cos,top); @@ -68,16 +79,23 @@ public void output(Resource resource) throws IOException { } private void writeWARCInfo(OutputStream recOut, MetaData md) throws IOException { - String filename = JSONUtils.extractSingle(md, "Container.Filename"); - if(filename == null) { - throw new IOException("No Container.Filename..."); - } + String filename = outFilename; + + if (filename == null) { + filename = JSONUtils.extractSingle(md, "Container.Filename"); + + if(filename == null) { + throw new IOException("No Container.Filename..."); + } + } + HttpHeaders headers = new HttpHeaders(); headers.add("Software-Info", IAUtils.COMMONS_VERSION); headers.addDateHeader("Extracted-Date", new Date()); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); headers.write(baos); - recW.writeWARCInfoRecord(recOut,filename,baos.toByteArray()); + recW.writeWARCInfoRecord(recOut,filename,baos.toByteArray()); } private String extractOrIO(MetaData md, String path) throws IOException { diff --git a/src/main/java/org/archive/extract/WETExtractorOutput.java b/src/main/java/org/archive/extract/WETExtractorOutput.java new file mode 100644 index 00000000..b306f59b --- /dev/null +++ b/src/main/java/org/archive/extract/WETExtractorOutput.java @@ -0,0 +1,167 @@ +package org.archive.extract; + +import org.archive.format.gzip.GZIPMemberWriter; +import org.archive.format.gzip.GZIPMemberWriterCommittedOutputStream; +import org.archive.format.http.HttpHeaders; +import org.archive.format.json.JSONUtils; +import org.archive.format.warc.WARCRecordWriter; +import org.archive.resource.MetaData; +import org.archive.resource.Resource; +import org.archive.util.DateUtils; +import org.archive.util.IAUtils; +import org.archive.util.StreamCopy; +import org.archive.util.io.CommitedOutputStream; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.Charset; +import java.text.ParseException; +import java.util.Date; + +/** + * This is for generating a WARC Encapsulated Text file + * + * These are implemented as WARC conversion records. Only + * Envelope.Payload-Metadata.HTTP-Response-Metadata.HTML-Metadata.Text fields are included + */ +public class WETExtractorOutput implements ExtractorOutput { + WARCRecordWriter recW; + private boolean wroteFirst; + private GZIPMemberWriter gzW; + private static int DEFAULT_BUFFER_RAM = 1024 * 1024; + private int bufferRAM = DEFAULT_BUFFER_RAM; + private final static Charset UTF8 = Charset.forName("UTF-8"); + private String outFilename; + + public WETExtractorOutput(OutputStream out) { + this(out, null); + } + + public WETExtractorOutput(OutputStream out, String filename) { + gzW = new GZIPMemberWriter(out); + recW = new WARCRecordWriter(); + wroteFirst = false; + outFilename = filename; + } + + private CommitedOutputStream getOutput() { + return new GZIPMemberWriterCommittedOutputStream(gzW,bufferRAM); + } + + + private String extractOrIO(MetaData md, String path) throws IOException { + String value = JSONUtils.extractSingle(md, path); + if(value == null) { + throw new IOException("No "+path+" found."); + } + return value; + } + + public void output(Resource resource) throws IOException { + StreamCopy.readToEOF(resource.getInputStream()); + MetaData top = resource.getMetaData().getTopMetaData(); + CommitedOutputStream cos; + + if(!wroteFirst) { + cos = getOutput(); + writeWARCInfo(cos, top); + cos.commit(); + wroteFirst = true; + } + String envelopeFormat = JSONUtils.extractSingle(top, "Envelope.Format"); + if(envelopeFormat == null) { + throw new IOException("Missing Envelope.Format"); + } + + String warctype = JSONUtils.extractSingle(top, "Envelope.WARC-Header-Metadata.WARC-Type"); + if (warctype != null && warctype.equals("response")) { + String textExtract = JSONUtils.extractSingle(top, "Envelope.Payload-Metadata.HTTP-Response-Metadata.HTML-Metadata.Text"); + + if (textExtract != null) { + cos = getOutput(); + if(envelopeFormat.equals("WARC")) { + writeWARC(cos, top, textExtract); + } else { + // hrm... + throw new IOException("Unknown Envelope.Format"); + } + cos.commit(); + } + } + } + + private void writeWARCInfo(OutputStream recOut, MetaData md) throws IOException { + String filename = outFilename; + + if (filename == null) { + filename = JSONUtils.extractSingle(md, "Container.Filename"); + + if(filename == null) { + throw new IOException("No Container.Filename..."); + } + } + + HttpHeaders headers = new HttpHeaders(); + headers.add("Software-Info", IAUtils.COMMONS_VERSION); + headers.addDateHeader("Extracted-Date", new Date()); + + // Dup out some useful headers from the incoming warcinfo + String warctype = JSONUtils.extractSingle(md, "Envelope.WARC-Header-Metadata.WARC-Type"); + if (warctype != null && warctype.equals("warcinfo")) { + final String[] usefulHeaders = {"robots", "isPartOf", "operator", "description", "publisher"}; + + for (String header : usefulHeaders) { + String value = JSONUtils.extractSingle(md, "Envelope.Payload-Metadata.WARC-Info-Metadata." + header); + if (value != null) { + headers.add(header, value); + } + } + } + + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + headers.write(baos); + recW.writeWARCInfoRecord(recOut, filename, baos.toByteArray()); + } + + private void writeWARC(OutputStream recOut, MetaData md, String textExtract) throws IOException { + String targetURI = extractOrIO(md, "Envelope.WARC-Header-Metadata.WARC-Target-URI"); + + String capDateString = extractOrIO(md, "Envelope.WARC-Header-Metadata.WARC-Date"); + capDateString = transformWARCDate(capDateString); + String recId = extractOrIO(md, "Envelope.WARC-Header-Metadata.WARC-Record-ID"); + writeWARCMDRecord(recOut, targetURI, capDateString, recId, textExtract); + } + + private void writeWARCMDRecord(OutputStream recOut, String targetURI, String capDateString, String recId, + String textExtract) + throws IOException { + + Date capDate; + try { + capDate = DateUtils.getSecondsSinceEpoch(capDateString); + + } catch (ParseException e) { + e.printStackTrace(); + // TODO... not the write thing... + capDate = new Date(); + } + + recW.writeTextConversionRecord(recOut, textExtract.getBytes("UTF-8"), targetURI, capDate, recId); + } + + private static String transformWARCDate(final String input) { + + StringBuilder output = new StringBuilder(14); + + output.append(input.substring(0,4)); + output.append(input.substring(5,7)); + output.append(input.substring(8,10)); + output.append(input.substring(11,13)); + output.append(input.substring(14,16)); + output.append(input.substring(17,19)); + + return output.toString(); + } +} diff --git a/src/main/java/org/archive/format/json/JSONUtils.java b/src/main/java/org/archive/format/json/JSONUtils.java index 28f4f43e..946b633b 100644 --- a/src/main/java/org/archive/format/json/JSONUtils.java +++ b/src/main/java/org/archive/format/json/JSONUtils.java @@ -114,4 +114,14 @@ private static void extractRecursive(JSONObject json, String path[], int idx, Li } } } + public static boolean removeObject(JSONObject json, String path, String node) { + JSONObject obj = extractObject(json, "Envelope.Payload-Metadata.HTTP-Response-Metadata.HTML-Metadata"); + if (obj != null) { + if (obj.remove("Text") != null) { + return true; + } + } + + return false; + } } diff --git a/src/main/java/org/archive/format/warc/WARCConstants.java b/src/main/java/org/archive/format/warc/WARCConstants.java index c9f6cbf3..6e625183 100644 --- a/src/main/java/org/archive/format/warc/WARCConstants.java +++ b/src/main/java/org/archive/format/warc/WARCConstants.java @@ -211,7 +211,9 @@ enum WARCRecordType { "application/http; msgtype=request"; public static final String HTTP_RESPONSE_MIMETYPE = "application/http; msgtype=response"; - + public static final String HTTP_RESPONSE_MIMETYPE_NS = + "application/http;msgtype=response"; // wget does this + public static final String FTP_CONTROL_CONVERSATION_MIMETYPE = "text/x-ftp-control-conversation"; diff --git a/src/main/java/org/archive/format/warc/WARCRecordWriter.java b/src/main/java/org/archive/format/warc/WARCRecordWriter.java index 0aab83b7..ae6d8d67 100644 --- a/src/main/java/org/archive/format/warc/WARCRecordWriter.java +++ b/src/main/java/org/archive/format/warc/WARCRecordWriter.java @@ -2,18 +2,33 @@ import java.io.IOException; import java.io.OutputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.Date; import java.util.UUID; import org.archive.format.http.HttpConstants; import org.archive.format.http.HttpHeaders; +import org.archive.util.Base32; import org.archive.util.DateUtils; public class WARCRecordWriter implements WARCConstants, HttpConstants { private static final String SCHEME = "urn:uuid"; private static final String SCHEME_COLON = SCHEME + ":"; - + private MessageDigest sha1; + private Base32 base32; + + public WARCRecordWriter() { + try { + sha1 = MessageDigest.getInstance("SHA1"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + + base32 = new Base32(); + } + /** * Write the headers and contents as a WARC record to the given * output stream. @@ -97,6 +112,29 @@ public void writeJSONMetadataRecord( OutputStream out, writeRecord(out, headers, contents); } + public void writeTextConversionRecord( OutputStream out, + byte[] contents, + String targetURI, + Date originalDate, + String origRecordId) throws IOException + { + HttpHeaders headers = new HttpHeaders(); + headers.add(HEADER_KEY_TYPE, WARCRecordType.conversion.name()); + headers.add(HEADER_KEY_URI, targetURI); + headers.add(HEADER_KEY_DATE, DateUtils.getLog14Date(originalDate)); + headers.add(HEADER_KEY_ID, makeRecordId()); + headers.add(HEADER_KEY_REFERS_TO, origRecordId); + headers.add(HEADER_KEY_BLOCK_DIGEST, contentHash(contents)); + + headers.add(CONTENT_TYPE, "text/plain"); + writeRecord(out, headers, contents); + } + + private String contentHash(byte[] content) { + sha1.reset(); + return "sha1:" + base32.encode(sha1.digest(content)); + } + private String makeRecordId() { StringBuilder recID = new StringBuilder(); diff --git a/src/main/java/org/archive/hadoop/ResourceRecordReader.java b/src/main/java/org/archive/hadoop/ResourceRecordReader.java index 06d3ce2e..933c4f28 100644 --- a/src/main/java/org/archive/hadoop/ResourceRecordReader.java +++ b/src/main/java/org/archive/hadoop/ResourceRecordReader.java @@ -75,16 +75,16 @@ public void initialize(InputSplit inputSplit, TaskAttemptContext context) if(inputSplit instanceof FileSplit) { FileSplit fs = (FileSplit) inputSplit; Path fsPath = fs.getPath(); - FileSystem fSys = fsPath.getFileSystem(context.getConfiguration()); - FSDataInputStream fsdis = fSys.open(fsPath); - String path = fsPath.getName(); - name = fsPath.getName(); - stream = new HDFSStream(fsdis); - startOffset = fs.getStart(); + FileSystem fSys = fsPath.getFileSystem(context.getConfiguration()); + FSDataInputStream fsdis = fSys.open(fsPath); + String path = fsPath.getName(); + name = fsPath.getName(); + stream = new HDFSStream(fsdis); + startOffset = fs.getStart(); length = fs.getLength(); long endOffset = startOffset + length; stream.setOffset(startOffset); - series = new GZIPMemberSeries(stream, name, startOffset); + series = new GZIPMemberSeries(stream, name, startOffset); GZIPResourceContainer prod = new GZIPResourceContainer(series,endOffset); ResourceProducer envelope; @@ -95,8 +95,8 @@ public void initialize(InputSplit inputSplit, TaskAttemptContext context) } else { throw new IOException("arguments must be arc.gz or warc.gz"); } - ResourceFactoryMapper mapper = new ExtractingResourceFactoryMapper(); - producer = new ExtractingResourceProducer(envelope, mapper); + ResourceFactoryMapper mapper = new ExtractingResourceFactoryMapper(); + producer = new ExtractingResourceProducer(envelope, mapper); } else { throw new IOException("Need FileSplit input..."); diff --git a/src/main/java/org/archive/resource/ResourceConstants.java b/src/main/java/org/archive/resource/ResourceConstants.java index dd04fcfe..ccc587e2 100644 --- a/src/main/java/org/archive/resource/ResourceConstants.java +++ b/src/main/java/org/archive/resource/ResourceConstants.java @@ -114,5 +114,5 @@ public interface ResourceConstants { public static final String HTML_LINK_TAGS = "Link"; public static final String HTML_META_TAGS = "Metas"; public static final String HTML_SCRIPT_TAGS = "Scripts"; - + public static final String HTML_TEXT = "Text"; } diff --git a/src/main/java/org/archive/resource/html/ExtractingParseObserver.java b/src/main/java/org/archive/resource/html/ExtractingParseObserver.java index e1f57b55..6a1f02e9 100644 --- a/src/main/java/org/archive/resource/html/ExtractingParseObserver.java +++ b/src/main/java/org/archive/resource/html/ExtractingParseObserver.java @@ -11,14 +11,17 @@ import org.htmlparser.nodes.RemarkNode; import org.htmlparser.nodes.TagNode; import org.htmlparser.nodes.TextNode; +import org.htmlparser.util.Translate; public class ExtractingParseObserver implements ParseObserver { HTMLMetaData data; Stack> openAnchors; Stack openAnchorTexts; + StringBuffer textExtract; String title = null; boolean inTitle = false; + boolean inPre = false; protected static String cssUrlPatString = "url\\s*\$\\s*([\\\\\"']*.+?[\\\\\"']*)\\s*\$"; @@ -59,6 +62,7 @@ public ExtractingParseObserver(HTMLMetaData data) { this.data = data; openAnchors = new Stack>(); openAnchorTexts = new Stack(); + textExtract = new StringBuffer(8192); } public void handleDocumentStart() { @@ -66,7 +70,10 @@ public void handleDocumentStart() { } public void handleDocumentComplete() { - // no-op + if (textExtract.length() > 0) { + data.setTextExtract(textExtract.toString()); + textExtract = new StringBuffer(8192); + } } public void handleTagEmpty(TagNode tag) { @@ -78,7 +85,10 @@ public void handleTagOpen(TagNode tag) { if(name.equals("TITLE")) { inTitle = !tag.isEmptyXmlTag(); return; - } + } else if (name.equals("PRE")) { + inPre = true; + } + // first the global attributes: // background String v = tag.getAttribute("background"); @@ -101,6 +111,7 @@ public void handleTagClose(TagNode tag) { // probably the right thing.. return; } + // Only interesting if it's a : if(tag.getTagName().equals("A")) { if(openAnchors.size() > 0) { @@ -122,13 +133,41 @@ public void handleTagClose(TagNode tag) { data.addHref(vals); } } - } + } else if (tag.getTagName().equals("PRE")) { + inPre = false; + } } public void handleTextNode(TextNode text) { // TODO: OPTIMIZ: This can be a lot smarter, if StringBuilders are full, // this result is thrown away. - String t = text.getText().replaceAll("\\s+", " "); + //System.out.println("JDBUG: Got text from node: " + text.getText().toString()); + + String txt = text.getText(); + if (!inPre) { + txt = Translate.decode(txt); + txt = txt.replace('\u00a0', ' '); + + char c = ' '; + if (textExtract.length() > 0) { + c = textExtract.charAt(textExtract.length()-1); + } + for (int i = 0; i < txt.length(); i++) { + char c2 = txt.charAt(i); + // Translate so output is a bit cleaner + if (c2 == '\r') { + c2 = '\n'; + } + if (!Character.isWhitespace(c) || !Character.isWhitespace(c2)) { + textExtract.append(c2); + } + c = c2; + } + } + else + textExtract.append(txt); + + String t = text.getText().replaceAll("\\s+", " "); if(t.length() > MAX_TEXT_LEN) { t = t.substring(0,MAX_TEXT_LEN); diff --git a/src/main/java/org/archive/resource/html/HTMLMetaData.java b/src/main/java/org/archive/resource/html/HTMLMetaData.java index 024d9677..b9dcc7ea 100644 --- a/src/main/java/org/archive/resource/html/HTMLMetaData.java +++ b/src/main/java/org/archive/resource/html/HTMLMetaData.java @@ -69,6 +69,10 @@ public void addHref(String...a) { appendObj2(this,HTML_LINKS,a); } + public void setTextExtract(String textExtract) { + putUnlessNull(this,HTML_TEXT, textExtract); + } + private void appendObj2(JSONObject o, String arr, String... a) { if(o == null) { return; diff --git a/src/main/java/org/archive/resource/http/HTTPResponseResourceFactory.java b/src/main/java/org/archive/resource/http/HTTPResponseResourceFactory.java index 135691b6..c2eb7b05 100644 --- a/src/main/java/org/archive/resource/http/HTTPResponseResourceFactory.java +++ b/src/main/java/org/archive/resource/http/HTTPResponseResourceFactory.java @@ -23,7 +23,6 @@ public Resource getResource(InputStream is, MetaData metaData, ResourceContainer container) throws ResourceParseException, IOException { try { - HttpResponse response = parser.parse(is); metaData.putString(PAYLOAD_CONTENT_TYPE, PAYLOAD_TYPE_HTTP_RESPONSE); diff --git a/src/main/java/org/archive/resource/warc/WARCResourceFactory.java b/src/main/java/org/archive/resource/warc/WARCResourceFactory.java index 14bd53e1..137ee0ff 100644 --- a/src/main/java/org/archive/resource/warc/WARCResourceFactory.java +++ b/src/main/java/org/archive/resource/warc/WARCResourceFactory.java @@ -23,7 +23,6 @@ public Resource getResource(InputStream is, MetaData parentMetaData, ResourceContainer container) throws ResourceParseException, IOException { try { - HttpResponse response = parser.parse(is); WARCResource r = new WARCResource(parentMetaData.createChild(ENVELOPE), container, response); From 347e6f133bffbf8c0fd7558141fedc0e67d7b8ae Mon Sep 17 00:00:00 2001 From: Jordan Mendelson Date: Wed, 5 Feb 2014 11:04:51 -0800 Subject: [PATCH 02/83] Hack to limit CSS size to 100K. --- .../resource/html/ExtractingParseObserver.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/archive/resource/html/ExtractingParseObserver.java b/src/main/java/org/archive/resource/html/ExtractingParseObserver.java index 6a1f02e9..23f52473 100644 --- a/src/main/java/org/archive/resource/html/ExtractingParseObserver.java +++ b/src/main/java/org/archive/resource/html/ExtractingParseObserver.java @@ -408,9 +408,18 @@ public void extract(HTMLMetaData data, TagNode node, ExtractingParseObserver obs } } private void patternCSSExtract(HTMLMetaData data, Pattern pattern, String content) { - Matcher m = pattern.matcher(content); + String newcontent; + int contentLen = content.length(); + if (contentLen > 100000) { + newcontent = content.substring(100000); + contentLen = newcontent.length(); + } else { + newcontent = content; + } + + Matcher m = pattern.matcher(newcontent); int idx = 0; - int contentLen = content.length(); + while((idx < contentLen) && m.find(idx)) { String url = m.group(1); int origUrlLength = url.length(); From 04d556c9747679f9dc5304914cbbf13eb713431b Mon Sep 17 00:00:00 2001 From: Jordan Mendelson Date: Wed, 19 Mar 2014 23:21:03 -0700 Subject: [PATCH 03/83] htmlparser bump to 2.1 to fix some serious bugs with jsp parsing --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 03b1240d..6860ed02 100644 --- a/pom.xml +++ b/pom.xml @@ -38,7 +38,7 @@ org.htmlparser htmlparser - 1.6 + 2.1 From edc38202a347307595d6d6b6cc596974b09a96f3 Mon Sep 17 00:00:00 2001 From: Jordan Mendelson Date: Thu, 26 Jun 2014 12:46:38 -0700 Subject: [PATCH 04/83] If there is an error parsing a resource, return the version one level higher (ie, http response instead of html response) --- .../extract/ExtractingResourceProducer.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/archive/extract/ExtractingResourceProducer.java b/src/main/java/org/archive/extract/ExtractingResourceProducer.java index de671bee..ccfd1ee6 100644 --- a/src/main/java/org/archive/extract/ExtractingResourceProducer.java +++ b/src/main/java/org/archive/extract/ExtractingResourceProducer.java @@ -37,8 +37,18 @@ public Resource getNext() throws ResourceParseException, IOException { current.getClass().toString(), f.getClass().toString())); } - current = f.getResource(current.getInputStream(), - current.getMetaData(), current.getContainer()); + + Resource previous = current; + try { + current = f.getResource(current.getInputStream(), + current.getMetaData(), current.getContainer()); + } catch (ResourceParseException e) { + if(LOG.isLoggable(Level.WARNING)) { + LOG.warning("Error creating resource, returning more generic version: " + e); + } + // If we end up with some kind of parse error, return the resource one level higher + return previous; + } } } From 7eddd644160a30d174423f142b266f6b99c24864 Mon Sep 17 00:00:00 2001 From: Jordan Mendelson Date: Thu, 26 Jun 2014 15:02:18 -0700 Subject: [PATCH 05/83] Remove the code that returns an easier version if there was a parse error since it was caused by a s3 problem. --- .../org/archive/extract/ExtractingResourceProducer.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/archive/extract/ExtractingResourceProducer.java b/src/main/java/org/archive/extract/ExtractingResourceProducer.java index ccfd1ee6..0e938579 100644 --- a/src/main/java/org/archive/extract/ExtractingResourceProducer.java +++ b/src/main/java/org/archive/extract/ExtractingResourceProducer.java @@ -37,7 +37,8 @@ public Resource getNext() throws ResourceParseException, IOException { current.getClass().toString(), f.getClass().toString())); } - + + /* Resource previous = current; try { current = f.getResource(current.getInputStream(), @@ -49,6 +50,10 @@ public Resource getNext() throws ResourceParseException, IOException { // If we end up with some kind of parse error, return the resource one level higher return previous; } + */ + current = f.getResource(current.getInputStream(), + current.getMetaData(), current.getContainer()); + } } From cd7b7d96c6a3fefeb58ecaffb5528824821426d6 Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Tue, 5 Jul 2016 08:17:03 +0200 Subject: [PATCH 06/83] Support for file:/ URLs which are erroneously interpreted as relative paths /current_dir/file:/path/file.warc.gz --- src/main/java/org/archive/extract/ProducerUtils.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/archive/extract/ProducerUtils.java b/src/main/java/org/archive/extract/ProducerUtils.java index 666b0714..d8db9630 100644 --- a/src/main/java/org/archive/extract/ProducerUtils.java +++ b/src/main/java/org/archive/extract/ProducerUtils.java @@ -65,7 +65,15 @@ public static ResourceProducer getProducer(String path, long offset) throws IOEx } else { - if(!(file.exists() && file.canRead())) { + if(path.startsWith("file:/")) { + file = new File(new URL(path).getPath()); + } + + if(!file.exists()) { + System.err.println(path + ": file not found."); + return null; + } + if(!file.canRead()) { System.err.println(path + " is not a readable file."); return null; } From 52bd2599fd5f7d954908ed97651617e57610953c Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Tue, 5 Jul 2016 09:33:38 +0200 Subject: [PATCH 07/83] Fixed unit test doubleToString which fails on Java 8 OpenJDK because of a fix when rounding doubles cf. https://bugs.openjdk.java.net/browse/JDK-7131459 Test double value not close to a tie (1.344 instead of 1.345) --- src/test/java/org/archive/util/ArchiveUtilsTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/archive/util/ArchiveUtilsTest.java b/src/test/java/org/archive/util/ArchiveUtilsTest.java index 8251615a..e74763b3 100644 --- a/src/test/java/org/archive/util/ArchiveUtilsTest.java +++ b/src/test/java/org/archive/util/ArchiveUtilsTest.java @@ -229,7 +229,7 @@ public void testByteArrayEquals() { /** test doubleToString() */ public void testDoubleToString(){ - double test = 12.345; + double test = 12.344d; assertTrue( "cecking zero precision", ArchiveUtils.doubleToString(test, 0).equals("12")); @@ -238,7 +238,7 @@ public void testDoubleToString(){ ArchiveUtils.doubleToString(test, 2).equals("12.34")); assertTrue( "cecking precision higher then the double has", - ArchiveUtils.doubleToString(test, 65).equals("12.345")); + ArchiveUtils.doubleToString(test, 65).equals("12.344")); } From adc1345e6c65c1cee0c2c5ec8fd3f547a12e1b5c Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Tue, 5 Jul 2016 10:03:10 +0200 Subject: [PATCH 08/83] Merge solution from iipc/webarchive-commons for double rounding problem, see https://github.com/iipc/webarchive-commons/pull/33 --- .../java/org/archive/util/ArchiveUtilsTest.java | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/test/java/org/archive/util/ArchiveUtilsTest.java b/src/test/java/org/archive/util/ArchiveUtilsTest.java index e74763b3..586a1821 100644 --- a/src/test/java/org/archive/util/ArchiveUtilsTest.java +++ b/src/test/java/org/archive/util/ArchiveUtilsTest.java @@ -229,16 +229,19 @@ public void testByteArrayEquals() { /** test doubleToString() */ public void testDoubleToString(){ - double test = 12.344d; - assertTrue( + double test = 12.121d; + assertEquals( "cecking zero precision", - ArchiveUtils.doubleToString(test, 0).equals("12")); - assertTrue( + "12", + ArchiveUtils.doubleToString(test, 0)); + assertEquals( "cecking 2 character precision", - ArchiveUtils.doubleToString(test, 2).equals("12.34")); - assertTrue( + "12.12", + ArchiveUtils.doubleToString(test, 2)); + assertEquals( "cecking precision higher then the double has", - ArchiveUtils.doubleToString(test, 65).equals("12.344")); + "12.121", + ArchiveUtils.doubleToString(test, 65)); } From b9b9b8af43b1b33fa224486d98b110df859817ed Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Tue, 5 Jul 2016 10:46:28 +0200 Subject: [PATCH 09/83] Fix StringIndexOutOfBoundsException in WAT/WET generation (fixes #1) - correct check for min. required URL lenght when stripping 4 characters (2 at each end) - simplified code in method patternCSSExtract - in case CSS is larger than 100 kB: process first 100kB and not everything else except the first 100kB (this wasn't the original intention, probably) - improved regular expression matching URLs in CSS: use non-capturing groups --- .../html/ExtractingParseObserver.java | 47 ++++++------------- 1 file changed, 15 insertions(+), 32 deletions(-) diff --git a/src/main/java/org/archive/resource/html/ExtractingParseObserver.java b/src/main/java/org/archive/resource/html/ExtractingParseObserver.java index 23f52473..b2fc99a6 100644 --- a/src/main/java/org/archive/resource/html/ExtractingParseObserver.java +++ b/src/main/java/org/archive/resource/html/ExtractingParseObserver.java @@ -26,7 +26,7 @@ public class ExtractingParseObserver implements ParseObserver { protected static String cssUrlPatString = "url\\s*\$\\s*([\\\\\"']*.+?[\\\\\"']*)\\s*\$"; protected static String cssImportNoUrlPatString = - "@import\\s+(('[^']+')|(\"[^\"]+\")|(\$'[^']+'\$)|(\$\"[^\"]+\"\$)|(\$[^)]+\$)|([a-z0-9_.:/\\\\-]+))\\s*;"; + "@import\\s+((?:'[^']+')|(?:\"[^\"]+\")|(?:\$'[^']+'\$)|(?:\$\"[^\"]+\"\$)|(?:\$[^)]+\$)|(?:[a-z0-9_.:/\\\\-]+))\\s*;"; protected static Pattern cssImportNoUrlPattern = Pattern .compile(cssImportNoUrlPatString); @@ -408,48 +408,31 @@ public void extract(HTMLMetaData data, TagNode node, ExtractingParseObserver obs } } private void patternCSSExtract(HTMLMetaData data, Pattern pattern, String content) { - String newcontent; - int contentLen = content.length(); - if (contentLen > 100000) { - newcontent = content.substring(100000); - contentLen = newcontent.length(); - } else { - newcontent = content; - } - - Matcher m = pattern.matcher(newcontent); + Matcher m = pattern.matcher(content); int idx = 0; - - while((idx < contentLen) && m.find(idx)) { + int contentLen = content.length(); + if (contentLen > 100000) + // extract URLs only from the first 100 kB + contentLen = 100000; + while((idx < contentLen) && m.find()) { + idx = m.end(); String url = m.group(1); - int origUrlLength = url.length(); - int urlStart = m.start(1); - int urlEnd = m.end(1); - idx = urlEnd; if(url.length() < 2) { continue; } if ((url.charAt(0) == '(') - && (url.charAt(origUrlLength-1) == ')')) { - url = url.substring(1, origUrlLength - 1); - urlStart += 1; - origUrlLength -= 2; + && (url.charAt(url.length()-1) == ')')) { + url = url.substring(1, url.length() - 1); } - if (url.charAt(0) == '"') { - url = url.substring(1, origUrlLength - 1); - urlStart += 1; - } else if (url.charAt(0) == '\'') { - url = url.substring(1, origUrlLength - 1); - urlStart += 1; + if (url.charAt(0) == '"' || url.charAt(0) == '\'') { + url = url.substring(1, url.length() - 1); } else if (url.charAt(0) == '\\') { - if(url.length() == 2) + if(url.length() <= 4) { continue; - url = url.substring(2, origUrlLength - 2); - urlStart += 2; + } + url = url.substring(2, url.length() - 2); } - int urlLength = url.length(); data.addHref("path","STYLE/#text","href",url); - idx += urlLength; } } } From fbb6dd024dcf5c83e6c937b28010f03bcc5b0b51 Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Tue, 2 Aug 2016 17:31:16 +0200 Subject: [PATCH 10/83] pom.xml to build with CDH 5 --- pom-cdh5.xml | 229 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 229 insertions(+) create mode 100644 pom-cdh5.xml diff --git a/pom-cdh5.xml b/pom-cdh5.xml new file mode 100644 index 00000000..3619206d --- /dev/null +++ b/pom-cdh5.xml @@ -0,0 +1,229 @@ + + 4.0.0 + + org.archive + ia-web-commons + 1.0-SNAPSHOT + jar + + ia-web-commons + http://maven.apache.org + + + UTF-8 + ${maven.build.timestamp} + yyyyMMddhhmmss + + + + + junit + junit + 3.8.1 + test + + + + com.google.guava + guava + 14.0.1 + + + + org.json + json + 20090211 + + + org.htmlparser + htmlparser + 2.1 + + + + org.mozilla + juniversalchardet + 1.0.3 + + + + commons-httpclient + commons-httpclient + 3.1 + + + + org.apache.hadoop + hadoop-client + 2.6.0-cdh5.8.0 + + + commons-httpclient + commons-httpclient + + + javax.servlet + servlet-api + + + javax.servlet.jsp + jsp-api + + + org.mortbay.jetty + jetty + + + org.mortbay.jetty + jetty-util + + + tomcat + jasper-runtime + + + tomcat + jasper-compiler + + + + + org.apache.hadoop + hadoop-common + 2.6.0-cdh5.8.0 + + + org.apache.hadoop + hadoop-mapreduce-client-common + 2.6.0-cdh5.8.0 + + + org.apache.hadoop + hadoop-mapreduce-client-core + 2.6.0-cdh5.8.0 + + + + org.apache.pig + pig + 0.11.1 + provided + + + + commons-lang + commons-lang + 2.5 + + + + commons-io + commons-io + 2.4 + + + + org.gnu.inet + libidn + 1.15 + + + it.unimi.dsi + mg4j + 1.0.1 + compile + + + org.apache.httpcomponents + httpcore + 4.3 + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 2.3.2 + + 1.6 + 1.6 + + + + maven-assembly-plugin + 2.4 + + + jar-with-dependencies + + ia-web-commons + + + + package + + single + + + + + + + + src/main/resources + true + + + + + + + internetarchive + Internet Archive Maven Repository + http://builds.archive.org:8080/maven2 + default + + + true + daily + warn + + + true + daily + warn + + + + + cloudera + Cloudera Hadoop + https://repository.cloudera.com/artifactory/cloudera-repos/ + default + + + true + daily + warn + + + true + daily + warn + + + + + + + + repository + + ${repository.url} + + + + From e379dcc038150ed866c6825f875e2f2a1ee47fb0 Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Sun, 7 Aug 2016 16:37:30 +0200 Subject: [PATCH 11/83] Make regular expression to extract URLs from CSS more restrictive regarding leading and trailing quotes to avoid long-runners due to heavy back-tracking. This closes #2. --- .../java/org/archive/resource/html/ExtractingParseObserver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/archive/resource/html/ExtractingParseObserver.java b/src/main/java/org/archive/resource/html/ExtractingParseObserver.java index b2fc99a6..6e56a12d 100644 --- a/src/main/java/org/archive/resource/html/ExtractingParseObserver.java +++ b/src/main/java/org/archive/resource/html/ExtractingParseObserver.java @@ -24,7 +24,7 @@ public class ExtractingParseObserver implements ParseObserver { boolean inPre = false; protected static String cssUrlPatString = - "url\\s*\$\\s*([\\\\\"']*.+?[\\\\\"']*)\\s*\$"; + "url\\s*\$\\s*((?:\\\\?[\"'])?.+?(?:\\\\?[\"'])?)\\s*\$"; protected static String cssImportNoUrlPatString = "@import\\s+((?:'[^']+')|(?:\"[^\"]+\")|(?:\$'[^']+'\$)|(?:\$\"[^\"]+\"\$)|(?:\$[^)]+\$)|(?:[a-z0-9_.:/\\\\-]+))\\s*;"; From 3763ccbb2121a7d3745b0d5d1f381a5395658b6b Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Thu, 29 Sep 2016 11:44:18 +0200 Subject: [PATCH 12/83] Extract also `property` attributes of HTML meta elements, this fixes #3 --- .../java/org/archive/resource/html/ExtractingParseObserver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/archive/resource/html/ExtractingParseObserver.java b/src/main/java/org/archive/resource/html/ExtractingParseObserver.java index 6e56a12d..9e0f5c2f 100644 --- a/src/main/java/org/archive/resource/html/ExtractingParseObserver.java +++ b/src/main/java/org/archive/resource/html/ExtractingParseObserver.java @@ -388,7 +388,7 @@ public void extract(HTMLMetaData data, TagNode node, ExtractingParseObserver obs } private static class MetaTagExtractor implements TagExtractor { public void extract(HTMLMetaData data, TagNode node, ExtractingParseObserver obs) { - ArrayList l = getAttrList(node,"name","rel","content","http-equiv"); + ArrayList l = getAttrList(node,"name","rel","content","http-equiv","property"); if(l != null) { data.addMeta(l); } From f4ce8828ccee9ff85d5f77d860fc0c3e068caf7e Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Thu, 24 Nov 2016 11:52:27 +0100 Subject: [PATCH 13/83] Use CharsetDetector to guess encoding of HTML document, fixes #4 --- .../resource/html/HTMLResourceFactory.java | 32 +++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/archive/resource/html/HTMLResourceFactory.java b/src/main/java/org/archive/resource/html/HTMLResourceFactory.java index 935843f1..34062ed9 100644 --- a/src/main/java/org/archive/resource/html/HTMLResourceFactory.java +++ b/src/main/java/org/archive/resource/html/HTMLResourceFactory.java @@ -1,9 +1,14 @@ package org.archive.resource.html; +import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; import java.io.UnsupportedEncodingException; +import org.archive.format.http.HttpHeaders; +import org.archive.format.json.JSONUtils; +import org.archive.format.text.charset.CharsetDetector; +import org.archive.format.text.charset.StandardCharsetDetector; import org.archive.format.text.html.CDATALexer; import org.archive.format.text.html.LexParser; import org.archive.resource.MetaData; @@ -13,17 +18,40 @@ import org.archive.resource.ResourceParseException; import org.htmlparser.lexer.Page; import org.htmlparser.util.ParserException; +import org.json.JSONException; +import org.json.JSONObject; public class HTMLResourceFactory implements ResourceFactory { + protected static final int CHARSET_GUESS_CHUNK_SIZE = 8192; + protected static final String HTTP_HEADER_PATH = "Envelope.Payload-Metadata.HTTP-Response-Metadata.Headers"; + + protected CharsetDetector charSetDetector = new StandardCharsetDetector(); + + public Resource getResource(InputStream is, MetaData parentMetaData, ResourceContainer container) throws ResourceParseException, IOException { HTMLMetaData hmd = new HTMLMetaData(parentMetaData); ExtractingParseObserver epo = new ExtractingParseObserver(hmd); LexParser parser = new LexParser(epo); CDATALexer lex = new CDATALexer(); - // TODO: figure out charset: - String charset = "UTF-8"; + + // guess charset based on HTTP header and sniffed content chunk + is = new BufferedInputStream(is, CHARSET_GUESS_CHUNK_SIZE); + byte[] chunk = new byte[CHARSET_GUESS_CHUNK_SIZE]; + is.mark(0); + int chunkSize = is.read(chunk, 0, CHARSET_GUESS_CHUNK_SIZE); + is.reset(); + JSONObject headers = JSONUtils.extractObject(hmd.getTopMetaData(), HTTP_HEADER_PATH); + HttpHeaders httpHeaders = new HttpHeaders(); + if (headers.has("Content-Type")) { + try { + httpHeaders.add("Content-Type", headers.getString("Content-Type")); + } catch (JSONException e) { } + } + + String charset = charSetDetector.getCharset(chunk, chunkSize, httpHeaders); + Page page; try { page = new Page(is, charset); From a0427962f7d7996367f17232ee007775664fc4fa Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Thu, 24 Nov 2016 14:03:38 +0100 Subject: [PATCH 14/83] HTML encoding detection: fix errors with empty content or empty charset values --- .../format/text/charset/CharsetDetector.java | 2 ++ .../resource/html/HTMLResourceFactory.java | 25 +++++++++++++------ 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/archive/format/text/charset/CharsetDetector.java b/src/main/java/org/archive/format/text/charset/CharsetDetector.java index d391aac3..f550e342 100644 --- a/src/main/java/org/archive/format/text/charset/CharsetDetector.java +++ b/src/main/java/org/archive/format/text/charset/CharsetDetector.java @@ -179,6 +179,8 @@ protected String getCharsetFromMeta(byte buffer[],int len) throws IOException { private static String trimAttrValue(String value) { String result = value; + if (result.isEmpty()) + return result; if (result.charAt(0) == '"') { result = result.substring(1, result.length() - 1); } else if (result.charAt(0) == '\'') { diff --git a/src/main/java/org/archive/resource/html/HTMLResourceFactory.java b/src/main/java/org/archive/resource/html/HTMLResourceFactory.java index 34062ed9..32ffc143 100644 --- a/src/main/java/org/archive/resource/html/HTMLResourceFactory.java +++ b/src/main/java/org/archive/resource/html/HTMLResourceFactory.java @@ -5,6 +5,8 @@ import java.io.InputStream; import java.io.UnsupportedEncodingException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.archive.format.http.HttpHeaders; import org.archive.format.json.JSONUtils; import org.archive.format.text.charset.CharsetDetector; @@ -23,6 +25,8 @@ public class HTMLResourceFactory implements ResourceFactory { + public static final Log LOG = LogFactory.getLog(HTMLResourceFactory.class); + protected static final int CHARSET_GUESS_CHUNK_SIZE = 8192; protected static final String HTTP_HEADER_PATH = "Envelope.Payload-Metadata.HTTP-Response-Metadata.Headers"; @@ -37,21 +41,28 @@ public Resource getResource(InputStream is, MetaData parentMetaData, CDATALexer lex = new CDATALexer(); // guess charset based on HTTP header and sniffed content chunk + String charset = "UTF-8"; is = new BufferedInputStream(is, CHARSET_GUESS_CHUNK_SIZE); byte[] chunk = new byte[CHARSET_GUESS_CHUNK_SIZE]; is.mark(0); int chunkSize = is.read(chunk, 0, CHARSET_GUESS_CHUNK_SIZE); is.reset(); - JSONObject headers = JSONUtils.extractObject(hmd.getTopMetaData(), HTTP_HEADER_PATH); - HttpHeaders httpHeaders = new HttpHeaders(); - if (headers.has("Content-Type")) { + if (chunkSize > 0) { + JSONObject headers = JSONUtils.extractObject(hmd.getTopMetaData(), HTTP_HEADER_PATH); + HttpHeaders httpHeaders = new HttpHeaders(); + if (headers.has("Content-Type")) { + try { + httpHeaders.add("Content-Type", headers.getString("Content-Type")); + } catch (JSONException e) { } + } try { - httpHeaders.add("Content-Type", headers.getString("Content-Type")); - } catch (JSONException e) { } + charset = charSetDetector.getCharset(chunk, chunkSize, httpHeaders); + LOG.info("Guessed charset: " + charset); + } catch (Exception e) { + LOG.error("Failed to guess charset: " + e.getMessage()); + } } - String charset = charSetDetector.getCharset(chunk, chunkSize, httpHeaders); - Page page; try { page = new Page(is, charset); From 01d076a855e11af16175a9053ffedbe8f6a2aaa1 Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Thu, 24 Nov 2016 14:07:21 +0100 Subject: [PATCH 15/83] Match http-equiv meta elements with unquoted attribute values, e.g. --- .../org/archive/format/text/charset/CharsetDetector.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/archive/format/text/charset/CharsetDetector.java b/src/main/java/org/archive/format/text/charset/CharsetDetector.java index f550e342..1c0fd227 100644 --- a/src/main/java/org/archive/format/text/charset/CharsetDetector.java +++ b/src/main/java/org/archive/format/text/charset/CharsetDetector.java @@ -60,7 +60,8 @@ public abstract class CharsetDetector { private final static String META_CONTENT_ATTR_PATTERN_STRING = "\\b" + META_CONTENT_ATTRIBUTE + "\\s*=\\s*(" + ANY_ATTR_VALUE + ")(?:\\s|>)?"; private final static String META_HTTP_EQUIV_ATTR_PATTERN_STRING = "\\b" + - META_HTTP_EQUIV_ATTRIBUTE + "\\s*=\\s*(" + ANY_ATTR_VALUE + ")(?:\\s|>)?"; + META_HTTP_EQUIV_ATTRIBUTE + "\\s*=\\s*(" + META_CONTENT_TYPE + "|" + + ANY_ATTR_VALUE + ")(?:\\s|>)?"; @@ -180,7 +181,7 @@ protected String getCharsetFromMeta(byte buffer[],int len) throws IOException { private static String trimAttrValue(String value) { String result = value; if (result.isEmpty()) - return result; + return result; if (result.charAt(0) == '"') { result = result.substring(1, result.length() - 1); } else if (result.charAt(0) == '\'') { @@ -229,7 +230,6 @@ public static String findMetaContentType(String pageSample) { protected String getCharsetFromBytes(byte buffer[], int len) throws IOException { String charsetName = null; - UniversalDetector detector = new UniversalDetector(null); detector.handleData(buffer, 0, len); detector.dataEnd(); From da92adb1b7561d3c7fd29794375310436f4e750f Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Fri, 9 Dec 2016 15:35:10 +0100 Subject: [PATCH 16/83] Strip empty port, do not fail, fixes #5 --- src/main/java/org/archive/url/URLParser.java | 24 +++++++++++-------- .../archive/url/IAURLCanonicalizerTest.java | 1 + .../archive/url/WaybackURLKeyMakerTest.java | 1 + 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/archive/url/URLParser.java b/src/main/java/org/archive/url/URLParser.java index 98e4c1aa..83d3c386 100644 --- a/src/main/java/org/archive/url/URLParser.java +++ b/src/main/java/org/archive/url/URLParser.java @@ -246,16 +246,20 @@ public static HandyURL parse(String urlString) throws URISyntaxException { colonPort = uriAuthority.substring(portColonIndex); } if(colonPort != null) { - if(colonPort.startsWith(":")) { - try { - port = Integer.parseInt(colonPort.substring(1)); - } catch(NumberFormatException e) { - throw new URISyntaxException(urlString, "bad port " - + colonPort.substring(1)); - } - } else { - // XXX: what's happened?! - } + if(colonPort.startsWith(":")) { + if (colonPort.length() == 1) { + // a bare colon (http://example.com:/), use default port + } else { + try { + port = Integer.parseInt(colonPort.substring(1)); + } catch(NumberFormatException e) { + throw new URISyntaxException(urlString, "bad port " + + colonPort.substring(1)); + } + } + } else { + // XXX: what's happened?! + } } if(userInfo != null) { int passColonIndex = userInfo.indexOf(COLON); diff --git a/src/test/java/org/archive/url/IAURLCanonicalizerTest.java b/src/test/java/org/archive/url/IAURLCanonicalizerTest.java index 3263edc7..8a7a18eb 100644 --- a/src/test/java/org/archive/url/IAURLCanonicalizerTest.java +++ b/src/test/java/org/archive/url/IAURLCanonicalizerTest.java @@ -12,6 +12,7 @@ public void testFull() throws URISyntaxException { compCan(iaC,"https://www.archive.org:80/","https://archive.org:80/"); compCan(iaC,"http://www.archive.org:443/","http://archive.org:443/"); compCan(iaC,"https://www.archive.org:443/","https://archive.org/"); + compCan(iaC,"http://www.archive.org:/","http://archive.org/"); compCan(iaC,"http://www.archive.org/big/","http://archive.org/big"); compCan(iaC,"dns:www.archive.org","dns:www.archive.org"); diff --git a/src/test/java/org/archive/url/WaybackURLKeyMakerTest.java b/src/test/java/org/archive/url/WaybackURLKeyMakerTest.java index 34bfe625..26161456 100644 --- a/src/test/java/org/archive/url/WaybackURLKeyMakerTest.java +++ b/src/test/java/org/archive/url/WaybackURLKeyMakerTest.java @@ -22,6 +22,7 @@ public void testMakeKey() throws URISyntaxException { assertEquals("org,archive)/goo", km.makeKey("http://archive.org/goo/?")); assertEquals("org,archive)/goo?a&b", km.makeKey("http://archive.org/goo/?b&a")); assertEquals("org,archive)/goo?a=1&a=2&b", km.makeKey("http://archive.org/goo/?a=2&b&a=1")); + assertEquals("org,archive)/", km.makeKey("http://archive.org:/")); } } From eb66fc448110fac39b3692b7843d7c84c8b35112 Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Wed, 18 Jan 2017 16:18:55 +0100 Subject: [PATCH 17/83] Make regular expression to extract URLs from CSS more restrictive: merged improvements from iipc/webarchive-commons#63 --- .../html/ExtractingParseObserver.java | 25 ++++----- .../html/ExtractingParseObserverTest.java | 51 +++++++++++-------- 2 files changed, 39 insertions(+), 37 deletions(-) diff --git a/src/main/java/org/archive/resource/html/ExtractingParseObserver.java b/src/main/java/org/archive/resource/html/ExtractingParseObserver.java index 9e0f5c2f..0fce1b2a 100644 --- a/src/main/java/org/archive/resource/html/ExtractingParseObserver.java +++ b/src/main/java/org/archive/resource/html/ExtractingParseObserver.java @@ -25,13 +25,18 @@ public class ExtractingParseObserver implements ParseObserver { protected static String cssUrlPatString = "url\\s*\$\\s*((?:\\\\?[\"'])?.+?(?:\\\\?[\"'])?)\\s*\$"; + protected static String cssUrlTrimPatString = + "^(?:\\\\?[\"'])+|(?:\\\\?[\"'])+$"; protected static String cssImportNoUrlPatString = - "@import\\s+((?:'[^']+')|(?:\"[^\"]+\")|(?:\$'[^']+'\$)|(?:\$\"[^\"]+\"\$)|(?:\$[^)]+\$)|(?:[a-z0-9_.:/\\\\-]+))\\s*;"; + "@import\\s+((?:'[^']+')|(?:\"[^\"]+\")|(?:\$'[^']+'\$)|(?:\$\"[^\"]+\"\$)|(?:\$[^)]+\$)|(?:[a-z0-9_.:/\\\\-]+))\\s*;"; protected static Pattern cssImportNoUrlPattern = Pattern .compile(cssImportNoUrlPatString); protected static Pattern cssUrlPattern = Pattern.compile(cssUrlPatString); + + protected static Pattern cssUrlTrimPattern = Pattern.compile(cssUrlTrimPatString); + private final static int MAX_TEXT_LEN = 100; // private static String GLOBAL_ATTR[] = {"background"}; @@ -417,22 +422,10 @@ private void patternCSSExtract(HTMLMetaData data, Pattern pattern, String conten while((idx < contentLen) && m.find()) { idx = m.end(); String url = m.group(1); - if(url.length() < 2) { - continue; - } - if ((url.charAt(0) == '(') - && (url.charAt(url.length()-1) == ')')) { - url = url.substring(1, url.length() - 1); - } - if (url.charAt(0) == '"' || url.charAt(0) == '\'') { - url = url.substring(1, url.length() - 1); - } else if (url.charAt(0) == '\\') { - if(url.length() <= 4) { - continue; - } - url = url.substring(2, url.length() - 2); + url = cssUrlTrimPattern.matcher(url).replaceAll(""); + if (!url.isEmpty()) { + data.addHref("path","STYLE/#text","href", url); } - data.addHref("path","STYLE/#text","href",url); } } } diff --git a/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java b/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java index 24b6c18a..b052e375 100644 --- a/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java +++ b/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java @@ -19,7 +19,9 @@ public void testHandleStyleNodeExceptions() throws Exception { "url (' ')", "url('\")", "url(')", - "url('\"')" + "url('\"')", + "url('\\\"\"')", + "url(''''')" }; boolean except = false; HTMLMetaData md = new HTMLMetaData(new MetaData()); @@ -37,6 +39,7 @@ public void testHandleStyleNodeExceptions() throws Exception { assertFalse(except); } } + public void testHandleStyleNode() throws Exception { String[][] tests = { {""}, @@ -45,31 +48,36 @@ public void testHandleStyleNode() throws Exception { {"url(\"foo.gif\")","foo.gif"}, {"url(\\\"foo.gif\\\")","foo.gif"}, {"url(\\'foo.gif\\')","foo.gif"}, - - }; + {"url(''foo.gif'')","foo.gif"}, + {"url( foo.gif )","foo.gif"}, + {"url('''')"}, + {"url('foo.gif'')","foo.gif"}, + }; for(String[] testa : tests) { checkExtract(testa); } - // boolean except = false; -// HTMLMetaData md = new HTMLMetaData(new MetaData()); -// ExtractingParseObserver epo = new ExtractingParseObserver(md); -// for(String css : tests) { -// try { -// TextNode tn = new TextNode(css); -// epo.handleStyleNode(tn); -// } catch(Exception e) { -// System.err.format("And the winner is....(%s)\n", css); -// e.printStackTrace(); -// except = true; -// throw e; -// } -// assertFalse(except); -// } } + + /** + * Test whether the pattern matcher does extract nothing and also does not + * not hang-up if an overlong CSS link is truncated. + */ + public void testHandleStyleNodeNoHangupTruncated() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("url("); + for (int i = 0; i < 500000; i++) + sb.append('\''); + sb.append("foo.gif"); + for (int i = 0; i < 499000; i++) + sb.append('\''); + String[] test = new String[1]; + test[0] = sb.toString(); + checkExtract(test); + } + private void checkExtract(String[] data) throws JSONException { // System.err.format("CSS(%s) want[0](%s)\n",css,want[0]); String css = data[0]; - boolean except = false; HTMLMetaData md = new HTMLMetaData(new MetaData()); ExtractingParseObserver epo = new ExtractingParseObserver(md); try { @@ -87,10 +95,11 @@ private void checkExtract(String[] data) throws JSONException { assertTrue(o instanceof JSONObject); JSONObject jo = (JSONObject) o; - assertEquals(data[i],jo.getString("href")); + assertEquals("CSS link extraction failed for <" + css + ">", + data[i], jo.getString("href")); } } else { - assertNull(a); + assertNull("Expected no extracted link for <" + css + ">", a); } } From bf22eecbb18c75fa34b00f3f4ed9f863c089ea03 Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Fri, 27 Jan 2017 09:05:19 +0100 Subject: [PATCH 18/83] CharsetDetector: sync with iipc/webarchive-commons --- .../org/archive/format/text/charset/CharsetDetector.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/archive/format/text/charset/CharsetDetector.java b/src/main/java/org/archive/format/text/charset/CharsetDetector.java index 1c0fd227..690f8b99 100644 --- a/src/main/java/org/archive/format/text/charset/CharsetDetector.java +++ b/src/main/java/org/archive/format/text/charset/CharsetDetector.java @@ -179,9 +179,10 @@ protected String getCharsetFromMeta(byte buffer[],int len) throws IOException { } private static String trimAttrValue(String value) { + if (value.isEmpty()) { + return value; + } String result = value; - if (result.isEmpty()) - return result; if (result.charAt(0) == '"') { result = result.substring(1, result.length() - 1); } else if (result.charAt(0) == '\'') { From 7b9e81233512ff9754995e0aa83788e8b814b8dc Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Mon, 20 Feb 2017 11:54:59 +0100 Subject: [PATCH 19/83] fix indentation (use tab) --- .../archive/extract/ResourceExtractor.java | 17 ++-- .../archive/extract/WATExtractorOutput.java | 31 +++--- .../html/ExtractingParseObserver.java | 96 +++++++++---------- 3 files changed, 71 insertions(+), 73 deletions(-) diff --git a/src/main/java/org/archive/extract/ResourceExtractor.java b/src/main/java/org/archive/extract/ResourceExtractor.java index ff7d8c50..5d71bbd5 100644 --- a/src/main/java/org/archive/extract/ResourceExtractor.java +++ b/src/main/java/org/archive/extract/ResourceExtractor.java @@ -53,8 +53,7 @@ private static int USAGE(int exitCode) { System.err.println("\t\t\t (note that column 1 is NOT standard Wayback canonicalized)\n"); System.err.println("\t\t-wat\tembed JSON output in a compressed WARC" + "wrapper, for storage, or sharing."); - System.err.println("\t\t-wet\tembed text extracts in a compressed WARC" + - "wrapper, for storage, or sharing."); + System.err.println("\t\t-wet\tembed text extracts in a compressed WARC" + "wrapper, for storage, or sharing."); return exitCode; } @@ -103,13 +102,13 @@ public int run(String[] args) } else if(args[arg].equals("-wat")) { path = args[arg+1]; out = new WATExtractorOutput(os); - } else if(args[arg].equals("-wet")) { - path = args[arg+1]; - out = new WETExtractorOutput(os); - } else { - String filter = args[arg+1]; - out = new JSONViewExtractorOutput(os, filter); - } + } else if (args[arg].equals("-wet")) { + path = args[arg + 1]; + out = new WETExtractorOutput(os); + } else { + String filter = args[arg + 1]; + out = new JSONViewExtractorOutput(os, filter); + } } else { out = new DumpingExtractorOutput(os); } diff --git a/src/main/java/org/archive/extract/WATExtractorOutput.java b/src/main/java/org/archive/extract/WATExtractorOutput.java index ee803672..7bb9fb88 100644 --- a/src/main/java/org/archive/extract/WATExtractorOutput.java +++ b/src/main/java/org/archive/extract/WATExtractorOutput.java @@ -29,17 +29,17 @@ public class WATExtractorOutput implements ExtractorOutput { private static int DEFAULT_BUFFER_RAM = 1024 * 1024; private int bufferRAM = DEFAULT_BUFFER_RAM; private final static Charset UTF8 = Charset.forName("UTF-8"); - private String outFilename; + private String outFilename; - public WATExtractorOutput(OutputStream out) { - this(out, null); - } + public WATExtractorOutput(OutputStream out) { + this(out, null); + } public WATExtractorOutput(OutputStream out, String filename) { gzW = new GZIPMemberWriter(out); recW = new WARCRecordWriter(); wroteFirst = false; - outFilename = filename; + outFilename = filename; } private CommitedOutputStream getOutput() { @@ -62,9 +62,8 @@ public void output(Resource resource) throws IOException { throw new IOException("Missing Envelope.Format"); } - // remove the text extracts if it exists - JSONUtils.removeObject(top, "Envelope.Payload-Metadata.HTTP-Response-Metadata.HTML-Metadata", "Text"); - + // remove the text extracts if it exists + JSONUtils.removeObject(top, "Envelope.Payload-Metadata.HTTP-Response-Metadata.HTML-Metadata", "Text"); cos = getOutput(); if(envelopeFormat.equals("ARC")) { @@ -79,15 +78,15 @@ public void output(Resource resource) throws IOException { } private void writeWARCInfo(OutputStream recOut, MetaData md) throws IOException { - String filename = outFilename; + String filename = outFilename; - if (filename == null) { - filename = JSONUtils.extractSingle(md, "Container.Filename"); + if (filename == null) { + filename = JSONUtils.extractSingle(md, "Container.Filename"); - if(filename == null) { - throw new IOException("No Container.Filename..."); - } - } + if (filename == null) { + throw new IOException("No Container.Filename..."); + } + } HttpHeaders headers = new HttpHeaders(); headers.add("Software-Info", IAUtils.COMMONS_VERSION); @@ -95,7 +94,7 @@ private void writeWARCInfo(OutputStream recOut, MetaData md) throws IOException ByteArrayOutputStream baos = new ByteArrayOutputStream(); headers.write(baos); - recW.writeWARCInfoRecord(recOut,filename,baos.toByteArray()); + recW.writeWARCInfoRecord(recOut, filename, baos.toByteArray()); } private String extractOrIO(MetaData md, String path) throws IOException { diff --git a/src/main/java/org/archive/resource/html/ExtractingParseObserver.java b/src/main/java/org/archive/resource/html/ExtractingParseObserver.java index 0fce1b2a..1ed61497 100644 --- a/src/main/java/org/archive/resource/html/ExtractingParseObserver.java +++ b/src/main/java/org/archive/resource/html/ExtractingParseObserver.java @@ -18,10 +18,10 @@ public class ExtractingParseObserver implements ParseObserver { HTMLMetaData data; Stack> openAnchors; Stack openAnchorTexts; - StringBuffer textExtract; + StringBuffer textExtract; String title = null; boolean inTitle = false; - boolean inPre = false; + boolean inPre = false; protected static String cssUrlPatString = "url\\s*\$\\s*((?:\\\\?[\"'])?.+?(?:\\\\?[\"'])?)\\s*\$"; @@ -67,7 +67,7 @@ public ExtractingParseObserver(HTMLMetaData data) { this.data = data; openAnchors = new Stack>(); openAnchorTexts = new Stack(); - textExtract = new StringBuffer(8192); + textExtract = new StringBuffer(8192); } public void handleDocumentStart() { @@ -75,10 +75,10 @@ public void handleDocumentStart() { } public void handleDocumentComplete() { - if (textExtract.length() > 0) { - data.setTextExtract(textExtract.toString()); - textExtract = new StringBuffer(8192); - } + if (textExtract.length() > 0) { + data.setTextExtract(textExtract.toString()); + textExtract = new StringBuffer(8192); + } } public void handleTagEmpty(TagNode tag) { @@ -91,8 +91,8 @@ public void handleTagOpen(TagNode tag) { inTitle = !tag.isEmptyXmlTag(); return; } else if (name.equals("PRE")) { - inPre = true; - } + inPre = true; + } // first the global attributes: // background @@ -139,59 +139,59 @@ public void handleTagClose(TagNode tag) { } } } else if (tag.getTagName().equals("PRE")) { - inPre = false; - } + inPre = false; + } } public void handleTextNode(TextNode text) { // TODO: OPTIMIZ: This can be a lot smarter, if StringBuilders are full, - // this result is thrown away. - //System.out.println("JDBUG: Got text from node: " + text.getText().toString()); - - String txt = text.getText(); - if (!inPre) { - txt = Translate.decode(txt); - txt = txt.replace('\u00a0', ' '); - - char c = ' '; - if (textExtract.length() > 0) { - c = textExtract.charAt(textExtract.length()-1); - } - for (int i = 0; i < txt.length(); i++) { - char c2 = txt.charAt(i); - // Translate so output is a bit cleaner - if (c2 == '\r') { - c2 = '\n'; - } - if (!Character.isWhitespace(c) || !Character.isWhitespace(c2)) { - textExtract.append(c2); - } - c = c2; - } - } - else - textExtract.append(txt); - - String t = text.getText().replaceAll("\\s+", " "); - - if(t.length() > MAX_TEXT_LEN) { - t = t.substring(0,MAX_TEXT_LEN); + // this result is thrown away. + // System.out.println("JDBUG: Got text from node: " + + // text.getText().toString()); + + String txt = text.getText(); + if (!inPre) { + txt = Translate.decode(txt); + txt = txt.replace('\u00a0', ' '); + + char c = ' '; + if (textExtract.length() > 0) { + c = textExtract.charAt(textExtract.length() - 1); + } + for (int i = 0; i < txt.length(); i++) { + char c2 = txt.charAt(i); + // Translate so output is a bit cleaner + if (c2 == '\r') { + c2 = '\n'; + } + if (!Character.isWhitespace(c) || !Character.isWhitespace(c2)) { + textExtract.append(c2); + } + c = c2; + } + } else + textExtract.append(txt); + + String t = text.getText().replaceAll("\\s+", " "); + + if (t.length() > MAX_TEXT_LEN) { + t = t.substring(0, MAX_TEXT_LEN); } - if(inTitle) { + if (inTitle) { title = t; } else { - - for(StringBuilder s : openAnchorTexts) { - if(s.length() >= MAX_TEXT_LEN) { + + for (StringBuilder s : openAnchorTexts) { + if (s.length() >= MAX_TEXT_LEN) { // if we are full, parents enclosing us should be too.. break; } - if(s.length() + t.length() < MAX_TEXT_LEN) { + if (s.length() + t.length() < MAX_TEXT_LEN) { s.append(t); } else { // only add as much as we can: - s.append(t.substring(0,MAX_TEXT_LEN - s.length())); + s.append(t.substring(0, MAX_TEXT_LEN - s.length())); } // BUGBUG: check now for multiple trailing spaces, and strip: } From 4a56f814ab0689562d7e2d0a51464a25b367be24 Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Mon, 20 Feb 2017 11:55:38 +0100 Subject: [PATCH 20/83] upgrade to CDH 5.10.0 --- pom-cdh5.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pom-cdh5.xml b/pom-cdh5.xml index 3619206d..f0c90ac2 100644 --- a/pom-cdh5.xml +++ b/pom-cdh5.xml @@ -51,12 +51,12 @@ commons-httpclient commons-httpclient 3.1 - + org.apache.hadoop hadoop-client - 2.6.0-cdh5.8.0 + 2.6.0-cdh5.10.0 commons-httpclient @@ -85,8 +85,8 @@ tomcat jasper-compiler - - + + org.apache.hadoop @@ -128,7 +128,7 @@ libidn 1.15 - + it.unimi.dsi mg4j 1.0.1 From 5626c90ad5cfeed215a733c8cb756648d180ecde Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Tue, 21 Feb 2017 13:42:43 +0100 Subject: [PATCH 21/83] Improve HTML link extraction, fixes #9 - add extractors for more elements which can take URLs as attribute values, complete attributes - add unit test to verify link extraction --- .../html/ExtractingParseObserver.java | 48 ++++++- .../html/ExtractingParseObserverTest.java | 116 +++++++++++++++ .../resource/html/link-extraction-test.warc | 136 ++++++++++++++++++ 3 files changed, 298 insertions(+), 2 deletions(-) create mode 100644 src/test/resources/org/archive/resource/html/link-extraction-test.warc diff --git a/src/main/java/org/archive/resource/html/ExtractingParseObserver.java b/src/main/java/org/archive/resource/html/ExtractingParseObserver.java index 1ed61497..c97e0d42 100644 --- a/src/main/java/org/archive/resource/html/ExtractingParseObserver.java +++ b/src/main/java/org/archive/resource/html/ExtractingParseObserver.java @@ -60,6 +60,17 @@ public class ExtractingParseObserver implements ParseObserver { extractors.put("META", new MetaTagExtractor()); extractors.put("OBJECT", new ObjectTagExtractor()); extractors.put("SCRIPT", new ScriptTagExtractor()); + extractors.put("Q", new QuotationLinkTagExtractor()); + extractors.put("BLOCKQUOTE", new QuotationLinkTagExtractor()); + extractors.put("DEL", new QuotationLinkTagExtractor()); + extractors.put("INS", new QuotationLinkTagExtractor()); + // HTML5: + extractors.put("BUTTON", new ButtonTagExtractor()); + extractors.put("MENUITEM", new MenuitemTagExtractor()); + extractors.put("VIDEO", new EmbedVideoTagExtractor()); + extractors.put("AUDIO", new EmbedTagExtractor()); + extractors.put("TRACK", new EmbedTagExtractor()); + extractors.put("SOURCE", new EmbedTagExtractor()); } @@ -335,12 +346,24 @@ public void extract(HTMLMetaData data, TagNode node, ExtractingParseObserver obs } } + private static class ButtonTagExtractor implements TagExtractor { + public void extract(HTMLMetaData data, TagNode node, ExtractingParseObserver obs) { + addBasicHrefs(data,node,"formaction"); + } + } + private static class EmbedTagExtractor implements TagExtractor { public void extract(HTMLMetaData data, TagNode node, ExtractingParseObserver obs) { addBasicHrefs(data,node,"src"); } } + private static class EmbedVideoTagExtractor implements TagExtractor { + public void extract(HTMLMetaData data, TagNode node, ExtractingParseObserver obs) { + addBasicHrefs(data,node,"src","poster"); + } + } + private static class FormTagExtractor implements TagExtractor { public void extract(HTMLMetaData data, TagNode node, ExtractingParseObserver obs) { ArrayList l = new ArrayList(); @@ -368,21 +391,26 @@ public void extract(HTMLMetaData data, TagNode node, ExtractingParseObserver obs addBasicHrefs(data,node,"src"); } } + private static class IFrameTagExtractor implements TagExtractor { public void extract(HTMLMetaData data, TagNode node, ExtractingParseObserver obs) { addBasicHrefs(data,node,"src"); } } + private static class ImgTagExtractor implements TagExtractor { public void extract(HTMLMetaData data, TagNode node, ExtractingParseObserver obs) { addHrefWithAttrs(data,node,"src","alt","title"); + addBasicHrefs(data,node,"longdesc"); } } + private static class InputTagExtractor implements TagExtractor { public void extract(HTMLMetaData data, TagNode node, ExtractingParseObserver obs) { - addBasicHrefs(data,node,"src"); + addBasicHrefs(data,node,"src","formaction"); } } + private static class LinkTagExtractor implements TagExtractor { public void extract(HTMLMetaData data, TagNode node, ExtractingParseObserver obs) { ArrayList l = getAttrListUrl(node,"href","rel","type"); @@ -391,6 +419,13 @@ public void extract(HTMLMetaData data, TagNode node, ExtractingParseObserver obs } } } + + private static class MenuitemTagExtractor implements TagExtractor { + public void extract(HTMLMetaData data, TagNode node, ExtractingParseObserver obs) { + addBasicHrefs(data,node,"icon"); + } + } + private static class MetaTagExtractor implements TagExtractor { public void extract(HTMLMetaData data, TagNode node, ExtractingParseObserver obs) { ArrayList l = getAttrList(node,"name","rel","content","http-equiv","property"); @@ -399,11 +434,19 @@ public void extract(HTMLMetaData data, TagNode node, ExtractingParseObserver obs } } } + private static class ObjectTagExtractor implements TagExtractor { public void extract(HTMLMetaData data, TagNode node, ExtractingParseObserver obs) { - addBasicHrefs(data,node,"codebase","cdata"); + addBasicHrefs(data,node,"codebase","cdata","data"); } } + + private static class QuotationLinkTagExtractor implements TagExtractor { + public void extract(HTMLMetaData data, TagNode node, ExtractingParseObserver obs) { + addBasicHrefs(data,node,"cite"); + } + } + private static class ScriptTagExtractor implements TagExtractor { public void extract(HTMLMetaData data, TagNode node, ExtractingParseObserver obs) { ArrayList l = getAttrListUrl(node,"src","type"); @@ -412,6 +455,7 @@ public void extract(HTMLMetaData data, TagNode node, ExtractingParseObserver obs } } } + private void patternCSSExtract(HTMLMetaData data, Pattern pattern, String content) { Matcher m = pattern.matcher(content); int idx = 0; diff --git a/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java b/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java index b052e375..a8b5213b 100644 --- a/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java +++ b/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java @@ -1,15 +1,33 @@ package org.archive.resource.html; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Logger; + +import org.archive.extract.ExtractingResourceFactoryMapper; +import org.archive.extract.ExtractingResourceProducer; +import org.archive.extract.ProducerUtils; +import org.archive.extract.ResourceFactoryMapper; import org.archive.resource.MetaData; +import org.archive.resource.Resource; +import org.archive.resource.ResourceParseException; +import org.archive.resource.ResourceProducer; import org.htmlparser.nodes.TextNode; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; + import junit.framework.TestCase; public class ExtractingParseObserverTest extends TestCase { + private static final Logger LOG = + Logger.getLogger(ExtractingParseObserverTest.class.getName()); + public void testHandleStyleNodeExceptions() throws Exception { String[] tests = { "some css", @@ -103,5 +121,103 @@ private void checkExtract(String[] data) throws JSONException { } } + private void checkLink(Multimap links, String url, String path) { + assertTrue("Link with URL " + url + " not found", links.containsKey(url)); + assertTrue("Wrong path " + path + " for " + url, links.get(url).contains(path)); + } + + private void checkLinks(Resource resource, String[][] expectedLinks) { + assertNotNull(resource); + assertTrue(resource instanceof HTMLResource); + MetaData md = resource.getMetaData(); + LOG.info(md.toString()); + Multimap links = ArrayListMultimap.create(); + try { + // + String baseUrl = (String) md.getJSONObject("Head").opt("Base"); + if (baseUrl != null) { + links.put(baseUrl, "__base__"); + } + // + JSONArray metas = md.getJSONObject("Head").optJSONArray("Metas"); + if (metas != null) { + for (int i = 0; i < metas.length(); i++) { + JSONObject o = (JSONObject) metas.optJSONObject(i); + if (o.getString("http-equiv").equals("Refresh")) { + String metaRefreshTarget = o.getString("content").replaceFirst("(?i)(?:^\\d+\\s*;)?\\s*url=", ""); + LOG.info(metaRefreshTarget); + links.put(metaRefreshTarget, "__meta_refresh__"); + } + } + } + } catch (JSONException e) { + fail("Failed to parse JSON: " + e.getMessage()); + } + // extract outlinks + List linkArrays = new ArrayList(); + if (md.optJSONArray("Links") != null) { + linkArrays.add(md.optJSONArray("Links")); + } + try { + if (md.getJSONObject("Head") != null && md.getJSONObject("Head").getJSONArray("Link") != null) { + linkArrays.add(md.getJSONObject("Head").getJSONArray("Link")); + } + } catch (JSONException e1) { + } + for (JSONArray ldata : linkArrays) { + for (int i = 0; i < ldata.length(); i++) { + JSONObject o = (JSONObject) ldata.optJSONObject(i); + try { + String url = o.getString("url"); + links.put(url, o.getString("path")); + LOG.info(" found link: " + o.getString("url") + " " + o.getString("path")); + } catch (JSONException e) { + fail("Failed to extract URL from link: " + e.getMessage()); + } + } + } + assertEquals("Unexpected number of links", expectedLinks.length, links.size()); + for (String[] l : expectedLinks) { + checkLink(links, l[0], l[1]); + } + } + + public void testLinkExtraction() throws ResourceParseException, IOException { + String testFileName = "link-extraction-test.warc"; + ResourceProducer producer = ProducerUtils.getProducer(getClass().getResource(testFileName).toString()); + ResourceFactoryMapper mapper = new ExtractingResourceFactoryMapper(); + ExtractingResourceProducer extractor = + new ExtractingResourceProducer(producer, mapper); + extractor.getNext(); // skip warcinfo record + String[][] html4links = { + {"http://www.example.com/", "__base__"}, + {"http://www.example.com/redirected.html", "__meta_refresh__"}, + {"background.jpg", "BODY@/background"}, + {"http://www.example.com/a-href.html", "A@/href"}, + {"#anchor", "A@/href"}, + {"image.png", "IMG@/src"}, + {"image.gif", "IMG@/src"}, + {"http://example.com/image-description.html#image.gif", "IMG@/longdesc"}, + {"helloworld.swf", "OBJECT@/data"}, + {"http://www.example.com/shakespeare.html", "Q@/cite"}, + {"http://www.example.com/shakespeare-long.html", "BLOCKQUOTE@/cite"} + }; + checkLinks(extractor.getNext(), html4links); + String[][] html5links = { + {"http:///www.example.com/video.html", "LINK@/href", "canonical"}, + {"video.rss", "LINK@/href", "alternate"}, + {"https://archive.org/download/WebmVp8Vorbis/webmvp8.gif", "VIDEO@/poster"}, + {"https://archive.org/download/WebmVp8Vorbis/webmvp8.webm", "SOURCE@/src"}, + {"https://archive.org/download/WebmVp8Vorbis/webmvp8_512kb.mp4", "SOURCE@/src"}, + {"https://archive.org/download/WebmVp8Vorbis/webmvp8.ogv", "SOURCE@/src"} + }; + checkLinks(extractor.getNext(), html5links); + String[][] fbVideoLinks = { + {"https://www.facebook.com/facebook/videos/10153231379946729/", "BLOCKQUOTE@/cite"}, + {"https://www.facebook.com/facebook/videos/10153231379946729/", "A@/href"}, + {"https://www.facebook.com/facebook/", "A@/href"}, + }; + checkLinks(extractor.getNext(), fbVideoLinks); + } } diff --git a/src/test/resources/org/archive/resource/html/link-extraction-test.warc b/src/test/resources/org/archive/resource/html/link-extraction-test.warc new file mode 100644 index 00000000..aed76aad --- /dev/null +++ b/src/test/resources/org/archive/resource/html/link-extraction-test.warc @@ -0,0 +1,136 @@ +WARC/1.0 +WARC-Type: warcinfo +Content-Type: application/warc-fields +WARC-Date: 2017-02-20T14:00:56Z +Content-Length: 128 + +format: WARC File Format 1.0 +conformsTo: http://bibnum.bnf.fr/WARC/WARC_ISO_28500_version1_latestdraft.pdf +robots: classic + + + +WARC/1.0 +WARC-Type: response +WARC-Date: 2017-02-20T14:00:56Z +WARC-Target-URI: http://www.example.com/html4.html +Content-Type: application/http;msgtype=response +Content-Length: 1243 + +HTTP/1.1 200 OK +Date: Mon, 20 Feb 2017 14:00:56 GMT +Content-Length: 1125 +Content-Type: application/xhtml+xml + + + + + + + +Test XHTML Link Extraction + + +A@/href +

+ anchor only + IMG@/src + IMG@/longdesc + +

+ To be or not to be. +

+To be, or not to be, that is the question:
+Whether 'tis nobler in the mind to suffer
+The slings and arrows of outrageous fortune, â€¦ +

+ + + + +WARC/1.0 +WARC-Type: response +WARC-Target-URI: http://www.example.com/link-extraction-test-html5-video.html +WARC-Date: 2017-02-20T21:35:03Z +Content-Type: application/http;msgtype=response +Content-Length: 890 + +HTTP/1.1 200 OK +Date: Mon, 20 Feb 2017 21:35:03 GMT +Content-Length: 789 +Content-Type: text/html + + + + +Test HTML5 Video Tag + + + + + + +

+ + + + +WARC/1.0 +WARC-Type: response +WARC-Target-URI: http://www.example.com/fb-video.html +WARC-Date: 2017-02-20T16:58:50Z +Content-Type: application/http;msgtype=response +Content-Length: 1330 + +HTTP/1.1 200 OK +Date: Mon, 20 Feb 2017 16:58:50 GMT +Content-Length: 1194 +Content-Type: text/html + + + + + fb-video - Embedded Videos - Social Plugins + + + + +

+ + + +

+ How to Share With Just Friends +
How to share with just friends.
+ Posted by Facebook on Friday, December 5, 2014 +

+ + + + + From 4b0deb414db6917611b4df054cd4cbcd87eb8c89 Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Wed, 22 Feb 2017 10:15:04 +0100 Subject: [PATCH 22/83] Extract links/URLs from data-href and data-uri attributes, fixes #7 --- .../html/ExtractingParseObserver.java | 30 ++- .../html/ExtractingParseObserverTest.java | 63 +++++- .../resource/html/link-extraction-test.warc | 184 ++++++++++++++++++ 3 files changed, 261 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/archive/resource/html/ExtractingParseObserver.java b/src/main/java/org/archive/resource/html/ExtractingParseObserver.java index c97e0d42..8821d4cd 100644 --- a/src/main/java/org/archive/resource/html/ExtractingParseObserver.java +++ b/src/main/java/org/archive/resource/html/ExtractingParseObserver.java @@ -2,12 +2,17 @@ import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; +import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.Stack; +import java.util.Vector; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.archive.format.text.html.ParseObserver; +import org.htmlparser.Attribute; import org.htmlparser.nodes.RemarkNode; import org.htmlparser.nodes.TagNode; import org.htmlparser.nodes.TextNode; @@ -39,11 +44,10 @@ public class ExtractingParseObserver implements ParseObserver { private final static int MAX_TEXT_LEN = 100; -// private static String GLOBAL_ATTR[] = {"background"}; - private static final String PATH = "path"; private static final String PATH_SEPARATOR = "@/"; - private final static Map extractors; + private static final Map extractors; + private static final Set globalHrefAttributes; static { extractors = new HashMap(); extractors.put("A", new AnchorTagExtractor()); @@ -71,6 +75,11 @@ public class ExtractingParseObserver implements ParseObserver { extractors.put("AUDIO", new EmbedTagExtractor()); extractors.put("TRACK", new EmbedTagExtractor()); extractors.put("SOURCE", new EmbedTagExtractor()); + + globalHrefAttributes = new HashSet(); + globalHrefAttributes.add("background"); + globalHrefAttributes.add("data-href"); + globalHrefAttributes.add("data-uri"); } @@ -106,10 +115,17 @@ public void handleTagOpen(TagNode tag) { } // first the global attributes: - // background - String v = tag.getAttribute("background"); - if(v != null) { - data.addHref(PATH,makePath(name,"background"),"url",v); + Vector attributes = tag.getAttributesEx(); + for (Attribute a : attributes) { + String attrName = a.getName(); + String attrValue = a.getValue(); + if (attrName == null || attrValue == null) { + continue; + } + attrName = attrName.toLowerCase(Locale.ROOT); + if (globalHrefAttributes.contains(attrName)) { + data.addHref(PATH,makePath(name,attrName),"url",attrValue); + } } // TODO: style attribute, BASE(href) tag, Resolve URLs diff --git a/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java b/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java index a8b5213b..b1b800c2 100644 --- a/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java +++ b/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java @@ -132,26 +132,28 @@ private void checkLinks(Resource resource, String[][] expectedLinks) { MetaData md = resource.getMetaData(); LOG.info(md.toString()); Multimap links = ArrayListMultimap.create(); - try { + JSONObject head = md.optJSONObject("Head"); + if (head != null) { // - String baseUrl = (String) md.getJSONObject("Head").opt("Base"); + String baseUrl = (String) head.opt("Base"); if (baseUrl != null) { links.put(baseUrl, "__base__"); } // - JSONArray metas = md.getJSONObject("Head").optJSONArray("Metas"); + JSONArray metas = head.optJSONArray("Metas"); if (metas != null) { for (int i = 0; i < metas.length(); i++) { JSONObject o = (JSONObject) metas.optJSONObject(i); - if (o.getString("http-equiv").equals("Refresh")) { - String metaRefreshTarget = o.getString("content").replaceFirst("(?i)(?:^\\d+\\s*;)?\\s*url=", ""); - LOG.info(metaRefreshTarget); - links.put(metaRefreshTarget, "__meta_refresh__"); + String httpEquiv = o.optString("http-equiv"); + if (httpEquiv != null && httpEquiv.equalsIgnoreCase("Refresh")) { + String metaRefreshTarget = o.optString("content"); + if (metaRefreshTarget != null) { + metaRefreshTarget = metaRefreshTarget.replaceFirst("(?i)(?:^\\d+\\s*;)?\\s*url=", ""); + links.put(metaRefreshTarget, "__meta_refresh__"); + } } } } - } catch (JSONException e) { - fail("Failed to parse JSON: " + e.getMessage()); } // extract outlinks List linkArrays = new ArrayList(); @@ -212,12 +214,55 @@ public void testLinkExtraction() throws ResourceParseException, IOException { {"https://archive.org/download/WebmVp8Vorbis/webmvp8.ogv", "SOURCE@/src"} }; checkLinks(extractor.getNext(), html5links); + String[][] html5links2 = { + {"http://www.example.com/", "A@/href"}, + }; + checkLinks(extractor.getNext(), html5links2); String[][] fbVideoLinks = { {"https://www.facebook.com/facebook/videos/10153231379946729/", "BLOCKQUOTE@/cite"}, {"https://www.facebook.com/facebook/videos/10153231379946729/", "A@/href"}, {"https://www.facebook.com/facebook/", "A@/href"}, + {"https://www.facebook.com/facebook/videos/10153231379946729/", "DIV@/data-href"} }; checkLinks(extractor.getNext(), fbVideoLinks); + String[][] dataHrefLinks = { + {"standard.css", "LINK@/href", "stylesheet"}, + {"https://www.facebook.com/elegantthemes/videos/10153760379211923/", "DIV@/data-href"}, + {"https://www.facebook.com/facebook/videos/10153231379946729/", "DIV@/data-href"}, + {"https://www.facebook.com/facebook/videos/10153231379946729/", "BLOCKQUOTE@/cite"}, + {"https://www.facebook.com/facebook/videos/10153231379946729/", "A@/href"}, + {"https://www.facebook.com/facebook/", "A@/href"}, + {"//edge.flowplayer.org/bauhaus.webm", "SOURCE@/src"}, + {"//edge.flowplayer.org/bauhaus.mp4", "SOURCE@/src"}, + {"//edge.flowplayer.org/functional.webm", "BUTTON@/data-href"}, + {"/content-page", "ARTICLE@/data-href"}, + {"/content-page", "A@/href"}, + {"/tags/content","A@/href"}, + {"/tags/headlines", "A@/href"}, + {"http://grabaperch.com", "DIV@/data-href"}, + {"green.css", "LINK@/data-href"}, + {"blue.css", "LINK@/data-href"}, + {"http://codecanyon.net/user/CodingJack", "A@/data-href"}, + {"jackbox/img/thumbs/4.jpg", "IMG@/src"}, + {"//venobox-destination", "A@/data-href"}, + {"#", "A@/href"}, + {"http://www.youtube.com/v/itTskyFLSS8&rel=0&autohide=1&showinfo=0&autoplay=1", "DIV@/data-href"}, + {"#", "A@/href"}, + {"http://www.youtube.com/v/itTskyFLSS8&rel=0&autohide=1&showinfo=0", "IFRAME@/src"} + }; + checkLinks(extractor.getNext(), dataHrefLinks); + String[][] fbSocialLinks = { + {"http://www.your-domain.com/your-page.html", "DIV@/data-uri"}, + {"https://developers.facebook.com/docs/plugins/comments#configurator", "DIV@/data-href"}, + {"https://www.facebook.com/zuck/posts/10102735452532991?comment_id=1070233703036185", "DIV@/data-href"}, + {"https://www.facebook.com/zuck", "DIV@/data-href"}, + {"https://developers.facebook.com/docs/plugins/", "DIV@/data-href"}, + {"https://www.facebook.com/facebook", "DIV@/data-href"}, + {"https://www.facebook.com/facebook", "BLOCKQUOTE@/cite"}, + {"https://www.facebook.com/facebook", "A@/href"}, + {"http://www.your-domain.com/your-page.html", "DIV@/data-href"} + }; + checkLinks(extractor.getNext(), fbSocialLinks); } } diff --git a/src/test/resources/org/archive/resource/html/link-extraction-test.warc b/src/test/resources/org/archive/resource/html/link-extraction-test.warc index aed76aad..1781168c 100644 --- a/src/test/resources/org/archive/resource/html/link-extraction-test.warc +++ b/src/test/resources/org/archive/resource/html/link-extraction-test.warc @@ -90,6 +90,42 @@ Content-Type: text/html +WARC/1.0 +WARC-Type: response +WARC-Target-URI: http://www.example.com/poor_html5.html +WARC-Date: 2017-02-21T15:50:40Z +Content-Type: application/http;msgtype=response +Content-Length: 594 + +HTTP/1.1 200 OK +Date: Tue, 21 Feb 2017 15:50:40 GMT +Content-Length: 486 +Content-Type: text/html + + +Testing poor HTML5 + + + + + +This is valid HTML5! + + + +

header

+ +

headline

+ +

paragraph one with link. + + WARC/1.0 WARC-Type: response WARC-Target-URI: http://www.example.com/fb-video.html @@ -134,3 +170,151 @@ Content-Type: text/html +WARC/1.0 +WARC-Type: response +WARC-Target-URI: http://www.example.com/data-href.examples.html +WARC-Date: 2017-02-21T21:05:10Z +Content-Type: application/http;msgtype=response +Content-Length: 3160 + +HTTP/1.1 200 OK +Date: Tue, 21 Feb 2017 21:05:10 GMT +Content-Length: 3057 +Content-Type: text/html + + + + + + + + + + + + +

+ + +

+ How to Share With Just Friends +
How to share with just friends.
+ Posted by Facebook on Friday, December 5, 2014 +

+ + +

+ +

+ + +

Headline goes here.

And here goes a bit of copy about the content of the article.

+ Tags: content, headlines +

+ + +

+ + + +

+ + + +venobox + + + + + +WARC/1.0 +WARC-Type: response +WARC-Target-URI: http://www.example.com/fb-social-plugins.html +WARC-Date: 2017-02-22T09:33:02Z +Content-Type: application/http;msgtype=response +Content-Length: 1870 + +HTTP/1.1 200 OK +Date: Wed, 22 Feb 2017 09:33:02 GMT +Content-Length: 1767 +Content-Type: text/html + + +

+ + +

Facebook

+ + + + + From 5ac0e781cc194e1c31560d7f7c5ad240ae7a074d Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Wed, 22 Feb 2017 14:12:18 +0100 Subject: [PATCH 23/83] sync unit test with iipc/webarchive-commons (including iipc/webarchive-commons#72) --- .../resource/html/ExtractingParseObserverTest.java | 6 +++--- .../archive/resource/html/link-extraction-test.warc | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java b/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java index b1b800c2..8f690a06 100644 --- a/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java +++ b/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java @@ -39,7 +39,7 @@ public void testHandleStyleNodeExceptions() throws Exception { "url(')", "url('\"')", "url('\\\"\"')", - "url(''''')" + "url(''''')" }; boolean except = false; HTMLMetaData md = new HTMLMetaData(new MetaData()); @@ -128,7 +128,7 @@ private void checkLink(Multimap links, String url, String path) { private void checkLinks(Resource resource, String[][] expectedLinks) { assertNotNull(resource); - assertTrue(resource instanceof HTMLResource); + assertTrue("Wrong instance type of Resource: " + resource.getClass(), resource instanceof HTMLResource); MetaData md = resource.getMetaData(); LOG.info(md.toString()); Multimap links = ArrayListMultimap.create(); @@ -186,7 +186,7 @@ private void checkLinks(Resource resource, String[][] expectedLinks) { public void testLinkExtraction() throws ResourceParseException, IOException { String testFileName = "link-extraction-test.warc"; - ResourceProducer producer = ProducerUtils.getProducer(getClass().getResource(testFileName).toString()); + ResourceProducer producer = ProducerUtils.getProducer(getClass().getResource(testFileName).getPath()); ResourceFactoryMapper mapper = new ExtractingResourceFactoryMapper(); ExtractingResourceProducer extractor = new ExtractingResourceProducer(producer, mapper); diff --git a/src/test/resources/org/archive/resource/html/link-extraction-test.warc b/src/test/resources/org/archive/resource/html/link-extraction-test.warc index 1781168c..ab0e54c8 100644 --- a/src/test/resources/org/archive/resource/html/link-extraction-test.warc +++ b/src/test/resources/org/archive/resource/html/link-extraction-test.warc @@ -14,7 +14,7 @@ WARC/1.0 WARC-Type: response WARC-Date: 2017-02-20T14:00:56Z WARC-Target-URI: http://www.example.com/html4.html -Content-Type: application/http;msgtype=response +Content-Type: application/http; msgtype=response Content-Length: 1243 HTTP/1.1 200 OK @@ -55,7 +55,7 @@ WARC/1.0 WARC-Type: response WARC-Target-URI: http://www.example.com/link-extraction-test-html5-video.html WARC-Date: 2017-02-20T21:35:03Z -Content-Type: application/http;msgtype=response +Content-Type: application/http; msgtype=response Content-Length: 890 HTTP/1.1 200 OK @@ -94,7 +94,7 @@ WARC/1.0 WARC-Type: response WARC-Target-URI: http://www.example.com/poor_html5.html WARC-Date: 2017-02-21T15:50:40Z -Content-Type: application/http;msgtype=response +Content-Type: application/http; msgtype=response Content-Length: 594 HTTP/1.1 200 OK @@ -130,7 +130,7 @@ WARC/1.0 WARC-Type: response WARC-Target-URI: http://www.example.com/fb-video.html WARC-Date: 2017-02-20T16:58:50Z -Content-Type: application/http;msgtype=response +Content-Type: application/http; msgtype=response Content-Length: 1330 HTTP/1.1 200 OK @@ -174,7 +174,7 @@ WARC/1.0 WARC-Type: response WARC-Target-URI: http://www.example.com/data-href.examples.html WARC-Date: 2017-02-21T21:05:10Z -Content-Type: application/http;msgtype=response +Content-Type: application/http; msgtype=response Content-Length: 3160 HTTP/1.1 200 OK @@ -261,7 +261,7 @@ WARC/1.0 WARC-Type: response WARC-Target-URI: http://www.example.com/fb-social-plugins.html WARC-Date: 2017-02-22T09:33:02Z -Content-Type: application/http;msgtype=response +Content-Type: application/http; msgtype=response Content-Length: 1870 HTTP/1.1 200 OK From 5536dbec62d4d26adc3a9b896567f313d9aefc6d Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Wed, 22 Feb 2017 14:35:19 +0100 Subject: [PATCH 24/83] upgrade to CDH 5.10.0 --- pom-cdh5.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom-cdh5.xml b/pom-cdh5.xml index f0c90ac2..149a07e1 100644 --- a/pom-cdh5.xml +++ b/pom-cdh5.xml @@ -91,17 +91,17 @@ org.apache.hadoop hadoop-common - 2.6.0-cdh5.8.0 + 2.6.0-cdh5.10.0 org.apache.hadoop hadoop-mapreduce-client-common - 2.6.0-cdh5.8.0 + 2.6.0-cdh5.10.0 org.apache.hadoop hadoop-mapreduce-client-core - 2.6.0-cdh5.8.0 + 2.6.0-cdh5.10.0 From 58e85a60d75707da55ed499f836e57d49347484a Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Fri, 28 Apr 2017 17:04:24 +0200 Subject: [PATCH 25/83] Merge with recent master of iipc/webarchive-commons, transfer all changes made to commoncrawl/ia-web-commons unless already included: - WET extraction - htmlparser bump to 2.1 - support file:// input - extract `property` attributes of HTML meta elements - add pom.xml to build with Cloudera CDH (5.11.0) --- pom-cdh5.xml | 296 ++++++++++++++++++ pom.xml | 22 +- .../ExtractingResourceFactoryMapper.java | 5 +- .../org/archive/extract/ProducerUtils.java | 12 +- .../archive/extract/ResourceExtractor.java | 4 + .../archive/extract/WATExtractorOutput.java | 12 +- .../archive/extract/WETExtractorOutput.java | 167 ++++++++++ .../org/archive/format/json/JSONUtils.java | 10 + .../archive/format/warc/WARCConstants.java | 4 +- .../archive/format/warc/WARCRecordWriter.java | 47 ++- .../archive/resource/ResourceConstants.java | 2 +- .../html/ExtractingParseObserver.java | 44 ++- .../archive/resource/html/HTMLMetaData.java | 4 + 13 files changed, 603 insertions(+), 26 deletions(-) create mode 100644 pom-cdh5.xml create mode 100644 src/main/java/org/archive/extract/WETExtractorOutput.java diff --git a/pom-cdh5.xml b/pom-cdh5.xml new file mode 100644 index 00000000..217e499d --- /dev/null +++ b/pom-cdh5.xml @@ -0,0 +1,296 @@ + + 4.0.0 + + + org.sonatype.oss + oss-parent + 7 + + + org.commoncrawl + ia-web-commons + 1.1.8-SNAPSHOT + jar + + ia-web-commons + https://github.com/commoncrawl/ia-web-commons + + + The International Internet Preservation Consortium + http://netpreserve.org/ + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + many-devs + Many Others Developers Proceed Me + many@dev.org + + + anjackson + Andrew Jackson + Andrew.Jackson@bl.uk + + + + GitHub Issues + https://github.com/iipc/webarchive-commons/issues + + + scm:git:git@github.com:iipc/webarchive-commons.git + scm:git:git@github.com:iipc/webarchive-commons.git + git@github.com:iipc/webarchive-commons.git + + + + UTF-8 + ${maven.build.timestamp} + yyyyMMddhhmmss + + + sonatype-nexus-staging + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + sonatype-nexus-snapshots + https://oss.sonatype.org/content/repositories/snapshots/ + + + + + junit + junit + 3.8.1 + + + + com.google.guava + guava + 17.0 + + + + org.json + json + 20131018 + + + org.htmlparser + htmlparser + 2.1 + + + + com.googlecode.juniversalchardet + juniversalchardet + 1.0.3 + + + + commons-httpclient + commons-httpclient + 3.1 + + + + org.apache.hadoop + hadoop-client + 2.6.0-cdh5.11.0 + + + commons-httpclient + commons-httpclient + + + javax.servlet + servlet-api + + + javax.servlet.jsp + jsp-api + + + org.mortbay.jetty + jetty + + + org.mortbay.jetty + jetty-util + + + tomcat + jasper-runtime + + + tomcat + jasper-compiler + + + hsqldb + hsqldb + + + + + org.apache.hadoop + hadoop-common + 2.6.0-cdh5.11.0 + + + org.apache.hadoop + hadoop-mapreduce-client-common + 2.6.0-cdh5.11.0 + + + org.apache.hadoop + hadoop-mapreduce-client-core + 2.6.0-cdh5.11.0 + + + + org.apache.pig + pig + 0.11.1 + provided + + + + commons-lang + commons-lang + 2.5 + + + + commons-io + commons-io + 2.4 + + + + org.gnu.inet + libidn + 1.15 + + + it.unimi.dsi + dsiutils + 2.0.12 + compile + + + ch.qos.logback + logback-classic + + + + + org.apache.httpcomponents + httpcore + 4.3 + + + joda-time + joda-time + 1.6 + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 2.3.2 + + 1.6 + 1.6 + + + + maven-assembly-plugin + 2.4 + + + jar-with-dependencies + + ia-web-commons + + + + package + + single + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + 1.3.1 + + + enforce-maven + + enforce + + + + + This project requires Maven 3.0.5 or higher + 3.0.5 + + + + + + + + + + + src/main/resources + true + + + + + + + cloudera + Cloudera Hadoop + https://repository.cloudera.com/artifactory/cloudera-repos/ + default + + + true + daily + warn + + + true + daily + warn + + + + + + + + diff --git a/pom.xml b/pom.xml index 24780063..7d008123 100644 --- a/pom.xml +++ b/pom.xml @@ -7,13 +7,13 @@ 7 - org.netpreserve.commons - webarchive-commons + org.commoncrawl + ia-web-commons 1.1.8-SNAPSHOT jar - webarchive-commons - https://github.com/iipc/webarchive-commons + ia-web-commons + https://github.com/commoncrawl/ia-web-commons The International Internet Preservation Consortium @@ -81,7 +81,7 @@ org.htmlparser htmlparser - 1.6 + 2.1 @@ -94,7 +94,7 @@ commons-httpclient commons-httpclient 3.1 - + org.apache.hadoop @@ -128,12 +128,12 @@ tomcat jasper-compiler - + hsqldb hsqldb - - + + @@ -160,7 +160,7 @@ libidn 1.15 - + it.unimi.dsi dsiutils 2.0.12 @@ -176,7 +176,7 @@ org.apache.httpcomponents httpcore 4.3 - + joda-time joda-time diff --git a/src/main/java/org/archive/extract/ExtractingResourceFactoryMapper.java b/src/main/java/org/archive/extract/ExtractingResourceFactoryMapper.java index ad10be40..0afe16fb 100644 --- a/src/main/java/org/archive/extract/ExtractingResourceFactoryMapper.java +++ b/src/main/java/org/archive/extract/ExtractingResourceFactoryMapper.java @@ -153,7 +153,10 @@ private boolean isWARCInfoResource(MetaData envelope) { private boolean isHTTPResponseWARCResource(MetaData envelope) { return childFieldEquals(envelope,WARC_HEADER_METADATA, WARCConstants.CONTENT_TYPE, - WARCConstants.HTTP_RESPONSE_MIMETYPE); + WARCConstants.HTTP_RESPONSE_MIMETYPE) + || childFieldEquals(envelope,WARC_HEADER_METADATA, + WARCConstants.CONTENT_TYPE, + WARCConstants.HTTP_RESPONSE_MIMETYPE_NS); } private boolean isWARCJSONResource(MetaData envelope) { return childFieldEquals(envelope,WARC_HEADER_METADATA, diff --git a/src/main/java/org/archive/extract/ProducerUtils.java b/src/main/java/org/archive/extract/ProducerUtils.java index b75d2f15..d8db9630 100644 --- a/src/main/java/org/archive/extract/ProducerUtils.java +++ b/src/main/java/org/archive/extract/ProducerUtils.java @@ -29,7 +29,7 @@ public static ResourceProducer getProducer(String path, long offset) throws IOEx wf.setStrict(STRICT_GZ); File file = new File(path); - if(path.startsWith("hdfs://")) { + if(path.startsWith("hdfs://") || path.startsWith("s3a://")) { String name = file.getName(); Path fsPath = new Path(path); FileSystem fs = fsPath.getFileSystem(new Configuration()); @@ -65,7 +65,15 @@ public static ResourceProducer getProducer(String path, long offset) throws IOEx } else { - if(!(file.exists() && file.canRead())) { + if(path.startsWith("file:/")) { + file = new File(new URL(path).getPath()); + } + + if(!file.exists()) { + System.err.println(path + ": file not found."); + return null; + } + if(!file.canRead()) { System.err.println(path + " is not a readable file."); return null; } diff --git a/src/main/java/org/archive/extract/ResourceExtractor.java b/src/main/java/org/archive/extract/ResourceExtractor.java index 2812aa5b..d15cbed1 100644 --- a/src/main/java/org/archive/extract/ResourceExtractor.java +++ b/src/main/java/org/archive/extract/ResourceExtractor.java @@ -54,6 +54,7 @@ private static int USAGE(int exitCode) { System.err.println("\t\t\t (note that column 1 is NOT standard Wayback canonicalized)\n"); System.err.println("\t\t-wat\tembed JSON output in a compressed WARC" + "wrapper, for storage, or sharing."); + System.err.println("\t\t-wet\tembed text extracts in a compressed WARC" + "wrapper, for storage, or sharing."); return exitCode; } @@ -109,6 +110,9 @@ public int run(String[] args) } else if(args[arg].equals("-wat")) { path = args[arg+1]; out = new WATExtractorOutput(os, outputFile); + } else if (args[arg].equals("-wet")) { + path = args[arg + 1]; + out = new WETExtractorOutput(os); } else { String filter = args[arg+1]; out = new JSONViewExtractorOutput(os, filter); diff --git a/src/main/java/org/archive/extract/WATExtractorOutput.java b/src/main/java/org/archive/extract/WATExtractorOutput.java index 3bcfa924..4d574b91 100644 --- a/src/main/java/org/archive/extract/WATExtractorOutput.java +++ b/src/main/java/org/archive/extract/WATExtractorOutput.java @@ -41,6 +41,10 @@ public class WATExtractorOutput implements ExtractorOutput { private static final Logger LOG = Logger.getLogger(WATExtractorOutput.class.getName()); + public WATExtractorOutput(OutputStream out) { + this(out, null); + } + public WATExtractorOutput(OutputStream out, String outputFile) { gzW = new GZIPMemberWriter(out); recW = new WARCRecordWriter(); @@ -67,6 +71,10 @@ public void output(Resource resource) throws IOException { // hrm... throw new IOException("Missing Envelope.Format"); } + + // remove the text extracts if it exists + JSONUtils.removeObject(top, "Envelope.Payload-Metadata.HTTP-Response-Metadata.HTML-Metadata", "Text"); + cos = getOutput(); if(envelopeFormat.startsWith("ARC")) { writeARC(cos,top); @@ -100,8 +108,8 @@ private void writeWARCInfo(OutputStream recOut, MetaData md) throws IOException File tmpFile = new File(filename); filename = tmpFile.getName(); HttpHeaders headers = new HttpHeaders(); - headers.add("software", IAUtils.COMMONS_VERSION); - headers.addDateHeader("extractedDate", new Date()); + headers.add("Software-Info", IAUtils.COMMONS_VERSION); + headers.addDateHeader("Extracted-Date", new Date()); // add ip, hostname try { diff --git a/src/main/java/org/archive/extract/WETExtractorOutput.java b/src/main/java/org/archive/extract/WETExtractorOutput.java new file mode 100644 index 00000000..14b9553f --- /dev/null +++ b/src/main/java/org/archive/extract/WETExtractorOutput.java @@ -0,0 +1,167 @@ +package org.archive.extract; + +import org.archive.format.gzip.GZIPMemberWriter; +import org.archive.format.gzip.GZIPMemberWriterCommittedOutputStream; +import org.archive.format.http.HttpHeaders; +import org.archive.format.json.JSONUtils; +import org.archive.format.warc.WARCRecordWriter; +import org.archive.resource.MetaData; +import org.archive.resource.Resource; +import org.archive.util.DateUtils; +import org.archive.util.IAUtils; +import org.archive.util.StreamCopy; +import org.archive.util.io.CommitedOutputStream; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.Charset; +import java.text.ParseException; +import java.util.Date; + +/** + * This is for generating a WARC Encapsulated Text file + * + * These are implemented as WARC conversion records. Only + * Envelope.Payload-Metadata.HTTP-Response-Metadata.HTML-Metadata.Text fields are included + */ +public class WETExtractorOutput implements ExtractorOutput { + WARCRecordWriter recW; + private boolean wroteFirst; + private GZIPMemberWriter gzW; + private static int DEFAULT_BUFFER_RAM = 1024 * 1024; + private int bufferRAM = DEFAULT_BUFFER_RAM; + private final static Charset UTF8 = Charset.forName("UTF-8"); + private String outFilename; + + public WETExtractorOutput(OutputStream out) { + this(out, null); + } + + public WETExtractorOutput(OutputStream out, String filename) { + gzW = new GZIPMemberWriter(out); + recW = new WARCRecordWriter(); + wroteFirst = false; + outFilename = filename; + } + + private CommitedOutputStream getOutput() { + return new GZIPMemberWriterCommittedOutputStream(gzW,bufferRAM); + } + + + private String extractOrIO(MetaData md, String path) throws IOException { + String value = JSONUtils.extractSingle(md, path); + if(value == null) { + throw new IOException("No "+path+" found."); + } + return value; + } + + public void output(Resource resource) throws IOException { + StreamCopy.readToEOF(resource.getInputStream()); + MetaData top = resource.getMetaData().getTopMetaData(); + CommitedOutputStream cos; + + if(!wroteFirst) { + cos = getOutput(); + writeWARCInfo(cos, top); + cos.commit(); + wroteFirst = true; + } + String envelopeFormat = JSONUtils.extractSingle(top, "Envelope.Format"); + if(envelopeFormat == null) { + throw new IOException("Missing Envelope.Format"); + } + + String warctype = JSONUtils.extractSingle(top, "Envelope.WARC-Header-Metadata.WARC-Type"); + if (warctype != null && warctype.equals("response")) { + String textExtract = JSONUtils.extractSingle(top, "Envelope.Payload-Metadata.HTTP-Response-Metadata.HTML-Metadata.Text"); + + if (textExtract != null) { + cos = getOutput(); + if(envelopeFormat.startsWith("WARC")) { + writeWARC(cos, top, textExtract); + } else { + // hrm... + throw new IOException("Unknown Envelope.Format"); + } + cos.commit(); + } + } + } + + private void writeWARCInfo(OutputStream recOut, MetaData md) throws IOException { + String filename = outFilename; + + if (filename == null) { + filename = JSONUtils.extractSingle(md, "Container.Filename"); + + if(filename == null) { + throw new IOException("No Container.Filename..."); + } + } + + HttpHeaders headers = new HttpHeaders(); + headers.add("Software-Info", IAUtils.COMMONS_VERSION); + headers.addDateHeader("Extracted-Date", new Date()); + + // Dup out some useful headers from the incoming warcinfo + String warctype = JSONUtils.extractSingle(md, "Envelope.WARC-Header-Metadata.WARC-Type"); + if (warctype != null && warctype.equals("warcinfo")) { + final String[] usefulHeaders = {"robots", "isPartOf", "operator", "description", "publisher"}; + + for (String header : usefulHeaders) { + String value = JSONUtils.extractSingle(md, "Envelope.Payload-Metadata.WARC-Info-Metadata." + header); + if (value != null) { + headers.add(header, value); + } + } + } + + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + headers.write(baos); + recW.writeWARCInfoRecord(recOut, filename, baos.toByteArray()); + } + + private void writeWARC(OutputStream recOut, MetaData md, String textExtract) throws IOException { + String targetURI = extractOrIO(md, "Envelope.WARC-Header-Metadata.WARC-Target-URI"); + + String capDateString = extractOrIO(md, "Envelope.WARC-Header-Metadata.WARC-Date"); + capDateString = transformWARCDate(capDateString); + String recId = extractOrIO(md, "Envelope.WARC-Header-Metadata.WARC-Record-ID"); + writeWARCMDRecord(recOut, targetURI, capDateString, recId, textExtract); + } + + private void writeWARCMDRecord(OutputStream recOut, String targetURI, String capDateString, String recId, + String textExtract) + throws IOException { + + Date capDate; + try { + capDate = DateUtils.getSecondsSinceEpoch(capDateString); + + } catch (ParseException e) { + e.printStackTrace(); + // TODO... not the write thing... + capDate = new Date(); + } + + recW.writeTextConversionRecord(recOut, textExtract.getBytes("UTF-8"), targetURI, capDate, recId); + } + + private static String transformWARCDate(final String input) { + + StringBuilder output = new StringBuilder(14); + + output.append(input.substring(0,4)); + output.append(input.substring(5,7)); + output.append(input.substring(8,10)); + output.append(input.substring(11,13)); + output.append(input.substring(14,16)); + output.append(input.substring(17,19)); + + return output.toString(); + } +} diff --git a/src/main/java/org/archive/format/json/JSONUtils.java b/src/main/java/org/archive/format/json/JSONUtils.java index 28f4f43e..946b633b 100644 --- a/src/main/java/org/archive/format/json/JSONUtils.java +++ b/src/main/java/org/archive/format/json/JSONUtils.java @@ -114,4 +114,14 @@ private static void extractRecursive(JSONObject json, String path[], int idx, Li } } } + public static boolean removeObject(JSONObject json, String path, String node) { + JSONObject obj = extractObject(json, "Envelope.Payload-Metadata.HTTP-Response-Metadata.HTML-Metadata"); + if (obj != null) { + if (obj.remove("Text") != null) { + return true; + } + } + + return false; + } } diff --git a/src/main/java/org/archive/format/warc/WARCConstants.java b/src/main/java/org/archive/format/warc/WARCConstants.java index 93a81f96..4f2fa574 100644 --- a/src/main/java/org/archive/format/warc/WARCConstants.java +++ b/src/main/java/org/archive/format/warc/WARCConstants.java @@ -209,7 +209,9 @@ enum WARCRecordType { "application/http; msgtype=request"; public static final String HTTP_RESPONSE_MIMETYPE = "application/http; msgtype=response"; - + public static final String HTTP_RESPONSE_MIMETYPE_NS = + "application/http;msgtype=response"; // wget does this + public static final String FTP_CONTROL_CONVERSATION_MIMETYPE = "text/x-ftp-control-conversation"; diff --git a/src/main/java/org/archive/format/warc/WARCRecordWriter.java b/src/main/java/org/archive/format/warc/WARCRecordWriter.java index 0aab83b7..943410b9 100644 --- a/src/main/java/org/archive/format/warc/WARCRecordWriter.java +++ b/src/main/java/org/archive/format/warc/WARCRecordWriter.java @@ -2,18 +2,32 @@ import java.io.IOException; import java.io.OutputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.Date; import java.util.UUID; import org.archive.format.http.HttpConstants; import org.archive.format.http.HttpHeaders; +import org.archive.util.Base32; import org.archive.util.DateUtils; -public class WARCRecordWriter implements WARCConstants, HttpConstants -{ - private static final String SCHEME = "urn:uuid"; - private static final String SCHEME_COLON = SCHEME + ":"; - +public class WARCRecordWriter implements WARCConstants, HttpConstants { + private static final String SCHEME = "urn:uuid"; + private static final String SCHEME_COLON = SCHEME + ":"; + private MessageDigest sha1; + private Base32 base32; + + public WARCRecordWriter() { + try { + sha1 = MessageDigest.getInstance("SHA1"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + + base32 = new Base32(); + } + /** * Write the headers and contents as a WARC record to the given * output stream. @@ -97,6 +111,29 @@ public void writeJSONMetadataRecord( OutputStream out, writeRecord(out, headers, contents); } + public void writeTextConversionRecord( OutputStream out, + byte[] contents, + String targetURI, + Date originalDate, + String origRecordId) throws IOException + { + HttpHeaders headers = new HttpHeaders(); + headers.add(HEADER_KEY_TYPE, WARCRecordType.conversion.name()); + headers.add(HEADER_KEY_URI, targetURI); + headers.add(HEADER_KEY_DATE, DateUtils.getLog14Date(originalDate)); + headers.add(HEADER_KEY_ID, makeRecordId()); + headers.add(HEADER_KEY_REFERS_TO, origRecordId); + headers.add(HEADER_KEY_BLOCK_DIGEST, contentHash(contents)); + + headers.add(CONTENT_TYPE, "text/plain"); + writeRecord(out, headers, contents); + } + + private String contentHash(byte[] content) { + sha1.reset(); + return "sha1:" + base32.encode(sha1.digest(content)); + } + private String makeRecordId() { StringBuilder recID = new StringBuilder(); diff --git a/src/main/java/org/archive/resource/ResourceConstants.java b/src/main/java/org/archive/resource/ResourceConstants.java index 3b8bea1c..9eea22b5 100644 --- a/src/main/java/org/archive/resource/ResourceConstants.java +++ b/src/main/java/org/archive/resource/ResourceConstants.java @@ -115,5 +115,5 @@ public interface ResourceConstants { public static final String HTML_LINK_TAGS = "Link"; public static final String HTML_META_TAGS = "Metas"; public static final String HTML_SCRIPT_TAGS = "Scripts"; - + public static final String HTML_TEXT = "Text"; } diff --git a/src/main/java/org/archive/resource/html/ExtractingParseObserver.java b/src/main/java/org/archive/resource/html/ExtractingParseObserver.java index 826851e0..0af6c018 100644 --- a/src/main/java/org/archive/resource/html/ExtractingParseObserver.java +++ b/src/main/java/org/archive/resource/html/ExtractingParseObserver.java @@ -16,14 +16,17 @@ import org.htmlparser.nodes.RemarkNode; import org.htmlparser.nodes.TagNode; import org.htmlparser.nodes.TextNode; +import org.htmlparser.util.Translate; public class ExtractingParseObserver implements ParseObserver { HTMLMetaData data; Stack> openAnchors; Stack openAnchorTexts; + StringBuffer textExtract; String title = null; boolean inTitle = false; + boolean inPre = false; protected static String cssUrlPatString = "url\\s*\$\\s*((?:\\\\?[\"'])?.+?(?:\\\\?[\"'])?)\\s*\$"; @@ -84,6 +87,7 @@ public ExtractingParseObserver(HTMLMetaData data) { this.data = data; openAnchors = new Stack>(); openAnchorTexts = new Stack(); + textExtract = new StringBuffer(8192); } public void handleDocumentStart() { @@ -91,7 +95,10 @@ public void handleDocumentStart() { } public void handleDocumentComplete() { - // no-op + if (textExtract.length() > 0) { + data.setTextExtract(textExtract.toString()); + textExtract = new StringBuffer(8192); + } } public void handleTagEmpty(TagNode tag) { @@ -103,6 +110,8 @@ public void handleTagOpen(TagNode tag) { if(name.equals("TITLE")) { inTitle = !tag.isEmptyXmlTag(); return; + } else if (name.equals("PRE")) { + inPre = true; } // first the global attributes: @@ -134,6 +143,7 @@ public void handleTagClose(TagNode tag) { // probably the right thing.. return; } + // Only interesting if it's a : if(tag.getTagName().equals("A")) { if(openAnchors.size() > 0) { @@ -155,12 +165,40 @@ public void handleTagClose(TagNode tag) { data.addHref(vals); } } + } else if (tag.getTagName().equals("PRE")) { + inPre = false; } } public void handleTextNode(TextNode text) { // TODO: OPTIMIZ: This can be a lot smarter, if StringBuilders are full, - // this result is thrown away. + // this result is thrown away. + // System.out.println("JDBUG: Got text from node: " + + // text.getText().toString()); + + String txt = text.getText(); + if (!inPre) { + txt = Translate.decode(txt); + txt = txt.replace('\u00a0', ' '); + + char c = ' '; + if (textExtract.length() > 0) { + c = textExtract.charAt(textExtract.length() - 1); + } + for (int i = 0; i < txt.length(); i++) { + char c2 = txt.charAt(i); + // Translate so output is a bit cleaner + if (c2 == '\r') { + c2 = '\n'; + } + if (!Character.isWhitespace(c) || !Character.isWhitespace(c2)) { + textExtract.append(c2); + } + c = c2; + } + } else + textExtract.append(txt); + String t = text.getText().replaceAll("\\s+", " "); if(t.length() > MAX_TEXT_LEN) { @@ -406,7 +444,7 @@ public void extract(HTMLMetaData data, TagNode node, ExtractingParseObserver obs private static class MetaTagExtractor implements TagExtractor { public void extract(HTMLMetaData data, TagNode node, ExtractingParseObserver obs) { - ArrayList l = getAttrList(node,"name","rel","content","http-equiv"); + ArrayList l = getAttrList(node,"name","rel","content","http-equiv","property"); if(l != null) { data.addMeta(l); } diff --git a/src/main/java/org/archive/resource/html/HTMLMetaData.java b/src/main/java/org/archive/resource/html/HTMLMetaData.java index 024d9677..b95dcad3 100644 --- a/src/main/java/org/archive/resource/html/HTMLMetaData.java +++ b/src/main/java/org/archive/resource/html/HTMLMetaData.java @@ -69,6 +69,10 @@ public void addHref(String...a) { appendObj2(this,HTML_LINKS,a); } + public void setTextExtract(String textExtract) { + putUnlessNull(this,HTML_TEXT, textExtract); + } + private void appendObj2(JSONObject o, String arr, String... a) { if(o == null) { return; From aad4e05bc8bf25405747d5110eb2aea178a9a6f0 Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Fri, 28 Apr 2017 22:41:56 +0200 Subject: [PATCH 26/83] Do not add value of preceding HTTP header field if there is no value (or only white space), this fixes #11 --- .../archive/format/http/HttpHeaderParser.java | 4 ++-- .../format/http/HttpResponseParserTest.java | 24 +++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/archive/format/http/HttpHeaderParser.java b/src/main/java/org/archive/format/http/HttpHeaderParser.java index d63ec405..bee3c28b 100755 --- a/src/main/java/org/archive/format/http/HttpHeaderParser.java +++ b/src/main/java/org/archive/format/http/HttpHeaderParser.java @@ -301,8 +301,9 @@ public ParseState handleByte(byte b, HttpHeaderParser parser) throws HttpParseEx if(isLWSP(b)) { return parser.postColonState; } + // reset previous value also in case the header value is empty + parser.setValueStartIdx(); if(b == CR) { - // TODO: THINK more... parser.valuePreCRState = parser.postColonState; return parser.valuePostCRState; } @@ -310,7 +311,6 @@ public ParseState handleByte(byte b, HttpHeaderParser parser) throws HttpParseEx // TODO: this is lax, is LFLF an OK terminator? return parser.lineStartState; } - parser.setValueStartIdx(); parser.addValueByte(b); return parser.valueState; } diff --git a/src/test/java/org/archive/format/http/HttpResponseParserTest.java b/src/test/java/org/archive/format/http/HttpResponseParserTest.java index c0d13230..ea076a69 100644 --- a/src/test/java/org/archive/format/http/HttpResponseParserTest.java +++ b/src/test/java/org/archive/format/http/HttpResponseParserTest.java @@ -57,4 +57,28 @@ public void testParseWithLf() throws IOException { } + public void testParseEmptyHeaderField() throws IOException { + + HttpResponseParser parser = new HttpResponseParser(); + String message = "200 OK\r\nContent-Type: text/plain\r\nServer: \r\n\r\nHi there"; + try { + HttpResponse response = + parser.parse(new ByteArrayInputStream(message.getBytes(IAUtils.UTF8))); + assertNotNull(response); + HttpHeaders headers = response.getHeaders(); + assertNotNull(headers); + assertEquals(2, headers.size()); + HttpHeader header = headers.get(1); + assertEquals("Server",header.getName()); + System.err.println(header.getValue()); + assertFalse("text/plain".equals(header.getValue())); + TestUtils.assertStreamEquals(response, "Hi there".getBytes(IAUtils.UTF8)); + + } catch (HttpParseException e) { + e.printStackTrace(); + fail(); + } + + } + } From 3fac1894535661de2841dcd0fa79d09ef4a8c7b4 Mon Sep 17 00:00:00 2001 From: Lauren Ko Date: Tue, 2 May 2017 15:15:06 -0500 Subject: [PATCH 27/83] Fix HTTP-Response-Metadata for wget WARCs. Changes came from https://github.com/commoncrawl/ia-web-commons/commit/58e85a60d75707da55ed499f836e57d49347484a --- src/main/java/org/archive/format/warc/WARCConstants.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/archive/format/warc/WARCConstants.java b/src/main/java/org/archive/format/warc/WARCConstants.java index 4f2fa574..504dc380 100644 --- a/src/main/java/org/archive/format/warc/WARCConstants.java +++ b/src/main/java/org/archive/format/warc/WARCConstants.java @@ -209,8 +209,8 @@ enum WARCRecordType { "application/http; msgtype=request"; public static final String HTTP_RESPONSE_MIMETYPE = "application/http; msgtype=response"; - public static final String HTTP_RESPONSE_MIMETYPE_NS = - "application/http;msgtype=response"; // wget does this + public static final String HTTP_RESPONSE_MIMETYPE_NS = + "application/http;msgtype=response"; // wget does this public static final String FTP_CONTROL_CONVERSATION_MIMETYPE = "text/x-ftp-control-conversation"; From 9722bcc65f74c17489329388ffb64b6151d28d97 Mon Sep 17 00:00:00 2001 From: Lauren Ko Date: Tue, 2 May 2017 15:41:23 -0500 Subject: [PATCH 28/83] Update with fixes for 1.1.9 --- CHANGES.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index ccdc1ce7..1ba5c1de 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,9 @@ +1.1.9 +----- +* [Extract `property` attributes of HTML meta elements](https://github.com/iipc/webarchive-commons/pull/75) +* [Do not add value of preceding HTTP header field if there is no value](https://github.com/iipc/webarchive-commons/pull/74) +* [Fix WAT records corresponding to response records of Wget generated WARCs](https://github.com/iipc/webarchive-commons/pull/74) + 1.1.8 ----- * [Improve HTML link extraction](https://github.com/iipc/webarchive-commons/pull/72) From a0dcbc545349732432b1a99deaec89e3ff737ce7 Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Thu, 4 May 2017 13:46:16 +0200 Subject: [PATCH 29/83] bump version, sync with iipc/webarchive-commons --- pom-cdh5.xml | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom-cdh5.xml b/pom-cdh5.xml index 217e499d..6ffe1233 100644 --- a/pom-cdh5.xml +++ b/pom-cdh5.xml @@ -9,7 +9,7 @@ org.commoncrawl ia-web-commons - 1.1.8-SNAPSHOT + 1.1.9-SNAPSHOT jar ia-web-commons diff --git a/pom.xml b/pom.xml index 7d008123..56fdc5f6 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ org.commoncrawl ia-web-commons - 1.1.8-SNAPSHOT + 1.1.9-SNAPSHOT jar ia-web-commons From ea00e6dbda39b82ddfa3f82e4865e559d544fa1c Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Mon, 5 Jun 2017 13:27:13 +0200 Subject: [PATCH 30/83] Limit pattern matching URLs embedded in CSS to match max. 8000 characters, add unit test, fixes commoncrawl/ia-web-commons#12 --- .../resource/html/ExtractingParseObserver.java | 2 +- .../html/ExtractingParseObserverTest.java | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/archive/resource/html/ExtractingParseObserver.java b/src/main/java/org/archive/resource/html/ExtractingParseObserver.java index 0af6c018..6d72bc63 100644 --- a/src/main/java/org/archive/resource/html/ExtractingParseObserver.java +++ b/src/main/java/org/archive/resource/html/ExtractingParseObserver.java @@ -29,7 +29,7 @@ public class ExtractingParseObserver implements ParseObserver { boolean inPre = false; protected static String cssUrlPatString = - "url\\s*\$\\s*((?:\\\\?[\"'])?.+?(?:\\\\?[\"'])?)\\s*\$"; + "url\\s*\$\\s*([^)\\s]{1,8000}?)\\s*\$"; protected static String cssUrlTrimPatString = "^(?:\\\\?[\"'])+|(?:\\\\?[\"'])+$"; protected static String cssImportNoUrlPatString = diff --git a/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java b/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java index 8f690a06..a7fa272f 100644 --- a/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java +++ b/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java @@ -93,6 +93,24 @@ public void testHandleStyleNodeNoHangupTruncated() throws Exception { checkExtract(test); } + /** + * Test whether the pattern matcher does not stack overflow with overlong + * sequence of quote characters around a CSS link. + */ + public void testHandleStyleNodeNoStackOverflow() throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("url("); + for (int i = 0; i < 20000; i++) + sb.append('\''); + sb.append("foos.gif"); + for (int i = 0; i < 20000; i++) + sb.append('\''); + sb.append(");"); + String[] test = new String[1]; + test[0] = sb.toString(); + checkExtract(test); + } + private void checkExtract(String[] data) throws JSONException { // System.err.format("CSS(%s) want[0](%s)\n",css,want[0]); String css = data[0]; From a9d9ed9b4291a6a2e496235acf287ab3fb0a2a37 Mon Sep 17 00:00:00 2001 From: Naomi Dushay Date: Tue, 8 Aug 2017 16:08:43 -0700 Subject: [PATCH 31/83] use commons-collections v3.2.2 to avoid v3.2.1 vulnerability --- pom.xml | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 56fdc5f6..ae1787a7 100644 --- a/pom.xml +++ b/pom.xml @@ -72,7 +72,7 @@ guava 17.0 - + org.json json @@ -89,7 +89,7 @@ juniversalchardet 1.0.3 - + commons-httpclient commons-httpclient @@ -170,8 +170,21 @@ ch.qos.logback logback-classic + + + commons-collections + commons-collections + + + + + commons-collections + commons-collections + 3.2.2 + + org.apache.httpcomponents httpcore From b2252fc05e9b25509c9a3ee422a821b3b03cd99b Mon Sep 17 00:00:00 2001 From: Lauren Ko Date: Wed, 9 Aug 2017 10:57:28 -0500 Subject: [PATCH 32/83] Update CHANGES.md for PR 77 --- CHANGES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.md b/CHANGES.md index 1ba5c1de..dcb598d9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,6 @@ 1.1.9 ----- +* [Use commons-collections v3.2.2 to avoid v3.2.1 vulnerability](https://github.com/iipc/webarchive-commons/pull/77) * [Extract `property` attributes of HTML meta elements](https://github.com/iipc/webarchive-commons/pull/75) * [Do not add value of preceding HTTP header field if there is no value](https://github.com/iipc/webarchive-commons/pull/74) * [Fix WAT records corresponding to response records of Wget generated WARCs](https://github.com/iipc/webarchive-commons/pull/74) From 7ce4e8849fd4a8ff31ec56875bf9022f481072c1 Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Wed, 23 Aug 2017 13:25:47 +0200 Subject: [PATCH 33/83] WET extractor: improve spacing of textual content, fixes #13 - rely on HTML elements (block vs. inline) for spacing and line breaks --- .../html/ExtractingParseObserver.java | 84 ++++++++++++++---- .../html/ExtractingParseObserverTest.java | 30 +++++++ .../resource/html/text-extraction-test.warc | Bin 0 -> 3649 bytes 3 files changed, 99 insertions(+), 15 deletions(-) create mode 100644 src/test/resources/org/archive/resource/html/text-extraction-test.warc diff --git a/src/main/java/org/archive/resource/html/ExtractingParseObserver.java b/src/main/java/org/archive/resource/html/ExtractingParseObserver.java index 6d72bc63..e183c25f 100644 --- a/src/main/java/org/archive/resource/html/ExtractingParseObserver.java +++ b/src/main/java/org/archive/resource/html/ExtractingParseObserver.java @@ -23,7 +23,7 @@ public class ExtractingParseObserver implements ParseObserver { HTMLMetaData data; Stack> openAnchors; Stack openAnchorTexts; - StringBuffer textExtract; + StringBuilder textExtract; String title = null; boolean inTitle = false; boolean inPre = false; @@ -44,6 +44,27 @@ public class ExtractingParseObserver implements ParseObserver { private final static int MAX_TEXT_LEN = 100; + private final static String[] BLOCK_ELEMENTS = { "address", "article", "aside", "blockquote", "body", "br", + "button", "canvas", "caption", "col", "colgroup", "dd", "div", "dl", "dt", "embed", "fieldset", + "figcaption", "figure", "footer", "form", "h1", "h2", "h3", "h4", "h5", "h6", "header", "hgroup", "hr", + "li", "map", "noscript", "object", "ol", "output", "p", "pre", "progress", "section", "table", "tbody", + "textarea", "tfoot", "th", "thead", "tr", "ul", "video" }; + private static final Set blockElements; + /* inline elements which content is not melted with surrounding words */ + private final static String[] INLINE_ELEMENTS_SPACING = { "address", "cite", "details", "datalist", "iframe", "img", + "input", "label", "legend", "optgroup", "q", "select", "summary", "tbody", "td", "time" }; + private static final Set inlineSpacingElements; + static { + blockElements = new HashSet(); + for (String el : BLOCK_ELEMENTS) { + blockElements.add(el.toUpperCase(Locale.ROOT)); + } + inlineSpacingElements = new HashSet(); + for (String el : INLINE_ELEMENTS_SPACING) { + inlineSpacingElements.add(el.toUpperCase(Locale.ROOT)); + } + } + private static final String PATH = "path"; private static final String PATH_SEPARATOR = "@/"; private static final Map extractors; @@ -87,7 +108,7 @@ public ExtractingParseObserver(HTMLMetaData data) { this.data = data; openAnchors = new Stack>(); openAnchorTexts = new Stack(); - textExtract = new StringBuffer(8192); + textExtract = new StringBuilder(8192); } public void handleDocumentStart() { @@ -97,7 +118,7 @@ public void handleDocumentStart() { public void handleDocumentComplete() { if (textExtract.length() > 0) { data.setTextExtract(textExtract.toString()); - textExtract = new StringBuffer(8192); + textExtract = new StringBuilder(8192); } } @@ -114,6 +135,12 @@ public void handleTagOpen(TagNode tag) { inPre = true; } + if (blockElements.contains(name)) { + appendParagraphSeparator(textExtract); + } else if (inlineSpacingElements.contains(name)) { + appendSpace(textExtract); + } + // first the global attributes: Vector attributes = tag.getAttributesEx(); for (Attribute a : attributes) { @@ -136,16 +163,22 @@ public void handleTagOpen(TagNode tag) { } public void handleTagClose(TagNode tag) { + String name = tag.getTagName(); + if(inTitle) { inTitle = false; data.setTitle(title); title = null; - // probably the right thing.. - return; + } + + if (blockElements.contains(name)) { + appendParagraphSeparator(textExtract); + } else if (inlineSpacingElements.contains(name)) { + appendSpace(textExtract); } // Only interesting if it's a : - if(tag.getTagName().equals("A")) { + if(name.equals("A")) { if(openAnchors.size() > 0) { // TODO: what happens here when we get unaligned (extra 's?) ArrayList vals = openAnchors.pop(); @@ -173,12 +206,12 @@ public void handleTagClose(TagNode tag) { public void handleTextNode(TextNode text) { // TODO: OPTIMIZ: This can be a lot smarter, if StringBuilders are full, // this result is thrown away. - // System.out.println("JDBUG: Got text from node: " + - // text.getText().toString()); String txt = text.getText(); - if (!inPre) { - txt = Translate.decode(txt); + txt = Translate.decode(txt); + if (inPre) { + textExtract.append(txt); + } else { txt = txt.replace('\u00a0', ' '); char c = ' '; @@ -187,17 +220,15 @@ public void handleTextNode(TextNode text) { } for (int i = 0; i < txt.length(); i++) { char c2 = txt.charAt(i); - // Translate so output is a bit cleaner - if (c2 == '\r') { - c2 = '\n'; + if (c2 == '\r' || c2 == '\n') { + c2 = ' '; } if (!Character.isWhitespace(c) || !Character.isWhitespace(c2)) { textExtract.append(c2); } c = c2; } - } else - textExtract.append(txt); + } String t = text.getText().replaceAll("\\s+", " "); @@ -308,6 +339,29 @@ private static void addHrefWithAttrs(HTMLMetaData data, TagNode node, } } + private static void appendParagraphSeparator(StringBuilder sb) { + int length = sb.length(); + if (length > 0) { + // remove white space before paragraph break + while (length > 0 && sb.charAt(length - 1) == ' ') { + sb.deleteCharAt(--length); + } + if (length > 0 && sb.charAt(length - 1) != '\n') { + sb.append('\n'); + } + } + } + + private static void appendSpace(StringBuilder sb) { + int length = sb.length(); + if (length > 0) { + char lastBufferChar = sb.charAt(length - 1); + if (lastBufferChar != ' ' && lastBufferChar != '\n') { + sb.append(' '); + } + } + } + private interface TagExtractor { public void extract(HTMLMetaData data, TagNode node, ExtractingParseObserver obs); } diff --git a/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java b/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java index a7fa272f..67f38a8d 100644 --- a/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java +++ b/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java @@ -11,6 +11,7 @@ import org.archive.extract.ResourceFactoryMapper; import org.archive.resource.MetaData; import org.archive.resource.Resource; +import org.archive.resource.ResourceConstants; import org.archive.resource.ResourceParseException; import org.archive.resource.ResourceProducer; import org.htmlparser.nodes.TextNode; @@ -283,4 +284,33 @@ public void testLinkExtraction() throws ResourceParseException, IOException { checkLinks(extractor.getNext(), fbSocialLinks); } + public void testTextExtraction() throws ResourceParseException, IOException { + String testFileName = "text-extraction-test.warc"; + ResourceProducer producer = ProducerUtils.getProducer(getClass().getResource(testFileName).getPath()); + ResourceFactoryMapper mapper = new ExtractingResourceFactoryMapper(); + ExtractingResourceProducer extractor = new ExtractingResourceProducer(producer, mapper); + extractor.getNext(); // skip warcinfo record + Resource resource = extractor.getNext(); + assertNotNull(resource); + assertTrue("Wrong instance type of Resource: " + resource.getClass(), resource instanceof HTMLResource); + String text = resource.getMetaData().getString(ResourceConstants.HTML_TEXT); + System.out.println(text); + assertTrue(text.contains("text\nThere should be a paragraph break after ")); + assertTrue(text.contains("آ«foobarfooآ»")); + assertFalse(text.contains("آ«foo bar fooآ»")); + assertTrue(text.contains("comments: nospace")); + assertFalse(text.contains("before an imageand after")); + assertFalse(text.contains("firstsecond line")); + assertFalse(text.contains("first linediv element")); + assertFalse(text.contains("div elementsecond line")); + assertFalse(text.contains("2017by")); + assertFalse(text.contains("Heath9")); + assertFalse(text.contains("readAdd")); + assertTrue(text.contains("read\nAdd")); + assertFalse(text.contains("first linesecond line")); + assertTrue(text.contains("first line\nsecond line\n")); + // TODO: CDATA in mathml not correctly parsed + // assertTrue(text.matches("CDATA in MathML:\\W*xA6L30Tx`|t|(uH9WG+cq$qAvq)mp-5sl zw4@w@AZStalxq+D3tFH+552U90zMb$r9dye^j7rJV~Zd`^3724%4@}N^Vo$Xa`-)e z-;-yPlU>8o&5n8oS6Z;Fh>PZ!MYKrDj=EbGoD_U@j;kuC5$3ck3_+tMl;m-};@!g> zcx}@fYNoB(W?=PQ({!D|PtWDtAw|kF7g_eGqjuD!WHZcNBy^(#niF&&Pmr`ID!~Re ztAo--8RwO27$FUdW~PTlq9@D{Z1U^D^z`lnd*qnrgNKCGV4w9M2Xks3Gn{a}ij$7Y z%CO|Mi=rIYHI0OR@h^p3M1YaHDvO$|VZkgVTzmUu`q+~Be6EuPo>e)~qjF|se3sjs zEVx!z7|=^~ak2%Pk!Fnp+jOn5YmPRcX`$bQSonN%R;L_-vU6_HIwor@ZF4vjjvNNT zJqV=*Au!S1+a0wH$Qg-mAiIYqO)7(fEYr1Z(2Z^#1s(O2Fo4WO6RUL43XL<&2sHwD{Vi(sf)UUxTiVJa4@!dR&2(2)a|H~C?XZt zPH>SDh(fr8@N7wp;Q%t@9n6+sQp>SlM4-M%JL+vhDs7U}hmdES&Pdtt9p?s`;l+++ zLJ9*i1&Ks@y6BwtTuBSD2$Bq!0L{CycXv1V$-RBZuvzY_UPf>XKb-QM_-7gA1c`tW zgo_wenBkP+DnlV7`28BqGg2VPQ-F|ziRdsmx`TL$gbvSWg{a<)aEzi1vzqXoZo?C8 z)CC

=Qlq(!chH_dE3BU~`(o#o4L?ze}JQX(~6L-2QoT~ie6|ZMh z(#-~dM)2q5)W@uB09jfHp@`&U2K}Z+*Q8!4N=UlyqDQ|?%F+w{5HrsReYkn_C(i&0 zk41q52+1|Ugr8u-C}Skq=`NNZ$g1qNLhv>rP|MN+7d-Z^Xc|OU9w1HgWg@I~dWLYL zb#Mx;g7FrhZBB*+8ktBTL}xf99-XDAX3<~m5b``8m?wCG&ioNB5Cq8`E(ew;) zutn%5gnKkeXw>9flurh3=VY7x7xQFJd4_gRPoW90lomB2Jkp`N*#}a&gnC045NE&?9f1f}*F_FvwD{ zy9wpw5D8Wm5ySJ`pjexeP-~BBgVuFJYQySv{ja8a&FXmu_T4pH0{NAzavsqlhLQq* z&1!M8>w6*?9;9;NA3zm1&}xwlT~rKi@JXDv0ox!9muMG@Rv>Ehkig~Blw&lb1@r?P zZ?dobuF9J0QA)f1B#zN;TT{a96|qbaK)9_Z*-sd0le6s^6(hbXQ&i@dx{*$*AO{yI zUkVN(!jR;xu-&~$Fp=#SN6_`4LmZc4Q+X;TVx-S|ddX5FIFS>=G3sT(@j`Zu*P{z#ZZl(e{CvxdoF@j14S~%s?;YRjs6S8@Md?fr_fFvVlK5PH5x)QM z!&hH`pBK}~(L_;x|H&(#;g{eZMTaLxit@o#MfvDwit;HOee^FyX-q}=_j`)s{z_54 zU;X-zKl0$g3*qfsd&(ET_~b8thP#*f^l1O(e_sAx$9dy#cTfHSN3Tqw)-L}&`rluH z_|hfiH;;ZkfaBNs{*O5n4{;;t51c5_YX+J*)~tSD_gu5@I>R?j*h$U*-v5vO1j9^h zo=d^tXr`svy})wB>f9f{Xc<*;_z4!4~=DFCxME$N(^#n#7i?!(< z^nr#IZqv}%?cK*BVhy-%Mim|2j ziPh8kb}!V%aUYx=nqzC=M0PR=*CK+Aw~DoSVl)(u>_K&zaCpli#vI-~^&XIU7Hz4CW>X$A|UDyBs literal 0 HcmV?d00001 From dda2c89230603e4ce1f7f93cba6e4ad5472d5d43 Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Wed, 23 Aug 2017 14:48:05 +0200 Subject: [PATCH 34/83] WAT extracor: add rel attribute to A@/href links, fixes #10 - add "rel" attribute to A and AREA links - add attributes "hreflang" and "type" (MIME type) to A@/href links --- .../html/ExtractingParseObserver.java | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/archive/resource/html/ExtractingParseObserver.java b/src/main/java/org/archive/resource/html/ExtractingParseObserver.java index e183c25f..87769d90 100644 --- a/src/main/java/org/archive/resource/html/ExtractingParseObserver.java +++ b/src/main/java/org/archive/resource/html/ExtractingParseObserver.java @@ -376,7 +376,7 @@ public void extract(HTMLMetaData data, TagNode node, ExtractingParseObserver obs l.add(makePath("A","href")); l.add("url"); l.add(url); - for(String a : new String[] {"target","alt","title"}) { + for(String a : new String[] {"target","alt","title","rel","hreflang","type"}) { String v = node.getAttribute(a); if(v != null) { l.add(a); @@ -403,7 +403,22 @@ public void extract(HTMLMetaData data, TagNode node, ExtractingParseObserver obs private static class AreaTagExtractor implements TagExtractor { public void extract(HTMLMetaData data, TagNode node, ExtractingParseObserver obs) { - addBasicHrefs(data,node,"href"); + String url = node.getAttribute("href"); + if(url != null) { + ArrayList l = new ArrayList(); + l.add(PATH); + l.add(makePath("AREA","href")); + l.add("url"); + l.add(url); + for(String a : new String[] {"rel"}) { + String v = node.getAttribute(a); + if(v != null) { + l.add(a); + l.add(v); + } + } + data.addHref(l); + } } } From ea0729b91593c798a93f54a1462c76d09ef4d628 Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Wed, 23 Aug 2017 17:04:52 +0200 Subject: [PATCH 35/83] WAT extractor: get links from onClick attributes, fixes #8 - extract links from JavaScript code snippets in onClick attributes of INPUT and DIV elements --- .../html/ExtractingParseObserver.java | 40 +++++++++++++++++- .../html/ExtractingParseObserverTest.java | 10 +++++ .../resource/html/link-extraction-test.warc | 42 +++++++++++++++++++ 3 files changed, 91 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/archive/resource/html/ExtractingParseObserver.java b/src/main/java/org/archive/resource/html/ExtractingParseObserver.java index 87769d90..8b4c36d6 100644 --- a/src/main/java/org/archive/resource/html/ExtractingParseObserver.java +++ b/src/main/java/org/archive/resource/html/ExtractingParseObserver.java @@ -42,6 +42,15 @@ public class ExtractingParseObserver implements ParseObserver { protected static Pattern cssUrlTrimPattern = Pattern.compile(cssUrlTrimPatString); + protected static String jsOnClickUrl1PatString = + "(?i)^(?:javascript:)?(?:(?:window|top|document|self|parent)\\.)?location(?:\\.href)?\\s*=\\s*('|')([^'\"]{3,256})\\1$"; + protected static String jsOnClickUrl2PatString = + "(?i)^(?:javascript:)?(?:window|parent)\\.open\\((['\"]|')([^\"']{3,256}?)\\1[,)]"; + protected static Pattern[] jsOnClickUrlPatterns = { + Pattern.compile(jsOnClickUrl1PatString), + Pattern.compile(jsOnClickUrl2PatString) + }; + private final static int MAX_TEXT_LEN = 100; private final static String[] BLOCK_ELEMENTS = { "address", "article", "aside", "blockquote", "body", "br", @@ -75,6 +84,7 @@ public class ExtractingParseObserver implements ParseObserver { extractors.put("APPLET", new AppletTagExtractor()); extractors.put("AREA", new AreaTagExtractor()); extractors.put("BASE", new BaseTagExtractor()); + extractors.put("DIV", new DivTagExtractor()); extractors.put("EMBED", new EmbedTagExtractor()); extractors.put("FORM", new FormTagExtractor()); extractors.put("FRAME", new FrameTagExtractor()); @@ -337,7 +347,20 @@ private static void addHrefWithAttrs(HTMLMetaData data, TagNode node, if(l != null) { data.addHref(l); } - } + } + + private static void addHrefsOnclick(HTMLMetaData data, TagNode node) { + String onclick = node.getAttribute("onclick"); + if (onclick != null) { + String path = makePath(node.getTagName(), "onclick"); + for (Pattern pattern : jsOnClickUrlPatterns) { + String url = patternJSExtract(pattern, onclick); + if (url != null) { + data.addHref(PATH, path, "url", url); + } + } + } + } private static void appendParagraphSeparator(StringBuilder sb) { int length = sb.length(); @@ -437,6 +460,12 @@ public void extract(HTMLMetaData data, TagNode node, ExtractingParseObserver obs } } + private static class DivTagExtractor implements TagExtractor { + public void extract(HTMLMetaData data, TagNode node, ExtractingParseObserver obs) { + addHrefsOnclick(data,node); + } + } + private static class EmbedTagExtractor implements TagExtractor { public void extract(HTMLMetaData data, TagNode node, ExtractingParseObserver obs) { addBasicHrefs(data,node,"src"); @@ -493,6 +522,7 @@ public void extract(HTMLMetaData data, TagNode node, ExtractingParseObserver obs private static class InputTagExtractor implements TagExtractor { public void extract(HTMLMetaData data, TagNode node, ExtractingParseObserver obs) { addBasicHrefs(data,node,"src","formaction"); + addHrefsOnclick(data,node); } } @@ -557,4 +587,12 @@ private void patternCSSExtract(HTMLMetaData data, Pattern pattern, String conten } } } + + private static String patternJSExtract(Pattern pattern, String content) { + Matcher m = pattern.matcher(content); + if (m.find()) { + return m.group(2); + } + return null; + } } diff --git a/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java b/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java index 67f38a8d..39199819 100644 --- a/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java +++ b/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java @@ -282,6 +282,16 @@ public void testLinkExtraction() throws ResourceParseException, IOException { {"http://www.your-domain.com/your-page.html", "DIV@/data-href"} }; checkLinks(extractor.getNext(), fbSocialLinks); + String[][] onClickLinks = { + {"webpage.html", "DIV@/onclick"}, + {"index.html", "INPUT@/onclick"}, + {"http://www.x.com/", "INPUT@/onclick"}, + {"button-child.php", "INPUT@/onclick"}, + {"http://example.com/", "INPUT@/onclick"}, + {"http://example.com/location/href/1.html", "INPUT@/onclick"}, + {"http://example.com/location/href/2.html", "INPUT@/onclick"} + }; + checkLinks(extractor.getNext(), onClickLinks); } public void testTextExtraction() throws ResourceParseException, IOException { diff --git a/src/test/resources/org/archive/resource/html/link-extraction-test.warc b/src/test/resources/org/archive/resource/html/link-extraction-test.warc index ab0e54c8..1a30598e 100644 --- a/src/test/resources/org/archive/resource/html/link-extraction-test.warc +++ b/src/test/resources/org/archive/resource/html/link-extraction-test.warc @@ -318,3 +318,45 @@ Content-Type: text/html +WARC/1.0 +WARC-Type: response +WARC-Date: 2017-08-23T13:54:59Z +Content-Type: application/http;msgtype=response +Content-Length: 1279 + +HTTP/1.1 200 OK +Date: Wed, 23 Aug 2017 13:54:59 GMT +Server: Apache/2.4.18 (Ubuntu) +Last-Modified: Wed, 23 Aug 2017 13:54:03 GMT +ETag: "3ca-5576c0b718ab3" +Accept-Ranges: bytes +Content-Length: 971 +Vary: Accept-Encoding +Keep-Alive: timeout=5, max=100 +Connection: Keep-Alive +Content-Type: text/html + + + +Test Extraction of URLs from INPUT onClick Attributes + + + + +

Click to load webpage

Here is the logo

+Examples & more + image URL containing escaped ampersand ("&") +

+ + + + From 4a1f3238662786445c54131d67b768a557696acf Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Thu, 27 Jun 2019 16:56:47 +0200 Subject: [PATCH 39/83] WAT/WET extraction: use compiled regex Pattern to get better performance --- .../org/archive/resource/html/ExtractingParseObserver.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/archive/resource/html/ExtractingParseObserver.java b/src/main/java/org/archive/resource/html/ExtractingParseObserver.java index 0ca97bb9..afc2b91f 100644 --- a/src/main/java/org/archive/resource/html/ExtractingParseObserver.java +++ b/src/main/java/org/archive/resource/html/ExtractingParseObserver.java @@ -51,6 +51,8 @@ public class ExtractingParseObserver implements ParseObserver { Pattern.compile(jsOnClickUrl2PatString) }; + protected static Pattern wsPattern = Pattern.compile("\\s+"); + private final static int MAX_TEXT_LEN = 100; private final static String[] BLOCK_ELEMENTS = { "address", "article", "aside", "blockquote", "body", "br", @@ -197,7 +199,7 @@ public void handleTagClose(TagNode tag) { if((vals != null) && (vals.size() > 0)) { if(text != null) { // contained an href - we want to ignore : - String trimmed = Translate.decode(text.toString()).trim().replaceAll("\\s+", " "); + String trimmed = wsPattern.matcher(Translate.decode(text.toString()).trim()).replaceAll(" "); if(trimmed.length() > MAX_TEXT_LEN) { trimmed = trimmed.substring(0,MAX_TEXT_LEN); } @@ -241,7 +243,7 @@ public void handleTextNode(TextNode text) { } } - String t = txt.replaceAll("\\s+", " "); + String t = wsPattern.matcher(txt).replaceAll(" "); if(t.length() > MAX_TEXT_LEN) { t = t.substring(0,MAX_TEXT_LEN); From fb514ab5e70bc2c19cd2d32716e06773571b88cf Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Mon, 8 Jul 2019 16:48:11 +0200 Subject: [PATCH 40/83] Add unit test for HTML entity decoding - includes character entities not supported by htmlparser.org as TODOs --- .../html/ExtractingParseObserverTest.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java b/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java index c1b63f5f..54fe2ffb 100644 --- a/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java +++ b/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java @@ -15,6 +15,7 @@ import org.archive.resource.ResourceParseException; import org.archive.resource.ResourceProducer; import org.htmlparser.nodes.TextNode; +import org.htmlparser.util.Translate; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; @@ -371,4 +372,31 @@ public void testTextExtraction() throws ResourceParseException, IOException { // assertTrue(text.matches("CDATA in MathML:\\W*x Date: Mon, 22 Jul 2019 15:53:14 +0200 Subject: [PATCH 41/83] Replace the org.json dependency by Ted Dunning's open-json library - fixes incompatible license, see https://wiki.debian.org/qa.debian.org/jsonevil https://lwn.net/Articles/707510/ - improves performance when writing JSON --- pom.xml | 5 +++-- src/main/java/org/archive/extract/WATExtractorOutput.java | 8 ++------ 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/pom.xml b/pom.xml index ae1787a7..9d247d01 100644 --- a/pom.xml +++ b/pom.xml @@ -74,10 +74,11 @@ - org.json + com.tdunning json - 20131018 + 1.8 + org.htmlparser htmlparser diff --git a/src/main/java/org/archive/extract/WATExtractorOutput.java b/src/main/java/org/archive/extract/WATExtractorOutput.java index 4d574b91..22c04574 100644 --- a/src/main/java/org/archive/extract/WATExtractorOutput.java +++ b/src/main/java/org/archive/extract/WATExtractorOutput.java @@ -177,12 +177,8 @@ private void writeWARCMDRecord(OutputStream recOut, MetaData md, ByteArrayOutputStream bos = new ByteArrayOutputStream(); OutputStreamWriter osw = new OutputStreamWriter(bos, UTF8); - try { - md.write(osw); - } catch (JSONException e1) { - e1.printStackTrace(); - throw new IOException(e1); - } + String contents = md.toString(); + osw.write(contents, 0, contents.length()); osw.flush(); // ByteArrayInputStream bais = new ByteArrayInputStream(md.toString().getBytes("UTF-8")); Date capDate; From d188b9e902c98d8df486c3cdcafdddcaf9183b68 Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Tue, 23 Jul 2019 13:20:58 +0200 Subject: [PATCH 42/83] Use openjson as replacement for the org.json dependency (https://github.com/openjson/openjson) - see https://github.com/tdunning/open-json/pull/13 --- pom.xml | 6 +++--- .../java/org/archive/extract/DumpingExtractorOutput.java | 2 +- .../archive/extract/ExtractingResourceFactoryMapper.java | 4 ++-- .../java/org/archive/extract/RealCDXExtractorOutput.java | 6 +++--- .../extract/WARCMetadataRecordExtractorOutput.java | 6 +++--- src/main/java/org/archive/extract/WATExtractorOutput.java | 2 +- src/main/java/org/archive/format/arc/FiledescRecord.java | 8 ++++---- .../org/archive/format/json/CompoundORJSONPathSpec.java | 2 +- src/main/java/org/archive/format/json/JSONPathSpec.java | 2 +- src/main/java/org/archive/format/json/JSONUtils.java | 6 +++--- src/main/java/org/archive/format/json/JSONView.java | 2 +- .../java/org/archive/format/json/SimpleJSONPathSpec.java | 6 +++--- .../java/org/archive/hadoop/ArchiveJSONViewLoader.java | 4 ++-- .../java/org/archive/hadoop/func/JSONViewEvalFunc.java | 4 ++-- src/main/java/org/archive/io/WriterPool.java | 6 +++--- src/main/java/org/archive/resource/MetaData.java | 8 ++++---- src/main/java/org/archive/resource/gzip/GZIPMetaData.java | 4 ++-- src/main/java/org/archive/resource/html/HTMLMetaData.java | 6 +++--- .../org/archive/resource/html/HTMLResourceFactory.java | 4 ++-- .../org/archive/resource/warc/record/DNSResource.java | 4 ++-- .../warc/record/WARCJSONMetaDataResourceFactory.java | 4 ++-- .../archive/format/json/CompoundORJSONPathSpecTest.java | 4 ++-- .../org/archive/format/json/JSONPathSpecFactoryTest.java | 4 ++-- src/test/java/org/archive/format/json/JSONViewTest.java | 4 ++-- .../org/archive/format/json/SimpleJSONPathSpecTest.java | 4 ++-- .../resource/html/ExtractingParseObserverTest.java | 6 +++--- .../java/org/archive/resource/html/HTMLMetaDataTest.java | 6 +++--- 27 files changed, 62 insertions(+), 62 deletions(-) diff --git a/pom.xml b/pom.xml index 9d247d01..f3d4e586 100644 --- a/pom.xml +++ b/pom.xml @@ -74,9 +74,9 @@ - com.tdunning - json - 1.8 + com.github.openjson + openjson + 1.0.11 diff --git a/src/main/java/org/archive/extract/DumpingExtractorOutput.java b/src/main/java/org/archive/extract/DumpingExtractorOutput.java index 69591931..43a81608 100644 --- a/src/main/java/org/archive/extract/DumpingExtractorOutput.java +++ b/src/main/java/org/archive/extract/DumpingExtractorOutput.java @@ -7,7 +7,7 @@ import org.archive.resource.Resource; import org.archive.util.StreamCopy; -import org.json.JSONException; +import com.github.openjson.JSONException; import com.google.common.io.ByteStreams; import com.google.common.io.CountingOutputStream; diff --git a/src/main/java/org/archive/extract/ExtractingResourceFactoryMapper.java b/src/main/java/org/archive/extract/ExtractingResourceFactoryMapper.java index 0afe16fb..eb749d7d 100644 --- a/src/main/java/org/archive/extract/ExtractingResourceFactoryMapper.java +++ b/src/main/java/org/archive/extract/ExtractingResourceFactoryMapper.java @@ -20,8 +20,8 @@ import org.archive.resource.warc.record.DNSResourceFactory; import org.archive.resource.warc.record.WARCJSONMetaDataResourceFactory; import org.archive.resource.warc.record.WARCMetaDataResourceFactory; -import org.json.JSONException; -import org.json.JSONObject; +import com.github.openjson.JSONException; +import com.github.openjson.JSONObject; public class ExtractingResourceFactoryMapper implements ResourceFactoryMapper { diff --git a/src/main/java/org/archive/extract/RealCDXExtractorOutput.java b/src/main/java/org/archive/extract/RealCDXExtractorOutput.java index e6f6e82f..1fffe4fc 100644 --- a/src/main/java/org/archive/extract/RealCDXExtractorOutput.java +++ b/src/main/java/org/archive/extract/RealCDXExtractorOutput.java @@ -21,9 +21,9 @@ import org.archive.url.WaybackURLKeyMaker; import org.archive.util.IAUtils; import org.archive.util.StreamCopy; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; +import com.github.openjson.JSONArray; +import com.github.openjson.JSONException; +import com.github.openjson.JSONObject; import com.google.common.io.ByteStreams; import com.google.common.io.CountingOutputStream; diff --git a/src/main/java/org/archive/extract/WARCMetadataRecordExtractorOutput.java b/src/main/java/org/archive/extract/WARCMetadataRecordExtractorOutput.java index 68f9d1c8..42bb4de9 100644 --- a/src/main/java/org/archive/extract/WARCMetadataRecordExtractorOutput.java +++ b/src/main/java/org/archive/extract/WARCMetadataRecordExtractorOutput.java @@ -18,9 +18,9 @@ import org.archive.resource.Resource; import org.archive.util.IAUtils; import org.archive.util.StreamCopy; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; +import com.github.openjson.JSONArray; +import com.github.openjson.JSONException; +import com.github.openjson.JSONObject; import com.google.common.io.ByteStreams; import com.google.common.io.CountingOutputStream; diff --git a/src/main/java/org/archive/extract/WATExtractorOutput.java b/src/main/java/org/archive/extract/WATExtractorOutput.java index 22c04574..149aa7ac 100644 --- a/src/main/java/org/archive/extract/WATExtractorOutput.java +++ b/src/main/java/org/archive/extract/WATExtractorOutput.java @@ -22,7 +22,7 @@ import org.archive.util.DateUtils; import org.archive.util.StreamCopy; import org.archive.util.io.CommitedOutputStream; -import org.json.JSONException; +import com.github.openjson.JSONException; import java.net.InetAddress; import java.text.DateFormat; diff --git a/src/main/java/org/archive/format/arc/FiledescRecord.java b/src/main/java/org/archive/format/arc/FiledescRecord.java index 9af3d461..dc43765b 100644 --- a/src/main/java/org/archive/format/arc/FiledescRecord.java +++ b/src/main/java/org/archive/format/arc/FiledescRecord.java @@ -2,9 +2,9 @@ import java.util.logging.Logger; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; +import com.github.openjson.JSONArray; +import com.github.openjson.JSONException; +import com.github.openjson.JSONObject; public class FiledescRecord { private static final Logger LOG = @@ -87,4 +87,4 @@ public String getFormat() { public void setFormat(String format) { this.format = format; } -} \ No newline at end of file +} diff --git a/src/main/java/org/archive/format/json/CompoundORJSONPathSpec.java b/src/main/java/org/archive/format/json/CompoundORJSONPathSpec.java index b99e4f23..aa6911e4 100644 --- a/src/main/java/org/archive/format/json/CompoundORJSONPathSpec.java +++ b/src/main/java/org/archive/format/json/CompoundORJSONPathSpec.java @@ -3,7 +3,7 @@ import java.util.ArrayList; import java.util.List; -import org.json.JSONObject; +import com.github.openjson.JSONObject; public class CompoundORJSONPathSpec implements JSONPathSpec { ArrayList parts; diff --git a/src/main/java/org/archive/format/json/JSONPathSpec.java b/src/main/java/org/archive/format/json/JSONPathSpec.java index 68adf0bd..f78eaaff 100644 --- a/src/main/java/org/archive/format/json/JSONPathSpec.java +++ b/src/main/java/org/archive/format/json/JSONPathSpec.java @@ -2,7 +2,7 @@ import java.util.List; -import org.json.JSONObject; +import com.github.openjson.JSONObject; public interface JSONPathSpec { public static final String EMPTY = ""; diff --git a/src/main/java/org/archive/format/json/JSONUtils.java b/src/main/java/org/archive/format/json/JSONUtils.java index 946b633b..0dc6ad24 100644 --- a/src/main/java/org/archive/format/json/JSONUtils.java +++ b/src/main/java/org/archive/format/json/JSONUtils.java @@ -4,9 +4,9 @@ import java.util.List; import java.util.logging.Logger; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; +import com.github.openjson.JSONArray; +import com.github.openjson.JSONException; +import com.github.openjson.JSONObject; public class JSONUtils { private static final Logger LOG = diff --git a/src/main/java/org/archive/format/json/JSONView.java b/src/main/java/org/archive/format/json/JSONView.java index b73c0666..99da2364 100644 --- a/src/main/java/org/archive/format/json/JSONView.java +++ b/src/main/java/org/archive/format/json/JSONView.java @@ -6,7 +6,7 @@ import java.util.logging.Logger; import org.apache.commons.lang.StringUtils; -import org.json.JSONObject; +import com.github.openjson.JSONObject; /** * diff --git a/src/main/java/org/archive/format/json/SimpleJSONPathSpec.java b/src/main/java/org/archive/format/json/SimpleJSONPathSpec.java index c0b1a8d6..f114d30c 100644 --- a/src/main/java/org/archive/format/json/SimpleJSONPathSpec.java +++ b/src/main/java/org/archive/format/json/SimpleJSONPathSpec.java @@ -4,9 +4,9 @@ import java.util.List; import java.util.logging.Logger; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; +import com.github.openjson.JSONArray; +import com.github.openjson.JSONException; +import com.github.openjson.JSONObject; public class SimpleJSONPathSpec implements JSONPathSpec { private static final Logger LOG = diff --git a/src/main/java/org/archive/hadoop/ArchiveJSONViewLoader.java b/src/main/java/org/archive/hadoop/ArchiveJSONViewLoader.java index e92ed7e1..766e33e6 100644 --- a/src/main/java/org/archive/hadoop/ArchiveJSONViewLoader.java +++ b/src/main/java/org/archive/hadoop/ArchiveJSONViewLoader.java @@ -11,8 +11,8 @@ import org.apache.pig.data.Tuple; import org.apache.pig.data.TupleFactory; import org.archive.format.json.JSONView; -import org.json.JSONException; -import org.json.JSONObject; +import com.github.openjson.JSONException; +import com.github.openjson.JSONObject; public class ArchiveJSONViewLoader extends ArchiveMetadataLoader { private final static Logger LOG = diff --git a/src/main/java/org/archive/hadoop/func/JSONViewEvalFunc.java b/src/main/java/org/archive/hadoop/func/JSONViewEvalFunc.java index 8d4446b5..bc390ff6 100644 --- a/src/main/java/org/archive/hadoop/func/JSONViewEvalFunc.java +++ b/src/main/java/org/archive/hadoop/func/JSONViewEvalFunc.java @@ -8,8 +8,8 @@ import org.apache.pig.data.Tuple; import org.apache.pig.data.TupleFactory; import org.archive.format.json.JSONUtils; -import org.json.JSONException; -import org.json.JSONObject; +import com.github.openjson.JSONException; +import com.github.openjson.JSONObject; public class JSONViewEvalFunc extends EvalFunc { private static final Logger LOG = diff --git a/src/main/java/org/archive/io/WriterPool.java b/src/main/java/org/archive/io/WriterPool.java index 2dc385a1..c5e175e0 100644 --- a/src/main/java/org/archive/io/WriterPool.java +++ b/src/main/java/org/archive/io/WriterPool.java @@ -30,9 +30,9 @@ import java.util.logging.Level; import java.util.logging.Logger; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; +import com.github.openjson.JSONArray; +import com.github.openjson.JSONException; +import com.github.openjson.JSONObject; /** * Pool of Writers. diff --git a/src/main/java/org/archive/resource/MetaData.java b/src/main/java/org/archive/resource/MetaData.java index a975b0d4..1237a51c 100755 --- a/src/main/java/org/archive/resource/MetaData.java +++ b/src/main/java/org/archive/resource/MetaData.java @@ -2,10 +2,10 @@ import java.util.logging.Logger; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONTokener; +import com.github.openjson.JSONArray; +import com.github.openjson.JSONException; +import com.github.openjson.JSONObject; +import com.github.openjson.JSONTokener; public class MetaData extends JSONObject { diff --git a/src/main/java/org/archive/resource/gzip/GZIPMetaData.java b/src/main/java/org/archive/resource/gzip/GZIPMetaData.java index 0fc18162..104e5fa9 100644 --- a/src/main/java/org/archive/resource/gzip/GZIPMetaData.java +++ b/src/main/java/org/archive/resource/gzip/GZIPMetaData.java @@ -12,8 +12,8 @@ import org.archive.resource.MetaData; import org.archive.resource.ResourceConstants; import org.archive.util.ByteOp; -import org.json.JSONException; -import org.json.JSONObject; +import com.github.openjson.JSONException; +import com.github.openjson.JSONObject; public class GZIPMetaData extends MetaData implements ResourceConstants { private static final Logger LOG = Logger.getLogger(GZIPMetaData.class.getName()); diff --git a/src/main/java/org/archive/resource/html/HTMLMetaData.java b/src/main/java/org/archive/resource/html/HTMLMetaData.java index b95dcad3..460d28a0 100644 --- a/src/main/java/org/archive/resource/html/HTMLMetaData.java +++ b/src/main/java/org/archive/resource/html/HTMLMetaData.java @@ -5,9 +5,9 @@ import org.archive.resource.MetaData; import org.archive.resource.ResourceConstants; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; +import com.github.openjson.JSONArray; +import com.github.openjson.JSONException; +import com.github.openjson.JSONObject; public class HTMLMetaData extends MetaData implements ResourceConstants { diff --git a/src/main/java/org/archive/resource/html/HTMLResourceFactory.java b/src/main/java/org/archive/resource/html/HTMLResourceFactory.java index afb1c850..2532622e 100644 --- a/src/main/java/org/archive/resource/html/HTMLResourceFactory.java +++ b/src/main/java/org/archive/resource/html/HTMLResourceFactory.java @@ -20,8 +20,8 @@ import org.archive.resource.ResourceParseException; import org.htmlparser.lexer.Page; import org.htmlparser.util.ParserException; -import org.json.JSONException; -import org.json.JSONObject; +import com.github.openjson.JSONException; +import com.github.openjson.JSONObject; public class HTMLResourceFactory implements ResourceFactory { diff --git a/src/main/java/org/archive/resource/warc/record/DNSResource.java b/src/main/java/org/archive/resource/warc/record/DNSResource.java index 2bcb2bc1..86c56652 100644 --- a/src/main/java/org/archive/resource/warc/record/DNSResource.java +++ b/src/main/java/org/archive/resource/warc/record/DNSResource.java @@ -9,8 +9,8 @@ import org.archive.resource.MetaData; import org.archive.resource.ResourceConstants; import org.archive.resource.ResourceContainer; -import org.json.JSONException; -import org.json.JSONObject; +import com.github.openjson.JSONException; +import com.github.openjson.JSONObject; public class DNSResource extends AbstractEmptyResource implements ResourceConstants { private static final Logger LOG = diff --git a/src/main/java/org/archive/resource/warc/record/WARCJSONMetaDataResourceFactory.java b/src/main/java/org/archive/resource/warc/record/WARCJSONMetaDataResourceFactory.java index 43041efb..6173bfdc 100644 --- a/src/main/java/org/archive/resource/warc/record/WARCJSONMetaDataResourceFactory.java +++ b/src/main/java/org/archive/resource/warc/record/WARCJSONMetaDataResourceFactory.java @@ -11,8 +11,8 @@ import org.archive.resource.ResourceContainer; import org.archive.resource.ResourceFactory; import org.archive.resource.ResourceParseException; -import org.json.JSONException; -import org.json.JSONTokener; +import com.github.openjson.JSONException; +import com.github.openjson.JSONTokener; public class WARCJSONMetaDataResourceFactory implements ResourceFactory, ResourceConstants { private static final Charset UTF8 = Charset.forName("UTF-8"); diff --git a/src/test/java/org/archive/format/json/CompoundORJSONPathSpecTest.java b/src/test/java/org/archive/format/json/CompoundORJSONPathSpecTest.java index 57c21965..c10241d2 100644 --- a/src/test/java/org/archive/format/json/CompoundORJSONPathSpecTest.java +++ b/src/test/java/org/archive/format/json/CompoundORJSONPathSpecTest.java @@ -3,8 +3,8 @@ import java.util.ArrayList; import org.archive.util.TestUtils; -import org.json.JSONException; -import org.json.JSONObject; +import com.github.openjson.JSONException; +import com.github.openjson.JSONObject; import junit.framework.TestCase; diff --git a/src/test/java/org/archive/format/json/JSONPathSpecFactoryTest.java b/src/test/java/org/archive/format/json/JSONPathSpecFactoryTest.java index ab999dca..b351a120 100644 --- a/src/test/java/org/archive/format/json/JSONPathSpecFactoryTest.java +++ b/src/test/java/org/archive/format/json/JSONPathSpecFactoryTest.java @@ -1,8 +1,8 @@ package org.archive.format.json; import org.archive.util.TestUtils; -import org.json.JSONException; -import org.json.JSONObject; +import com.github.openjson.JSONException; +import com.github.openjson.JSONObject; import junit.framework.TestCase; diff --git a/src/test/java/org/archive/format/json/JSONViewTest.java b/src/test/java/org/archive/format/json/JSONViewTest.java index 20bd4fe6..b75a4e0a 100644 --- a/src/test/java/org/archive/format/json/JSONViewTest.java +++ b/src/test/java/org/archive/format/json/JSONViewTest.java @@ -1,8 +1,8 @@ package org.archive.format.json; import org.archive.util.TestUtils; -import org.json.JSONException; -import org.json.JSONObject; +import com.github.openjson.JSONException; +import com.github.openjson.JSONObject; import junit.framework.TestCase; diff --git a/src/test/java/org/archive/format/json/SimpleJSONPathSpecTest.java b/src/test/java/org/archive/format/json/SimpleJSONPathSpecTest.java index a703b49a..773df618 100644 --- a/src/test/java/org/archive/format/json/SimpleJSONPathSpecTest.java +++ b/src/test/java/org/archive/format/json/SimpleJSONPathSpecTest.java @@ -1,8 +1,8 @@ package org.archive.format.json; import org.archive.util.TestUtils; -import org.json.JSONException; -import org.json.JSONObject; +import com.github.openjson.JSONException; +import com.github.openjson.JSONObject; import junit.framework.TestCase; diff --git a/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java b/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java index 54fe2ffb..796a1bb4 100644 --- a/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java +++ b/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java @@ -16,9 +16,9 @@ import org.archive.resource.ResourceProducer; import org.htmlparser.nodes.TextNode; import org.htmlparser.util.Translate; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; +import com.github.openjson.JSONArray; +import com.github.openjson.JSONException; +import com.github.openjson.JSONObject; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Multimap; diff --git a/src/test/java/org/archive/resource/html/HTMLMetaDataTest.java b/src/test/java/org/archive/resource/html/HTMLMetaDataTest.java index fb255d3c..49be94d5 100644 --- a/src/test/java/org/archive/resource/html/HTMLMetaDataTest.java +++ b/src/test/java/org/archive/resource/html/HTMLMetaDataTest.java @@ -1,8 +1,8 @@ package org.archive.resource.html; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; +import com.github.openjson.JSONArray; +import com.github.openjson.JSONException; +import com.github.openjson.JSONObject; import junit.framework.TestCase; From 60b6f7eae35264b466c4daa8379704647a9892a1 Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Mon, 19 Aug 2019 23:07:51 +0200 Subject: [PATCH 43/83] WAT: only unescape complete XML/HTML character entities (fixes #19) - replace org.htmlparser's decode method by Apache commons `unescapeHtml` --- .../html/ExtractingParseObserver.java | 30 +++++++++++-------- .../html/ExtractingParseObserverTest.java | 6 ++-- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/archive/resource/html/ExtractingParseObserver.java b/src/main/java/org/archive/resource/html/ExtractingParseObserver.java index afc2b91f..e64df095 100644 --- a/src/main/java/org/archive/resource/html/ExtractingParseObserver.java +++ b/src/main/java/org/archive/resource/html/ExtractingParseObserver.java @@ -163,7 +163,7 @@ public void handleTagOpen(TagNode tag) { } attrName = attrName.toLowerCase(Locale.ROOT); if (globalHrefAttributes.contains(attrName)) { - attrValue = Translate.decode(attrValue); + attrValue = decodeCharEnt(attrValue); data.addHref(PATH,makePath(name,attrName),"url",attrValue); } } @@ -199,7 +199,7 @@ public void handleTagClose(TagNode tag) { if((vals != null) && (vals.size() > 0)) { if(text != null) { // contained an href - we want to ignore : - String trimmed = wsPattern.matcher(Translate.decode(text.toString()).trim()).replaceAll(" "); + String trimmed = wsPattern.matcher(decodeCharEnt(text.toString()).trim()).replaceAll(" "); if(trimmed.length() > MAX_TEXT_LEN) { trimmed = trimmed.substring(0,MAX_TEXT_LEN); } @@ -221,7 +221,7 @@ public void handleTextNode(TextNode text) { // this result is thrown away. String txt = text.getText(); - txt = Translate.decode(txt); + txt = decodeCharEnt(txt); if (inPre) { textExtract.append(txt); } else { @@ -274,7 +274,7 @@ public void handleScriptNode(TextNode text) { } public void handleStyleNode(TextNode text) { - String cssStr = Translate.decode(text.getText()); + String cssStr = decodeCharEnt(text.getText()); patternCSSExtract(data, cssUrlPattern, cssStr); patternCSSExtract(data, cssImportNoUrlPattern, cssStr); } @@ -303,7 +303,7 @@ private static void addBasicHrefs(HTMLMetaData data, TagNode node, String... att for(String attr : attrs) { String val = node.getAttribute(attr); if(val != null) { - val = Translate.decode(val); + val = decodeCharEnt(val); data.addHref(PATH,makePath(node.getTagName(),attr),"url",val); } } @@ -314,7 +314,7 @@ private static ArrayList getAttrList(TagNode node, String... attrs) { for(String attr : attrs) { String val = node.getAttribute(attr); if(val != null) { - val = Translate.decode(val); + val = decodeCharEnt(val); l.add(attr); l.add(val); } @@ -330,7 +330,7 @@ private static ArrayList getAttrListUrl(TagNode node, String url = node.getAttribute(urlAttr); ArrayList l = null; if(url != null) { - url = Translate.decode(url); + url = decodeCharEnt(url); l = new ArrayList(); l.add(PATH); l.add(makePath(node.getTagName(),urlAttr)); @@ -340,7 +340,7 @@ private static ArrayList getAttrListUrl(TagNode node, for(String attr : optionalAttrs) { String val = node.getAttribute(attr); if(val != null) { - val = Translate.decode(val); + val = decodeCharEnt(val); l.add(attr); l.add(val); } @@ -404,7 +404,7 @@ public void extract(HTMLMetaData data, TagNode node, ExtractingParseObserver obs String url = node.getAttribute("href"); if(url != null) { // got data: - url = Translate.decode(url); + url = decodeCharEnt(url); l.add(PATH); l.add(makePath("A","href")); l.add("url"); @@ -412,7 +412,7 @@ public void extract(HTMLMetaData data, TagNode node, ExtractingParseObserver obs for(String a : new String[] {"target","alt","title","rel","hreflang","type"}) { String v = node.getAttribute(a); if(v != null) { - v = Translate.decode(v); + v = decodeCharEnt(v); l.add(a); l.add(v); } @@ -439,7 +439,7 @@ private static class AreaTagExtractor implements TagExtractor { public void extract(HTMLMetaData data, TagNode node, ExtractingParseObserver obs) { String url = node.getAttribute("href"); if(url != null) { - url = Translate.decode(url); + url = decodeCharEnt(url); ArrayList l = new ArrayList(); l.add(PATH); l.add(makePath("AREA","href")); @@ -461,7 +461,7 @@ private static class BaseTagExtractor implements TagExtractor { public void extract(HTMLMetaData data, TagNode node, ExtractingParseObserver obs) { String url = node.getAttribute("href"); if(url != null) { - url = Translate.decode(url); + url = decodeCharEnt(url); data.setBaseHref(url); } } @@ -496,7 +496,7 @@ public void extract(HTMLMetaData data, TagNode node, ExtractingParseObserver obs ArrayList l = new ArrayList(); String url = node.getAttribute("action"); if(url != null) { - url = Translate.decode(url); + url = decodeCharEnt(url); // got data: l.add(PATH); l.add(makePath("FORM","action")); @@ -609,4 +609,8 @@ private static String patternJSExtract(Pattern pattern, String content) { } return null; } + + public static String decodeCharEnt(String ent) { + return org.apache.commons.lang.StringEscapeUtils.unescapeHtml(ent); + } } diff --git a/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java b/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java index 796a1bb4..95f3109d 100644 --- a/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java +++ b/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java @@ -385,16 +385,18 @@ public void testHtmlParserEntityDecoding() { // â€™ right single quotation mark { "’", "\u2019" }, // آ» right-pointing double angle quotation mark - { "»", "\u00bb" }, + { "»", "\u00bb" }, // â€¦ horizontal ellipsis { "…", "\u2026" }, // ً¤†‘ CJK UNIFIED IDEOGRAPH-24191 // TODO: { "𤆑", new String(Character.toChars(0x24191)) }, // ًںکٹ U+1F60A SMILING FACE WITH SMILING EYES // TODO: { "😊", new String(Character.toChars(0x1f60a)) }, + // must not touch "&order=" and never decode "&or" as "∨" + { "&order=lexical", "&order=lexical" }, }; for (String[] ent : entities) { - String decoded = Translate.decode(ent[0]); + String decoded = ExtractingParseObserver.decodeCharEnt(ent[0]); assertEquals("Entity " + ent[0] + " not properly decoded", ent[1], decoded); } } From 716eae928246799c70e0d35ed5d6cdb09d973977 Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Tue, 27 Aug 2019 14:56:14 +0200 Subject: [PATCH 44/83] WAT: only unescape complete XML/HTML character entities (fixes #19) - replace org.htmlparser's decode method by Apache commons-text `unescapeHtml4` --- pom.xml | 8 ++- .../html/ExtractingParseObserver.java | 8 ++- .../html/ExtractingParseObserverTest.java | 49 +++++++++++++------ 3 files changed, 47 insertions(+), 18 deletions(-) diff --git a/pom.xml b/pom.xml index f3d4e586..c761c729 100644 --- a/pom.xml +++ b/pom.xml @@ -147,7 +147,13 @@ commons-lang commons-lang - 2.5 + 2.6 + + + + org.apache.commons + commons-text + 1.7 diff --git a/src/main/java/org/archive/resource/html/ExtractingParseObserver.java b/src/main/java/org/archive/resource/html/ExtractingParseObserver.java index e64df095..3d24d783 100644 --- a/src/main/java/org/archive/resource/html/ExtractingParseObserver.java +++ b/src/main/java/org/archive/resource/html/ExtractingParseObserver.java @@ -610,7 +610,11 @@ private static String patternJSExtract(Pattern pattern, String content) { return null; } - public static String decodeCharEnt(String ent) { - return org.apache.commons.lang.StringEscapeUtils.unescapeHtml(ent); + public static String decodeCharEnt(String text) { + try { + return org.apache.commons.text.StringEscapeUtils.unescapeHtml4(text); + } catch (Throwable e) { + return text; + } } } diff --git a/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java b/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java index 95f3109d..14256a5a 100644 --- a/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java +++ b/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java @@ -374,30 +374,49 @@ public void testTextExtraction() throws ResourceParseException, IOException { public void testHtmlParserEntityDecoding() { String[][] entities = { // - // ampersand + /* ampersand */ { "&", "&" }, - // apostrophe + /* apostrophe */ // TODO: { "'", "'" }, - // comma + /* comma */ // TODO: { ",", "," }, - // % percent - // TODO: { "percnt", "%" }, - // â€™ right single quotation mark + /* % percent */ + // TODO: { "%", "%" }, + /* â€™ right single quotation mark */ { "’", "\u2019" }, - // آ» right-pointing double angle quotation mark + /* آ» right-pointing double angle quotation mark */ { "»", "\u00bb" }, - // â€¦ horizontal ellipsis + /* â€¦ horizontal ellipsis */ { "…", "\u2026" }, - // ً¤†‘ CJK UNIFIED IDEOGRAPH-24191 - // TODO: { "𤆑", new String(Character.toChars(0x24191)) }, - // ًںکٹ U+1F60A SMILING FACE WITH SMILING EYES - // TODO: { "😊", new String(Character.toChars(0x1f60a)) }, - // must not touch "&order=" and never decode "&or" as "∨" - { "&order=lexical", "&order=lexical" }, + /* ً¤†‘ CJK UNIFIED IDEOGRAPH-24191 */ + { "𤆑", new String(Character.toChars(0x24191)) }, + /* ًںکٹ U+1F60A SMILING FACE WITH SMILING EYES */ + { "😊", new String(Character.toChars(0x1f60a)) }, + /* + * must not decode "&or" in "&order" as "∨" (âˆ¨ U+2228) to + * avoid that unescaped ampersands in URLs cause erroneous + * replacements + */ + { "https://example.org/search?q=example&order=lexical", + "https://example.org/search?q=example&order=lexical" }, + { "https://example.org/search?q=example&order=lexical", + "https://example.org/search?q=example&order=lexical" }, + { "∨", "\u2228" }, + /* ًں‘ژ U+1F44E THUMBS DOWN SIGN (must not decode 0x1f44) */ + { "👎", new String(Character.toChars(0x1f44e)) }, + /* + * invalid Unicode code point: make sure that exceptions are + * handled, the actual character may appear as (? or ï؟½) + */ + { "�", null }, // single char of surrogate pair + { "�", null }, // + { "�", null }, // }; for (String[] ent : entities) { String decoded = ExtractingParseObserver.decodeCharEnt(ent[0]); - assertEquals("Entity " + ent[0] + " not properly decoded", ent[1], decoded); + if (ent[1] != null) { + assertEquals("Entity " + ent[0] + " not properly decoded", ent[1], decoded); + } } } From bb754a3558154e47ab96fe048ffdab8f4c11b71c Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Wed, 28 Aug 2019 11:16:29 +0200 Subject: [PATCH 45/83] WAT: only unescape complete XML/HTML character entities (fixes #19) - use jsoup's Parser.unescapeEntities(...) instead which * supports HTML 5 entities * provides a safer mode when entities in attributes are decoded --- pom.xml | 6 ++-- .../html/ExtractingParseObserver.java | 16 ++++++--- .../html/ExtractingParseObserverTest.java | 36 ++++++++++++++++--- 3 files changed, 47 insertions(+), 11 deletions(-) diff --git a/pom.xml b/pom.xml index c761c729..7efadd0a 100644 --- a/pom.xml +++ b/pom.xml @@ -151,9 +151,9 @@ - org.apache.commons - commons-text - 1.7 + org.jsoup + jsoup + 1.12.1 diff --git a/src/main/java/org/archive/resource/html/ExtractingParseObserver.java b/src/main/java/org/archive/resource/html/ExtractingParseObserver.java index 3d24d783..e494e92d 100644 --- a/src/main/java/org/archive/resource/html/ExtractingParseObserver.java +++ b/src/main/java/org/archive/resource/html/ExtractingParseObserver.java @@ -16,7 +16,6 @@ import org.htmlparser.nodes.RemarkNode; import org.htmlparser.nodes.TagNode; import org.htmlparser.nodes.TextNode; -import org.htmlparser.util.Translate; public class ExtractingParseObserver implements ParseObserver { @@ -199,7 +198,7 @@ public void handleTagClose(TagNode tag) { if((vals != null) && (vals.size() > 0)) { if(text != null) { // contained an href - we want to ignore : - String trimmed = wsPattern.matcher(decodeCharEnt(text.toString()).trim()).replaceAll(" "); + String trimmed = wsPattern.matcher(decodeCharEnt(text.toString(), false).trim()).replaceAll(" "); if(trimmed.length() > MAX_TEXT_LEN) { trimmed = trimmed.substring(0,MAX_TEXT_LEN); } @@ -221,7 +220,7 @@ public void handleTextNode(TextNode text) { // this result is thrown away. String txt = text.getText(); - txt = decodeCharEnt(txt); + txt = decodeCharEnt(txt, false); if (inPre) { textExtract.append(txt); } else { @@ -611,9 +610,18 @@ private static String patternJSExtract(Pattern pattern, String content) { } public static String decodeCharEnt(String text) { + return decodeCharEnt(text, true); + } + + public static String decodeCharEnt(String text, boolean inAttribute) { + if (text.indexOf('&') == -1) { + return text; + } try { - return org.apache.commons.text.StringEscapeUtils.unescapeHtml4(text); + return org.jsoup.parser.Parser.unescapeEntities(text, inAttribute); } catch (Throwable e) { + System.err.println(text); + e.printStackTrace(); return text; } } diff --git a/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java b/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java index 14256a5a..96c96bee 100644 --- a/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java +++ b/src/test/java/org/archive/resource/html/ExtractingParseObserverTest.java @@ -15,7 +15,6 @@ import org.archive.resource.ResourceParseException; import org.archive.resource.ResourceProducer; import org.htmlparser.nodes.TextNode; -import org.htmlparser.util.Translate; import com.github.openjson.JSONArray; import com.github.openjson.JSONException; import com.github.openjson.JSONObject; @@ -377,11 +376,12 @@ public void testHtmlParserEntityDecoding() { /* ampersand */ { "&", "&" }, /* apostrophe */ - // TODO: { "'", "'" }, + { "'", "'" }, + { "'", "'" }, /* comma */ - // TODO: { ",", "," }, + { ",", "," }, /* % percent */ - // TODO: { "%", "%" }, + { "%", "%" }, /* â€™ right single quotation mark */ { "’", "\u2019" }, /* آ» right-pointing double angle quotation mark */ @@ -411,9 +411,37 @@ public void testHtmlParserEntityDecoding() { { "�", null }, // single char of surrogate pair { "�", null }, // { "�", null }, // + { "�", null }, // + /* + * for better text conversion, some entities might be decoded + * even if not closed by a ; + */ + { " ", "\u00a0\u00a0\u00a0" }, // + { " ", "\u00a0" }, // + { "&order", "&order" }, // + /* but never in URLs */ + { "https://example.org/search?q=example =value", + "https://example.org/search?q=example =value" }, // + /* + * test more aggressive replacement in text mode (not + * inAttribute) + */ + { "law&order", "law&order", "false" }, // + { "a ∨ b", "a \u2228 b", "false" }, // + { "a &or b", "a &or b", "false" }, // + { "a & b", "a & b", "false" }, // + /* comparison of text vs. attribute mode */ + { "a = b", "a =\u00a0b", "true" }, // + { "a = c", "a\u00a0=\u00a0c", "false" }, // + { "a = &order=true", "a =\u00a0&order=true", "true" }, // + { "a = &order=true", "a\u00a0=\u00a0&order=true", "false" }, // }; for (String[] ent : entities) { String decoded = ExtractingParseObserver.decodeCharEnt(ent[0]); + if (ent.length > 2) { + // test for text nodes + decoded = ExtractingParseObserver.decodeCharEnt(ent[0], Boolean.valueOf(ent[2])); + } if (ent[1] != null) { assertEquals("Entity " + ent[0] + " not properly decoded", ent[1], decoded); } From 248fb98e19351091a6d2b0c7422a912d62c920d4 Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Wed, 28 Aug 2019 11:53:49 +0200 Subject: [PATCH 46/83] Remove pom-cdh5.xml as changes required to run under Cloudera Hadoop (CDH) are hold in a separate branch now --- pom-cdh5.xml | 296 --------------------------------------------------- 1 file changed, 296 deletions(-) delete mode 100644 pom-cdh5.xml diff --git a/pom-cdh5.xml b/pom-cdh5.xml deleted file mode 100644 index aa29fd89..00000000 --- a/pom-cdh5.xml +++ /dev/null @@ -1,296 +0,0 @@ - - 4.0.0 - - - org.sonatype.oss - oss-parent - 7 - - - org.commoncrawl - ia-web-commons - 1.1.9-SNAPSHOT - jar - - ia-web-commons - https://github.com/commoncrawl/ia-web-commons - - - The International Internet Preservation Consortium - http://netpreserve.org/ - - - - The Apache Software License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0.txt - repo - - - - - many-devs - Many Others Developers Proceed Me - many@dev.org - - - anjackson - Andrew Jackson - Andrew.Jackson@bl.uk - - - - GitHub Issues - https://github.com/iipc/webarchive-commons/issues - - - scm:git:git@github.com:iipc/webarchive-commons.git - scm:git:git@github.com:iipc/webarchive-commons.git - git@github.com:iipc/webarchive-commons.git - - - - UTF-8 - ${maven.build.timestamp} - yyyyMMddhhmmss - - - sonatype-nexus-staging - https://oss.sonatype.org/service/local/staging/deploy/maven2/ - sonatype-nexus-snapshots - https://oss.sonatype.org/content/repositories/snapshots/ - - - - - junit - junit - 3.8.1 - - - - com.google.guava - guava - 17.0 - - - - org.json - json - 20131018 - - - org.htmlparser - htmlparser - 2.1 - - - - com.googlecode.juniversalchardet - juniversalchardet - 1.0.3 - - - - commons-httpclient - commons-httpclient - 3.1 - - - - org.apache.hadoop - hadoop-client - 2.6.0-cdh5.12.0 - - - commons-httpclient - commons-httpclient - - - javax.servlet - servlet-api - - - javax.servlet.jsp - jsp-api - - - org.mortbay.jetty - jetty - - - org.mortbay.jetty - jetty-util - - - tomcat - jasper-runtime - - - tomcat - jasper-compiler - - - hsqldb - hsqldb - - - - - org.apache.hadoop - hadoop-common - 2.6.0-cdh5.11.0 - - - org.apache.hadoop - hadoop-mapreduce-client-common - 2.6.0-cdh5.11.0 - - - org.apache.hadoop - hadoop-mapreduce-client-core - 2.6.0-cdh5.11.0 - - - - org.apache.pig - pig - 0.11.1 - provided - - - - commons-lang - commons-lang - 2.5 - - - - commons-io - commons-io - 2.4 - - - - org.gnu.inet - libidn - 1.15 - - - it.unimi.dsi - dsiutils - 2.0.12 - compile - - - ch.qos.logback - logback-classic - - - - - org.apache.httpcomponents - httpcore - 4.3 - - - joda-time - joda-time - 1.6 - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 2.3.2 - - 1.6 - 1.6 - - - - maven-assembly-plugin - 2.4 - - - jar-with-dependencies - - ia-web-commons - - - - package - - single - - - - - - org.apache.maven.plugins - maven-enforcer-plugin - 1.3.1 - - - enforce-maven - - enforce - - - - - This project requires Maven 3.0.5 or higher - 3.0.5 - - - - - - - - - - - src/main/resources - true - - - - - - - cloudera - Cloudera Hadoop - https://repository.cloudera.com/artifactory/cloudera-repos/ - default - - - true - daily - warn - - - true - daily - warn - - - - - - - - From 00109ece7cb6d5aa7c2a32a3a8cae25af250d60a Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Tue, 22 Oct 2019 13:27:35 +0200 Subject: [PATCH 47/83] Upgrade dependencies --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 9cd3f253..9d9a7be2 100644 --- a/pom.xml +++ b/pom.xml @@ -159,7 +159,7 @@ commons-io commons-io - 2.4 + 2.6 @@ -195,7 +195,7 @@ org.apache.httpcomponents httpcore - 4.3 + 4.4.12 joda-time From 428022bc9a2638337ecb9f08fea5f89d155e2443 Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Thu, 12 Dec 2019 20:45:03 +0100 Subject: [PATCH 48/83] Work-around issue processing WARC request records with invalid message line (missing/empty HTTP version string) - make HttpRequestMessageParser.parseLax(...) not throw an exception --- .../java/org/archive/format/http/HttpRequestMessageParser.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/java/org/archive/format/http/HttpRequestMessageParser.java b/src/main/java/org/archive/format/http/HttpRequestMessageParser.java index f7bc43c7..159c2b73 100644 --- a/src/main/java/org/archive/format/http/HttpRequestMessageParser.java +++ b/src/main/java/org/archive/format/http/HttpRequestMessageParser.java @@ -181,9 +181,6 @@ public int parseLax(byte buf[], int len, HttpRequestMessageObserver obs) } while(buf[idx] == SP) { idx++; - if(idx >= len) { - throw new HttpParseException("No spaces in message"); - } } vs = idx; while(idx < len) { From 37b40f5bc718bf80e0815d421316d0ccecdcb6e7 Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Fri, 5 Jun 2020 15:13:04 +0200 Subject: [PATCH 49/83] - simplify date parsing resp. do not parse if not needed - use static calls for static methods --- .../archive/extract/WATExtractorOutput.java | 39 +++++-------- .../archive/extract/WETExtractorOutput.java | 57 +++++++------------ .../archive/format/warc/WARCRecordWriter.java | 7 +-- 3 files changed, 36 insertions(+), 67 deletions(-) diff --git a/src/main/java/org/archive/extract/WATExtractorOutput.java b/src/main/java/org/archive/extract/WATExtractorOutput.java index 149aa7ac..f531e182 100644 --- a/src/main/java/org/archive/extract/WATExtractorOutput.java +++ b/src/main/java/org/archive/extract/WATExtractorOutput.java @@ -1,6 +1,5 @@ package org.archive.extract; -import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; @@ -22,11 +21,8 @@ import org.archive.util.DateUtils; import org.archive.util.StreamCopy; import org.archive.util.io.CommitedOutputStream; -import com.github.openjson.JSONException; import java.net.InetAddress; -import java.text.DateFormat; -import java.text.SimpleDateFormat; import java.util.logging.Logger; @@ -164,23 +160,29 @@ private void writeWARC(OutputStream recOut, MetaData md) throws IOException { targetURI = extractOrIO(md, "Envelope.WARC-Header-Metadata.WARC-Target-URI"); } // handle date of generation in WARC format - DateFormat dateFormat = new SimpleDateFormat("yyyyMMddHHmmss"); - String capDateString = dateFormat.format(new Date()); + Date date = new Date(); String recId = extractOrIO(md, "Envelope.WARC-Header-Metadata.WARC-Record-ID"); - writeWARCMDRecord(recOut,md,targetURI,capDateString,recId); + writeWARCMDRecord(recOut,md,targetURI,date,recId); } private void writeWARCMDRecord(OutputStream recOut, MetaData md, - String targetURI, String capDateString, String recId) + String targetURI, Date capDate, String recId) throws IOException { - ByteArrayOutputStream bos = new ByteArrayOutputStream(); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); OutputStreamWriter osw = new OutputStreamWriter(bos, UTF8); String contents = md.toString(); osw.write(contents, 0, contents.length()); osw.flush(); -// ByteArrayInputStream bais = new ByteArrayInputStream(md.toString().getBytes("UTF-8")); + + recW.writeJSONMetadataRecord(recOut, bos.toByteArray(), + targetURI, capDate, recId); + } + + private void writeWARCMDRecord(OutputStream recOut, MetaData md, + String targetURI, String capDateString, String recId) + throws IOException { Date capDate; try { capDate = DateUtils.getSecondsSinceEpoch(capDateString); @@ -190,22 +192,7 @@ private void writeWARCMDRecord(OutputStream recOut, MetaData md, // TODO... not the write thing... capDate = new Date(); } - - recW.writeJSONMetadataRecord(recOut, bos.toByteArray(), - targetURI, capDate, recId); + writeWARCMDRecord(recOut, md, targetURI, capDate, recId); } - private static String transformWARCDate(final String input) { - - StringBuilder output = new StringBuilder(14); - - output.append(input.substring(0,4)); - output.append(input.substring(5,7)); - output.append(input.substring(8,10)); - output.append(input.substring(11,13)); - output.append(input.substring(14,16)); - output.append(input.substring(17,19)); - - return output.toString(); - } } diff --git a/src/main/java/org/archive/extract/WETExtractorOutput.java b/src/main/java/org/archive/extract/WETExtractorOutput.java index 14b9553f..57e62910 100644 --- a/src/main/java/org/archive/extract/WETExtractorOutput.java +++ b/src/main/java/org/archive/extract/WETExtractorOutput.java @@ -7,7 +7,6 @@ import org.archive.format.warc.WARCRecordWriter; import org.archive.resource.MetaData; import org.archive.resource.Resource; -import org.archive.util.DateUtils; import org.archive.util.IAUtils; import org.archive.util.StreamCopy; import org.archive.util.io.CommitedOutputStream; @@ -15,8 +14,11 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; -import java.nio.charset.Charset; -import java.text.ParseException; +import java.nio.charset.StandardCharsets; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; import java.util.Date; /** @@ -31,7 +33,6 @@ public class WETExtractorOutput implements ExtractorOutput { private GZIPMemberWriter gzW; private static int DEFAULT_BUFFER_RAM = 1024 * 1024; private int bufferRAM = DEFAULT_BUFFER_RAM; - private final static Charset UTF8 = Charset.forName("UTF-8"); private String outFilename; public WETExtractorOutput(OutputStream out) { @@ -63,24 +64,25 @@ public void output(Resource resource) throws IOException { MetaData top = resource.getMetaData().getTopMetaData(); CommitedOutputStream cos; - if(!wroteFirst) { + if (!wroteFirst) { cos = getOutput(); writeWARCInfo(cos, top); cos.commit(); wroteFirst = true; } String envelopeFormat = JSONUtils.extractSingle(top, "Envelope.Format"); - if(envelopeFormat == null) { + if (envelopeFormat == null) { throw new IOException("Missing Envelope.Format"); } String warctype = JSONUtils.extractSingle(top, "Envelope.WARC-Header-Metadata.WARC-Type"); + if (warctype != null && warctype.equals("response")) { String textExtract = JSONUtils.extractSingle(top, "Envelope.Payload-Metadata.HTTP-Response-Metadata.HTML-Metadata.Text"); if (textExtract != null) { cos = getOutput(); - if(envelopeFormat.startsWith("WARC")) { + if (envelopeFormat.startsWith("WARC")) { writeWARC(cos, top, textExtract); } else { // hrm... @@ -97,7 +99,7 @@ private void writeWARCInfo(OutputStream recOut, MetaData md) throws IOException if (filename == null) { filename = JSONUtils.extractSingle(md, "Container.Filename"); - if(filename == null) { + if (filename == null) { throw new IOException("No Container.Filename..."); } } @@ -129,39 +131,22 @@ private void writeWARC(OutputStream recOut, MetaData md, String textExtract) thr String targetURI = extractOrIO(md, "Envelope.WARC-Header-Metadata.WARC-Target-URI"); String capDateString = extractOrIO(md, "Envelope.WARC-Header-Metadata.WARC-Date"); - capDateString = transformWARCDate(capDateString); String recId = extractOrIO(md, "Envelope.WARC-Header-Metadata.WARC-Record-ID"); - writeWARCMDRecord(recOut, targetURI, capDateString, recId, textExtract); + writeWARCMDRecord(recOut, targetURI, parseWarcDate(capDateString), recId, textExtract); + } + + private static Date parseWarcDate(String capDateString) { + Date capDate; + ZonedDateTime zdt = ZonedDateTime.from( + DateTimeFormatter.ISO_INSTANT.withZone(ZoneId.of(ZoneOffset.UTC.toString())).parse(capDateString)); + capDate = Date.from(zdt.toInstant()); + return capDate; } - private void writeWARCMDRecord(OutputStream recOut, String targetURI, String capDateString, String recId, + private void writeWARCMDRecord(OutputStream recOut, String targetURI, Date capDate, String recId, String textExtract) throws IOException { - - Date capDate; - try { - capDate = DateUtils.getSecondsSinceEpoch(capDateString); - - } catch (ParseException e) { - e.printStackTrace(); - // TODO... not the write thing... - capDate = new Date(); - } - - recW.writeTextConversionRecord(recOut, textExtract.getBytes("UTF-8"), targetURI, capDate, recId); + recW.writeTextConversionRecord(recOut, textExtract.getBytes(StandardCharsets.UTF_8), targetURI, capDate, recId); } - private static String transformWARCDate(final String input) { - - StringBuilder output = new StringBuilder(14); - - output.append(input.substring(0,4)); - output.append(input.substring(5,7)); - output.append(input.substring(8,10)); - output.append(input.substring(11,13)); - output.append(input.substring(14,16)); - output.append(input.substring(17,19)); - - return output.toString(); - } } diff --git a/src/main/java/org/archive/format/warc/WARCRecordWriter.java b/src/main/java/org/archive/format/warc/WARCRecordWriter.java index 943410b9..398ca59e 100644 --- a/src/main/java/org/archive/format/warc/WARCRecordWriter.java +++ b/src/main/java/org/archive/format/warc/WARCRecordWriter.java @@ -16,7 +16,6 @@ public class WARCRecordWriter implements WARCConstants, HttpConstants { private static final String SCHEME = "urn:uuid"; private static final String SCHEME_COLON = SCHEME + ":"; private MessageDigest sha1; - private Base32 base32; public WARCRecordWriter() { try { @@ -24,8 +23,6 @@ public WARCRecordWriter() { } catch (NoSuchAlgorithmException e) { throw new RuntimeException(e); } - - base32 = new Base32(); } /** @@ -111,7 +108,7 @@ public void writeJSONMetadataRecord( OutputStream out, writeRecord(out, headers, contents); } - public void writeTextConversionRecord( OutputStream out, + public void writeTextConversionRecord(OutputStream out, byte[] contents, String targetURI, Date originalDate, @@ -131,7 +128,7 @@ public void writeTextConversionRecord( OutputStream out, private String contentHash(byte[] content) { sha1.reset(); - return "sha1:" + base32.encode(sha1.digest(content)); + return "sha1:" + Base32.encode(sha1.digest(content)); } private String makeRecordId() From a1f07adbe8b6d9f1059e50d9e69580428c464cff Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Fri, 5 Jun 2020 22:04:24 +0200 Subject: [PATCH 50/83] WET extractor: add identified natural language of text content (#22) - add WARC header `WARC-Identified-Content-Language to WET record - restore Text extract in WAT extractor, so that WET extractor can be called later --- .../archive/extract/WATExtractorOutput.java | 14 +++++++- .../archive/extract/WETExtractorOutput.java | 35 ++++++++++--------- .../org/archive/format/json/JSONUtils.java | 10 ------ .../archive/format/warc/WARCRecordWriter.java | 10 ++++-- 4 files changed, 40 insertions(+), 29 deletions(-) diff --git a/src/main/java/org/archive/extract/WATExtractorOutput.java b/src/main/java/org/archive/extract/WATExtractorOutput.java index f531e182..00e0875c 100644 --- a/src/main/java/org/archive/extract/WATExtractorOutput.java +++ b/src/main/java/org/archive/extract/WATExtractorOutput.java @@ -22,6 +22,8 @@ import org.archive.util.StreamCopy; import org.archive.util.io.CommitedOutputStream; +import com.github.openjson.JSONObject; + import java.net.InetAddress; import java.util.logging.Logger; @@ -69,7 +71,12 @@ public void output(Resource resource) throws IOException { } // remove the text extracts if it exists - JSONUtils.removeObject(top, "Envelope.Payload-Metadata.HTTP-Response-Metadata.HTML-Metadata", "Text"); + String textExtract = null; + JSONObject htmlMeta = JSONUtils.extractObject(top, "Envelope.Payload-Metadata.HTTP-Response-Metadata.HTML-Metadata"); + if (htmlMeta != null && htmlMeta.has("Text")) { + textExtract = htmlMeta.getString("Text"); + htmlMeta.remove("Text"); + } cos = getOutput(); if(envelopeFormat.startsWith("ARC")) { @@ -81,6 +88,11 @@ public void output(Resource resource) throws IOException { throw new IOException("Unknown Envelope.Format"); } cos.commit(); + + // restore text extract + if (textExtract != null) { + htmlMeta.put("Text", textExtract); + } } private void writeWARCInfo(OutputStream recOut, MetaData md) throws IOException { diff --git a/src/main/java/org/archive/extract/WETExtractorOutput.java b/src/main/java/org/archive/extract/WETExtractorOutput.java index 57e62910..b126096f 100644 --- a/src/main/java/org/archive/extract/WETExtractorOutput.java +++ b/src/main/java/org/archive/extract/WETExtractorOutput.java @@ -11,6 +11,8 @@ import org.archive.util.StreamCopy; import org.archive.util.io.CommitedOutputStream; +import com.github.openjson.JSONObject; + import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; @@ -20,6 +22,8 @@ import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.Date; +import java.util.Map; +import java.util.TreeMap; /** * This is for generating a WARC Encapsulated Text file @@ -51,14 +55,6 @@ private CommitedOutputStream getOutput() { } - private String extractOrIO(MetaData md, String path) throws IOException { - String value = JSONUtils.extractSingle(md, path); - if(value == null) { - throw new IOException("No "+path+" found."); - } - return value; - } - public void output(Resource resource) throws IOException { StreamCopy.readToEOF(resource.getInputStream()); MetaData top = resource.getMetaData().getTopMetaData(); @@ -76,8 +72,10 @@ public void output(Resource resource) throws IOException { } String warctype = JSONUtils.extractSingle(top, "Envelope.WARC-Header-Metadata.WARC-Type"); + if (warctype == null) + return; - if (warctype != null && warctype.equals("response")) { + if (warctype.equals("response")) { String textExtract = JSONUtils.extractSingle(top, "Envelope.Payload-Metadata.HTTP-Response-Metadata.HTML-Metadata.Text"); if (textExtract != null) { @@ -128,11 +126,16 @@ private void writeWARCInfo(OutputStream recOut, MetaData md) throws IOException } private void writeWARC(OutputStream recOut, MetaData md, String textExtract) throws IOException { - String targetURI = extractOrIO(md, "Envelope.WARC-Header-Metadata.WARC-Target-URI"); - - String capDateString = extractOrIO(md, "Envelope.WARC-Header-Metadata.WARC-Date"); - String recId = extractOrIO(md, "Envelope.WARC-Header-Metadata.WARC-Record-ID"); - writeWARCMDRecord(recOut, targetURI, parseWarcDate(capDateString), recId, textExtract); + JSONObject headers = JSONUtils.extractObject(md, "Envelope.WARC-Header-Metadata"); + String targetURI = headers.getString("WARC-Target-URI"); + String capDateString = headers.getString("WARC-Date"); + String recId = headers.getString("WARC-Record-ID"); + Map addHeaders = null; + if (headers.has("WARC-Identified-Content-Language")) { + addHeaders = new TreeMap(); + addHeaders.put("WARC-Identified-Content-Language", headers.getString("WARC-Identified-Content-Language")); + } + writeWARCMDRecord(recOut, targetURI, parseWarcDate(capDateString), recId, textExtract, addHeaders); } private static Date parseWarcDate(String capDateString) { @@ -144,9 +147,9 @@ private static Date parseWarcDate(String capDateString) { } private void writeWARCMDRecord(OutputStream recOut, String targetURI, Date capDate, String recId, - String textExtract) + String textExtract, Map addHeaders) throws IOException { - recW.writeTextConversionRecord(recOut, textExtract.getBytes(StandardCharsets.UTF_8), targetURI, capDate, recId); + recW.writeTextConversionRecord(recOut, textExtract.getBytes(StandardCharsets.UTF_8), targetURI, capDate, recId, addHeaders); } } diff --git a/src/main/java/org/archive/format/json/JSONUtils.java b/src/main/java/org/archive/format/json/JSONUtils.java index 0dc6ad24..6fff07bb 100644 --- a/src/main/java/org/archive/format/json/JSONUtils.java +++ b/src/main/java/org/archive/format/json/JSONUtils.java @@ -114,14 +114,4 @@ private static void extractRecursive(JSONObject json, String path[], int idx, Li } } } - public static boolean removeObject(JSONObject json, String path, String node) { - JSONObject obj = extractObject(json, "Envelope.Payload-Metadata.HTTP-Response-Metadata.HTML-Metadata"); - if (obj != null) { - if (obj.remove("Text") != null) { - return true; - } - } - - return false; - } } diff --git a/src/main/java/org/archive/format/warc/WARCRecordWriter.java b/src/main/java/org/archive/format/warc/WARCRecordWriter.java index 398ca59e..4f5e7461 100644 --- a/src/main/java/org/archive/format/warc/WARCRecordWriter.java +++ b/src/main/java/org/archive/format/warc/WARCRecordWriter.java @@ -5,6 +5,7 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.Date; +import java.util.Map; import java.util.UUID; import org.archive.format.http.HttpConstants; @@ -112,7 +113,8 @@ public void writeTextConversionRecord(OutputStream out, byte[] contents, String targetURI, Date originalDate, - String origRecordId) throws IOException + String origRecordId, + Map addHeaders) throws IOException { HttpHeaders headers = new HttpHeaders(); headers.add(HEADER_KEY_TYPE, WARCRecordType.conversion.name()); @@ -121,7 +123,11 @@ public void writeTextConversionRecord(OutputStream out, headers.add(HEADER_KEY_ID, makeRecordId()); headers.add(HEADER_KEY_REFERS_TO, origRecordId); headers.add(HEADER_KEY_BLOCK_DIGEST, contentHash(contents)); - + if (addHeaders != null) { + for (Map.Entry e : addHeaders.entrySet()) { + headers.add(e.getKey(), e.getValue()); + } + } headers.add(CONTENT_TYPE, "text/plain"); writeRecord(out, headers, contents); } From d2c73795cd70ac6f8e0d872ed7493b348f7dee05 Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Thu, 11 Jun 2020 14:24:03 +0200 Subject: [PATCH 51/83] WAT extractor: do not fail on missing WARC-Filename in warcinfo record, fixes #88 - do not throw IOException if there is no WARC-Filename in warcinfo record - write metadata record (corresponding to warcinfo) without WARC-Target-URI --- src/main/java/org/archive/extract/WATExtractorOutput.java | 2 +- src/main/java/org/archive/format/warc/WARCRecordWriter.java | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/archive/extract/WATExtractorOutput.java b/src/main/java/org/archive/extract/WATExtractorOutput.java index 00e0875c..2e25abf1 100644 --- a/src/main/java/org/archive/extract/WATExtractorOutput.java +++ b/src/main/java/org/archive/extract/WATExtractorOutput.java @@ -167,7 +167,7 @@ private void writeWARC(OutputStream recOut, MetaData md) throws IOException { String warcType = extractOrIO(md, "Envelope.WARC-Header-Metadata.WARC-Type"); String targetURI; if(warcType.equals("warcinfo")) { - targetURI = extractOrIO(md, "Envelope.WARC-Header-Metadata.WARC-Filename"); + targetURI = JSONUtils.extractSingle(md, "Envelope.WARC-Header-Metadata.WARC-Filename"); } else { targetURI = extractOrIO(md, "Envelope.WARC-Header-Metadata.WARC-Target-URI"); } diff --git a/src/main/java/org/archive/format/warc/WARCRecordWriter.java b/src/main/java/org/archive/format/warc/WARCRecordWriter.java index 4f5e7461..02e6700e 100644 --- a/src/main/java/org/archive/format/warc/WARCRecordWriter.java +++ b/src/main/java/org/archive/format/warc/WARCRecordWriter.java @@ -100,7 +100,10 @@ public void writeJSONMetadataRecord( OutputStream out, { HttpHeaders headers = new HttpHeaders(); headers.add(HEADER_KEY_TYPE, WARCRecordType.metadata.name()); - headers.add(HEADER_KEY_URI, targetURI); + if (targetURI != null) { + // WARC-Target-URI is optional in metadata records + headers.add(HEADER_KEY_URI, targetURI); + } headers.add(HEADER_KEY_DATE, DateUtils.getLog14Date(originalDate)); headers.add(HEADER_KEY_ID, makeRecordId()); headers.add(HEADER_KEY_REFERS_TO, origRecordId); From c0381bcef400a79546789b0a3959ea8ff5e675e2 Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Mon, 15 Jun 2020 13:29:25 +0200 Subject: [PATCH 52/83] Update change log to include #85, #86 and #89 --- CHANGES.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index dcb598d9..bf985ada 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,10 @@ +1.1.10 +------ +* [WAT extractor: do not fail on missing WARC-Filename in warcinfo record](https://github.com/iipc/webarchive-commons/pull/89) +* [ExtractingParseObserver: extract rel, hreflang and type attributes](https://github.com/iipc/webarchive-commons/pull/86) +* [ExtractingParseObserver: extract links from onClick attributes](https://github.com/iipc/webarchive-commons/pull/85) +* [Update TravisCI config](https://github.com/iipc/webarchive-commons/pull/83) + 1.1.9 ----- * [Use commons-collections v3.2.2 to avoid v3.2.1 vulnerability](https://github.com/iipc/webarchive-commons/pull/77) From 111524684ec17d2a606d5f4a8a39a8bf2ee8b641 Mon Sep 17 00:00:00 2001 From: Lauren Ko Date: Mon, 15 Jun 2020 09:15:26 -0500 Subject: [PATCH 53/83] Merge pull request #89 from sebastian-nagel/webarchive-commons-88, fixes #23, closes #24 WAT extractor: do not fail on missing WARC-Filename in warcinfo record From 8accf193fab35d0d920498fa2941e416c0d6e01f Mon Sep 17 00:00:00 2001 From: Sebastian Nagel Date: Tue, 16 Mar 2021 11:40:20 +0100 Subject: [PATCH 54/83] Fix InterruptibleCharSequenceTest (testInterruptibility), fixes #25 - if thread running the regexp matching is already finished after the initial/current sleeping time, rerun the test again with a shorter sleeping time until the expected RuntimeException is hit --- .../util/InterruptibleCharSequenceTest.java | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/test/java/org/archive/util/InterruptibleCharSequenceTest.java b/src/test/java/org/archive/util/InterruptibleCharSequenceTest.java index a3a5f180..8b5c5d1b 100644 --- a/src/test/java/org/archive/util/InterruptibleCharSequenceTest.java +++ b/src/test/java/org/archive/util/InterruptibleCharSequenceTest.java @@ -107,14 +107,24 @@ public void testNoninterruptible() throws InterruptedException { } public void testInterruptibility() throws InterruptedException { - BlockingQueue