HTML encoding detection: fix errors with empty content or empty charset values

Author: sebastian-nagel

src/main/java/org/archive/format/text/charset/CharsetDetector.java

@@ -182,6 +182,8 @@ private static String trimAttrValue(String value) {
 			return value;
 		}
 		String result = value;
+		if (result.isEmpty())
+		    return result;
 		if (result.charAt(0) == '"') {
 			result = result.substring(1, result.length() - 1);
 		} else if (result.charAt(0) == '\'') {

src/main/java/org/archive/resource/html/HTMLResourceFactory.java

@@ -5,6 +5,8 @@
 import java.io.InputStream;
 import java.io.UnsupportedEncodingException;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.archive.format.http.HttpHeaders;
 import org.archive.format.json.JSONUtils;
 import org.archive.format.text.charset.CharsetDetector;
@@ -23,6 +25,8 @@
 
 public class HTMLResourceFactory implements ResourceFactory {
 
+	public static final Log LOG = LogFactory.getLog(HTMLResourceFactory.class);
+
 	protected static final int CHARSET_GUESS_CHUNK_SIZE = 8192;
 	protected static final String HTTP_HEADER_PATH = "Envelope.Payload-Metadata.HTTP-Response-Metadata.Headers";
 
@@ -37,21 +41,27 @@ public Resource getResource(InputStream is, MetaData parentMetaData,
 		CDATALexer lex = new CDATALexer();
 
 		// guess charset based on HTTP header and sniffed content chunk
+		String charset = "UTF-8";
 		is = new BufferedInputStream(is, CHARSET_GUESS_CHUNK_SIZE);
 		byte[] chunk = new byte[CHARSET_GUESS_CHUNK_SIZE];
 		is.mark(0);
 		int chunkSize = is.read(chunk, 0, CHARSET_GUESS_CHUNK_SIZE);
 		is.reset();
-		JSONObject headers = JSONUtils.extractObject(hmd.getTopMetaData(), HTTP_HEADER_PATH);
-		HttpHeaders httpHeaders = new HttpHeaders();
-		if (headers.has("Content-Type")) {
+		if (chunkSize > 0) {
+			JSONObject headers = JSONUtils.extractObject(hmd.getTopMetaData(), HTTP_HEADER_PATH);
+			HttpHeaders httpHeaders = new HttpHeaders();
+			if (headers.has("Content-Type")) {
+				try {
+					httpHeaders.add("Content-Type", headers.getString("Content-Type"));
+				} catch (JSONException e) { }
+			}
 			try {
-				httpHeaders.add("Content-Type", headers.getString("Content-Type"));
-			} catch (JSONException e) { }
+				charset = charSetDetector.getCharset(chunk, chunkSize, httpHeaders);
+			} catch (Exception e) {
+				LOG.error("Failed to guess charset: " + e.getMessage());
+			}
 		}
 
-		String charset = charSetDetector.getCharset(chunk, chunkSize, httpHeaders);
-
 		Page page;
 		try {
 			page = new Page(is, charset);