Initialize PrintStreams using UTF-8 charset,

call String.getBytes() with charset.

Author: sebastian-nagel

src/main/java/org/archive/extract/DumpingExtractorOutput.java

@@ -3,6 +3,7 @@
 import java.io.IOException;
 import java.io.OutputStream;
 import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
 import java.util.logging.Logger;
 
 import org.archive.resource.Resource;
@@ -12,13 +13,18 @@
 import com.google.common.io.ByteStreams;
 import com.google.common.io.CountingOutputStream;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+
 public class DumpingExtractorOutput implements ExtractorOutput {
 	private static final Logger LOG = 
 		Logger.getLogger(DumpingExtractorOutput.class.getName());
 
 	private PrintStream out;
 	public DumpingExtractorOutput(OutputStream out) {
-		this.out = new PrintStream(out);
+		try {
+			this.out = new PrintStream(out, false, UTF_8.name());
+		} catch (UnsupportedEncodingException e) {
+		}
 	}
 
 	public void output(Resource resource) throws IOException {

src/main/java/org/archive/extract/JSONViewExtractorOutput.java

@@ -3,19 +3,25 @@
 import java.io.IOException;
 import java.io.OutputStream;
 import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
 import java.util.List;
 
 import org.apache.commons.lang3.StringUtils;
 import org.archive.format.json.JSONView;
 import org.archive.resource.Resource;
 import org.archive.util.StreamCopy;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+
 public class JSONViewExtractorOutput implements ExtractorOutput {
 	private PrintStream out;
 	private JSONView view;
 	public JSONViewExtractorOutput(OutputStream out, String filterPath) {
 		view = new JSONView(filterPath.split(","));
-		this.out = new PrintStream(out);
+		try {
+			this.out = new PrintStream(out, false, UTF_8.name());
+		} catch (UnsupportedEncodingException e) {
+		}
 	}
 	public void output(Resource resource) throws IOException {
 		StreamCopy.readToEOF(resource.getInputStream());

src/main/java/org/archive/format/gzip/zipnum/ZipNumCluster.java

@@ -13,6 +13,7 @@
 import java.io.BufferedReader;
 import java.io.FileReader;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
@@ -35,6 +36,8 @@
 import org.archive.util.binsearch.impl.HTTPSeekableLineReader;
 import org.archive.util.iterator.CloseableIterator;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+
 public class ZipNumCluster extends ZipNumIndex {
 
 	final static Logger LOGGER = Logger.getLogger(ZipNumCluster.class.getName());

src/main/java/org/archive/io/arc/ARC2WCDX.java

@@ -32,6 +32,8 @@
 import org.archive.util.ArchiveUtils;
 import org.archive.util.SURT;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+
 /**
  * Create a 'Wide' CDX from an ARC. Takes one argument, the path to the ARC.
  * Writes .wcdx.gz in same directory.
@@ -61,7 +63,7 @@ public static Object[] createWcdx(ARCReader reader) {
         PrintStream writer = null;
         long count = 0;
         try {
-            writer = new PrintStream(new GZIPOutputStream(new FileOutputStream(wcdxFile)));
+            writer = new PrintStream(new GZIPOutputStream(new FileOutputStream(wcdxFile)), false, UTF_8.name());
 
             // write header: legend + timestamp
             StringBuilder legend = new StringBuilder();

src/main/java/org/archive/io/warc/WARCWriter.java

@@ -45,6 +45,8 @@
 
 import static org.archive.format.warc.WARCConstants.*;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+
 
 /**
  * WARC implementation.
@@ -357,12 +359,12 @@ public URI writeWarcinfoRecord(String filename, final String description)
         byte [] warcinfoBody = null;
         if (settings.getMetadata() == null) {
         	// TODO: What to write into a warcinfo?  What to associate?
-        	warcinfoBody = "TODO: Unimplemented".getBytes();
+        	warcinfoBody = "TODO: Unimplemented".getBytes(UTF_8);
         } else {
         	ByteArrayOutputStream baos = new ByteArrayOutputStream();
         	for (final Iterator<String> i = settings.getMetadata().iterator();
         			i.hasNext();) {
-        		baos.write(i.next().toString().getBytes(UTF8Bytes.UTF8));
+        		baos.write(i.next().toString().getBytes(UTF_8));
         	}
         	warcinfoBody = baos.toByteArray();
         }

src/main/java/org/archive/url/URI.java

@@ -42,6 +42,8 @@
 import java.util.Hashtable;
 import java.util.Locale;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+
 /**
  * The interface for the URI(Uniform Resource Identifiers) version of RFC 2396.
  * This class has the purpose of supportting of parsing a URI reference to
@@ -1696,7 +1698,7 @@ private static byte[] getBytes(String original, String charset) {
         try {
             return original.getBytes(charset);
         } catch (UnsupportedEncodingException e) {
-            return original.getBytes();
+            return original.getBytes(UTF_8);
         }
     }
 

src/main/java/org/archive/util/SURT.java

@@ -32,6 +32,8 @@
 import org.archive.url.URIException;
 import org.archive.url.UsableURIFactory;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+
 /**
  * Sort-friendly URI Reordering Transform.
  * 
@@ -238,7 +240,7 @@ public static void main(String[] args) throws IOException {
         InputStream in = args.length > 0 ? new BufferedInputStream(
                 new FileInputStream(args[0])) : System.in;
         PrintStream out = args.length > 1 ? new PrintStream(
-                new BufferedOutputStream(new FileOutputStream(args[1])))
+                new BufferedOutputStream(new FileOutputStream(args[1])), false, UTF_8.name())
                 : System.out;
         BufferedReader br =
             new BufferedReader(new InputStreamReader(in));

src/main/java/org/archive/util/SurtPrefixSet.java

@@ -37,6 +37,8 @@
 import org.archive.util.iterator.LineReadingIterator;
 import org.archive.util.iterator.RegexLineIterator;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+
 /**
  * Specialized TreeSet for keeping a set of String prefixes. 
  * 
@@ -343,10 +345,10 @@ public static void main(String[] args) throws IOException {
         InputStream in = args.length > 0 ? new BufferedInputStream(
                 new FileInputStream(args[0])) : System.in;
         PrintStream out = args.length > 1 ? new PrintStream(
-                new BufferedOutputStream(new FileOutputStream(args[1])))
+                new BufferedOutputStream(new FileOutputStream(args[1])), false, UTF_8.name())
                 : System.out;
         BufferedReader br =
-            new BufferedReader(new InputStreamReader(in));
+            new BufferedReader(new InputStreamReader(in, UTF_8.name()));
         String line;
         while((line = br.readLine())!=null) {
             if(line.indexOf("#")>0) line=line.substring(0,line.indexOf("#"));

src/main/java/org/archive/util/TextUtils.java

@@ -40,6 +40,8 @@
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+
 public class TextUtils {
     private static final String FIRSTWORD = "^([^\\s]*).*$";
 
@@ -279,14 +281,11 @@ public static String exceptionToString(String  message, Throwable e) {
      * @param s String to escape
      * @return URL-escaped string
      */
-    @SuppressWarnings("deprecation")
     public static String urlEscape(String s) {
         try {
-            return URLEncoder.encode(s,"UTF8");
+            return URLEncoder.encode(s, UTF_8.name());
         } catch (UnsupportedEncodingException e) {
-            // should be impossible; all JVMs must support UTF8
-            // but have a fallback just in case
-            return URLEncoder.encode(s); 
+            return s;
         }
     }
 
@@ -296,14 +295,11 @@ public static String urlEscape(String s) {
      * @param s String do unescape
      * @return URL-unescaped String
      */
-    @SuppressWarnings("deprecation")
     public static String urlUnescape(String s) {
         try {
-            return URLDecoder.decode(s, "UTF8");
+            return URLDecoder.decode(s, UTF_8.name());
         } catch (UnsupportedEncodingException e) {
-            // should be impossible; all JVMs must support UTF8
-            // but have a fallback just in case
-            return URLDecoder.decode(s);
+            return s;
         }
     }
 }

src/main/java/org/archive/util/binsearch/SortedTextFile.java

@@ -9,6 +9,8 @@
 import org.archive.util.GeneralURIStreamFactory;
 import org.archive.util.iterator.CloseableIterator;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+
 public class SortedTextFile {
 
 	public static class NumericComparator implements Comparator<String>
@@ -371,7 +373,7 @@ private long searchOffset(SeekableLineReader slr,
 	    String prev = null;
 	    while(true) {
 	    	if (line != null) {
-	    		offset += line.getBytes().length + 1;
+	    		offset += line.getBytes(UTF_8).length + 1;
 	    	}
 	    	line = slr.readLine();
 	    	if(line == null) break;
@@ -380,7 +382,7 @@ private long searchOffset(SeekableLineReader slr,
 	    }
 
 	    if (lessThan && prev != null) {
-	    	offset -= prev.getBytes().length + 1;
+	    	offset -= prev.getBytes(UTF_8).length + 1;
 	    }
 
 	    return offset;