Merge branch 'master' into release

Author: garydgregory

src/changes/changes.xml

@@ -55,6 +55,7 @@ The <action> type attribute can be add,update,fix,remove.
       <action issue="CODEC-273" dev="ggregory" type="add" due-to="Gary Gregory">Add Path APIs to org.apache.commons.codec.digest.DigestUtils similar to File APIs.</action>
       <action issue="CODEC-274" dev="ggregory" type="add" due-to="Gary Gregory">Add SHA-512/224 and SHA-512/256 to DigestUtils for Java 9 and up.</action>
       <action issue="CODEC-275" dev="ggregory" type="add" due-to="Claude Warren">Add missing note in javadoc when sign extension error is present #34.</action>
+      <action issue="CODEC-276" dev="ggregory" type="add" due-to="Gary Gregory">Reliance on default encoding in MurmurHash2 and MurmurHash3.</action>
     </release>
 
     <release version="1.13" date="2019-07-20" description="Feature and fix release.">

src/main/java/org/apache/commons/codec/binary/Base32.java

@@ -338,17 +338,16 @@ public Base32(final int lineLength, final byte[] lineSeparator, final boolean us
      * silently ignored, but has implications for other bytes, too. This method subscribes to the garbage-in,
      * garbage-out philosophy: it will not check the provided data for validity.
      * </p>
+     * <p>
+     * Output is written to {@link org.apache.commons.codec.binary.BaseNCodec.Context#buffer Context#buffer} as 8-bit
+     * octets, using {@link org.apache.commons.codec.binary.BaseNCodec.Context#pos Context#pos} as the buffer position
+     * </p>
      *
-     * @param in
-     *            byte[] array of ascii data to Base32 decode.
-     * @param inPos
-     *            Position to start reading data from.
-     * @param inAvail
-     *            Amount of bytes available from input for decoding.
+     * @param in byte[] array of ascii data to Base32 decode.
+     * @param inPos Position to start reading data from.
+     * @param inAvail Amount of bytes available from input for decoding.
      * @param context the context to be used
      *
-     * Output is written to {@link org.apache.commons.codec.binary.BaseNCodec.Context#buffer Context#buffer}
-     * as 8-bit octets, using {@link org.apache.commons.codec.binary.BaseNCodec.Context#pos Context#pos} as the buffer position
      */
     @Override
     void decode(final byte[] in, int inPos, final int inAvail, final Context context) {

src/main/java/org/apache/commons/codec/binary/BaseNCodecOutputStream.java

@@ -49,7 +49,7 @@ public class BaseNCodecOutputStream extends FilterOutputStream {
 
     /**
      * TODO should this be protected?
-     * 
+     *
      * @param out the underlying output or null.
      * @param basedCodec a BaseNCodec.
      * @param doEncode true to encode, false to decode, TODO should be an enum?

src/main/java/org/apache/commons/codec/digest/DigestUtils.java

@@ -134,7 +134,8 @@ public static byte[] digest(final MessageDigest messageDigest, final InputStream
      *             On error reading from the stream
      * @since 1.14
      */
-    public static byte[] digest(final MessageDigest messageDigest, final Path data, final OpenOption... options) throws IOException {
+    public static byte[] digest(final MessageDigest messageDigest, final Path data, final OpenOption... options)
+        throws IOException {
         return updateDigest(messageDigest, data, options).digest();
     }
 
@@ -1195,7 +1196,7 @@ public static String sha512_224Hex(final byte[] data) {
      * @param data
      *            Data to digest
      * @return SHA-512/224 digest as a hex string
-     * @throws IOException 
+     * @throws IOException
      *             On error reading from the stream
      * @since 1.14
      */
@@ -1271,7 +1272,7 @@ public static String sha512_256Hex(final byte[] data) {
      * @param data
      *            Data to digest
      * @return SHA-512/256 digest as a hex string
-     * @throws IOException 
+     * @throws IOException
      *             On error reading from the stream
      * @since 1.14
      */
@@ -1452,7 +1453,8 @@ private static MessageDigest updateDigest(final MessageDigest digest, final File
      *             On error reading from the stream
      * @since 1.8
      */
-    public static MessageDigest updateDigest(final MessageDigest digest, final InputStream inputStream) throws IOException {
+    public static MessageDigest updateDigest(final MessageDigest digest, final InputStream inputStream)
+        throws IOException {
         final byte[] buffer = new byte[STREAM_BUFFER_LENGTH];
         int read = inputStream.read(buffer, 0, STREAM_BUFFER_LENGTH);
 
@@ -1478,13 +1480,13 @@ public static MessageDigest updateDigest(final MessageDigest digest, final Input
      *             On error reading from the stream
      * @since 1.14
      */
-    public static MessageDigest updateDigest(final MessageDigest digest, final Path path, final OpenOption... options) throws IOException {
+    public static MessageDigest updateDigest(final MessageDigest digest, final Path path, final OpenOption... options)
+        throws IOException {
         try (final BufferedInputStream inputStream = new BufferedInputStream(Files.newInputStream(path, options))) {
             return updateDigest(digest, inputStream);
         }
     }
 
-
     /**
      * Reads through a RandomAccessFile and updates the digest for the data using non-blocking-io (NIO)
      *

src/main/java/org/apache/commons/codec/digest/MessageDigestAlgorithms.java

@@ -98,7 +98,7 @@ public class MessageDigestAlgorithms {
      * <p>
      * Included starting in Oracle Java 9.
      * </p>
-     * 
+     *
      * @since 1.14
      */
     public static final String SHA_512_224 = "SHA-512/224";
@@ -108,7 +108,7 @@ public class MessageDigestAlgorithms {
      * <p>
      * Included starting in Oracle Java 9.
      * </p>
-     * 
+     *
      * @since 1.14
      */
     public static final String SHA_512_256 = "SHA-512/256";

src/main/java/org/apache/commons/codec/digest/MurmurHash2.java

@@ -17,6 +17,9 @@
 
 package org.apache.commons.codec.digest;
 
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+
 /**
  * Implementation of the MurmurHash2 32-bit and 64-bit hash functions.
  *
@@ -48,6 +51,13 @@
  */
 public final class MurmurHash2 {
 
+    /**
+     * Default Charset used to convert strings into bytes.
+     * 
+     * Consider private; package private for tests only.
+     */
+    static final Charset GET_BYTES_CHARSET = StandardCharsets.UTF_8;
+
     // Constants for 32-bit variant
     private static final int M32 = 0x5bd1e995;
     private static final int R32 = 24;
@@ -132,7 +142,7 @@ public static int hash32(final byte[] data, final int length) {
      *
      * <pre>
      * int seed = 0x9747b28c;
-     * byte[] bytes = data.getBytes();
+     * byte[] bytes = data.getBytes(StandardCharsets.UTF_8);
      * int hash = MurmurHash2.hash32(bytes, bytes.length, seed);
      * </pre>
      *
@@ -141,7 +151,7 @@ public static int hash32(final byte[] data, final int length) {
      * @see #hash32(byte[], int, int)
      */
     public static int hash32(final String text) {
-        final byte[] bytes = text.getBytes();
+        final byte[] bytes = text.getBytes(GET_BYTES_CHARSET);
         return hash32(bytes, bytes.length);
     }
 
@@ -152,7 +162,7 @@ public static int hash32(final String text) {
      *
      * <pre>
      * int seed = 0x9747b28c;
-     * byte[] bytes = text.substring(from, from + length).getBytes();
+     * byte[] bytes = text.substring(from, from + length).getBytes(StandardCharsets.UTF_8);
      * int hash = MurmurHash2.hash32(bytes, bytes.length, seed);
      * </pre>
      *
@@ -243,7 +253,7 @@ public static long hash64(final byte[] data, final int length) {
      *
      * <pre>
      * int seed = 0xe17a1465;
-     * byte[] bytes = data.getBytes();
+     * byte[] bytes = data.getBytes(StandardCharsets.UTF_8);
      * int hash = MurmurHash2.hash64(bytes, bytes.length, seed);
      * </pre>
      *
@@ -252,7 +262,7 @@ public static long hash64(final byte[] data, final int length) {
      * @see #hash64(byte[], int, int)
      */
     public static long hash64(final String text) {
-        final byte[] bytes = text.getBytes();
+        final byte[] bytes = text.getBytes(GET_BYTES_CHARSET);
         return hash64(bytes, bytes.length);
     }
 
@@ -263,7 +273,7 @@ public static long hash64(final String text) {
      *
      * <pre>
      * int seed = 0xe17a1465;
-     * byte[] bytes = text.substring(from, from + length).getBytes();
+     * byte[] bytes = text.substring(from, from + length).getBytes(StandardCharsets.UTF_8);
      * int hash = MurmurHash2.hash64(bytes, bytes.length, seed);
      * </pre>
      *

src/main/java/org/apache/commons/codec/digest/MurmurHash3.java

@@ -17,41 +17,54 @@
 
 package org.apache.commons.codec.digest;
 
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+
 /**
  * Implementation of the MurmurHash3 32-bit and 128-bit hash functions.
  *
- * <p>MurmurHash is a non-cryptographic hash function suitable for general
- * hash-based lookup. The name comes from two basic operations, multiply (MU)
- * and rotate (R), used in its inner loop. Unlike cryptographic hash functions,
- * it is not specifically designed to be difficult to reverse by an adversary,
- * making it unsuitable for cryptographic purposes.</p>
+ * <p>
+ * MurmurHash is a non-cryptographic hash function suitable for general hash-based lookup. The name comes from two basic
+ * operations, multiply (MU) and rotate (R), used in its inner loop. Unlike cryptographic hash functions, it is not
+ * specifically designed to be difficult to reverse by an adversary, making it unsuitable for cryptographic purposes.
+ * </p>
  *
- * <p>This contains a Java port of the 32-bit hash function {@code MurmurHash3_x86_32}
- * and the 128-bit hash function {@code MurmurHash3_x64_128} from Austin Applyby's
- * original {@code c++} code in SMHasher.</p>
+ * <p>
+ * This contains a Java port of the 32-bit hash function {@code MurmurHash3_x86_32} and the 128-bit hash function
+ * {@code MurmurHash3_x64_128} from Austin Applyby's original {@code c++} code in SMHasher.
+ * </p>
  *
- * <p>This is public domain code with no copyrights. From home page of
- * <a href="https://github.com/aappleby/smhasher">SMHasher</a>:</p>
+ * <p>
+ * This is public domain code with no copyrights. From home page of
+ * <a href="https://github.com/aappleby/smhasher">SMHasher</a>:
+ * </p>
  *
- * <blockquote>
- * "All MurmurHash versions are public domain software, and the author
- * disclaims all copyright to their code."
- * </blockquote>
+ * <blockquote> "All MurmurHash versions are public domain software, and the author disclaims all copyright to their
+ * code." </blockquote>
  *
- * <p>Original adaption from Apache Hive. That adaption contains a {@code hash64} method
- * that is not part of the original MurmurHash3 code. It is not recommended to use these methods.
- * They will be removed in a future release. To obtain a 64-bit hash use half of the bits
- * from the {@code hash128x64} methods using the input data converted to bytes.<p>
+ * <p>
+ * Original adaption from Apache Hive. That adaption contains a {@code hash64} method that is not part of the original
+ * MurmurHash3 code. It is not recommended to use these methods. They will be removed in a future release. To obtain a
+ * 64-bit hash use half of the bits from the {@code hash128x64} methods using the input data converted to bytes.
+ * <p>
  *
  * @see <a href="https://en.wikipedia.org/wiki/MurmurHash">MurmurHash</a>
- * @see <a href="https://github.com/aappleby/smhasher/blob/master/src/MurmurHash3.cpp">
- *   Original MurmurHash3 c++ code</a>
- * @see <a href="https://github.com/apache/hive/blob/master/storage-api/src/java/org/apache/hive/common/util/Murmur3.java">
- *   Apache Hive Murmer3</a>
+ * @see <a href="https://github.com/aappleby/smhasher/blob/master/src/MurmurHash3.cpp"> Original MurmurHash3 c++
+ *      code</a>
+ * @see <a href=
+ *      "https://github.com/apache/hive/blob/master/storage-api/src/java/org/apache/hive/common/util/Murmur3.java">
+ *      Apache Hive Murmer3</a>
  * @since 1.13
  */
 public final class MurmurHash3 {
 
+    /**
+     * Default Charset used to convert strings into bytes.
+     * 
+     * Consider private; package private for tests only.
+     */
+    static final Charset GET_BYTES_CHARSET = StandardCharsets.UTF_8;
+
     /**
      * A random number to use for a hash code.
      *
@@ -230,7 +243,7 @@ public static int hash32(final byte[] data) {
      * <pre>
      * int offset = 0;
      * int seed = 104729;
-     * byte[] bytes = data.getBytes();
+     * byte[] bytes = data.getBytes(StandardCharsets.UTF_8);
      * int hash = MurmurHash3.hash32(bytes, offset, bytes.length, seed);
      * </pre>
      *
@@ -246,7 +259,7 @@ public static int hash32(final byte[] data) {
      */
     @Deprecated
     public static int hash32(final String data) {
-        final byte[] bytes = data.getBytes();
+        final byte[] bytes = data.getBytes(GET_BYTES_CHARSET);
         return hash32(bytes, 0, bytes.length, DEFAULT_SEED);
     }
 
@@ -748,7 +761,7 @@ public static long[] hash128x64(final byte[] data) {
      * <pre>
      * int offset = 0;
      * int seed = 104729;
-     * byte[] bytes = data.getBytes();
+     * byte[] bytes = data.getBytes(StandardCharsets.UTF_8);
      * int hash = MurmurHash3.hash128(bytes, offset, bytes.length, seed);
      * </pre>
      *
@@ -763,7 +776,7 @@ public static long[] hash128x64(final byte[] data) {
      */
     @Deprecated
     public static long[] hash128(final String data) {
-        final byte[] bytes = data.getBytes();
+        final byte[] bytes = data.getBytes(GET_BYTES_CHARSET);
         return hash128(bytes, 0, bytes.length, DEFAULT_SEED);
     }
 

src/main/java/org/apache/commons/codec/digest/XXHash32.java

@@ -24,9 +24,14 @@
 /**
  * Implementation of the xxhash32 hash algorithm.
  *
- * <p>Copied from Commons Compress 1.14
- * <a href="https://git-wip-us.apache.org/repos/asf?p=commons-compress.git;a=blob;f=src/main/java/org/apache/commons/compress/compressors/lz4/XXHash32.java;h=a406ffc197449be594d46f0d2712b2d4786a1e68;hb=HEAD">https://git-wip-us.apache.org/repos/asf?p=commons-compress.git;a=blob;f=src/main/java/org/apache/commons/compress/compressors/lz4/XXHash32.java;h=a406ffc197449be594d46f0d2712b2d4786a1e68;hb=HEAD</a></p>
- * <p>NotThreadSafe</p>
+ * <p>
+ * Copied from Commons Compress 1.14 <a href=
+ * "https://git-wip-us.apache.org/repos/asf?p=commons-compress.git;a=blob;f=src/main/java/org/apache/commons/compress/compressors/lz4/XXHash32.java;h=a406ffc197449be594d46f0d2712b2d4786a1e68;hb=HEAD">https://git-wip-us.apache.org/repos/asf?p=commons-compress.git;a=blob;f=src/main/java/org/apache/commons/compress/compressors/lz4/XXHash32.java;h=a406ffc197449be594d46f0d2712b2d4786a1e68;hb=HEAD</a>
+ * </p>
+ * <p>
+ * NotThreadSafe
+ * </p>
+ *
  * @see <a href="http://cyan4973.github.io/xxHash/">xxHash</a>
  * @since 1.11
  */

src/main/java/org/apache/commons/codec/language/ColognePhonetic.java

@@ -241,8 +241,8 @@ public CologneOutputBuffer(final int buffSize) {
          * '0' is only stored if it is the first entry.
          * Ignored chars are never stored.
          * If the code is the same as the last code (whether stored or not) it is not stored.
-         * 
-         * @param code the code to store. 
+         *
+         * @param code the code to store.
          */
         public void put(final char code) {
             if (code != CHAR_IGNORE && lastCode != code && (code != '0' || length == 0)) {
@@ -404,7 +404,7 @@ public String encode(final String text) {
 
     /**
      * Compares the first encoded string to the second encoded string.
-     * 
+     *
      * @param text1 source text to encode before testing for equality.
      * @param text2 source text to encode before testing for equality.
      * @return {@code true} if the encoding the first string equals the encoding of the second string, {@code false}

src/test/java/org/apache/commons/codec/digest/DigestUtilsTest.java

@@ -198,7 +198,7 @@ public void testMd2Length() {
         hash = DigestUtils.md2(getBytesUtf8(hashMe));
         assertEquals(16, hash.length);
     }
-    
+
     @Test
     public void testMd5Hex() throws IOException {
         // Examples from RFC 1321