Merge branch 'master' into fix/CODEC-340_base58_custom_alphabet

Author: garydgregory

src/changes/changes.xml

@@ -46,8 +46,13 @@ The <action> type attribute can be add,update,fix,remove.
     <release version="1.22.1" date="YYYY-MM-DD" description="This is a feature and maintenance release. Java 8 or later is required.">
       <!-- FIX -->
       <action type="fix" issue="CODEC-340" dev="ggregory" due-to="Ruiqi Dong, Gary Gregory">Base58.Builder.setEncodeTable(byte...) is ignored when encoding and decoding.</action>
-      <action type="add" issue="CODEC-337" dev="pkarwasz" due-to="Ruiqi Dong, Gary Gregory">Digest ALL reuses System.in, so only the first algorithm sees the real input (#431).</action>
+      <action type="fix" issue="CODEC-342" dev="ggregory" due-to="Ruiqi Dong, Gary Gregory">Base32.Builder.setEncodeTable(byte...) can create a codec that cannot decode its own output.</action>
+      <action type="fix" issue="CODEC-343" dev="ggregory" due-to="Ruiqi Dong, Gary Gregory">Base32.Builder.setHexDecodeTable(boolean) sets the encode table to a decode lookup table.</action>
+      <action type="fix" issue="CODEC-341" dev="ggregory" due-to="Ruiqi Dong, Gary Gregory">Base16.Builder.setEncodeTable(byte...) can create a codec that cannot decode its own output.</action>
+      <action type="fix" issue="CODEC-339" dev="ggregory" due-to="Ruiqi Dong, Gary Gregory">URLCodec.encodeUrl(BitSet, byte[]) allows custom safe sets to emit URL encoding control characters.</action>
+      <action type="fix" issue="CODEC-338" dev="ggregory" due-to="Ruiqi Dong, Gary Gregory">PercentCodec loses literal '+' when plusForSpace is enabled.</action>
       <!-- ADD -->
+      <action type="add" issue="CODEC-337" dev="pkarwasz" due-to="Ruiqi Dong, Gary Gregory">Digest ALL reuses System.in, so only the first algorithm sees the real input (#431).</action>
       <!-- UPDATE -->
       <action type="update" dev="ggregory" due-to="Gary Gregory">Bump org.apache.commons:commons-parent from 98 to 102.</action>
       <action type="update" dev="ggregory" due-to="Gary Gregory">Bump commons-io:commons-io from 2.21.0 to 2.22.0.</action>

src/main/java/org/apache/commons/codec/binary/Base16.java

@@ -33,7 +33,7 @@
  * </p>
  * <p>
  * The only additional feature above those specified in RFC 4648 is support for working with a lower-case alphabet in addition to the default upper-case
- * alphabet.
+ * alphabet, and configuring a custom 16-byte alphabet with {@link Builder#setEncodeTable(byte...)}.
  * </p>
  *
  * @see Base16InputStream
@@ -78,10 +78,18 @@ public Base16 get() {
             return new Base16(this);
         }
 
+        /**
+         * Sets the Base16 encode table and derives the matching decode table.
+         *
+         * @param encodeTable 16 unique bytes, null resets to the default upper-case table.
+         * @return {@code this} instance.
+         * @throws IllegalArgumentException if {@code encodeTable} does not contain 16 unique bytes.
+         */
         @Override
         public Builder setEncodeTable(final byte... encodeTable) {
-            super.setDecodeTableRaw(Arrays.equals(encodeTable, LOWER_CASE_ENCODE_TABLE) ? LOWER_CASE_DECODE_TABLE : UPPER_CASE_DECODE_TABLE);
-            return super.setEncodeTable(encodeTable);
+            final byte[] table = encodeTable != null ? encodeTable : UPPER_CASE_ENCODE_TABLE;
+            super.setDecodeTableRaw(toDecodeTable(table));
+            return super.setEncodeTable(table);
         }
 
         /**
@@ -91,8 +99,7 @@ public Builder setEncodeTable(final byte... encodeTable) {
          * @return {@code this} instance.
          */
         public Builder setLowerCase(final boolean lowerCase) {
-            setEncodeTableRaw(lowerCase ? LOWER_CASE_ENCODE_TABLE : UPPER_CASE_ENCODE_TABLE);
-            return asThis();
+            return setEncodeTable(lowerCase ? LOWER_CASE_ENCODE_TABLE : UPPER_CASE_ENCODE_TABLE);
         }
 
     }
@@ -162,6 +169,32 @@ public static Builder builder() {
         return new Builder();
     }
 
+    private static byte[] toDecodeTable(final byte[] encodeTable) {
+        if (Arrays.equals(encodeTable, UPPER_CASE_ENCODE_TABLE)) {
+            return UPPER_CASE_DECODE_TABLE;
+        }
+        if (Arrays.equals(encodeTable, LOWER_CASE_ENCODE_TABLE)) {
+            return LOWER_CASE_DECODE_TABLE;
+        }
+        if (encodeTable.length != 1 << BITS_PER_ENCODED_BYTE) {
+            throw new IllegalArgumentException("Base16 encode table must contain 16 entries.");
+        }
+        int max = -1;
+        for (final byte b : encodeTable) {
+            max = Math.max(max, b & 0xff);
+        }
+        final byte[] decodeTable = new byte[max + 1];
+        Arrays.fill(decodeTable, (byte) -1);
+        for (int i = 0; i < encodeTable.length; i++) {
+            final int b = encodeTable[i] & 0xff;
+            if (decodeTable[b] != -1) {
+                throw new IllegalArgumentException("Duplicate value in Base16 encode table: " + b);
+            }
+            decodeTable[b] = (byte) i;
+        }
+        return decodeTable;
+    }
+
     /**
      * Constructs a Base16 codec used for decoding and encoding.
      */
@@ -241,8 +274,9 @@ void decode(final byte[] data, int offset, final int length, final Context conte
 
     private int decodeOctet(final byte octet) {
         int decoded = -1;
-        if ((octet & 0xff) < decodeTable.length) {
-            decoded = decodeTable[octet];
+        final int b = octet & 0xff;
+        if (b < decodeTable.length) {
+            decoded = decodeTable[b];
         }
         if (decoded == -1) {
             throw new IllegalArgumentException("Invalid octet in encoded value: " + (int) octet);
@@ -282,7 +316,8 @@ void encode(final byte[] data, final int offset, final int length, final Context
      */
     @Override
     public boolean isInAlphabet(final byte octet) {
-        return isInAlphabet((byte) (octet & 0xff), decodeTable);
+        final int b = octet & 0xff;
+        return b < decodeTable.length && decodeTable[b] != -1;
     }
 
     /**

src/main/java/org/apache/commons/codec/binary/Base32.java

@@ -96,14 +96,24 @@ public Base32 get() {
             return new Base32(this);
         }
 
+        /**
+         * Sets the encode table and derives the matching decode table.
+         * <p>
+         * The RFC 4648 Base32 and Base32 Hex tables keep their case-insensitive decoders.
+         * </p>
+         *
+         * @param encodeTable the encode table with exactly 32 unique entries, null resets to the default.
+         * @return {@code this} instance.
+         * @throws IllegalArgumentException if the encode table does not contain exactly 32 unique entries.
+         */
         @Override
         public Builder setEncodeTable(final byte... encodeTable) {
-            super.setDecodeTableRaw(Arrays.equals(encodeTable, HEX_ENCODE_TABLE) ? HEX_DECODE_TABLE : DECODE_TABLE);
+            super.setDecodeTableRaw(toDecodeTable(encodeTable));
             return super.setEncodeTable(encodeTable);
         }
 
         /**
-         * Sets the decode table to use Base32 hexadecimal if {@code true}, otherwise use the Base32 alphabet.
+         * Sets the encode and decode tables to use Base32 hexadecimal if {@code true}, otherwise use the Base32 alphabet.
          * <p>
          * This overrides a value previously set with {@link #setEncodeTable(byte...)}.
          * </p>
@@ -113,7 +123,7 @@ public Builder setEncodeTable(final byte... encodeTable) {
          * @since 1.18.0
          */
         public Builder setHexDecodeTable(final boolean useHex) {
-            return setEncodeTable(decodeTable(useHex));
+            return setEncodeTable(encodeTable(useHex));
         }
 
         /**
@@ -145,6 +155,8 @@ public Builder setHexEncodeTable(final boolean useHex) {
 
     private static final int BYTES_PER_ENCODED_BLOCK = 8;
     private static final int BYTES_PER_UNENCODED_BLOCK = 5;
+    private static final int DECODING_TABLE_LENGTH = 256;
+    private static final int ENCODING_TABLE_LENGTH = 1 << BITS_PER_ENCODED_BYTE;
 
     /**
      * This array is a lookup table that translates Unicode characters drawn from the "Base32 Alphabet" (as specified in Table 3 of RFC 4648) into their 5-bit
@@ -256,6 +268,29 @@ public static Builder builder() {
         return new Builder();
     }
 
+    /**
+     * Calculates a decode table for a given encode table.
+     *
+     * @param encodeTable that is used to determine decode lookup table.
+     * @return A new decode table.
+     * @throws IllegalArgumentException if the encode table does not contain exactly 32 unique entries.
+     */
+    private static byte[] calculateDecodeTable(final byte[] encodeTable) {
+        if (encodeTable.length != ENCODING_TABLE_LENGTH) {
+            throw new IllegalArgumentException("encodeTable must have exactly 32 entries.");
+        }
+        final byte[] decodeTable = new byte[DECODING_TABLE_LENGTH];
+        Arrays.fill(decodeTable, (byte) -1);
+        for (int i = 0; i < encodeTable.length; i++) {
+            final int encodedByte = encodeTable[i] & 0xff;
+            if (decodeTable[encodedByte] != -1) {
+                throw new IllegalArgumentException("encodeTable must not contain duplicate entries.");
+            }
+            decodeTable[encodedByte] = (byte) i;
+        }
+        return decodeTable;
+    }
+
     private static byte[] decodeTable(final boolean useHex) {
         return useHex ? HEX_DECODE_TABLE : DECODE_TABLE;
     }
@@ -276,6 +311,23 @@ private static byte[] encodeTable(final boolean useHex) {
         return useHex ? HEX_ENCODE_TABLE : ENCODE_TABLE;
     }
 
+    /**
+     * Gets the decode table that matches the given encode table.
+     *
+     * @param encodeTable that is used to determine decode lookup table.
+     * @return the matching decode table.
+     */
+    private static byte[] toDecodeTable(final byte[] encodeTable) {
+        final byte[] table = encodeTable != null ? encodeTable : ENCODE_TABLE;
+        if (Arrays.equals(table, ENCODE_TABLE)) {
+            return DECODE_TABLE;
+        }
+        if (Arrays.equals(table, HEX_ENCODE_TABLE)) {
+            return HEX_DECODE_TABLE;
+        }
+        return calculateDecodeTable(table);
+    }
+
     /**
      * Convenience variable to help us determine when our buffer is going to run out of room and needs resizing. {@code encodeSize = {@link
      * #BYTES_PER_ENCODED_BLOCK} + lineSeparator.length;}
@@ -530,14 +582,14 @@ void decode(final byte[] input, int inPos, final int inAvail, final Context cont
         }
         final int decodeSize = this.encodeSize - 1;
         for (int i = 0; i < inAvail; i++) {
-            final byte b = input[inPos++];
-            if (b == pad) {
+            final int b = input[inPos++] & 0xff;
+            if (b == (pad & 0xff)) {
                 // We're done.
                 context.eof = true;
                 break;
             }
             final byte[] buffer = ensureBufferSize(decodeSize, context);
-            if (b >= 0 && b < this.decodeTable.length) {
+            if (b < this.decodeTable.length) {
                 final int result = this.decodeTable[b];
                 if (result >= 0) {
                     context.modulus = (context.modulus + 1) % BYTES_PER_ENCODED_BLOCK;
@@ -738,7 +790,8 @@ byte[] getLineSeparator() {
      */
     @Override
     public boolean isInAlphabet(final byte octet) {
-        return isInAlphabet(octet, decodeTable);
+        final int value = octet & 0xff;
+        return value < decodeTable.length && decodeTable[value] != -1;
     }
 
     /**

src/main/java/org/apache/commons/codec/binary/Base58.java

@@ -92,7 +92,7 @@ public Base58.Builder setEncodeTable(final byte... encodeTable) {
     private static final int DECODING_TABLE_LENGTH = 256;
     private static final int ENCODING_TABLE_LENGTH = 58;
 
-    private static final byte[] EMPTY = new byte[0];
+    private static final byte[] EMPTY = {};
 
     /**
      * Base58 alphabet: 123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz

src/main/java/org/apache/commons/codec/binary/BaseNCodec.java

@@ -58,6 +58,17 @@ public abstract class BaseNCodec implements BinaryEncoder, BinaryDecoder {
      */
     public abstract static class AbstractBuilder<T, B extends AbstractBuilder<T, B>> implements Supplier<T> {
 
+        /**
+         * Clones the given array or returns a default array if the array is null.
+         *
+         * @param array        The array to test and clone if not null.
+         * @param defaultArray The default array to return if the array is null.
+         * @return A clone of the array or the default array if the array is null.
+         */
+        static byte[] clone(final byte[] array, final byte[] defaultArray) {
+            return array != null ? array.clone() : defaultArray;
+        }
+
         private int unencodedBlockSize;
         private int encodedBlockSize;
         private CodecPolicy decodingPolicy = DECODING_POLICY_DEFAULT;
@@ -132,7 +143,7 @@ int getUnencodedBlockSize() {
          * @since 1.20.0
          */
         public B setDecodeTable(final byte[] decodeTable) {
-            this.decodeTable = decodeTable != null ? decodeTable.clone() : null;
+            this.decodeTable = clone(decodeTable, null);
             return asThis();
         }
 
@@ -176,7 +187,7 @@ B setEncodedBlockSize(final int encodedBlockSize) {
          * @return {@code this} instance.
          */
         public B setEncodeTable(final byte... encodeTable) {
-            this.encodeTable = encodeTable != null ? encodeTable.clone() : defaultEncodeTable;
+            this.encodeTable = clone(encodeTable, defaultEncodeTable);
             return asThis();
         }
 
@@ -209,7 +220,7 @@ public B setLineLength(final int lineLength) {
          * @return {@code this} instance.
          */
         public B setLineSeparator(final byte... lineSeparator) {
-            this.lineSeparator = lineSeparator != null ? lineSeparator.clone() : CHUNK_SEPARATOR;
+            this.lineSeparator = clone(lineSeparator , CHUNK_SEPARATOR);
             return asThis();
         }
 

src/main/java/org/apache/commons/codec/net/PercentCodec.java

@@ -44,6 +44,11 @@ public class PercentCodec implements BinaryEncoder, BinaryDecoder {
      */
     private static final byte ESCAPE_CHAR = '%';
 
+    /**
+     * The plus character used to encode spaces when plusForSpace is true.
+     */
+    private static final byte PLUS_CHAR = '+';
+
     /**
      * The bit set used to store the character that should be always encoded.
      */
@@ -80,6 +85,9 @@ public PercentCodec() {
     public PercentCodec(final byte[] alwaysEncodeChars, final boolean plusForSpace) {
         this.plusForSpace = plusForSpace;
         insertAlwaysEncodeChars(alwaysEncodeChars);
+        if (plusForSpace) {
+            insertAlwaysEncodeChar(PLUS_CHAR);
+        }
     }
 
     private boolean canEncode(final byte c) {

src/main/java/org/apache/commons/codec/net/QCodec.java

@@ -101,8 +101,6 @@ public class QCodec extends RFC1522Codec implements StringEncoder, StringDecoder
         PRINTABLE_CHARS.set('}');
         PRINTABLE_CHARS.set('~');
     }
-    private static final byte SPACE = 32;
-
     private static final byte UNDERSCORE = 95;
 
     private boolean encodeBlanks;
@@ -201,7 +199,7 @@ protected byte[] doDecoding(final byte[] bytes) throws DecoderException {
                 if (b != UNDERSCORE) {
                     tmp[i] = b;
                 } else {
-                    tmp[i] = SPACE;
+                    tmp[i] = Utils.SPACE;
                 }
             }
             return QuotedPrintableCodec.decodeQuotedPrintable(tmp);
@@ -217,7 +215,7 @@ protected byte[] doEncoding(final byte[] bytes) {
         final byte[] data = QuotedPrintableCodec.encodeQuotedPrintable(PRINTABLE_CHARS, bytes);
         if (this.encodeBlanks) {
             for (int i = 0; i < data.length; i++) {
-                if (data[i] == SPACE) {
+                if (data[i] == Utils.SPACE) {
                     data[i] = UNDERSCORE;
                 }
             }

src/main/java/org/apache/commons/codec/net/QuotedPrintableCodec.java

@@ -72,7 +72,6 @@ public class QuotedPrintableCodec implements BinaryEncoder, BinaryDecoder, Strin
     private static final BitSet PRINTABLE_CHARS = new BitSet(256);
     private static final byte ESCAPE_CHAR = '=';
     private static final byte TAB = 9;
-    private static final byte SPACE = 32;
     private static final byte CR = 13;
     private static final byte LF = 10;
 
@@ -96,7 +95,7 @@ public class QuotedPrintableCodec implements BinaryEncoder, BinaryDecoder, Strin
             PRINTABLE_CHARS.set(i);
         }
         PRINTABLE_CHARS.set(TAB);
-        PRINTABLE_CHARS.set(SPACE);
+        PRINTABLE_CHARS.set(Utils.SPACE);
     }
 
     /**
@@ -284,7 +283,7 @@ private static int getUnsignedOctet(final int index, final byte[] bytes) {
      * @return {@code true} if the byte is either a space or tab character.
      */
     private static boolean isWhitespace(final int b) {
-        return b == SPACE || b == TAB;
+        return b == Utils.SPACE || b == TAB;
     }
 
     /**