org.apache.commons.codec.language.DaitchMokotoffSoundex.cleanup(String)

does not remove special characters like punctuation

Author: garydgregory

src/changes/changes.xml

@@ -57,6 +57,7 @@ The <action> type attribute can be add,update,fix,remove.
       <action type="fix" dev="ggregory" due-to="Sebastian Baunsgaard">Javadoc typo in Base16.java #380.</action>
       <action type="fix" dev="ggregory" due-to="Gary Gregory">Deprecate unused constant org.apache.commons.codec.language.bm.Rule.ALL.</action>
       <action type="fix" dev="ggregory" issue="CODEC-331" due-to="IlikeCode, Gary Gregory">org.apache.commons.codec.language.bm.Rule.parsePhonemeExpr(String) adds duplicate empty phoneme when input ends with |.</action>
+      <action type="fix" dev="ggregory" issue="CODEC-331" due-to="IlikeCode, Gary Gregory">org.apache.commons.codec.language.DaitchMokotoffSoundex.cleanup(String) does not remove special characters like punctuation.</action>
       <!-- ADD -->
       <action type="add" dev="ggregory" due-to="Gary Gregory">Add HmacUtils.hmac(Path).</action>      
       <action type="add" dev="ggregory" due-to="Gary Gregory">Add HmacUtils.hmacHex(Path).</action>      

src/main/java/org/apache/commons/codec/language/DaitchMokotoffSoundex.java

@@ -347,7 +347,7 @@ public DaitchMokotoffSoundex(final boolean folding) {
     private String cleanup(final String input) {
         final StringBuilder sb = new StringBuilder();
         for (char ch : input.toCharArray()) {
-            if (Character.isWhitespace(ch)) {
+            if (Character.isWhitespace(ch) | !Character.isLetter(ch)) {
                 continue;
             }
             ch = Character.toLowerCase(ch);

src/test/java/org/apache/commons/codec/AbstractStringEncoderTest.java

@@ -29,59 +29,56 @@
  */
 public abstract class AbstractStringEncoderTest<T extends StringEncoder> {
 
-    protected T stringEncoder = this.createStringEncoder();
+    protected T stringEncoder = createStringEncoder();
 
     public void checkEncoding(final String expected, final String source) throws EncoderException {
-        assertEquals(expected, this.getStringEncoder().encode(source), "Source: " + source);
+        assertEquals(expected, getStringEncoder().encode(source), "Source: " + source);
     }
 
     protected void checkEncodings(final String[][] data) throws EncoderException {
         for (final String[] element : data) {
-            this.checkEncoding(element[1], element[0]);
+            checkEncoding(element[1], element[0]);
         }
     }
 
-    protected void checkEncodingVariations(final String expected, final String[] data) throws EncoderException {
+    protected void checkEncodingVariations(final String expected, final String... data) throws EncoderException {
         for (final String element : data) {
-            this.checkEncoding(expected, element);
+            checkEncoding(expected, element);
         }
     }
 
     protected abstract T createStringEncoder();
 
     public T getStringEncoder() {
-        return this.stringEncoder;
+        return stringEncoder;
     }
 
     @Test
     void testEncodeEmpty() throws Exception {
-        final Encoder encoder = this.getStringEncoder();
+        final Encoder encoder = getStringEncoder();
         encoder.encode("");
         encoder.encode(" ");
         encoder.encode("\t");
     }
 
     @Test
     void testEncodeNull() throws EncoderException {
-        final StringEncoder encoder = this.getStringEncoder();
+        final StringEncoder encoder = getStringEncoder();
         encoder.encode(null);
     }
 
     @Test
     void testEncodeWithInvalidObject() throws Exception {
-        final StringEncoder encoder = this.getStringEncoder();
-        assertThrows(EncoderException.class, () -> encoder.encode(Float.valueOf(3.4f)),
-                "An exception was not thrown when we tried to encode a Float object");
+        final StringEncoder encoder = getStringEncoder();
+        assertThrows(EncoderException.class, () -> encoder.encode(Float.valueOf(3.4f)), "An exception was not thrown when we tried to encode a Float object");
     }
+
     @Test
     void testLocaleIndependence() throws Exception {
-        final StringEncoder encoder = this.getStringEncoder();
-
+        final StringEncoder encoder = getStringEncoder();
         final String[] data = { "I", "i" };
-
         final Locale orig = Locale.getDefault();
         final Locale[] locales = { Locale.ENGLISH, new Locale("tr"), Locale.getDefault() };
-
         try {
             for (final String element : data) {
                 String ref = null;
@@ -104,5 +101,4 @@ void testLocaleIndependence() throws Exception {
             Locale.setDefault(orig);
         }
     }
-
 }

src/test/java/org/apache/commons/codec/language/DaitchMokotoffSoundexTest.java

@@ -14,13 +14,18 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.commons.codec.language;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
+import java.util.stream.IntStream;
+
 import org.apache.commons.codec.AbstractStringEncoderTest;
 import org.apache.commons.codec.EncoderException;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
 
 /**
  * Tests {@link DaitchMokotoffSoundex}.
@@ -47,7 +52,6 @@ private String soundex(final String source) {
     void testAccentedCharacterFolding() {
         assertEquals("294795", soundex("Straßburg"));
         assertEquals("294795", soundex("Strasburg"));
-
         assertEquals("095600", soundex("Éregon"));
         assertEquals("095600", soundex("Eregon"));
     }
@@ -59,7 +63,6 @@ void testAdjacentCodes() {
         // 0-54-4---8 -> wrong
         // 0-54-----8 -> correct
         assertEquals("054800", soundex("AKSSOL"));
-
         // GERSCHFELD
         // G-E-RS-CH-F-E-L-D
         // 5--4/94-5/4-7-8-3 -> wrong
@@ -82,18 +85,18 @@ void testEncodeBasic() {
 
     @Test
     void testEncodeIgnoreApostrophes() throws EncoderException {
-        checkEncodingVariations("079600", new String[] { "OBrien", "'OBrien", "O'Brien", "OB'rien", "OBr'ien",
-                "OBri'en", "OBrie'n", "OBrien'" });
+        checkEncodingVariations("079600", "OBrien", "'OBrien", "O'Brien", "OB'rien", "OBr'ien", "OBri'en", "OBrie'n", "OBrien'");
     }
 
     /**
      * Test data from http://www.myatt.demon.co.uk/sxalg.htm
      *
-     * @throws EncoderException for some failure scenarios     */
+     * @throws EncoderException for some failure scenarios
+     */
     @Test
     void testEncodeIgnoreHyphens() throws EncoderException {
-        checkEncodingVariations("565463", new String[] { "KINGSMITH", "-KINGSMITH", "K-INGSMITH", "KI-NGSMITH",
-                "KIN-GSMITH", "KING-SMITH", "KINGS-MITH", "KINGSM-ITH", "KINGSMI-TH", "KINGSMIT-H", "KINGSMITH-" });
+        checkEncodingVariations("565463", "KINGSMITH", "-KINGSMITH", "K-INGSMITH", "KI-NGSMITH", "KIN-GSMITH", "KING-SMITH", "KINGS-MITH", "KINGSM-ITH",
+                "KINGSMI-TH", "KINGSMIT-H", "KINGSMITH-");
     }
 
     @Test
@@ -102,6 +105,18 @@ void testEncodeIgnoreTrimmable() {
         assertEquals("746536", encode("Washington"));
     }
 
+    static IntStream getNonLetters() {
+        return IntStream.rangeClosed(Character.MIN_VALUE, Character.MAX_VALUE).filter(c -> !Character.isLetter(c));
+    }
+
+    @ParameterizedTest
+    @MethodSource("getNonLetters")
+    void testEncodeIgnoreNonLetters(final int nonLetterInt) throws EncoderException {
+        final char nonLetterChar = (char) nonLetterInt;
+        checkEncodingVariations("746536", "Washington" + nonLetterChar, nonLetterChar + "Washington", nonLetterChar + "Washington" + nonLetterChar,
+                "Washi" + nonLetterChar + "ngton");
+    }
+
     /**
      * Examples from http://www.jewishgen.org/infofiles/soundex.html
      */
@@ -116,7 +131,6 @@ void testSoundexBasic() {
         assertEquals("370000", soundex("Topf"));
         assertEquals("586660", soundex("Kleinmann"));
         assertEquals("769600", soundex("Ben Aron"));
-
         assertEquals("097400|097500", soundex("AUERBACH"));
         assertEquals("097400|097500", soundex("OHRBACH"));
         assertEquals("874400", soundex("LIPSHITZ"));
@@ -166,5 +180,4 @@ void testSpecialRomanianCharacters() {
         assertEquals("364000|464000", soundex("ţamas")); // t-cedilla
         assertEquals("364000|464000", soundex("țamas")); // t-comma
     }
-
 }