[CODEC-187] Apply patch to make BeiderMorse phonetic engine compatible with v3.3 of the reference implementation.

git-svn-id: https://svn.apache.org/repos/asf/commons/proper/codec/trunk@1608115 13f79535-47bb-0310-9956-ffa450edef68

Author: netomi

src/main/java/org/apache/commons/codec/language/bm/BeiderMorseEncoder.java

@@ -64,9 +64,12 @@
  * encodings as they consider a wider range of possible, approximate phonetic interpretations of the original word.
  * Down-stream applications may wish to further process the encoding for indexing or lookup purposes, for example, by
  * splitting on pipe (<code>|</code>) and indexing under each of these alternatives.
+ * <p>
+ * <b>Note</b>: this version of the Beider-Morse encoding is equivalent with v3.3 of the reference implementation.
  *
  * @see <a href="http://stevemorse.org/phonetics/bmpm.htm">Beider-Morse Phonetic Matching</a>
  * @see <a href="http://stevemorse.org/phoneticinfo.htm">Reference implementation</a>
+ *
  * @since 1.6
  * @version $Id$
  */

src/main/java/org/apache/commons/codec/language/bm/Lang.java

@@ -95,11 +95,11 @@ public boolean matches(final String txt) {
 
     private static final Map<NameType, Lang> Langs = new EnumMap<NameType, Lang>(NameType.class);
 
-    private static final String LANGUAGE_RULES_RN = "org/apache/commons/codec/language/bm/lang.txt";
+    private static final String LANGUAGE_RULES_RN = "org/apache/commons/codec/language/bm/%s_lang.txt";
 
     static {
         for (final NameType s : NameType.values()) {
-            Langs.put(s, loadFromResource(LANGUAGE_RULES_RN, Languages.getInstance(s)));
+            Langs.put(s, loadFromResource(String.format(LANGUAGE_RULES_RN, s.getName()), Languages.getInstance(s)));
         }
     }
 

src/main/java/org/apache/commons/codec/language/bm/Languages.java

@@ -73,6 +73,8 @@ public static LanguageSet from(final Set<String> langs) {
         public abstract boolean isSingleton();
 
         public abstract LanguageSet restrictTo(LanguageSet other);
+
+        public abstract LanguageSet merge(LanguageSet other);
     }
 
     /**
@@ -127,6 +129,22 @@ public LanguageSet restrictTo(final LanguageSet other) {
             }
         }
 
+        @Override
+        public LanguageSet merge(final LanguageSet other) {
+            if (other == NO_LANGUAGES) {
+                return this;
+            } else if (other == ANY_LANGUAGE) {
+                return other;
+            } else {
+                final SomeLanguages sl = (SomeLanguages) other;
+                final Set<String> ls = new HashSet<String>(languages);
+                for (String lang : sl.languages) {
+                  ls.add(lang);
+                }
+                return from(ls);
+            }
+        }
+
         @Override
         public String toString() {
             return "Languages(" + languages.toString() + ")";
@@ -216,6 +234,11 @@ public LanguageSet restrictTo(final LanguageSet other) {
             return this;
         }
 
+        @Override
+        public LanguageSet merge(final LanguageSet other) {
+            return other;
+        }
+
         @Override
         public String toString() {
             return "NO_LANGUAGES";
@@ -251,6 +274,11 @@ public LanguageSet restrictTo(final LanguageSet other) {
             return other;
         }
 
+        @Override
+        public LanguageSet merge(final LanguageSet other) {
+            return other;
+        }
+
         @Override
         public String toString() {
             return "ANY_LANGUAGE";

src/main/java/org/apache/commons/codec/language/bm/PhoneticEngine.java

@@ -28,7 +28,7 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
-import java.util.TreeSet;
+import java.util.TreeMap;
 
 import org.apache.commons.codec.language.bm.Languages.LanguageSet;
 import org.apache.commons.codec.language.bm.Rule.Phoneme;
@@ -335,7 +335,8 @@ private PhonemeBuilder applyFinalRules(final PhonemeBuilder phonemeBuilder,
             return phonemeBuilder;
         }
 
-        final Set<Rule.Phoneme> phonemes = new TreeSet<Rule.Phoneme>(Rule.Phoneme.COMPARATOR);
+        final Map<Rule.Phoneme, Rule.Phoneme> phonemes =
+            new TreeMap<Rule.Phoneme, Rule.Phoneme>(Rule.Phoneme.COMPARATOR);
 
         for (final Rule.Phoneme phoneme : phonemeBuilder.getPhonemes()) {
             PhonemeBuilder subBuilder = PhonemeBuilder.empty(phoneme.getLanguages());
@@ -355,10 +356,21 @@ private PhonemeBuilder applyFinalRules(final PhonemeBuilder phonemeBuilder,
                 i = rulesApplication.getI();
             }
 
-            phonemes.addAll(subBuilder.getPhonemes());
+            // the phonemes map orders the phonemes only based on their text, but ignores the language set
+            // when adding new phonemes, check for equal phonemes and merge their language set, otherwise
+            // phonemes with the same text but different language set get lost
+            for (final Rule.Phoneme newPhoneme : subBuilder.getPhonemes()) {
+                if (phonemes.containsKey(newPhoneme)) {
+                    final Rule.Phoneme oldPhoneme = phonemes.remove(newPhoneme);
+                    final Rule.Phoneme mergedPhoneme = oldPhoneme.mergeWithLanguage(newPhoneme.getLanguages());
+                    phonemes.put(mergedPhoneme, mergedPhoneme);
+                } else {
+                    phonemes.put(newPhoneme, newPhoneme);
+                }
+            }
         }
 
-        return new PhonemeBuilder(phonemes);
+        return new PhonemeBuilder(phonemes.keySet());
     }
 
     /**

src/main/java/org/apache/commons/codec/language/bm/Rule.java

@@ -147,6 +147,22 @@ public Phoneme join(final Phoneme right) {
             return new Phoneme(this.phonemeText.toString() + right.phonemeText.toString(),
                                this.languages.restrictTo(right.languages));
         }
+
+        /**
+         * Returns a new Phoneme with the same text but a union of its
+         * current language set and the given one.
+         *
+         * @param lang the language set to merge
+         * @return a new Phoneme
+         */
+        public Phoneme mergeWithLanguage(final LanguageSet lang) {
+          return new Phoneme(this.phonemeText.toString(), this.languages.merge(lang));
+        }
+
+        @Override
+        public String toString() {
+          return phonemeText.toString() + "[" + languages + "]";
+        }
     }
 
     public interface PhonemeExpr {
@@ -442,6 +458,9 @@ public String toString() {
                                         sb.append("Rule");
                                         sb.append("{line=").append(myLine);
                                         sb.append(", loc='").append(loc).append('\'');
+                                        sb.append(", pat='").append(pat).append('\'');
+                                        sb.append(", lcon='").append(lCon).append('\'');
+                                        sb.append(", rcon='").append(rCon).append('\'');
                                         sb.append('}');
                                         return sb.toString();
                                     }

src/main/resources/org/apache/commons/codec/language/bm/ash_exact_russian.txt

@@ -16,4 +16,4 @@
  */
 
 "E" "" "" "e"
-"I "" "" "i"
+"I" "" "" "i"

src/main/resources/org/apache/commons/codec/language/bm/ash_hebrew_common.txt

@@ -24,6 +24,7 @@
 "b" "^" "" "b"    
 "b" "" "" "(b|v)"    
 
+"J" "" "" "l" 
 "ja" "" "" "i"
 "jA" "" "" "i"
 "jB" "" "" "i"
@@ -75,17 +76,20 @@
 "ou" "^" "" "(u|v|1)"
 "o" "^" "" "(u|v|1)"
 "O" "^" "" "(u|v|1)"
+"P" "^" "" "(u|v|1)" 
 "U" "^" "" "(u|v|1)"
 "u" "^" "" "(u|v|1)"
 
 "o" "" "$" "(u|1)"
 "O" "" "$" "(u|1)"
+"P" "" "$" "(u|1)" 
 "u" "" "$" "(u|1)"
 "U" "" "$" "(u|1)"
 
 "ou" "" "" "u"
 "o" "" "" "u"
 "O" "" "" "u"
+"P" "" "" "u" 
 "U" "" "" "u"
 
 "VV" "" "" "u" // alef/ayin + vov from ruleshebrew
@@ -102,8 +106,9 @@
     //"z" "" "" "(z|Z)"
     //"d" "" "" "(d|dZ)"
 
-"TB" "" "$" "(t|s)" // tav from ruleshebrew; only Ashkenazic
-"TB" "" "" "t" // tav from ruleshebrew; only Ashkenazic
+"TB" "^" "" "t" // tav from ruleshebrew; only Ashkenazic
+"TB" "" "$" "s" // tav from ruleshebrew; only Ashkenazic
+"TB" "" "" "(t|s)" // tav from ruleshebrew; only Ashkenazic
 "T" "" "" "t"   // tet from ruleshebrew
 
    //"k" "" "" "(k|x)"