|
| 1 | +/* |
| 2 | + * Licensed to the Apache Software Foundation (ASF) under one or more |
| 3 | + * contributor license agreements. See the NOTICE file distributed with |
| 4 | + * this work for additional information regarding copyright ownership. |
| 5 | + * The ASF licenses this file to You under the Apache License, Version 2.0 |
| 6 | + * (the "License"); you may not use this file except in compliance with |
| 7 | + * the License. You may obtain a copy of the License at |
| 8 | + * |
| 9 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | + * |
| 11 | + * Unless required by applicable law or agreed to in writing, software |
| 12 | + * distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | + * See the License for the specific language governing permissions and |
| 15 | + * limitations under the License. |
| 16 | + */ |
| 17 | + |
| 18 | +package org.apache.commons.codec.language.bm; |
| 19 | + |
| 20 | +import java.io.InputStream; |
| 21 | +import java.util.ArrayList; |
| 22 | +import java.util.Arrays; |
| 23 | +import java.util.Collections; |
| 24 | +import java.util.EnumMap; |
| 25 | +import java.util.HashSet; |
| 26 | +import java.util.List; |
| 27 | +import java.util.Map; |
| 28 | +import java.util.Scanner; |
| 29 | +import java.util.Set; |
| 30 | +import java.util.regex.Pattern; |
| 31 | + |
| 32 | +/** |
| 33 | + * <p> |
| 34 | + * Language guessing utility. |
| 35 | + * </p> |
| 36 | + * <p> |
| 37 | + * This class encapsulates rules used to guess the possible languages that a word originates from. This is done by reference to a whole |
| 38 | + * series of rules distributed in resource files. |
| 39 | + * </p> |
| 40 | + * <p> |
| 41 | + * Instances of this class are typically managed through the static factory method instance(). Unless you are developing your own language |
| 42 | + * guessing rules, you will not need to interact with this class directly. |
| 43 | + * </p> |
| 44 | + * <p> |
| 45 | + * This class is intended to be immutable and thread-safe. |
| 46 | + * </p> |
| 47 | + * <h2>Lang resources</h2 |
| 48 | + * <p> |
| 49 | + * Language guessing rules are typically loaded from resource files. These are UTF-8 encoded text files. They are systematically named |
| 50 | + * following the pattern: <blockquote>org/apache/commons/codec/language/bm/lang.txt</blockquote> The format of these resources is the |
| 51 | + * following: |
| 52 | + * </p> |
| 53 | + * <ul> |
| 54 | + * <li><b>Rules:</b> whitespace separated strings. There should be 3 columns to each row, and these will be interpreted as: |
| 55 | + * <ol> |
| 56 | + * <li>pattern: a regular expression.</li> |
| 57 | + * <li>languages: a '+'-separated list of languages.</li> |
| 58 | + * <li>acceptOnMatch: 'true' or 'false' indicating if a match rules in or rules out the language.</li> |
| 59 | + * </ol> |
| 60 | + * </li> |
| 61 | + * <li><b>End-of-line comments:</b> Any occurance of '//' will cause all text following on that line to be discarded as a comment.</li> |
| 62 | + * <li><b>Multi-line comments:</b> Any line starting with '/*' will start multi-line commenting mode. This will skip all content until a |
| 63 | + * line ending in '*' and '/' is found.</li> |
| 64 | + * <li><b>Blank lines:</b> All blank lines will be skipped.</li> |
| 65 | + * </ul> |
| 66 | + * <p/> |
| 67 | + * Port of lang.php |
| 68 | + * |
| 69 | + * @author Apache Software Foundation |
| 70 | + * @since 2.0 |
| 71 | + */ |
| 72 | +public class Lang { |
| 73 | + |
| 74 | + private static class LangRule { |
| 75 | + private boolean acceptOnMatch; |
| 76 | + private Set<String> languages; |
| 77 | + private Pattern pattern; |
| 78 | + |
| 79 | + private LangRule(Pattern pattern, Set<String> languages, boolean acceptOnMatch) { |
| 80 | + this.pattern = pattern; |
| 81 | + this.languages = languages; |
| 82 | + this.acceptOnMatch = acceptOnMatch; |
| 83 | + } |
| 84 | + |
| 85 | + public boolean matches(String txt) { |
| 86 | + return this.pattern.matcher(txt).find(); |
| 87 | + } |
| 88 | + } |
| 89 | + |
| 90 | + private static final Map<NameType, Lang> Langs = new EnumMap<NameType, Lang>(NameType.class); |
| 91 | + |
| 92 | + private static final String LANGUAGE_RULES_RN = "org/apache/commons/codec/language/bm/lang.txt"; |
| 93 | + |
| 94 | + static { |
| 95 | + for (NameType s : NameType.values()) { |
| 96 | + Langs.put(s, loadFromResource(LANGUAGE_RULES_RN, Languages.instance(s))); |
| 97 | + } |
| 98 | + } |
| 99 | + |
| 100 | + /** |
| 101 | + * Gets a Lang instance for one of the supported NameTypes. |
| 102 | + * |
| 103 | + * @param nameType |
| 104 | + * the NameType to look up |
| 105 | + * @return a Lang encapsulating the language guessing rules for that name type |
| 106 | + */ |
| 107 | + public static Lang instance(NameType nameType) { |
| 108 | + return Langs.get(nameType); |
| 109 | + } |
| 110 | + |
| 111 | + /** |
| 112 | + * <p> |
| 113 | + * Loads language rules from a resource. |
| 114 | + * </p> |
| 115 | + * <p> |
| 116 | + * In normal use, you will obtain instances of Lang through the {@link #instance(NameType)} method. You will only need to call this |
| 117 | + * yourself if you are developing custom language mapping rules. |
| 118 | + * </p> |
| 119 | + * |
| 120 | + * @param languageRulesResourceName |
| 121 | + * the fully-qualified resource name to load |
| 122 | + * @param languages |
| 123 | + * the languages that these rules will support |
| 124 | + * @return a Lang encapsulating the loaded language-guessing rules. |
| 125 | + */ |
| 126 | + public static Lang loadFromResource(String languageRulesResourceName, Languages languages) { |
| 127 | + List<LangRule> rules = new ArrayList<LangRule>(); |
| 128 | + InputStream lRulesIS = Lang.class.getClassLoader().getResourceAsStream(languageRulesResourceName); |
| 129 | + |
| 130 | + if (lRulesIS == null) { |
| 131 | + throw new IllegalStateException("Unable to resolve required resource:" + LANGUAGE_RULES_RN); |
| 132 | + } |
| 133 | + |
| 134 | + Scanner scanner = new Scanner(lRulesIS, ResourceConstants.ENCODING); |
| 135 | + boolean inExtendedComment = false; |
| 136 | + while (scanner.hasNextLine()) { |
| 137 | + String rawLine = scanner.nextLine(); |
| 138 | + String line = rawLine; |
| 139 | + |
| 140 | + if (inExtendedComment) { |
| 141 | + if (line.endsWith(ResourceConstants.EXT_CMT_END)) { |
| 142 | + inExtendedComment = false; |
| 143 | + } else { |
| 144 | + // discard doc comment line |
| 145 | + } |
| 146 | + } else { |
| 147 | + if (line.startsWith(ResourceConstants.EXT_CMT_START)) { |
| 148 | + inExtendedComment = true; |
| 149 | + } else { |
| 150 | + // discard comments |
| 151 | + int cmtI = line.indexOf(ResourceConstants.CMT); |
| 152 | + if (cmtI >= 0) { |
| 153 | + // System.err.println("index of comment: " + cmtI); |
| 154 | + line = line.substring(0, cmtI); |
| 155 | + } |
| 156 | + |
| 157 | + // trim leading-trailing whitespace |
| 158 | + line = line.trim(); |
| 159 | + |
| 160 | + if (line.length() == 0) |
| 161 | + continue; // empty lines can be safely skipped |
| 162 | + |
| 163 | + // split it up |
| 164 | + String[] parts = line.split("\\s+"); |
| 165 | + // System.err.println("part count: " + parts.length); |
| 166 | + |
| 167 | + if (parts.length != 3) { |
| 168 | + // fixme: we really need to log this somewhere |
| 169 | + System.err.println("Warning: malformed line '" + rawLine + "'"); |
| 170 | + continue; |
| 171 | + } |
| 172 | + |
| 173 | + Pattern pattern = Pattern.compile(parts[0]); |
| 174 | + String[] langs = parts[1].split("\\+"); |
| 175 | + boolean accept = parts[2].equals("true"); |
| 176 | + |
| 177 | + rules.add(new LangRule(pattern, new HashSet<String>(Arrays.asList(langs)), accept)); |
| 178 | + } |
| 179 | + } |
| 180 | + } |
| 181 | + |
| 182 | + return new Lang(rules, languages); |
| 183 | + } |
| 184 | + |
| 185 | + private final Languages languages; |
| 186 | + private final List<LangRule> rules; |
| 187 | + |
| 188 | + private Lang(List<LangRule> rules, Languages languages) { |
| 189 | + this.rules = Collections.unmodifiableList(rules); |
| 190 | + this.languages = languages; |
| 191 | + } |
| 192 | + |
| 193 | + /** |
| 194 | + * Guesses the language of a word. |
| 195 | + * |
| 196 | + * @param text |
| 197 | + * the word |
| 198 | + * @return the language that the word originates from or {@link Languages#ANY} if there was no unique match |
| 199 | + */ |
| 200 | + public String guessLanguage(String text) { |
| 201 | + Set<String> ls = guessLanguages(text); |
| 202 | + if (ls.size() == 1) { |
| 203 | + return ls.iterator().next(); |
| 204 | + } else { |
| 205 | + return Languages.ANY; |
| 206 | + } |
| 207 | + } |
| 208 | + |
| 209 | + /** |
| 210 | + * Guesses the languages of a word. |
| 211 | + * |
| 212 | + * @param text |
| 213 | + * the word |
| 214 | + * @return a Set of Strings of language names that are potential matches for the word |
| 215 | + */ |
| 216 | + public Set<String> guessLanguages(String text) { |
| 217 | + text = text.toLowerCase(); // todo: locale? |
| 218 | + // System.out.println("Testing text: '" + text + "'"); |
| 219 | + |
| 220 | + Set<String> langs = new HashSet<String>(this.languages.getLanguages()); |
| 221 | + for (LangRule rule : this.rules) { |
| 222 | + if (rule.matches(text)) { |
| 223 | + // System.out.println("Rule " + rule.pattern + " matches " + text); |
| 224 | + if (rule.acceptOnMatch) { |
| 225 | + // System.out.println("Retaining " + rule.languages); |
| 226 | + langs.retainAll(rule.languages); |
| 227 | + } else { |
| 228 | + // System.out.println("Removing " + rule.languages); |
| 229 | + langs.removeAll(rule.languages); |
| 230 | + } |
| 231 | + // System.out.println("Current languages: " + langs); |
| 232 | + } else { |
| 233 | + // System.out.println("Rule " + rule.pattern + " does not match " + text); |
| 234 | + } |
| 235 | + } |
| 236 | + |
| 237 | + return langs; |
| 238 | + } |
| 239 | +} |
0 commit comments