A higher score indicates a greater distance.
- * *- * The previous implementation of the Levenshtein distance algorithm was - * from http://www.merriampark.com - * /ld.htm + * This returns the default instance that uses a version + * of the algorithm that does not use a threshold parameter. *
* + * @see {@link #getDefaultInstance()} + */ + public LevenshteinDistance() { + this(null); + } + + /** *
- * Chas Emerick has written an implementation in Java, which avoids an
- * OutOfMemoryError which can occur when my Java implementation is used with
- * very large strings.
- * This implementation of the Levenshtein distance algorithm is from http://www.merriampark.com/
- * ldjava.htm
+ * If the threshold is not null, distance calculations will be limited to a maximum length.
+ * If the threshold is null, the unlimited version of the algorithm will be used.
*
Find the Levenshtein distance between two Strings.
+ * + *A higher score indicates a greater distance.
+ * + *The previous implementation of the Levenshtein distance algorithm + * was from http://www.merriampark.com/ld.htm
+ * + *Chas Emerick has written an implementation in Java, which avoids an OutOfMemoryError
+ * which can occur when my Java implementation is used with very large strings.
+ * This implementation of the Levenshtein distance algorithm
+ * is from http://www.merriampark.com/ldjava.htm
* distance.compare(null, *) = IllegalArgumentException
* distance.compare(*, null) = IllegalArgumentException
@@ -68,14 +93,25 @@ public class LevenshteinDistance implements StringMetric {
* distance.compare("hello", "hallo") = 1
*
*
- * @param left the first string, must not be null
- * @param right the second string, must not be null
- * @return result distance
+ * @param s the first String, must not be null
+ * @param t the second String, must not be null
+ * @return result distance, or -1
* @throws IllegalArgumentException if either String input {@code null}
*/
- @Override
- public Integer compare(CharSequence left, CharSequence right) {
- return compare(left, right, Integer.MAX_VALUE);
+ public Integer compare(CharSequence s, CharSequence t) {
+ if (threshold != null) {
+ return limitedCompare(s, t, threshold);
+ } else {
+ return unlimitedCompare(s, t);
+ }
+ }
+
+ public static LevenshteinDistance getDefaultInstance() {
+ return DEFAULT_INSTANCE;
+ }
+
+ public Integer getThreshold() {
+ return threshold;
}
/**
@@ -91,27 +127,25 @@ public Integer compare(CharSequence left, CharSequence right) {
*
*
*
- * distance.compare(null, *, *) = IllegalArgumentException
- * distance.compare(*, null, *) = IllegalArgumentException
- * distance.compare(*, *, -1) = IllegalArgumentException
- * distance.compare("","", 0) = 0
- * distance.compare("aaapppp", "", 8) = 7
- * distance.compare("aaapppp", "", 7) = 7
- * distance.compare("aaapppp", "", 6)) = -1
- * distance.compare("elephant", "hippo", 7) = 7
- * distance.compare("elephant", "hippo", 6) = -1
- * distance.compare("hippo", "elephant", 7) = 7
- * distance.compare("hippo", "elephant", 6) = -1
+ * limitedCompare(null, *, *) = IllegalArgumentException
+ * limitedCompare(*, null, *) = IllegalArgumentException
+ * limitedCompare(*, *, -1) = IllegalArgumentException
+ * limitedCompare("","", 0) = 0
+ * limitedCompare("aaapppp", "", 8) = 7
+ * limitedCompare("aaapppp", "", 7) = 7
+ * limitedCompare("aaapppp", "", 6)) = -1
+ * limitedCompare("elephant", "hippo", 7) = 7
+ * limitedCompare("elephant", "hippo", 6) = -1
+ * limitedCompare("hippo", "elephant", 7) = 7
+ * limitedCompare("hippo", "elephant", 6) = -1
*
*
* @param left the first string, must not be null
* @param right the second string, must not be null
* @param threshold the target threshold, must not be negative
- * @return result distance
- * @throws IllegalArgumentException if either String input {@code null} or
- * negative threshold
+ * @return result distance, or -1
*/
- public Integer compare(CharSequence left, CharSequence right, int threshold) {
+ private static int limitedCompare(CharSequence left, CharSequence right, int threshold) {
if (left == null || right == null) {
throw new IllegalArgumentException("Strings must not be null");
}
@@ -124,7 +158,7 @@ public Integer compare(CharSequence left, CharSequence right, int threshold) {
* equal to the threshold value, returning -1 if it's greater. The
* advantage is performance: unbounded distance is O(nm), but a bound of
* k allows us to reduce it to O(km) time by only computing a diagonal
- * stripe of width 2k + 1 of the cost table. It is also possible to use
+ * stripe of width 2k + 1 of the cost table. It is also possible to use* this to compute the unbounded Levenshtein distance by starting the
* this to compute the unbounded Levenshtein distance by starting the
* threshold at 1 and doubling each time until the distance is found;
* this is O(dm), where d is the distance.
@@ -143,8 +177,16 @@ public Integer compare(CharSequence left, CharSequence right, int threshold) {
* and our threshold is 1. In this case we're going to walk a stripe of
* length 3. The matrix would look like so:
*
- * 1 2 3 4 5 1 |#|#| | | | 2 |#|#|#| | | 3 | |#|#|#| | 4 | | |#|#|#| 5 |
- * | | |#|#| 6 | | | | |#| 7 | | | | | |
+ * + * 1 2 3 4 5 + * 1 |#|#| | | | + * 2 |#|#|#| | | + * 3 | |#|#|#| | + * 4 | | |#|#|#| + * 5 | | | |#|#| + * 6 | | | | |#| + * 7 | | | | | | + ** * Note how the stripe leads off the table as there is no possible way * to turn a string of length 5 into one of length 7 in edit distance of @@ -243,4 +285,113 @@ public Integer compare(CharSequence left, CharSequence right, int threshold) { return -1; } + /** + *
Find the Levenshtein distance between two Strings.
+ * + *A higher score indicates a greater distance.
+ * + *The previous implementation of the Levenshtein distance algorithm + * was from http://www.merriampark.com/ld.htm
+ * + *Chas Emerick has written an implementation in Java, which avoids an OutOfMemoryError
+ * which can occur when my Java implementation is used with very large strings.
+ * This implementation of the Levenshtein distance algorithm
+ * is from http://www.merriampark.com/ldjava.htm
+ * unlimitedCompare(null, *) = IllegalArgumentException
+ * unlimitedCompare(*, null) = IllegalArgumentException
+ * unlimitedCompare("","") = 0
+ * unlimitedCompare("","a") = 1
+ * unlimitedCompare("aaapppp", "") = 7
+ * unlimitedCompare("frog", "fog") = 1
+ * unlimitedCompare("fly", "ant") = 3
+ * unlimitedCompare("elephant", "hippo") = 7
+ * unlimitedCompare("hippo", "elephant") = 7
+ * unlimitedCompare("hippo", "zzzzzzzz") = 8
+ * unlimitedCompare("hello", "hallo") = 1
+ *
+ *
+ * @param s the first String, must not be null
+ * @param t the second String, must not be null
+ * @return result distance, or -1
+ * @throws IllegalArgumentException if either String input {@code null}
+ */
+ private static int unlimitedCompare(CharSequence s, CharSequence t) {
+ if (s == null || t == null) {
+ throw new IllegalArgumentException("Strings must not be null");
+ }
+
+ /*
+ The difference between this impl. and the previous is that, rather
+ than creating and retaining a matrix of size s.length() + 1 by t.length() + 1,
+ we maintain two single-dimensional arrays of length s.length() + 1. The first, d,
+ is the 'current working' distance array that maintains the newest distance cost
+ counts as we iterate through the characters of String s. Each time we increment
+ the index of String t we are comparing, d is copied to p, the second int[]. Doing so
+ allows us to retain the previous cost counts as required by the algorithm (taking
+ the minimum of the cost count to the left, up one, and diagonally up and to the left
+ of the current cost count being calculated). (Note that the arrays aren't really
+ copied anymore, just switched...this is clearly much better than cloning an array
+ or doing a System.arraycopy() each time through the outer loop.)
+
+ Effectively, the difference between the two implementations is this one does not
+ cause an out of memory condition when calculating the LD over two very large strings.
+ */
+
+ int n = s.length(); // length of s
+ int m = t.length(); // length of t
+
+ if (n == 0) {
+ return m;
+ } else if (m == 0) {
+ return n;
+ }
+
+ if (n > m) {
+ // swap the input strings to consume less memory
+ final CharSequence tmp = s;
+ s = t;
+ t = tmp;
+ n = m;
+ m = t.length();
+ }
+
+ int p[] = new int[n + 1]; //'previous' cost array, horizontally
+ int d[] = new int[n + 1]; // cost array, horizontally
+ int _d[]; //placeholder to assist in swapping p and d
+
+ // indexes into strings s and t
+ int i; // iterates through s
+ int j; // iterates through t
+
+ char t_j; // jth character of t
+
+ int cost; // cost
+
+ for (i = 0; i <= n; i++) {
+ p[i] = i;
+ }
+
+ for (j = 1; j <= m; j++) {
+ t_j = t.charAt(j - 1);
+ d[0] = j;
+
+ for (i = 1; i <= n; i++) {
+ cost = s.charAt(i - 1) == t_j ? 0 : 1;
+ // minimum of cell to the left+1, to the top+1, diagonally left and up +cost
+ d[i] = Math.min(Math.min(d[i - 1] + 1, p[i] + 1), p[i - 1] + cost);
+ }
+
+ // copy current distance counts to 'previous row' distance counts
+ _d = p;
+ p = d;
+ d = _d;
+ }
+
+ // our last action in the above loop was to switch d and p, so p now
+ // actually has the most recent cost counts
+ return p[n];
+ }
+
}
diff --git a/src/test/java/org/apache/commons/text/similarity/LevenshteinDistanceTest.java b/src/test/java/org/apache/commons/text/similarity/LevenshteinDistanceTest.java
index 1d4e6fa2ce..ea5bd348e6 100644
--- a/src/test/java/org/apache/commons/text/similarity/LevenshteinDistanceTest.java
+++ b/src/test/java/org/apache/commons/text/similarity/LevenshteinDistanceTest.java
@@ -18,6 +18,9 @@
import static org.junit.Assert.assertEquals;
+import java.util.Map;
+import java.util.TreeMap;
+
import org.junit.BeforeClass;
import org.junit.Test;
@@ -26,15 +29,10 @@
*/
public class LevenshteinDistanceTest {
- private static LevenshteinDistance distance;
-
- @BeforeClass
- public static void setUp() {
- distance = new LevenshteinDistance();
- }
-
@Test
public void testGetLevenshteinDistance_StringString() {
+ LevenshteinDistance distance = new LevenshteinDistance();
+
assertEquals(0, (int) distance.compare("", ""));
assertEquals(1, (int) distance.compare("", "a"));
assertEquals(7, (int) distance.compare("aaapppp", ""));
@@ -49,98 +47,112 @@ public void testGetLevenshteinDistance_StringString() {
@Test(expected = IllegalArgumentException.class)
public void testGetLevenshteinDistance_NullString() throws Exception {
+ LevenshteinDistance distance = new LevenshteinDistance();
+
distance.compare("a", null);
}
@Test(expected = IllegalArgumentException.class)
public void testGetLevenshteinDistance_StringNull() throws Exception {
+ LevenshteinDistance distance = new LevenshteinDistance();
+
distance.compare(null, "a");
}
@Test
public void testGetLevenshteinDistance_StringStringInt() {
+ Map+ * This returns an instance that call {@link Locale#getLocale()} + * to get a {@link Locale} for each + * call to {@link #compare(CharSequence, CharSequence)} + *
+ * + * @see {@link #getDefaultInstance()} + */ + public FuzzyScore() { + this(null); + } + + public FuzzyScore(final Locale locale) { + this.locale = locale; + } + /** ** Find the Fuzzy Score which indicates the similarity score between two - * Strings. This method uses the default locale. + * Strings. + *
+ *+ * If this instance's {@link Locale} is null, the current default will be + * determined using {@link Locale#getDefault()}. *
* * @param term a full term that should be matched against, must not be null @@ -47,7 +72,8 @@ public class FuzzyScore implements StringMetric- * This returns an instance that call {@link Locale#getLocale()} - * to get a {@link Locale} for each - * call to {@link #compare(CharSequence, CharSequence)} - *
- * - * @see {@link #getDefaultInstance()} - */ - public FuzzyScore() { - this(null); - } - - public FuzzyScore(final Locale locale) { - this.locale = locale; - } /** - *- * Find the Fuzzy Score which indicates the similarity score between two - * Strings. - *
- *- * If this instance's {@link Locale} is null, the current default will be - * determined using {@link Locale#getDefault()}. - *
+ *This returns a {@link Locale}-specific {@link FuzzyScore}.
* - * @param term a full term that should be matched against, must not be null - * @param query the query that will be matched against a term, must not be - * null - * @return result score - * @throws IllegalArgumentException if either String input {@code null} + * @param locale The string matching logic is case insensitive. + A {@link Locale} is necessary to normalize both Strings to lower case. + * @throws IllegalArgumentException + * This is thrown if the {@link Locale} parameter is {@code null}. */ - @Override - public Integer compare(CharSequence term, CharSequence query) { - Locale locale = this.locale != null ? this.locale : Locale.getDefault(); - return compare(term, query, locale); + public FuzzyScore(final Locale locale) { + if (locale == null) { + throw new IllegalArgumentException("Locale must not be null"); + } + this.locale = locale; } /** @@ -96,17 +71,14 @@ public Integer compare(CharSequence term, CharSequence query) { * @param term a full term that should be matched against, must not be null * @param query the query that will be matched against a term, must not be * null - * @param locale This string matching logic is case insensitive. A locale is - * necessary to normalize both Strings to lower case. * @return result score * @throws IllegalArgumentException if either String input {@code null} or * Locale input {@code null} */ - private static int compare(CharSequence term, CharSequence query, Locale locale) { + @Override + public Integer compare(CharSequence term, CharSequence query) { if (term == null || query == null) { throw new IllegalArgumentException("Strings must not be null"); - } else if (locale == null) { - throw new IllegalArgumentException("Locale must not be null"); } // fuzzy logic is case insensitive. We normalize the Strings to lower @@ -156,10 +128,6 @@ private static int compare(CharSequence term, CharSequence query, Locale locale) return score; } - public static FuzzyScore getDefaultInstance() { - return DEFAULT_INSTANCE; - } - public Locale getLocale() { return locale; } diff --git a/src/test/java/org/apache/commons/text/similarity/FuzzyScoreTest.java b/src/test/java/org/apache/commons/text/similarity/FuzzyScoreTest.java index b54215df8d..5d5d995bde 100644 --- a/src/test/java/org/apache/commons/text/similarity/FuzzyScoreTest.java +++ b/src/test/java/org/apache/commons/text/similarity/FuzzyScoreTest.java @@ -67,4 +67,9 @@ public void testGetFuzzyScore_NullNullLocale() throws Exception { score.compare(null, null); } + @Test(expected = IllegalArgumentException.class) + public void testMissingLocale() throws Exception { + FuzzyScore score = new FuzzyScore((Locale) null); + } + } From 10cfb00ca8a8d21cc3109ca90fbdbe21ca1c014f Mon Sep 17 00:00:00 2001 From: j--baker