Add a variant of the Levenshtein string-distance function that lets the user

specify the cost values to use, instead of always using 1's.
Volkan Yazici

In passing, remove fuzzystrmatch.h, which contained a bunch of stuff that had
no business being in a .h file; fold it into its only user, fuzzystrmatch.c.

contrib/fuzzystrmatch/fuzzystrmatch.c

@@ -5,7 +5,7 @@
  *
  * Joe Conway <mail@joeconway.com>
  *
- * $PostgreSQL: pgsql/contrib/fuzzystrmatch/fuzzystrmatch.c,v 1.26 2008/03/25 22:42:41 tgl Exp $
+ * $PostgreSQL: pgsql/contrib/fuzzystrmatch/fuzzystrmatch.c,v 1.27 2008/04/03 21:13:07 tgl Exp $
  * Copyright (c) 2001-2008, PostgreSQL Global Development Group
  * ALL RIGHTS RESERVED;
  *
@@ -15,6 +15,8 @@
  * found at http://www.merriampark.com/ld.htm
  * Also looked at levenshtein.c in the PHP 4.0.6 distribution for
  * inspiration.
+ * Configurable penalty costs extension is introduced by Volkan
+ * YAZICI <volkan.yazici@gmail.com>.
  *
  * metaphone()
  * -----------
@@ -43,153 +45,251 @@
  *
  */
 
-#include "fuzzystrmatch.h"
+#include "postgres.h"
+
+#include <ctype.h>
+
+#include "fmgr.h"
+#include "utils/builtins.h"
 
 PG_MODULE_MAGIC;
 
+
 /*
- * Calculates Levenshtein Distance between two strings.
+ * External declarations for exported functions
- * Uses simplest and fastest cost model only, i.e. assumes a cost of 1 for
- * each deletion, substitution, or insertion.
  */
-PG_FUNCTION_INFO_V1(levenshtein);
+extern Datum levenshtein_with_costs(PG_FUNCTION_ARGS);
-Datum
+extern Datum levenshtein(PG_FUNCTION_ARGS);
-levenshtein(PG_FUNCTION_ARGS)
+extern Datum metaphone(PG_FUNCTION_ARGS);
+extern Datum soundex(PG_FUNCTION_ARGS);
+extern Datum difference(PG_FUNCTION_ARGS);
+
+/*
+ * Soundex
+ */
+static void _soundex(const char *instr, char *outstr);
+
+#define SOUNDEX_LEN 4
+
+/*									ABCDEFGHIJKLMNOPQRSTUVWXYZ */
+static const char *soundex_table = "01230120022455012623010202";
+
+#define soundex_code(letter) soundex_table[toupper((unsigned char) (letter)) - 'A']
+
+
+/*
+ * Levenshtein
+ */
+#define MAX_LEVENSHTEIN_STRLEN		255
+
+static int	levenshtein_internal(const char *s, const char *t,
+								 int ins_c, int del_c, int sub_c);
+
+
+/*
+ * Metaphone
+ */
+#define MAX_METAPHONE_STRLEN		255
+
+/*
+ * Original code by Michael G Schwern starts here.
+ * Code slightly modified for use as PostgreSQL function.
+ */
+
+
+/**************************************************************************
+	metaphone -- Breaks english phrases down into their phonemes.
+
+	Input
+		word			--	An english word to be phonized
+		max_phonemes	--	How many phonemes to calculate.  If 0, then it
+							will phonize the entire phrase.
+		phoned_word		--	The final phonized word.  (We'll allocate the
+							memory.)
+	Output
+		error	--	A simple error flag, returns TRUE or FALSE
+
+	NOTES:	ALL non-alpha characters are ignored, this includes whitespace,
+	although non-alpha characters will break up phonemes.
+****************************************************************************/
+
+
+/**************************************************************************
+	my constants -- constants I like
+
+	Probably redundant.
+
+***************************************************************************/
+
+#define META_ERROR			FALSE
+#define META_SUCCESS		TRUE
+#define META_FAILURE		FALSE
+
+
+/*	I add modifications to the traditional metaphone algorithm that you
+	might find in books.  Define this if you want metaphone to behave
+	traditionally */
+#undef USE_TRADITIONAL_METAPHONE
+
+/* Special encodings */
+#define  SH		'X'
+#define  TH		'0'
+
+static char Lookahead(char *word, int how_far);
+static int	_metaphone(char *word, int max_phonemes, char **phoned_word);
+
+/* Metachar.h ... little bits about characters for metaphone */
+
+
+/*-- Character encoding array & accessing macros --*/
+/* Stolen directly out of the book... */
+char		_codes[26] = {
+	1, 16, 4, 16, 9, 2, 4, 16, 9, 2, 0, 2, 2, 2, 1, 4, 0, 2, 4, 4, 1, 0, 0, 0, 8, 0
+/*	a  b c	d e f g  h i j k l m n o p q r s t u v w x y z */
+};
+
+
+#define ENCODE(c) (isalpha((unsigned char) (c)) ? _codes[((toupper((unsigned char) (c))) - 'A')] : 0)
+
+#define isvowel(c)	(ENCODE(c) & 1)		/* AEIOU */
+
+/* These letters are passed through unchanged */
+#define NOCHANGE(c) (ENCODE(c) & 2)		/* FJMNR */
+
+/* These form dipthongs when preceding H */
+#define AFFECTH(c)	(ENCODE(c) & 4)		/* CGPST */
+
+/* These make C and G soft */
+#define MAKESOFT(c) (ENCODE(c) & 8)		/* EIY */
+
+/* These prevent GH from becoming F */
+#define NOGHTOF(c)	(ENCODE(c) & 16)	/* BDH */
+
+
+/*
+ * levenshtein_internal - Calculates Levenshtein distance metric
+ *                        between supplied strings. Generally
+ *                        (1, 1, 1) penalty costs suffices common
+ *                        cases, but your mileage may vary.
+ */
+static int
+levenshtein_internal(const char *s, const char *t,
+					 int ins_c, int del_c, int sub_c)
 {
-	char	   *str_s = TextDatumGetCString(PG_GETARG_DATUM(0));
+	int		 m, n;
-	char	   *str_t = TextDatumGetCString(PG_GETARG_DATUM(1));
+	int		*prev;
-	int			cols = strlen(str_s) + 1;
+	int		*curr;
-	int			rows = strlen(str_t) + 1;
+	int		 i, j;
-	char	   *str_s0;
+	const char *x;
-	int		   *u_cells;
+	const char *y;
-	int		   *l_cells;
+
-	int		   *tmp;
+	m = strlen(s);
-	int			i;
+	n = strlen(t);
-	int			j;
 
 	/*
-	 * str_s is referred to as the "source", str_t is referred to as the
+	 * If either m or n is 0, the answer is the other value. This makes
-	 * "target", cols = length of source + 1 to allow for the initialization
+	 * sense since it would take that many insertions to build a matching
-	 * column, rows = length of target + 1 to allow for the initialization row
+	 * string
 	 */
+	if (!m)
+		return n;
+	if (!n)
+		return m;
 
 	/*
-	 * Restrict the length of the strings being compared to something
+	 * For security concerns, restrict excessive CPU+RAM usage. (This
-	 * reasonable because we will have to perform rows * cols calculations. If
+	 * implementation uses O(m) memory and has O(mn) complexity.)
-	 * longer strings need to be compared, increase MAX_LEVENSHTEIN_STRLEN to
-	 * suit (but within your tolerance for speed and memory usage).
 	 */
-	if ((cols > MAX_LEVENSHTEIN_STRLEN + 1) || (rows > MAX_LEVENSHTEIN_STRLEN + 1))
+	if (m > MAX_LEVENSHTEIN_STRLEN ||
+		n > MAX_LEVENSHTEIN_STRLEN)
 		ereport(ERROR,
 				(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
 				 errmsg("argument exceeds the maximum length of %d bytes",
 						MAX_LEVENSHTEIN_STRLEN)));
 
-	/*
+	/* One more cell for initialization column and row. */
-	 * If either rows or cols is 0, the answer is the other value. This makes
+	++m;
-	 * sense since it would take that many insertions the build a matching
+	++n;
-	 * string
-	 */
-
-	if (cols == 0)
-		PG_RETURN_INT32(rows);
-
-	if (rows == 0)
-		PG_RETURN_INT32(cols);
 
 	/*
-	 * Allocate two vectors of integers. One will be used for the "upper" row,
+	 * Instead of building an (m+1)x(n+1) array, we'll use two
-	 * the other for the "lower" row. Initialize the "upper" row to 0..cols
+	 * different arrays of size m+1 for storing accumulated values.
+	 * At each step one represents the "previous" row and one is the
+	 * "current" row of the notional large array.
 	 */
-	u_cells = palloc(sizeof(int) * cols);
+	prev = (int *) palloc(2 * m * sizeof(int));
-	for (i = 0; i < cols; i++)
+	curr = prev + m;
-		u_cells[i] = i;
 
-	l_cells = palloc(sizeof(int) * cols);
+	/* Initialize the "previous" row to 0..cols */
+	for (i = 0; i < m; i++)
+		prev[i] = i;
 
-	/*
+	/* Loop through rows of the notional array */
-	 * Use str_s0 to "rewind" the pointer to str_s in the nested for loop
+	for (y = t, j = 1; j < n; y++, j++)
-	 * below
-	 */
-	str_s0 = str_s;
-
-	/*
-	 * Loop through the rows, starting at row 1. Row 0 is used for the initial
-	 * "upper" row.
-	 */
-	for (j = 1; j < rows; j++)
 	{
-		/*
+		int *temp;
-		 * We'll always start with col 1, and initialize lower row col 0 to j
-		 */
-		l_cells[0] = j;
 
-		for (i = 1; i < cols; i++)
+		/*
+		 * First cell must increment sequentially, as we're on the
+		 * j'th row of the (m+1)x(n+1) array.
+		 */
+		curr[0] = j;
+		
+		for (x = s, i = 1; i < m; x++, i++)
 		{
-			int			c = 0;
+			int	ins;
-			int			c1 = 0;
+			int	del;
-			int			c2 = 0;
+			int	sub;
-			int			c3 = 0;
 
-			/*
+			/* Calculate costs for probable operations. */
-			 * The "cost" value is 0 if the character at the current col
+			ins = prev[i] + ins_c;						/* Insertion    */
-			 * position in the source string, matches the character at the
+			del = curr[i-1] + del_c;					/* Deletion     */
-			 * current row position in the target string; cost is 1 otherwise.
+			sub = prev[i-1] + ((*x == *y) ? 0 : sub_c);	/* Substitution */
-			 */
-			c = (*str_s != *str_t);
 
-			/*
+			/* Take the one with minimum cost. */
-			 * c1 is upper right cell plus 1
+			curr[i] = Min(ins, del);
-			 */
+			curr[i] = Min(curr[i], sub);
-			c1 = u_cells[i] + 1;
+		}
 
-			/*
+		/* Swap current row with previous row. */
-			 * c2 is lower left cell plus 1
+		temp = curr;
-			 */
+		curr = prev;
-			c2 = l_cells[i - 1] + 1;
+		prev = temp;
-
-			/*
-			 * c3 is cell diagonally above to the left plus "cost"
-			 */
-			c3 = u_cells[i - 1] + c;
-
-			/*
-			 * The lower right cell is set to the minimum of c1, c2, c3
-			 */
-			l_cells[i] = (c1 < c2 ? c1 : c2) < c3 ? (c1 < c2 ? c1 : c2) : c3;
-
-			/*
-			 * Increment the pointer to str_s
-			 */
-			str_s++;
 	}
 
 	/*
-		 * Lower row now becomes the upper row, and the upper row gets reused
+	 * Because the final value was swapped from the previous row to
-		 * as the new lower row.
+	 * the current row, that's where we'll find it.
 	 */
-		tmp = u_cells;
+	return prev[m-1];
-		u_cells = l_cells;
-		l_cells = tmp;
-
-		/*
-		 * Increment the pointer to str_t
-		 */
-		str_t++;
-
-		/*
-		 * Rewind the pointer to str_s
-		 */
-		str_s = str_s0;
-	}
-
-	/*
-	 * Because the final value (at position row, col) was swapped from the
-	 * lower row to the upper row, that's where we'll find it.
-	 */
-	PG_RETURN_INT32(u_cells[cols - 1]);
 }
 
+
+PG_FUNCTION_INFO_V1(levenshtein_with_costs);
+Datum
+levenshtein_with_costs(PG_FUNCTION_ARGS)
+{
+	char	*src = TextDatumGetCString(PG_GETARG_DATUM(0));
+	char	*dst = TextDatumGetCString(PG_GETARG_DATUM(1));
+	int		 ins_c = PG_GETARG_INT32(2);
+	int		 del_c = PG_GETARG_INT32(3);
+	int		 sub_c = PG_GETARG_INT32(4);
+
+	PG_RETURN_INT32(levenshtein_internal(src, dst, ins_c, del_c, sub_c));
+}
+
+
+PG_FUNCTION_INFO_V1(levenshtein);
+Datum
+levenshtein(PG_FUNCTION_ARGS)
+{
+	char	*src = TextDatumGetCString(PG_GETARG_DATUM(0));
+	char	*dst = TextDatumGetCString(PG_GETARG_DATUM(1));
+
+	PG_RETURN_INT32(levenshtein_internal(src, dst, 1, 1, 1));
+}
+
+
 /*
  * Calculates the metaphone of an input string.
  * Returns number of characters requested
@@ -249,9 +349,8 @@ metaphone(PG_FUNCTION_ARGS)
 /*
  * Original code by Michael G Schwern starts here.
  * Code slightly modified for use as PostgreSQL
- * function (palloc, etc). Original includes
+ * function (palloc, etc).
- * are rolled into fuzzystrmatch.h
+ */
- *------------------------------------------------------------------*/
 
 /* I suppose I could have been using a character pointer instead of
  * accessing the array directly... */
@@ -273,7 +372,7 @@ metaphone(PG_FUNCTION_ARGS)
 
 /* Allows us to safely look ahead an arbitrary # of letters */
 /* I probably could have just used strlen... */
-char
+static char
 Lookahead(char *word, int how_far)
 {
 	char		letter_ahead = '\0';	/* null by default */
@@ -299,14 +398,10 @@ Lookahead(char *word, int how_far)
 #define Isbreak(c)	(!isalpha((unsigned char) (c)))
 
 
-int
+static int
-_metaphone(
+_metaphone(char *word,				/* IN */
- /* IN */
-		   char *word,
 		   int max_phonemes,
- /* OUT */
+		   char **phoned_word)		/* OUT */
-		   char **phoned_word
-)
 {
 	int			w_idx = 0;		/* point in the phonization we're at. */
 	int			p_idx = 0;		/* end of the phoned phrase */

contrib/fuzzystrmatch/fuzzystrmatch.h

@@ -1,171 +0,0 @@
-/*
- * fuzzystrmatch.h
- *
- * Functions for "fuzzy" comparison of strings
- *
- * Joe Conway <mail@joeconway.com>
- *
- * $PostgreSQL: pgsql/contrib/fuzzystrmatch/fuzzystrmatch.h,v 1.17 2008/03/25 22:42:41 tgl Exp $
- * Copyright (c) 2001-2008, PostgreSQL Global Development Group
- * ALL RIGHTS RESERVED;
- *
- * levenshtein()
- * -------------
- * Written based on a description of the algorithm by Michael Gilleland
- * found at http://www.merriampark.com/ld.htm
- * Also looked at levenshtein.c in the PHP 4.0.6 distribution for
- * inspiration.
- *
- * metaphone()
- * -----------
- * Modified for PostgreSQL by Joe Conway.
- * Based on CPAN's "Text-Metaphone-1.96" by Michael G Schwern <schwern@pobox.com>
- * Code slightly modified for use as PostgreSQL function (palloc, elog, etc).
- * Metaphone was originally created by Lawrence Philips and presented in article
- * in "Computer Language" December 1990 issue.
- *
- * Permission to use, copy, modify, and distribute this software and its
- * documentation for any purpose, without fee, and without a written agreement
- * is hereby granted, provided that the above copyright notice and this
- * paragraph and the following two paragraphs appear in all copies.
- *
- * IN NO EVENT SHALL THE AUTHORS OR DISTRIBUTORS BE LIABLE TO ANY PARTY FOR
- * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING
- * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS
- * DOCUMENTATION, EVEN IF THE AUTHOR OR DISTRIBUTORS HAVE BEEN ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- *
- * THE AUTHORS AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES,
- * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
- * AND FITNESS FOR A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS
- * ON AN "AS IS" BASIS, AND THE AUTHOR AND DISTRIBUTORS HAS NO OBLIGATIONS TO
- * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
- *
- */
-
-#ifndef FUZZYSTRMATCH_H
-#define FUZZYSTRMATCH_H
-
-#include "postgres.h"
-
-#include <ctype.h>
-
-#include "fmgr.h"
-#include "utils/builtins.h"
-
-
-
-/*
- * External declarations
- */
-extern Datum levenshtein(PG_FUNCTION_ARGS);
-extern Datum metaphone(PG_FUNCTION_ARGS);
-extern Datum soundex(PG_FUNCTION_ARGS);
-extern Datum difference(PG_FUNCTION_ARGS);
-
-/*
- * Soundex
- */
-static void _soundex(const char *instr, char *outstr);
-
-#define SOUNDEX_LEN 4
-
-/*									ABCDEFGHIJKLMNOPQRSTUVWXYZ */
-static const char *soundex_table = "01230120022455012623010202";
-
-#define soundex_code(letter) soundex_table[toupper((unsigned char) (letter)) - 'A']
-
-
-/*
- * Levenshtein
- */
-#define MAX_LEVENSHTEIN_STRLEN		255
-
-
-/*
- * Metaphone
- */
-#define MAX_METAPHONE_STRLEN		255
-
-/*
- * Original code by Michael G Schwern starts here.
- * Code slightly modified for use as PostgreSQL
- * function (combined *.h into here).
- *------------------------------------------------------------------*/
-
-/**************************************************************************
-	metaphone -- Breaks english phrases down into their phonemes.
-
-	Input
-		word			--	An english word to be phonized
-		max_phonemes	--	How many phonemes to calculate.  If 0, then it
-							will phonize the entire phrase.
-		phoned_word		--	The final phonized word.  (We'll allocate the
-							memory.)
-	Output
-		error	--	A simple error flag, returns TRUE or FALSE
-
-	NOTES:	ALL non-alpha characters are ignored, this includes whitespace,
-	although non-alpha characters will break up phonemes.
-****************************************************************************/
-
-
-/**************************************************************************
-	my constants -- constants I like
-
-	Probably redundant.
-
-***************************************************************************/
-
-#define META_ERROR			FALSE
-#define META_SUCCESS		TRUE
-#define META_FAILURE		FALSE
-
-
-/*	I add modifications to the traditional metaphone algorithm that you
-	might find in books.  Define this if you want metaphone to behave
-	traditionally */
-#undef USE_TRADITIONAL_METAPHONE
-
-/* Special encodings */
-#define  SH		'X'
-#define  TH		'0'
-
-char		Lookahead(char *word, int how_far);
-int
-_metaphone(
- /* IN */
-		   char *word,
-		   int max_phonemes,
- /* OUT */
-		   char **phoned_word
-);
-
-/* Metachar.h ... little bits about characters for metaphone */
-
-
-/*-- Character encoding array & accessing macros --*/
-/* Stolen directly out of the book... */
-char		_codes[26] = {
-	1, 16, 4, 16, 9, 2, 4, 16, 9, 2, 0, 2, 2, 2, 1, 4, 0, 2, 4, 4, 1, 0, 0, 0, 8, 0
-/*	a  b c	d e f g  h i j k l m n o p q r s t u v w x y z */
-};
-
-
-#define ENCODE(c) (isalpha((unsigned char) (c)) ? _codes[((toupper((unsigned char) (c))) - 'A')] : 0)
-
-#define isvowel(c)	(ENCODE(c) & 1)		/* AEIOU */
-
-/* These letters are passed through unchanged */
-#define NOCHANGE(c) (ENCODE(c) & 2)		/* FJMNR */
-
-/* These form dipthongs when preceding H */
-#define AFFECTH(c)	(ENCODE(c) & 4)		/* CGPST */
-
-/* These make C and G soft */
-#define MAKESOFT(c) (ENCODE(c) & 8)		/* EIY */
-
-/* These prevent GH from becoming F */
-#define NOGHTOF(c)	(ENCODE(c) & 16)	/* BDH */
-
-#endif   /* FUZZYSTRMATCH_H */

contrib/fuzzystrmatch/fuzzystrmatch.sql.in

@@ -1,4 +1,4 @@
-/* $PostgreSQL: pgsql/contrib/fuzzystrmatch/fuzzystrmatch.sql.in,v 1.9 2007/11/13 04:24:28 momjian Exp $ */
+/* $PostgreSQL: pgsql/contrib/fuzzystrmatch/fuzzystrmatch.sql.in,v 1.10 2008/04/03 21:13:07 tgl Exp $ */
 
 -- Adjust this setting to control where the objects get created.
 SET search_path = public;
@@ -7,6 +7,10 @@ CREATE OR REPLACE FUNCTION levenshtein (text,text) RETURNS int
 AS 'MODULE_PATHNAME','levenshtein'
 LANGUAGE C IMMUTABLE STRICT;
 
+CREATE OR REPLACE FUNCTION levenshtein (text,text,int,int,int) RETURNS int
+AS 'MODULE_PATHNAME','levenshtein_with_costs'
+LANGUAGE C IMMUTABLE STRICT;
+
 CREATE OR REPLACE FUNCTION metaphone (text,int) RETURNS text
 AS 'MODULE_PATHNAME','metaphone'
 LANGUAGE C IMMUTABLE STRICT;

contrib/fuzzystrmatch/uninstall_fuzzystrmatch.sql

@@ -1,4 +1,4 @@
-/* $PostgreSQL: pgsql/contrib/fuzzystrmatch/uninstall_fuzzystrmatch.sql,v 1.3 2007/11/13 04:24:28 momjian Exp $ */
+/* $PostgreSQL: pgsql/contrib/fuzzystrmatch/uninstall_fuzzystrmatch.sql,v 1.4 2008/04/03 21:13:07 tgl Exp $ */
 
 -- Adjust this setting to control where the objects get dropped.
 SET search_path = public;
@@ -15,4 +15,6 @@ DROP FUNCTION soundex(text);
 
 DROP FUNCTION metaphone (text,int);
 
+DROP FUNCTION levenshtein (text,text,int,int,int);
+
 DROP FUNCTION levenshtein (text,text);

doc/src/sgml/fuzzystrmatch.sgml

@@ -1,4 +1,4 @@
-<!-- $PostgreSQL: pgsql/doc/src/sgml/fuzzystrmatch.sgml,v 1.3 2007/12/06 04:12:10 tgl Exp $ -->
+<!-- $PostgreSQL: pgsql/doc/src/sgml/fuzzystrmatch.sgml,v 1.4 2008/04/03 21:13:07 tgl Exp $ -->
 
 <sect1 id="fuzzystrmatch">
  <title>fuzzystrmatch</title>
@@ -74,16 +74,20 @@ SELECT * FROM s WHERE difference(s.nm, 'john') &gt; 2;
   </para>
 
   <programlisting>
+   levenshtein(text source, text target, int ins_cost, int del_cost, int sub_cost) returns int
    levenshtein(text source, text target) returns int
   </programlisting>
 
   <para>
    Both <literal>source</literal> and <literal>target</literal> can be any
-   non-null string, with a maximum of 255 characters.
+   non-null string, with a maximum of 255 bytes.  The cost parameters
+   specify how much to charge for a character insertion, deletion, or
+   substitution, respectively.  You can omit the cost parameters, as in
+   the second version of the function; in that case they all default to 1.
   </para>
 
   <para>
-   Example:
+   Examples:
   </para>
 
   <programlisting>
@@ -92,6 +96,12 @@ test=# SELECT levenshtein('GUMBO', 'GAMBOL');
 -------------
            2
 (1 row)
+
+test=# SELECT levenshtein('GUMBO', 'GAMBOL', 2,1,1);
+ levenshtein
+-------------
+           3
+(1 row)
   </programlisting>
  </sect2>