diff --git a/doc/src/sgml/gist.sgml b/doc/src/sgml/gist.sgml
index 587517da1d1..b0ac8e52b6a 100644
--- a/doc/src/sgml/gist.sgml
+++ b/doc/src/sgml/gist.sgml
@@ -147,6 +147,8 @@
        Returns a value indicating the <quote>cost</quote> of inserting the new
        entry into a particular branch of the tree.  items will be inserted
        down the path of least <function>penalty</function> in the tree.
+       Values returned by <function>penalty</function> should be non-negative.
+       If a negative value is returned, it will be treated as zero.
       </para>
      </listitem>
     </varlistentry>
diff --git a/src/backend/access/gist/gistutil.c b/src/backend/access/gist/gistutil.c
index d88f5ea0f81..5bd6a9893b0 100644
--- a/src/backend/access/gist/gistutil.c
+++ b/src/backend/access/gist/gistutil.c
@@ -13,6 +13,8 @@
  */
 #include "postgres.h"
 
+#include <math.h>
+
 #include "access/gist_private.h"
 #include "access/heapam.h"
 #include "access/reloptions.h"
@@ -530,16 +532,22 @@ gistpenalty(GISTSTATE *giststate, int attno,
 {
 	float		penalty = 0.0;
 
-	if (giststate->penaltyFn[attno].fn_strict == FALSE || (isNullOrig == FALSE && isNullAdd == FALSE))
+	if (giststate->penaltyFn[attno].fn_strict == FALSE ||
+		(isNullOrig == FALSE && isNullAdd == FALSE))
+	{
 		FunctionCall3(&giststate->penaltyFn[attno],
 					  PointerGetDatum(orig),
 					  PointerGetDatum(add),
 					  PointerGetDatum(&penalty));
+		/* disallow negative or NaN penalty */
+		if (isnan(penalty) || penalty < 0.0)
+			penalty = 0.0;
+	}
 	else if (isNullOrig && isNullAdd)
 		penalty = 0.0;
 	else
-		penalty = 1e10;			/* try to prevent to mix null and non-null
-								 * value */
+		penalty = 1e10;			/* try to prevent mixing null and non-null
+								 * values */
 
 	return penalty;
 }