sharpetronics/PostgreSQL
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2025-10-31 00:03:57 -04:00
Code Issues Projects Releases Wiki Activity

Fix an O(N^2) performance issue for sessions modifying many relations.

Browse Source 
AtEOXact_RelationCache() scanned the entire relation cache at the end of
any transaction that created a new relation or assigned a new relfilenode.
Thus, clients such as pg_restore had an O(N^2) performance problem that
would start to be noticeable after creating 10000 or so tables.  Since
typically only a small number of relcache entries need any cleanup, we
can fix this by keeping a small list of their OIDs and doing hash_searches
for them.  We fall back to the full-table scan if the list overflows.

Ideally, the maximum list length would be set at the point where N
hash_searches would cost just less than the full-table scan.  Some quick
experimentation says that point might be around 50-100; I (tgl)
conservatively set MAX_EOXACT_LIST = 32.  For the case that we're worried
about here, which is short single-statement transactions, it's unlikely
there would ever be more than about a dozen list entries anyway; so it's
probably not worth being too tense about the value.

We could avoid the hash_searches by instead keeping the target relcache
entries linked into a list, but that would be noticeably more complicated
and bug-prone because of the need to maintain such a list in the face of
relcache entry drops.  Since a relcache entry can only need such cleanup
after a somewhat-heavyweight filesystem operation, trying to save a
hash_search per cleanup doesn't seem very useful anyway --- it's the scan
over all the not-needing-cleanup entries that we wish to avoid here.

Jeff Janes, reviewed and tweaked a bit by Tom Lane
This commit is contained in:
Tom Lane 2013-01-20 13:44:49 -05:00
parent 0a2da5282a
commit d5b31cc32b
1 changed files with 127 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

164
src/backend/utils/cache/relcache.c vendored
View File

@ -137,9 +137,27 @@ static long relcacheInvalsReceived = 0L;
static List *initFileRelationIds = NIL;
/*
* This flag lets us optimize away work in AtEO(Sub)Xact_RelationCache().
* eoxact_list[] stores the OIDs of relations that (might) need AtEOXact
* cleanup work. This list intentionally has limited size; if it overflows,
* we fall back to scanning the whole hashtable. There is no value in a very
* large list because (1) at some point, a hash_seq_search scan is faster than
* retail lookups, and (2) the value of this is to reduce EOXact work for
* short transactions, which can't have dirtied all that many tables anyway.
* EOXactListAdd() does not bother to prevent duplicate list entries, so the
* cleanup processing must be idempotent.
*/
static bool need_eoxact_work = false;
#define MAX_EOXACT_LIST 32
static Oid eoxact_list[MAX_EOXACT_LIST];
static int eoxact_list_len = 0;
static bool eoxact_list_overflowed = false;
#define EOXactListAdd(rel) \
do { \
if (eoxact_list_len < MAX_EOXACT_LIST) \
eoxact_list[eoxact_list_len++] = (rel)->rd_id; \
else \
eoxact_list_overflowed = true; \
} while (0)
/*
@ -204,6 +222,9 @@ static void RelationClearRelation(Relation relation, bool rebuild);
static void RelationReloadIndexInfo(Relation relation);
static void RelationFlushRelation(Relation relation);
static void AtEOXact_cleanup(Relation relation, bool isCommit);
static void AtEOSubXact_cleanup(Relation relation, bool isCommit,
SubTransactionId mySubid, SubTransactionId parentSubid);
static bool load_relcache_init_file(bool shared);
static void write_relcache_init_file(bool shared);
static void write_item(const void *data, Size len, FILE *fp);
@ -2275,31 +2296,56 @@ AtEOXact_RelationCache(bool isCommit)
{
HASH_SEQ_STATUS status;
RelIdCacheEnt *idhentry;
int i;
/*
* To speed up transaction exit, we want to avoid scanning the relcache
* unless there is actually something for this routine to do. Other than
* the debug-only Assert checks, most transactions don't create any work
* for us to do here, so we keep a static flag that gets set if there is
* anything to do. (Currently, this means either a relation is created in
* the current xact, or one is given a new relfilenode, or an index list
* is forced.) For simplicity, the flag remains set till end of top-level
* transaction, even though we could clear it at subtransaction end in
* some cases.
* Unless the eoxact_list[] overflowed, we only need to examine the rels
* listed in it. Otherwise fall back on a hash_seq_search scan.
*
* For simplicity, eoxact_list[] entries are not deleted till end of
* top-level transaction, even though we could remove them at
* subtransaction end in some cases, or remove relations from the list if
* they are cleared for other reasons. Therefore we should expect the
* case that list entries are not found in the hashtable; if not, there's
* nothing to do for them.
*/
if (!need_eoxact_work
#ifdef USE_ASSERT_CHECKING
&& !assert_enabled
#endif
)
return;
if (eoxact_list_overflowed)
{
hash_seq_init(&status, RelationIdCache);
while ((idhentry = (RelIdCacheEnt *) hash_seq_search(&status)) != NULL)
{
Relation relation = idhentry->reldesc;
AtEOXact_cleanup(idhentry->reldesc, isCommit);
}
}
else
{
for (i = 0; i < eoxact_list_len; i++)
{
idhentry = (RelIdCacheEnt *) hash_search(RelationIdCache,
(void *) &eoxact_list[i],
HASH_FIND,
NULL);
if (idhentry != NULL)
AtEOXact_cleanup(idhentry->reldesc, isCommit);
}
}
/* Now we're out of the transaction and can clear the list */
eoxact_list_len = 0;
eoxact_list_overflowed = false;
}
/*
* AtEOXact_cleanup
*
* Clean up a single rel at main-transaction commit or abort
*
* NB: this processing must be idempotent, because EOXactListAdd() doesn't
* bother to prevent duplicate entries in eoxact_list[].
*/
static void
AtEOXact_cleanup(Relation relation, bool isCommit)
{
/*
* The relcache entry's ref count should be back to its normal
* not-in-a-transaction state: 0 unless it's nailed in cache.
@ -2307,6 +2353,12 @@ AtEOXact_RelationCache(bool isCommit)
* In bootstrap mode, this is NOT true, so don't check it --- the
* bootstrap code expects relations to stay open across start/commit
* transaction calls. (That seems bogus, but it's not worth fixing.)
*
* Note: ideally this check would be applied to every relcache entry,
* not just those that have eoxact work to do. But it's not worth
* forcing a scan of the whole relcache just for this. (Moreover,
* doing so would mean that assert-enabled testing never tests the
* hash_search code path above, which seems a bad idea.)
*/
#ifdef USE_ASSERT_CHECKING
if (!IsBootstrapProcessingMode())
@ -2335,7 +2387,7 @@ AtEOXact_RelationCache(bool isCommit)
else
{
RelationClearRelation(relation, false);
continue;
return;
}
}
@ -2354,10 +2406,6 @@ AtEOXact_RelationCache(bool isCommit)
relation->rd_oidindex = InvalidOid;
relation->rd_indexvalid = 0;
}
}
/* Once done with the transaction, we can reset need_eoxact_work */
need_eoxact_work = false;
}
/*
@ -2373,20 +2421,51 @@ AtEOSubXact_RelationCache(bool isCommit, SubTransactionId mySubid,
{
HASH_SEQ_STATUS status;
RelIdCacheEnt *idhentry;
int i;
/*
* Skip the relcache scan if nothing to do --- see notes for
* AtEOXact_RelationCache.
* Unless the eoxact_list[] overflowed, we only need to examine the rels
* listed in it. Otherwise fall back on a hash_seq_search scan. Same
* logic as in AtEOXact_RelationCache.
*/
if (!need_eoxact_work)
return;
if (eoxact_list_overflowed)
{
hash_seq_init(&status, RelationIdCache);
while ((idhentry = (RelIdCacheEnt *) hash_seq_search(&status)) != NULL)
{
Relation relation = idhentry->reldesc;
AtEOSubXact_cleanup(idhentry->reldesc, isCommit,
mySubid, parentSubid);
}
}
else
{
for (i = 0; i < eoxact_list_len; i++)
{
idhentry = (RelIdCacheEnt *) hash_search(RelationIdCache,
(void *) &eoxact_list[i],
HASH_FIND,
NULL);
if (idhentry != NULL)
AtEOSubXact_cleanup(idhentry->reldesc, isCommit,
mySubid, parentSubid);
}
}
/* Don't reset the list; we still need more cleanup later */
}
/*
* AtEOSubXact_cleanup
*
* Clean up a single rel at subtransaction commit or abort
*
* NB: this processing must be idempotent, because EOXactListAdd() doesn't
* bother to prevent duplicate entries in eoxact_list[].
*/
static void
AtEOSubXact_cleanup(Relation relation, bool isCommit,
SubTransactionId mySubid, SubTransactionId parentSubid)
{
/*
* Is it a relation created in the current subtransaction?
*
@ -2400,7 +2479,7 @@ AtEOSubXact_RelationCache(bool isCommit, SubTransactionId mySubid,
else
{
RelationClearRelation(relation, false);
continue;
return;
}
}
@ -2426,7 +2505,6 @@ AtEOSubXact_RelationCache(bool isCommit, SubTransactionId mySubid,
relation->rd_oidindex = InvalidOid;
relation->rd_indexvalid = 0;
}
}
}
@ -2516,9 +2594,6 @@ RelationBuildLocalRelation(const char *relname,
rel->rd_createSubid = GetCurrentSubTransactionId();
rel->rd_newRelfilenodeSubid = InvalidSubTransactionId;
/* must flag that we have rels created in this transaction */
need_eoxact_work = true;
/*
* create a new tuple descriptor from the one passed in. We do this
* partly to copy it into the cache context, and partly because the new
@ -2609,6 +2684,12 @@ RelationBuildLocalRelation(const char *relname,
*/
RelationCacheInsert(rel);
/*
* Flag relation as needing eoxact cleanup (to clear rd_createSubid).
* We can't do this before storing relid in it.
*/
EOXactListAdd(rel);
/*
* done building relcache entry.
*/
@ -2732,8 +2813,9 @@ RelationSetNewRelfilenode(Relation relation, TransactionId freezeXid)
* operations on the rel in the same transaction.
*/
relation->rd_newRelfilenodeSubid = GetCurrentSubTransactionId();
/* ... and now we have eoxact cleanup work to do */
need_eoxact_work = true;
/* Flag relation as needing eoxact cleanup (to remove the hint) */
EOXactListAdd(relation);
}
@ -3513,8 +3595,8 @@ RelationSetIndexList(Relation relation, List *indexIds, Oid oidIndex)
relation->rd_indexlist = indexIds;
relation->rd_oidindex = oidIndex;
relation->rd_indexvalid = 2; /* mark list as forced */
/* must flag that we have a forced index list */
need_eoxact_work = true;
/* Flag relation as needing eoxact cleanup (to reset the list) */
EOXactListAdd(relation);
}
/*