sharpetronics/PostgreSQL
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2025-05-14 00:03:46 -04:00
Code Issues Projects Releases Wiki Activity

Allow usage of huge maintenance_work_mem for GIN build.

Browse Source 
Currently, in-memory posting list during GIN build process is limited 1GB
because of using repalloc. The patch replaces call of repalloc to repalloc_huge.
It increases limit of posting list from 180 millions
(1GB / sizeof(ItemPointerData)) to 4 billions limited by maxcount/count fields
in GinEntryAccumulator and subsequent calls. Check added.

Also, fix accounting of allocatedMemory during build to prevent integer
overflow with maintenance_work_mem > 4GB.

Robert Abraham <robert.abraham86@googlemail.com> with additions by me
This commit is contained in:
Teodor Sigaev 2015-09-02 20:08:58 +03:00
parent 075ab425bd
commit 30bb26b5e0
4 changed files with 12 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/backend/access/gin/ginbulk.c
View File

@ -14,6 +14,8 @@
#include "postgres.h" #include "postgres.h"
#include <limits.h>
#include "access/gin_private.h" #include "access/gin_private.h"
#include "utils/datum.h" #include "utils/datum.h"
#include "utils/memutils.h" #include "utils/memutils.h"
@ -36,10 +38,16 @@ ginCombineData(RBNode *existing, const RBNode *newdata, void *arg)
*/ */
if (eo->count >= eo->maxcount) if (eo->count >= eo->maxcount)
{ {
if (eo->maxcount > INT_MAX)
ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("posting list is too long"),
errhint("Reduce maintenance_work_mem")));
accum->allocatedMemory -= GetMemoryChunkSpace(eo->list); accum->allocatedMemory -= GetMemoryChunkSpace(eo->list);
eo->maxcount *= 2; eo->maxcount *= 2;
eo->list = (ItemPointerData *) eo->list = (ItemPointerData *)
repalloc(eo->list, sizeof(ItemPointerData) * eo->maxcount); repalloc_huge(eo->list, sizeof(ItemPointerData) * eo->maxcount);
accum->allocatedMemory += GetMemoryChunkSpace(eo->list); accum->allocatedMemory += GetMemoryChunkSpace(eo->list);
} }

2
src/backend/access/gin/ginfast.c
View File

@ -814,7 +814,7 @@ ginInsertCleanup(GinState *ginstate,
*/ */
if (GinPageGetOpaque(page)->rightlink == InvalidBlockNumber || if (GinPageGetOpaque(page)->rightlink == InvalidBlockNumber ||
(GinPageHasFullRow(page) && (GinPageHasFullRow(page) &&
(accum.allocatedMemory >= maintenance_work_mem * 1024L))) (accum.allocatedMemory >= (Size)maintenance_work_mem * 1024L)))
{ {
ItemPointerData *list; ItemPointerData *list;
uint32 nlist; uint32 nlist;

2
src/backend/access/gin/gininsert.c
View File

@ -281,7 +281,7 @@ ginBuildCallback(Relation index, HeapTuple htup, Datum *values,
&htup->t_self); &htup->t_self);
/* If we've maxed out our available memory, dump everything to the index */ /* If we've maxed out our available memory, dump everything to the index */
if (buildstate->accum.allocatedMemory >= maintenance_work_mem * 1024L) if (buildstate->accum.allocatedMemory >= (Size)maintenance_work_mem * 1024L)
{ {
ItemPointerData *list; ItemPointerData *list;
Datum key; Datum key;

2
src/include/access/gin_private.h
View File

@ -903,7 +903,7 @@ typedef struct GinEntryAccumulator
typedef struct typedef struct
{ {
GinState *ginstate; GinState *ginstate;
long allocatedMemory; Size allocatedMemory;
GinEntryAccumulator *entryallocator; GinEntryAccumulator *entryallocator;
uint32 eas_used; uint32 eas_used;
RBTree *tree; RBTree *tree;