Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Generic WAL support. Script for replicas #2

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Generic WAL support. Script for replicas #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 4 additions & 10 deletions Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ RUN localedef -i en_US -c -f UTF-8 en_US.UTF-8
RUN groupadd -r postgres --gid=999 \
&& useradd -m -r -g postgres --uid=999 postgres
WORKDIR /home/postgres
RUN su postgres -c "git clone https://github.com/postgres/postgres.git --depth=1 --branch=REL_11_STABLE"
RUN su postgres -c "git clone https://github.com/postgres/postgres.git --depth=1 --branch=master"
WORKDIR /home/postgres/postgres
COPY . ./contrib/ags
RUN su postgres -c "./configure \
Expand All @@ -45,12 +45,6 @@ RUN make install
WORKDIR /home/postgres/postgres/contrib/ags
RUN make install
RUN chown -R postgres:postgres /usr/local/pgsql
RUN su postgres -c "export PATH=$PATH:/usr/local/pgsql/bin && ./test.sh"
# ENTRYPOINT [ "./test.sh" ]

# RUN make install
# RUN make -C contrib install

# RUN cd project && make

# CMD ["/bin/bash", "~/project/text.sh"]
# RUN su postgres -c "export PATH=$PATH:/usr/local/pgsql/bin && ./test.sh"
ENV PGINSTALL /usr/local/pgsql
RUN su postgres -c "export PATH=$PATH:/usr/local/pgsql/bin && ./create_replica.sh"
73 changes: 73 additions & 0 deletions create_replica.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
#!/usr/bin/env bash
# Taken from https://github.com/afiskon/pgscripts/blob/master/install.sh

set -e

if [[ -z $PGINSTALL ]]; then
echo "ERROR: \$PGINSTALL environment variable is empty"
exit 1
fi

M=$PGINSTALL
U=`whoami`

pkill -9 postgres || true

# rm -rf $M || true
# mkdir $M

make install

$M/bin/initdb -D $M/data-master

# probably not cheap!
echo "wal_consistency_checking = 'all'" >> $M/data-master/postgresql.conf
echo "max_prepared_transactions = 100" >> $M/data-master/postgresql.conf
echo "wal_level = logical" >> $M/data-master/postgresql.conf
echo "wal_keep_segments = 128" >> $M/data-master/postgresql.conf
# keep max_connections large enough or `make installcheck-world` may fail during `prep` test
echo "max_connections = 100" >> $M/data-master/postgresql.conf
echo "wal_log_hints = on" >> $M/data-master/postgresql.conf
echo "max_wal_senders = 8" >> $M/data-master/postgresql.conf
echo "wal_keep_segments = 64" >> $M/data-master/postgresql.conf
echo "listen_addresses = '*'" >> $M/data-master/postgresql.conf
echo "hot_standby = on" >> $M/data-master/postgresql.conf
echo "log_statement = all" >> $M/data-master/postgresql.conf
echo "max_locks_per_transaction = 256" >> $M/data-master/postgresql.conf
#echo "shared_buffers = 1GB" >> $M/data-master/postgresql.conf
#echo "fsync = off" >> $M/data-master/postgresql.conf
#echo "autovacuum = off" >> $M/data-master/postgresql.conf

echo "host replication $U 127.0.0.1/24 trust" >> $M/data-master/pg_hba.conf
echo "host all $U 127.0.0.1/24 trust" >> $M/data-master/pg_hba.conf
echo "host all all 10.128.0.0/16 trust" >> $M/data-master/pg_hba.conf

# CREATE ROLE scram_role LOGIN PASSWORD ('pass' USING 'scram');
# CREATE DATABASE scram_role;
# GRANT ALL privileges ON DATABASE scram_role TO scram_role;
# psql -U scram_role

echo '' > $M/data-master/logfile

echo "=== STARTING MASTER ==="

$M/bin/pg_ctl -w -D $M/data-master -l $M/data-master/logfile start
# $M/bin/createdb $U
$M/bin/psql -c "create table test(k int primary key, v text);"

echo "=== RUNNING PG_BASEBACKUP ==="

$M/bin/pg_basebackup -P -R -X stream -c fast -h 127.0.0.1 -U $U -D $M/data-slave
echo "port = 5433" >> $M/data-slave/postgresql.conf

echo "=== STARTING SLAVE ==="

$M/bin/pg_ctl -w -D $M/data-slave -l $M/data-slave/logfile start

$M/bin/psql postgres -c "create extension cube;"
$M/bin/psql postgres -c "create extension ags;"

$M/bin/psql postgres -c "create table x as select cube(random()) c from generate_series(1,10000) y;"
$M/bin/psql postgres -c "create index on x using ags(c);"
$M/bin/psql postgres -c "set enable_bitmapscan to off;explain analyze select * from x where c <@ cube(0,0.1);"
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this query needs to be executed on replica
--port=5433

$M/bin/psql postgres -c "delete from x where (c~>1)>0.1;"
75 changes: 38 additions & 37 deletions gist.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

#include "gist_private.h"
#include "gistscan.h"
#include "access/generic_xlog.h"
#include "catalog/pg_collation.h"
#include "miscadmin.h"
#include "storage/lmgr.h"
Expand All @@ -30,14 +31,14 @@
/* non-export function prototypes */
static void gistfixsplit(GISTInsertState *state, GISTSTATE *giststate);
static bool gistinserttuple(GISTInsertState *state, GISTInsertStack *stack,
GISTSTATE *giststate, IndexTuple tuple, OffsetNumber oldoffnum);
GISTSTATE *giststate, IndexTuple tuple, OffsetNumber oldoffnum);
static bool gistinserttuples(GISTInsertState *state, GISTInsertStack *stack,
GISTSTATE *giststate,
IndexTuple *tuples, int ntup, OffsetNumber oldoffnum,
Buffer leftchild, Buffer rightchild,
bool unlockbuf, bool unlockleftchild, int ndeltup, OffsetNumber skipoffnum);
GISTSTATE *giststate,
IndexTuple *tuples, int ntup, OffsetNumber oldoffnum,
Buffer leftchild, Buffer rightchild,
bool unlockbuf, bool unlockleftchild, int ndeltup, OffsetNumber skipoffnum);
static void gistfinishsplit(GISTInsertState *state, GISTInsertStack *stack,
GISTSTATE *giststate, List *splitinfo, bool releasebuf);
GISTSTATE *giststate, List *splitinfo, bool releasebuf);
static void gistprunepage(Relation rel, Page page, Buffer buffer,
Relation heapRel);

Expand All @@ -59,7 +60,7 @@ PG_FUNCTION_INFO_V1(agshandler);
* and callbacks.
*/
Datum
agshandler(PG_FUNCTION_ARGS)
agshandler(PG_FUNCTION_ARGS)
{
IndexAmRoutine *amroutine = makeNode(IndexAmRoutine);

Expand Down Expand Up @@ -220,17 +221,18 @@ gistplacetopage(Relation rel, Size freespace, GISTSTATE *giststate,
Buffer buffer,
IndexTuple *itup, int ntup, OffsetNumber oldoffnum,
BlockNumber *newblkno,
Buffer leftchildbuf,
List **splitinfo,
bool markfollowright,
Relation heapRel,
bool is_build,
int ndeltup,
Buffer leftchildbuf,
List **splitinfo,
bool markfollowright,
Relation heapRel,
bool is_build,
int ndeltup,
OffsetNumber skipoffnum)
{
BlockNumber blkno = BufferGetBlockNumber(buffer);
Page page = BufferGetPage(buffer);
bool is_leaf = (GistPageIsLeaf(page)) ? true : false;
GenericXLogState *state;
XLogRecPtr recptr;
int i;
bool is_split;
Expand Down Expand Up @@ -525,9 +527,16 @@ gistplacetopage(Relation rel, Size freespace, GISTSTATE *giststate,
else
{
if (RelationNeedsWAL(rel))
recptr = gistXLogSplit(is_leaf,
dist, oldrlink, oldnsn, leftchildbuf,
markfollowright);
{
state = GenericXLogStart(rel);
for (ptr = dist; ptr; ptr = ptr->next)
{
GenericXLogRegisterBuffer(state, ptr->buffer, GENERIC_XLOG_FULL_IMAGE);
}

recptr = GenericXLogFinish(state);
}

else
recptr = gistGetFakeLSN(rel);
}
Expand Down Expand Up @@ -604,19 +613,10 @@ gistplacetopage(Relation rel, Size freespace, GISTSTATE *giststate,
{
if (RelationNeedsWAL(rel))
{
OffsetNumber ndeloffs = 0,
deloffs[BLCKSZ /sizeof(ItemIdData)];

if (OffsetNumberIsValid(oldoffnum))
{
for (i = 0; i < ndeltup; i++)
deloffs[i] = oldoffnum + i;
ndeloffs = ndeltup;
}

recptr = gistXLogUpdate(buffer,
deloffs, ndeloffs, itup, ntup,
leftchildbuf, skipoffnum);
state = GenericXLogStart(rel);
GenericXLogRegisterBuffer(state, buffer, GENERIC_XLOG_FULL_IMAGE);
GenericXLogRegisterBuffer(state, leftchildbuf, GENERIC_XLOG_FULL_IMAGE);
recptr = GenericXLogFinish(state);
}
else
recptr = gistGetFakeLSN(rel);
Expand Down Expand Up @@ -1414,7 +1414,7 @@ inline void
gistcheckskippage(Page page)
{
OffsetNumber i,
maxoff;
maxoff;
int skiplast = 0;
bool wasskip = false;
Assert(false);
Expand Down Expand Up @@ -1485,7 +1485,7 @@ gisttestskipgroup(GISTInsertState *state, GISTInsertStack *stack,
}
}
gistinserttuples(state, stack, giststate, itvec, totalsize, skipoffnum,
InvalidBuffer, InvalidBuffer, false, false, skipsize + 1, InvalidOffsetNumber);
InvalidBuffer, InvalidBuffer, false, false, skipsize + 1, InvalidOffsetNumber);
}
}

Expand Down Expand Up @@ -1647,7 +1647,7 @@ gistSplit(Relation r,
if (!gistfitpage(lvectup, v.splitVector.spl_nleft))
{
SplitedPageLayout *resptr,
*subres;
*subres;

resptr = subres = gistSplit(r, page, lvectup, v.splitVector.spl_nleft, giststate);

Expand Down Expand Up @@ -1677,7 +1677,7 @@ gistSplitBySkipgroup(Relation r,
GISTSTATE *giststate)
{
IndexTuple *lvectup,
*rvectup,
*rvectup,
*skiptuples;
OffsetNumber *skipoffsets;
GistSplitVector v;
Expand Down Expand Up @@ -1760,7 +1760,7 @@ gistSplitBySkipgroup(Relation r,
if (!gistfitpage(lvectup, v.splitVector.spl_nleft))
{
SplitedPageLayout *resptr,
*subres;
*subres;

resptr = subres = gistSplitBySkipgroup(r, page, lvectup, v.splitVector.spl_nleft, giststate);
if (subres == NULL)
Expand Down Expand Up @@ -1972,10 +1972,11 @@ gistprunepage(Relation rel, Page page, Buffer buffer, Relation heapRel)
if (RelationNeedsWAL(rel))
{
XLogRecPtr recptr;
GenericXLogState *state;

recptr = gistXLogDelete(buffer,
deletable, ndeletable,
heapRel->rd_node);
state = GenericXLogStart(rel);
GenericXLogRegisterBuffer(state, buffer, GENERIC_XLOG_FULL_IMAGE);
recptr = GenericXLogFinish(state);

PageSetLSN(page, recptr);
}
Expand Down
17 changes: 13 additions & 4 deletions gistvacuum.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
#include "access/genam.h"
#include "gist_private.h"
#include "access/transam.h"
#include "access/generic_xlog.h"
#include "commands/vacuum.h"
#include "lib/integerset.h"
#include "miscadmin.h"
Expand Down Expand Up @@ -367,10 +368,12 @@ gistvacuumpage(GistVacState *vstate, BlockNumber blkno, BlockNumber orig_blkno)
if (RelationNeedsWAL(rel))
{
XLogRecPtr recptr;
GenericXLogState *state;

state = GenericXLogStart(rel);
GenericXLogRegisterBuffer(state, buffer, GENERIC_XLOG_FULL_IMAGE);
recptr = GenericXLogFinish(state);

recptr = gistXLogDelete(buffer,
todelete, ntodelete,
rel->rd_node);
PageSetLSN(page, recptr);
}
else
Expand Down Expand Up @@ -663,7 +666,13 @@ gistdeletepage(IndexVacuumInfo *info, GistBulkDeleteResult *stats,
PageIndexTupleDelete(parentPage, downlink);

if (RelationNeedsWAL(info->index))
recptr = gistXLogPageDelete(leafBuffer, txid, parentBuffer, downlink);
{
GenericXLogState *state;

state = GenericXLogStart(info->index);
GenericXLogRegisterBuffer(state, leafBuffer, GENERIC_XLOG_FULL_IMAGE);
recptr = GenericXLogFinish(state);
}
else
recptr = gistGetFakeLSN(info->index);
PageSetLSN(parentPage, recptr);
Expand Down