feat: performance optimization

Optimize performance of index create, bulk extend and so on.

contrib/bloom/blinsert.c

@@ -166,7 +166,7 @@ blbuildempty(Relation index)
 	Page		metapage;
 
 	/* Construct metapage. */
-	metapage = (Page) palloc_io_aligned(BLCKSZ, 0);
+	metapage = (Page) palloc_aligned(BLCKSZ, PG_IO_ALIGN_SIZE, 0);
 	BloomFillMetapage(index, metapage);
 
 	/*
@@ -178,7 +178,7 @@ blbuildempty(Relation index)
 	 */
 	PageSetChecksumInplace(metapage, BLOOM_METAPAGE_BLKNO);
 	smgrwrite(RelationGetSmgr(index), INIT_FORKNUM, BLOOM_METAPAGE_BLKNO,
-			  (char *) metapage, true);
+			  metapage, true);
 	log_newpage(&(RelationGetSmgr(index))->smgr_rnode.node, INIT_FORKNUM,
 				BLOOM_METAPAGE_BLKNO, metapage, true);
 

contrib/bloom/blutils.c

@@ -15,6 +15,7 @@
 
 #include "access/amapi.h"
 #include "access/generic_xlog.h"
+#include "access/hio.h"
 #include "access/reloptions.h"
 #include "bloom.h"
 #include "catalog/index.h"
@@ -391,7 +392,7 @@ BloomNewBuffer(Relation index)
 	if (needLock)
 		LockRelationForExtension(index, ExclusiveLock);
 
-	buffer = ReadBuffer(index, P_NEW);
+	buffer = polar_index_add_blocks(index);
 	LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
 
 	if (needLock)

contrib/pageinspect/Makefile

@@ -22,7 +22,7 @@ DATA =  pageinspect--1.10--1.11.sql \
 	pageinspect--1.0--1.1.sql
 PGFILEDESC = "pageinspect - functions to inspect contents of database pages"
 
-REGRESS = page btree brin gin gist hash checksum oldextversions
+REGRESS = page btree brin gin gist hash checksum oldextversions index_bulk_extend
 
 ifdef USE_PGXS
 PG_CONFIG = pg_config

contrib/pageinspect/btreefuncs.c

@@ -520,6 +520,18 @@ bt_page_items_internal(PG_FUNCTION_ARGS, enum pageinspect_version ext_version)
 		uargs->page = palloc(BLCKSZ);
 		memcpy(uargs->page, BufferGetPage(buffer), BLCKSZ);
 
+		/*
+		 * POLAR: During bulk extend, page inspect may access zero page, which
+		 * makes PageGetSpecialPointer Assert error. In order to handle this
+		 * situation, we init these zero pages.
+		 */
+		if (PageIsNew(uargs->page))
+		{
+			_bt_pageinit(uargs->page, BufferGetPageSize(buffer));
+			elog(DEBUG1, "page from block" INT64_FORMAT " is new in index bulk extend", blkno);
+		}
+		/* POLAR end */
+
 		UnlockReleaseBuffer(buffer);
 		relation_close(rel, AccessShareLock);
 

contrib/pageinspect/expected/gist.out

@@ -1,13 +1,6 @@
--- The gist_page_opaque_info() function prints the page's LSN. Normally,
--- that's constant 1 (GistBuildLSN) on every page of a freshly built GiST
--- index. But with wal_level=minimal, the whole relation is dumped to WAL at
--- the end of the transaction if it's smaller than wal_skip_threshold, which
--- updates the LSNs. Wrap the tests on gist_page_opaque_info() in the
--- same transaction with the CREATE INDEX so that we see the LSNs before
--- they are possibly overwritten at end of transaction.
-BEGIN;
--- Create a test table and GiST index.
-CREATE TABLE test_gist AS SELECT point(i,i) p, i::text t FROM
+-- The gist_page_opaque_info() function prints the page's LSN.
+-- Use an unlogged index, so that the LSN is predictable.
+CREATE UNLOGGED TABLE test_gist AS SELECT point(i,i) p, i::text t FROM
     generate_series(1,1000) i;
 CREATE INDEX test_gist_idx ON test_gist USING gist (p);
 -- Page 0 is the root, the rest are leaf pages
@@ -29,7 +22,6 @@ SELECT * FROM gist_page_opaque_info(get_raw_page('test_gist_idx', 2));
  0/1 | 0/0 |         1 | {leaf}
 (1 row)
 
-COMMIT;
 SELECT * FROM gist_page_items(get_raw_page('test_gist_idx', 0), 'test_gist_idx');
  itemoffset |   ctid    | itemlen | dead |             keys              
 ------------+-----------+---------+------+-------------------------------

contrib/pageinspect/expected/index_bulk_extend.out

@@ -0,0 +1,45 @@
+create schema test_index_bulk_extend;
+set search_path to test_index_bulk_extend;
+set client_min_messages to error; 
+create extension if not exists pageinspect;
+show polar_index_bulk_extend_size;
+ polar_index_bulk_extend_size 
+------------------------------
+ 1MB
+(1 row)
+
+drop table if exists p;
+create table p(a int, b varchar, c numeric,d int8);
+-- INSERT DATA
+insert into p select
+i,
+md5(i::text),
+i / 982.0,
+i * -1
+from
+generate_series(0,100000 - 1)i;
+-- INIT INDEX
+create index p_c_d_idx on p(c,d);
+-- DML
+insert into p select
+i,
+i::text || 'sdha&$#*&',
+i / 160.0,
+i / 32
+from
+generate_series(0,100000 - 1)i;
+-- For index bulk extend expansion
+select pg_relation_size('p_c_d_idx') < 3 * pg_relation_size('p');
+ ?column? 
+----------
+ t
+(1 row)
+
+-- For index bulk extend core
+select count(*) from generate_series(1, pg_relation_size('p_c_d_idx') / current_setting('block_size')::bigint - 1) AS blkno,bt_page_items('p_c_d_idx', blkno);
+ count  
+--------
+ 202716
+(1 row)
+
+drop schema test_index_bulk_extend cascade;

contrib/pageinspect/sql/gist.sql

@@ -1,14 +1,6 @@
--- The gist_page_opaque_info() function prints the page's LSN. Normally,
--- that's constant 1 (GistBuildLSN) on every page of a freshly built GiST
--- index. But with wal_level=minimal, the whole relation is dumped to WAL at
--- the end of the transaction if it's smaller than wal_skip_threshold, which
--- updates the LSNs. Wrap the tests on gist_page_opaque_info() in the
--- same transaction with the CREATE INDEX so that we see the LSNs before
--- they are possibly overwritten at end of transaction.
-BEGIN;
-
--- Create a test table and GiST index.
-CREATE TABLE test_gist AS SELECT point(i,i) p, i::text t FROM
+-- The gist_page_opaque_info() function prints the page's LSN.
+-- Use an unlogged index, so that the LSN is predictable.
+CREATE UNLOGGED TABLE test_gist AS SELECT point(i,i) p, i::text t FROM
     generate_series(1,1000) i;
 CREATE INDEX test_gist_idx ON test_gist USING gist (p);
 
@@ -17,8 +9,6 @@ SELECT * FROM gist_page_opaque_info(get_raw_page('test_gist_idx', 0));
 SELECT * FROM gist_page_opaque_info(get_raw_page('test_gist_idx', 1));
 SELECT * FROM gist_page_opaque_info(get_raw_page('test_gist_idx', 2));
 
-COMMIT;
-
 SELECT * FROM gist_page_items(get_raw_page('test_gist_idx', 0), 'test_gist_idx');
 SELECT * FROM gist_page_items(get_raw_page('test_gist_idx', 1), 'test_gist_idx') LIMIT 5;
 

contrib/pageinspect/sql/index_bulk_extend.sql

@@ -0,0 +1,38 @@
+create schema test_index_bulk_extend;
+set search_path to test_index_bulk_extend;
+set client_min_messages to error; 
+
+create extension if not exists pageinspect;
+
+show polar_index_bulk_extend_size;
+
+drop table if exists p;
+create table p(a int, b varchar, c numeric,d int8);
+
+
+-- INSERT DATA
+insert into p select
+i,
+md5(i::text),
+i / 982.0,
+i * -1
+from
+generate_series(0,100000 - 1)i;
+-- INIT INDEX
+create index p_c_d_idx on p(c,d);
+
+-- DML
+insert into p select
+i,
+i::text || 'sdha&$#*&',
+i / 160.0,
+i / 32
+from
+generate_series(0,100000 - 1)i;
+
+-- For index bulk extend expansion
+select pg_relation_size('p_c_d_idx') < 3 * pg_relation_size('p');
+-- For index bulk extend core
+select count(*) from generate_series(1, pg_relation_size('p_c_d_idx') / current_setting('block_size')::bigint - 1) AS blkno,bt_page_items('p_c_d_idx', blkno);
+
+drop schema test_index_bulk_extend cascade;

contrib/pg_freespacemap/expected/pg_freespacemap.out

@@ -54,17 +54,12 @@ WITH rel AS (SELECT oid::regclass AS id FROM pg_class WHERE relname ~ 'freespace
 
 DELETE FROM freespace_tab;
 VACUUM freespace_tab;
--- In bulk extend, we will pre-extend pages.
--- And these pages will not be expected to vacuum truncated to avoid
--- repeating bulk extenion and truncating.
--- So the relation will exist in free space map.
 WITH rel AS (SELECT oid::regclass AS id FROM pg_class WHERE relname ~ 'freespace')
   SELECT rel.id, fsm.blkno, (fsm.avail > 0) AS is_avail
     FROM rel, LATERAL pg_freespace(rel.id) AS fsm
     ORDER BY 1, 2;
        id        | blkno | is_avail 
 -----------------+-------+----------
- freespace_tab   |     0 | t
  freespace_brin  |     0 | f
  freespace_brin  |     1 | f
  freespace_brin  |     2 | t
@@ -80,7 +75,7 @@ WITH rel AS (SELECT oid::regclass AS id FROM pg_class WHERE relname ~ 'freespace
  freespace_hash  |     7 | f
  freespace_hash  |     8 | f
  freespace_hash  |     9 | f
-(16 rows)
+(15 rows)
 
 -- failures with incorrect block number
 SELECT * FROM pg_freespace('freespace_tab', -1);

contrib/pg_freespacemap/sql/pg_freespacemap.sql

@@ -20,10 +20,6 @@ WITH rel AS (SELECT oid::regclass AS id FROM pg_class WHERE relname ~ 'freespace
 
 DELETE FROM freespace_tab;
 VACUUM freespace_tab;
--- In bulk extend, we will pre-extend pages.
--- And these pages will not be expected to vacuum truncated to avoid
--- repeating bulk extenion and truncating.
--- So the relation will exist in free space map.
 WITH rel AS (SELECT oid::regclass AS id FROM pg_class WHERE relname ~ 'freespace')
   SELECT rel.id, fsm.blkno, (fsm.avail > 0) AS is_avail
     FROM rel, LATERAL pg_freespace(rel.id) AS fsm

contrib/pg_prewarm/pg_prewarm.c

@@ -36,7 +36,7 @@ typedef enum
 	PREWARM_BUFFER
 } PrewarmType;
 
-static PGAlignedBlock blockbuffer;
+static PGIOAlignedBlock blockbuffer;
 
 /*
  * pg_prewarm(regclass, mode text, fork text,

external/Makefile

@@ -4,7 +4,7 @@ subdir = external
 top_builddir = ..
 include $(top_builddir)/src/Makefile.global
 
-SUBDIRS = 
+SUBDIRS =
 
 # multi-arch/0-dependency/fast-compile extensions can be added here
 # sort extention by names, less git conflict
@@ -15,6 +15,7 @@ SUBDIRS += polar_monitor_preload
 SUBDIRS += polar_parameter_manager
 SUBDIRS += polar_proxy_utils
 SUBDIRS += polar_resource_manager
+SUBDIRS += polar_smgrperf
 SUBDIRS += polar_stat_env
 SUBDIRS += polar_worker
 

external/polar_monitor/polar_monitor--1.0.sql

@@ -674,32 +674,6 @@ LANGUAGE C PARALLEL SAFE;
 
 REVOKE ALL ON FUNCTION polar_xlog_buffer_stat_reset() FROM PUBLIC;
 
-/* Per Index */
-CREATE FUNCTION polar_pg_stat_get_bulk_create_index_extend_times(
-    IN oid,
-    OUT int8
-)
-AS 'MODULE_PATHNAME', 'polar_pg_stat_get_bulk_create_index_extend_times'
-LANGUAGE C PARALLEL SAFE;
-
--- Create View for create index extend stats
-CREATE VIEW polar_pg_stat_all_index_extend_stats AS
-    SELECT
-        C.oid AS relid,
-        N.nspname AS schemaname,
-        C.relname AS relname,
-        polar_pg_stat_get_bulk_create_index_extend_times(C.oid) AS idx_create_extend_times
-    FROM pg_class C LEFT JOIN
-            pg_index I ON C.oid = I.indrelid LEFT JOIN
-            pg_class T ON C.reltoastrelid = T.oid LEFT JOIN
-            pg_index X ON T.oid = X.indrelid
-            LEFT JOIN pg_namespace N ON (N.oid = C.relnamespace)
-    WHERE C.relkind IN ('r', 't', 'm')
-    GROUP BY C.oid, N.nspname, C.relname, T.oid, X.indrelid;
-
-REVOKE ALL ON FUNCTION polar_pg_stat_get_bulk_create_index_extend_times(IN oid,OUT int8) FROM PUBLIC;
-/* POLAR end */
-
 CREATE FUNCTION polar_get_slot_node_type(slot_name text)
 RETURNS text
 AS 'MODULE_PATHNAME', 'polar_get_slot_node_type'

external/polar_monitor/polar_monitor.c

@@ -220,24 +220,6 @@ polar_pg_stat_get_bulk_read_blocks_IO(PG_FUNCTION_ARGS)
 	PG_RETURN_INT64(result);
 }
 
-/* POLAR: Bulk create index extend stats */
-/* Per table (or index) */
-PG_FUNCTION_INFO_V1(polar_pg_stat_get_bulk_create_index_extend_times);
-Datum
-polar_pg_stat_get_bulk_create_index_extend_times(PG_FUNCTION_ARGS)
-{
-	Oid			relid = PG_GETARG_OID(0);
-	int64		result;
-	PgStat_StatTabEntry *tabentry;
-
-	if ((tabentry = pgstat_fetch_stat_tabentry(relid)) == NULL)
-		result = 0;
-	else
-		result = (int64) (tabentry->polar_bulk_create_index_extends_times);
-
-	PG_RETURN_INT64(result);
-}
-
 PG_FUNCTION_INFO_V1(polar_get_slot_node_type);
 Datum
 polar_get_slot_node_type(PG_FUNCTION_ARGS)

external/polar_smgrperf/.gitignore

@@ -0,0 +1,4 @@
+# Generated subdirectories
+/log/
+/results/
+/tmp_check/

external/polar_smgrperf/Makefile

@@ -0,0 +1,21 @@
+# external/polar_smgrperf/Makefile
+
+MODULE_big = polar_smgrperf
+OBJS = polar_smgrperf.o $(WIN32RES)
+
+EXTENSION = polar_smgrperf
+DATA = polar_smgrperf--1.0.sql
+PGFILEDESC = "polar_smgrperf - perf test on smgr"
+
+TAP_TESTS = 1
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = external/polar_smgrperf
+top_builddir = ../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif

external/polar_smgrperf/polar_smgrperf--1.0.sql

@@ -0,0 +1,45 @@
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "CREATE EXTENSION polar_smgrperf" to load this file. \quit
+
+CREATE FUNCTION polar_smgrperf_prepare(
+    nblocks INT DEFAULT 131072)
+RETURNS VOID
+AS 'MODULE_PATHNAME'
+LANGUAGE C STRICT;
+
+CREATE FUNCTION polar_smgrperf_cleanup()
+RETURNS VOID
+AS 'MODULE_PATHNAME'
+LANGUAGE C STRICT;
+
+CREATE FUNCTION polar_smgrperf_read(
+    bs INT DEFAULT 1,
+    begin_blkno INT DEFAULT 0,
+    end_blkno INT DEFAULT 131072,
+    sequential BOOLEAN DEFAULT TRUE)
+RETURNS VOID
+AS 'MODULE_PATHNAME'
+LANGUAGE C STRICT;
+
+CREATE FUNCTION polar_smgrperf_write(
+    bs INT DEFAULT 1,
+    begin_blkno INT DEFAULT 0,
+    end_blkno INT DEFAULT 131072,
+    sequential BOOLEAN DEFAULT TRUE)
+RETURNS VOID
+AS 'MODULE_PATHNAME'
+LANGUAGE C STRICT;
+
+CREATE FUNCTION polar_smgrperf_extend(
+    bs INT DEFAULT 1)
+RETURNS VOID
+AS 'MODULE_PATHNAME'
+LANGUAGE C STRICT;
+
+CREATE FUNCTION polar_smgrperf_nblocks(
+    relnumber OID DEFAULT 1,
+    nblocks_cached BOOLEAN DEFAULT FALSE,
+    fd_cached BOOLEAN DEFAULT TRUE
+) RETURNS VOID
+AS 'MODULE_PATHNAME'
+LANGUAGE C STRICT;