From ef95fe228e0d797d91f26e6620f69c3820e934a3 Mon Sep 17 00:00:00 2001 From: Domino Valdano <2644901+reductionista@users.noreply.github.com> Date: Fri, 23 Feb 2024 11:28:16 -0800 Subject: [PATCH] Update to postgresql 12, add note about size of multicolumn index --- core/chains/evm/logpoller/orm.go | 2 +- ...g_poller_filters_add_topics_logs_per_block.sql | 15 ++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/core/chains/evm/logpoller/orm.go b/core/chains/evm/logpoller/orm.go index a6bc173db23..b190fa1be09 100644 --- a/core/chains/evm/logpoller/orm.go +++ b/core/chains/evm/logpoller/orm.go @@ -128,7 +128,7 @@ func (o *DbORM) InsertFilter(filter Filter, qopts ...pg.QOpt) (err error) { (SELECT unnest(:address_array ::::BYTEA[]) addr) a, (SELECT unnest(:event_sig_array ::::BYTEA[]) ev) e %s - ON CONFLICT (evm.f_log_filter_row_id(name, evm_chain_id, address, event, topic2, topic3, topic4)) + ON CONFLICT (evm.f_log_poller_filter_hash(name, evm_chain_id, address, event, topic2, topic3, topic4)) DO UPDATE SET retention=:retention ::::BIGINT, max_logs_kept=:max_logs_kept ::::NUMERIC, logs_per_block=:logs_per_block ::::NUMERIC`, topicsColumns.String(), topicsSql.String()) diff --git a/core/store/migrate/migrations/0225_log_poller_filters_add_topics_logs_per_block.sql b/core/store/migrate/migrations/0225_log_poller_filters_add_topics_logs_per_block.sql index fb3f2bb46dc..90c6d4d0451 100644 --- a/core/store/migrate/migrations/0225_log_poller_filters_add_topics_logs_per_block.sql +++ b/core/store/migrate/migrations/0225_log_poller_filters_add_topics_logs_per_block.sql @@ -1,11 +1,12 @@ -- +goose Up -- This generates a unique BIGINT for the log_poller_filters table from hashing (name, evm_chain_id, address, event, topic2, topic3, topic4). --- An ordinary UNIQUE CONSTRAINT can't work for this because the topics can be NULL. Any row with any column being NULL automatically satisfies the unique constraint (NULL != NULL) --- There are simpler ways of doing this in postgres 12, 13, 14, and especially 15. But for now, we still officially support postgresql 11 and this should be just as efficient. -CREATE OR REPLACE FUNCTION evm.f_log_filter_row_id(name TEXT, evm_chain_id NUMERIC, address BYTEA, event BYTEA, topic2 BYTEA, topic3 BYTEA, topic4 BYTEA) +-- Using an ordinary multi-column index on 7 columns would require a lot of extra storage space, and there are additional complications due to the topics being allowed to be NULL. +-- Note for updating this if and when we drop support for postgresql 12 & 13: hashrecordextended() can be used directly in postgresql 14, avoiding the need for a helper function. +-- The helper function is necessary only for the IMMUTABLE keyword +CREATE OR REPLACE FUNCTION evm.f_log_poller_filter_hash(name TEXT, evm_chain_id NUMERIC, address BYTEA, event BYTEA, topic2 BYTEA, topic3 BYTEA, topic4 BYTEA) RETURNS BIGINT - LANGUAGE SQL IMMUTABLE COST 25 PARALLEL SAFE AS 'SELECT 2^32 * hashtext(textin(record_out(($1,$3,$5,$7)))) + hashtext(textin(record_out(($2, $4, $6))))'; + LANGUAGE SQL IMMUTABLE COST 25 PARALLEL SAFE AS 'SELECT hashtextextended(textin(record_out(($1,$2,$3,$4,$5,$6,$7))), 0)'; ALTER TABLE evm.log_poller_filters ADD COLUMN topic2 BYTEA CHECK (octet_length(topic2) = 32), @@ -14,7 +15,7 @@ ALTER TABLE evm.log_poller_filters ADD COLUMN max_logs_kept BIGINT, ADD COLUMN logs_per_block BIGINT; -CREATE UNIQUE INDEX evm_log_poller_filters_name_chain_address_event_topics_key ON evm.log_poller_filters (evm.f_log_filter_row_id(name, evm_chain_id, address, event, topic2, topic3, topic4)); +CREATE UNIQUE INDEX log_poller_filters_hash_key ON evm.log_poller_filters (evm.f_log_poller_filter_hash(name, evm_chain_id, address, event, topic2, topic3, topic4)); ALTER TABLE evm.log_poller_filters DROP CONSTRAINT evm_log_poller_filters_name_evm_chain_id_address_event_key; @@ -23,7 +24,7 @@ ALTER TABLE evm.log_poller_filters ALTER TABLE evm.log_poller_filters ADD CONSTRAINT evm_log_poller_filters_name_evm_chain_id_address_event_key UNIQUE (name, evm_chain_id, address, event); -DROP INDEX IF EXISTS evm_log_poller_filters_name_chain_address_event_topics_key; +DROP INDEX IF EXISTS log_poller_filters_hash_key; ALTER TABLE evm.log_poller_filters DROP COLUMN topic2, @@ -32,4 +33,4 @@ ALTER TABLE evm.log_poller_filters DROP COLUMN max_logs_kept, DROP COLUMN logs_per_block; -DROP FUNCTION IF EXISTS evm.f_log_filter_row_id(TEXT, NUMERIC, BYTEA, BYTEA, BYTEA, BYTEA, BYTEA); +DROP FUNCTION IF EXISTS evm.f_log_poller_filter_hash(TEXT, NUMERIC, BYTEA, BYTEA, BYTEA, BYTEA, BYTEA);