From a147fc1d01ec188f0fb5f9104df40ed88b0b6bb8 Mon Sep 17 00:00:00 2001 From: RdLrT <3169068+rdlrt@users.noreply.github.com> Date: Mon, 27 Nov 2023 15:03:08 +1100 Subject: [PATCH] Move dbscripts SQL to koios-artifacts repo, and prep v1.1.0 (#1708) ## Description - [x] Move /scripts/gnode-helper-scripts/dbscripts folder to koios-artifacts repo (v1.1.0 tag), as it only contains SQL files. Accordingly update setup-grest.sh references. - [x] Some left-over hard-coded entries (other than documentation ones) for `cardano-community` replaced by `${G_ACCOUNT}` - [x] Update dbsync.sh to also pre-fill conway genesis hash - [x] Install cron job for asset_txo_cache_update - [x] Bump setup-grest.sh tag reference to v1.1.0 ## How has this been tested? Tested on v1 instance --- scripts/cnode-helper-scripts/dbsync.sh | 6 +- scripts/cnode-helper-scripts/env | 1 - scripts/cnode-helper-scripts/gLiveView.sh | 2 +- scripts/cnode-helper-scripts/guild-deploy.sh | 4 +- .../db-scripts/basics.sql | 173 ------------------ .../db-scripts/genesis_table.sql | 18 -- .../db-scripts/reset_grest.sql | 21 --- scripts/grest-helper-scripts/setup-grest.sh | 11 +- 8 files changed, 15 insertions(+), 221 deletions(-) delete mode 100644 scripts/grest-helper-scripts/db-scripts/basics.sql delete mode 100644 scripts/grest-helper-scripts/db-scripts/genesis_table.sql delete mode 100644 scripts/grest-helper-scripts/db-scripts/reset_grest.sql diff --git a/scripts/cnode-helper-scripts/dbsync.sh b/scripts/cnode-helper-scripts/dbsync.sh index 8ca762472..db18ee067 100755 --- a/scripts/cnode-helper-scripts/dbsync.sh +++ b/scripts/cnode-helper-scripts/dbsync.sh @@ -70,10 +70,12 @@ check_config_sanity() { SHGENHASHCFG=$(jq '.ShelleyGenesisHash' <"${CONFIG}" 2>/dev/null) ALGENHASH=$("${CCLI}" genesis hash --genesis "${ALONZO_GENESIS_JSON}" 2>/dev/null) ALGENHASHCFG=$(jq '.AlonzoGenesisHash' <"${CONFIG}" 2>/dev/null) + CWGENHASH=$("${CCLI}" genesis hash --genesis "${CONWAY_GENESIS_JSON}" 2>/dev/null) + CWGENHASHCFG=$(jq '.ConwayGenesisHash' <"${CONFIG}" 2>/dev/null) # If hash are missing/do not match, add that to the end of config. We could have sorted it based on logic, but that would mess up sdiff comparison outputs - if [[ "${BYGENHASH}" != "${BYGENHASHCFG}" ]] || [[ "${SHGENHASH}" != "${SHGENHASHCFG}" ]] || [[ "${ALGENHASH}" != "${ALGENHASHCFG}" ]]; then + if [[ "${BYGENHASH}" != "${BYGENHASHCFG}" ]] || [[ "${SHGENHASH}" != "${SHGENHASHCFG}" ]] || [[ "${ALGENHASH}" != "${ALGENHASHCFG}" ]] || [[ "${CWGENHASH}" != "${CWGENHASHCFG}" ]]; then cp "${CONFIG}" "${CONFIG}".tmp - jq --arg BYGENHASH ${BYGENHASH} --arg SHGENHASH ${SHGENHASH} --arg ALGENHASH ${ALGENHASH} '.ByronGenesisHash = $BYGENHASH | .ShelleyGenesisHash = $SHGENHASH | .AlonzoGenesisHash = $ALGENHASH' <"${CONFIG}" >"${CONFIG}".tmp + jq --arg BYGENHASH ${BYGENHASH} --arg SHGENHASH ${SHGENHASH} --arg ALGENHASH ${ALGENHASH} --arg CWGENHASH ${CWGENHASH} '.ByronGenesisHash = $BYGENHASH | .ShelleyGenesisHash = $SHGENHASH | .AlonzoGenesisHash = $ALGENHASH | .ConwayGenesisHash = $CWGENHASH' <"${CONFIG}" >"${CONFIG}".tmp [[ -s "${CONFIG}".tmp ]] && mv -f "${CONFIG}".tmp "${CONFIG}" fi } diff --git a/scripts/cnode-helper-scripts/env b/scripts/cnode-helper-scripts/env index 5701fa085..1304f5f24 100644 --- a/scripts/cnode-helper-scripts/env +++ b/scripts/cnode-helper-scripts/env @@ -781,7 +781,6 @@ set_default_vars() { [[ -z ${G_ACCOUNT} ]] && G_ACCOUNT="cardano-community" URL_RAW="https://raw.githubusercontent.com/${G_ACCOUNT}/guild-operators/${BRANCH}" - DB_SCRIPTS_URL="${URL_RAW}/scripts/grest-helper-scripts/db-scripts" export LC_ALL=C.UTF-8 # special mapping of coreutils gdate to date for MacOS if [[ $(uname) == Darwin ]]; then diff --git a/scripts/cnode-helper-scripts/gLiveView.sh b/scripts/cnode-helper-scripts/gLiveView.sh index 9a9f03665..4b5927830 100755 --- a/scripts/cnode-helper-scripts/gLiveView.sh +++ b/scripts/cnode-helper-scripts/gLiveView.sh @@ -125,7 +125,7 @@ if [[ ${UPDATE_CHECK} = Y && ${SKIP_UPDATE} != Y ]]; then if command -v cncli >/dev/null && command -v systemctl >/dev/null && systemctl is-active --quiet ${CNODE_VNAME}-cncli-sync.service; then vcur=$(cncli -V | sed 's/cncli /v/g') - vrem=$(curl -s https://api.github.com/repos/cardano-community/cncli/releases/latest | jq -r .tag_name) + vrem=$(curl -s https://api.github.com/repos/${G_ACCOUNT}/cncli/releases/latest | jq -r .tag_name) [[ ${vcur} != ${vrem} ]] && printf "${FG_MAGENTA}CNCLI current version (${vcur}) different from repo (${vrem}), consider upgrading!.${NC}" && waitToProceed fi diff --git a/scripts/cnode-helper-scripts/guild-deploy.sh b/scripts/cnode-helper-scripts/guild-deploy.sh index 63f770109..bb2153f40 100755 --- a/scripts/cnode-helper-scripts/guild-deploy.sh +++ b/scripts/cnode-helper-scripts/guild-deploy.sh @@ -356,11 +356,11 @@ download_cncli() { [[ -z ${ARCH##*aarch64*} ]] && err_exit " The cncli pre-compiled binary is not available for ARM, you might need to build them!" echo -e "\nInstalling CNCLI.." if command -v cncli >/dev/null; then cncli_version="v$(cncli -V 2>/dev/null | cut -d' ' -f2)"; else cncli_version="v0.0.0"; fi - cncli_git_version="$(curl -s https://api.github.com/repos/cardano-community/cncli/releases/latest | jq -r '.tag_name')" + cncli_git_version="$(curl -s https://api.github.com/repos/${G_ACCOUNT}/cncli/releases/latest | jq -r '.tag_name')" echo -e "\n Downloading CNCLI..." rm -rf /tmp/cncli-bin && mkdir /tmp/cncli-bin pushd /tmp/cncli-bin >/dev/null || err_exit - cncli_asset_url="$(curl -s https://api.github.com/repos/cardano-community/cncli/releases/latest | jq -r '.assets[].browser_download_url' | grep 'linux-musl.tar.gz')" + cncli_asset_url="$(curl -s https://api.github.com/repos/${G_ACCOUNT}/cncli/releases/latest | jq -r '.assets[].browser_download_url' | grep 'linux-musl.tar.gz')" if curl -sL -f -m ${CURL_TIMEOUT} -o cncli.tar.gz ${cncli_asset_url}; then tar zxf cncli.tar.gz &>/dev/null rm -f cncli.tar.gz diff --git a/scripts/grest-helper-scripts/db-scripts/basics.sql b/scripts/grest-helper-scripts/db-scripts/basics.sql deleted file mode 100644 index 641b05d43..000000000 --- a/scripts/grest-helper-scripts/db-scripts/basics.sql +++ /dev/null @@ -1,173 +0,0 @@ --------------------------------------------------------------------------------- --- Entry point for Koios node DB setup: --- 1) grest schema that will hold all RPC functions/views and cached tables --- 2) web_anon user --- 3) grest.control_table --- 4) grest.genesis --- 5) drop existing functions --- 6) helper functions --- 7) optional db indexes on important public tables --------------------------------------------------------------------------------- --- GREST SCHEMA -- -CREATE SCHEMA IF NOT EXISTS grest; - --- WEB_ANON USER -- -DO $$ -BEGIN - CREATE ROLE web_anon nologin; -EXCEPTION - WHEN DUPLICATE_OBJECT THEN - RAISE NOTICE 'web_anon exists, skipping...'; -END -$$; - -DO $$ -BEGIN - CREATE ROLE authenticator LOGIN; - EXECUTE FORMAT('GRANT CONNECT ON DATABASE %I to authenticator', current_database()); -EXCEPTION - WHEN DUPLICATE_OBJECT THEN - RAISE NOTICE 'authenticator exists, skipping...'; -END; -$$; - -GRANT USAGE ON SCHEMA public TO authenticator,web_anon; -GRANT USAGE ON SCHEMA grest TO authenticator,web_anon; -GRANT SELECT ON ALL TABLES IN SCHEMA public TO authenticator,web_anon; -GRANT SELECT ON ALL TABLES IN SCHEMA grest TO authenticator,web_anon; -GRANT web_anon TO authenticator; -ALTER ROLE authenticator SET statement_timeout = 125000; - -ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT -SELECT - ON TABLES TO web_anon,authenticator; - -ALTER DEFAULT PRIVILEGES IN SCHEMA grest GRANT -SELECT - ON TABLES TO web_anon,authenticator; - -ALTER ROLE web_anon SET search_path TO grest, public; -ALTER ROLE authenticator SET search_path TO grest, public; - --- CONTROL TABLE -- -CREATE TABLE IF NOT EXISTS GREST.CONTROL_TABLE ( - key text PRIMARY KEY, - last_value text NOT NULL, - artifacts text -); - --- GENESIS TABLE -- -DROP TABLE IF EXISTS grest.genesis; - --- Data Types are intentionally kept varchar for single ID row to avoid future edge cases -CREATE TABLE grest.genesis ( - NETWORKMAGIC varchar, - NETWORKID varchar, - ACTIVESLOTCOEFF varchar, - UPDATEQUORUM varchar, - MAXLOVELACESUPPLY varchar, - EPOCHLENGTH varchar, - SYSTEMSTART varchar, - SLOTSPERKESPERIOD varchar, - SLOTLENGTH varchar, - MAXKESREVOLUTIONS varchar, - SECURITYPARAM varchar, - ALONZOGENESIS varchar -); - --- DROP EXISTING FUNCTIONS -DO -$do$ -DECLARE - _sql text; -BEGIN - SELECT INTO _sql - string_agg( - format( - 'DROP %s %s CASCADE;', - CASE prokind - WHEN 'f' THEN 'FUNCTION' - WHEN 'a' THEN 'AGGREGATE' - WHEN 'p' THEN 'PROCEDURE' - WHEN 'w' THEN 'FUNCTION' -- window function (rarely applicable) - END, - oid::regprocedure - ), - E'\n' - ) - FROM - pg_proc - WHERE - pronamespace = 'grest'::regnamespace -- schema name here - AND prokind = ANY ('{f,a,p,w}'); -- optionally filter kinds - - IF _sql IS NOT NULL THEN - RAISE NOTICE '%', _sql; -- debug - EXECUTE _sql; - ELSE - RAISE NOTICE 'No fuctions found in schema %', quote_ident('grest'); - END IF; -END -$do$; - --- HELPER FUNCTIONS -- -CREATE FUNCTION grest.get_query_pids_partial_match (_query text) - RETURNS TABLE ( - pid integer) - LANGUAGE plpgsql - AS $$ -BEGIN - RETURN QUERY - SELECT - pg_stat_activity.pid - FROM - pg_stat_activity - WHERE - query ILIKE '%' || _query || '%' - AND query NOT ILIKE '%grest.get_query_pids_partial_match%' - AND query NOT ILIKE '%grest.kill_queries_partial_match%' - AND datname = (SELECT current_database()); -END; -$$; - -CREATE PROCEDURE grest.kill_queries_partial_match (_query text) -LANGUAGE plpgsql -AS $$ -DECLARE - _pids integer[]; - _pid integer; -BEGIN - _pids := ARRAY ( - SELECT grest.get_query_pids_partial_match (_query) - ); - FOREACH _pid IN ARRAY _pids - LOOP - RAISE NOTICE 'Cancelling PID: %', _pid; - PERFORM PG_TERMINATE_BACKEND(_pid); - END LOOP; -END; -$$; - -CREATE FUNCTION grest.update_control_table (_key text, _last_value text, _artifacts text default null) - RETURNS void - LANGUAGE plpgsql - AS -$$ - BEGIN - INSERT INTO - GREST.CONTROL_TABLE (key, last_value, artifacts) - VALUES - (_key, _last_value, _artifacts) - ON CONFLICT ( - key - ) DO UPDATE - SET - last_value = _last_value, - artifacts = _artifacts; - END; -$$; - --- Refresh asset token registry cache from github, to avoid stale deletes -DELETE FROM grest.control_table WHERE key='asset_registry_commit'; --- DATABASE INDEXES -- --- Empty diff --git a/scripts/grest-helper-scripts/db-scripts/genesis_table.sql b/scripts/grest-helper-scripts/db-scripts/genesis_table.sql deleted file mode 100644 index 7e4bb6df2..000000000 --- a/scripts/grest-helper-scripts/db-scripts/genesis_table.sql +++ /dev/null @@ -1,18 +0,0 @@ -DROP TABLE IF EXISTS grest.genesis; - --- Data Types are intentionally kept varchar for single ID row to avoid future edge cases -CREATE TABLE grest.genesis ( - NETWORKMAGIC varchar, - NETWORKID varchar, - ACTIVESLOTCOEFF varchar, - UPDATEQUORUM varchar, - MAXLOVELACESUPPLY varchar, - EPOCHLENGTH varchar, - SYSTEMSTART varchar, - SLOTSPERKESPERIOD varchar, - SLOTLENGTH varchar, - MAXKESREVOLUTIONS varchar, - SECURITYPARAM varchar, - ALONZOGENESIS varchar -); - diff --git a/scripts/grest-helper-scripts/db-scripts/reset_grest.sql b/scripts/grest-helper-scripts/db-scripts/reset_grest.sql deleted file mode 100644 index a022ef6e5..000000000 --- a/scripts/grest-helper-scripts/db-scripts/reset_grest.sql +++ /dev/null @@ -1,21 +0,0 @@ --- Drop triggers first that depend on grest.functions() -SELECT - 'DROP TRIGGER ' || trigger_name || ' ON ' || event_object_table || ';' -FROM - information_schema.triggers -WHERE - trigger_schema = 'public'; - --- Recreate grest schema -DROP SCHEMA IF EXISTS grest CASCADE; - -CREATE SCHEMA grest; - -GRANT USAGE ON SCHEMA grest TO authenticator,web_anon; - -GRANT SELECT ON ALL TABLES IN SCHEMA grest TO authenticator,web_anon; - -ALTER DEFAULT PRIVILEGES IN SCHEMA grest GRANT -SELECT - ON TABLES TO authenticator,web_anon; - diff --git a/scripts/grest-helper-scripts/setup-grest.sh b/scripts/grest-helper-scripts/setup-grest.sh index 4261e4d03..cc3420a33 100755 --- a/scripts/grest-helper-scripts/setup-grest.sh +++ b/scripts/grest-helper-scripts/setup-grest.sh @@ -16,7 +16,7 @@ # Do NOT modify code below # ###################################### -SGVERSION=v1.1.0rc +SGVERSION=v1.1.0 ######## Functions ######## usage() { @@ -91,7 +91,7 @@ SGVERSION=v1.1.0rc get_cron_job_executable() { local job=$1 local job_path="${CRON_SCRIPTS_DIR}/${job}.sh" - local job_url="https://raw.githubusercontent.com/cardano-community/koios-artifacts/${SGVERSION}/files/grest/cron/jobs/${job}.sh" + local job_url="https://raw.githubusercontent.com/${G_ACCOUNT}/koios-artifacts/${SGVERSION}/files/grest/cron/jobs/${job}.sh" if curl -s -f -m "${CURL_TIMEOUT}" -o "${job_path}" "${job_url}"; then printf "\n Downloaded \e[32m${job_path}\e[0m" chmod +x "${job_path}" @@ -172,7 +172,11 @@ SGVERSION=v1.1.0rc get_cron_job_executable "asset-info-cache-update" set_cron_variables "asset-info-cache-update" - install_cron_job "asset-info-cache-update" "* * * * *" + install_cron_job "asset-info-cache-update" "*/2 * * * *" + + get_cron_job_executable "asset-txo-cache-update" + set_cron_variables "asset-txo-cache-update" + install_cron_job "asset-txo-cache-update" "*/5 * * * *" get_cron_job_executable "epoch-summary-corrections-update" set_cron_variables "epoch-summary-corrections-update" @@ -205,6 +209,7 @@ SGVERSION=v1.1.0rc [[ -z "${CRON_DIR}" ]] && CRON_DIR="/etc/cron.d" [[ -z "${PGDATABASE}" ]] && PGDATABASE="cexplorer" [[ -z "${HAPROXY_CFG}" ]] && HAPROXY_CFG="${CNODE_HOME}/files/haproxy.cfg" + [[ -z "${DB_SCRIPTS_URL}" ]] && DB_SCRIPTS_URL="https://raw.githubusercontent.com/${G_ACCOUNT}/koios-artifacts/${SGVERSION}/files/grest/rpc/db-scripts" DOCS_URL="https://cardano-community.github.io/guild-operators" [[ -z "${PGPASSFILE}" ]] && export PGPASSFILE="${CNODE_HOME}"/priv/.pgpass case ${NWMAGIC} in