From 916af3b8c9ef5c107128e0b95785beee10e16261 Mon Sep 17 00:00:00 2001 From: RdLrT <3169068+rdlrt@users.noreply.github.com> Date: Mon, 27 Nov 2023 15:03:08 +1100 Subject: [PATCH] Move dbscripts SQL to koios-artifacts repo, and prep v1.1.0 (#1708) ## Description - [x] Move /scripts/gnode-helper-scripts/dbscripts folder to koios-artifacts repo (v1.1.0 tag), as it only contains SQL files. Accordingly update setup-grest.sh references. - [x] Some left-over hard-coded entries (other than documentation ones) for `cardano-community` replaced by `${G_ACCOUNT}` - [x] Update dbsync.sh to also pre-fill conway genesis hash - [x] Install cron job for asset_txo_cache_update - [x] Bump setup-grest.sh tag reference to v1.1.0 ## How has this been tested? Tested on v1 instance --- docs/Scripts/gliveview.md | 22 +-- scripts/cnode-helper-scripts/dbsync.sh | 6 +- scripts/cnode-helper-scripts/env | 1 - scripts/cnode-helper-scripts/gLiveView.sh | 2 +- scripts/cnode-helper-scripts/guild-deploy.sh | 4 +- .../db-scripts/basics.sql | 173 ------------------ .../db-scripts/genesis_table.sql | 18 -- .../db-scripts/reset_grest.sql | 21 --- scripts/grest-helper-scripts/setup-grest.sh | 11 +- 9 files changed, 17 insertions(+), 241 deletions(-) delete mode 100644 scripts/grest-helper-scripts/db-scripts/basics.sql delete mode 100644 scripts/grest-helper-scripts/db-scripts/genesis_table.sql delete mode 100644 scripts/grest-helper-scripts/db-scripts/reset_grest.sql diff --git a/docs/Scripts/gliveview.md b/docs/Scripts/gliveview.md index 7ba9da737..56eb8ead2 100644 --- a/docs/Scripts/gliveview.md +++ b/docs/Scripts/gliveview.md @@ -3,27 +3,9 @@ **Koios gLiveView** is a local monitoring tool to use in addition to remote monitoring tools like Prometheus/Grafana, Zabbix or IOG's RTView. This is especially useful when moving to a systemd deployment - if you haven't done so already - as it offers an intuitive UI to monitor the node status. -The tool is independent from other files and can run as a standalone utility that can be stopped/started without affecting the status of `cardano-node`. - -##### Download - -If you've used [guild-deploy.sh](../basics.md#pre-requisites), you can skip this part, as this is already set up for you. The tool relies on the common `env` configuration file. -To get current epoch blocks, the [logMonitor.sh](../Scripts/logmonitor.md) script is needed (and can be combined with [CNCLI](../Scripts/cncli.md)). This is optional and **Koios gLiveView** will function without it. - -!!! info "Note" - For those who follow the [folder structure](../basics.md#folder-structure) in this repo and do not wish to run `guild-deploy.sh`, you can run the below in `$CNODE_HOME/scripts` folder - -To download the script: - -```bash -curl -s -o gLiveView.sh https://raw.githubusercontent.com/cardano-community/guild-operators/master/scripts/cnode-helper-scripts/gLiveView.sh -curl -s -o env https://raw.githubusercontent.com/cardano-community/guild-operators/master/scripts/cnode-helper-scripts/env -chmod 755 gLiveView.sh -``` - ##### Configuration & Startup -For most setups, it's enough to set `CNODE_PORT` in the `env` file. The rest of the variables should automatically be detected. If required, modify User Variables in `env` and `gLiveView.sh` to suit your environment (if folder structure you use is different). This should lead you to a stage where you can now start running `./gLiveView.sh` in the folder you downloaded the script (the default location would be `$CNODE_HOME/scripts`). Note that the script is smart enough to automatically detect when you're running as a Core or Relay and will show fields accordingly. +For most setups, it's enough to set `CNODE_PORT` in the `env` file. The rest of the variables should automatically be detected. If required, modify User Variables in `env` and `gLiveView.sh` to suit your environment (if the environment is customised). This should lead you to a stage where you can now start running `./gLiveView.sh` in the folder you downloaded the script (the default location would be `$CNODE_HOME/scripts`). Note that the script is smart enough to automatically detect when you're running as a Core or Relay and will show fields accordingly. The tool can be run in legacy mode with only standard ASCII characters for terminals with trouble displaying the box-drawing characters. Run `./gLiveView.sh -h` to show available command-line parameters or permanently set it directly in script. @@ -65,7 +47,7 @@ Displays live metrics from cardano-node gathered through the nodes EKG/Prometheu - **Mem (RSS)** - RSS is the Resident Set Size and shows how much memory is allocated to cardano-node and that is in RAM. It does not include memory that is swapped out. It does include memory from shared libraries as long as the pages from those libraries are actually in memory. It does include all stack and heap memory. - **Mem (Live) / (Heap)** - GC (Garbage Collector) values that show how much memory is used for live/heap data. A large difference between them (or the heap approaching the physical memory limit) means the node is struggling with the garbage collector and/or may begin swapping. - **GC Minor / Major** - Collecting garbage from "Young space" is called a Minor GC. Major (Full) GC is done more rarily and is a more expensive operation. Explaining garbage collection is a topic outside the scope of this documentation and google is your friend for this. -- **Block propagation** - Last delay measures the duration between when the last block was scheduled to be produced and when the node learned about it. Late blocks are blocks whose delay is larger than 5s. If the node is not synching, the number of late blocks needs to stay low. Within 1/3/5s estimates the chance of observing a delay of 1/3/5s (based on the delays observed for previous blocks). A healthy node needs to stay above 95% of blocks within 3s. Finally, served blocks counts how many blocks were fetched by "in" peers. If this does not increase for a long time, it means the "in" peers are learning about new blocks from somewhere else (and therefore this node is not contributing towards accelerating the propagation). Overall, these metrics are helpful in tweaking the topology and/or performance of the network links. +- **Block propagation** - Last Block measures the duration between when the last block was scheduled to be produced and when the node learned about it. Late blocks are blocks whose delay is larger than 5s. If the node is not synching, the number of late blocks needs to stay low. Within 1/3/5s estimates the chance of observing a delay of 1/3/5s (based on the delays observed for previous blocks). A healthy node needs to stay above 95% of blocks within 3s. Finally, served blocks counts how many blocks were fetched by "in" peers. If this does not increase for a long time, it means the "in" peers are learning about new blocks from somewhere else (and therefore this node is not contributing towards accelerating the propagation). Overall, these metrics are helpful in tweaking the topology and/or performance of the network links. ###### Core section diff --git a/scripts/cnode-helper-scripts/dbsync.sh b/scripts/cnode-helper-scripts/dbsync.sh index 8ca762472..db18ee067 100755 --- a/scripts/cnode-helper-scripts/dbsync.sh +++ b/scripts/cnode-helper-scripts/dbsync.sh @@ -70,10 +70,12 @@ check_config_sanity() { SHGENHASHCFG=$(jq '.ShelleyGenesisHash' <"${CONFIG}" 2>/dev/null) ALGENHASH=$("${CCLI}" genesis hash --genesis "${ALONZO_GENESIS_JSON}" 2>/dev/null) ALGENHASHCFG=$(jq '.AlonzoGenesisHash' <"${CONFIG}" 2>/dev/null) + CWGENHASH=$("${CCLI}" genesis hash --genesis "${CONWAY_GENESIS_JSON}" 2>/dev/null) + CWGENHASHCFG=$(jq '.ConwayGenesisHash' <"${CONFIG}" 2>/dev/null) # If hash are missing/do not match, add that to the end of config. We could have sorted it based on logic, but that would mess up sdiff comparison outputs - if [[ "${BYGENHASH}" != "${BYGENHASHCFG}" ]] || [[ "${SHGENHASH}" != "${SHGENHASHCFG}" ]] || [[ "${ALGENHASH}" != "${ALGENHASHCFG}" ]]; then + if [[ "${BYGENHASH}" != "${BYGENHASHCFG}" ]] || [[ "${SHGENHASH}" != "${SHGENHASHCFG}" ]] || [[ "${ALGENHASH}" != "${ALGENHASHCFG}" ]] || [[ "${CWGENHASH}" != "${CWGENHASHCFG}" ]]; then cp "${CONFIG}" "${CONFIG}".tmp - jq --arg BYGENHASH ${BYGENHASH} --arg SHGENHASH ${SHGENHASH} --arg ALGENHASH ${ALGENHASH} '.ByronGenesisHash = $BYGENHASH | .ShelleyGenesisHash = $SHGENHASH | .AlonzoGenesisHash = $ALGENHASH' <"${CONFIG}" >"${CONFIG}".tmp + jq --arg BYGENHASH ${BYGENHASH} --arg SHGENHASH ${SHGENHASH} --arg ALGENHASH ${ALGENHASH} --arg CWGENHASH ${CWGENHASH} '.ByronGenesisHash = $BYGENHASH | .ShelleyGenesisHash = $SHGENHASH | .AlonzoGenesisHash = $ALGENHASH | .ConwayGenesisHash = $CWGENHASH' <"${CONFIG}" >"${CONFIG}".tmp [[ -s "${CONFIG}".tmp ]] && mv -f "${CONFIG}".tmp "${CONFIG}" fi } diff --git a/scripts/cnode-helper-scripts/env b/scripts/cnode-helper-scripts/env index 5701fa085..1304f5f24 100644 --- a/scripts/cnode-helper-scripts/env +++ b/scripts/cnode-helper-scripts/env @@ -781,7 +781,6 @@ set_default_vars() { [[ -z ${G_ACCOUNT} ]] && G_ACCOUNT="cardano-community" URL_RAW="https://raw.githubusercontent.com/${G_ACCOUNT}/guild-operators/${BRANCH}" - DB_SCRIPTS_URL="${URL_RAW}/scripts/grest-helper-scripts/db-scripts" export LC_ALL=C.UTF-8 # special mapping of coreutils gdate to date for MacOS if [[ $(uname) == Darwin ]]; then diff --git a/scripts/cnode-helper-scripts/gLiveView.sh b/scripts/cnode-helper-scripts/gLiveView.sh index 9a9f03665..4b5927830 100755 --- a/scripts/cnode-helper-scripts/gLiveView.sh +++ b/scripts/cnode-helper-scripts/gLiveView.sh @@ -125,7 +125,7 @@ if [[ ${UPDATE_CHECK} = Y && ${SKIP_UPDATE} != Y ]]; then if command -v cncli >/dev/null && command -v systemctl >/dev/null && systemctl is-active --quiet ${CNODE_VNAME}-cncli-sync.service; then vcur=$(cncli -V | sed 's/cncli /v/g') - vrem=$(curl -s https://api.github.com/repos/cardano-community/cncli/releases/latest | jq -r .tag_name) + vrem=$(curl -s https://api.github.com/repos/${G_ACCOUNT}/cncli/releases/latest | jq -r .tag_name) [[ ${vcur} != ${vrem} ]] && printf "${FG_MAGENTA}CNCLI current version (${vcur}) different from repo (${vrem}), consider upgrading!.${NC}" && waitToProceed fi diff --git a/scripts/cnode-helper-scripts/guild-deploy.sh b/scripts/cnode-helper-scripts/guild-deploy.sh index 63f770109..bb2153f40 100755 --- a/scripts/cnode-helper-scripts/guild-deploy.sh +++ b/scripts/cnode-helper-scripts/guild-deploy.sh @@ -356,11 +356,11 @@ download_cncli() { [[ -z ${ARCH##*aarch64*} ]] && err_exit " The cncli pre-compiled binary is not available for ARM, you might need to build them!" echo -e "\nInstalling CNCLI.." if command -v cncli >/dev/null; then cncli_version="v$(cncli -V 2>/dev/null | cut -d' ' -f2)"; else cncli_version="v0.0.0"; fi - cncli_git_version="$(curl -s https://api.github.com/repos/cardano-community/cncli/releases/latest | jq -r '.tag_name')" + cncli_git_version="$(curl -s https://api.github.com/repos/${G_ACCOUNT}/cncli/releases/latest | jq -r '.tag_name')" echo -e "\n Downloading CNCLI..." rm -rf /tmp/cncli-bin && mkdir /tmp/cncli-bin pushd /tmp/cncli-bin >/dev/null || err_exit - cncli_asset_url="$(curl -s https://api.github.com/repos/cardano-community/cncli/releases/latest | jq -r '.assets[].browser_download_url' | grep 'linux-musl.tar.gz')" + cncli_asset_url="$(curl -s https://api.github.com/repos/${G_ACCOUNT}/cncli/releases/latest | jq -r '.assets[].browser_download_url' | grep 'linux-musl.tar.gz')" if curl -sL -f -m ${CURL_TIMEOUT} -o cncli.tar.gz ${cncli_asset_url}; then tar zxf cncli.tar.gz &>/dev/null rm -f cncli.tar.gz diff --git a/scripts/grest-helper-scripts/db-scripts/basics.sql b/scripts/grest-helper-scripts/db-scripts/basics.sql deleted file mode 100644 index 641b05d43..000000000 --- a/scripts/grest-helper-scripts/db-scripts/basics.sql +++ /dev/null @@ -1,173 +0,0 @@ --------------------------------------------------------------------------------- --- Entry point for Koios node DB setup: --- 1) grest schema that will hold all RPC functions/views and cached tables --- 2) web_anon user --- 3) grest.control_table --- 4) grest.genesis --- 5) drop existing functions --- 6) helper functions --- 7) optional db indexes on important public tables --------------------------------------------------------------------------------- --- GREST SCHEMA -- -CREATE SCHEMA IF NOT EXISTS grest; - --- WEB_ANON USER -- -DO $$ -BEGIN - CREATE ROLE web_anon nologin; -EXCEPTION - WHEN DUPLICATE_OBJECT THEN - RAISE NOTICE 'web_anon exists, skipping...'; -END -$$; - -DO $$ -BEGIN - CREATE ROLE authenticator LOGIN; - EXECUTE FORMAT('GRANT CONNECT ON DATABASE %I to authenticator', current_database()); -EXCEPTION - WHEN DUPLICATE_OBJECT THEN - RAISE NOTICE 'authenticator exists, skipping...'; -END; -$$; - -GRANT USAGE ON SCHEMA public TO authenticator,web_anon; -GRANT USAGE ON SCHEMA grest TO authenticator,web_anon; -GRANT SELECT ON ALL TABLES IN SCHEMA public TO authenticator,web_anon; -GRANT SELECT ON ALL TABLES IN SCHEMA grest TO authenticator,web_anon; -GRANT web_anon TO authenticator; -ALTER ROLE authenticator SET statement_timeout = 125000; - -ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT -SELECT - ON TABLES TO web_anon,authenticator; - -ALTER DEFAULT PRIVILEGES IN SCHEMA grest GRANT -SELECT - ON TABLES TO web_anon,authenticator; - -ALTER ROLE web_anon SET search_path TO grest, public; -ALTER ROLE authenticator SET search_path TO grest, public; - --- CONTROL TABLE -- -CREATE TABLE IF NOT EXISTS GREST.CONTROL_TABLE ( - key text PRIMARY KEY, - last_value text NOT NULL, - artifacts text -); - --- GENESIS TABLE -- -DROP TABLE IF EXISTS grest.genesis; - --- Data Types are intentionally kept varchar for single ID row to avoid future edge cases -CREATE TABLE grest.genesis ( - NETWORKMAGIC varchar, - NETWORKID varchar, - ACTIVESLOTCOEFF varchar, - UPDATEQUORUM varchar, - MAXLOVELACESUPPLY varchar, - EPOCHLENGTH varchar, - SYSTEMSTART varchar, - SLOTSPERKESPERIOD varchar, - SLOTLENGTH varchar, - MAXKESREVOLUTIONS varchar, - SECURITYPARAM varchar, - ALONZOGENESIS varchar -); - --- DROP EXISTING FUNCTIONS -DO -$do$ -DECLARE - _sql text; -BEGIN - SELECT INTO _sql - string_agg( - format( - 'DROP %s %s CASCADE;', - CASE prokind - WHEN 'f' THEN 'FUNCTION' - WHEN 'a' THEN 'AGGREGATE' - WHEN 'p' THEN 'PROCEDURE' - WHEN 'w' THEN 'FUNCTION' -- window function (rarely applicable) - END, - oid::regprocedure - ), - E'\n' - ) - FROM - pg_proc - WHERE - pronamespace = 'grest'::regnamespace -- schema name here - AND prokind = ANY ('{f,a,p,w}'); -- optionally filter kinds - - IF _sql IS NOT NULL THEN - RAISE NOTICE '%', _sql; -- debug - EXECUTE _sql; - ELSE - RAISE NOTICE 'No fuctions found in schema %', quote_ident('grest'); - END IF; -END -$do$; - --- HELPER FUNCTIONS -- -CREATE FUNCTION grest.get_query_pids_partial_match (_query text) - RETURNS TABLE ( - pid integer) - LANGUAGE plpgsql - AS $$ -BEGIN - RETURN QUERY - SELECT - pg_stat_activity.pid - FROM - pg_stat_activity - WHERE - query ILIKE '%' || _query || '%' - AND query NOT ILIKE '%grest.get_query_pids_partial_match%' - AND query NOT ILIKE '%grest.kill_queries_partial_match%' - AND datname = (SELECT current_database()); -END; -$$; - -CREATE PROCEDURE grest.kill_queries_partial_match (_query text) -LANGUAGE plpgsql -AS $$ -DECLARE - _pids integer[]; - _pid integer; -BEGIN - _pids := ARRAY ( - SELECT grest.get_query_pids_partial_match (_query) - ); - FOREACH _pid IN ARRAY _pids - LOOP - RAISE NOTICE 'Cancelling PID: %', _pid; - PERFORM PG_TERMINATE_BACKEND(_pid); - END LOOP; -END; -$$; - -CREATE FUNCTION grest.update_control_table (_key text, _last_value text, _artifacts text default null) - RETURNS void - LANGUAGE plpgsql - AS -$$ - BEGIN - INSERT INTO - GREST.CONTROL_TABLE (key, last_value, artifacts) - VALUES - (_key, _last_value, _artifacts) - ON CONFLICT ( - key - ) DO UPDATE - SET - last_value = _last_value, - artifacts = _artifacts; - END; -$$; - --- Refresh asset token registry cache from github, to avoid stale deletes -DELETE FROM grest.control_table WHERE key='asset_registry_commit'; --- DATABASE INDEXES -- --- Empty diff --git a/scripts/grest-helper-scripts/db-scripts/genesis_table.sql b/scripts/grest-helper-scripts/db-scripts/genesis_table.sql deleted file mode 100644 index 7e4bb6df2..000000000 --- a/scripts/grest-helper-scripts/db-scripts/genesis_table.sql +++ /dev/null @@ -1,18 +0,0 @@ -DROP TABLE IF EXISTS grest.genesis; - --- Data Types are intentionally kept varchar for single ID row to avoid future edge cases -CREATE TABLE grest.genesis ( - NETWORKMAGIC varchar, - NETWORKID varchar, - ACTIVESLOTCOEFF varchar, - UPDATEQUORUM varchar, - MAXLOVELACESUPPLY varchar, - EPOCHLENGTH varchar, - SYSTEMSTART varchar, - SLOTSPERKESPERIOD varchar, - SLOTLENGTH varchar, - MAXKESREVOLUTIONS varchar, - SECURITYPARAM varchar, - ALONZOGENESIS varchar -); - diff --git a/scripts/grest-helper-scripts/db-scripts/reset_grest.sql b/scripts/grest-helper-scripts/db-scripts/reset_grest.sql deleted file mode 100644 index a022ef6e5..000000000 --- a/scripts/grest-helper-scripts/db-scripts/reset_grest.sql +++ /dev/null @@ -1,21 +0,0 @@ --- Drop triggers first that depend on grest.functions() -SELECT - 'DROP TRIGGER ' || trigger_name || ' ON ' || event_object_table || ';' -FROM - information_schema.triggers -WHERE - trigger_schema = 'public'; - --- Recreate grest schema -DROP SCHEMA IF EXISTS grest CASCADE; - -CREATE SCHEMA grest; - -GRANT USAGE ON SCHEMA grest TO authenticator,web_anon; - -GRANT SELECT ON ALL TABLES IN SCHEMA grest TO authenticator,web_anon; - -ALTER DEFAULT PRIVILEGES IN SCHEMA grest GRANT -SELECT - ON TABLES TO authenticator,web_anon; - diff --git a/scripts/grest-helper-scripts/setup-grest.sh b/scripts/grest-helper-scripts/setup-grest.sh index 4261e4d03..cc3420a33 100755 --- a/scripts/grest-helper-scripts/setup-grest.sh +++ b/scripts/grest-helper-scripts/setup-grest.sh @@ -16,7 +16,7 @@ # Do NOT modify code below # ###################################### -SGVERSION=v1.1.0rc +SGVERSION=v1.1.0 ######## Functions ######## usage() { @@ -91,7 +91,7 @@ SGVERSION=v1.1.0rc get_cron_job_executable() { local job=$1 local job_path="${CRON_SCRIPTS_DIR}/${job}.sh" - local job_url="https://raw.githubusercontent.com/cardano-community/koios-artifacts/${SGVERSION}/files/grest/cron/jobs/${job}.sh" + local job_url="https://raw.githubusercontent.com/${G_ACCOUNT}/koios-artifacts/${SGVERSION}/files/grest/cron/jobs/${job}.sh" if curl -s -f -m "${CURL_TIMEOUT}" -o "${job_path}" "${job_url}"; then printf "\n Downloaded \e[32m${job_path}\e[0m" chmod +x "${job_path}" @@ -172,7 +172,11 @@ SGVERSION=v1.1.0rc get_cron_job_executable "asset-info-cache-update" set_cron_variables "asset-info-cache-update" - install_cron_job "asset-info-cache-update" "* * * * *" + install_cron_job "asset-info-cache-update" "*/2 * * * *" + + get_cron_job_executable "asset-txo-cache-update" + set_cron_variables "asset-txo-cache-update" + install_cron_job "asset-txo-cache-update" "*/5 * * * *" get_cron_job_executable "epoch-summary-corrections-update" set_cron_variables "epoch-summary-corrections-update" @@ -205,6 +209,7 @@ SGVERSION=v1.1.0rc [[ -z "${CRON_DIR}" ]] && CRON_DIR="/etc/cron.d" [[ -z "${PGDATABASE}" ]] && PGDATABASE="cexplorer" [[ -z "${HAPROXY_CFG}" ]] && HAPROXY_CFG="${CNODE_HOME}/files/haproxy.cfg" + [[ -z "${DB_SCRIPTS_URL}" ]] && DB_SCRIPTS_URL="https://raw.githubusercontent.com/${G_ACCOUNT}/koios-artifacts/${SGVERSION}/files/grest/rpc/db-scripts" DOCS_URL="https://cardano-community.github.io/guild-operators" [[ -z "${PGPASSFILE}" ]] && export PGPASSFILE="${CNODE_HOME}"/priv/.pgpass case ${NWMAGIC} in