diff --git a/.github/workflows/docker_bin.yml b/.github/workflows/docker_bin.yml index a9bacbbd6..3b13315cc 100644 --- a/.github/workflows/docker_bin.yml +++ b/.github/workflows/docker_bin.yml @@ -32,10 +32,13 @@ jobs: sudo apt-get update -y sudo apt-get remove -y '^dotnet-.*' sudo apt-get remove -y 'php.*' - sudo apt-get remove -y azure-cli google-cloud-sdk google-chrome-stable firefox powershell mono-devel + sudo apt-get remove -y azure-cli google-chrome-stable firefox powershell mono-devel sudo apt-get autoremove -y sudo apt-get clean - sudo rm -rf /usr/share/dotnet + sudo rm -rf "/usr/share/dotnet" + sudo rm -rf "/usr/local/lib/android" + sudo rm -rf "/opt/ghc" + sudo rm -rf "/opt/hostedtoolcache/CodeQL" sudo rm -rf "/usr/local/share/boost" sudo rm -rf "$AGENT_TOOLSDIRECTORY" - name: docker push latest diff --git a/docs/Build/grest-changelog.md b/docs/Build/grest-changelog.md index 727fff133..c09de3ab3 100644 --- a/docs/Build/grest-changelog.md +++ b/docs/Build/grest-changelog.md @@ -1,5 +1,62 @@ # Koios gRest Changelog +## [1.1.0rc] - For all networks. + +This will be first major [breaking] release for Koios consumers in a while, and will be rolled out under new base prefix (`/api/v1`). +The major work with this release was to start making use of newer flags in dbsync which help performance of queries under new endpoints. Also, you'd see quite a few new endpoint additions below, that'd be helping out with slightly lighter version of queries. To keep migration paths easier, we will ensure both v0 and v1 versions of the release is up for a month post release, before retiring v0. + +### New endpoints added: +- `/pool_registrations` - List of all pool registrations initiated in the requested epoch [#239](https://github.com/cardano-community/koios-artifacts/pull/239) +- `/pool_retirements` - List of all pool retirements initiated in the requested epoch [#239](https://github.com/cardano-community/koios-artifacts/pull/239) +- `/treasury_withdrawals` - List of withdrawals made from treasury [#239](https://github.com/cardano-community/koios-artifacts/pull/239) +- `/reserve_withdrawals` - List of withdrawals made from reserves (MIRs) [#239](https://github.com/cardano-community/koios-artifacts/pull/239) +- `/account_txs` - Transactions associated with a given stake address [#239](https://github.com/cardano-community/koios-artifacts/pull/239) +- `/address_utxos` - Get UTxO details for requested addresses [#239](https://github.com/cardano-community/koios-artifacts/pull/239) +- `/asset_utxos` - Get UTxO details for requested assets [#239](https://github.com/cardano-community/koios-artifacts/pull/239) +- `/script_utxos` - Get UTxO details for requested script hashes [#239](https://github.com/cardano-community/koios-artifacts/pull/239) +- `/utxo_info` - Details for requested UTxO arrays [#239](https://github.com/cardano-community/koios-artifacts/pull/239) +- `/script_info` - Information about a given script FROM script hashes [#239](https://github.com/cardano-community/koios-artifacts/pull/239) +- `/ogmios/?EvaluateTransaction` - Expose [Ogmios EvaluateTransaction](https://ogmios.dev/api/#operation-publish-/?EvaluateTransaction) as an endpoint [#1690](https://github.com/cardano-community/guild-operators/pull/1690) +- `/ogmios/?SubmitTransaction` - Expose [Ogmios SubmitTransaction](https://ogmios.dev/api/#operation-publish-/?SubmitTransaction) as an endpoint [#1690](https://github.com/cardano-community/guild-operators/pull/1690) + +### Data Input/Output Changes: +- Input - `/account_utxos` , `/credential_utxos` - Accept `extended` as an additional flag - which enables `asset_list`, `reference_script` and `inline_datum` to the output [#239](https://github.com/cardano-community/koios-artifacts/pull/239) +- Output - `/block_txs` - Flatten output with transaction details (`tx_hash`, `epoch_no`, `block_height`, `block_time`) instead of `tx_hashes` array [#239](https://github.com/cardano-community/koios-artifacts/pull/239) +- Output - `/epoch_params` - Update `cost_models` to JSON (upstream change in node) [#239](https://github.com/cardano-community/koios-artifacts/pull/239) +- Output - `/account_assets` , `/address_assets` - Flatten the output result (instead of `asset_list` array) making it easier to apply horizontal filtering based on any of the fields +- Output - Align output fields for `/account_utxos` , `/address_utxos`, `/asset_utxos` , `/script_utxos` and `/utxo_info` to return same schema giving complete details about UTxOs involved, with few fields toggled based on `extended` input flag [#239](https://github.com/cardano-community/koios-artifacts/pull/239) +- Output - `/pool_list` - Add various details to the endpoint for each pool (`pool_id_hex`,`active_epoch_no`,`margin`,`fixed_cost`,`pledge`,`reward_addr`,`owners`,`relays`,`ticker`,`meta_url`,`meta_hash`,`pool_status`,`retiring_epoch`) - this should mean *some* of the requests to `pool_info` should no longer be required [#239](https://github.com/cardano-community/koios-artifacts/pull/239) +- Output - `/pool_updates` - In v0, `pool_updates` only provided pool registration updates, while `pool_status` corresponded to current status of pool. With v1, we will have registration as well as deregistration transactions, and each transaction will have `update_type` (enum of either `registration` or `deregistration`) instead of `pool_status`. As a side-effect, since a registration transaction only has `retiring_epoch` as metadata, all the other fields will show up as `null` for such a transaction [#239](https://github.com/cardano-community/koios-artifacts/pull/239) +- Output - `/pool_metadata` , `/pool_relays` - Add `pool_status` field to denote whether pool is retired [#239](https://github.com/cardano-community/koios-artifacts/pull/239) +- Output - `/datum_info` - Rename `hash` to `datum_hash` and add `creation_tx_hash` [#239](https://github.com/cardano-community/koios-artifacts/pull/239) +- Output - `/native_script_list` - Remove `script` column (as it has pretty large output better queried against `script_info`), add `size` and change `type` to text [#239](https://github.com/cardano-community/koios-artifacts/pull/239) +- Output - `/plutus_script_list` - Add `type` and `size` to output [#239](https://github.com/cardano-community/koios-artifacts/pull/239) +- Output - `/asset_info` - Add `cip68_metadata` JSONB field [#239](https://github.com/cardano-community/koios-artifacts/pull/227) +- Output - `/pool_history` - Add member_rewards [#225](https://github.com/cardano-community/koios-artifacts/pull/225) + +### Deprecations: +- `/tx_utxos` - No longer required as replaced by `/utxo_info` [#239](https://github.com/cardano-community/koios-artifacts/pull/239) + +### Chores: +- Update base version to `v1` from `v0` [#1690](https://github.com/cardano-community/guild-operators/pull/1690) +- Allow Bearer Authentication against endpoints [#239](https://github.com/cardano-community/koios-artifacts/pull/239) +- Cron job to apply corrections to epoch info [#239](https://github.com/cardano-community/koios-artifacts/pull/239) +- `epoch_info_cache` Remove protocol parameters, as they can be queried from live table. Accordingly update dependent queries [#239](https://github.com/cardano-community/koios-artifacts/pull/239) +- Make use of new `consumed_by_tx_in_id` column in `tx_out` from dbsync 13.1.1.3 across endpoints [#239](https://github.com/cardano-community/koios-artifacts/pull/239) +- Fix `_last_active_stake_validated_epoch` in active_stake_cache [#222](https://github.com/cardano-community/koios-artifacts/pull/222) +- Typo for pool_history_cache.sql as well as add a check to ensure epoch_info_cache has run at least once prior to pool_history_cache [#223](https://github.com/cardano-community/koios-artifacts/pull/223) +- Move control_table entry in cache tables to the end (instead of start) [#226](https://github.com/cardano-community/koios-artifacts/pull/226) +- Fix Asset Info Cache (include mint/burn tx rather than sum for meta consideration) [#226](https://github.com/cardano-community/koios-artifacts/pull/226) +- Update SQLs as per SQLFluff linting guidelines [#226](https://github.com/cardano-community/koios-artifacts/pull/226) +- Fix for tip check in cron jobs [#217](https://github.com/cardano-community/koios-artifacts/pull/217) +- Update cron jobs to exit if the database has not received block in 5 mins [#200](https://github.com/cardano-community/koios-artifacts/pull/200) +- Update active stake cache to use control table [#196](https://github.com/cardano-community/koios-artifacts/pull/196) +- Update Asset Info Cache entry whenever asset registry cache has an update [#194](https://github.com/cardano-community/koios-artifacts/pull/194) +- Bump up margin for tx rollback lookup for asset_info_cache to 1000 , as 100 is too small a margin for 2-3 blocks , which can contain more than 100 transactions (of which if oldest transaction contains a mint, it will not get into the cache) [#177](https://github.com/cardano-community/koios-artifacts/pull/177) +- Swap grestrpcs file to list exceptions and treat everything else as RPC [#1690](https://github.com/cardano-community/guild-operators/pull/1690) +- Update grest-poll.sh to have stricter spec validation and add health check for asset_registry [#1690](https://github.com/cardano-community/guild-operators/pull/1690) +- Update guild-deploy.sh to include latest pre-release for ogmios [#1690](https://github.com/cardano-community/guild-operators/pull/1690) + ## [1.0.10] - For all networks. The release is effectively same as `1.0.10rc` except with one minor modification below. diff --git a/docs/Scripts/cntools-changelog.md b/docs/Scripts/cntools-changelog.md index 97d68ff87..30b9c9751 100644 --- a/docs/Scripts/cntools-changelog.md +++ b/docs/Scripts/cntools-changelog.md @@ -6,6 +6,10 @@ All notable changes to this tool will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [11.0.1] - 2023-10-25 +#### Fixed +- Fix display for Pool Cost and Pledge to accept integer as well as decimal format of ADA + ## [11.0.0] - 2023-07-05 #### Changed - CNTools now part of Koios brand diff --git a/scripts/cnode-helper-scripts/cntools.library b/scripts/cnode-helper-scripts/cntools.library index 73590d666..b6719dfac 100644 --- a/scripts/cnode-helper-scripts/cntools.library +++ b/scripts/cnode-helper-scripts/cntools.library @@ -15,7 +15,7 @@ CNTOOLS_MAJOR_VERSION=11 # Minor: Changes and features of minor character that can be applied without breaking existing functionality or workflow CNTOOLS_MINOR_VERSION=0 # Patch: Backwards compatible bug fixes. No additional functionality or major changes -CNTOOLS_PATCH_VERSION=0 +CNTOOLS_PATCH_VERSION=1 CNTOOLS_VERSION="${CNTOOLS_MAJOR_VERSION}.${CNTOOLS_MINOR_VERSION}.${CNTOOLS_PATCH_VERSION}" diff --git a/scripts/cnode-helper-scripts/cntools.sh b/scripts/cnode-helper-scripts/cntools.sh index 4ed90ab1a..e63c8e5f1 100755 --- a/scripts/cnode-helper-scripts/cntools.sh +++ b/scripts/cnode-helper-scripts/cntools.sh @@ -3551,9 +3551,9 @@ function main { echo println DEBUG "Pool name : ${FG_LGRAY}$(jq -r '."pool-metadata".name' <<< ${offlineJSON})${NC}" println DEBUG "Ticker : ${FG_LGRAY}$(jq -r '."pool-metadata".ticker' <<< ${offlineJSON})${NC}" - println DEBUG "Pledge : ${FG_LBLUE}$(formatAsset "$(jq -r '."pool-pledge"' <<< ${offlineJSON})")${NC} Ada" + println DEBUG "Pledge : ${FG_LBLUE}$(formatLovelace "$(AdaToLovelace "$(jq -r '."pool-pledge"' <<< ${offlineJSON})")")${NC} Ada" println DEBUG "Margin : ${FG_LBLUE}$(jq -r '."pool-margin"' <<< ${offlineJSON})${NC} %" - println DEBUG "Cost : ${FG_LBLUE}$(formatAsset "$(jq -r '."pool-cost"' <<< ${offlineJSON})")${NC} Ada" + println DEBUG "Cost : ${FG_LBLUE}$(formatLovelace "$(AdaToLovelace "$(jq -r '."pool-cost"' <<< ${offlineJSON})")")${NC} Ada" for otx_signing_file in $(jq -r '."signing-file"[] | @base64' <<< "${offlineJSON}"); do _jq() { base64 -d <<< ${otx_signing_file} | jq -r "${1}"; } otx_signing_name=$(_jq '.name') @@ -3725,9 +3725,9 @@ function main { jq -er '.metadata' <<< ${offlineJSON} &>/dev/null && println DEBUG "Metadata :\n$(jq -r '.metadata' <<< ${offlineJSON})\n" [[ ${otx_type} = "Pool Registration" || ${otx_type} = "Pool Update" ]] && println DEBUG "Pool name : ${FG_LGRAY}$(jq -r '."pool-metadata".name' <<< ${offlineJSON})${NC}" [[ ${otx_type} = "Pool Registration" || ${otx_type} = "Pool Update" ]] && println DEBUG "Ticker : ${FG_LGRAY}$(jq -r '."pool-metadata".ticker' <<< ${offlineJSON})${NC}" - [[ ${otx_type} = "Pool Registration" || ${otx_type} = "Pool Update" ]] && println DEBUG "Pledge : ${FG_LBLUE}$(formatAsset "$(jq -r '."pool-pledge"' <<< ${offlineJSON})")${NC} Ada" + [[ ${otx_type} = "Pool Registration" || ${otx_type} = "Pool Update" ]] && println DEBUG "Pledge : ${FG_LBLUE}$(formatLovelace "$(AdaToLovelace "$(jq -r '."pool-pledge"' <<< ${offlineJSON})")")${NC} Ada" [[ ${otx_type} = "Pool Registration" || ${otx_type} = "Pool Update" ]] && println DEBUG "Margin : ${FG_LBLUE}$(jq -r '."pool-margin"' <<< ${offlineJSON})${NC} %" - [[ ${otx_type} = "Pool Registration" || ${otx_type} = "Pool Update" ]] && println DEBUG "Cost : ${FG_LBLUE}$(formatAsset "$(jq -r '."pool-cost"' <<< ${offlineJSON})")${NC} Ada" + [[ ${otx_type} = "Pool Registration" || ${otx_type} = "Pool Update" ]] && println DEBUG "Cost : ${FG_LBLUE}$(formatLovelace "$(AdaToLovelace "$(jq -r '."pool-cost"' <<< ${offlineJSON})")")${NC} Ada" [[ ${otx_type} = "Asset Minting" || ${otx_type} = "Asset Burning" ]] && println DEBUG "Policy Name : ${FG_LGRAY}$(jq -r '."policy-name"' <<< ${offlineJSON})${NC}" [[ ${otx_type} = "Asset Minting" || ${otx_type} = "Asset Burning" ]] && println DEBUG "Policy ID : ${FG_LGRAY}$(jq -r '."policy-id"' <<< ${offlineJSON})${NC}" [[ ${otx_type} = "Asset Minting" || ${otx_type} = "Asset Burning" ]] && println DEBUG "Asset Name : ${FG_LGRAY}$(jq -r '."asset-name"' <<< ${offlineJSON})${NC}" diff --git a/scripts/cnode-helper-scripts/env b/scripts/cnode-helper-scripts/env index 2357f18b7..50f6a6b7c 100644 --- a/scripts/cnode-helper-scripts/env +++ b/scripts/cnode-helper-scripts/env @@ -34,7 +34,7 @@ #TIMEOUT_LEDGER_STATE=300 # Timeout in seconds for querying and dumping ledger-state #IP_VERSION=4 # The IP version to use for push and fetch, valid options: 4 | 6 | mix (Default: 4) #ENABLE_KOIOS=Y # (Y|N) Enable KOIOS API. If disabled, local node queries will be used instead with increased system resource requirements (default Y) -#KOIOS_API="https://api.koios.rest/api/v0" # Koios API for blockchain queries instead of local cli lookup. +#KOIOS_API="https://api.koios.rest/api/v1" # Koios API for blockchain queries instead of local cli lookup. # Leave commented for automatic network detection between MainNet, Preview, Preprod or Guild network. # https://www.koios.rest/ #DBSYNC_QUERY_FOLDER="${CNODE_HOME}/files/dbsync/queries" # [advanced feature] Folder containing DB-Sync chain analysis queries @@ -1010,19 +1010,19 @@ case ${NWMAGIC} in 764824073) NETWORK_NAME="Mainnet" SHELLEY_TRANS_EPOCH=208 - [[ -z ${KOIOS_API} ]] && KOIOS_API="https://api.koios.rest/api/v0" ;; + [[ -z ${KOIOS_API} ]] && KOIOS_API="https://api.koios.rest/api/v1" ;; 1) NETWORK_NAME="PreProd" SHELLEY_TRANS_EPOCH=4 - [[ -z ${KOIOS_API} ]] && KOIOS_API="https://preprod.koios.rest/api/v0";; + [[ -z ${KOIOS_API} ]] && KOIOS_API="https://preprod.koios.rest/api/v1";; 141) NETWORK_NAME="Guild" SHELLEY_TRANS_EPOCH=2 - [[ -z ${KOIOS_API} ]] && KOIOS_API="https://guild.koios.rest/api/v0" ;; + [[ -z ${KOIOS_API} ]] && KOIOS_API="https://guild.koios.rest/api/v1";; 2) NETWORK_NAME="Preview" SHELLEY_TRANS_EPOCH=0 - [[ -z ${KOIOS_API} ]] && KOIOS_API="https://preview.koios.rest/api/v0" ;; + [[ -z ${KOIOS_API} ]] && KOIOS_API="https://preview.koios.rest/api/v1";; *) NETWORK_NAME="Custom" [[ -z ${SHELLEY_TRANS_EPOCH} ]] && SHELLEY_TRANS_EPOCH=0 diff --git a/scripts/cnode-helper-scripts/gLiveView.sh b/scripts/cnode-helper-scripts/gLiveView.sh index 5741dd36c..61ac1b027 100755 --- a/scripts/cnode-helper-scripts/gLiveView.sh +++ b/scripts/cnode-helper-scripts/gLiveView.sh @@ -57,7 +57,7 @@ setTheme() { # Do NOT modify code below # ###################################### -GLV_VERSION=v1.27.5 +GLV_VERSION=v1.27.6 PARENT="$(dirname $0)" @@ -656,7 +656,7 @@ while true; do tlines=$(tput lines) # update terminal lines tcols=$(tput cols) # update terminal columns done - while [[ ${line} -ge $((tlines - 1)) ]]; do + while [[ ${line} -gt ${tlines} ]]; do mvPos 2 2 printf "${style_status_3}Terminal height too small!${NC}" mvPos 4 2 diff --git a/scripts/cnode-helper-scripts/guild-deploy.sh b/scripts/cnode-helper-scripts/guild-deploy.sh index fe7162cd2..66d86c61d 100755 --- a/scripts/cnode-helper-scripts/guild-deploy.sh +++ b/scripts/cnode-helper-scripts/guild-deploy.sh @@ -429,14 +429,14 @@ download_ogmios() { if command -v ogmios >/dev/null; then ogmios_version="$(ogmios --version)"; else ogmios_version="v0.0.0"; fi rm -rf /tmp/ogmios && mkdir /tmp/ogmios pushd /tmp/ogmios >/dev/null || err_exit - ogmios_asset_url="$(curl -s https://api.github.com/repos/CardanoSolutions/ogmios/releases/latest | jq -r '.assets[].browser_download_url')" + ogmios_asset_url="$(curl -s https://api.github.com/repos/CardanoSolutions/ogmios/releases | jq -r '.[0].assets[].browser_download_url')" if curl -sL -f -m ${CURL_TIMEOUT} -o ogmios.zip ${ogmios_asset_url}; then unzip ogmios.zip &>/dev/null rm -f ogmios.zip [[ -f bin/ogmios ]] && OGMIOSPATH=bin/ogmios [[ -f ogmios ]] && OGMIOSPATH=ogmios [[ -n ${OGMIOSPATH} ]] || err_exit "ogmios downloaded but binary not found after extracting package!" - ogmios_git_version="$(curl -s https://api.github.com/repos/CardanoSolutions/ogmios/releases/latest | jq -r '.tag_name')" + ogmios_git_version="$(curl -s https://api.github.com/repos/CardanoSolutions/ogmios/releases | jq -r '.[0].tag_name')" if ! versionCheck "${ogmios_git_version}" "${ogmios_version}"; then [[ "${ogmios_version}" = "0.0.0" ]] && echo -e "\n latest version: ${ogmios_git_version}" || echo -e "\n installed version: ${ogmios_version} | latest version: ${ogmios_git_version}" chmod +x /tmp/ogmios/${OGMIOSPATH} diff --git a/scripts/grest-helper-scripts/db-scripts/basics.sql b/scripts/grest-helper-scripts/db-scripts/basics.sql index e81da525b..641b05d43 100644 --- a/scripts/grest-helper-scripts/db-scripts/basics.sql +++ b/scripts/grest-helper-scripts/db-scripts/basics.sql @@ -36,7 +36,7 @@ GRANT USAGE ON SCHEMA grest TO authenticator,web_anon; GRANT SELECT ON ALL TABLES IN SCHEMA public TO authenticator,web_anon; GRANT SELECT ON ALL TABLES IN SCHEMA grest TO authenticator,web_anon; GRANT web_anon TO authenticator; -ALTER ROLE authenticator SET statement_timeout = 65000; +ALTER ROLE authenticator SET statement_timeout = 125000; ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT diff --git a/scripts/grest-helper-scripts/grest-poll.sh b/scripts/grest-helper-scripts/grest-poll.sh index 8346624af..d1455eb41 100755 --- a/scripts/grest-helper-scripts/grest-poll.sh +++ b/scripts/grest-helper-scripts/grest-poll.sh @@ -119,7 +119,7 @@ function chk_tip() { } function chk_rpc_struct() { - srvr_spec="$(curl -skL "${1}" | jq 'leaf_paths as $p | [$p[] | tostring] |join(".")' 2>/dev/null)" + srvr_spec="$(curl -skL "${1}" | jq '[leaf_paths as $p | { "key": $p | map(tostring) | join("_"), "value": getpath($p) }] | from_entries' | awk '{print $1 " " $2}' | grep -e ^\"paths -e ^\"parameters -e ^\"definitions 2>/dev/null)" api_endpts="$(grep ^\ \ / "${LOCAL_SPEC}" | sed -e 's# /#/#g' -e 's#:##' | sort)" for endpt in ${api_endpts} do @@ -155,7 +155,7 @@ function chk_cache_status() { else [[ -z "${GENESIS_JSON}" ]] && GENESIS_JSON="${PARENT}"/../files/shelley-genesis.json epoch_length=$(jq -r .epochLength "${GENESIS_JSON}" 2>/dev/null) - if [[ ${epoch_slot} -ge $(( epoch_length / 10 )) ]]; then + if [[ ${epoch_slot} -ge $(( epoch_length / 8 )) ]]; then if [[ "${last_actvstake_epoch}" != "${epoch}" ]]; then log_err "Active Stake cache for epoch ${epoch} still not populated as of ${epoch_slot} slot, maximum tolerance was $(( epoch_length / 10 )) !!" optexit @@ -194,7 +194,7 @@ function chk_endpt_post() { } function chk_asset_registry() { - ct=$(curl -sfkL -H 'Prefer: count=exact' "${GURL}/asset_registry_cache?select=name&limit=1" -I 2>/dev/null | grep -i "content-range" | cut -d/ -f2 | tr -d '[:space:]') + ct=$(curl -sfkL -H 'Prefer: count=exact' "${GURL}/asset_token_registry?select=asset_name&limit=1" -I 2>/dev/null | grep -i "content-range" | cut -d/ -f2 | tr -d '[:space:]') if [[ "${ct}" == "" ]] || [[ $ct -lt 150 ]]; then log_err "Asset registry cache seems incomplete (<150) assets, try deleting key: asset_registry_commit in control_table and wait for next cron run" optexit diff --git a/scripts/grest-helper-scripts/setup-grest.sh b/scripts/grest-helper-scripts/setup-grest.sh index 3a04a52e3..3870be6f7 100755 --- a/scripts/grest-helper-scripts/setup-grest.sh +++ b/scripts/grest-helper-scripts/setup-grest.sh @@ -16,7 +16,7 @@ # Do NOT modify code below # ###################################### -SGVERSION=v1.0.11rc +SGVERSION=v1.1.0rc ######## Functions ######## usage() { @@ -275,7 +275,8 @@ SGVERSION=v1.0.11rc common_update() { # Create skeleton whitelist URL file if one does not already exist using most common option curl -sfkL "https://${KOIOS_SRV}/koiosapi.yaml" -o "${CNODE_HOME}"/files/koiosapi.yaml 2>/dev/null - grep " #RPC" "${CNODE_HOME}"/files/koiosapi.yaml | sed -e 's#^ /#/#' | cut -d: -f1 | sort > "${CNODE_HOME}"/files/grestrpcs 2>/dev/null + grep "^ /" "${CNODE_HOME}"/files/koiosapi.yaml | grep -v -e submittx -e "#RPC" | sed -e 's#^ /#/#' | cut -d: -f1 | sort > "${CNODE_HOME}"/files/grestrpcs 2>/dev/null + echo "/control_table" >> "${CNODE_HOME}"/files/grestrpcs 2>/dev/null checkUpdate grest-poll.sh Y N N grest-helper-scripts >/dev/null sed -i "s# API_STRUCT_DEFINITION=\"https://api.koios.rest/koiosapi.yaml\"# API_STRUCT_DEFINITION=\"https://${KOIOS_SRV}/koiosapi.yaml\"#g" grest-poll.sh checkUpdate checkstatus.sh Y N N grest-helper-scripts >/dev/null @@ -341,9 +342,9 @@ SGVERSION=v1.0.11rc } deploy_haproxy() { - printf "[Re]Installing HAProxy.." + printf "\n[Re]Installing HAProxy.." pushd ~/tmp >/dev/null || err_exit - haproxy_url="http://www.haproxy.org/download/2.6/src/haproxy-2.6.5.tar.gz" + haproxy_url="http://www.haproxy.org/download/2.8/src/haproxy-2.8.3.tar.gz" if curl -sL -f -m ${CURL_TIMEOUT} -o haproxy.tar.gz "${haproxy_url}"; then tar xf haproxy.tar.gz &>/dev/null && rm -f haproxy.tar.gz if command -v apt-get >/dev/null; then @@ -352,7 +353,7 @@ SGVERSION=v1.0.11rc if command -v yum >/dev/null; then sudo yum -y install pcre-devel >/dev/null || err_exit "'sudo yum -y install prce-devel' failed!" fi - cd haproxy-2.6.5 || return + cd haproxy-2.8.3 || return make clean >/dev/null make -j $(nproc) TARGET=linux-glibc USE_ZLIB=1 USE_LIBCRYPT=1 USE_OPENSSL=1 USE_PCRE=1 USE_SYSTEMD=1 USE_PROMEX=1 >/dev/null sudo make install-bin >/dev/null @@ -440,23 +441,27 @@ SGVERSION=v1.0.11rc bind 0.0.0.0:8053 ## If using SSL, comment line above and uncomment line below #bind :8453 ssl crt /etc/ssl/server.pem no-sslv3 + compression direction response + compression algo-res gzip + compression type-res application/json + option http-buffer-request http-request set-log-level silent acl srv_down nbsrv(grest_postgrest) eq 0 acl is_wss hdr(Upgrade) -i websocket http-request use-service prometheus-exporter if { path /metrics } http-request track-sc0 src table flood_lmt_rate - http-request deny deny_status 429 if { sc_http_req_rate(0) gt 250 } - use_backend ogmios if { path_beg /api/v0/ogmios } || { path_beg /dashboard.js } || { path_beg /assets } || { path_beg /health } || is_wss - use_backend submitapi if { path_beg /api/v0/submittx } + http-request deny deny_status 429 if { sc_http_req_rate(0) gt 500 } + use_backend ogmios if { path_beg /api/ogmios } || { path_beg /dashboard.js } || { path_beg /assets } || { path_beg /health } || is_wss + use_backend submitapi if { path_beg /api/v1/submittx } use_backend grest_failover if srv_down default_backend grest_postgrest backend grest_postgrest balance first #option external-check - acl grestrpcs path_beg -f \"\\\$GRESTTOP\"/files/grestrpcs - http-request set-path \"%[path,regsub(^/api/v0/,/)]\" - http-request set-path \"%[path,regsub(^/,/rpc/)]\" if grestrpcs + acl grestviews path_beg -f \"\\\$GRESTTOP\"/files/grestrpcs + http-request set-path \"%[path,regsub(^/api/v1/,/)]\" + http-request set-path \"%[path,regsub(^/,/rpc/)]\" if !grestviews !{ path_beg /rpc } !{ path -i / } #external-check path \"/usr/bin:/bin:/tmp:/sbin:/usr/sbin\" #external-check command \"\\\$GRESTTOP\"/scripts/grest-poll.sh server local 127.0.0.1:8050 check inter 20000 fall 1 rise 2 @@ -468,7 +473,7 @@ SGVERSION=v1.0.11rc backend ogmios balance first - http-request set-path \"%[path,regsub(^/api/v0/ogmios/,/)]\" + http-request set-path \"%[path,regsub(^/api/ogmios/,/)]\" option httpchk GET /health http-check expect status 200 default-server inter 20s fall 1 rise 2 @@ -477,7 +482,7 @@ SGVERSION=v1.0.11rc backend submitapi balance first option httpchk POST /api/submit/tx - http-request set-path \"%[path,regsub(^/api/v0/submittx,/api/submit/tx)]\" + http-request set-path \"%[path,regsub(^/api/v1/submittx,/api/submit/tx)]\" http-check expect status 415 default-server inter 20s fall 1 rise 2 server local 127.0.0.1:8090 check @@ -656,7 +661,9 @@ SGVERSION=v1.0.11rc # Description : Update the setup-grest.sh version used in the database. update_grest_version() { - [[ "${RESET_GREST}" == "Y" ]] && artifacts=['reset'] || artifacts='' + koios_release_commit="$(curl -s https://api.github.com/repos/${G_ACCOUNT}/koios-artifacts/commits/${SGVERSION} | jq -r '.sha')" + [[ -z ${koios_release_commit} ]] && koios_release_commit="null" + [[ "${RESET_GREST}" == "Y" ]] && artifacts=['reset',"${koios_release_commit}"] || artifacts=["${koios_release_commit}"] ! output=$(psql ${PGDATABASE} -qbt -c "SELECT GREST.update_control_table( 'version',