From 17387e96c0fa16bb86aae3005d7fbb86ffb91853 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9gory=20Saramite?= Date: Wed, 8 Nov 2023 17:39:36 +0100 Subject: [PATCH] Patches to SEEDS and PEERS + Uses Snapshot Sync as default (#2) * Broken Genesys, seed and peers on Evmos/tesnet https://github.com/evmos/testnets/issues/2862 - Switch to Genesys file from qubelabs - Add -s flag to sync from Polkadot Snapshot - Fetch last Polkadot snapshot's URL Current setup fails (even with snapshot sync) setup `panic: leveldb` * Removes deprecated README notes --- evmos/manifests/statefulset.yaml | 4 +- evmos/tesnet-validator/README.md | 40 --------- evmos/tesnet-validator/snapshot.sh | 51 ++++------- evmos/tesnet-validator/testnet_node.sh | 112 ++++++++++++++----------- k8s/README.md | 33 +++++--- k8s/private-cluster/README.md | 26 ------ 6 files changed, 102 insertions(+), 164 deletions(-) delete mode 100644 evmos/tesnet-validator/README.md mode change 100644 => 100755 evmos/tesnet-validator/snapshot.sh delete mode 100644 k8s/private-cluster/README.md diff --git a/evmos/manifests/statefulset.yaml b/evmos/manifests/statefulset.yaml index 292593f..3b96096 100644 --- a/evmos/manifests/statefulset.yaml +++ b/evmos/manifests/statefulset.yaml @@ -27,7 +27,7 @@ spec: privileged: true readOnlyRootFilesystem: false # to be changed in prod runAsNonRoot: false - image: gregsaram/evmos:15.0.0-rc2-testnet-0.2 + image: gregsaram/evmos:15.0.0-rc2-testnet-0.3 resources: requests: memory: "20Gi" @@ -35,7 +35,7 @@ spec: limits: memory: "25Gi" cpu: "4" - args: ["sh", "-c", "/usr/bin/testnet_node.sh -y & sleep 84000s"] + args: ["sh", "-c", "/usr/bin/testnet_node.sh -y -s & sleep 84000s"] ## sleeps for troubleshooting ports: - containerPort: 26656 name: p2p diff --git a/evmos/tesnet-validator/README.md b/evmos/tesnet-validator/README.md deleted file mode 100644 index 744943c..0000000 --- a/evmos/tesnet-validator/README.md +++ /dev/null @@ -1,40 +0,0 @@ -## Evmos testnet - -Evmosd Dockerile and coniguration scripts based tharsishq/evmos:v15.0.0-rc2 -This is intended to run on Kubernetes (k8s), see manifests directory for more info on the k8s setup. - -The steps involved in setting up the CI/CD pipeline are listed below: - -1. Setting up the Continuous Integration Pipeline - -1.1 Configuring the GitHub repositories. - -1.2 Configuring Cloud Build GitHub Trigger - -2. Setting up the Continuous Delivery Pipeline - -2.1 Configuring the GKE cluster - -2.2 Configuring ArgoCD on GKE - - -Pre-Requisites: - - Two code repositories, one containing application code and the other containing Kubernetes manifest files and deployment templates. Here we have used GitHub repositories for demonstration. - Cloud build configured with GitHub application code repository to execute a build based on the events in the repository. - Artifact Registry to store the images built from cloud build. - A GKE cluster for running the Docker images of the applications and an ArgoCD server used for continuous delivery. - - -WorkFlow: - -1. Continuous Integration(CI Pipeline): - - A Developer makes the changes in the code, fixes the bugs, and pushes it to the application repository. - Cloud build then invokes triggers either manually or automatically by the events on the repository such as pushes or pull requests. - Once the trigger gets invoked by any events, cloud build then executes the instructions written in the build config file (cloudbuild.yaml) such as building the docker image from Dockerfile provided and pushing it to the artifact registry configured. - Once the image with the new tag got pushed to the registry, it will get updated in the Kubernetes manifest repository. - -2. Continuous Deployment(CD Pipeline) - - ArgoCD continuously monitors the git repository for any changes that happen and then pulls the changes. It compares the current state of the deployed application with the desired state in the git repository and then applies the changes by automatically deploying the manifest on the GKE cluster. \ No newline at end of file diff --git a/evmos/tesnet-validator/snapshot.sh b/evmos/tesnet-validator/snapshot.sh old mode 100644 new mode 100755 index 409d212..18b7487 --- a/evmos/tesnet-validator/snapshot.sh +++ b/evmos/tesnet-validator/snapshot.sh @@ -16,21 +16,28 @@ HOMEDIR="/evmos/evmosd" CONFIGDIR="$HOMEDIR/config" DATADIR="$HOMEDIR/data" -SNAPSHOTNAME="evmos_18472724.tar.lz4" +# Fetch current snapshot URL +SNAPSHOTURL=$(curl -s https://polkachu.com/testnets/evmos/snapshots | grep -o 'https[^"]*.lz4' | head -n 1) + +echo $SNAPSHOTURL +echo $SNAPSHOTNAME # Path variables CONFIG="$CONFIGDIR/config.toml" APP_TOML="$CONFIGDIR/app.toml" GENESIS="$CONFIGDIR/genesis.json" -for cmd in jq curl lz4 evmosd; do +for cmd in curl lz4 evmosd; do command -v $cmd >/dev/null 2>&1 || { echo >&2 "$cmd is required but it's not installed. Aborting."; exit 1; } done +# Stream the snapshot into database location. +curl -o - -L ${SNAPSHOTURL} \ + | lz4 -c -d - \ + | tar -x -C $HOMEDIR \ + || { echo "Failed to download & decompress snapshot ${SNAPSHOTURL}. check https://polkachu.com/testnets/evmos/snapshots"; exit 1; } -# Setup client config -evmosd config chain-id "$CHAINID" --home "$HOMEDIR" -evmosd config keyring-backend "$KEYRING" --home "$HOMEDIR" +cp ${DATADIR}/priv_validator_state.json ${CONFIGDIR}/priv_validator_state.json update_or_add() { key=$1 @@ -52,36 +59,6 @@ update_or_add "pruning-keep-every" "0" "$APP_TOML" update_or_add "pruning-interval" "10" "$APP_TOML" sed -i 's/indexer = kv/indexer = null/g' "$CONFIG" -# Enable prometheus metrics and all APIs for dev node -sed -i 's/prometheus = false/prometheus = true/' "$CONFIG" -sed -i 's/prometheus-retention-time = "0"/prometheus-retention-time = "1000000000000"/g' "$APP_TOML" -sed -i 's/enabled = false/enabled = true/g' "$APP_TOML" -sed -i 's/enable = false/enable = true/g' "$APP_TOML" -# Don't enable memiavl by default -grep -q -F '[memiavl]' "$APP_TOML" && sed -i '/\[memiavl\]/,/^\[/ s/enable = true/enable = false/' "$APP_TOML" - -# Stream the snapshot into database location. - -curl -o - -L https://snapshots.polkachu.com/testnet-snapshots/evmos/${SNAPSHOTNAME} \ - | lz4 -c -d - \ - | tar -x -C $HOMEDIR \ - || { echo "snapshot ${SNAPSHOTNAME} not available, use the latest one. check https://polkachu.com/testnets/evmos/snapshots"; exit 1; } - -cp ${DATADIR}/priv_validator_state.json ${CONFIGDIR}/priv_validator_state.json - -evmosd tendermint unsafe-reset-all --home $HOME/.evmosd --keep-addr-book -echo "evmosd tx staking create-validator ..." -evmosd tx staking create-validator \ ---amount=1000000atevmos \ ---pubkey=$(evmosd tendermint show-validator) \ ---moniker="${MONIKER}" \ ---chain-id='evmos_9000-4' \ ---commission-rate="0.05" \ ---commission-max-rate="0.10" \ ---commission-max-change-rate="0.01" \ ---min-self-delegation="1000000" \ ---gas="auto" \ ---gas-prices="0.025atevmos" \ ---from='mykey' \ ---home="${HOMEDIR}" +# Remove all the data and WAL, reset this node's validator to genesis state +#evmosd tendermint unsafe-reset-all --home ${HOMEDIR} --keep-addr-book --keyring-backend ${KEYRING} diff --git a/evmos/tesnet-validator/testnet_node.sh b/evmos/tesnet-validator/testnet_node.sh index 567e14e..f1e0b25 100755 --- a/evmos/tesnet-validator/testnet_node.sh +++ b/evmos/tesnet-validator/testnet_node.sh @@ -37,8 +37,8 @@ command -v wget >/dev/null 2>&1 || { set -e # Parse input flags -install=true overwrite="" +snapshotsync=0 while [[ $# -gt 0 ]]; do key="$1" @@ -49,10 +49,15 @@ while [[ $# -gt 0 ]]; do shift # Move past the flag ;; -n) - echo "Flag -n passed -> Not overwriting the previous chain data." + echo "Flag -n passed -> Not overwriting the previous config " overwrite="n" shift # Move past the argument ;; + -s) + echo "Flag -s passed -> Overwriting data with latest testnet snapshot" + snapshotsync=1 + shift # Move past the argument + ;; *) echo "Unknown flag passed: $key -> Exiting script!" exit 1 @@ -75,11 +80,12 @@ fi # Setup local node if overwrite is set to Yes, otherwise skip setup if [[ $overwrite == "y" || $overwrite == "Y" ]]; then # Remove the previous folder - rm -rf "$HOMEDIR" + rm -rf "$HOMEDIR/*" + evmosd init $MONIKER --chain-id "$CHAINID" --home "$HOMEDIR" + evmosd config keyring-backend "$KEYRING" --home "$HOMEDIR" # Set client config echo "client config" - evmosd config keyring-backend "$KEYRING" --home "$HOMEDIR" evmosd config chain-id "$CHAINID" --home "$HOMEDIR" # myKey address 0x7cb61d4117ae31a12e393a1cfa3bac666481d02e | evmos10jmp6sgh4cc6zt3e8gw05wavvejgr5pwjnpcky @@ -94,36 +100,38 @@ if [[ $overwrite == "y" || $overwrite == "Y" ]]; then node_address=$(evmosd keys show -a "$VAL_KEY" --keyring-backend "$KEYRING" --home "$HOMEDIR") # Set moniker and chain-id for Evmos (Moniker can be anything, chain-id must be an integer) - evmosd tendermint unsafe-reset-all --home "$HOMEDIR" - evmosd init $MONIKER --chain-id "$CHAINID" --home "$HOMEDIR" - - # Download Genesis file for testnet evmos_9000-4 -# wget -O ${GENESIS} https://archive.evmos.dev/evmos_9000-4/genesis.json - wget -O ${GENESIS} https://snapshots.polkachu.com/testnet-genesis/evmos/genesis.json + #evmosd tendermint unsafe-reset-all --home "$HOMEDIR" + + # Download Genesis file for testnet evmos_9000-4, not needed with snapshot sync + if [ "$snapshotsync" -eq 0 ]; then + if [ -f ${GENESIS}] ; then + rm -f ${GENESIS} + fi + # wget -O ${GENESIS} https://archive.evmos.dev/evmos_9000-4/genesis.json + # wget -O ${GENESIS} https://snapshots.polkachu.com/testnet-genesis/evmos/genesis.json + wget -O ${GENESIS} https://qubelabs.io/evmos/genesis.json + fi # Update persistent_peers and seeds for testnet setup - PEERS=`curl -sL https://raw.githubusercontent.com/evmos/testnets/main/evmos_9000-4/peers.txt | sort -r | head -n 10 | awk '{print $1}' | paste -s -d, -` + # https://github.com/evmos/testnets/issues/2862 + # PEERS=`curl -sL https://raw.githubusercontent.com/evmos/testnets/main/evmos_9000-4/peers.txt | sort -r | head -n 10 | awk '{print $1}' | paste -s -d, -` + # sed -i.bak -e "s/^persistent_peers *=.*/persistent_peers = \"$PEERS\"/" ${CONFIG} + + PEERS="182be4eedbc3a3b6d3174f39c18a23ccc86b04ca@142.132.130.222:26661,81d580f9d75f5bb224d262f3d2960fb3a74d1f58@65.109.39.50:33656,940a407248badb7c77397f22e48fb31c690184d0@27.75.78.95:16758,2dd78738cf0cb2779d6dfa79236c0236f1d58248@65.109.24.78:15656,7797a37a0b3f0296949c3a537dcc151b4ce86e84@78.46.107.187:17056" sed -i.bak -e "s/^persistent_peers *=.*/persistent_peers = \"$PEERS\"/" ${CONFIG} - SEEDS=`curl -sL https://raw.githubusercontent.com/evmos/testnets/main/evmos_9000-4/seeds.txt | awk '{print $1}' | paste -s -d, -` + SEEDS="ade4d8bc8cbe014af6ebdf3cb7b1e9ad36f412c0@testnet-seeds.polkachu.com:13456" sed -i '/^seeds =/s/^/#/; /^#seeds =/a seeds = "'"$SEEDS"'"' ${CONFIG} # enable prometheus metrics and all APIs for dev node - if [[ "$OSTYPE" == "darwin"* ]]; then - sed -i '' 's/prometheus = false/prometheus = true/' "$CONFIG" - sed -i '' 's/prometheus-retention-time = 0/prometheus-retention-time = 1000000000000/g' "$APP_TOML" - sed -i '' 's/enabled = false/enabled = true/g' "$APP_TOML" - sed -i '' 's/enable = false/enable = true/g' "$APP_TOML" - # Don't enable memiavl by default - grep -q -F '[memiavl]' "$APP_TOML" && sed -i '' '/\[memiavl\]/,/^\[/ s/enable = true/enable = false/' "$APP_TOML" - else - sed -i 's/prometheus = false/prometheus = true/' "$CONFIG" - sed -i 's/prometheus-retention-time = "0"/prometheus-retention-time = "1000000000000"/g' "$APP_TOML" - sed -i 's/enabled = false/enabled = true/g' "$APP_TOML" - sed -i 's/enable = false/enable = true/g' "$APP_TOML" - # Don't enable memiavl by default - grep -q -F '[memiavl]' "$APP_TOML" && sed -i '/\[memiavl\]/,/^\[/ s/enable = true/enable = false/' "$APP_TOML" - fi + + sed -i 's/prometheus = false/prometheus = true/' "$CONFIG" + sed -i 's/prometheus-retention-time = "0"/prometheus-retention-time = "1000000000000"/g' "$APP_TOML" + sed -i 's/enabled = false/enabled = true/g' "$APP_TOML" + sed -i 's/enable = false/enable = true/g' "$APP_TOML" + # Don't enable memiavl by default + grep -q -F '[memiavl]' "$APP_TOML" && sed -i '/\[memiavl\]/,/^\[/ s/enable = true/enable = false/' "$APP_TOML" + # Edit genesys file in python to parse the JSON file in a streaming manner # Less memory intensive @@ -131,24 +139,30 @@ if [[ $overwrite == "y" || $overwrite == "Y" ]]; then # /usr/bin/genesis_edit.py $GENESIS # Run this to ensure everything worked and that the genesis file is setup correctly - echo "validating genesis" - evmosd validate-genesis --home "$HOMEDIR" + # echo "validating genesis" + # evmosd validate-genesis --home "$HOMEDIR" +fi + +# Download Snapshot +if [ "$snapshotsync" -eq 1 ]; then + echo "Fetching snapshot and overwriting ${HOMEDIR}/data" + (/usr/bin/snapshot.sh) fi - echo "evmosd tx staking create-validator ..." - evmosd tx staking create-validator \ - --amount=1000000atevmos \ - --pubkey=$(evmosd tendermint show-validator) \ - --moniker="${MONIKER}" \ - --chain-id='evmos_9000-4' \ - --commission-rate="0.05" \ - --commission-max-rate="0.10" \ - --commission-max-change-rate="0.01" \ - --min-self-delegation="1000000" \ - --gas="auto" \ - --gas-prices="0.025atevmos" \ - --from='mykey' \ - --home="${HOMEDIR}" + # echo "evmosd tx staking create-validator ..." + # evmosd tx staking create-validator \ + # --amount=1000000atevmos \ + # --pubkey=$(evmosd tendermint show-validator) \ + # --moniker="${MONIKER}" \ + # --chain-id='evmos_9000-4' \ + # --commission-rate="0.05" \ + # --commission-max-rate="0.10" \ + # --commission-max-change-rate="0.01" \ + # --min-self-delegation="1000000" \ + # --gas="auto" \ + # --gas-prices="0.025atevmos" \ + # --from='mykey' \ + # --home="${HOMEDIR}" # evmosd tx staking create-validator \ # --amount=1000000000atevmos \ @@ -163,6 +177,8 @@ fi # --gas-prices="0.025atevmos" \ # --from='mykey' + + # Start the node evmosd start \ --metrics "$TRACE" \ @@ -171,8 +187,8 @@ evmosd start \ --home "$HOMEDIR" -evmosd start \ - --metrics "" \ - --log_level info \ - --json-rpc.api eth,txpool,personal,net,debug,web3 \ - --home "/evmos/evmosd" +# evmosd start \ + # --metrics "" \ + # --log_level info \ + # --json-rpc.api eth,txpool,personal,net,debug,web3 \ + # --home "/evmos/evmosd" diff --git a/k8s/README.md b/k8s/README.md index 050eb03..2c62a8c 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -1,15 +1,26 @@ -# Terraform K8s setup in GKE -Terraform K8s setup of a public and a private GKE cluster -Requires access to gcloud via oauth +# Private Zonal Cluster Terraform Setup +Create a simple private or public cluster in a single zone. +## Inputs +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| cluster\_name\_suffix | A suffix to append to the default cluster name | `string` | `""` | no | +| ip\_range\_pods | The secondary ip range to use for pods | `any` | n/a | yes | +| ip\_range\_services | The secondary ip range to use for services | `any` | n/a | yes | +| network | The VPC network to host the cluster in | `any` | n/a | yes | +| project\_id | The project ID to host the cluster in | `any` | n/a | yes | +| region | The region to host the cluster in | `any` | n/a | yes | +| subnetwork | The subnetwork to host the cluster in | `any` | n/a | yes | +| zones | The zone to host the cluster in (required if is a zonal cluster) | `list(string)` | n/a | yes | - Use an oauth token, such as this example from a GKE cluster. The google_client_config data source fetches a token from the Google Authorization server, which expires in 1 hour by default. -```json -provider "kubernetes" { - host = "https://${data.google_container_cluster.my_cluster.endpoint}" - token = data.google_client_config.default.access_token - cluster_ca_certificate = base64decode(data.google_container_cluster.my_cluster.master_auth[0].cluster_ca_certificate) -} -``` +## Update Inputs In terraform.tfvars +The code is pretty generic, most variable can directly be set in the .tfvars file +A generic VPC confuration is set in private-vpc.tf. Edit as needed. + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure \ No newline at end of file diff --git a/k8s/private-cluster/README.md b/k8s/private-cluster/README.md deleted file mode 100644 index b2e4ffa..0000000 --- a/k8s/private-cluster/README.md +++ /dev/null @@ -1,26 +0,0 @@ -# Private Zonal Cluster Terraform Setup - -Create a simple private cluster in a single zone. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| cluster\_name\_suffix | A suffix to append to the default cluster name | `string` | `""` | no | -| ip\_range\_pods | The secondary ip range to use for pods | `any` | n/a | yes | -| ip\_range\_services | The secondary ip range to use for services | `any` | n/a | yes | -| network | The VPC network to host the cluster in | `any` | n/a | yes | -| project\_id | The project ID to host the cluster in | `any` | n/a | yes | -| region | The region to host the cluster in | `any` | n/a | yes | -| subnetwork | The subnetwork to host the cluster in | `any` | n/a | yes | -| zones | The zone to host the cluster in (required if is a zonal cluster) | `list(string)` | n/a | yes | - -## Update Inputs In terraform.tfvars -The code is pretty generic, most variable can directly be set in the .tfvars file -A generic VPC confuration is set in private-vpc.tf. Edit as needed. - -To provision this example, run the following from within this directory: -- `terraform init` to get the plugins -- `terraform plan` to see the infrastructure plan -- `terraform apply` to apply the infrastructure build -- `terraform destroy` to destroy the built infrastructure \ No newline at end of file