Integration Testing setup

.github/workflows/go_test.yml

@@ -0,0 +1,28 @@
+name: Go_test
+on: [push]
+
+jobs:
+    build:
+        runs-on: ubuntu-latest
+        services:
+            dind:
+                image: docker:23.0-rc-dind-rootless
+                ports:
+                    - 2375:2375
+        steps:
+            - name: Install and start kurtosis
+              run: |
+                  echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
+                  sudo apt update
+                  sudo apt install kurtosis-cli
+                  kurtosis engine restart
+
+            - uses: actions/checkout@v3
+            - name: Setup Go
+              uses: actions/setup-go@v4
+              with:
+                  go-version: "1.19.x"
+            - name: Install dependencies
+              run: go mod download all
+            - name: Test with the Go CLI
+              run: go test ./... -v

go.mod

@@ -3,29 +3,32 @@ module github.com/stader-labs/stader-node
 go 1.16
 
 require (
-	github.com/Microsoft/go-winio v0.6.0 // indirect
+	github.com/Masterminds/semver/v3 v3.2.1 // indirect
 	github.com/a8m/envsubst v1.3.0
 	github.com/alessio/shellescape v1.4.1
 	github.com/blang/semver/v4 v4.0.0
 	github.com/btcsuite/btcd v0.23.1
 	github.com/btcsuite/btcd/btcutil v1.1.2
 	github.com/docker/distribution v2.8.1+incompatible // indirect
 	github.com/docker/docker v23.0.3+incompatible
-	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/dustin/go-humanize v1.0.0
 	github.com/ethereum/go-ethereum v1.10.26
 	github.com/fatih/color v1.13.0
 	github.com/ferranbt/fastssz v0.1.2
 	github.com/glendc/go-external-ip v0.1.0
 	github.com/google/uuid v1.3.0
-	github.com/herumi/bls-eth-go-binary v1.28.1 // indirect
+	github.com/herumi/bls-eth-go-binary v1.28.1
 	github.com/imdario/mergo v0.3.13
 	github.com/klauspost/cpuid/v2 v2.1.1
+	github.com/kurtosis-tech/kurtosis-portal/api/golang v0.0.0-20230411133558-b983d9bebe4f // indirect
+	github.com/kurtosis-tech/kurtosis/api/golang v0.80.4
+	github.com/kurtosis-tech/kurtosis/grpc-file-transfer/golang v0.0.0-20230609162710-10b6b91dc9e8 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
-	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.0.2 // indirect
+	github.com/ory/dockertest/v3 v3.10.0
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
 	github.com/prometheus/client_golang v1.13.0
 	github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7
@@ -35,19 +38,23 @@ require (
 	github.com/shirou/gopsutil/v3 v3.23.1
 	github.com/stader-labs/ethcli-ui/configuration v0.0.0-20230619122118-1d15353af546
 	github.com/stader-labs/ethcli-ui/wizard v0.0.0-20230619122118-1d15353af546
+	github.com/stretchr/testify v1.8.4
 	github.com/tyler-smith/go-bip39 v1.1.0
+	github.com/ulikunitz/xz v0.5.11 // indirect
 	github.com/urfave/cli v1.22.10
 	github.com/wealdtech/go-eth2-types/v2 v2.7.0
 	github.com/wealdtech/go-eth2-util v1.7.0
 	github.com/wealdtech/go-eth2-wallet-encryptor-keystorev4 v1.3.0
 	go.uber.org/atomic v1.11.0 // indirect
 	golang.org/x/crypto v0.7.0
+	golang.org/x/net v0.10.0 // indirect
 	golang.org/x/sync v0.1.0
-	golang.org/x/term v0.9.0
+	golang.org/x/term v0.8.0
 	golang.org/x/text v0.10.0 // indirect
-	google.golang.org/grpc v1.49.0 // indirect
+	google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
 	gopkg.in/yaml.v2 v2.4.0
-	gotest.tools/v3 v3.3.0 // indirect
 )
 
 replace github.com/rivo/tview => github.com/hamidraza/tview v1.0.0

shared/services/bc-manager.go

@@ -34,13 +34,17 @@ import (
 )
 
 // This is a proxy for multiple Beacon clients, providing natural fallback support if one of them fails.
+
 type BeaconClientManager struct {
 	primaryBc       beacon.Client
 	fallbackBc      beacon.Client
 	logger          log.ColorLogger
 	primaryReady    bool
 	fallbackReady   bool
 	ignoreSyncCheck bool
+
+	//SAFETY: This flag using in local test only, never set to true in prod
+	localTestnet bool
 }
 
 // This is a signature for a wrapped Beacon client function that only returns an error
@@ -201,7 +205,18 @@ func (m *BeaconClientManager) GetValidatorStatusByIndex(index string, opts *beac
 }
 
 // Get a validator's status by its pubkey
-func (m *BeaconClientManager) GetValidatorStatus(pubkey types.ValidatorPubkey, opts *beacon.ValidatorStatusOptions) (beacon.ValidatorStatus, error) {
+func (m *BeaconClientManager) GetValidatorStatus(
+	pubkey types.ValidatorPubkey,
+	opts *beacon.ValidatorStatusOptions,
+) (beacon.ValidatorStatus, error) {
+	//SAFETY: This flag is true in local test only
+	if m.localTestnet {
+		return beacon.ValidatorStatus{
+			Exists: true,
+			Status: beacon.ValidatorState_PendingInitialized,
+		}, nil
+	}
+
 	result, err := m.runFunction1(func(client beacon.Client) (interface{}, error) {
 		return client.GetValidatorStatus(pubkey, opts)
 	})
@@ -304,6 +319,10 @@ func (m *BeaconClientManager) GetCommitteesForEpoch(epoch *uint64) ([]beacon.Com
 	return result.([]beacon.Committee), nil
 }
 
+func (m *BeaconClientManager) IsLocalTestnet() bool {
+	return m.localTestnet
+}
+
 /// ==================
 /// Internal Functions
 /// ==================

shared/services/config/besu-params.go

@@ -123,6 +123,7 @@ func NewBesuConfig(cfg *StaderConfig) *BesuConfig {
 				config.Network_Prater:   besuTagTest,
 				config.Network_Devnet:   besuTagTest,
 				config.Network_Zhejiang: besuTagTest,
+				config.Network_Local:    besuTagTest,
 			},
 			AffectsContainers:    []config.ContainerID{config.ContainerID_Eth1},
 			EnvironmentVariables: []string{"EC_CONTAINER_TAG"},

shared/services/config/external-configs.go

@@ -197,6 +197,7 @@ func NewExternalLighthouseConfig(cfg *StaderConfig) *ExternalLighthouseConfig {
 				config.Network_Prater:   getLighthouseTagTest(),
 				config.Network_Devnet:   getLighthouseTagTest(),
 				config.Network_Zhejiang: getLighthouseTagTest(),
+				config.Network_Local:    getLighthouseTagTest(),
 			},
 			AffectsContainers:    []config.ContainerID{config.ContainerID_Validator},
 			EnvironmentVariables: []string{"VC_CONTAINER_TAG"},
@@ -282,6 +283,7 @@ func NewExternalPrysmConfig(cfg *StaderConfig) *ExternalPrysmConfig {
 				config.Network_Prater:   getPrysmVcTestTag(),
 				config.Network_Devnet:   getPrysmVcTestTag(),
 				config.Network_Zhejiang: getLighthouseTagTest(),
+				config.Network_Local:    getLighthouseTagTest(),
 			},
 			AffectsContainers:    []config.ContainerID{config.ContainerID_Validator},
 			EnvironmentVariables: []string{"VC_CONTAINER_TAG"},
@@ -356,6 +358,7 @@ func NewExternalNimbusConfig(cfg *StaderConfig) *ExternalNimbusConfig {
 				config.Network_Prater:   nimbusVcTagTest,
 				config.Network_Devnet:   nimbusVcTagTest,
 				config.Network_Zhejiang: nimbusVcTagTest,
+				config.Network_Local:    nimbusVcTagTest,
 			},
 			AffectsContainers:    []config.ContainerID{config.ContainerID_Validator},
 			EnvironmentVariables: []string{"VC_CONTAINER_TAG"},
@@ -417,6 +420,7 @@ func NewExternalTekuConfig(cfg *StaderConfig) *ExternalTekuConfig {
 				config.Network_Prater:   tekuTagTest,
 				config.Network_Devnet:   tekuTagTest,
 				config.Network_Zhejiang: tekuTagTest,
+				config.Network_Local:    tekuTagTest,
 			},
 			AffectsContainers:    []config.ContainerID{config.ContainerID_Validator},
 			EnvironmentVariables: []string{"VC_CONTAINER_TAG"},

shared/services/config/geth-params.go

@@ -124,6 +124,7 @@ func NewGethConfig(cfg *StaderConfig) *GethConfig {
 				config.Network_Prater:   gethTagTest,
 				config.Network_Devnet:   gethTagTest,
 				config.Network_Zhejiang: gethTagTest,
+				config.Network_Local:    gethTagTest,
 			},
 			AffectsContainers:    []config.ContainerID{config.ContainerID_Eth1},
 			EnvironmentVariables: []string{"EC_CONTAINER_TAG"},

shared/services/config/lighthouse-config.go

@@ -79,6 +79,7 @@ func NewLighthouseConfig(cfg *StaderConfig) *LighthouseConfig {
 				config.Network_Prater:   getLighthouseTagTest(),
 				config.Network_Devnet:   getLighthouseTagTest(),
 				config.Network_Zhejiang: getLighthouseTagTest(),
+				config.Network_Local:    getLighthouseTagTest(),
 			},
 			AffectsContainers:    []config.ContainerID{config.ContainerID_Eth2, config.ContainerID_Validator},
 			EnvironmentVariables: []string{"BN_CONTAINER_TAG", "VC_CONTAINER_TAG"},

shared/services/config/local-testnet-presign-public-key.txt

@@ -0,0 +1 @@
+LS0tLS1CRUdJTiBSU0EgUFVCTElDIEtFWS0tLS0tCk1JSUNJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBZzhBTUlJQ0NnS0NBZ0VBdkJYVm5iWit0QWMzQy81eWpsQ2oKdnhXN3UwVEpTNjBPN3hjeHB1TEE0OUdhM0xYT0V4UG9wY1RhU3U1MTRkSkFYOC9Bdm5XZEtwQ29hL2x0V05wQgplNzk1eXRYNU53eGFiV3kvd2Y4ZG04ZEJBZ2lMUGhNdGZwTXd5eThNS1QwOXhkcU1oRENPQ3BwcG51a1Q5UU5KClVyMGlkV2tNU3FTQnM4Tm8yMjlCTG1CWVFTbWJlV3Y0QmRDK3lEL0ZicUVoSXdDL2ZrVy9Nc210dE9WU1RScksKMzhHcm85SmVWVGhBMUpyTXIzUWgyMzJOejhmNzRHbG5qb2ZUVmhjNzVOd00xcWgvV1lPb1N0ZjVobnJ5K1dvSgpieURpb2tpRm9YS1BPcy83NDNkdDJWQWllZTdFd0JDOTZzKzhoSnNLTW5hZzYyUFRtRDQwS212V1Z1aU9UTngxCmNhWG1RNzQ2eVJkaHVubU8wMjNJb3JaRGsrVTZTQStwcmpXaFE5c1oxWWw5NFA5NklBZzFvV2VtQmZLb3lHYWIKZG9ndTdRU0QyRit1cHJ0TEFOYks4TlhVL1pUT2l5MUtBd01tWjVYaWdDODRySUFaeEo2RUo5M0hheDZRM2hIagpBM0grUGdqTzFYSVdPL1hybmh0SWFoM2J6c2N6U2RJMkRMOHFyaWNnOThxY3FlbnVyYWh1dW5SVEFiKzR0M1B1CmNEclVXZVFYa3I1eFVESEQxTHpEeDlvSjI1TmdJVFA2dmtMei9xTjVpZ3BTMnQ1c285d24yWHdlZVZSWC9zRnYKNlpDc1ZOUkhGOS92UHpQcWRJR210OS9NZUZRd2xHSnNURENMLzExWjFQc01iNGpYZlpMNnMvb2xDekVtbVVjTwpTVXpncUNJSXF5UDhwVkJQQ0MwdzkzOENBd0VBQVE9PQotLS0tLUVORCBSU0EgUFVCTElDIEtFWS0tLS0tCg==

shared/services/config/mev-boost-config.go

@@ -417,6 +417,7 @@ func createDefaultRelays() []config.MevRelay {
 				config.Network_Mainnet: "https://0xac6e77dfe25ecd6110b8e780608cce0dab71fdd5ebea22a16c0205200f2f8e2e3ad3b71d3499c54ad14d6c21b41a37ae@boost-relay.flashbots.net?id=staderlabs",
 				config.Network_Prater:  "https://0xafa4c6985aa049fb79dd37010438cfebeb0f2bd42b115b89dd678dab0670c1de38da0c4e9138c9290a398ecd9a0b3110@builder-relay-goerli.flashbots.net?id=staderlabs",
 				config.Network_Devnet:  "https://0xafa4c6985aa049fb79dd37010438cfebeb0f2bd42b115b89dd678dab0670c1de38da0c4e9138c9290a398ecd9a0b3110@builder-relay-goerli.flashbots.net?id=staderlabs",
+				config.Network_Local:   "",
 			},
 			Regulated:     true,
 			NoSandwiching: false,
@@ -568,6 +569,7 @@ func generateProfileParameter(id string, relays []config.MevRelay, regulated boo
 		DescriptionsByNetwork: map[config.Network]string{
 			config.Network_Mainnet: mainnetDescription,
 			config.Network_Prater:  praterDescription,
+			config.Network_Local:   praterDescription,
 		},
 	}
 }

shared/services/config/nethermind-params.go

@@ -155,6 +155,7 @@ func NewNethermindConfig(cfg *StaderConfig) *NethermindConfig {
 				config.Network_Prater:   nethermindTagTest,
 				config.Network_Devnet:   nethermindTagTest,
 				config.Network_Zhejiang: nethermindTagTest,
+				config.Network_Local:    nethermindTagTest,
 			},
 			AffectsContainers:    []config.ContainerID{config.ContainerID_Eth1},
 			EnvironmentVariables: []string{"EC_CONTAINER_TAG"},

shared/services/config/nimbus-config.go

@@ -91,6 +91,7 @@ func NewNimbusConfig(cfg *StaderConfig) *NimbusConfig {
 				config.Network_Prater:   nimbusBnTagTest,
 				config.Network_Devnet:   nimbusBnTagTest,
 				config.Network_Zhejiang: nimbusBnTagTest,
+				config.Network_Local:    nimbusBnTagTest,
 			},
 			AffectsContainers:    []config.ContainerID{config.ContainerID_Eth2},
 			EnvironmentVariables: []string{"BN_CONTAINER_TAG"},
@@ -108,6 +109,7 @@ func NewNimbusConfig(cfg *StaderConfig) *NimbusConfig {
 				config.Network_Prater:   nimbusVcTagTest,
 				config.Network_Devnet:   nimbusVcTagTest,
 				config.Network_Zhejiang: nimbusVcTagTest,
+				config.Network_Local:    nimbusVcTagTest,
 			},
 			AffectsContainers:    []config.ContainerID{config.ContainerID_Validator},
 			EnvironmentVariables: []string{"VC_CONTAINER_TAG"},

shared/services/config/prysm-config.go

@@ -120,6 +120,7 @@ func NewPrysmConfig(cfg *StaderConfig) *PrysmConfig {
 				config.Network_Prater:   getPrysmBnTestTag(),
 				config.Network_Devnet:   getPrysmBnTestTag(),
 				config.Network_Zhejiang: getPrysmBnTestTag(),
+				config.Network_Local:    getPrysmBnTestTag(),
 			},
 			AffectsContainers:    []config.ContainerID{config.ContainerID_Eth2},
 			EnvironmentVariables: []string{"BN_CONTAINER_TAG"},
@@ -137,6 +138,7 @@ func NewPrysmConfig(cfg *StaderConfig) *PrysmConfig {
 				config.Network_Prater:   getPrysmVcTestTag(),
 				config.Network_Devnet:   getPrysmVcTestTag(),
 				config.Network_Zhejiang: getPrysmVcTestTag(),
+				config.Network_Local:    getPrysmVcTestTag(),
 			},
 			AffectsContainers:    []config.ContainerID{config.ContainerID_Validator},
 			EnvironmentVariables: []string{"VC_CONTAINER_TAG"},

shared/services/config/stadernode-config.go

@@ -59,6 +59,11 @@ var stageEncryptionKey string
 //go:embed dev-presign-public-key.txt
 var devEncryptionKey string
 
+// SAFETY: This file using in local test only.
+//
+//go:embed local-testnet-presign-public-key.txt
+var localTestnetEncryptionKey string
+
 // --ignore-sync-check
 // Defaults
 const defaultProjectName string = "stader"
@@ -235,6 +240,7 @@ func NewStadernodeConfig(cfg *StaderConfig) *StaderNodeConfig {
 			config.Network_Mainnet: "https://beaconcha.in",
 			config.Network_Prater:  "https://prater.beaconcha.in",
 			config.Network_Devnet:  "https://prater.beaconcha.in",
+			config.Network_Local:   "https://prater.beaconcha.in",
 		},
 
 		txWatchUrl: map[config.Network]string{
@@ -248,34 +254,39 @@ func NewStadernodeConfig(cfg *StaderConfig) *StaderNodeConfig {
 			config.Network_Prater:   5,       // Goerli
 			config.Network_Devnet:   5,       // Also goerli
 			config.Network_Zhejiang: 1337803, // Zhejiang
+			config.Network_Local:    31337,   // Local testnet
 		},
 
 		ethxTokenAddress: map[config.Network]string{
 			config.Network_Prater:   "0x3338eCd3ab3d3503c55c931d759fA6d78d287236",
 			config.Network_Devnet:   "0x38DE8Df722B4032Cc6987F00bCA0d9B37d9F9438",
 			config.Network_Mainnet:  "0xA35b1B31Ce002FBF2058D22F30f95D405200A15b",
 			config.Network_Zhejiang: "0x90Da3CA75532A17ca38440a32595F036ecE46E85",
+			config.Network_Local:    "0xA15BB66138824a1c7167f5E85b957d04Dd34E468",
 		},
 
 		staderConfigAddress: map[config.Network]string{
 			config.Network_Prater:   "0x2aa6cEd8Cf0a93884216BaE5dbF4299932aB577B",
 			config.Network_Devnet:   "0x749Ed651c4F41E0D705960e815A58815ffFd3afe",
 			config.Network_Mainnet:  "0x4ABEF2263d5A5ED582FC9A9789a41D85b68d69DB",
 			config.Network_Zhejiang: "0x90Da3CA75532A17ca38440a32595F036ecE46E85",
+			config.Network_Local:    "0x700b6A60ce7EaaEA56F065753d8dcB9653dbAD35",
 		},
 
 		baseStaderBackendUrl: map[config.Network]string{
 			config.Network_Prater:   "https://ethx-offchain-preprod.staderlabs.com",
 			config.Network_Devnet:   "https://stage-ethx-offchain.staderlabs.click",
 			config.Network_Mainnet:  "https://ethx-offchain.staderlabs.com",
 			config.Network_Zhejiang: "0x90Da3CA75532A17ca38440a32595F036ecE46E85",
+			config.Network_Local:    "http://localhost:9989",
 		},
 
 		preSignEncryptionKey: map[config.Network]string{
 			config.Network_Prater:   stageEncryptionKey,
 			config.Network_Devnet:   devEncryptionKey,
 			config.Network_Mainnet:  prodEncryptionKey,
 			config.Network_Zhejiang: stageEncryptionKey,
+			config.Network_Local:    localTestnetEncryptionKey,
 		},
 	}
 }
@@ -454,11 +465,12 @@ func (cfg *StaderNodeConfig) GetSpRewardCyclePath(cycle int64, daemon bool) stri
 }
 
 func (cfg *StaderNodeConfig) GetFeeRecipientFilePath() string {
-	if !cfg.parent.IsNativeMode {
+	if cfg.parent != nil && !cfg.parent.IsNativeMode {
 		return filepath.Join(DaemonDataPath, "validators", FeeRecipientFilename)
 	}
 
 	return filepath.Join(cfg.DataPath.Value.(string), "validators", NativeFeeRecipientFilename)
+
 }
 
 func (cfg *StaderNodeConfig) GetClaimData(cycles []*big.Int) ([]*big.Int, []*big.Int, [][][32]byte, error) {

shared/services/config/teku-config.go

@@ -111,6 +111,7 @@ func NewTekuConfig(cfg *StaderConfig) *TekuConfig {
 				config.Network_Prater:   tekuTagTest,
 				config.Network_Devnet:   tekuTagTest,
 				config.Network_Zhejiang: tekuTagTest,
+				config.Network_Local:    tekuTagTest,
 			},
 			AffectsContainers:    []config.ContainerID{config.ContainerID_Eth2, config.ContainerID_Validator},
 			EnvironmentVariables: []string{"BN_CONTAINER_TAG", "VC_CONTAINER_TAG"},

shared/services/gas/gas.go

@@ -220,7 +220,7 @@ func handleEtherchainGasPrices(gasSuggestion etherchain.GasFeeSuggestion, gasInf
 
 		desiredPriceFloat, err := strconv.ParseFloat(desiredPrice, 64)
 		if err != nil {
-			fmt.Printf("Not a valid gas price (%s), try again.", err.Error())
+			fmt.Printf("Not a valid gas price (%+v), try again. \n", err.Error())
 			continue
 		}
 		if desiredPriceFloat <= 0 {
@@ -298,7 +298,7 @@ func handleEtherscanGasPrices(gasSuggestion etherscan.GasFeeSuggestion, gasInfo
 
 		desiredPriceFloat, err := strconv.ParseFloat(desiredPrice, 64)
 		if err != nil {
-			fmt.Println("Not a valid gas price (%s), try again.", err.Error())
+			fmt.Printf("Not a valid gas price (%s), try again.\n", err.Error())
 			continue
 		}
 		if desiredPriceFloat <= 0 {

shared/services/requirements.go

@@ -23,12 +23,13 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"github.com/ethereum/go-ethereum/common"
 	"log"
 	"math/big"
 	"sync"
 	"time"
 
+	"github.com/ethereum/go-ethereum/common"
+
 	"github.com/stader-labs/stader-node/shared/services/config"
 	"github.com/stader-labs/stader-node/stader-lib/node"
 	"github.com/stader-labs/stader-node/stader-lib/stader"
@@ -175,8 +176,13 @@ func WaitNodeRegistered(c *cli.Context, operatorAddress common.Address, verbose
 		}
 		operatorId, err := node.GetOperatorId(pnr, operatorAddress, nil)
 		if err != nil {
+			if verbose {
+				log.Printf("The node is not registered with Stader, retrying in %s...\n", checkNodeRegisteredInterval.String())
+			}
+			time.Sleep(checkNodeRegisteredInterval)
 			return err
 		}
+
 		if operatorId.Cmp(big.NewInt(0)) != 0 {
 			return nil
 		}
@@ -498,7 +504,7 @@ func waitBeaconClientSynced(c *cli.Context, verbose bool, timeout int64) (bool,
 		// Check sync status
 		if syncStatus.Syncing {
 			if verbose {
-				log.Println("Eth 2.0 node syncing: %.2f%%\n", syncStatus.Progress*100)
+				log.Printf("Eth 2.0 node syncing: %.2f%% \n", syncStatus.Progress*100)
 			}
 		} else {
 			return true, nil