feat: allow consumers to have different keys than providers (#1175) (#…

…1180)

* feat: allow consumers to have different keys than providers

This reflects how ICS chains are launched in the wild, with validators
setting their consensus keys before the chain launches

* Parameterize ICS_SPAWN_TIME_WAIT

chain/cosmos/cosmos_chain.go

@@ -832,6 +832,10 @@ type ValidatorWithIntPower struct {
 
 // Bootstraps the chain and starts it from genesis
 func (c *CosmosChain) Start(testName string, ctx context.Context, additionalGenesisWallets ...ibc.WalletAmount) error {
+	if c.cfg.InterchainSecurityConfig.ConsumerCopyProviderKey != nil && c.Provider == nil {
+		return fmt.Errorf("don't set ConsumerCopyProviderKey if it's not a consumer chain")
+	}
+
 	chainCfg := c.Config()
 
 	decimalPow := int64(math.Pow10(int(*chainCfg.CoinDecimals)))

chain/cosmos/ics.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"path"
 	"strconv"
+	"strings"
 	"time"
 
 	sdkmath "cosmossdk.io/math"
@@ -20,6 +21,7 @@ import (
 	"github.com/strangelove-ventures/interchaintest/v8/ibc"
 	"github.com/strangelove-ventures/interchaintest/v8/testreporter"
 	"github.com/strangelove-ventures/interchaintest/v8/testutil"
+	"github.com/tidwall/gjson"
 	"go.uber.org/zap"
 	"golang.org/x/mod/semver"
 	"golang.org/x/sync/errgroup"
@@ -55,23 +57,27 @@ func (c *CosmosChain) FinishICSProviderSetup(ctx context.Context, r ibc.Relayer,
 
 	providerVal := stakingVals[0]
 
-	beforeDel, err := c.StakingQueryDelegationsTo(ctx, providerVal.OperatorAddress)
-	if err != nil {
-		return fmt.Errorf("failed to query delegations to validator: %w", err)
-	}
-
 	err = c.GetNode().StakingDelegate(ctx, "validator", providerVal.OperatorAddress, fmt.Sprintf("1000000%s", c.Config().Denom))
 	if err != nil {
 		return fmt.Errorf("failed to delegate to validator: %w", err)
 	}
 
-	afterDel, err := c.StakingQueryDelegationsTo(ctx, providerVal.OperatorAddress)
+	stakingVals, err = c.StakingQueryValidators(ctx, stakingttypes.BondStatusBonded)
 	if err != nil {
-		return fmt.Errorf("failed to query delegations to validator: %w", err)
+		return fmt.Errorf("failed to query validators: %w", err)
 	}
-
-	if afterDel[0].Balance.Amount.LT(beforeDel[0].Balance.Amount) {
-		return fmt.Errorf("delegation failed: %w", err)
+	var providerAfter *stakingttypes.Validator
+	for _, val := range stakingVals {
+		if val.OperatorAddress == providerVal.OperatorAddress {
+			providerAfter = &val
+			break
+		}
+	}
+	if providerAfter == nil {
+		return fmt.Errorf("failed to find provider validator after delegation")
+	}
+	if providerAfter.Tokens.LT(providerVal.Tokens) {
+		return fmt.Errorf("delegation failed; before: %v, after: %v", providerVal.Tokens, providerAfter.Tokens)
 	}
 
 	return c.FlushPendingICSPackets(ctx, r, eRep, ibcPath)
@@ -96,6 +102,9 @@ func (c *CosmosChain) FlushPendingICSPackets(ctx context.Context, r ibc.Relayer,
 
 // Bootstraps the provider chain and starts it from genesis
 func (c *CosmosChain) StartProvider(testName string, ctx context.Context, additionalGenesisWallets ...ibc.WalletAmount) error {
+	if c.cfg.InterchainSecurityConfig.ConsumerCopyProviderKey != nil {
+		return fmt.Errorf("don't set ConsumerCopyProviderKey in the provider chain")
+	}
 	existingFunc := c.cfg.ModifyGenesis
 	c.cfg.ModifyGenesis = func(cc ibc.ChainConfig, b []byte) ([]byte, error) {
 		var err error
@@ -140,6 +149,14 @@ func (c *CosmosChain) StartProvider(testName string, ctx context.Context, additi
 		return fmt.Errorf("failed to parse trusting period in 'StartProvider': %w", err)
 	}
 
+	spawnTimeWait := os.Getenv("ICS_SPAWN_TIME_WAIT")
+	if spawnTimeWait == "" {
+		spawnTimeWait = "45s"
+	}
+	spawnTimeWaitDuration, err := time.ParseDuration(spawnTimeWait)
+	if err != nil {
+		return fmt.Errorf("invalid ICS_SPAWN_TIME_WAIT %s: %w", spawnTimeWait, err)
+	}
 	for _, consumer := range c.Consumers {
 		prop := ccvclient.ConsumerAdditionProposalJSON{
 			Title:         fmt.Sprintf("Addition of %s consumer chain", consumer.cfg.Name),
@@ -148,7 +165,7 @@ func (c *CosmosChain) StartProvider(testName string, ctx context.Context, additi
 			InitialHeight: clienttypes.Height{RevisionNumber: clienttypes.ParseChainID(consumer.cfg.ChainID), RevisionHeight: 1},
 			GenesisHash:   []byte("gen_hash"),
 			BinaryHash:    []byte("bin_hash"),
-			SpawnTime:     time.Now(), // Client on provider tracking consumer will be created as soon as proposal passes
+			SpawnTime:     time.Now().Add(spawnTimeWaitDuration),
 
 			// TODO fetch or default variables
 			BlocksPerDistributionTransmission: 1000,
@@ -230,13 +247,34 @@ func (c *CosmosChain) StartConsumer(testName string, ctx context.Context, additi
 
 	// Copy provider priv val keys to these nodes
 	for i, val := range c.Provider.Validators {
-		privVal, err := val.PrivValFileContent(ctx)
-		if err != nil {
-			return err
-		}
-		if err := c.Validators[i].OverwritePrivValFile(ctx, privVal); err != nil {
-			return err
-		}
+		i := i
+		val := val
+		eg.Go(func() error {
+			copy := c.cfg.InterchainSecurityConfig.ConsumerCopyProviderKey != nil && c.cfg.InterchainSecurityConfig.ConsumerCopyProviderKey(i)
+			if copy {
+				privVal, err := val.PrivValFileContent(ctx)
+				if err != nil {
+					return err
+				}
+				if err := c.Validators[i].OverwritePrivValFile(ctx, privVal); err != nil {
+					return err
+				}
+			} else {
+				key, _, err := c.Validators[i].ExecBin(ctx, "tendermint", "show-validator")
+				if err != nil {
+					return fmt.Errorf("failed to get consumer validator pubkey: %w", err)
+				}
+				keyStr := strings.TrimSpace(string(key))
+				_, err = c.Provider.Validators[i].ExecTx(ctx, valKey, "provider", "assign-consensus-key", c.cfg.ChainID, keyStr)
+				if err != nil {
+					return fmt.Errorf("failed to assign consumer validator pubkey: %w", err)
+				}
+			}
+			return nil
+		})
+	}
+	if err := eg.Wait(); err != nil {
+		return err
 	}
 
 	if c.cfg.PreGenesis != nil {
@@ -246,6 +284,18 @@ func (c *CosmosChain) StartConsumer(testName string, ctx context.Context, additi
 		}
 	}
 
+	// Wait for spawn time
+	proposals, _, err := c.Provider.GetNode().ExecQuery(ctx, "gov", "proposals")
+	if err != nil {
+		return fmt.Errorf("failed to query proposed chains: %w", err)
+	}
+	spawnTime := gjson.GetBytes(proposals, fmt.Sprintf("proposals.#(messages.0.content.chain_id==%q).messages.0.content.spawn_time", c.cfg.ChainID)).Time()
+	c.log.Info("Waiting for chain to spawn", zap.Time("spawn_time", spawnTime), zap.String("chain_id", c.cfg.ChainID))
+	time.Sleep(time.Until(spawnTime))
+	if err := testutil.WaitForBlocks(ctx, 2, c.Provider); err != nil {
+		return err
+	}
+
 	validator0 := c.Validators[0]
 
 	for _, wallet := range additionalGenesisWallets {

docs/envOptions.md

@@ -19,3 +19,5 @@
     - Set to `"always"` to show logs for both pass and fail.
     - Set to `"never"` to never show any logs.
     - Leave unset to show logs only for failed tests.
+
+- `ICS_SPAWN_TIME_WAIT`: A go duration (e.g. 2m) that specifies how long to wait for ICS chains to spawn

examples/ibc/ics_test.go

@@ -20,7 +20,7 @@ import (
 
 var (
 	icsVersions     = []string{"v3.1.0", "v3.3.0", "v4.0.0"}
-	vals            = 1
+	vals            = 2
 	fNodes          = 0
 	providerChainID = "provider-1"
 )
@@ -63,7 +63,11 @@ func icsTest(t *testing.T, version string) {
 		{
 			Name: "ics-consumer", Version: version,
 			NumValidators: &vals, NumFullNodes: &fNodes,
-			ChainConfig: ibc.ChainConfig{GasAdjustment: 1.5, ChainID: "consumer-1", Bech32Prefix: consumerBechPrefix},
+			ChainConfig: ibc.ChainConfig{GasAdjustment: 1.5, ChainID: "consumer-1", Bech32Prefix: consumerBechPrefix, InterchainSecurityConfig: ibc.ICSConfig{
+				ConsumerCopyProviderKey: func(i int) bool {
+					return i == 0
+				},
+			}},
 		},
 	})
 
@@ -128,6 +132,20 @@ func icsTest(t *testing.T, version string) {
 		require.EqualValues(t, amt, bal)
 	})
 
+	t.Run("validate consumer keys copied", func(t *testing.T) {
+		providerKey0, err := provider.Validators[0].PrivValFileContent(ctx)
+		require.NoError(t, err)
+		consumerKey0, err := consumer.Validators[0].PrivValFileContent(ctx)
+		require.NoError(t, err)
+		require.Equal(t, providerKey0, consumerKey0)
+
+		providerKey1, err := provider.Validators[1].PrivValFileContent(ctx)
+		require.NoError(t, err)
+		consumerKey1, err := consumer.Validators[1].PrivValFileContent(ctx)
+		require.NoError(t, err)
+		require.NotEqual(t, providerKey1, consumerKey1)
+	})
+
 	t.Run("provider -> consumer IBC transfer", func(t *testing.T) {
 		providerChannelInfo, err := r.GetChannels(ctx, eRep, provider.Config().ChainID)
 		require.NoError(t, err)

go.mod

@@ -46,6 +46,7 @@ require (
 	github.com/rivo/tview v0.0.0-20220307222120-9994674d60a8
 	github.com/spf13/cobra v1.8.0
 	github.com/stretchr/testify v1.9.0
+	github.com/tidwall/gjson v1.17.1
 	github.com/tyler-smith/go-bip32 v1.0.0
 	github.com/tyler-smith/go-bip39 v1.1.0
 	go.uber.org/multierr v1.11.0
@@ -234,6 +235,8 @@ require (
 	github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect
 	github.com/tendermint/go-amino v0.16.0 // indirect
 	github.com/tidwall/btree v1.7.0 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/ulikunitz/xz v0.5.11 // indirect
 	github.com/zondax/hid v0.9.2 // indirect
 	github.com/zondax/ledger-go v0.14.3 // indirect

go.sum

@@ -1132,6 +1132,12 @@ github.com/tendermint/go-amino v0.16.0 h1:GyhmgQKvqF82e2oZeuMSp9JTN0N09emoSZlb2l
 github.com/tendermint/go-amino v0.16.0/go.mod h1:TQU0M1i/ImAo+tYpZi73AU3V/dKeCoMC9Sphe2ZwGME=
 github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI=
 github.com/tidwall/btree v1.7.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY=
+github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U=
+github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU=
 github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
 github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk=

ibc/types.go

@@ -15,6 +15,7 @@ import (
 	ibcexported "github.com/cosmos/ibc-go/v8/modules/core/03-connection/types"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/client"
+	"github.com/google/go-cmp/cmp"
 )
 
 // ChainConfig defines the chain parameters requires to run an interchaintest testnet for a chain.
@@ -220,7 +221,7 @@ func (c ChainConfig) MergeChainSpecConfig(other ChainConfig) ChainConfig {
 		c.ExposeAdditionalPorts = append(c.ExposeAdditionalPorts, other.ExposeAdditionalPorts...)
 	}
 
-	if other.InterchainSecurityConfig != (ICSConfig{}) {
+	if !cmp.Equal(other.InterchainSecurityConfig, ICSConfig{}) {
 		c.InterchainSecurityConfig = other.InterchainSecurityConfig
 	}
 
@@ -419,6 +420,7 @@ type PathUpdateOptions struct {
 }
 
 type ICSConfig struct {
-	ProviderVerOverride string `yaml:"provider,omitempty" json:"provider,omitempty"`
-	ConsumerVerOverride string `yaml:"consumer,omitempty" json:"consumer,omitempty"`
+	ProviderVerOverride     string `yaml:"provider,omitempty" json:"provider,omitempty"`
+	ConsumerVerOverride     string `yaml:"consumer,omitempty" json:"consumer,omitempty"`
+	ConsumerCopyProviderKey func(int) bool
 }

local-interchain/go.mod

@@ -222,6 +222,9 @@ require (
 	github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect
 	github.com/tendermint/go-amino v0.16.0 // indirect
 	github.com/tidwall/btree v1.7.0 // indirect
+	github.com/tidwall/gjson v1.17.1 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/tyler-smith/go-bip32 v1.0.0 // indirect
 	github.com/tyler-smith/go-bip39 v1.1.0 // indirect
 	github.com/ulikunitz/xz v0.5.11 // indirect

local-interchain/go.sum

@@ -1113,6 +1113,12 @@ github.com/tendermint/go-amino v0.16.0 h1:GyhmgQKvqF82e2oZeuMSp9JTN0N09emoSZlb2l
 github.com/tendermint/go-amino v0.16.0/go.mod h1:TQU0M1i/ImAo+tYpZi73AU3V/dKeCoMC9Sphe2ZwGME=
 github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI=
 github.com/tidwall/btree v1.7.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY=
+github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U=
+github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU=
 github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
 github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk=