diff --git a/test/testground/go.sum b/test/testground/go.sum index 7817f58b04..d4069c77ca 100644 --- a/test/testground/go.sum +++ b/test/testground/go.sum @@ -192,8 +192,8 @@ github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOC github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/celestiaorg/celestia-app v1.0.0-rc0.0.20230919215657-e2d325755a40 h1:cgvExGQAZExH+rnQL6zOHsDewm4XG6ZwfkJ5JPD+Udc= github.com/celestiaorg/celestia-app v1.0.0-rc0.0.20230919215657-e2d325755a40/go.mod h1:mlISBhKW66V9PkDqQegyJPS4380TT0Y6qNw/5IurpcM= -github.com/celestiaorg/celestia-core v1.26.2-tm-v0.34.28 h1:2efXQaggLFknz0wQufr4nUEz5G7pSVHS1j7NuJDsvII= -github.com/celestiaorg/celestia-core v1.26.2-tm-v0.34.28/go.mod h1:++dNzzzjP9jYg+NopN9G8sg1HEZ58lv1TPtg71evZ0E= +github.com/celestiaorg/celestia-core v1.27.0-tm-v0.34.28 h1:BE7JFZ1SYpwM9OfL9cPcjlO5xjIbDPgdFkJNouyl6jA= +github.com/celestiaorg/celestia-core v1.27.0-tm-v0.34.28/go.mod h1:1GT0RfdNqOXvyR3Hq4ROcNBknQNz9E6K5l3Cla9eFFk= github.com/celestiaorg/cosmos-sdk v1.18.0-sdk-v0.46.14 h1:dDfoQJOlVNj4HufJ1lBLTo2k3/L/255MIiKmEQziDmw= github.com/celestiaorg/cosmos-sdk v1.18.0-sdk-v0.46.14/go.mod h1:kkdiHo/zG6ar80730+bG1owdMAQXrGp4utFu7mbfADo= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 h1:CJdIpo8n5MFP2MwK0gSRcOVlDlFdQJO1p+FqdxYzmvc= diff --git a/test/testground/network/consensus_node.go b/test/testground/network/consensus_node.go index 10401625e2..50f1624f15 100644 --- a/test/testground/network/consensus_node.go +++ b/test/testground/network/consensus_node.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "errors" - "fmt" "os" "path/filepath" @@ -14,9 +13,9 @@ import ( appns "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/celestia-app/pkg/user" "github.com/celestiaorg/celestia-app/test/util/blobfactory" + "github.com/celestiaorg/celestia-app/test/util/genesis" "github.com/celestiaorg/celestia-app/test/util/testnode" blobtypes "github.com/celestiaorg/celestia-app/x/blob/types" - "github.com/cosmos/cosmos-sdk/crypto/hd" "github.com/cosmos/cosmos-sdk/crypto/keyring" srvconfig "github.com/cosmos/cosmos-sdk/server/config" srvtypes "github.com/cosmos/cosmos-sdk/server/types" @@ -29,63 +28,31 @@ import ( "github.com/tendermint/tendermint/node" "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/privval" + "github.com/testground/sdk-go/run" "github.com/testground/sdk-go/runtime" ) -// NodeConfig is a portable configuration for a node. This is originally created -// and published by the Leader node and then downloaded by the other follower -// nodes. It is used to create a consensus node that -type NodeConfig struct { - Status Status `json:"status"` - NodeType string `json:"node_type"` - Name string `json:"name"` - ChainID string `json:"chain_id,omitempty"` - StartHeight int64 `json:"start_height"` - Keys KeySet `json:"keys"` - CmtConfig tmconfig.Config `json:"cmt_config"` - AppConfig srvconfig.Config `json:"app_config"` - P2PID string `json:"p2p_id"` - // HaltHeight is the height at which all nodes will halt and finish the - // execution portion of the test. - HaltHeight int `json:"halt_height"` -} - -type KeySet struct { +// ConsensusNode is the node type used by testground instances to run a +// celestia-app full node. It can optionally be configured to be a validator, +// and has methods to boostrap a network, initialize itself, start, and stop. +type ConsensusNode struct { + Name string // NetworkKey is the key used for signing gossiped messages. - NetworkKey ed25519.PrivKey `json:"network_key"` + networkKey ed25519.PrivKey // ConsensusKey is the key used for signing votes. - ConsensusKey ed25519.PrivKey `json:"consensus_key"` - // AccountKey is the key used for signing transactions. - AccountMnemonic string `json:"account_mnemonic"` -} + consensusKey ed25519.PrivKey -func (c *Config) ConsensusNode(globalSequence int) (*ConsensusNode, error) { - if len(c.Nodes) < globalSequence { - return nil, fmt.Errorf("node %d not found", globalSequence) - } - // find a node with the provided global sequence - var ncfg NodeConfig - for _, cfg := range c.Nodes { - if cfg.Status.GlobalSequence == int64(globalSequence) { - ncfg = cfg - break - } - } - ncfg.ChainID = c.ChainID - return NewConsensusNode(c.Genesis, ncfg) -} + kr keyring.Keyring + ecfg encoding.Config -// NodeID creates the ID for each node. This is currently just the global -// sequence. -func NodeID(globalSequence int64) string { - return fmt.Sprintf("%d", globalSequence) -} + params *Params + CmtNode *node.Node + CmtConfig *tmconfig.Config + AppConfig *srvconfig.Config + baseDir string + + cctx testnode.Context -type ConsensusNode struct { - NodeConfig - kr keyring.Keyring - genesis json.RawMessage - ecfg encoding.Config stopFuncs []func() error // AppOptions are the application options of the test node. AppOptions *testnode.KVAppOptions @@ -93,108 +60,149 @@ type ConsensusNode struct { AppCreator srvtypes.AppCreator cmtNode *node.Node - cctx testnode.Context } -func NewConsensusNode(genesis json.RawMessage, cfg NodeConfig) (*ConsensusNode, error) { - ecfg := encoding.MakeConfig(app.ModuleEncodingRegisters...) - kr, err := ImportKey(keyring.NewInMemory(ecfg.Codec), cfg.Keys.AccountMnemonic, cfg.Name) +// Bootstrap is the first function called in a test by each node. It is +// responsible for initializing the node and creating a gentx if this node is a +// validator. +func (cn *ConsensusNode) Bootstrap(ctx context.Context, runenv *runtime.RunEnv, initCtx *run.InitContext) ([]PeerPacket, error) { + cn.ecfg = encoding.MakeConfig(app.ModuleBasics) + + ip, err := initCtx.NetClient.GetDataNetworkIP() + if err != nil { + return nil, err + } + + params, err := ParseParams(runenv) + if err != nil { + return nil, err + } + cn.params = params + + nodeID := NodeID(initCtx.GlobalSeq) + cn.Name = nodeID + + kr, addrs := testnode.NewKeyring(nodeID, TxSimAccountName) + cn.kr = kr + + val := genesis.NewDefaultValidator(nodeID) + cn.consensusKey = val.ConsensusKey + cn.networkKey = val.NetworkKey + + var bz []byte + if runenv.TestGroupID == ValidatorGroupID { + gentx, err := val.GenTx(cn.ecfg, cn.kr, cn.params.ChainID) + if err != nil { + return nil, err + } + bz, err = cn.ecfg.TxConfig.TxJSONEncoder()(gentx) + if err != nil { + return nil, err + } + } + + pubKs, err := getPublicKeys(cn.kr, nodeID, TxSimAccountName) + if err != nil { + return nil, err + } + + pp := PeerPacket{ + PeerID: peerID(ip.String(), cn.networkKey), + IP: ip.String(), + GroupID: runenv.TestGroupID, + GlobalSequence: initCtx.GlobalSeq, + GenesisAccounts: addrsToStrings(addrs...), + GenesisPubKeys: pubKs, + GenTx: json.RawMessage(bz), + } + + _, err = initCtx.SyncClient.Publish(ctx, PeerPacketTopic, pp) if err != nil { - return nil, fmt.Errorf("failed to import key: %w", err) - } - return &ConsensusNode{ - genesis: genesis, - NodeConfig: cfg, - AppCreator: cmd.NewAppServer, - AppOptions: testnode.DefaultAppOptions(), - ecfg: ecfg, - kr: kr, - }, nil + return nil, err + } + + return DownloadSync(ctx, initCtx, PeerPacketTopic, PeerPacket{}, runenv.TestInstanceCount) } // Init creates the files required by tendermint and celestia-app using the data // downloaded from the Leader node. -func (c *ConsensusNode) Init(baseDir string) (string, error) { - basePath := filepath.Join(baseDir, ".celestia-app") - c.CmtConfig.SetRoot(basePath) +func (cn *ConsensusNode) Init(baseDir string, genesis json.RawMessage, mcfg ConsensusNodeMetaConfig) error { + cn.CmtConfig = mcfg.CmtConfig + cn.AppConfig = mcfg.AppConfig + cn.AppCreator = cmd.NewAppServer + cn.AppOptions = testnode.DefaultAppOptions() + + baseDir = filepath.Join(baseDir, ".celestia-app") + cn.baseDir = baseDir + + cn.CmtConfig.SetRoot(baseDir) // save the genesis file - configPath := filepath.Join(basePath, "config") + configPath := filepath.Join(baseDir, "config") err := os.MkdirAll(configPath, os.ModePerm) if err != nil { - return "", err + return err } // save the genesis file as configured - err = cmtos.WriteFile(c.CmtConfig.GenesisFile(), c.genesis, 0o644) + err = cmtos.WriteFile(cn.CmtConfig.GenesisFile(), genesis, 0o644) if err != nil { - return "", err + return err } - pvStateFile := c.CmtConfig.PrivValidatorStateFile() + pvStateFile := cn.CmtConfig.PrivValidatorStateFile() if err := tmos.EnsureDir(filepath.Dir(pvStateFile), 0o777); err != nil { - return "", err + return err } - pvKeyFile := c.CmtConfig.PrivValidatorKeyFile() + pvKeyFile := cn.CmtConfig.PrivValidatorKeyFile() if err := tmos.EnsureDir(filepath.Dir(pvKeyFile), 0o777); err != nil { - return "", err + return err } - filePV := privval.NewFilePV(c.Keys.ConsensusKey, pvKeyFile, pvStateFile) + filePV := privval.NewFilePV(cn.consensusKey, pvKeyFile, pvStateFile) filePV.Save() - nodeKeyFile := c.CmtConfig.NodeKeyFile() + nodeKeyFile := cn.CmtConfig.NodeKeyFile() if err := tmos.EnsureDir(filepath.Dir(nodeKeyFile), 0o777); err != nil { - return "", err + return err } nodeKey := &p2p.NodeKey{ - PrivKey: c.Keys.NetworkKey, + PrivKey: cn.networkKey, } if err := nodeKey.SaveAs(nodeKeyFile); err != nil { - return "", err + return err } - return basePath, nil + return nil } // StartNode uses the testnode package to start a tendermint node with // celestia-app and the provided configuration. -func (c *ConsensusNode) StartNode(ctx context.Context, baseDir string) error { - ucfg := c.UniversalTestingConfig() +func (cn *ConsensusNode) StartNode(ctx context.Context, baseDir string) error { + ucfg := cn.UniversalTestingConfig() tmNode, app, err := testnode.NewCometNode(baseDir, &ucfg) if err != nil { return err } - c.cmtNode = tmNode - cctx := testnode.NewContext(ctx, c.kr, ucfg.TmConfig, c.ChainID) + cn.cmtNode = tmNode + cctx := testnode.NewContext(ctx, cn.kr, ucfg.TmConfig, cn.params.ChainID) cctx, stopNode, err := testnode.StartNode(tmNode, cctx) - c.stopFuncs = append(c.stopFuncs, stopNode) + cn.stopFuncs = append(cn.stopFuncs, stopNode) if err != nil { return err } cctx, cleanupGRPC, err := testnode.StartGRPCServer(app, ucfg.AppConfig, cctx) - c.stopFuncs = append(c.stopFuncs, cleanupGRPC) + cn.stopFuncs = append(cn.stopFuncs, cleanupGRPC) - c.cctx = cctx + cn.cctx = cctx return err } -// UniversalTestingConfig returns the configuration used by the testnode package. -func (c *ConsensusNode) UniversalTestingConfig() testnode.UniversalTestingConfig { - return testnode.UniversalTestingConfig{ - TmConfig: &c.CmtConfig, - AppConfig: &c.AppConfig, - AppOptions: c.AppOptions, - AppCreator: c.AppCreator, - SupressLogs: false, - } -} - // Stop stops the node and cleans up the data directory. -func (c *ConsensusNode) Stop() error { +func (cn *ConsensusNode) Stop() error { var err error - for _, stop := range c.stopFuncs { + for _, stop := range cn.stopFuncs { if sterr := stop(); err != nil { err = sterr } @@ -202,6 +210,17 @@ func (c *ConsensusNode) Stop() error { return err } +// UniversalTestingConfig returns the configuration used by the testnode package. +func (cn *ConsensusNode) UniversalTestingConfig() testnode.UniversalTestingConfig { + return testnode.UniversalTestingConfig{ + TmConfig: cn.CmtConfig, + AppConfig: cn.AppConfig, + AppOptions: cn.AppOptions, + AppCreator: cn.AppCreator, + SupressLogs: false, + } +} + // SubmitRandomPFB will submit a single PFB using the consensus node's tx // signing account. One blob will be included for each size provided in a single PFB. func (c *ConsensusNode) SubmitRandomPFB(ctx context.Context, runenv *runtime.RunEnv, blobSizes ...int) (*sdk.TxResponse, error) { @@ -241,15 +260,26 @@ func (c *ConsensusNode) SubmitRandomPFB(ctx context.Context, runenv *runtime.Run return signer.SubmitPayForBlob(ctx, blobs, user.SetGasLimitAndFee(limit, 0.1)) } -// ImportKey imports the provided mnemonic into the keyring with the provided name. -func ImportKey(kr keyring.Keyring, accountMnemonic string, name string) (keyring.Keyring, error) { - if accountMnemonic == "" { - return kr, fmt.Errorf("account mnemonic cannot be empty") +func addrsToStrings(addrs ...sdk.AccAddress) []string { + strs := make([]string, len(addrs)) + for i, addr := range addrs { + strs[i] = addr.String() } - _, err := kr.Key(name) - if err == nil { - return kr, fmt.Errorf("key %s already exists", name) + return strs +} + +func getPublicKeys(kr keyring.Keyring, accounts ...string) ([]string, error) { + keys := make([]string, 0, len(accounts)) + for _, acc := range accounts { + rec, err := kr.Key(acc) + if err != nil { + return nil, err + } + pubK, err := rec.GetPubKey() + if err != nil { + return nil, err + } + keys = append(keys, SerializePublicKey(pubK)) } - _, err = kr.NewAccount(name, accountMnemonic, "", "", hd.Secp256k1) - return kr, err + return keys, nil } diff --git a/test/testground/network/entry_point.go b/test/testground/network/entry_point.go index df16c5aaf2..4d02e62356 100644 --- a/test/testground/network/entry_point.go +++ b/test/testground/network/entry_point.go @@ -16,16 +16,6 @@ func EntryPoint(runenv *runtime.RunEnv, initCtx *run.InitContext) error { } defer cancel() - // publish and download the ip addresses of all nodes - statuses, err := SyncStatus(ctx, runenv, initCtx) - if err != nil { - runenv.RecordFailure(err) - initCtx.SyncClient.MustSignalAndWait(ctx, FailedState, runenv.TestInstanceCount) - return err - } - - runenv.RecordMessage("statuses: %v", statuses) - // determine roles based only on the global sequence number. This allows for // us to deterministically calculate the IP addresses of each node. role, err := NewRole(runenv, initCtx) @@ -40,7 +30,7 @@ func EntryPoint(runenv *runtime.RunEnv, initCtx *run.InitContext) error { // using the parameters defined in the manifest and plan toml files. The // single "leader" role performs creation and publishing of the configs, // while the "follower" roles download the configs from the leader. - err = role.Plan(ctx, statuses, runenv, initCtx) + err = role.Plan(ctx, runenv, initCtx) if err != nil { runenv.RecordFailure(err) initCtx.SyncClient.MustSignalAndWait(ctx, FailedState, runenv.TestInstanceCount) diff --git a/test/testground/network/follower.go b/test/testground/network/follower.go index 07388b9208..d066ed8d99 100644 --- a/test/testground/network/follower.go +++ b/test/testground/network/follower.go @@ -22,9 +22,8 @@ type Follower struct { // NewFollower creates a new follower role. func NewFollower() *Follower { - f := &Follower{} op := NewOperator() - + f := &Follower{&ConsensusNode{}, nil} op.RegisterCommand( RunTxSimCommandID, func(ctx context.Context, runenv *runtime.RunEnv, _ *run.InitContext, args json.RawMessage) error { @@ -39,35 +38,43 @@ func NewFollower() *Follower { ) op.RegisterCommand(RunSubmitRandomPFBs, f.SubmitRandomPFBsHandler) - f.op = op return f } // Plan is the method that downloads the genesis, configurations, and keys for // all of the other nodes in the network. -func (f *Follower) Plan(ctx context.Context, _ []Status, runenv *runtime.RunEnv, initCtx *run.InitContext) error { +func (f *Follower) Plan(ctx context.Context, runenv *runtime.RunEnv, initCtx *run.InitContext) error { + _, err := f.Bootstrap(ctx, runenv, initCtx) + if err != nil { + return err + } - cfg, err := DownloadNetworkConfig(ctx, initCtx) + tcfg, err := DownloadTestgroundConfig(ctx, initCtx) if err != nil { return err } - f.ConsensusNode, err = cfg.ConsensusNode(int(initCtx.GlobalSeq)) - return err -} + err = f.Init(homeDir, tcfg.Genesis, tcfg.ConsensusNodeConfigs[f.Name]) + if err != nil { + return err + } -func (f *Follower) Execute(ctx context.Context, runenv *runtime.RunEnv, initCtx *run.InitContext) error { - baseDir, err := f.ConsensusNode.Init(homeDir) + err = f.ConsensusNode.StartNode(ctx, f.baseDir) if err != nil { return err } - err = f.ConsensusNode.StartNode(ctx, baseDir) + + _, err = f.cctx.WaitForHeightWithTimeout(int64(2), time.Minute*5) if err != nil { return err } - runenv.RecordMessage(fmt.Sprintf("follower %d waiting for commands", f.Status.GlobalSequence)) + return err +} + +func (f *Follower) Execute(ctx context.Context, runenv *runtime.RunEnv, initCtx *run.InitContext) error { + runenv.RecordMessage(fmt.Sprintf("follower %d waiting for commands")) return f.ListenForCommands(ctx, runenv, initCtx) } diff --git a/test/testground/network/leader.go b/test/testground/network/leader.go index afae8c9a8a..c9fa839d1c 100644 --- a/test/testground/network/leader.go +++ b/test/testground/network/leader.go @@ -10,7 +10,6 @@ import ( "github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1" cryptotypes "github.com/cosmos/cosmos-sdk/crypto/types" - "github.com/tendermint/tendermint/types" coretypes "github.com/tendermint/tendermint/types" "github.com/testground/sdk-go/run" "github.com/testground/sdk-go/runtime" @@ -19,48 +18,66 @@ import ( // Leader is the role for the leader node in a test. It is responsible for // creating the genesis block and distributing it to all nodes. type Leader struct { - *FullNode + *ConsensusNode } // Plan is the method that creates and distributes the genesis, configurations, // and keys for all of the other nodes in the network. -func (l *Leader) Plan(ctx context.Context, statuses []Status, runenv *runtime.RunEnv, initCtx *run.InitContext) error { - packets, err := l.BootstrapPeers(ctx, runenv, initCtx) +func (l *Leader) Plan(ctx context.Context, runenv *runtime.RunEnv, initCtx *run.InitContext) error { + packets, err := l.Bootstrap(ctx, runenv, initCtx) if err != nil { return err } // create Genesis and distribute it to all nodes - genesis, tgCfg, err := l.GenesisEvent(ctx, runenv, initCtx, packets) + genesis, err := l.GenesisEvent(ctx, runenv, initCtx, packets) if err != nil { return err } - baseDir, err := l.Init(homeDir, genesis, tgCfg[l.Name]) + // create all of the configs using the delivered packets + tcfg, err := NewTestgroundConfig(l.params, genesis, packets) if err != nil { return err } - l.baseDir = baseDir + // apply the topology functions to the configs to create a specific network. + for _, configurator := range l.params.Configurators { + tcfg, err = configurator(tcfg) + if err != nil { + return err + } + } - return nil -} + err = PublishTestgroundConfig(ctx, initCtx, tcfg) + if err != nil { + return err + } -func (l *Leader) Execute(ctx context.Context, runenv *runtime.RunEnv, initCtx *run.InitContext) error { - baseDir, err := l.ConsensusNode.Init(homeDir) + err = l.Init(homeDir, tcfg.Genesis, tcfg.ConsensusNodeConfigs[l.Name]) if err != nil { return err } - err = l.ConsensusNode.StartNode(ctx, baseDir) + err = l.ConsensusNode.StartNode(ctx, l.baseDir) if err != nil { return err } - time.Sleep(time.Second * 20) + _, err = l.cctx.WaitForHeightWithTimeout(int64(2), time.Minute*5) + if err != nil { + return err + } + // this is a helpful sanity check that logs the blocks from the POV of the + // leader in a testground viewable way. go l.subscribeAndRecordBlocks(ctx, runenv) + return nil +} + +func (l *Leader) Execute(ctx context.Context, runenv *runtime.RunEnv, initCtx *run.InitContext) error { + time.Sleep(time.Second * 20) // seqs := runenv.IntParam(BlobSequencesParam) @@ -79,7 +96,7 @@ func (l *Leader) Execute(ctx context.Context, runenv *runtime.RunEnv, initCtx *r sizes..., ) - _, err = initCtx.SyncClient.Publish(ctx, CommandTopic, cmd) + _, err := initCtx.SyncClient.Publish(ctx, CommandTopic, cmd) if err != nil { return err } @@ -99,8 +116,8 @@ func (l *Leader) Execute(ctx context.Context, runenv *runtime.RunEnv, initCtx *r // runenv.RecordMessage(fmt.Sprintf("leader submittedPFB code %d space %s", resp.Code, resp.Codespace)) - runenv.RecordMessage(fmt.Sprintf("leader waiting for halt height %d", l.HaltHeight)) - _, err = l.cctx.WaitForHeightWithTimeout(int64(l.ConsensusNode.HaltHeight), time.Minute*30) + runenv.RecordMessage(fmt.Sprintf("leader waiting for halt height %d", l.params.HaltHeight)) + _, err = l.cctx.WaitForHeightWithTimeout(int64(l.params.HaltHeight), time.Minute*30) if err != nil { return err } @@ -152,8 +169,7 @@ func (l *Leader) GenesisEvent(ctx context.Context, runenv *runtime.RunEnv, initC gentxs = append(gentxs, packet.GenTx) } - // save and gossip the genesis doc and configs to all of nodes and then we done - doc, err := GenesisDoc(l.ecfg, l.params.ChainID, gentxs, addrs, pubKeys) + return GenesisDoc(l.ecfg, l.params.ChainID, gentxs, addrs, pubKeys) } func SerializePublicKey(pubKey cryptotypes.PubKey) string { @@ -185,7 +201,7 @@ func (l *Leader) subscribeAndRecordBlocks(ctx context.Context, runenv *runtime.R for { select { case ev := <-events: - newBlock, ok := ev.Data.(types.EventDataNewBlock) + newBlock, ok := ev.Data.(coretypes.EventDataNewBlock) if !ok { return fmt.Errorf("unexpected event type: %T", ev.Data) } diff --git a/test/testground/network/params.go b/test/testground/network/params.go index eb9aeb1a6a..4f3b13afd7 100644 --- a/test/testground/network/params.go +++ b/test/testground/network/params.go @@ -5,12 +5,10 @@ import ( "fmt" "time" - mrand "math/rand" - "github.com/celestiaorg/celestia-app/app" - "github.com/celestiaorg/celestia-app/test/util/genesis" + srvconfig "github.com/cosmos/cosmos-sdk/server/config" + tmconfig "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/crypto/ed25519" - cmtjson "github.com/tendermint/tendermint/libs/json" "github.com/tendermint/tendermint/p2p" "github.com/testground/sdk-go/runtime" ) @@ -40,7 +38,7 @@ type Params struct { HaltHeight int Timeout time.Duration Pex bool - TopologyFns []TopologyFn + Configurators []Configurator PerPeerBandwidth int BlobsPerSeq int BlobSequences int @@ -94,7 +92,7 @@ func ParseParams(runenv *runtime.RunEnv) (*Params, error) { return nil, err } - p.TopologyFns, err = GetTopologyFns(runenv) + p.Configurators, err = GetConfigurators(runenv) if err != nil { return nil, err } @@ -119,120 +117,24 @@ func (p *Params) NodeCount() int { return p.FullNodes + p.Validators } -func (p *Params) StandardConfig(statuses []Status) (Config, error) { - // set the global configs for each node +func StandardCometConfig(params *Params) *tmconfig.Config { cmtcfg := app.DefaultConsensusConfig() cmtcfg.Instrumentation.PrometheusListenAddr = "0.0.0.0:26660" cmtcfg.Instrumentation.Prometheus = true - cmtcfg.P2P.PexReactor = p.Pex - cmtcfg.P2P.SendRate = int64(p.PerPeerBandwidth) - cmtcfg.P2P.RecvRate = int64(p.PerPeerBandwidth) - cmtcfg.Consensus.TimeoutCommit = p.TimeoutCommit - cmtcfg.Consensus.TimeoutPropose = p.TimeoutPropose + cmtcfg.P2P.PexReactor = params.Pex + cmtcfg.P2P.SendRate = int64(params.PerPeerBandwidth) + cmtcfg.P2P.RecvRate = int64(params.PerPeerBandwidth) + cmtcfg.Consensus.TimeoutCommit = params.TimeoutCommit + cmtcfg.Consensus.TimeoutPropose = params.TimeoutPropose cmtcfg.TxIndex.Indexer = "kv" + return cmtcfg +} - vals := make([]genesis.Validator, 0) - accs := make([]genesis.Account, 0) - networkKeys := make([]ed25519.PrivKey, 0, len(statuses)) - r := mrand.New(mrand.NewSource(time.Now().UnixNano())) - - nodes := []NodeConfig{} - for i, status := range statuses { - networkKeys = append(networkKeys, genesis.GenerateEd25519(genesis.NewSeed(r))) - nodeName := fmt.Sprintf("%d", status.GlobalSequence) - - consensusKey := ed25519.GenPrivKey() - switch status.NodeType { - case "validators": - val := genesis.NewDefaultValidator(nodeName) - consensusKey = val.ConsensusKey - vals = append(vals, val) - case "full_nodes": - accs = append(accs, genesis.NewAccounts(999999999999999999, nodeName)...) - default: - return Config{}, fmt.Errorf("unknown node type %s", status.NodeType) - } - - nodes = append(nodes, NodeConfig{ - Status: status, - NodeType: status.NodeType, - Name: nodeName, - StartHeight: 0, - HaltHeight: p.HaltHeight, - Keys: KeySet{ - NetworkKey: networkKeys[i], - ConsensusKey: consensusKey, - }, - CmtConfig: *cmtcfg, - AppConfig: *app.DefaultAppConfig(), - P2PID: peerID(status, networkKeys[i]), - }) - } - - g := genesis.NewDefaultGenesis(). - WithValidators(vals...). - WithAccounts(accs...) - - gDoc, err := g.Export() - if err != nil { - return Config{}, nil - } - - genDocBytes, err := cmtjson.MarshalIndent(gDoc, "", " ") - if err != nil { - return Config{}, err - } - - nodes, err = setMnemomics(g.Accounts(), nodes) - if err != nil { - return Config{}, err - } - - for _, node := range nodes { - if node.Keys.AccountMnemonic == "" { - return Config{}, fmt.Errorf("mnemonic not found for account %s", node.Name) - } - } - - for _, top := range p.TopologyFns { - nodes, err = top(nodes) - } - - cfg := Config{ - ChainID: g.ChainID, - Genesis: genDocBytes, - Nodes: nodes, - } - - return cfg, nil +func StandardAppConfig() *srvconfig.Config { + return app.DefaultAppConfig() } func peerID(ip string, networkKey ed25519.PrivKey) string { nodeID := string(p2p.PubKeyToID(networkKey.PubKey())) return fmt.Sprintf("%s@%s:26656", nodeID, ip) } - -func setMnemomics(accs []genesis.Account, nodeCfgs []NodeConfig) ([]NodeConfig, error) { - accountMap := make(map[string]genesis.Account) - for _, acc := range accs { - accountMap[acc.Name] = acc - } - if len(accountMap) != len(accs) { - return nil, fmt.Errorf("duplicate account names found") - } - if len(nodeCfgs) > len(accountMap) { - - return nil, fmt.Errorf("node count and account count mismatch: accounts %d node configs %d", len(accountMap), len(nodeCfgs)) - } - for i, cfg := range nodeCfgs { - if acc, ok := accountMap[cfg.Name]; ok { - if acc.Mnemonic == "" { - return nil, fmt.Errorf("mnemonic not found for account %s", acc.Name) - } - nodeCfgs[i].Keys.AccountMnemonic = acc.Mnemonic - continue - } - return nil, fmt.Errorf("account not found for node %s", cfg.Name) - } - return nodeCfgs, nil -} diff --git a/test/testground/network/params_test.go b/test/testground/network/params_test.go deleted file mode 100644 index 952a7d3412..0000000000 --- a/test/testground/network/params_test.go +++ /dev/null @@ -1,126 +0,0 @@ -package network - -import ( - "testing" - - "github.com/celestiaorg/celestia-app/app" - "github.com/celestiaorg/celestia-app/app/encoding" - "github.com/celestiaorg/celestia-app/test/util/genesis" - "github.com/cosmos/cosmos-sdk/crypto/hd" - "github.com/cosmos/cosmos-sdk/crypto/keyring" - "github.com/influxdata/influxdb/pkg/testing/assert" - "github.com/stretchr/testify/require" - // Replace with the imports where your types are defined - // "your_package/tmconfig" - // "your_package/srvconfig" -) - -func TestSetMnemonics(t *testing.T) { - // Initialize example data - accounts := []genesis.Account{ - {Name: "Alice", Mnemonic: "AliceMnemonic"}, - {Name: "Bob", Mnemonic: "BobMnemonic"}, - } - - nodeConfigs := []NodeConfig{ - { - Name: "Alice", - Keys: KeySet{}, - }, - { - Name: "Bob", - Keys: KeySet{}, - }, - } - - // Test setting mnemonics - modifiedConfigs, err := setMnemomics(accounts, nodeConfigs) - require.NoError(t, err) - - for _, cfg := range modifiedConfigs { - require.NotEmpty(t, cfg.Keys.AccountMnemonic) - } -} - -func TestConfigGeneration(t *testing.T) { - p := Params{ - Validators: 3, - FullNodes: 0, - Timeout: 0, - TopologyFns: []TopologyFn{ConnectAll}, - HaltHeight: 100, - Pex: false, - } - - ss := []Status{ - { - IP: "10.32.0.21", - GlobalSequence: 1, - GroupSequence: 1, - Group: "validators", - NodeType: "validators", - }, - { - IP: "10.45.0.19", - GlobalSequence: 2, - GroupSequence: 2, - Group: "validators", - NodeType: "validators", - }, - { - IP: "10.44.128.11", - GlobalSequence: 3, - GroupSequence: 3, - Group: "validators", - NodeType: "validators", - }, - } - cfg, err := p.StandardConfig(ss) - require.NoError(t, err) - - require.Equal(t, 3, len(cfg.Nodes)) - ecfg := encoding.MakeConfig(app.ModuleEncodingRegisters...) - for _, node := range cfg.Nodes { - require.NotEmpty(t, node.Keys.NetworkKey) - require.NotEmpty(t, node.Keys.ConsensusKey) - require.NotEmpty(t, node.Keys.AccountMnemonic) - require.Equal(t, "validators", node.NodeType) - - kr := keyring.NewInMemory(ecfg.Codec) - - kr, err = ImportKey(kr, node.Keys.AccountMnemonic, node.Name) - require.NoError(t, err) - rec, err := kr.Key(node.Name) - require.NoError(t, err) - _, err = rec.GetAddress() - require.NoError(t, err) - } -} - -func TestExportImportKey(t *testing.T) { - ecfg := encoding.MakeConfig(app.ModuleEncodingRegisters...) - kr := keyring.NewInMemory(ecfg.Codec) - _, mn, err := kr.NewMnemonic("test", keyring.English, "", "", hd.Secp256k1) - require.NoError(t, err) - rec, err := kr.Key("test") - require.NoError(t, err) - err = kr.Delete("test") - require.NoError(t, err) - rec, err = kr.NewAccount("test", mn, "", "", hd.Secp256k1) - require.NoError(t, err) - _, err = rec.GetAddress() - require.NoError(t, err) - - _, _, err = kr.NewMnemonic("test-2", keyring.English, "", "", hd.Secp256k1) - require.NoError(t, err) - rec, err = kr.Key("test-2") - require.NoError(t, err) - armor, err := kr.ExportPrivKeyArmor("test-2", "") - require.NoError(t, err) - err = kr.Delete("test-2") - require.NoError(t, err) - err = kr.ImportPrivKey("test-2", armor, "") - assert.NoError(t, err) - rec, err = kr.Key("test-2") - require.NoError(t, err) -} diff --git a/test/testground/network/role.go b/test/testground/network/role.go index e7d7c1312d..13bef57a07 100644 --- a/test/testground/network/role.go +++ b/test/testground/network/role.go @@ -2,34 +2,8 @@ package network import ( "context" - "encoding/json" - "errors" "fmt" - "os" - "path/filepath" - "github.com/celestiaorg/celestia-app/app" - "github.com/celestiaorg/celestia-app/app/encoding" - "github.com/celestiaorg/celestia-app/cmd/celestia-appd/cmd" - appns "github.com/celestiaorg/celestia-app/pkg/namespace" - "github.com/celestiaorg/celestia-app/pkg/user" - "github.com/celestiaorg/celestia-app/test/util/blobfactory" - "github.com/celestiaorg/celestia-app/test/util/genesis" - "github.com/celestiaorg/celestia-app/test/util/testnode" - blobtypes "github.com/celestiaorg/celestia-app/x/blob/types" - "github.com/cosmos/cosmos-sdk/crypto/keyring" - cryptotypes "github.com/cosmos/cosmos-sdk/crypto/types" - srvconfig "github.com/cosmos/cosmos-sdk/server/config" - srvtypes "github.com/cosmos/cosmos-sdk/server/types" - sdk "github.com/cosmos/cosmos-sdk/types" - tmconfig "github.com/tendermint/tendermint/config" - "github.com/tendermint/tendermint/crypto/ed25519" - cmtos "github.com/tendermint/tendermint/libs/os" - tmos "github.com/tendermint/tendermint/libs/os" - tmrand "github.com/tendermint/tendermint/libs/rand" - "github.com/tendermint/tendermint/node" - "github.com/tendermint/tendermint/p2p" - "github.com/tendermint/tendermint/privval" "github.com/testground/sdk-go/run" "github.com/testground/sdk-go/runtime" ) @@ -49,7 +23,7 @@ const ( type Role interface { // Plan is the first function called in a test by each node. It is responsible // for creating the genesis block and distributing it to all nodes. - Plan(ctx context.Context, statuses []Status, runenv *runtime.RunEnv, initCtx *run.InitContext) error + Plan(ctx context.Context, runenv *runtime.RunEnv, initCtx *run.InitContext) error // Execute is the second function called in a test by each node. It is // responsible for starting the node and/or running any tests. Execute(ctx context.Context, runenv *runtime.RunEnv, initCtx *run.InitContext) error @@ -70,306 +44,9 @@ func NewRole(runenv *runtime.RunEnv, initCtx *run.InitContext) (Role, error) { // TODO: throw and error if there is more than a single leader case 1: runenv.RecordMessage("red leader sitting by") - return &Leader{FullNode: &FullNode{}}, nil + return &Leader{ConsensusNode: &ConsensusNode{}}, nil default: runenv.RecordMessage(fmt.Sprintf("red %d sitting by", seq)) return NewFollower(), nil } } - -// PeerPacket is the message that is sent to other nodes upon network -// initialization. It contains the necessary info from this node to start the -// network. -type PeerPacket struct { - PeerID string `json:"peer_id"` - IP string `json:"ip"` - GroupID string `json:"group_id"` - GlobalSequence int64 `json:"global_sequence"` - GenesisAccounts []string `json:"genesis_accounts"` - GenesisPubKeys []string `json:"pub_keys"` - GenTx json.RawMessage `json:"gen_tx"` -} - -func (pp *PeerPacket) IsValidator() bool { - return pp.GroupID == ValidatorGroupID -} - -func (pp *PeerPacket) IsLeader() bool { - return pp.GlobalSequence == LeaderGlobalSequence -} - -func (pp *PeerPacket) Name() string { - return NodeID(pp.GlobalSequence) -} - -func (pp *PeerPacket) GetPubKeys() ([]cryptotypes.PubKey, error) { - pks := make([]cryptotypes.PubKey, 0, len(pp.GenesisPubKeys)) - for _, pk := range pp.GenesisPubKeys { - sdkpk, err := DeserializeAccountPublicKey(pk) - if err != nil { - return nil, err - } - pks = append(pks, sdkpk) - } - return pks, nil -} - -// TestgroundConfig is the first message sent by the Leader to the rest of the -// Follower nodes after the network has been configured. -type TestgroundConfig struct { - Genesis json.RawMessage `json:"genesis"` - ConsensusNodeConfigs map[string]ConsensusNodeMetaConfig -} - -type ConsensusNodeMetaConfig struct { - CmtConfig tmconfig.Config `json:"cmt_config"` - AppConfig srvconfig.Config `json:"app_config"` -} - -// todo rename to consensus node after refactor -type FullNode struct { - Name string - // NetworkKey is the key used for signing gossiped messages. - networkKey ed25519.PrivKey - // ConsensusKey is the key used for signing votes. - consensusKey ed25519.PrivKey - - kr keyring.Keyring - ecfg encoding.Config - - params *Params - CmtNode *node.Node - CmtConfig tmconfig.Config - AppConfig srvconfig.Config - baseDir string - - cctx testnode.Context - - stopFuncs []func() error - // AppOptions are the application options of the test node. - AppOptions *testnode.KVAppOptions - // AppCreator is used to create the application for the testnode. - AppCreator srvtypes.AppCreator - - cmtNode *node.Node -} - -// BootstrapPeers is the first function called in a test by each node. It is -// responsible for initializing the node and creating a gentx if this node is a -// validator. -func (cn *FullNode) BootstrapPeers(ctx context.Context, runenv *runtime.RunEnv, initCtx *run.InitContext) ([]PeerPacket, error) { - ip, err := initCtx.NetClient.GetDataNetworkIP() - if err != nil { - return nil, err - } - - cn.ecfg = encoding.MakeConfig(app.ModuleEncodingRegisters...) - - params, err := ParseParams(runenv) - if err != nil { - return nil, err - } - cn.params = params - - nodeID := NodeID(initCtx.GlobalSeq) - cn.Name = nodeID - - kr, addrs := testnode.NewKeyring(nodeID, TxSimAccountName) - cn.kr = kr - - val := genesis.NewDefaultValidator(nodeID) - cn.consensusKey = val.ConsensusKey - cn.networkKey = val.NetworkKey - - var bz []byte - if runenv.TestGroupID == ValidatorGroupID { - gentx, err := val.GenTx(cn.ecfg, cn.kr, cn.params.ChainID) - if err != nil { - return nil, err - } - bz, err = cn.ecfg.TxConfig.TxJSONEncoder()(gentx) - if err != nil { - return nil, err - } - } - - pubKs, err := getPublicKeys(cn.kr, nodeID, TxSimAccountName) - if err != nil { - return nil, err - } - - pp := PeerPacket{ - PeerID: peerID(ip.String(), cn.networkKey), - IP: ip.String(), - GroupID: runenv.TestGroupID, - GlobalSequence: initCtx.GlobalSeq, - GenesisAccounts: addrsToStrings(addrs...), - GenesisPubKeys: pubKs, - GenTx: json.RawMessage(bz), - } - - _, err = initCtx.SyncClient.Publish(ctx, PeerPacketTopic, pp) - if err != nil { - return nil, err - } - - return DownloadSync(ctx, initCtx, PeerPacketTopic, PeerPacket{}, runenv.TestInstanceCount) -} - -// Init creates the files required by tendermint and celestia-app using the data -// downloaded from the Leader node. -func (cn *FullNode) Init(baseDir string, genesis json.RawMessage, mcfg ConsensusNodeMetaConfig) (string, error) { - cn.CmtConfig = mcfg.CmtConfig - cn.AppConfig = mcfg.AppConfig - cn.AppCreator = cmd.NewAppServer - cn.AppOptions = testnode.DefaultAppOptions() - - basePath := filepath.Join(baseDir, ".celestia-app") - cn.CmtConfig.SetRoot(basePath) - - // save the genesis file - configPath := filepath.Join(basePath, "config") - err := os.MkdirAll(configPath, os.ModePerm) - if err != nil { - return "", err - } - // save the genesis file as configured - err = cmtos.WriteFile(cn.CmtConfig.GenesisFile(), genesis, 0o644) - if err != nil { - return "", err - } - pvStateFile := cn.CmtConfig.PrivValidatorStateFile() - if err := tmos.EnsureDir(filepath.Dir(pvStateFile), 0o777); err != nil { - return "", err - } - pvKeyFile := cn.CmtConfig.PrivValidatorKeyFile() - if err := tmos.EnsureDir(filepath.Dir(pvKeyFile), 0o777); err != nil { - return "", err - } - filePV := privval.NewFilePV(cn.consensusKey, pvKeyFile, pvStateFile) - filePV.Save() - - nodeKeyFile := cn.CmtConfig.NodeKeyFile() - if err := tmos.EnsureDir(filepath.Dir(nodeKeyFile), 0o777); err != nil { - return "", err - } - nodeKey := &p2p.NodeKey{ - PrivKey: cn.networkKey, - } - if err := nodeKey.SaveAs(nodeKeyFile); err != nil { - return "", err - } - - return basePath, nil -} - -// StartNode uses the testnode package to start a tendermint node with -// celestia-app and the provided configuration. -func (cn *FullNode) StartNode(ctx context.Context, baseDir string) error { - ucfg := cn.UniversalTestingConfig() - tmNode, app, err := testnode.NewCometNode(baseDir, &ucfg) - if err != nil { - return err - } - - cn.cmtNode = tmNode - cctx := testnode.NewContext(ctx, cn.kr, ucfg.TmConfig, cn.params.ChainID) - - cctx, stopNode, err := testnode.StartNode(tmNode, cctx) - cn.stopFuncs = append(cn.stopFuncs, stopNode) - if err != nil { - return err - } - - cctx, cleanupGRPC, err := testnode.StartGRPCServer(app, ucfg.AppConfig, cctx) - cn.stopFuncs = append(cn.stopFuncs, cleanupGRPC) - - cn.cctx = cctx - - return err -} - -// Stop stops the node and cleans up the data directory. -func (cn *FullNode) Stop() error { - var err error - for _, stop := range cn.stopFuncs { - if sterr := stop(); err != nil { - err = sterr - } - } - return err -} - -// UniversalTestingConfig returns the configuration used by the testnode package. -func (cn *FullNode) UniversalTestingConfig() testnode.UniversalTestingConfig { - return testnode.UniversalTestingConfig{ - TmConfig: &cn.CmtConfig, - AppConfig: &cn.AppConfig, - AppOptions: cn.AppOptions, - AppCreator: cn.AppCreator, - SupressLogs: false, - } -} - -// SubmitRandomPFB will submit a single PFB using the consensus node's tx -// signing account. One blob will be included for each size provided in a single PFB. -func (c *FullNode) SubmitRandomPFB(ctx context.Context, runenv *runtime.RunEnv, blobSizes ...int) (*sdk.TxResponse, error) { - runenv.RecordMessage("attempting to get the key") - if c.kr == nil { - return nil, errors.New("nil keyring") - } - rec, err := c.kr.Key(c.Name) - if err != nil { - return nil, err - } - runenv.RecordMessage("got key") - addr, err := rec.GetAddress() - if err != nil { - return nil, err - } - runenv.RecordMessage("got addr") - signer, err := user.SetupSigner(ctx, c.kr, c.cctx.GRPCClient, addr, c.ecfg) - if err != nil { - return nil, err - } - runenv.RecordMessage("created signer") - - r := tmrand.NewRand() - - blobs := blobfactory.RandBlobsWithNamespace(appns.RandomBlobNamespaces(r, len(blobSizes)), blobSizes) - runenv.RecordMessage("made blobs") - blobSizesU := make([]uint32, 0, len(blobSizes)) - for _, size := range blobSizes { - blobSizesU = append(blobSizesU, uint32(size)) - } - - limit := blobtypes.DefaultEstimateGas(blobSizesU) - - runenv.RecordMessage("finished prep for pfb") - - return signer.SubmitPayForBlob(ctx, blobs, user.SetGasLimitAndFee(limit, 0.1)) -} - -func addrsToStrings(addrs ...sdk.AccAddress) []string { - strs := make([]string, len(addrs)) - for i, addr := range addrs { - strs[i] = addr.String() - } - return strs -} - -func getPublicKeys(kr keyring.Keyring, accounts ...string) ([]string, error) { - keys := make([]string, 0, len(accounts)) - for _, acc := range accounts { - rec, err := kr.Key(acc) - if err != nil { - return nil, err - } - pubK, err := rec.GetPubKey() - if err != nil { - return nil, err - } - keys = append(keys, SerializePublicKey(pubK)) - } - return keys, nil -} diff --git a/test/testground/network/sync.go b/test/testground/network/sync.go index 85d8044780..80f5d2f871 100644 --- a/test/testground/network/sync.go +++ b/test/testground/network/sync.go @@ -4,9 +4,16 @@ import ( "context" "encoding/json" "errors" - + "fmt" + "sort" + + "github.com/celestiaorg/celestia-app/app" + cryptotypes "github.com/cosmos/cosmos-sdk/crypto/types" + srvconfig "github.com/cosmos/cosmos-sdk/server/config" + tmconfig "github.com/tendermint/tendermint/config" + cmtjson "github.com/tendermint/tendermint/libs/json" + coretypes "github.com/tendermint/tendermint/types" "github.com/testground/sdk-go/run" - "github.com/testground/sdk-go/runtime" "github.com/testground/sdk-go/sync" ) @@ -18,89 +25,122 @@ var ( GenesisTopic = sync.NewTopic("genesis", map[string]json.RawMessage{}) // NetworkConfigTopic is the topic used to exchange network configuration // between test instances. - ConfigTopic = sync.NewTopic("network-config", Config{}) - PeerPacketTopic = sync.NewTopic("peer_packet", PeerPacket{}) - CommandTopic = sync.NewTopic("command", Command{}) + TestgroundConfigTopic = sync.NewTopic("testground_config", TestgroundConfig{}) + PeerPacketTopic = sync.NewTopic("peer_packet", PeerPacket{}) + CommandTopic = sync.NewTopic("command", Command{}) ) -type Config struct { - ChainID string `json:"chain_id"` - Genesis json.RawMessage `json:"genesis"` - Nodes []NodeConfig `json:"nodes"` +// PeerPacket is the message that is sent to other nodes upon network +// initialization. It contains the necessary info from this node to start the +// network. +type PeerPacket struct { + PeerID string `json:"peer_id"` + IP string `json:"ip"` + GroupID string `json:"group_id"` + GlobalSequence int64 `json:"global_sequence"` + GenesisAccounts []string `json:"genesis_accounts"` + GenesisPubKeys []string `json:"pub_keys"` + GenTx json.RawMessage `json:"gen_tx"` } -// Status is used by followers to signal to the leader that they are -// online and thier network config. -type Status struct { - IP string `json:"ip"` - GlobalSequence int64 `json:"global_sequence"` - GroupSequence int64 `json:"group_sequence"` - Group string `json:"group"` - NodeType string `json:"node_type"` +func (pp *PeerPacket) IsValidator() bool { + return pp.GroupID == ValidatorGroupID } -func PublishConfig(ctx context.Context, initCtx *run.InitContext, cfg Config) error { - _, err := initCtx.SyncClient.Publish(ctx, ConfigTopic, cfg) - return err +func (pp *PeerPacket) IsLeader() bool { + return pp.GlobalSequence == LeaderGlobalSequence } -func DownloadNetworkConfig(ctx context.Context, initCtx *run.InitContext) (Config, error) { - cfgs, err := DownloadSync(ctx, initCtx, ConfigTopic, Config{}, 1) - if err != nil { - return Config{}, err - } - if len(cfgs) != 1 { - return Config{}, errors.New("no network config was downloaded despite there not being an error") +func (pp *PeerPacket) Name() string { + return NodeID(pp.GlobalSequence) +} + +func (pp *PeerPacket) GetPubKeys() ([]cryptotypes.PubKey, error) { + pks := make([]cryptotypes.PubKey, 0, len(pp.GenesisPubKeys)) + for _, pk := range pp.GenesisPubKeys { + sdkpk, err := DeserializeAccountPublicKey(pk) + if err != nil { + return nil, err + } + pks = append(pks, sdkpk) } - return cfgs[0], nil + return pks, nil } -func SyncStatus(ctx context.Context, runenv *runtime.RunEnv, initCtx *run.InitContext) ([]Status, error) { - ip, err := initCtx.NetClient.GetDataNetworkIP() +func NewTestgroundConfig(params *Params, genesis *coretypes.GenesisDoc, pps []PeerPacket) (TestgroundConfig, error) { + genBytes, err := cmtjson.MarshalIndent(genesis, "", " ") if err != nil { - return nil, err + return TestgroundConfig{}, err } - ns := Status{ - IP: ip.String(), - GlobalSequence: initCtx.GlobalSeq, - GroupSequence: initCtx.GroupSeq, - Group: runenv.TestGroupID, - NodeType: runenv.TestGroupID, + cfg := TestgroundConfig{ + Genesis: genBytes, + ConsensusNodeConfigs: make(map[string]ConsensusNodeMetaConfig), } - - err = publishStatus(ctx, runenv, initCtx, ns) - if err != nil { - return nil, err + for _, pp := range pps { + cfg.ConsensusNodeConfigs[pp.Name()] = ConsensusNodeMetaConfig{ + CmtConfig: StandardCometConfig(params), + AppConfig: app.DefaultAppConfig(), + } } + return cfg, nil +} - stats, err := downloadStatuses(ctx, initCtx, runenv.TestInstanceCount) - if err != nil { - return nil, err - } +// TestgroundConfig is the first message sent by the Leader to the rest of the +// Follower nodes after the network has been configured. +type TestgroundConfig struct { + Genesis json.RawMessage `json:"genesis"` + ConsensusNodeConfigs map[string]ConsensusNodeMetaConfig +} - stats = append(stats, ns) +type ConsensusNodeMetaConfig struct { + PeerPacket PeerPacket `json:"peer_packet"` + CmtConfig *tmconfig.Config `json:"cmt_config"` + AppConfig *srvconfig.Config `json:"app_config"` +} - // remove duplicate stats - seen := make(map[string]struct{}) - uniqueStats := make([]Status, 0, runenv.TestInstanceCount) - for _, s := range stats { - if _, ok := seen[s.IP]; !ok { - seen[s.IP] = struct{}{} - uniqueStats = append(uniqueStats, s) - } +// Nodes returns the list of nodes in the network sorted by global sequence. +func (tcfg *TestgroundConfig) Nodes() []ConsensusNodeMetaConfig { + nodes := make([]ConsensusNodeMetaConfig, 0, len(tcfg.ConsensusNodeConfigs)) + for _, n := range tcfg.ConsensusNodeConfigs { + nodes = append(nodes, n) } + sort.Slice(nodes, func(i, j int) bool { + return nodes[i].PeerPacket.GlobalSequence < nodes[j].PeerPacket.GlobalSequence + }) + return nodes +} + +func peerIDs(nodes []ConsensusNodeMetaConfig) []string { + peerIDs := make([]string, 0, len(nodes)) + for _, nodeCfg := range nodes { + peerIDs = append(peerIDs, nodeCfg.PeerPacket.PeerID) + } + return peerIDs +} - return uniqueStats, nil +func mapNodes(nodes []ConsensusNodeMetaConfig) map[string]ConsensusNodeMetaConfig { + m := make(map[string]ConsensusNodeMetaConfig) + for _, node := range nodes { + m[node.PeerPacket.Name()] = node + } + return m } -func publishStatus(ctx context.Context, runenv *runtime.RunEnv, initCtx *run.InitContext, ns Status) error { - _, err := initCtx.SyncClient.Publish(ctx, StatusTopic, ns) +func PublishTestgroundConfig(ctx context.Context, initCtx *run.InitContext, cfg TestgroundConfig) error { + _, err := initCtx.SyncClient.Publish(ctx, TestgroundConfigTopic, cfg) return err } -func downloadStatuses(ctx context.Context, initCtx *run.InitContext, count int) ([]Status, error) { - return DownloadSync(ctx, initCtx, StatusTopic, Status{}, count) +func DownloadTestgroundConfig(ctx context.Context, initCtx *run.InitContext) (TestgroundConfig, error) { + cfgs, err := DownloadSync(ctx, initCtx, TestgroundConfigTopic, TestgroundConfig{}, 1) + if err != nil { + return TestgroundConfig{}, err + } + if len(cfgs) != 1 { + return TestgroundConfig{}, errors.New("no network config was downloaded despite there not being an error") + } + return cfgs[0], nil } func DownloadSync[T any](ctx context.Context, initCtx *run.InitContext, topic *sync.Topic, t T, count int) ([]T, error) { @@ -124,3 +164,7 @@ func DownloadSync[T any](ctx context.Context, initCtx *run.InitContext, topic *s } return output, nil } + +func NodeID(globalSeq int64) string { + return fmt.Sprintf("%d", globalSeq) +} diff --git a/test/testground/network/topology.go b/test/testground/network/topology.go index 7714239b6f..a9773a8abb 100644 --- a/test/testground/network/topology.go +++ b/test/testground/network/topology.go @@ -1,7 +1,6 @@ package network import ( - "errors" "fmt" "strings" @@ -9,7 +8,7 @@ import ( ) const ( - TopologyParamKey = "topology" + ConfiguratorParam = "configurator" ConnectAllTopology = "connect_all" ConnectSubsetTopology = "connect_subset" PersistentPeerCountParamKey = "persistent-peer-count" @@ -21,49 +20,48 @@ func DefaultTopologies() []string { } } -func GetTopologyFns(runenv *runtime.RunEnv) ([]TopologyFn, error) { - topology := runenv.StringParam(TopologyParamKey) +func GetConfigurators(runenv *runtime.RunEnv) ([]Configurator, error) { + topology := runenv.StringParam(ConfiguratorParam) if topology == "" { topology = ConnectAllTopology } - tops := make([]TopologyFn, 0) + ops := make([]Configurator, 0) // TODO: fix the toml parser so that it can handle string arrays for _, topology := range []string{topology} { switch topology { case ConnectAllTopology: - tops = append(tops, ConnectAll) - case ConnectSubsetTopology: - numPeers := runenv.IntParam(PersistentPeerCountParamKey) - tops = append(tops, ConnectSubset(numPeers)) + ops = append(ops, ConnectAll) + // case ConnectSubsetTopology: + // numPeers := runenv.IntParam(PersistentPeerCountParamKey) + // ops = append(ops, ConnectSubset(numPeers)) default: return nil, fmt.Errorf("unknown topology func: %s", topology) } } - return tops, nil + return ops, nil } -// TopologyFn is a function that arbitarily modifies the provided node +// Configurator is a function that arbitarily modifies the provided node // configurations. It is used to generate the topology (which nodes are // connected to which) of the network, along with making other arbitrary changes // to the configs. -type TopologyFn func(nodes []NodeConfig) ([]NodeConfig, error) +type Configurator func(nodes TestgroundConfig) (TestgroundConfig, error) -var _ = TopologyFn(ConnectAll) -var _ = TopologyFn(ConnectSubset(10)) +var _ = Configurator(ConnectAll) -// ConnectAll is a TopologyFn that connects all nodes to each other via +// var _ = Configurator(ConnectSubset(10)) + +// ConnectAll is a Configurator that connects all nodes to each other via // persistent peers. -func ConnectAll(nodes []NodeConfig) ([]NodeConfig, error) { - peerIDs := make([]string, 0, len(nodes)) - for _, nodeCfg := range nodes { - peerIDs = append(peerIDs, nodeCfg.P2PID) - } +func ConnectAll(tcfg TestgroundConfig) (TestgroundConfig, error) { + nodes := tcfg.Nodes() + peerIDs := peerIDs(nodes) // For each node, generate the string that excludes its own P2PID - for nodeID, nodeConfig := range nodes { + for i, nodeConfig := range nodes { var filteredP2PIDs []string for _, pid := range peerIDs { - if pid != nodeConfig.P2PID { + if pid != nodeConfig.PeerPacket.PeerID { filteredP2PIDs = append(filteredP2PIDs, pid) } } @@ -71,55 +69,57 @@ func ConnectAll(nodes []NodeConfig) ([]NodeConfig, error) { // Here you could put the concatenated string into another field in NodeConfig // or do whatever you want with it. nodeConfig.CmtConfig.P2P.PersistentPeers = strings.Join(filteredP2PIDs, ",") - nodes[nodeID] = nodeConfig + nodes[i] = nodeConfig } - return nodes, nil -} - -// ConnectSubset is a TopologyFn that connects each node to a subset of other -// nodes via persistent peers. The subset is rotated for each node to minimize -// overlap. -func ConnectSubset(numPeers int) TopologyFn { - return func(nodes []NodeConfig) ([]NodeConfig, error) { - if len(nodes) < 1 { - return nil, errors.New("no nodes to generate topology for") - } - - if numPeers >= len(nodes) { - return nil, errors.New("number of peers to connect should be less than total number of nodes") - } - - peerIDs := make([]string, 0, len(nodes)) - for _, nodeCfg := range nodes { - peerIDs = append(peerIDs, nodeCfg.P2PID) - } + tcfg.ConsensusNodeConfigs = mapNodes(nodes) - // For each node, generate the list of peers that minimizes overlap - for i, nodeConfig := range nodes { - var filteredP2PIDs []string - - // Locate the index of this node in the peerIDs array - var startIndex int - for i, pid := range peerIDs { - if pid == nodeConfig.P2PID { - startIndex = i - break - } - } - - // Collect numPeers number of P2P IDs, skipping peers to minimize overlap - skip := len(peerIDs) / (numPeers + 1) // Number of peers to skip to get next peer - for i := 1; i <= numPeers; i++ { - targetIndex := (startIndex + i*skip) % len(peerIDs) - filteredP2PIDs = append(filteredP2PIDs, peerIDs[targetIndex]) - } - - // Put the concatenated string into the appropriate field in NodeConfig. - // Here I assume there is a CmtConfig field and a PersistentPeers field within it. - nodes[i].CmtConfig.P2P.PersistentPeers = strings.Join(filteredP2PIDs, ",") - } - - return nodes, nil - } + return tcfg, nil } + +// // ConnectSubset is a Configurator that connects each node to a subset of other +// // nodes via persistent peers. The subset is rotated for each node to minimize +// // overlap. +// func ConnectSubset(numPeers int) Configurator { +// return func(tcfg TestgroundConfig) (TestgroundConfig, error) { +// if len(nodes) < 1 { +// return nil, errors.New("no nodes to generate topology for") +// } + +// if numPeers >= len(nodes) { +// return nil, errors.New("number of peers to connect should be less than total number of nodes") +// } + +// peerIDs := make([]string, 0, len(nodes)) +// for _, nodeCfg := range nodes { +// peerIDs = append(peerIDs, nodeCfg.P2PID) +// } + +// // For each node, generate the list of peers that minimizes overlap +// for i, nodeConfig := range nodes { +// var filteredP2PIDs []string + +// // Locate the index of this node in the peerIDs array +// var startIndex int +// for i, pid := range peerIDs { +// if pid == nodeConfig.P2PID { +// startIndex = i +// break +// } +// } + +// // Collect numPeers number of P2P IDs, skipping peers to minimize overlap +// skip := len(peerIDs) / (numPeers + 1) // Number of peers to skip to get next peer +// for i := 1; i <= numPeers; i++ { +// targetIndex := (startIndex + i*skip) % len(peerIDs) +// filteredP2PIDs = append(filteredP2PIDs, peerIDs[targetIndex]) +// } + +// // Put the concatenated string into the appropriate field in NodeConfig. +// // Here I assume there is a CmtConfig field and a PersistentPeers field within it. +// nodes[i].CmtConfig.P2P.PersistentPeers = strings.Join(filteredP2PIDs, ",") +// } + +// return nodes, nil +// } +// }