Genesis Export Streaming (sei-protocol#1741)

* boilerplate code

* able to write the modules in proper json

* able to do validators and height

* all params in

* update sei-cosmos tag

* debug

* debug

* implemented StreamGenesis in all modules

* debug

* debug

* more prints in wasm section

* print out addr for large state sizes

* fix

* go back to flush after threshold

* add timing to evm

* add timing to prefixes in evm

* get len evm prefixes

* validate-genesis using streaming works

* comment out ibc next_validator_hash checks

* find incorrect next validators hash

* print next validators hash better

* print more info around consensus state

* skip first consensus client

* skip first consensus client 2

* only validate genesis for ibc

* run on all ibc clients

* ADD MARSHALJSON AND UNMARSHALJSON TO HEXBYTES

* checkpoint -- get import to work

* add back MarshalJSON and UnmarshalJSON

* export, validate, import work locally

* remove prints in import

* chunk up evm serialized for export

* update dependencies with their mains

* change seid start flag to config in app.toml

* cleanup

* change name of StreamGenesis to ExportGenesisStream

* change name of AppExporterToFile to AppExporterStream

* make validate-gensis streaming a flag

* change export-to-file to a flag in export cmd

* bump deps

* fix setting genesis to DefaultGenesis to avoid unmarshalling err

* bump deps

* fix deps

app/app.go

@@ -42,9 +42,11 @@ import (
 	"github.com/cosmos/cosmos-sdk/simapp"
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	sdkacltypes "github.com/cosmos/cosmos-sdk/types/accesscontrol"
-	sdkerrors "github.com/cosmos/cosmos-sdk/types/errors"
 	"github.com/cosmos/cosmos-sdk/types/module"
 	"github.com/cosmos/cosmos-sdk/version"
+
+	sdkerrors "github.com/cosmos/cosmos-sdk/types/errors"
+	genesistypes "github.com/cosmos/cosmos-sdk/types/genesis"
 	aclmodule "github.com/cosmos/cosmos-sdk/x/accesscontrol"
 	aclclient "github.com/cosmos/cosmos-sdk/x/accesscontrol/client"
 	aclconstants "github.com/cosmos/cosmos-sdk/x/accesscontrol/constants"
@@ -373,6 +375,8 @@ type App struct {
 	evmRPCConfig          evmrpc.Config
 	lightInvarianceConfig LightInvarianceConfig
 
+	genesisImportConfig genesistypes.GenesisImportConfig
+
 	receiptStore seidb.StateStore
 }
 
@@ -640,6 +644,12 @@ func New(
 	}
 	app.lightInvarianceConfig = lightInvarianceConfig
 
+	genesisImportConfig, err := ReadGenesisImportConfig(appOpts)
+	if err != nil {
+		panic(fmt.Sprintf("error reading genesis import config due to %s", err))
+	}
+	app.genesisImportConfig = genesisImportConfig
+
 	customDependencyGenerators := aclmapping.NewCustomDependencyGenerator()
 	aclOpts = append(aclOpts, aclkeeper.WithResourceTypeToStoreKeyMap(aclutils.ResourceTypeToStoreKeyMap))
 	aclOpts = append(aclOpts, aclkeeper.WithDependencyGeneratorMappings(customDependencyGenerators.GetCustomDependencyGenerators(app.EvmKeeper)))
@@ -1080,11 +1090,13 @@ func (app *App) EndBlocker(ctx sdk.Context, req abci.RequestEndBlock) abci.Respo
 // InitChainer application update at chain initialization
 func (app *App) InitChainer(ctx sdk.Context, req abci.RequestInitChain) abci.ResponseInitChain {
 	var genesisState GenesisState
-	if err := json.Unmarshal(req.AppStateBytes, &genesisState); err != nil {
-		panic(err)
+	if !app.genesisImportConfig.StreamGenesisImport {
+		if err := json.Unmarshal(req.AppStateBytes, &genesisState); err != nil {
+			panic(err)
+		}
 	}
 	app.UpgradeKeeper.SetModuleVersionMap(ctx, app.mm.GetVersionMap())
-	return app.mm.InitGenesis(ctx, app.appCodec, genesisState)
+	return app.mm.InitGenesis(ctx, app.appCodec, genesisState, app.genesisImportConfig)
 }
 
 func (app *App) PrepareProposalHandler(_ sdk.Context, req *abci.RequestPrepareProposal) (*abci.ResponsePrepareProposal, error) {

app/export.go

@@ -2,7 +2,9 @@ package app
 
 import (
 	"encoding/json"
+	"fmt"
 	"log"
+	"os"
 
 	"github.com/cosmos/cosmos-sdk/types/kv"
 
@@ -49,6 +51,41 @@ func (app *App) ExportAppStateAndValidators(
 	}, nil
 }
 
+func (app *App) ExportAppToFileStateAndValidators(
+	forZeroHeight bool, jailAllowedAddrs []string, file *os.File,
+) (servertypes.ExportedApp, error) {
+	// as if they could withdraw from the start of the next block
+	ctx := app.NewContext(true, tmproto.Header{Height: app.LastBlockHeight()})
+
+	// We export at last height + 1, because that's the height at which
+	// Tendermint will start InitChain.
+	height := app.LastBlockHeight() + 1
+	if forZeroHeight {
+		height = 0
+		app.prepForZeroHeightGenesis(ctx, jailAllowedAddrs)
+	}
+	err := app.mm.ProcessGenesisPerModule(ctx, app.appCodec, func(moduleName string, moduleJson json.RawMessage) error {
+		_, err := file.Write([]byte(fmt.Sprintf("{\"app_state\": {\"module\":\"%s\",\"data\":%s}}\n", moduleName, string(moduleJson))))
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+	if err != nil {
+		return servertypes.ExportedApp{}, err
+	}
+	validators, err := staking.WriteValidators(ctx, app.StakingKeeper)
+	if err != nil {
+		return servertypes.ExportedApp{}, err
+	}
+
+	return servertypes.ExportedApp{
+		Validators:      validators,
+		Height:          height,
+		ConsensusParams: app.BaseApp.GetConsensusParams(ctx),
+	}, nil
+}
+
 // AddressFromValidatorsKey creates the validator operator address from ValidatorsKey
 func AddressFromValidatorsKey(key []byte) []byte {
 	kv.AssertKeyAtLeastLength(key, 3)

app/genesis.go

@@ -4,11 +4,39 @@ import (
 	"encoding/json"
 
 	"github.com/cosmos/cosmos-sdk/codec"
+	genesistypes "github.com/cosmos/cosmos-sdk/types/genesis"
 	distrtypes "github.com/cosmos/cosmos-sdk/x/distribution/types"
 
 	sdk "github.com/cosmos/cosmos-sdk/types"
+
+	servertypes "github.com/cosmos/cosmos-sdk/server/types"
+	"github.com/spf13/cast"
+)
+
+var DefaultGenesisConfig = genesistypes.GenesisImportConfig{
+	StreamGenesisImport: false,
+	GenesisStreamFile:   "",
+}
+
+const (
+	flagGenesisStreamImport = "genesis.stream-import"
+	flagGenesisImportFile   = "genesis.import-file"
 )
 
+func ReadGenesisImportConfig(opts servertypes.AppOptions) (genesistypes.GenesisImportConfig, error) {
+	cfg := DefaultGenesisConfig // copy
+	var err error
+	if v := opts.Get(flagGenesisStreamImport); v != nil {
+		if cfg.StreamGenesisImport, err = cast.ToBoolE(v); err != nil {
+			return cfg, err
+		}
+	}
+	if v := opts.Get(flagGenesisImportFile); v != nil {
+		cfg.GenesisStreamFile = v.(string)
+	}
+	return cfg, nil
+}
+
 // The genesis state of the blockchain is represented here as a map of raw json
 // messages key'd by a identifier string.
 // The identifier is used to determine which module genesis information belongs

cmd/seid/cmd/root.go

@@ -313,27 +313,53 @@ func appExport(
 	forZeroHeight bool,
 	jailAllowedAddrs []string,
 	appOpts servertypes.AppOptions,
+	file *os.File,
 ) (servertypes.ExportedApp, error) {
+	exportableApp, err := getExportableApp(
+		logger,
+		db,
+		traceStore,
+		height,
+		appOpts,
+	)
+	if err != nil {
+		return servertypes.ExportedApp{}, err
+	}
+
+	if file == nil {
+		return exportableApp.ExportAppStateAndValidators(forZeroHeight, jailAllowedAddrs)
+	} else {
+		return exportableApp.ExportAppToFileStateAndValidators(forZeroHeight, jailAllowedAddrs, file)
+	}
+}
+
+func getExportableApp(
+	logger log.Logger,
+	db dbm.DB,
+	traceStore io.Writer,
+	height int64,
+	appOpts servertypes.AppOptions,
+) (*app.App, error) {
 	encCfg := app.MakeEncodingConfig()
 	encCfg.Marshaler = codec.NewProtoCodec(encCfg.InterfaceRegistry)
 
 	var exportableApp *app.App
 
 	homePath, ok := appOpts.Get(flags.FlagHome).(string)
 	if !ok || homePath == "" {
-		return servertypes.ExportedApp{}, errors.New("application home not set")
+		return nil, errors.New("application home not set")
 	}
 
 	if height != -1 {
 		exportableApp = app.New(logger, db, traceStore, false, map[int64]bool{}, cast.ToString(appOpts.Get(flags.FlagHome)), uint(1), true, nil, encCfg, app.GetWasmEnabledProposals(), appOpts, app.EmptyWasmOpts, app.EmptyACLOpts, app.EmptyAppOptions)
 		if err := exportableApp.LoadHeight(height); err != nil {
-			return servertypes.ExportedApp{}, err
+			return nil, err
 		}
 	} else {
 		exportableApp = app.New(logger, db, traceStore, true, map[int64]bool{}, cast.ToString(appOpts.Get(flags.FlagHome)), uint(1), true, nil, encCfg, app.GetWasmEnabledProposals(), appOpts, app.EmptyWasmOpts, app.EmptyACLOpts, app.EmptyAppOptions)
 	}
+	return exportableApp, nil
 
-	return exportableApp.ExportAppStateAndValidators(forZeroHeight, jailAllowedAddrs)
 }
 
 func getPrimeNums(lo int, hi int) []int {

go.mod

@@ -344,17 +344,17 @@ require (
 )
 
 replace (
-	github.com/CosmWasm/wasmd => github.com/sei-protocol/sei-wasmd v0.2.2
+	github.com/CosmWasm/wasmd => github.com/sei-protocol/sei-wasmd v0.2.3
 	github.com/confio/ics23/go => github.com/cosmos/cosmos-sdk/ics23/go v0.8.0
-	github.com/cosmos/cosmos-sdk => github.com/sei-protocol/sei-cosmos v0.3.27
+	github.com/cosmos/cosmos-sdk => github.com/sei-protocol/sei-cosmos v0.3.29
 	github.com/cosmos/iavl => github.com/sei-protocol/sei-iavl v0.1.9
-	github.com/cosmos/ibc-go/v3 => github.com/sei-protocol/sei-ibc-go/v3 v3.3.1
+	github.com/cosmos/ibc-go/v3 => github.com/sei-protocol/sei-ibc-go/v3 v3.3.2
 	github.com/ethereum/go-ethereum => github.com/sei-protocol/go-ethereum v1.13.5-sei-22
 	github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1
 	github.com/sei-protocol/sei-db => github.com/sei-protocol/sei-db v0.0.40
 	// Latest goleveldb is broken, we have to stick to this version
 	github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
-	github.com/tendermint/tendermint => github.com/sei-protocol/sei-tendermint v0.3.5
+	github.com/tendermint/tendermint => github.com/sei-protocol/sei-tendermint v0.3.6
 	github.com/tendermint/tm-db => github.com/sei-protocol/tm-db v0.0.4
 	google.golang.org/grpc => google.golang.org/grpc v1.33.2
 )

go.sum

@@ -1347,20 +1347,20 @@ github.com/sei-protocol/go-ethereum v1.13.5-sei-22 h1:t/m1qXER+DEMrcpqgoYmUxifkA
 github.com/sei-protocol/go-ethereum v1.13.5-sei-22/go.mod h1:kcRZmuzRn1lVejiFNTz4l4W7imnpq1bDAnuKS/RyhbQ=
 github.com/sei-protocol/goutils v0.0.2 h1:Bfa7Sv+4CVLNM20QcpvGb81B8C5HkQC/kW1CQpIbXDA=
 github.com/sei-protocol/goutils v0.0.2/go.mod h1:iYE2DuJfEnM+APPehr2gOUXfuLuPsVxorcDO+Tzq9q8=
-github.com/sei-protocol/sei-cosmos v0.3.27 h1:98hhUBySp3BWGjJ7jUuBxKUov4YitrQ2DUTExMG1I9k=
-github.com/sei-protocol/sei-cosmos v0.3.27/go.mod h1:og/KbejR/zSQ8otapODEDU9zYNhFSUDbq9+tgeYePyU=
+github.com/sei-protocol/sei-cosmos v0.3.29 h1:QQUTsqtLq9kpXCDZ+KHzEJKzdkgT7YlmXcZ3ygGFWQw=
+github.com/sei-protocol/sei-cosmos v0.3.29/go.mod h1:og/KbejR/zSQ8otapODEDU9zYNhFSUDbq9+tgeYePyU=
 github.com/sei-protocol/sei-db v0.0.40 h1:s6B3u9u0r2Ypd67P8Lrz2IR/QU/FXwtS2X/fnYEix2g=
 github.com/sei-protocol/sei-db v0.0.40/go.mod h1:F/ZKZA8HJPcUzSZPA8yt6pfwlGriJ4RDR4eHKSGLStI=
 github.com/sei-protocol/sei-iavl v0.1.9 h1:y4mVYftxLNRs6533zl7N0/Ch+CzRQc04JDfHolIxgBE=
 github.com/sei-protocol/sei-iavl v0.1.9/go.mod h1:7PfkEVT5dcoQE+s/9KWdoXJ8VVVP1QpYYPLdxlkSXFk=
-github.com/sei-protocol/sei-ibc-go/v3 v3.3.1 h1:BPG9LWe27x3SATpY9nj8JPe+0igyKyrcpB0z2ZvdcXQ=
-github.com/sei-protocol/sei-ibc-go/v3 v3.3.1/go.mod h1:VwB/vWu4ysT5DN2aF78d17LYmx3omSAdq6gpKvM7XRA=
-github.com/sei-protocol/sei-tendermint v0.3.5 h1:HNHCiLfiZs+O3azdrzGCPwhKaR3HPK2WAB+Ce/UP6Ns=
-github.com/sei-protocol/sei-tendermint v0.3.5/go.mod h1:4LSlJdhl3nf3OmohliwRNUFLOB1XWlrmSodrIP7fLh4=
+github.com/sei-protocol/sei-ibc-go/v3 v3.3.2 h1:BaMZ6gjwqe3R/5dLmcJ1TkSZ3omcWy2TjaAZAeOJH44=
+github.com/sei-protocol/sei-ibc-go/v3 v3.3.2/go.mod h1:VwB/vWu4ysT5DN2aF78d17LYmx3omSAdq6gpKvM7XRA=
+github.com/sei-protocol/sei-tendermint v0.3.6 h1:bvvyWHyJcdCo6+b5Da8p9STyC+7tSNKAI9XPQsYgBuU=
+github.com/sei-protocol/sei-tendermint v0.3.6/go.mod h1:4LSlJdhl3nf3OmohliwRNUFLOB1XWlrmSodrIP7fLh4=
 github.com/sei-protocol/sei-tm-db v0.0.5 h1:3WONKdSXEqdZZeLuWYfK5hP37TJpfaUa13vAyAlvaQY=
 github.com/sei-protocol/sei-tm-db v0.0.5/go.mod h1:Cpa6rGyczgthq7/0pI31jys2Fw0Nfrc+/jKdP1prVqY=
-github.com/sei-protocol/sei-wasmd v0.2.2 h1:0jwGW2E5xlmyP1z1BHEOa60DKMwXxkjd91htbr+bIR4=
-github.com/sei-protocol/sei-wasmd v0.2.2/go.mod h1:EnQkqvUA3tYpdgXjqatHK8ym9LCm1z+lM7XMqR9SA3o=
+github.com/sei-protocol/sei-wasmd v0.2.3 h1:txB7LIx4Luh7+fxVN4/PiHEAjYnkutKyrpsuIFW8lmA=
+github.com/sei-protocol/sei-wasmd v0.2.3/go.mod h1:EnQkqvUA3tYpdgXjqatHK8ym9LCm1z+lM7XMqR9SA3o=
 github.com/sei-protocol/tm-db v0.0.4 h1:7Y4EU62Xzzg6wKAHEotm7SXQR0aPLcGhKHkh3qd0tnk=
 github.com/sei-protocol/tm-db v0.0.4/go.mod h1:PWsIWOTwdwC7Ow/GUvx8HgUJTO691pBuorIQD8JvwAs=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=

scripts/initialize_local_chain.sh

@@ -60,7 +60,7 @@ cat ~/.sei/config/genesis.json | jq '.app_state["oracle"]["params"]["whitelist"]
 cat ~/.sei/config/genesis.json | jq '.app_state["distribution"]["params"]["community_tax"]="0.000000000000000000"' > ~/.sei/config/tmp_genesis.json && mv ~/.sei/config/tmp_genesis.json ~/.sei/config/genesis.json
 cat ~/.sei/config/genesis.json | jq '.consensus_params["block"]["max_gas"]="35000000"' > ~/.sei/config/tmp_genesis.json && mv ~/.sei/config/tmp_genesis.json ~/.sei/config/genesis.json
 cat ~/.sei/config/genesis.json | jq '.app_state["staking"]["params"]["max_voting_power_ratio"]="1.000000000000000000"' > ~/.sei/config/tmp_genesis.json && mv ~/.sei/config/tmp_genesis.json ~/.sei/config/genesis.json
-cat ~/.sei/config/genesis.json | jq '.app_state["bank"]["denom_metadata"]=[{"denom_units":[{"denom":"UATOM","exponent":6,"aliases":["UATOM"]}],"base":"uatom","display":"uatom","name":"UATOM","symbol":"UATOM"}]' > ~/.sei/config/tmp_genesis.json && mv ~/.sei/config/tmp_genesis.json ~/.sei/config/genesis.json
+cat ~/.sei/config/genesis.json | jq '.app_state["bank"]["denom_metadata"]=[{"denom_units":[{"denom":"usei","exponent":0,"aliases":["USEI"]}],"base":"usei","display":"usei","name":"USEI","symbol":"USEI"}]' > ~/.sei/config/tmp_genesis.json && mv ~/.sei/config/tmp_genesis.json ~/.sei/config/genesis.json
 
 # Use the Python command to get the dates
 START_DATE=$($PYTHON_CMD -c "from datetime import datetime; print(datetime.now().strftime('%Y-%m-%d'))")

x/epoch/module.go

@@ -74,6 +74,17 @@ func (AppModuleBasic) ValidateGenesis(cdc codec.JSONCodec, _ client.TxEncodingCo
 	return genState.Validate()
 }
 
+// ValidateGenesisStream performs genesis state validation for the capability module in a streaming fashion.
+func (am AppModuleBasic) ValidateGenesisStream(cdc codec.JSONCodec, config client.TxEncodingConfig, genesisCh <-chan json.RawMessage) error {
+	for genesis := range genesisCh {
+		err := am.ValidateGenesis(cdc, config, genesis)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // RegisterRESTRoutes registers the capability module's REST service handlers.
 func (AppModuleBasic) RegisterRESTRoutes(_ client.Context, _ *mux.Router) {}
 
@@ -164,6 +175,16 @@ func (am AppModule) ExportGenesis(ctx sdk.Context, cdc codec.JSONCodec) json.Raw
 	return cdc.MustMarshalJSON(genState)
 }
 
+// ExportGenesisStream returns the capability module's exported genesis state as raw JSON bytes in a streaming fashion.
+func (am AppModule) ExportGenesisStream(ctx sdk.Context, cdc codec.JSONCodec) <-chan json.RawMessage {
+	ch := make(chan json.RawMessage)
+	go func() {
+		ch <- am.ExportGenesis(ctx, cdc)
+		close(ch)
+	}()
+	return ch
+}
+
 // ConsensusVersion implements ConsensusVersion.
 func (AppModule) ConsensusVersion() uint64 { return 2 }
 

x/evm/genesis.go

@@ -83,6 +83,91 @@ func ExportGenesis(ctx sdk.Context, k *keeper.Keeper) *types.GenesisState {
 	return genesis
 }
 
+// TODO: move to better location
+var GENESIS_EXPORT_STREAM_SERIALIZED_LEN_MAX = 1000
+
+func ExportGenesisStream(ctx sdk.Context, k *keeper.Keeper) <-chan *types.GenesisState {
+	ch := make(chan *types.GenesisState)
+	go func() {
+		genesis := types.DefaultGenesis()
+		genesis.Params = k.GetParams(ctx)
+		ch <- genesis
+
+		k.IterateSeiAddressMapping(ctx, func(evmAddr common.Address, seiAddr sdk.AccAddress) bool {
+			var genesis types.GenesisState
+			genesis.Params = k.GetParams(ctx)
+			genesis.AddressAssociations = append(genesis.AddressAssociations, &types.AddressAssociation{
+				SeiAddress: seiAddr.String(),
+				EthAddress: evmAddr.Hex(),
+			})
+			ch <- &genesis
+			return false
+		})
+
+		k.IterateAllCode(ctx, func(addr common.Address, code []byte) bool {
+			var genesis types.GenesisState
+			genesis.Params = k.GetParams(ctx)
+			genesis.Codes = append(genesis.Codes, &types.Code{
+				Address: addr.Hex(),
+				Code:    code,
+			})
+			ch <- &genesis
+			return false
+		})
+
+		k.IterateState(ctx, func(addr common.Address, key, val common.Hash) bool {
+			var genesis types.GenesisState
+			genesis.Params = k.GetParams(ctx)
+			genesis.States = append(genesis.States, &types.ContractState{
+				Address: addr.Hex(),
+				Key:     key[:],
+				Value:   val[:],
+			})
+			ch <- &genesis
+			return false
+		})
+
+		k.IterateAllNonces(ctx, func(addr common.Address, nonce uint64) bool {
+			var genesis types.GenesisState
+			genesis.Params = k.GetParams(ctx)
+			genesis.Nonces = append(genesis.Nonces, &types.Nonce{
+				Address: addr.Hex(),
+				Nonce:   nonce,
+			})
+			ch <- &genesis
+			return false
+		})
+
+		for _, prefix := range [][]byte{
+			types.ReceiptKeyPrefix,
+			types.BlockBloomPrefix,
+			types.TxHashesPrefix,
+			types.PointerRegistryPrefix,
+			types.PointerCWCodePrefix,
+			types.PointerReverseRegistryPrefix,
+		} {
+			genesis := types.DefaultGenesis()
+			genesis.Params = k.GetParams(ctx)
+			k.IterateAll(ctx, prefix, func(key, val []byte) bool {
+				genesis.Serialized = append(genesis.Serialized, &types.Serialized{
+					Prefix: prefix,
+					Key:    key,
+					Value:  val,
+				})
+				if len(genesis.Serialized) > GENESIS_EXPORT_STREAM_SERIALIZED_LEN_MAX {
+					ch <- genesis
+					genesis = types.DefaultGenesis()
+					genesis.Params = k.GetParams(ctx)
+				}
+				return false
+			})
+			ch <- genesis
+		}
+		close(ch)
+	}()
+	return ch
+}
+
 // GetGenesisStateFromAppState returns x/evm GenesisState given raw application
 // genesis state.
 func GetGenesisStateFromAppState(cdc codec.JSONCodec, appState map[string]json.RawMessage) *types.GenesisState {