node/state: Refactor node pkg -> nodebuilder and split services into …

…modules (#997) * refactor: implement better node pkg structuring for node/state pkg * refactor: rename node/config pkg to node/node * refactor(node/state): Use fx annotation from #923 * refactor(node/state): use constructor from state pkg directly * fix: review suggestions * refactor(node): state module improvements and PR review in the form of a PR (#90) * refactor(node): state module improvements * Move settings, opts and config out of `node/node` back to `node` * `state` pkg module improvements * Absorb Config and options from `node/key` pkg * Don't use core.Config on Module constructor and take from the DI instead * Use node Type and Config on Module constructor * first steps towarads `core` pkg modulazation * chore(swamp): fix linter by shortening import alias * refactor: moving header to subpackage * refactor: moving share components to share subpackage * chore: basic linting * refactor/chore: renaming services and methods to satisfy linter * refactor: moving node construction to node/module.go * refactor: rpc and core subpackages * refactor: building node in /node/module.go, rpc and core fixes for tests * refactor: modularized daser and simplified node/module.go * refactor: passing configs by ref to actually modify node.Config, moving p2p opts, removing config_opts, renaming files to match new scheme * refactor(node): move some global node options to their respective node modules * refactor: unexporting moduleOpts and fields * refactor: variadic options for modules * refactor: removing lifecycle hooks * refactor: renaming node to nodebuilder * refactor: implementing review comments * fix: renaming test directories to nodebuilder * refactor: config setters separated from options, writing config to env for persisting options during initialization * refactor: minor review comments (cleaning clutter to get to the juicy stuff) * refactor: exporting share.Discovery, providing it directly, removing Full and Light availability constructors in nodebuilder/share * refactor: providing header service directly in module * refactor: making InitStore invoke directly without returning a function, removing dead code * fix: fixing issue where flags do not persist when init is called twice Co-authored-by: renaynay <[email protected]> * chore: removing dead code * refactor: deferring setting node config during flag parsing * refactor: collapsing switch in DefaultConfig * refactor: removing nodebuilder/daser/daser.go * refactor: fx lifecycle in for core/remote * chore: imports * refactor: removing all setters, moving checks to construction * refactor: adding cfg.ValidateBasic to each nodebuilder module * refactor: moving core sanity checks to cfg.ValidateBasic and populating default conf * fix: replacing IP on sanity check * renaming to sanitizeIP Co-authored-by @Wondertan Co-authored-by: Ryan <[email protected]> Co-authored-by: renaynay <[email protected]> * Fix issues with config persistence / invoking validation on config (#95) * fix config * make sure to save config before starting node, otherwise all overrides are lost * chore: lint * refactor|test(nodebuilder/tests): update fraud tests to use new nodebuilder configuration functionality * refactor(nodebuilder/header): adapt Fraud (with syncer) to refactoring changes * refactor(nodebuilder/header): restore logic that syncer fails with Warn if no chain head * fix(tests): fixing TestFraudProofSyncing * fix(cmd): not persisting config with flags on start * refactor(nodebuilder): consistent baseComponents, adding back daser proxy constructor, renaming Loader to ConfigLoader * lint: fixing goimports after rebase * refactor: implement better node pkg structuring for node/state pkg * refactor: rename node/config pkg to node/node * refactor(node/state): Use fx annotation from #923 * refactor(node/state): use constructor from state pkg directly * fix: review suggestions * refactor(node): state module improvements and PR review in the form of a PR (#90) * refactor(node): state module improvements * Move settings, opts and config out of `node/node` back to `node` * `state` pkg module improvements * Absorb Config and options from `node/key` pkg * Don't use core.Config on Module constructor and take from the DI instead * Use node Type and Config on Module constructor * first steps towarads `core` pkg modulazation * chore(swamp): fix linter by shortening import alias * refactor: moving header to subpackage * refactor: moving share components to share subpackage * chore: basic linting * refactor/chore: renaming services and methods to satisfy linter * refactor: moving node construction to node/module.go * refactor: rpc and core subpackages * refactor: building node in /node/module.go, rpc and core fixes for tests * refactor: modularized daser and simplified node/module.go * refactor: passing configs by ref to actually modify node.Config, moving p2p opts, removing config_opts, renaming files to match new scheme * refactor(node): move some global node options to their respective node modules * refactor: unexporting moduleOpts and fields * refactor: variadic options for modules * refactor: removing lifecycle hooks * refactor: renaming node to nodebuilder * refactor: implementing review comments * fix: renaming test directories to nodebuilder * refactor: config setters separated from options, writing config to env for persisting options during initialization * refactor: minor review comments (cleaning clutter to get to the juicy stuff) * refactor: exporting share.Discovery, providing it directly, removing Full and Light availability constructors in nodebuilder/share * refactor: providing header service directly in module * refactor: making InitStore invoke directly without returning a function, removing dead code * fix: fixing issue where flags do not persist when init is called twice Co-authored-by: renaynay <[email protected]> * chore: removing dead code * refactor: deferring setting node config during flag parsing * refactor: collapsing switch in DefaultConfig * refactor: removing nodebuilder/daser/daser.go * refactor: fx lifecycle in for core/remote * chore: imports * refactor: removing all setters, moving checks to construction * refactor: adding cfg.ValidateBasic to each nodebuilder module * refactor: moving core sanity checks to cfg.ValidateBasic and populating default conf * fix: replacing IP on sanity check * renaming to sanitizeIP Co-authored-by @Wondertan Co-authored-by: Ryan <[email protected]> Co-authored-by: renaynay <[email protected]> * Fix issues with config persistence / invoking validation on config (#95) * fix config * make sure to save config before starting node, otherwise all overrides are lost * chore: lint * refactor|test(nodebuilder/tests): update fraud tests to use new nodebuilder configuration functionality * refactor(nodebuilder/header): adapt Fraud (with syncer) to refactoring changes * refactor(nodebuilder/header): restore logic that syncer fails with Warn if no chain head * fix(tests): fixing TestFraudProofSyncing * fix(cmd): not persisting config with flags on start * refactor(nodebuilder): consistent baseComponents, adding back daser proxy constructor, renaming Loader to ConfigLoader * lint: fixing goimports after rebase * chore(docs): removing multiline todo from godoc comment for new gofmt compliance * fix(test): Fixing rpc test from rebase * refactor(nodebuilder/fraud): Move fraud-related constructors into separate module * refactor(service/rpc): Wrap port parsing error * refactor(nodebuilder|service): Wrap config error messages with pkg name * refactor(nodebuilder): cfg.ValidateBasic -> cfg.Validate * refactor(nodebuilder): Move refresh routing period check into config validation and fix err check in rpc/config * refactor(das | nodebuilder/p2p): apply suggestions from @Wondertan approval review * fix(cmd): replace logger in cmd after rebase/merge conflict issues * refactor(cmd): set context with config at end of parsing, not in every individual flag parsing func * refactor(cmd): simplify returns in flags_p2p Co-authored-by: Ryan <[email protected]> Co-authored-by: Hlib Kanunnikov <[email protected]> Co-authored-by: renaynay <[email protected]>
celestiaorg · Sep 23, 2022 · cec6a2d · cec6a2d
1 parent 565b2b8
commit cec6a2d
Show file tree

Hide file tree

Showing 89 changed files with 1,767 additions and 1,657 deletions.
diff --git a/Makefile b/Makefile
@@ -82,25 +82,25 @@ lint:
 ## test-unit: Running unit tests
 test-unit:
 	@echo "--> Running unit tests"
-	@go test -v `go list ./... | grep -v node/tests` -covermode=atomic -coverprofile=coverage.out
+	@go test -v `go list ./... | grep -v nodebuilder/tests` -covermode=atomic -coverprofile=coverage.out
 .PHONY: test-unit
 
 ## test-unit-race: Running unit tests with data race detector
 test-unit-race:
 	@echo "--> Running unit tests with data race detector"
-	@go test -v -race `go list ./... | grep -v node/tests`
+	@go test -v -race `go list ./... | grep -v nodebuilder/tests`
 .PHONY: test-unit-race
 
-## test-swamp: Running swamp tests located in node/tests
+## test-swamp: Running swamp tests located in nodebuilder/tests
 test-swamp:
 	@echo "--> Running swamp tests"
-	@go test -v ./node/tests
+	@go test -v ./nodebuilder/tests
 .PHONY: test-swamp
 
 ## test-swamp: Running swamp tests with data race detector located in node/tests
 test-swamp-race:
 	@echo "--> Running swamp tests with data race detector"
-	@go test -v -race ./node/tests
+	@go test -v -race ./nodebuilder/tests
 .PHONY: test-swamp-race
 
 ## test-all: Running both unit and swamp tests

diff --git a/cmd/cel-shed/header.go b/cmd/cel-shed/header.go
@@ -8,7 +8,8 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/celestiaorg/celestia-node/header/store"
-	"github.com/celestiaorg/celestia-node/node"
+	"github.com/celestiaorg/celestia-node/nodebuilder"
+	"github.com/celestiaorg/celestia-node/nodebuilder/node"
 )
 
 func init() {
@@ -42,9 +43,8 @@ Custom store path is not supported yet.`,
 			return fmt.Errorf("invalid height: %w", err)
 		}
 
-		s, err := node.OpenStore(
-			fmt.Sprintf("~/.celestia-%s-%s", strings.ToLower(tp.String()), strings.ToLower(network)),
-		)
+		s, err := nodebuilder.OpenStore(fmt.Sprintf("~/.celestia-%s-%s", strings.ToLower(tp.String()),
+			strings.ToLower(network)))
 		if err != nil {
 			return err
 		}

diff --git a/cmd/celestia/bridge.go b/cmd/celestia/bridge.go
@@ -4,7 +4,7 @@ import (
 	"github.com/spf13/cobra"
 
 	cmdnode "github.com/celestiaorg/celestia-node/cmd"
-	"github.com/celestiaorg/celestia-node/node"
+	"github.com/celestiaorg/celestia-node/nodebuilder/node"
 )
 
 // NOTE: We should always ensure that the added Flags below are parsed somewhere, like in the PersistentPreRun func on
@@ -48,28 +48,28 @@ var bridgeCmd = &cobra.Command{
 			return err
 		}
 
-		ctx, err = cmdnode.ParseP2PFlags(ctx, cmd)
-		if err != nil {
-			return err
-		}
+		cfg := cmdnode.NodeConfig(ctx)
 
-		ctx, err = cmdnode.ParseCoreFlags(ctx, cmd)
+		err = cmdnode.ParseP2PFlags(cmd, &cfg)
 		if err != nil {
 			return err
 		}
 
-		ctx, err = cmdnode.ParseMiscFlags(ctx, cmd)
+		err = cmdnode.ParseCoreFlags(cmd, &cfg)
 		if err != nil {
 			return err
 		}
 
-		ctx, err = cmdnode.ParseRPCFlags(ctx, cmd)
+		ctx, err = cmdnode.ParseMiscFlags(ctx, cmd)
 		if err != nil {
 			return err
 		}
 
-		ctx = cmdnode.ParseKeyFlags(ctx, cmd)
+		cmdnode.ParseRPCFlags(cmd, &cfg)
+		cmdnode.ParseKeyFlags(cmd, &cfg)
 
+		// set config
+		ctx = cmdnode.WithNodeConfig(ctx, &cfg)
 		cmd.SetContext(ctx)
 		return nil
 	},

diff --git a/cmd/celestia/full.go b/cmd/celestia/full.go
@@ -5,7 +5,7 @@ import (
 	"github.com/spf13/cobra"
 
 	cmdnode "github.com/celestiaorg/celestia-node/cmd"
-	"github.com/celestiaorg/celestia-node/node"
+	"github.com/celestiaorg/celestia-node/nodebuilder/node"
 )
 
 // NOTE: We should always ensure that the added Flags below are parsed somewhere, like in the PersistentPreRun func on
@@ -55,33 +55,33 @@ var fullCmd = &cobra.Command{
 			return err
 		}
 
-		ctx, err = cmdnode.ParseP2PFlags(ctx, cmd)
-		if err != nil {
-			return err
-		}
+		cfg := cmdnode.NodeConfig(ctx)
 
-		ctx, err = cmdnode.ParseCoreFlags(ctx, cmd)
+		err = cmdnode.ParseP2PFlags(cmd, &cfg)
 		if err != nil {
 			return err
 		}
 
-		ctx, err = cmdnode.ParseHeadersFlags(ctx, cmd)
+		err = cmdnode.ParseCoreFlags(cmd, &cfg)
 		if err != nil {
 			return err
 		}
 
-		ctx, err = cmdnode.ParseMiscFlags(ctx, cmd)
+		ctx, err = cmdnode.ParseHeadersFlags(ctx, cmd, &cfg)
 		if err != nil {
 			return err
 		}
 
-		ctx, err = cmdnode.ParseRPCFlags(ctx, cmd)
+		ctx, err = cmdnode.ParseMiscFlags(ctx, cmd)
 		if err != nil {
 			return err
 		}
 
-		ctx = cmdnode.ParseKeyFlags(ctx, cmd)
+		cmdnode.ParseRPCFlags(cmd, &cfg)
+		cmdnode.ParseKeyFlags(cmd, &cfg)
 
+		// set config
+		ctx = cmdnode.WithNodeConfig(ctx, &cfg)
 		cmd.SetContext(ctx)
 		return nil
 	},

diff --git a/cmd/celestia/light.go b/cmd/celestia/light.go
@@ -5,7 +5,7 @@ import (
 	"github.com/spf13/cobra"
 
 	cmdnode "github.com/celestiaorg/celestia-node/cmd"
-	"github.com/celestiaorg/celestia-node/node"
+	"github.com/celestiaorg/celestia-node/nodebuilder/node"
 )
 
 // NOTE: We should always ensure that the added Flags below are parsed somewhere, like in the PersistentPreRun func on
@@ -50,38 +50,39 @@ var lightCmd = &cobra.Command{
 
 		ctx = cmdnode.WithNodeType(ctx, node.Light)
 
+		// loads existing config into the environment
 		ctx, err = cmdnode.ParseNodeFlags(ctx, cmd)
 		if err != nil {
 			return err
 		}
 
-		ctx, err = cmdnode.ParseP2PFlags(ctx, cmd)
-		if err != nil {
-			return err
-		}
+		cfg := cmdnode.NodeConfig(ctx)
 
-		ctx, err = cmdnode.ParseCoreFlags(ctx, cmd)
+		err = cmdnode.ParseP2PFlags(cmd, &cfg)
 		if err != nil {
 			return err
 		}
 
-		ctx, err = cmdnode.ParseHeadersFlags(ctx, cmd)
+		err = cmdnode.ParseCoreFlags(cmd, &cfg)
 		if err != nil {
 			return err
 		}
 
-		ctx, err = cmdnode.ParseMiscFlags(ctx, cmd)
+		ctx, err = cmdnode.ParseHeadersFlags(ctx, cmd, &cfg)
 		if err != nil {
 			return err
 		}
 
-		ctx, err = cmdnode.ParseRPCFlags(ctx, cmd)
+		ctx, err = cmdnode.ParseMiscFlags(ctx, cmd)
 		if err != nil {
 			return err
 		}
 
-		ctx = cmdnode.ParseKeyFlags(ctx, cmd)
+		cmdnode.ParseRPCFlags(cmd, &cfg)
+		cmdnode.ParseKeyFlags(cmd, &cfg)
 
+		// set config
+		ctx = cmdnode.WithNodeConfig(ctx, &cfg)
 		cmd.SetContext(ctx)
 		return nil
 	},

diff --git a/cmd/env.go b/cmd/env.go
@@ -4,13 +4,15 @@ import (
 	"context"
 
 	logging "github.com/ipfs/go-log/v2"
+	"go.uber.org/fx"
 
-	"github.com/celestiaorg/celestia-node/node"
+	"github.com/celestiaorg/celestia-node/nodebuilder"
+	"github.com/celestiaorg/celestia-node/nodebuilder/node"
 )
 
 var log = logging.Logger("cmd")
 
-// NodeType reads the node.Type from the context.
+// NodeType reads the node type from the context.
 func NodeType(ctx context.Context) node.Type {
 	return ctx.Value(nodeTypeKey{}).(node.Type)
 }
@@ -20,7 +22,17 @@ func StorePath(ctx context.Context) string {
 	return ctx.Value(storePathKey{}).(string)
 }
 
-// WithNodeType sets Node Type in the given context.
+// NodeConfig reads the node config from the context.
+func NodeConfig(ctx context.Context) nodebuilder.Config {
+	cfg, ok := ctx.Value(configKey{}).(nodebuilder.Config)
+	if !ok {
+		nodeType := NodeType(ctx)
+		cfg = *nodebuilder.DefaultConfig(nodeType)
+	}
+	return cfg
+}
+
+// WithNodeType sets the node type in the given context.
 func WithNodeType(ctx context.Context, tp node.Type) context.Context {
 	return context.WithValue(ctx, nodeTypeKey{}, tp)
 }
@@ -30,23 +42,29 @@ func WithStorePath(ctx context.Context, storePath string) context.Context {
 	return context.WithValue(ctx, storePathKey{}, storePath)
 }
 
-// NodeOptions returns node options parsed from Environment(Flags, ENV vars, etc)
-func NodeOptions(ctx context.Context) []node.Option {
-	options, ok := ctx.Value(optionsKey{}).([]node.Option)
+// NodeOptions returns config options parsed from Environment(Flags, ENV vars, etc)
+func NodeOptions(ctx context.Context) []fx.Option {
+	options, ok := ctx.Value(optionsKey{}).([]fx.Option)
 	if !ok {
-		return []node.Option{}
+		return []fx.Option{}
 	}
 	return options
 }
 
 // WithNodeOptions add new options to Env.
-func WithNodeOptions(ctx context.Context, opts ...node.Option) context.Context {
+func WithNodeOptions(ctx context.Context, opts ...fx.Option) context.Context {
 	options := NodeOptions(ctx)
 	return context.WithValue(ctx, optionsKey{}, append(options, opts...))
 }
 
+// WithNodeConfig sets the node config in the Env.
+func WithNodeConfig(ctx context.Context, config *nodebuilder.Config) context.Context {
+	return context.WithValue(ctx, configKey{}, *config)
+}
+
 type (
 	optionsKey   struct{}
+	configKey    struct{}
 	storePathKey struct{}
 	nodeTypeKey  struct{}
 )
diff --git a/cmd/flags_core.go b/cmd/flags_core.go
@@ -1,15 +1,12 @@
 package cmd
 
 import (
-	"context"
 	"fmt"
-	"strconv"
-	"strings"
 
 	"github.com/spf13/cobra"
 	flag "github.com/spf13/pflag"
 
-	"github.com/celestiaorg/celestia-node/node"
+	"github.com/celestiaorg/celestia-node/nodebuilder"
 )
 
 var (
@@ -38,54 +35,27 @@ func CoreFlags() *flag.FlagSet {
 		"9090",
 		"Set a custom gRPC port for the core node connection. The --core.ip flag must also be provided.",
 	)
-
 	return flags
 }
 
 // ParseCoreFlags parses Core flags from the given cmd and applies values to Env.
-func ParseCoreFlags(ctx context.Context, cmd *cobra.Command) (context.Context, error) {
+func ParseCoreFlags(
+	cmd *cobra.Command,
+	cfg *nodebuilder.Config,
+) error {
 	coreIP := cmd.Flag(coreFlag).Value.String()
 	if coreIP == "" {
 		if cmd.Flag(coreGRPCFlag).Changed || cmd.Flag(coreRPCFlag).Changed {
-			return ctx, fmt.Errorf("cannot specify RPC/gRPC ports without specifying an IP address for --core.ip")
+			return fmt.Errorf("cannot specify RPC/gRPC ports without specifying an IP address for --core.ip")
 		}
-		return ctx, nil
-	}
-	// sanity check given core ip addr and strip leading protocol
-	ip, err := sanityCheckIP(coreIP)
-	if err != nil {
-		return ctx, err
+		return nil
 	}
 
 	rpc := cmd.Flag(coreRPCFlag).Value.String()
-	// sanity check rpc endpoint
-	_, err = strconv.Atoi(rpc)
-	if err != nil {
-		return ctx, err
-	}
-	ctx = WithNodeOptions(ctx, node.WithRemoteCoreIP(ip), node.WithRemoteCorePort(rpc))
-
 	grpc := cmd.Flag(coreGRPCFlag).Value.String()
-	// sanity check gRPC endpoint
-	_, err = strconv.Atoi(grpc)
-	if err != nil {
-		return ctx, err
-	}
-	ctx = WithNodeOptions(ctx, node.WithGRPCPort(grpc))
-	return ctx, nil
-}
 
-// sanityCheckIP trims leading protocol scheme and port from the given
-// IP address if present.
-func sanityCheckIP(ip string) (string, error) {
-	original := ip
-	ip = strings.TrimPrefix(ip, "http://")
-	ip = strings.TrimPrefix(ip, "https://")
-	ip = strings.TrimPrefix(ip, "tcp://")
-	ip = strings.TrimSuffix(ip, "/")
-	ip = strings.Split(ip, ":")[0]
-	if ip == "" {
-		return "", fmt.Errorf("invalid IP addr given: %s", original)
-	}
-	return ip, nil
+	cfg.Core.IP = coreIP
+	cfg.Core.RPCPort = rpc
+	cfg.Core.GRPCPort = grpc
+	return nil
 }