From a474e4acd8429021994976f12a9db168920615ad Mon Sep 17 00:00:00 2001 From: Matt Curtis Date: Tue, 26 Mar 2024 12:31:35 +0000 Subject: [PATCH 1/4] WIP --- go/common/host/host_healthcheck.go | 19 +++ go/config/host_config.go | 10 +- go/host/container/cli.go | 9 +- go/host/container/cli_flags.go | 4 +- go/host/container/host_container.go | 12 +- go/host/enclave/service.go | 83 ++++++++---- go/host/host.go | 15 ++- go/host/rpc/enclaverpc/enclave_client.go | 87 ++++++------- go/node/config.go | 2 +- .../networktest/actions/node_actions.go | 9 +- integration/networktest/interfaces.go | 4 +- .../sequencer_multi_enclave_test.go | 34 +++++ integration/simulation/devnetwork/node.go | 118 ++++++++++++------ .../simulation/network/network_utils.go | 4 +- 14 files changed, 274 insertions(+), 136 deletions(-) diff --git a/go/common/host/host_healthcheck.go b/go/common/host/host_healthcheck.go index 3b186d3920..5a2791c920 100644 --- a/go/common/host/host_healthcheck.go +++ b/go/common/host/host_healthcheck.go @@ -24,3 +24,22 @@ func (l *BasicErrHealthStatus) OK() bool { func (l *BasicErrHealthStatus) Message() string { return l.ErrMsg } + +type GroupErrsHealthStatus struct { + Errors []error +} + +func (g *GroupErrsHealthStatus) OK() bool { + return len(g.Errors) == 0 +} + +func (g *GroupErrsHealthStatus) Message() string { + msg := "" + for i, err := range g.Errors { + if i > 0 { + msg += ", " + } + msg += err.Error() + } + return msg +} diff --git a/go/config/host_config.go b/go/config/host_config.go index 23fa494257..0367b3ef11 100644 --- a/go/config/host_config.go +++ b/go/config/host_config.go @@ -32,7 +32,7 @@ type HostInputConfig struct { // Host on which to handle client RPC requests ClientRPCHost string // Address on which to connect to the enclave - EnclaveRPCAddress string + EnclaveRPCAddresses []string // P2PBindAddress is the address where the P2P server is bound to P2PBindAddress string // P2PPublicAddress is the advertised P2P server address @@ -111,7 +111,7 @@ func (p HostInputConfig) ToHostConfig() *HostConfig { HasClientRPCWebsockets: p.HasClientRPCWebsockets, ClientRPCPortWS: p.ClientRPCPortWS, ClientRPCHost: p.ClientRPCHost, - EnclaveRPCAddress: p.EnclaveRPCAddress, + EnclaveRPCAddresses: p.EnclaveRPCAddresses, P2PBindAddress: p.P2PBindAddress, P2PPublicAddress: p.P2PPublicAddress, L1WebsocketURL: p.L1WebsocketURL, @@ -208,8 +208,8 @@ type HostConfig struct { ClientRPCPortWS uint64 // Host on which to handle client RPC requests ClientRPCHost string - // Address on which to connect to the enclave - EnclaveRPCAddress string + // Addresses on which to connect to the node's enclaves (HA setups may have multiple) + EnclaveRPCAddresses []string // P2PBindAddress is the address where the P2P server is bound to P2PBindAddress string // P2PPublicAddress is the advertised P2P server address @@ -244,7 +244,7 @@ func DefaultHostParsedConfig() *HostInputConfig { HasClientRPCWebsockets: true, ClientRPCPortWS: 81, ClientRPCHost: "127.0.0.1", - EnclaveRPCAddress: "127.0.0.1:11000", + EnclaveRPCAddresses: []string{"127.0.0.1:11000"}, P2PBindAddress: "0.0.0.0:10000", P2PPublicAddress: "127.0.0.1:10000", L1WebsocketURL: "ws://127.0.0.1:8546", diff --git a/go/host/container/cli.go b/go/host/container/cli.go index 3d2da90b3e..4a8efe67d8 100644 --- a/go/host/container/cli.go +++ b/go/host/container/cli.go @@ -4,6 +4,7 @@ import ( "flag" "fmt" "os" + "strings" "time" "github.com/ten-protocol/go-ten/go/common" @@ -24,7 +25,7 @@ type HostConfigToml struct { HasClientRPCWebsockets bool ClientRPCPortWS uint ClientRPCHost string - EnclaveRPCAddress string + EnclaveRPCAddresses string // comma-separated P2PBindAddress string P2PPublicAddress string L1WebsocketURL string @@ -66,7 +67,7 @@ func ParseConfig() (*config.HostInputConfig, error) { clientRPCPortHTTP := flag.Uint64(clientRPCPortHTTPName, cfg.ClientRPCPortHTTP, flagUsageMap[clientRPCPortHTTPName]) clientRPCPortWS := flag.Uint64(clientRPCPortWSName, cfg.ClientRPCPortWS, flagUsageMap[clientRPCPortWSName]) clientRPCHost := flag.String(clientRPCHostName, cfg.ClientRPCHost, flagUsageMap[clientRPCHostName]) - enclaveRPCAddress := flag.String(enclaveRPCAddressName, cfg.EnclaveRPCAddress, flagUsageMap[enclaveRPCAddressName]) + enclaveRPCAddressesStr := flag.String(enclaveRPCAddressesName, strings.Join(cfg.EnclaveRPCAddresses, ","), flagUsageMap[enclaveRPCAddressesName]) p2pBindAddress := flag.String(p2pBindAddressName, cfg.P2PBindAddress, flagUsageMap[p2pBindAddressName]) p2pPublicAddress := flag.String(p2pPublicAddressName, cfg.P2PPublicAddress, flagUsageMap[p2pPublicAddressName]) l1WSURL := flag.String(l1WebsocketURLName, cfg.L1WebsocketURL, flagUsageMap[l1WebsocketURLName]) @@ -112,7 +113,7 @@ func ParseConfig() (*config.HostInputConfig, error) { cfg.HasClientRPCWebsockets = true cfg.ClientRPCPortWS = *clientRPCPortWS cfg.ClientRPCHost = *clientRPCHost - cfg.EnclaveRPCAddress = *enclaveRPCAddress + cfg.EnclaveRPCAddresses = strings.Split(*enclaveRPCAddressesStr, ",") cfg.P2PBindAddress = *p2pBindAddress cfg.P2PPublicAddress = *p2pPublicAddress cfg.L1WebsocketURL = *l1WSURL @@ -189,7 +190,7 @@ func fileBasedConfig(configPath string) (*config.HostInputConfig, error) { HasClientRPCWebsockets: tomlConfig.HasClientRPCWebsockets, ClientRPCPortWS: uint64(tomlConfig.ClientRPCPortWS), ClientRPCHost: tomlConfig.ClientRPCHost, - EnclaveRPCAddress: tomlConfig.EnclaveRPCAddress, + EnclaveRPCAddresses: strings.Split(tomlConfig.EnclaveRPCAddresses, ","), P2PBindAddress: tomlConfig.P2PBindAddress, P2PPublicAddress: tomlConfig.P2PPublicAddress, L1WebsocketURL: tomlConfig.L1WebsocketURL, diff --git a/go/host/container/cli_flags.go b/go/host/container/cli_flags.go index ee280a1d41..db64a90f65 100644 --- a/go/host/container/cli_flags.go +++ b/go/host/container/cli_flags.go @@ -9,7 +9,7 @@ const ( clientRPCPortHTTPName = "clientRPCPortHttp" clientRPCPortWSName = "clientRPCPortWs" clientRPCHostName = "clientRPCHost" - enclaveRPCAddressName = "enclaveRPCAddress" + enclaveRPCAddressesName = "enclaveRPCAddresses" p2pBindAddressName = "p2pBindAddress" p2pPublicAddressName = "p2pPublicAddress" l1WebsocketURLName = "l1WSURL" @@ -49,7 +49,7 @@ func getFlagUsageMap() map[string]string { clientRPCPortHTTPName: "The port on which to listen for client application RPC requests over HTTP", clientRPCPortWSName: "The port on which to listen for client application RPC requests over websockets", clientRPCHostName: "The host on which to handle client application RPC requests", - enclaveRPCAddressName: "The address to use to connect to the Obscuro enclave service", + enclaveRPCAddressesName: "The comma-separated addresses to use to connect to the Ten enclaves", p2pBindAddressName: "The address where the p2p server is bound to. Defaults to 0.0.0.0:10000", p2pPublicAddressName: "The P2P address where the other servers should connect to. Defaults to 127.0.0.1:10000", l1WebsocketURLName: "The websocket RPC address the host can use for L1 requests", diff --git a/go/host/container/host_container.go b/go/host/container/host_container.go index 033ab80bbd..24340115b9 100644 --- a/go/host/container/host_container.go +++ b/go/host/container/host_container.go @@ -130,9 +130,13 @@ func NewHostContainerFromConfig(parsedConfig *config.HostInputConfig, logger get // set the Host ID as the Public Key Address cfg.ID = ethWallet.Address() + if len(cfg.EnclaveRPCAddresses) != 1 { + logger.Crit("expected exactly one enclave address", "enclave_addresses", cfg.EnclaveRPCAddresses) + } + fmt.Println("Connecting to the enclave...") services := host.NewServicesRegistry(logger) - enclaveClient := enclaverpc.NewClient(cfg, logger) + enclaveClients := []common.Enclave{enclaverpc.NewClient(cfg.EnclaveRPCAddresses[0], cfg.EnclaveRPCTimeout, logger)} p2pLogger := logger.New(log.CmpKey, log.P2PCmp) metricsService := metrics.New(cfg.MetricsEnabled, cfg.MetricsHTTPPort, logger) @@ -150,13 +154,13 @@ func NewHostContainerFromConfig(parsedConfig *config.HostInputConfig, logger get obscuroRelevantContracts := []gethcommon.Address{cfg.ManagementContractAddress, cfg.MessageBusAddress} l1Repo := l1.NewL1Repository(l1Client, obscuroRelevantContracts, logger) - return NewHostContainer(cfg, services, aggP2P, l1Client, l1Repo, enclaveClient, mgmtContractLib, ethWallet, rpcServer, logger, metricsService) + return NewHostContainer(cfg, services, aggP2P, l1Client, l1Repo, enclaveClients, mgmtContractLib, ethWallet, rpcServer, logger, metricsService) } // NewHostContainer builds a host container with dependency injection rather than from config. // Useful for testing etc. (want to be able to pass in logger, and also have option to mock out dependencies) -func NewHostContainer(cfg *config.HostConfig, services *host.ServicesRegistry, p2p hostcommon.P2PHostService, l1Client ethadapter.EthClient, l1Repo hostcommon.L1RepoService, enclaveClient common.Enclave, contractLib mgmtcontractlib.MgmtContractLib, hostWallet wallet.Wallet, rpcServer node.Server, logger gethlog.Logger, metricsService *metrics.Service) *HostContainer { - h := host.NewHost(cfg, services, p2p, l1Client, l1Repo, enclaveClient, hostWallet, contractLib, logger, metricsService.Registry()) +func NewHostContainer(cfg *config.HostConfig, services *host.ServicesRegistry, p2p hostcommon.P2PHostService, l1Client ethadapter.EthClient, l1Repo hostcommon.L1RepoService, enclaveClients []common.Enclave, contractLib mgmtcontractlib.MgmtContractLib, hostWallet wallet.Wallet, rpcServer node.Server, logger gethlog.Logger, metricsService *metrics.Service) *HostContainer { + h := host.NewHost(cfg, services, p2p, l1Client, l1Repo, enclaveClients, hostWallet, contractLib, logger, metricsService.Registry()) hostContainer := &HostContainer{ host: h, diff --git a/go/host/enclave/service.go b/go/host/enclave/service.go index 402603ba75..1c836a2ad8 100644 --- a/go/host/enclave/service.go +++ b/go/host/enclave/service.go @@ -23,31 +23,46 @@ type enclaveServiceLocator interface { type Service struct { hostData host.Identity sl enclaveServiceLocator - // eventually this service will support multiple enclaves for HA, but currently there's only one - // The service goes via the Guardian to talk to the enclave (because guardian knows if the enclave is healthy etc.) - enclaveGuardian *Guardian + + // The service goes via the Guardians to talk to the enclave (because guardian knows if the enclave is healthy etc.) + enclaveGuardians []*Guardian running atomic.Bool logger gethlog.Logger } -func NewService(hostData host.Identity, serviceLocator enclaveServiceLocator, enclaveGuardian *Guardian, logger gethlog.Logger) *Service { +func NewService(hostData host.Identity, serviceLocator enclaveServiceLocator, enclaveGuardians []*Guardian, logger gethlog.Logger) *Service { return &Service{ - hostData: hostData, - sl: serviceLocator, - enclaveGuardian: enclaveGuardian, - logger: logger, + hostData: hostData, + sl: serviceLocator, + enclaveGuardians: enclaveGuardians, + logger: logger, } } func (e *Service) Start() error { e.running.Store(true) - return e.enclaveGuardian.Start() + for _, guardian := range e.enclaveGuardians { + if err := guardian.Start(); err != nil { + // abandon starting the rest of the guardians if one fails + return err + } + } + return nil } func (e *Service) Stop() error { e.running.Store(false) - return e.enclaveGuardian.Stop() + var errors []error + for i, guardian := range e.enclaveGuardians { + if err := guardian.Stop(); err != nil { + errors = append(errors, fmt.Errorf("error stopping enclave guardian [%d]: %w", i, err)) + } + } + if len(errors) > 0 { + return fmt.Errorf("errors stopping enclave guardians: %v", errors) + } + return nil } func (e *Service) HealthStatus() host.HealthStatus { @@ -55,41 +70,57 @@ func (e *Service) HealthStatus() host.HealthStatus { return &host.BasicErrHealthStatus{ErrMsg: "not running"} } - // check the enclave health, which in turn checks the DB health - enclaveHealthy, err := e.enclaveGuardian.enclaveClient.HealthCheck() - if err != nil { - return &host.BasicErrHealthStatus{ErrMsg: fmt.Sprintf("unable to HealthCheck enclave - %s", err.Error())} - } else if !enclaveHealthy { - return &host.BasicErrHealthStatus{ErrMsg: "enclave reported itself as not healthy"} - } + errors := make([]error, 0, len(e.enclaveGuardians)) - if !e.enclaveGuardian.GetEnclaveState().InSyncWithL1() { - return &host.BasicErrHealthStatus{ErrMsg: "enclave not in sync with L1"} + for i, guardian := range e.enclaveGuardians { + // check the enclave health, which in turn checks the DB health + enclaveHealthy, err := guardian.enclaveClient.HealthCheck() + if err != nil { + errors = append(errors, fmt.Errorf("unable to HealthCheck enclave[%d] - %w", i, err)) + } else if !enclaveHealthy { + errors = append(errors, fmt.Errorf("enclave[%d] reported itself not healthy", i)) + } + + if !guardian.GetEnclaveState().InSyncWithL1() { + errors = append(errors, fmt.Errorf("enclave[%d] not in sync with L1", i)) + } } // empty error msg means healthy - return &host.BasicErrHealthStatus{ErrMsg: ""} + return &host.GroupErrsHealthStatus{Errors: errors} +} + +func (e *Service) HealthyGuardian() *Guardian { + for _, guardian := range e.enclaveGuardians { + if guardian.HealthStatus().OK() { + return guardian + } + } + return nil } // LookupBatchBySeqNo is used to fetch batch data from the enclave - it is only used as a fallback for the sequencer // host if it's missing a batch (other host services should use L2Repo to fetch batch data) func (e *Service) LookupBatchBySeqNo(seqNo *big.Int) (*common.ExtBatch, error) { - state := e.enclaveGuardian.GetEnclaveState() + hg := e.HealthyGuardian() + state := hg.GetEnclaveState() if state.GetEnclaveL2Head().Cmp(seqNo) < 0 { return nil, errutil.ErrNotFound } - client := e.enclaveGuardian.GetEnclaveClient() + client := hg.GetEnclaveClient() return client.GetBatchBySeqNo(seqNo.Uint64()) } func (e *Service) GetEnclaveClient() common.Enclave { - return e.enclaveGuardian.GetEnclaveClient() + // for now we always return first guardian's enclave client + // in future be good to load balance and failover but need to improve subscribe/unsubscribe (unsubscribe from same enclave) + return e.enclaveGuardians[0].GetEnclaveClient() } func (e *Service) SubmitAndBroadcastTx(encryptedParams common.EncryptedParamsSendRawTx) (*responses.RawTx, error) { encryptedTx := common.EncryptedTx(encryptedParams) - enclaveResponse, sysError := e.enclaveGuardian.GetEnclaveClient().SubmitTx(encryptedTx) + enclaveResponse, sysError := e.GetEnclaveClient().SubmitTx(encryptedTx) if sysError != nil { e.logger.Warn("Could not submit transaction due to sysError.", log.ErrKey, sysError) return nil, sysError @@ -110,9 +141,9 @@ func (e *Service) SubmitAndBroadcastTx(encryptedParams common.EncryptedParamsSen } func (e *Service) Subscribe(id rpc.ID, encryptedParams common.EncryptedParamsLogSubscription) error { - return e.enclaveGuardian.GetEnclaveClient().Subscribe(id, encryptedParams) + return e.GetEnclaveClient().Subscribe(id, encryptedParams) } func (e *Service) Unsubscribe(id rpc.ID) error { - return e.enclaveGuardian.GetEnclaveClient().Unsubscribe(id) + return e.GetEnclaveClient().Unsubscribe(id) } diff --git a/go/host/host.go b/go/host/host.go index 60c2ec4b03..35c5babf73 100644 --- a/go/host/host.go +++ b/go/host/host.go @@ -49,7 +49,7 @@ type host struct { l2MessageBusAddress *gethcommon.Address } -func NewHost(config *config.HostConfig, hostServices *ServicesRegistry, p2p hostcommon.P2PHostService, ethClient ethadapter.EthClient, l1Repo hostcommon.L1RepoService, enclaveClient common.Enclave, ethWallet wallet.Wallet, mgmtContractLib mgmtcontractlib.MgmtContractLib, logger gethlog.Logger, regMetrics gethmetrics.Registry) hostcommon.Host { +func NewHost(config *config.HostConfig, hostServices *ServicesRegistry, p2p hostcommon.P2PHostService, ethClient ethadapter.EthClient, l1Repo hostcommon.L1RepoService, enclaveClients []common.Enclave, ethWallet wallet.Wallet, mgmtContractLib mgmtcontractlib.MgmtContractLib, logger gethlog.Logger, regMetrics gethmetrics.Registry) hostcommon.Host { database, err := db.CreateDBFromConfig(config, regMetrics, logger) if err != nil { logger.Crit("unable to create database for host", log.ErrKey, err) @@ -72,8 +72,17 @@ func NewHost(config *config.HostConfig, hostServices *ServicesRegistry, p2p host stopControl: stopcontrol.New(), } - enclGuardian := enclave.NewGuardian(config, hostIdentity, hostServices, enclaveClient, database, host.stopControl, logger) - enclService := enclave.NewService(hostIdentity, hostServices, enclGuardian, logger) + enclGuardians := make([]*enclave.Guardian, 0, len(enclaveClients)) + for i, enclClient := range enclaveClients { + if i > 0 { + // only the first enclave can be the sequencer for now, others behave as read-only validators + hostIdentity.IsSequencer = false + } + enclGuardian := enclave.NewGuardian(config, hostIdentity, hostServices, enclClient, database, host.stopControl, logger) + enclGuardians = append(enclGuardians, enclGuardian) + } + + enclService := enclave.NewService(hostIdentity, hostServices, enclGuardians, logger) l2Repo := l2.NewBatchRepository(config, hostServices, database, logger) subsService := events.NewLogEventManager(hostServices, logger) diff --git a/go/host/rpc/enclaverpc/enclave_client.go b/go/host/rpc/enclaverpc/enclave_client.go index 7f8ce81bac..7ff31de0bb 100644 --- a/go/host/rpc/enclaverpc/enclave_client.go +++ b/go/host/rpc/enclaverpc/enclave_client.go @@ -20,7 +20,6 @@ import ( "github.com/ten-protocol/go-ten/go/common/rpc/generated" "github.com/ten-protocol/go-ten/go/common/syserr" "github.com/ten-protocol/go-ten/go/common/tracers" - "github.com/ten-protocol/go-ten/go/config" "github.com/ten-protocol/go-ten/go/responses" "google.golang.org/grpc" @@ -34,15 +33,16 @@ import ( // Client implements enclave.Enclave and should be used by the host when communicating with the enclave via RPC. type Client struct { - protoClient generated.EnclaveProtoClient - connection *grpc.ClientConn - config *config.HostConfig - logger gethlog.Logger + protoClient generated.EnclaveProtoClient + connection *grpc.ClientConn + enclaveRPCAddress string + enclaveRPCTimeout time.Duration + logger gethlog.Logger } -func NewClient(config *config.HostConfig, logger gethlog.Logger) common.Enclave { +func NewClient(enclaveRPCAddress string, enclaveRPCTimeout time.Duration, logger gethlog.Logger) common.Enclave { opts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())} - connection, err := grpc.Dial(config.EnclaveRPCAddress, opts...) + connection, err := grpc.Dial(enclaveRPCAddress, opts...) if err != nil { logger.Crit("Failed to connect to enclave RPC service.", log.ErrKey, err) } @@ -54,7 +54,7 @@ func NewClient(config *config.HostConfig, logger gethlog.Logger) common.Enclave err = retry.Do(func() error { currState := connection.GetState() if currState != connectivity.Ready { - logger.Info("retrying connection until enclave is available", "status", currState.String(), "rpcAddr", config.EnclaveRPCAddress) + logger.Info("retrying connection until enclave is available", "status", currState.String(), "rpcAddr", enclaveRPCAddress) connection.Connect() return fmt.Errorf("connection is not ready, status=%s", currState) } @@ -67,10 +67,11 @@ func NewClient(config *config.HostConfig, logger gethlog.Logger) common.Enclave } return &Client{ - protoClient: generated.NewEnclaveProtoClient(connection), - connection: connection, - config: config, - logger: logger, + protoClient: generated.NewEnclaveProtoClient(connection), + connection: connection, + enclaveRPCAddress: enclaveRPCAddress, + enclaveRPCTimeout: enclaveRPCTimeout, + logger: logger, } } @@ -84,7 +85,7 @@ func (c *Client) Status() (common.Status, common.SystemError) { return common.Status{StatusCode: common.Unavailable}, syserr.NewInternalError(fmt.Errorf("RPC connection is not ready")) } - timeoutCtx, cancel := context.WithTimeout(context.Background(), c.config.EnclaveRPCTimeout) + timeoutCtx, cancel := context.WithTimeout(context.Background(), c.enclaveRPCTimeout) defer cancel() response, err := c.protoClient.Status(timeoutCtx, &generated.StatusRequest{}) @@ -103,7 +104,7 @@ func (c *Client) Status() (common.Status, common.SystemError) { } func (c *Client) Attestation() (*common.AttestationReport, common.SystemError) { - timeoutCtx, cancel := context.WithTimeout(context.Background(), c.config.EnclaveRPCTimeout) + timeoutCtx, cancel := context.WithTimeout(context.Background(), c.enclaveRPCTimeout) defer cancel() response, err := c.protoClient.Attestation(timeoutCtx, &generated.AttestationRequest{}) @@ -117,7 +118,7 @@ func (c *Client) Attestation() (*common.AttestationReport, common.SystemError) { } func (c *Client) GenerateSecret() (common.EncryptedSharedEnclaveSecret, common.SystemError) { - timeoutCtx, cancel := context.WithTimeout(context.Background(), c.config.EnclaveRPCTimeout) + timeoutCtx, cancel := context.WithTimeout(context.Background(), c.enclaveRPCTimeout) defer cancel() response, err := c.protoClient.GenerateSecret(timeoutCtx, &generated.GenerateSecretRequest{}) @@ -132,7 +133,7 @@ func (c *Client) GenerateSecret() (common.EncryptedSharedEnclaveSecret, common.S } func (c *Client) InitEnclave(secret common.EncryptedSharedEnclaveSecret) common.SystemError { - timeoutCtx, cancel := context.WithTimeout(context.Background(), c.config.EnclaveRPCTimeout) + timeoutCtx, cancel := context.WithTimeout(context.Background(), c.enclaveRPCTimeout) defer cancel() response, err := c.protoClient.InitEnclave(timeoutCtx, &generated.InitEnclaveRequest{EncryptedSharedEnclaveSecret: secret}) @@ -146,7 +147,7 @@ func (c *Client) InitEnclave(secret common.EncryptedSharedEnclaveSecret) common. } func (c *Client) EnclaveID() (common.EnclaveID, common.SystemError) { - timeoutCtx, cancel := context.WithTimeout(context.Background(), c.config.EnclaveRPCTimeout) + timeoutCtx, cancel := context.WithTimeout(context.Background(), c.enclaveRPCTimeout) defer cancel() response, err := c.protoClient.EnclaveID(timeoutCtx, &generated.EnclaveIDRequest{}) @@ -160,7 +161,7 @@ func (c *Client) EnclaveID() (common.EnclaveID, common.SystemError) { } func (c *Client) SubmitL1Block(block types.Block, receipts types.Receipts, isLatest bool) (*common.BlockSubmissionResponse, common.SystemError) { - timeoutCtx, cancel := context.WithTimeout(context.Background(), c.config.EnclaveRPCTimeout) + timeoutCtx, cancel := context.WithTimeout(context.Background(), c.enclaveRPCTimeout) defer cancel() var buffer bytes.Buffer @@ -186,7 +187,7 @@ func (c *Client) SubmitL1Block(block types.Block, receipts types.Receipts, isLat } func (c *Client) SubmitTx(tx common.EncryptedTx) (*responses.RawTx, common.SystemError) { - timeoutCtx, cancel := context.WithTimeout(context.Background(), c.config.EnclaveRPCTimeout) + timeoutCtx, cancel := context.WithTimeout(context.Background(), c.enclaveRPCTimeout) defer cancel() response, err := c.protoClient.SubmitTx(timeoutCtx, &generated.SubmitTxRequest{EncryptedTx: tx}) @@ -203,7 +204,7 @@ func (c *Client) SubmitTx(tx common.EncryptedTx) (*responses.RawTx, common.Syste func (c *Client) SubmitBatch(batch *common.ExtBatch) common.SystemError { defer core.LogMethodDuration(c.logger, measure.NewStopwatch(), "SubmitBatch rpc call") - timeoutCtx, cancel := context.WithTimeout(context.Background(), c.config.EnclaveRPCTimeout) + timeoutCtx, cancel := context.WithTimeout(context.Background(), c.enclaveRPCTimeout) defer cancel() batchMsg := rpc.ToExtBatchMsg(batch) @@ -219,7 +220,7 @@ func (c *Client) SubmitBatch(batch *common.ExtBatch) common.SystemError { } func (c *Client) ObsCall(encryptedParams common.EncryptedParamsCall) (*responses.Call, common.SystemError) { - timeoutCtx, cancel := context.WithTimeout(context.Background(), c.config.EnclaveRPCTimeout) + timeoutCtx, cancel := context.WithTimeout(context.Background(), c.enclaveRPCTimeout) defer cancel() response, err := c.protoClient.ObsCall(timeoutCtx, &generated.ObsCallRequest{ @@ -236,7 +237,7 @@ func (c *Client) ObsCall(encryptedParams common.EncryptedParamsCall) (*responses } func (c *Client) GetTransactionCount(encryptedParams common.EncryptedParamsGetTxCount) (*responses.TxCount, common.SystemError) { - timeoutCtx, cancel := context.WithTimeout(context.Background(), c.config.EnclaveRPCTimeout) + timeoutCtx, cancel := context.WithTimeout(context.Background(), c.enclaveRPCTimeout) defer cancel() response, err := c.protoClient.GetTransactionCount(timeoutCtx, &generated.GetTransactionCountRequest{EncryptedParams: encryptedParams}) @@ -253,7 +254,7 @@ func (c *Client) GetTransactionCount(encryptedParams common.EncryptedParamsGetTx func (c *Client) Stop() common.SystemError { c.logger.Info("Shutting down enclave client.") - timeoutCtx, cancel := context.WithTimeout(context.Background(), c.config.EnclaveRPCTimeout) + timeoutCtx, cancel := context.WithTimeout(context.Background(), c.enclaveRPCTimeout) defer cancel() response, err := c.protoClient.Stop(timeoutCtx, &generated.StopRequest{}) @@ -267,7 +268,7 @@ func (c *Client) Stop() common.SystemError { } func (c *Client) GetTransaction(encryptedParams common.EncryptedParamsGetTxByHash) (*responses.TxByHash, common.SystemError) { - timeoutCtx, cancel := context.WithTimeout(context.Background(), c.config.EnclaveRPCTimeout) + timeoutCtx, cancel := context.WithTimeout(context.Background(), c.enclaveRPCTimeout) defer cancel() response, err := c.protoClient.GetTransaction(timeoutCtx, &generated.GetTransactionRequest{EncryptedParams: encryptedParams}) @@ -282,7 +283,7 @@ func (c *Client) GetTransaction(encryptedParams common.EncryptedParamsGetTxByHas } func (c *Client) GetTransactionReceipt(encryptedParams common.EncryptedParamsGetTxReceipt) (*responses.TxReceipt, common.SystemError) { - timeoutCtx, cancel := context.WithTimeout(context.Background(), c.config.EnclaveRPCTimeout) + timeoutCtx, cancel := context.WithTimeout(context.Background(), c.enclaveRPCTimeout) defer cancel() response, err := c.protoClient.GetTransactionReceipt(timeoutCtx, &generated.GetTransactionReceiptRequest{EncryptedParams: encryptedParams}) @@ -297,7 +298,7 @@ func (c *Client) GetTransactionReceipt(encryptedParams common.EncryptedParamsGet } func (c *Client) GetBalance(encryptedParams common.EncryptedParamsGetBalance) (*responses.Balance, common.SystemError) { - timeoutCtx, cancel := context.WithTimeout(context.Background(), c.config.EnclaveRPCTimeout) + timeoutCtx, cancel := context.WithTimeout(context.Background(), c.enclaveRPCTimeout) defer cancel() response, err := c.protoClient.GetBalance(timeoutCtx, &generated.GetBalanceRequest{ @@ -314,7 +315,7 @@ func (c *Client) GetBalance(encryptedParams common.EncryptedParamsGetBalance) (* } func (c *Client) GetCode(address gethcommon.Address, batchHash *gethcommon.Hash) ([]byte, common.SystemError) { - timeoutCtx, cancel := context.WithTimeout(context.Background(), c.config.EnclaveRPCTimeout) + timeoutCtx, cancel := context.WithTimeout(context.Background(), c.enclaveRPCTimeout) defer cancel() response, err := c.protoClient.GetCode(timeoutCtx, &generated.GetCodeRequest{ @@ -332,7 +333,7 @@ func (c *Client) GetCode(address gethcommon.Address, batchHash *gethcommon.Hash) } func (c *Client) Subscribe(id gethrpc.ID, encryptedParams common.EncryptedParamsLogSubscription) common.SystemError { - timeoutCtx, cancel := context.WithTimeout(context.Background(), c.config.EnclaveRPCTimeout) + timeoutCtx, cancel := context.WithTimeout(context.Background(), c.enclaveRPCTimeout) defer cancel() response, err := c.protoClient.Subscribe(timeoutCtx, &generated.SubscribeRequest{ @@ -349,7 +350,7 @@ func (c *Client) Subscribe(id gethrpc.ID, encryptedParams common.EncryptedParams } func (c *Client) Unsubscribe(id gethrpc.ID) common.SystemError { - timeoutCtx, cancel := context.WithTimeout(context.Background(), c.config.EnclaveRPCTimeout) + timeoutCtx, cancel := context.WithTimeout(context.Background(), c.enclaveRPCTimeout) defer cancel() response, err := c.protoClient.Unsubscribe(timeoutCtx, &generated.UnsubscribeRequest{ @@ -365,7 +366,7 @@ func (c *Client) Unsubscribe(id gethrpc.ID) common.SystemError { } func (c *Client) EstimateGas(encryptedParams common.EncryptedParamsEstimateGas) (*responses.Gas, common.SystemError) { - timeoutCtx, cancel := context.WithTimeout(context.Background(), c.config.EnclaveRPCTimeout) + timeoutCtx, cancel := context.WithTimeout(context.Background(), c.enclaveRPCTimeout) defer cancel() response, err := c.protoClient.EstimateGas(timeoutCtx, &generated.EstimateGasRequest{ @@ -382,7 +383,7 @@ func (c *Client) EstimateGas(encryptedParams common.EncryptedParamsEstimateGas) } func (c *Client) GetLogs(encryptedParams common.EncryptedParamsGetLogs) (*responses.Logs, common.SystemError) { - timeoutCtx, cancel := context.WithTimeout(context.Background(), c.config.EnclaveRPCTimeout) + timeoutCtx, cancel := context.WithTimeout(context.Background(), c.enclaveRPCTimeout) defer cancel() response, err := c.protoClient.GetLogs(timeoutCtx, &generated.GetLogsRequest{ @@ -399,7 +400,7 @@ func (c *Client) GetLogs(encryptedParams common.EncryptedParamsGetLogs) (*respon } func (c *Client) HealthCheck() (bool, common.SystemError) { - timeoutCtx, cancel := context.WithTimeout(context.Background(), c.config.EnclaveRPCTimeout) + timeoutCtx, cancel := context.WithTimeout(context.Background(), c.enclaveRPCTimeout) defer cancel() response, err := c.protoClient.HealthCheck(timeoutCtx, &generated.EmptyArgs{}) @@ -415,7 +416,7 @@ func (c *Client) HealthCheck() (bool, common.SystemError) { func (c *Client) CreateBatch(skipIfEmpty bool) common.SystemError { defer core.LogMethodDuration(c.logger, measure.NewStopwatch(), "CreateBatch rpc call") - timeoutCtx, cancel := context.WithTimeout(context.Background(), c.config.EnclaveRPCTimeout) + timeoutCtx, cancel := context.WithTimeout(context.Background(), c.enclaveRPCTimeout) defer cancel() response, err := c.protoClient.CreateBatch(timeoutCtx, &generated.CreateBatchRequest{SkipIfEmpty: skipIfEmpty}) @@ -431,7 +432,7 @@ func (c *Client) CreateBatch(skipIfEmpty bool) common.SystemError { func (c *Client) CreateRollup(fromSeqNo uint64) (*common.ExtRollup, common.SystemError) { defer core.LogMethodDuration(c.logger, measure.NewStopwatch(), "CreateRollup rpc call") - timeoutCtx, cancel := context.WithTimeout(context.Background(), c.config.EnclaveRPCTimeout) + timeoutCtx, cancel := context.WithTimeout(context.Background(), c.enclaveRPCTimeout) defer cancel() response, err := c.protoClient.CreateRollup(timeoutCtx, &generated.CreateRollupRequest{ @@ -448,7 +449,7 @@ func (c *Client) CreateRollup(fromSeqNo uint64) (*common.ExtRollup, common.Syste } func (c *Client) DebugTraceTransaction(hash gethcommon.Hash, config *tracers.TraceConfig) (json.RawMessage, common.SystemError) { - timeoutCtx, cancel := context.WithTimeout(context.Background(), c.config.EnclaveRPCTimeout) + timeoutCtx, cancel := context.WithTimeout(context.Background(), c.enclaveRPCTimeout) defer cancel() confBytes, err := json.Marshal(config) @@ -471,7 +472,7 @@ func (c *Client) DebugTraceTransaction(hash gethcommon.Hash, config *tracers.Tra } func (c *Client) GetBatch(hash common.L2BatchHash) (*common.ExtBatch, common.SystemError) { - timeoutCtx, cancel := context.WithTimeout(context.Background(), c.config.EnclaveRPCTimeout) + timeoutCtx, cancel := context.WithTimeout(context.Background(), c.enclaveRPCTimeout) defer cancel() batchMsg, err := c.protoClient.GetBatch(timeoutCtx, &generated.GetBatchRequest{KnownHead: hash.Bytes()}) @@ -483,7 +484,7 @@ func (c *Client) GetBatch(hash common.L2BatchHash) (*common.ExtBatch, common.Sys } func (c *Client) GetBatchBySeqNo(seqNo uint64) (*common.ExtBatch, common.SystemError) { - timeoutCtx, cancel := context.WithTimeout(context.Background(), c.config.EnclaveRPCTimeout) + timeoutCtx, cancel := context.WithTimeout(context.Background(), c.enclaveRPCTimeout) defer cancel() batchMsg, err := c.protoClient.GetBatchBySeqNo(timeoutCtx, &generated.GetBatchBySeqNoRequest{SeqNo: seqNo}) @@ -495,7 +496,7 @@ func (c *Client) GetBatchBySeqNo(seqNo uint64) (*common.ExtBatch, common.SystemE } func (c *Client) GetRollupData(hash common.L2RollupHash) (*common.PublicRollupMetadata, common.SystemError) { - timeoutCtx, cancel := context.WithTimeout(context.Background(), c.config.EnclaveRPCTimeout) + timeoutCtx, cancel := context.WithTimeout(context.Background(), c.enclaveRPCTimeout) defer cancel() response, err := c.protoClient.GetRollupData(timeoutCtx, &generated.GetRollupDataRequest{Hash: hash.Bytes()}) @@ -552,7 +553,7 @@ func (c *Client) StreamL2Updates() (chan common.StreamL2UpdatesResponse, func()) } func (c *Client) DebugEventLogRelevancy(hash gethcommon.Hash) (json.RawMessage, common.SystemError) { - timeoutCtx, cancel := context.WithTimeout(context.Background(), c.config.EnclaveRPCTimeout) + timeoutCtx, cancel := context.WithTimeout(context.Background(), c.enclaveRPCTimeout) defer cancel() response, err := c.protoClient.DebugEventLogRelevancy(timeoutCtx, &generated.DebugEventLogRelevancyRequest{ @@ -568,7 +569,7 @@ func (c *Client) DebugEventLogRelevancy(hash gethcommon.Hash) (json.RawMessage, } func (c *Client) GetTotalContractCount() (*big.Int, common.SystemError) { - timeoutCtx, cancel := context.WithTimeout(context.Background(), c.config.EnclaveRPCTimeout) + timeoutCtx, cancel := context.WithTimeout(context.Background(), c.enclaveRPCTimeout) defer cancel() response, err := c.protoClient.GetTotalContractCount(timeoutCtx, &generated.GetTotalContractCountRequest{}) @@ -582,7 +583,7 @@ func (c *Client) GetTotalContractCount() (*big.Int, common.SystemError) { } func (c *Client) GetCustomQuery(encryptedParams common.EncryptedParamsGetStorageAt) (*responses.Receipts, common.SystemError) { - timeoutCtx, cancel := context.WithTimeout(context.Background(), c.config.EnclaveRPCTimeout) + timeoutCtx, cancel := context.WithTimeout(context.Background(), c.enclaveRPCTimeout) defer cancel() response, err := c.protoClient.GetReceiptsByAddress(timeoutCtx, &generated.GetReceiptsByAddressRequest{ @@ -599,7 +600,7 @@ func (c *Client) GetCustomQuery(encryptedParams common.EncryptedParamsGetStorage } func (c *Client) GetPublicTransactionData(pagination *common.QueryPagination) (*common.TransactionListingResponse, common.SystemError) { - timeoutCtx, cancel := context.WithTimeout(context.Background(), c.config.EnclaveRPCTimeout) + timeoutCtx, cancel := context.WithTimeout(context.Background(), c.enclaveRPCTimeout) defer cancel() response, err := c.protoClient.GetPublicTransactionData(timeoutCtx, &generated.GetPublicTransactionDataRequest{ @@ -625,7 +626,7 @@ func (c *Client) GetPublicTransactionData(pagination *common.QueryPagination) (* } func (c *Client) EnclavePublicConfig() (*common.EnclavePublicConfig, common.SystemError) { - timeoutCtx, cancel := context.WithTimeout(context.Background(), c.config.EnclaveRPCTimeout) + timeoutCtx, cancel := context.WithTimeout(context.Background(), c.enclaveRPCTimeout) defer cancel() response, err := c.protoClient.EnclavePublicConfig(timeoutCtx, &generated.EnclavePublicConfigRequest{}) diff --git a/go/node/config.go b/go/node/config.go index 13a27978cb..3c406f402a 100644 --- a/go/node/config.go +++ b/go/node/config.go @@ -107,7 +107,7 @@ func (c *Config) ToHostConfig() *config.HostInputConfig { cfg.IsGenesis = c.isGenesis cfg.PrivateKeyString = c.privateKey - cfg.EnclaveRPCAddress = fmt.Sprintf("127.0.0.1:%d", c.enclaveWSPort) + cfg.EnclaveRPCAddresses = []string{fmt.Sprintf("127.0.0.1:%d", c.enclaveWSPort)} cfg.ClientRPCPortWS = uint64(c.hostWSPort) cfg.ClientRPCPortHTTP = uint64(c.hostHTTPPort) diff --git a/integration/networktest/actions/node_actions.go b/integration/networktest/actions/node_actions.go index 2ff7ed0be9..80590d23a5 100644 --- a/integration/networktest/actions/node_actions.go +++ b/integration/networktest/actions/node_actions.go @@ -20,7 +20,8 @@ type startValidatorEnclaveAction struct { func (s *startValidatorEnclaveAction) Run(ctx context.Context, network networktest.NetworkConnector) (context.Context, error) { fmt.Printf("Validator %d: starting enclave\n", s.validatorIdx) validator := network.GetValidatorNode(s.validatorIdx) - err := validator.StartEnclave() + // note: these actions are assuming single-enclave setups + err := validator.StartEnclave(0) if err != nil { return nil, err } @@ -42,7 +43,7 @@ type stopValidatorEnclaveAction struct { func (s *stopValidatorEnclaveAction) Run(ctx context.Context, network networktest.NetworkConnector) (context.Context, error) { fmt.Printf("Validator %d: stopping enclave\n", s.validatorIdx) validator := network.GetValidatorNode(s.validatorIdx) - err := validator.StopEnclave() + err := validator.StopEnclave(0) if err != nil { return nil, err } @@ -105,7 +106,7 @@ func StopSequencerEnclave() networktest.Action { return RunOnlyAction(func(ctx context.Context, network networktest.NetworkConnector) (context.Context, error) { fmt.Println("Sequencer: stopping enclave") sequencer := network.GetSequencerNode() - err := sequencer.StopEnclave() + err := sequencer.StopEnclave(0) if err != nil { return nil, err } @@ -117,7 +118,7 @@ func StartSequencerEnclave() networktest.Action { return RunOnlyAction(func(ctx context.Context, network networktest.NetworkConnector) (context.Context, error) { fmt.Println("Sequencer: starting enclave") sequencer := network.GetSequencerNode() - err := sequencer.StartEnclave() + err := sequencer.StartEnclave(0) if err != nil { return nil, err } diff --git a/integration/networktest/interfaces.go b/integration/networktest/interfaces.go index 1a74845d30..c751328ae8 100644 --- a/integration/networktest/interfaces.go +++ b/integration/networktest/interfaces.go @@ -58,8 +58,8 @@ type NodeOperator interface { Start() error Stop() error - StartEnclave() error - StopEnclave() error + StartEnclave(idx int) error + StopEnclave(idx int) error StartHost() error StopHost() error diff --git a/integration/networktest/tests/nodescenario/sequencer_multi_enclave_test.go b/integration/networktest/tests/nodescenario/sequencer_multi_enclave_test.go index ce3f6b11fe..af393420b9 100644 --- a/integration/networktest/tests/nodescenario/sequencer_multi_enclave_test.go +++ b/integration/networktest/tests/nodescenario/sequencer_multi_enclave_test.go @@ -1 +1,35 @@ package nodescenario + +import ( + "math/big" + "testing" + + "github.com/ten-protocol/go-ten/integration/networktest" + "github.com/ten-protocol/go-ten/integration/networktest/actions" + "github.com/ten-protocol/go-ten/integration/networktest/env" + "github.com/ten-protocol/go-ten/integration/simulation/devnetwork" +) + +var _transferAmount = big.NewInt(100_000_000) + +func TestMultiEnclaveSequencer(t *testing.T) { + networktest.TestOnlyRunsInIDE(t) + networktest.Run( + "multi-enclave-sequencer", + t, + env.LocalDevNetwork(devnetwork.WithHASequencer()), + actions.Series( + &actions.CreateTestUser{UserID: 0}, + &actions.CreateTestUser{UserID: 1}, + actions.SetContextValue(actions.KeyNumberOfTestUsers, 2), + + &actions.AllocateFaucetFunds{UserID: 0}, + actions.SnapshotUserBalances(actions.SnapAfterAllocation), // record user balances (we have no guarantee on how much the network faucet allocates) + + &actions.SendNativeFunds{FromUser: 0, ToUser: 1, Amount: _transferAmount}, + + &actions.VerifyBalanceAfterTest{UserID: 1, ExpectedBalance: _transferAmount}, + &actions.VerifyBalanceDiffAfterTest{UserID: 0, Snapshot: actions.SnapAfterAllocation, ExpectedDiff: big.NewInt(0).Neg(_transferAmount)}, + ), + ) +} diff --git a/integration/simulation/devnetwork/node.go b/integration/simulation/devnetwork/node.go index 1095043e78..0c92b9a13b 100644 --- a/integration/simulation/devnetwork/node.go +++ b/integration/simulation/devnetwork/node.go @@ -35,6 +35,8 @@ import ( integrationCommon "github.com/ten-protocol/go-ten/integration/common" ) +const _multiEnclaveOffset = 10 + // InMemNodeOperator represents an Obscuro node playing a role in a DevSimulation // // Anything a node operator could do, that we need to simulate belongs here, for example: @@ -51,11 +53,11 @@ type InMemNodeOperator struct { l1Client ethadapter.EthClient logger gethlog.Logger - host *hostcontainer.HostContainer - enclave *enclavecontainer.EnclaveContainer - l1Wallet wallet.Wallet - enclaveDBFilepath string - hostDBFilepath string + host *hostcontainer.HostContainer + enclaves []*enclavecontainer.EnclaveContainer + l1Wallet wallet.Wallet + enclaveDBFilepaths []string // 1 per enclave + hostDBFilepath string } func (n *InMemNodeOperator) StopHost() error { @@ -67,9 +69,17 @@ func (n *InMemNodeOperator) StopHost() error { } func (n *InMemNodeOperator) Start() error { - err := n.StartEnclave() - if err != nil { - return fmt.Errorf("failed to start enclave - %w", err) + var err error + numEnclaves := n.config.NumSeqEnclaves + if n.nodeType != common.Sequencer { + numEnclaves = 1 + } + n.enclaves = make([]*enclavecontainer.EnclaveContainer, numEnclaves) + for i := 0; i < numEnclaves; i++ { + err = n.StartEnclave(i) + if err != nil { + return fmt.Errorf("failed to start enclave[%d] - %w", i, err) + } } err = n.StartHost() if err != nil { @@ -94,16 +104,23 @@ func (n *InMemNodeOperator) StartHost() error { } // StartEnclave starts the enclave process in -func (n *InMemNodeOperator) StartEnclave() error { +func (n *InMemNodeOperator) StartEnclave(idx int) error { // even if enclave was running previously we recreate the container to ensure state is like a new process // todo (@matt) - check if enclave is still running? - n.enclave = n.createEnclaveContainer() - return n.enclave.Start() + n.enclaves[idx] = n.createEnclaveContainer(idx) + return n.enclaves[idx].Start() } func (n *InMemNodeOperator) createHostContainer() *hostcontainer.HostContainer { enclavePort := n.config.PortStart + integration.DefaultEnclaveOffset + n.operatorIdx - enclaveAddr := fmt.Sprintf("%s:%d", network.Localhost, enclavePort) + var enclaveAddresses []string + if n.nodeType == common.Sequencer { + for i := 0; i < n.config.NumSeqEnclaves; i++ { + enclaveAddresses = append(enclaveAddresses, fmt.Sprintf("%s:%d", network.Localhost, enclavePort+(i*_multiEnclaveOffset))) + } + } else { + enclaveAddresses = append(enclaveAddresses, fmt.Sprintf("%s:%d", network.Localhost, enclavePort)) + } p2pPort := n.config.PortStart + integration.DefaultHostP2pOffset + n.operatorIdx p2pAddr := fmt.Sprintf("%s:%d", network.Localhost, p2pPort) @@ -117,7 +134,7 @@ func (n *InMemNodeOperator) createHostContainer() *hostcontainer.HostContainer { HasClientRPCWebsockets: true, ClientRPCPortWS: uint64(n.config.PortStart + integration.DefaultHostRPCWSOffset + n.operatorIdx), ClientRPCHost: network.Localhost, - EnclaveRPCAddress: enclaveAddr, + EnclaveRPCAddresses: enclaveAddresses, P2PBindAddress: p2pAddr, P2PPublicAddress: p2pAddr, EnclaveRPCTimeout: network.EnclaveClientRPCTimeout, @@ -142,9 +159,12 @@ func (n *InMemNodeOperator) createHostContainer() *hostcontainer.HostContainer { p2pLogger := hostLogger.New(log.CmpKey, log.P2PCmp) svcLocator := host.NewServicesRegistry(n.logger) nodeP2p := p2p.NewSocketP2PLayer(hostConfig, svcLocator, p2pLogger, nil) - // create an enclave client - enclaveClient := enclaverpc.NewClient(hostConfig, testlog.Logger().New(log.NodeIDKey, n.l1Wallet.Address())) + var enclaveClients []common.Enclave + for i, enclaveAddr := range hostConfig.EnclaveRPCAddresses { + fmt.Println("Connecting to the enclave...", i, enclaveAddr) + enclaveClients = append(enclaveClients, enclaverpc.NewClient(enclaveAddr, hostConfig.EnclaveRPCTimeout, hostLogger.New(log.NodeIDKey, n.l1Wallet.Address(), "enclIdx", i))) + } rpcConfig := node.RPCConfig{ Host: hostConfig.ClientRPCHost, EnableHTTP: hostConfig.HasClientRPCHTTP, @@ -156,25 +176,29 @@ func (n *InMemNodeOperator) createHostContainer() *hostcontainer.HostContainer { rpcServer := node.NewServer(&rpcConfig, n.logger) mgmtContractLib := mgmtcontractlib.NewMgmtContractLib(&hostConfig.ManagementContractAddress, n.logger) l1Repo := l1.NewL1Repository(n.l1Client, []gethcommon.Address{hostConfig.ManagementContractAddress, hostConfig.MessageBusAddress}, n.logger) - return hostcontainer.NewHostContainer(hostConfig, svcLocator, nodeP2p, n.l1Client, l1Repo, enclaveClient, mgmtContractLib, n.l1Wallet, rpcServer, hostLogger, metrics.New(false, 0, n.logger)) + return hostcontainer.NewHostContainer(hostConfig, svcLocator, nodeP2p, n.l1Client, l1Repo, enclaveClients, mgmtContractLib, n.l1Wallet, rpcServer, hostLogger, metrics.New(false, 0, n.logger)) } -func (n *InMemNodeOperator) createEnclaveContainer() *enclavecontainer.EnclaveContainer { +func (n *InMemNodeOperator) createEnclaveContainer(idx int) *enclavecontainer.EnclaveContainer { enclaveLogger := testlog.Logger().New(log.NodeIDKey, n.l1Wallet.Address(), log.CmpKey, log.EnclaveCmp) - enclavePort := n.config.PortStart + integration.DefaultEnclaveOffset + n.operatorIdx + enclavePort := n.config.PortStart + integration.DefaultEnclaveOffset + n.operatorIdx + (idx * _multiEnclaveOffset) enclaveAddr := fmt.Sprintf("%s:%d", network.Localhost, enclavePort) hostPort := n.config.PortStart + integration.DefaultHostP2pOffset + n.operatorIdx hostAddr := fmt.Sprintf("%s:%d", network.Localhost, hostPort) defaultCfg := integrationCommon.DefaultEnclaveConfig() - + enclaveType := n.nodeType + if n.nodeType == common.Sequencer && idx > 0 { + // we only want one sequencer enclave for now + enclaveType = common.Validator + } enclaveConfig := &config.EnclaveConfig{ HostID: n.l1Wallet.Address(), SequencerID: n.config.SequencerID, HostAddress: hostAddr, Address: enclaveAddr, - NodeType: n.nodeType, + NodeType: enclaveType, L1ChainID: integration.EthereumChainID, ObscuroChainID: integration.TenChainID, ValidateL1Blocks: false, @@ -184,7 +208,7 @@ func (n *InMemNodeOperator) createEnclaveContainer() *enclavecontainer.EnclaveCo ManagementContractAddress: n.l1Data.MgmtContractAddress, MinGasPrice: gethcommon.Big1, MessageBusAddress: n.l1Data.MessageBusAddr, - SqliteDBPath: n.enclaveDBFilepath, + SqliteDBPath: n.enclaveDBFilepaths[idx], DebugNamespaceEnabled: true, MaxBatchSize: 1024 * 55, MaxRollupSize: 1024 * 64, @@ -197,13 +221,19 @@ func (n *InMemNodeOperator) createEnclaveContainer() *enclavecontainer.EnclaveCo } func (n *InMemNodeOperator) Stop() error { + errs := make([]error, 0) // collect errors to return after attempting all stops err := n.host.Stop() if err != nil { - return fmt.Errorf("failed to stop host - %w", err) + errs = append(errs, fmt.Errorf("failed to stop host - %w", err)) } - err = n.enclave.Stop() - if err != nil { - return fmt.Errorf("failed to stop enclave - %w", err) + for i := 0; i < len(n.enclaves); i++ { + err = n.enclaves[i].Stop() + if err != nil { + errs = append(errs, fmt.Errorf("failed to stop enclave[%d] - %w", i, err)) + } + } + if len(errs) > 0 { + return fmt.Errorf("node operator failed to stop - %v", errs) } return nil } @@ -218,13 +248,13 @@ func (n *InMemNodeOperator) HostRPCHTTPAddress() string { return fmt.Sprintf("http://%s:%d", network.Localhost, hostPort) } -func (n *InMemNodeOperator) StopEnclave() error { - err := n.enclave.Stop() +func (n *InMemNodeOperator) StopEnclave(idx int) error { + err := n.enclaves[idx].Stop() if err != nil { n.logger.Error("failed to stop enclave - %w", err) // try again - err := n.enclave.Stop() + err := n.enclaves[idx].Stop() if err != nil { return fmt.Errorf("failed to stop enclave after second attempt - %w", err) } @@ -236,9 +266,17 @@ func NewInMemNodeOperator(operatorIdx int, config *TenConfig, nodeType common.No l1Client ethadapter.EthClient, l1Wallet wallet.Wallet, logger gethlog.Logger, ) *InMemNodeOperator { // todo (@matt) - put sqlite and levelDB storage in the same temp dir - sqliteDBPath, err := sqlite.CreateTempDBFile() - if err != nil { - panic("failed to create temp sqlite db path") + numEnclaves := config.NumSeqEnclaves + if nodeType != common.Sequencer { + numEnclaves = 1 + } + sqliteDBPaths := make([]string, numEnclaves) + for i := 0; i < numEnclaves; i++ { + sqliteDBPath, err := sqlite.CreateTempDBFile() + if err != nil { + panic("failed to create temp sqlite db path") + } + sqliteDBPaths[i] = sqliteDBPath } levelDBPath, err := os.MkdirTemp("", "levelDB_*") if err != nil { @@ -252,14 +290,14 @@ func NewInMemNodeOperator(operatorIdx int, config *TenConfig, nodeType common.No l1Wallet.SetNonce(l1Nonce) return &InMemNodeOperator{ - operatorIdx: operatorIdx, - config: config, - nodeType: nodeType, - l1Data: l1Data, - l1Client: l1Client, - l1Wallet: l1Wallet, - logger: logger, - enclaveDBFilepath: sqliteDBPath, - hostDBFilepath: levelDBPath, + operatorIdx: operatorIdx, + config: config, + nodeType: nodeType, + l1Data: l1Data, + l1Client: l1Client, + l1Wallet: l1Wallet, + logger: logger, + enclaveDBFilepaths: sqliteDBPaths, + hostDBFilepath: levelDBPath, } } diff --git a/integration/simulation/network/network_utils.go b/integration/simulation/network/network_utils.go index 4bed4c2cc5..12962c8be6 100644 --- a/integration/simulation/network/network_utils.go +++ b/integration/simulation/network/network_utils.go @@ -99,13 +99,13 @@ func createInMemObscuroNode( } enclaveLogger := testlog.Logger().New(log.NodeIDKey, id, log.CmpKey, log.EnclaveCmp) - enclaveClient := enclave.NewEnclave(enclaveConfig, &genesis.TestnetGenesis, mgmtContractLib, enclaveLogger) + enclaveClients := []common.Enclave{enclave.NewEnclave(enclaveConfig, &genesis.TestnetGenesis, mgmtContractLib, enclaveLogger)} // create an in memory obscuro node hostLogger := testlog.Logger().New(log.NodeIDKey, id, log.CmpKey, log.HostCmp) metricsService := metrics.New(hostConfig.MetricsEnabled, hostConfig.MetricsHTTPPort, hostLogger) l1Repo := l1.NewL1Repository(ethClient, ethereummock.MgmtContractAddresses, hostLogger) - currentContainer := container.NewHostContainer(hostConfig, host.NewServicesRegistry(hostLogger), mockP2P, ethClient, l1Repo, enclaveClient, mgmtContractLib, ethWallet, nil, hostLogger, metricsService) + currentContainer := container.NewHostContainer(hostConfig, host.NewServicesRegistry(hostLogger), mockP2P, ethClient, l1Repo, enclaveClients, mgmtContractLib, ethWallet, nil, hostLogger, metricsService) return currentContainer } From f5079feaea02ea7811b654b0aafe82d6b3a7267f Mon Sep 17 00:00:00 2001 From: Matt Curtis Date: Tue, 26 Mar 2024 18:49:43 +0000 Subject: [PATCH 2/4] HA: support multiple enclaves on a single host --- go/host/host.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/go/host/host.go b/go/host/host.go index 35c5babf73..6df417d983 100644 --- a/go/host/host.go +++ b/go/host/host.go @@ -74,11 +74,14 @@ func NewHost(config *config.HostConfig, hostServices *ServicesRegistry, p2p host enclGuardians := make([]*enclave.Guardian, 0, len(enclaveClients)) for i, enclClient := range enclaveClients { + // clone the hostIdentity data for each enclave + enclHostID := hostIdentity if i > 0 { // only the first enclave can be the sequencer for now, others behave as read-only validators - hostIdentity.IsSequencer = false + enclHostID.IsSequencer = false + enclHostID.IsGenesis = false } - enclGuardian := enclave.NewGuardian(config, hostIdentity, hostServices, enclClient, database, host.stopControl, logger) + enclGuardian := enclave.NewGuardian(config, enclHostID, hostServices, enclClient, database, host.stopControl, logger) enclGuardians = append(enclGuardians, enclGuardian) } From 3ef4267365fd3a33298584e5a60d13164a78f90f Mon Sep 17 00:00:00 2001 From: Matt Curtis Date: Tue, 26 Mar 2024 21:05:49 +0000 Subject: [PATCH 3/4] fix toml --- go/host/container/test.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/host/container/test.toml b/go/host/container/test.toml index 3d63e17a5c..c88b6ba5b1 100644 --- a/go/host/container/test.toml +++ b/go/host/container/test.toml @@ -5,7 +5,7 @@ clientRPCPortHTTP = 80 HasClientRPCWebsockets = true ClientRPCPortWS = 81 ClientRPCHost = "127.0.0.1" -EnclaveRPCAddress = "127.0.0.1:11000" +EnclaveRPCAddresses = "127.0.0.1:11000" EnclaveRPCTimeout = 10 P2PBindAddress = "0.0.0.0:10000" P2PPublicAddress = "127.0.0.1:10000" From bf553d49bca95b2ad72983d21f8862d1cd3d0d8a Mon Sep 17 00:00:00 2001 From: Matt Curtis Date: Wed, 27 Mar 2024 12:04:38 +0000 Subject: [PATCH 4/4] support multi enclaves for host_container --- go/host/container/host_container.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/go/host/container/host_container.go b/go/host/container/host_container.go index 24340115b9..c4b33081ba 100644 --- a/go/host/container/host_container.go +++ b/go/host/container/host_container.go @@ -130,13 +130,12 @@ func NewHostContainerFromConfig(parsedConfig *config.HostInputConfig, logger get // set the Host ID as the Public Key Address cfg.ID = ethWallet.Address() - if len(cfg.EnclaveRPCAddresses) != 1 { - logger.Crit("expected exactly one enclave address", "enclave_addresses", cfg.EnclaveRPCAddresses) - } - fmt.Println("Connecting to the enclave...") services := host.NewServicesRegistry(logger) - enclaveClients := []common.Enclave{enclaverpc.NewClient(cfg.EnclaveRPCAddresses[0], cfg.EnclaveRPCTimeout, logger)} + enclaveClients := make([]common.Enclave, len(cfg.EnclaveRPCAddresses)) + for i, addr := range cfg.EnclaveRPCAddresses { + enclaveClients[i] = enclaverpc.NewClient(addr, cfg.EnclaveRPCTimeout, logger) + } p2pLogger := logger.New(log.CmpKey, log.P2PCmp) metricsService := metrics.New(cfg.MetricsEnabled, cfg.MetricsHTTPPort, logger)