Testnets: configure two enclaves for HA sequencer node (#2201)

.github/workflows/manual-deploy-testnet-l2.yml

@@ -174,6 +174,13 @@ jobs:
             host_id: 1
           - node_l1_ws_lookup: L1_WS_URL_2
             host_id: 2
+          # sequencer has an HA setup with 2 enclaves
+          - num_enclaves: 2
+            host_id: 0
+          - num_enclaves: 1
+            host_id: 1
+          - num_enclaves: 1
+            host_id: 2
 
     steps:
       - name: 'Extract branch name'
@@ -336,6 +343,7 @@ jobs:
             && sudo go run /home/obscuro/go-obscuro/go/node/cmd  \
                -is_genesis=${{ matrix.is_genesis }} \
                -node_type=${{ matrix.node_type }} \
+               -num_enclaves=${{ matrix.num_enclaves }} \
                -is_sgx_enabled=true \
                -host_id=${{ vars[matrix.node_addr_lookup] }} \
                -l1_ws_url=${{ secrets[matrix.node_l1_ws_lookup] }} \

.github/workflows/manual-upgrade-testnet-l2.yml

@@ -134,6 +134,13 @@ jobs:
             host_id: 1
           - node_l1_ws_lookup: L1_WS_URL_2
             host_id: 2
+          # sequencer has an HA setup with 2 enclaves
+          - num_enclaves: 2
+            host_id: 0
+          - num_enclaves: 1
+            host_id: 1
+          - num_enclaves: 1
+            host_id: 2
 
     steps:
       - name: 'Extract branch name'
@@ -162,6 +169,7 @@ jobs:
             && sudo go run /home/obscuro/go-obscuro/go/node/cmd  \
               -is_genesis=${{ matrix.is_genesis }} \
               -node_type=${{ matrix.node_type }} \
+              -num_enclaves=${{ matrix.num_enclaves }} \
               -is_sgx_enabled=true \
               -host_id=${{ vars[matrix.node_addr_lookup] }} \
               -l1_ws_url=${{ secrets[matrix.node_l1_ws_lookup] }} \

go/node/cmd/cli.go

@@ -23,6 +23,7 @@ type NodeConfigCLI struct {
 	nodeAction              string
 	nodeType                string
 	isGenesis               bool
+	numEnclaves             int
 	isSGXEnabled            bool
 	enclaveDockerImage      string
 	hostDockerImage         string
@@ -64,6 +65,7 @@ func ParseConfigCLI() *NodeConfigCLI {
 	nodeName := flag.String(nodeNameFlag, "obscuronode", flagUsageMap[nodeNameFlag])
 	nodeType := flag.String(nodeTypeFlag, "", flagUsageMap[nodeTypeFlag])
 	isGenesis := flag.Bool(isGenesisFlag, false, flagUsageMap[isGenesisFlag])
+	numEnclaves := flag.Int(numEnclavesFlag, 1, flagUsageMap[numEnclavesFlag])
 	isSGXEnabled := flag.Bool(isSGXEnabledFlag, false, flagUsageMap[isSGXEnabledFlag])
 	enclaveDockerImage := flag.String(enclaveDockerImageFlag, "", flagUsageMap[enclaveDockerImageFlag])
 	hostDockerImage := flag.String(hostDockerImageFlag, "", flagUsageMap[hostDockerImageFlag])
@@ -98,6 +100,7 @@ func ParseConfigCLI() *NodeConfigCLI {
 	cfg.nodeName = *nodeName
 	cfg.nodeType = *nodeType
 	cfg.isGenesis = *isGenesis
+	cfg.numEnclaves = *numEnclaves
 	cfg.isSGXEnabled = *isSGXEnabled
 	cfg.enclaveDockerImage = *enclaveDockerImage
 	cfg.hostDockerImage = *hostDockerImage
@@ -166,6 +169,11 @@ func NodeCLIConfigToTenConfig(cliCfg *NodeConfigCLI) *config.TenConfig {
 		fmt.Printf("Error loading default Ten config: %v\n", err)
 		os.Exit(1)
 	}
+	enclaveAddresses := make([]string, cliCfg.numEnclaves)
+	for i := 0; i < cliCfg.numEnclaves; i++ {
+		enclaveAddresses[i] = fmt.Sprintf("%s-enclave-%d:%d",
+			cliCfg.nodeName, i, cliCfg.enclaveWSPort)
+	}
 
 	tenCfg.Network.L1.ChainID = int64(cliCfg.l1ChainID)
 	tenCfg.Network.L1.L1Contracts.ManagementContract = gethcommon.HexToAddress(cliCfg.managementContractAddr)
@@ -199,7 +207,7 @@ func NodeCLIConfigToTenConfig(cliCfg *NodeConfigCLI) *config.TenConfig {
 	tenCfg.Host.DB.UseInMemory = false // these nodes always use a persistent DB
 	tenCfg.Host.DB.PostgresHost = cliCfg.postgresDBHost
 	tenCfg.Host.Debug.EnableDebugNamespace = cliCfg.isDebugNamespaceEnabled
-	tenCfg.Host.Enclave.RPCAddresses = []string{fmt.Sprintf("%s:%d", cliCfg.nodeName+"-enclave", cliCfg.enclaveWSPort)}
+	tenCfg.Host.Enclave.RPCAddresses = enclaveAddresses
 	tenCfg.Host.L1.WebsocketURL = cliCfg.l1WebsocketURL
 	tenCfg.Host.L1.L1BeaconUrl = cliCfg.l1BeaconUrl
 	tenCfg.Host.L1.L1BlobArchiveUrl = cliCfg.l1BlobArchiveUrl
@@ -209,8 +217,8 @@ func NodeCLIConfigToTenConfig(cliCfg *NodeConfigCLI) *config.TenConfig {
 	tenCfg.Host.RPC.WSPort = uint64(cliCfg.hostWSPort)
 	tenCfg.Host.Log.Level = cliCfg.logLevel
 
-	tenCfg.Enclave.DB.UseInMemory = false // these nodes always use a persistent DB
-	tenCfg.Enclave.DB.EdgelessDBHost = cliCfg.nodeName + "-edgelessdb"
+	tenCfg.Enclave.DB.UseInMemory = false                                     // these nodes always use a persistent DB
+	tenCfg.Enclave.DB.EdgelessDBHost = cliCfg.nodeName + "-edgelessdb-" + "0" // will be dynamically set for HA
 	tenCfg.Enclave.Debug.EnableDebugNamespace = cliCfg.isDebugNamespaceEnabled
 	tenCfg.Enclave.EnableAttestation = cliCfg.isSGXEnabled
 	tenCfg.Enclave.RPC.BindAddress = fmt.Sprintf("0.0.0.0:%d", cliCfg.enclaveWSPort)

go/node/cmd/cli_flags.go

@@ -5,6 +5,7 @@ const (
 	nodeNameFlag                = "node_name"
 	nodeTypeFlag                = "node_type"
 	isGenesisFlag               = "is_genesis"
+	numEnclavesFlag             = "num_enclaves"
 	hostIDFlag                  = "host_id"
 	isSGXEnabledFlag            = "is_sgx_enabled"
 	enclaveDockerImageFlag      = "enclave_docker_image"
@@ -43,7 +44,8 @@ func getFlagUsageMap() map[string]string {
 	return map[string]string{
 		nodeNameFlag:                "Specifies the node base name",
 		nodeTypeFlag:                "The node's type (e.g. sequencer, validator)",
-		isGenesisFlag:               "Wether the node is the genesis node of the network",
+		isGenesisFlag:               "Whether the node is the genesis node of the network",
+		numEnclavesFlag:             "The number of enclaves to run as an HA setup (default 1, no HA pool)",
 		hostIDFlag:                  "The 20 bytes of the address of the Obscuro host this enclave serves",
 		isSGXEnabledFlag:            "Whether the it should run on an SGX is enabled CPU",
 		enclaveDockerImageFlag:      "Docker image for the enclave",

go/node/cmd/main.go

@@ -9,7 +9,7 @@ func main() {
 
 	tenCfg := NodeCLIConfigToTenConfig(cliConfig)
 
-	dockerNode := node.NewDockerNode(tenCfg, cliConfig.hostDockerImage, cliConfig.enclaveDockerImage, cliConfig.edgelessDBImage, false, cliConfig.pccsAddr)
+	dockerNode := node.NewDockerNode(tenCfg, cliConfig.hostDockerImage, cliConfig.enclaveDockerImage, cliConfig.edgelessDBImage, false, cliConfig.pccsAddr, cliConfig.numEnclaves)
 	var err error
 	switch cliConfig.nodeAction {
 	case startAction:

go/node/docker_node.go

@@ -20,31 +20,36 @@ type DockerNode struct {
 	edgelessDBImage  string
 	enclaveDebugMode bool
 	pccsAddr         string // optional specified PCCS address
+	numEnclaves      int    // number of enclaves to start for the node as an HA setup
 }
 
-func NewDockerNode(cfg *config.TenConfig, hostImage, enclaveImage, edgelessDBImage string, enclaveDebug bool, pccsAddr string) *DockerNode {
+func NewDockerNode(cfg *config.TenConfig, hostImage, enclaveImage, edgelessDBImage string, enclaveDebug bool, pccsAddr string, numEnclaves int) *DockerNode {
 	return &DockerNode{
 		cfg:              cfg,
 		hostImage:        hostImage,
 		enclaveImage:     enclaveImage,
 		edgelessDBImage:  edgelessDBImage,
 		enclaveDebugMode: enclaveDebug,
 		pccsAddr:         pccsAddr,
+		numEnclaves:      numEnclaves,
 	}
 }
 
 func (d *DockerNode) Start() error {
 	// todo (@pedro) - this should probably be removed in the future
 	d.cfg.PrettyPrint() // dump config to stdout
 
-	err := d.startEdgelessDB()
-	if err != nil {
-		return fmt.Errorf("failed to start edgelessdb: %w", err)
-	}
+	var err error
+	for i := 0; i < d.numEnclaves; i++ {
+		err = d.startEdgelessDB(i)
+		if err != nil {
+			return fmt.Errorf("failed to start edgelessdb: %w", err)
+		}
 
-	err = d.startEnclave()
-	if err != nil {
-		return fmt.Errorf("failed to start enclave: %w", err)
+		err = d.startEnclave(i)
+		if err != nil {
+			return fmt.Errorf("failed to start enclave: %w", err)
+		}
 	}
 
 	err = d.startHost()
@@ -62,9 +67,11 @@ func (d *DockerNode) Stop() error {
 		return err
 	}
 
-	err = docker.StopAndRemove(d.cfg.Node.Name + "-enclave")
-	if err != nil {
-		return err
+	for i := 0; i < d.numEnclaves; i++ {
+		err = docker.StopAndRemove(d.cfg.Node.Name + "-enclave-" + strconv.Itoa(i))
+		if err != nil {
+			return err
+		}
 	}
 
 	return nil
@@ -84,10 +91,12 @@ func (d *DockerNode) Upgrade(networkCfg *NetworkConfig) error {
 	d.cfg.Network.L1.L1Contracts.MessageBusContract = common.HexToAddress(networkCfg.MessageBusAddress)
 	d.cfg.Network.L1.StartHash = common.HexToHash(networkCfg.L1StartHash)
 
-	fmt.Println("Starting upgraded host and enclave")
-	err = d.startEnclave()
-	if err != nil {
-		return err
+	fmt.Println("Starting upgraded host and enclaves")
+	for i := 0; i < d.numEnclaves; i++ {
+		err = d.startEnclave(i)
+		if err != nil {
+			return err
+		}
 	}
 
 	err = d.startHost()
@@ -124,7 +133,7 @@ func (d *DockerNode) startHost() error {
 	return err
 }
 
-func (d *DockerNode) startEnclave() error {
+func (d *DockerNode) startEnclave(enclaveIdx int) error {
 	devices := map[string]string{}
 	exposedPorts := []int{}
 
@@ -134,6 +143,9 @@ func (d *DockerNode) startEnclave() error {
 	}
 
 	if d.enclaveDebugMode {
+		if d.numEnclaves > 1 {
+			return fmt.Errorf("cannot run multiple enclaves in debug mode")
+		}
 		cmd = []string{
 			"dlv",
 			"--listen=:2345",
@@ -147,6 +159,9 @@ func (d *DockerNode) startEnclave() error {
 		exposedPorts = append(exposedPorts, 2345)
 	}
 
+	// we set the edgeless DB address dynamically for each enclave
+	d.cfg.Enclave.DB.EdgelessDBHost = fmt.Sprintf("%s-edgelessdb-%d", d.cfg.Node.Name, enclaveIdx)
+
 	envVariables := d.cfg.ToEnvironmentVariables()
 
 	if d.cfg.Enclave.EnableAttestation {
@@ -163,16 +178,21 @@ func (d *DockerNode) startEnclave() error {
 		cmd = append(cmd, "-willAttest=false")
 	}
 
+	volumeName := fmt.Sprintf("%s-enclave-volume-%d", d.cfg.Node.Name, enclaveIdx)
+	containerName := fmt.Sprintf("%s-enclave-%d", d.cfg.Node.Name, enclaveIdx)
+
 	// we need the enclave volume to store the db credentials
-	enclaveVolume := map[string]string{d.cfg.Node.Name + "-enclave-volume": _enclaveDataDir}
-	_, err := docker.StartNewContainer(d.cfg.Node.Name+"-enclave", d.enclaveImage, cmd, exposedPorts, envVariables, devices, enclaveVolume, true)
+	enclaveVolume := map[string]string{volumeName: _enclaveDataDir}
+	_, err := docker.StartNewContainer(containerName, d.enclaveImage, cmd, exposedPorts, envVariables, devices, enclaveVolume, true)
 
 	return err
 }
 
-func (d *DockerNode) startEdgelessDB() error {
+func (d *DockerNode) startEdgelessDB(enclaveIdx int) error {
+	containerName := fmt.Sprintf("%s-edgelessdb-%d", d.cfg.Node.Name, enclaveIdx)
+	volumeName := fmt.Sprintf("%s-db-volume-%d", d.cfg.Node.Name, enclaveIdx)
 	envs := map[string]string{
-		"EDG_EDB_CERT_DNS": d.cfg.Node.Name + "-edgelessdb",
+		"EDG_EDB_CERT_DNS": containerName,
 	}
 	devices := map[string]string{}
 
@@ -188,11 +208,8 @@ func (d *DockerNode) startEdgelessDB() error {
 		envs["PCCS_ADDR"] = d.pccsAddr
 	}
 
-	// todo - do we need this volume?
-	//dbVolume := map[string]string{d.cfg.Node.Name + "-db-volume": "/data"}
-	//_, err := docker.StartNewContainer(d.cfg.Node.Name+"-edgelessdb", d.cfg.edgelessDBImage, nil, nil, envs, devices, dbVolume)
-
-	_, err := docker.StartNewContainer(d.cfg.Node.Name+"-edgelessdb", d.edgelessDBImage, nil, nil, envs, devices, nil, true)
+	dbVolume := map[string]string{volumeName: "/data"}
+	_, err := docker.StartNewContainer(containerName, d.edgelessDBImage, nil, nil, envs, devices, dbVolume, true)
 
 	return err
 }

testnet/launcher/docker.go

@@ -62,13 +62,7 @@ func (t *Testnet) Start() error {
 	sequencerCfg.Network.L1.L1Contracts.ManagementContract = common.HexToAddress(networkConfig.ManagementContractAddress)
 	sequencerCfg.Network.L1.L1Contracts.MessageBusContract = common.HexToAddress(networkConfig.MessageBusAddress)
 
-	sequencerNode := node.NewDockerNode(sequencerCfg,
-		"testnetobscuronet.azurecr.io/obscuronet/host:latest",
-		"testnetobscuronet.azurecr.io/obscuronet/enclave:latest",
-		edgelessDBImage,
-		false,
-		"", // no PCCS override
-	)
+	sequencerNode := node.NewDockerNode(sequencerCfg, "testnetobscuronet.azurecr.io/obscuronet/host:latest", "testnetobscuronet.azurecr.io/obscuronet/enclave:latest", edgelessDBImage, false, "", 0)
 
 	err = sequencerNode.Start()
 	if err != nil {
@@ -93,13 +87,7 @@ func (t *Testnet) Start() error {
 	validatorNodeCfg.Network.L1.L1Contracts.ManagementContract = common.HexToAddress(networkConfig.ManagementContractAddress)
 	validatorNodeCfg.Network.L1.L1Contracts.MessageBusContract = common.HexToAddress(networkConfig.MessageBusAddress)
 
-	validatorNode := node.NewDockerNode(validatorNodeCfg,
-		"testnetobscuronet.azurecr.io/obscuronet/host:latest",
-		"testnetobscuronet.azurecr.io/obscuronet/enclave:latest",
-		edgelessDBImage,
-		false,
-		"", // no PCCS override
-	)
+	validatorNode := node.NewDockerNode(validatorNodeCfg, "testnetobscuronet.azurecr.io/obscuronet/host:latest", "testnetobscuronet.azurecr.io/obscuronet/enclave:latest", edgelessDBImage, false, "", 0)
 	err = validatorNode.Start()
 	if err != nil {
 		return fmt.Errorf("unable to start the obscuro node - %w", err)