From bcaf1d1d41382bcb04c80276a188217ccc950f56 Mon Sep 17 00:00:00 2001 From: Matt Curtis Date: Thu, 5 Dec 2024 12:22:05 +0000 Subject: [PATCH 1/5] Testnets: grant sequencer enclaveID permission on L1 --- .../workflows/manual-deploy-testnet-l2.yml | 34 ++++++- .../scripts/sequencer/001_grant_sequencers.ts | 10 +- testnet/launcher/docker.go | 34 ++----- testnet/launcher/l1grantsequencers/cmd/cli.go | 3 + .../l1grantsequencers/cmd/cli_flags.go | 6 +- .../launcher/l1grantsequencers/cmd/main.go | 7 ++ testnet/launcher/l1grantsequencers/config.go | 8 ++ testnet/launcher/l1grantsequencers/docker.go | 92 ++++++++++++++++++- 8 files changed, 157 insertions(+), 37 deletions(-) diff --git a/.github/workflows/manual-deploy-testnet-l2.yml b/.github/workflows/manual-deploy-testnet-l2.yml index 821bc1e374..1bbd685acd 100644 --- a/.github/workflows/manual-deploy-testnet-l2.yml +++ b/.github/workflows/manual-deploy-testnet-l2.yml @@ -359,11 +359,43 @@ jobs: -postgres_db_host=postgres://tenuser:${{ secrets.TEN_POSTGRES_USER_PWD }}@postgres-ten-${{ github.event.inputs.testnet_type }}.postgres.database.azure.com:5432/ \ start' - check-obscuro-is-healthy: + grant-sequencer-enclaves: needs: - build - deploy runs-on: ubuntu-latest + environment: + name: ${{ github.event.inputs.testnet_type }} + steps: + - uses: actions/checkout@v4 + + - name: 'Grant permission to sequencer enclave(s)' + id: grantSequencerPermission + shell: bash + run: | + go run ./testnet/launcher/l1grantsequencers/cmd \ + -l1_http_url=${{ secrets.L1_HTTP_URL }} \ + -private_key=${{ secrets.ACCOUNT_PK_WORKER }} \ + -management_contract_addr=${{ needs.build.outputs.MGMT_CONTRACT_ADDR }} \ + -docker_image=${{ vars.L2_HARDHATDEPLOYER_DOCKER_BUILD_TAG }} \ + -sequencer_url=http://obscuronode-0-${{ github.event.inputs.testnet_type }}.uksouth.cloudapp.azure.com:10000 + + - name: 'Save sequencer permissioning container logs' + run: | + docker logs `docker ps -aqf "name=grant-sequencers"` > grant-sequencers.out 2>&1 + + - name: 'Upload sequencer permissioning container logs' + uses: actions/upload-artifact@v4 + with: + name: grant-sequencers + path: | + grant-sequencers.out + retention-days: 7 + + check-obscuro-is-healthy: + needs: + - grant-sequencer-enclaves + runs-on: ubuntu-latest environment: name: ${{ github.event.inputs.testnet_type }} steps: diff --git a/contracts/scripts/sequencer/001_grant_sequencers.ts b/contracts/scripts/sequencer/001_grant_sequencers.ts index 2f56a101ab..d8f8ccedc7 100644 --- a/contracts/scripts/sequencer/001_grant_sequencers.ts +++ b/contracts/scripts/sequencer/001_grant_sequencers.ts @@ -30,12 +30,14 @@ const grantSequencerStatus = async function (mgmtContractAddr: string, enclaveID } }; -const args = process.argv.slice(2); -if (args.length !== 2) { - throw new Error("Required arguments: "); +const mgmtContractAddr = process.env.MGMT_CONTRACT_ADDRESS; +const enclaveIDs = process.env.ENCLAVE_IDS; + +if (!mgmtContractAddr || !enclaveIDs) { + console.error("Missing required environment variables: MGMT_CONTRACT_ADDRESS and ENCLAVE_IDS."); + process.exit(1); } -const [mgmtContractAddr, enclaveIDs] = args as [string, string]; grantSequencerStatus(mgmtContractAddr, enclaveIDs) .then(() => process.exit(0)) .catch((error) => { diff --git a/testnet/launcher/docker.go b/testnet/launcher/docker.go index 48a1993593..9f985eb979 100644 --- a/testnet/launcher/docker.go +++ b/testnet/launcher/docker.go @@ -2,7 +2,6 @@ package launcher import ( "fmt" - "strings" "time" "github.com/ethereum/go-ethereum/common" @@ -138,6 +137,11 @@ func (t *Testnet) Start() error { return fmt.Errorf("unable to configure the l2 contract deployer - %w", err) } + err = t.grantSequencerStatus(networkConfig.ManagementContractAddress) + if err != nil { + return fmt.Errorf("failed to grant sequencer status: %w", err) + } + err = l2ContractDeployer.Start() if err != nil { return fmt.Errorf("unable to start the l2 contract deployer - %w", err) @@ -149,11 +153,6 @@ func (t *Testnet) Start() error { } fmt.Println("L2 Contracts were successfully deployed...") - err = t.grantSequencerStatus(networkConfig.ManagementContractAddress) - if err != nil { - return fmt.Errorf("failed to grant sequencer status: %w", err) - } - faucetPort := 99 faucetInst, err := faucet.NewDockerFaucet( faucet.NewFaucetConfig( @@ -304,27 +303,6 @@ func waitForHealthyNode(port int) error { // todo: hook the cfg func (t *Testnet) grantSequencerStatus(mgmtContractAddr string) error { // fetch enclaveIDs hostURL := fmt.Sprintf("http://localhost:%d", 80) - client, err := rpc.NewNetworkClient(hostURL) - if err != nil { - return fmt.Errorf("failed to create network client: %w", err) - } - defer client.Stop() - - obsClient := obsclient.NewObsClient(client) - health, err := obsClient.Health() - if err != nil { - return fmt.Errorf("failed to get health status: %w", err) - } - - if len(health.Enclaves) == 0 { - return fmt.Errorf("could not retrieve enclave IDs from health endpoint") - } - - var enclaveIDs []string - for _, status := range health.Enclaves { - enclaveIDs = append(enclaveIDs, status.EnclaveID.String()) - } - enclaveIDsStr := strings.Join(enclaveIDs, ",") l1grantsequencers, err := l1gs.NewGrantSequencers( l1gs.NewGrantSequencerConfig( @@ -332,7 +310,7 @@ func (t *Testnet) grantSequencerStatus(mgmtContractAddr string) error { l1gs.WithPrivateKey("f52e5418e349dccdda29b6ac8b0abe6576bb7713886aa85abea6181ba731f9bb"), l1gs.WithDockerImage(t.cfg.contractDeployerDockerImage), l1gs.WithMgmtContractAddress(mgmtContractAddr), - l1gs.WithEnclaveIDs(enclaveIDsStr), + l1gs.WithSequencerURL(hostURL), ), ) if err != nil { diff --git a/testnet/launcher/l1grantsequencers/cmd/cli.go b/testnet/launcher/l1grantsequencers/cmd/cli.go index 15e1418144..9aa2a7347d 100644 --- a/testnet/launcher/l1grantsequencers/cmd/cli.go +++ b/testnet/launcher/l1grantsequencers/cmd/cli.go @@ -11,6 +11,7 @@ type L1GrantSequencersConfigCLI struct { mgmtContractAddress string enclaveIDs string dockerImage string + sequencerURL string } // ParseConfigCLI returns a NodeConfigCLI based the cli params and defaults. @@ -23,6 +24,7 @@ func ParseConfigCLI() *L1GrantSequencersConfigCLI { mgmtContractAddress := flag.String(mgmtContractAddressFlag, "", flagUsageMap[mgmtContractAddressFlag]) enclaveIDs := flag.String(enclaveIDsFlag, "", flagUsageMap[enclaveIDsFlag]) dockerImage := flag.String(dockerImageFlag, "testnetobscuronet.azurecr.io/obscuronet/hardhatdeployer:latest", flagUsageMap[dockerImageFlag]) + sequencerURL := flag.String(sequencerURLFlag, "", flagUsageMap[sequencerURLFlag]) flag.Parse() cfg.l1HTTPURL = *l1HTTPURL @@ -30,6 +32,7 @@ func ParseConfigCLI() *L1GrantSequencersConfigCLI { cfg.mgmtContractAddress = *mgmtContractAddress cfg.enclaveIDs = *enclaveIDs cfg.dockerImage = *dockerImage + cfg.sequencerURL = *sequencerURL return cfg } diff --git a/testnet/launcher/l1grantsequencers/cmd/cli_flags.go b/testnet/launcher/l1grantsequencers/cmd/cli_flags.go index e85a6a9aed..7408097f40 100644 --- a/testnet/launcher/l1grantsequencers/cmd/cli_flags.go +++ b/testnet/launcher/l1grantsequencers/cmd/cli_flags.go @@ -4,10 +4,11 @@ package main const ( l1HTTPURLFlag = "l1_http_url" privateKeyFlag = "private_key" - mgmtContractAddressFlag = "mgmt_contract_address" + mgmtContractAddressFlag = "management_contract_addr" enclaveIDsFlag = "enclave_ids" dockerImageFlag = "docker_image" contractsEnvFileFlag = "contracts_env_file" + sequencerURLFlag = "sequencer_url" ) // Returns a map of the flag usages. @@ -17,8 +18,9 @@ func getFlagUsageMap() map[string]string { l1HTTPURLFlag: "Layer 1 network http RPC addr", privateKeyFlag: "L1 and L2 private key used in the node", mgmtContractAddressFlag: "L1 management contract address", - enclaveIDsFlag: "List of enclave public keys", + enclaveIDsFlag: "List of enclave public keys to grant sequencer role", dockerImageFlag: "Docker image to run", contractsEnvFileFlag: "If set, it will write the contract addresses to the file", + sequencerURLFlag: "Sequencer RPC URL to fetch enclave IDs (required if enclaveIDs are not provided)", } } diff --git a/testnet/launcher/l1grantsequencers/cmd/main.go b/testnet/launcher/l1grantsequencers/cmd/main.go index 6273ecee02..36ac1798a9 100644 --- a/testnet/launcher/l1grantsequencers/cmd/main.go +++ b/testnet/launcher/l1grantsequencers/cmd/main.go @@ -17,6 +17,7 @@ func main() { l1gs.WithDockerImage(cliConfig.dockerImage), l1gs.WithMgmtContractAddress(cliConfig.mgmtContractAddress), l1gs.WithEnclaveIDs(cliConfig.enclaveIDs), + l1gs.WithSequencerURL(cliConfig.sequencerURL), ), ) if err != nil { @@ -30,5 +31,11 @@ func main() { os.Exit(1) } + err = l1grantsequencers.WaitForFinish() + if err != nil { + fmt.Println("unexpected error waiting for grant sequnecer permission script to finish - %w", err) + os.Exit(1) + } + fmt.Println("L1 Sequencer permissions were successfully granted...") os.Exit(0) } diff --git a/testnet/launcher/l1grantsequencers/config.go b/testnet/launcher/l1grantsequencers/config.go index a230311cd0..0345e6b59e 100644 --- a/testnet/launcher/l1grantsequencers/config.go +++ b/testnet/launcher/l1grantsequencers/config.go @@ -10,6 +10,8 @@ type Config struct { mgmtContractAddress string enclaveIDs string dockerImage string + sequencerURL string + // debugEnabled bool } @@ -53,6 +55,12 @@ func WithDockerImage(s string) Option { } } +func WithSequencerURL(s string) Option { + return func(c *Config) { + c.sequencerURL = s + } +} + //func WithDebugEnabled(b bool) Option { // return func(c *Config) { // c.debugEnabled = b diff --git a/testnet/launcher/l1grantsequencers/docker.go b/testnet/launcher/l1grantsequencers/docker.go index 41eb56cf51..4468de3921 100644 --- a/testnet/launcher/l1grantsequencers/docker.go +++ b/testnet/launcher/l1grantsequencers/docker.go @@ -1,9 +1,18 @@ package l1grantsequencers import ( + "bytes" + "context" "fmt" + "io" + "strings" + "time" + "github.com/docker/docker/api/types" + "github.com/docker/docker/client" "github.com/ten-protocol/go-ten/go/common/docker" + "github.com/ten-protocol/go-ten/go/obsclient" + "github.com/ten-protocol/go-ten/go/rpc" ) type GrantSequencers struct { @@ -18,14 +27,25 @@ func NewGrantSequencers(cfg *Config) (*GrantSequencers, error) { } func (s *GrantSequencers) Start() error { + var enclaveIDs string + var err error + if s.cfg.enclaveIDs != "" { + enclaveIDs = s.cfg.enclaveIDs + } else if s.cfg.sequencerURL != "" { + enclaveIDs, err = fetchEnclaveIDs(s.cfg.sequencerURL) + if err != nil { + return err + } + } else { + return fmt.Errorf("enclaveIDs or sequencerURL must be provided") + } cmds := []string{ "npx", + "hardhat", "run", "--network", "layer1", "scripts/sequencer/001_grant_sequencers.ts", - s.cfg.mgmtContractAddress, - s.cfg.enclaveIDs, } envs := map[string]string{ @@ -37,8 +57,12 @@ func (s *GrantSequencers) Start() error { "accounts": [ "%s" ] } }`, s.cfg.l1HTTPURL, s.cfg.privateKey), + "MGMT_CONTRACT_ADDRESS": s.cfg.mgmtContractAddress, + "ENCLAVE_IDS": enclaveIDs, } + fmt.Printf("Starting grant sequencer script. MgntContractAddress: %s, EnclaveIDs: %s\n", s.cfg.mgmtContractAddress, enclaveIDs) + containerID, err := docker.StartNewContainer( "grant-sequencers", s.cfg.dockerImage, @@ -55,3 +79,67 @@ func (s *GrantSequencers) Start() error { s.containerID = containerID return nil } + +func (s *GrantSequencers) WaitForFinish() error { + cli, err := client.NewClientWithOpts(client.FromEnv) + if err != nil { + return fmt.Errorf("failed to create docker client: %w", err) + } + defer cli.Close() + + // make sure the container has finished execution + err = docker.WaitForContainerToFinish(s.containerID, 15*time.Minute) + if err != nil { + fmt.Println("Error waiting for container to finish: ", err) + s.PrintLogs(cli) + return err + } + + return nil +} + +func fetchEnclaveIDs(url string) (string, error) { + // fetch enclaveIDs + client, err := rpc.NewNetworkClient(url) + if err != nil { + return "", fmt.Errorf("failed to create network client (%s): %w", url, err) + } + defer client.Stop() + + obsClient := obsclient.NewObsClient(client) + health, err := obsClient.Health() + if err != nil { + return "", fmt.Errorf("failed to get health status: %w", err) + } + + if len(health.Enclaves) == 0 { + return "", fmt.Errorf("could not retrieve enclave IDs from health endpoint - no enclaves found") + } + + var enclaveIDs []string + for _, status := range health.Enclaves { + enclaveIDs = append(enclaveIDs, status.EnclaveID.String()) + } + return strings.Join(enclaveIDs, ","), nil +} + +func (s *GrantSequencers) PrintLogs(cli *client.Client) { + logsOptions := types.ContainerLogsOptions{ + ShowStdout: true, + ShowStderr: true, + } + + // Read the container logs + out, err := cli.ContainerLogs(context.Background(), s.containerID, logsOptions) + if err != nil { + fmt.Printf("Error printing out container %s logs... %v\n", s.containerID, err) + } + defer out.Close() + + var buf bytes.Buffer + _, err = io.Copy(&buf, out) + if err != nil { + fmt.Printf("Error getting logs for container %s\n", s.containerID) + } + fmt.Println(buf.String()) +} From 4752c25c0026b6265cd59df9da52b45691f6fbad Mon Sep 17 00:00:00 2001 From: Matt Curtis Date: Mon, 9 Dec 2024 16:02:51 +0000 Subject: [PATCH 2/5] wait for node up before permissioning --- .../workflows/manual-deploy-testnet-l2.yml | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/.github/workflows/manual-deploy-testnet-l2.yml b/.github/workflows/manual-deploy-testnet-l2.yml index 1bbd685acd..854b5452cc 100644 --- a/.github/workflows/manual-deploy-testnet-l2.yml +++ b/.github/workflows/manual-deploy-testnet-l2.yml @@ -359,11 +359,28 @@ jobs: -postgres_db_host=postgres://tenuser:${{ secrets.TEN_POSTGRES_USER_PWD }}@postgres-ten-${{ github.event.inputs.testnet_type }}.postgres.database.azure.com:5432/ \ start' - grant-sequencer-enclaves: + check-obscuro-is-healthy: needs: - build - deploy runs-on: ubuntu-latest + environment: + name: ${{ github.event.inputs.testnet_type }} + steps: + - uses: actions/checkout@v4 + + - name: "Wait until obscuro node is healthy" + shell: bash + run: | + ./.github/workflows/runner-scripts/wait-node-healthy.sh --host=obscuronode-0-${{ github.event.inputs.testnet_type }}.uksouth.cloudapp.azure.com + ./.github/workflows/runner-scripts/wait-node-healthy.sh --host=obscuronode-1-${{ github.event.inputs.testnet_type }}.uksouth.cloudapp.azure.com + ./.github/workflows/runner-scripts/wait-node-healthy.sh --host=obscuronode-2-${{ github.event.inputs.testnet_type }}.uksouth.cloudapp.azure.com + + + grant-sequencer-enclaves: + needs: + - check-obscuro-is-healthy + runs-on: ubuntu-latest environment: name: ${{ github.event.inputs.testnet_type }} steps: @@ -392,22 +409,6 @@ jobs: grant-sequencers.out retention-days: 7 - check-obscuro-is-healthy: - needs: - - grant-sequencer-enclaves - runs-on: ubuntu-latest - environment: - name: ${{ github.event.inputs.testnet_type }} - steps: - - uses: actions/checkout@v4 - - - name: "Wait until obscuro node is healthy" - shell: bash - run: | - ./.github/workflows/runner-scripts/wait-node-healthy.sh --host=obscuronode-0-${{ github.event.inputs.testnet_type }}.uksouth.cloudapp.azure.com - ./.github/workflows/runner-scripts/wait-node-healthy.sh --host=obscuronode-1-${{ github.event.inputs.testnet_type }}.uksouth.cloudapp.azure.com - ./.github/workflows/runner-scripts/wait-node-healthy.sh --host=obscuronode-2-${{ github.event.inputs.testnet_type }}.uksouth.cloudapp.azure.com - deploy-l2-contracts: needs: - build From 5b4a4ea8695847f901e35b3f25425dfdf255e908 Mon Sep 17 00:00:00 2001 From: Matt Curtis Date: Mon, 9 Dec 2024 22:21:28 +0000 Subject: [PATCH 3/5] permission step needs build step for its output --- .github/workflows/manual-deploy-testnet-l2.yml | 7 ++++--- testnet/launcher/l1grantsequencers/docker.go | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/manual-deploy-testnet-l2.yml b/.github/workflows/manual-deploy-testnet-l2.yml index 854b5452cc..22f7ce90df 100644 --- a/.github/workflows/manual-deploy-testnet-l2.yml +++ b/.github/workflows/manual-deploy-testnet-l2.yml @@ -379,6 +379,7 @@ jobs: grant-sequencer-enclaves: needs: + - build - check-obscuro-is-healthy runs-on: ubuntu-latest environment: @@ -412,7 +413,7 @@ jobs: deploy-l2-contracts: needs: - build - - check-obscuro-is-healthy + - grant-sequencer-enclaves runs-on: ubuntu-latest environment: name: ${{ github.event.inputs.testnet_type }} @@ -450,7 +451,7 @@ jobs: update-loadbalancer: needs: - - check-obscuro-is-healthy + - grant-sequencer-enclaves runs-on: ubuntu-latest environment: name: ${{ github.event.inputs.testnet_type }} @@ -483,7 +484,7 @@ jobs: testnet_type: ${{ github.event.inputs.testnet_type }} secrets: inherit needs: - - check-obscuro-is-healthy + - grant-sequencer-enclaves obscuro-test-repository-dispatch: runs-on: ubuntu-latest diff --git a/testnet/launcher/l1grantsequencers/docker.go b/testnet/launcher/l1grantsequencers/docker.go index 4468de3921..dc17c5ca08 100644 --- a/testnet/launcher/l1grantsequencers/docker.go +++ b/testnet/launcher/l1grantsequencers/docker.go @@ -27,6 +27,7 @@ func NewGrantSequencers(cfg *Config) (*GrantSequencers, error) { } func (s *GrantSequencers) Start() error { + fmt.Printf("Starting grant sequencers with config: %s\n", s.cfg) var enclaveIDs string var err error if s.cfg.enclaveIDs != "" { From 46894db2111830ad45892825525aa9256e61ec0d Mon Sep 17 00:00:00 2001 From: Matt Curtis Date: Mon, 9 Dec 2024 23:50:09 +0000 Subject: [PATCH 4/5] Fix hardcoded values in wiring --- testnet/launcher/l1grantsequencers/cmd/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testnet/launcher/l1grantsequencers/cmd/main.go b/testnet/launcher/l1grantsequencers/cmd/main.go index 36ac1798a9..0eecf02957 100644 --- a/testnet/launcher/l1grantsequencers/cmd/main.go +++ b/testnet/launcher/l1grantsequencers/cmd/main.go @@ -12,8 +12,8 @@ func main() { l1grantsequencers, err := l1gs.NewGrantSequencers( l1gs.NewGrantSequencerConfig( - l1gs.WithL1HTTPURL("http://eth2network:8025"), - l1gs.WithPrivateKey("f52e5418e349dccdda29b6ac8b0abe6576bb7713886aa85abea6181ba731f9bb"), + l1gs.WithL1HTTPURL(cliConfig.l1HTTPURL), + l1gs.WithPrivateKey(cliConfig.privateKey), l1gs.WithDockerImage(cliConfig.dockerImage), l1gs.WithMgmtContractAddress(cliConfig.mgmtContractAddress), l1gs.WithEnclaveIDs(cliConfig.enclaveIDs), From a9deb2f096981f1bf8cf5af42577678c6913e10b Mon Sep 17 00:00:00 2001 From: Matt Curtis Date: Tue, 10 Dec 2024 12:53:02 +0000 Subject: [PATCH 5/5] fix rpc port --- .github/workflows/manual-deploy-testnet-l2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual-deploy-testnet-l2.yml b/.github/workflows/manual-deploy-testnet-l2.yml index 22f7ce90df..65c5095696 100644 --- a/.github/workflows/manual-deploy-testnet-l2.yml +++ b/.github/workflows/manual-deploy-testnet-l2.yml @@ -396,7 +396,7 @@ jobs: -private_key=${{ secrets.ACCOUNT_PK_WORKER }} \ -management_contract_addr=${{ needs.build.outputs.MGMT_CONTRACT_ADDR }} \ -docker_image=${{ vars.L2_HARDHATDEPLOYER_DOCKER_BUILD_TAG }} \ - -sequencer_url=http://obscuronode-0-${{ github.event.inputs.testnet_type }}.uksouth.cloudapp.azure.com:10000 + -sequencer_url=http://obscuronode-0-${{ github.event.inputs.testnet_type }}.uksouth.cloudapp.azure.com:80 - name: 'Save sequencer permissioning container logs' run: |