From 680df8a73ab539b176716f86c653e01cb741248a Mon Sep 17 00:00:00 2001 From: Javed Khan Date: Fri, 21 Apr 2023 08:45:01 +0530 Subject: [PATCH 01/11] da: use celestia as da build(deps): update go-da to v0.1.0 build(deps): update go-da to v0.2.0; add ctx da: add cli flags for da config da: add get timeout to da client build(deps): bump local-celestia-devnet to v0.12.7 fix(cli): fix da rpc check fix(cli): check return err fix(da): blob data source reuse DataFromEVMTransactions --- .github/workflows/fork-sync.yml | 18 ++++ README.md | 79 +++++++++++++++- bedrock-devnet/devnet/__init__.py | 7 +- .../understand/Celestia-logo-color-color.svg | 1 + go.mod | 12 ++- go.sum | 29 ++++-- op-batcher/batcher/config.go | 6 ++ op-batcher/batcher/driver.go | 12 +++ op-batcher/batcher/service.go | 19 ++++ op-batcher/flags/flags.go | 3 +- op-celestia/cli.go | 94 +++++++++++++++++++ op-celestia/da.go | 10 ++ op-celestia/da_client.go | 26 +++++ op-e2e/system/e2esys/setup.go | 4 + op-node/flags/flags.go | 2 + op-node/node/config.go | 6 ++ op-node/node/node.go | 7 ++ op-node/rollup/derive/blob_data_source.go | 21 ++++- .../rollup/derive/blob_data_source_test.go | 15 ++- op-node/rollup/derive/calldata_source.go | 69 ++++++++++++-- op-node/rollup/derive/calldata_source_test.go | 4 +- op-node/rollup/derive/data_source.go | 6 +- op-node/rollup/driver/da.go | 14 +++ op-node/service.go | 4 +- ops-bedrock/docker-compose.yml | 28 +++++- .../op-stack-go/Dockerfile.dockerignore | 1 + 26 files changed, 455 insertions(+), 42 deletions(-) create mode 100644 .github/workflows/fork-sync.yml create mode 100644 docs/op-stack/src/assets/docs/understand/Celestia-logo-color-color.svg create mode 100644 op-celestia/cli.go create mode 100644 op-celestia/da.go create mode 100644 op-celestia/da_client.go create mode 100644 op-node/rollup/driver/da.go diff --git a/.github/workflows/fork-sync.yml b/.github/workflows/fork-sync.yml new file mode 100644 index 000000000000..a2196fb742f4 --- /dev/null +++ b/.github/workflows/fork-sync.yml @@ -0,0 +1,18 @@ +name: Sync Fork + +on: + schedule: + - cron: '*/30 * * * *' # every 30 minutes + workflow_dispatch: # on button click + +jobs: + sync: + + runs-on: ubuntu-latest + + steps: + - uses: tgymnich/fork-sync@v1.8 + with: + base: celestia-develop + head: develop + token: ${{ secrets.REPO_PAT }} diff --git a/README.md b/README.md index d33f920c8398..bb0f0233114d 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,11 @@

Optimism +

Optimism is a low-cost and lightning-fast Ethereum L2 blockchain, built with the OP Stack.


-

Optimism is Ethereum, scaled.

+

+

+ Celestia +

Celestia is a modular data availability network that securely scales with the number of users, making it easy for anyone to launch their own blockchain.


@@ -29,8 +32,24 @@ ## What is Optimism? [Optimism](https://www.optimism.io/) is a project dedicated to scaling Ethereum's technology and expanding its ability to coordinate people from across the world to build effective decentralized economies and governance systems. The [Optimism Collective](https://www.optimism.io/vision) builds open-source software that powers scalable blockchains and aims to address key governance and economic challenges in the wider Ethereum ecosystem. Optimism operates on the principle of **impact=profit**, the idea that individuals who positively impact the Collective should be proportionally rewarded with profit. **Change the incentives and you change the world.** +If you're looking to run the OP Stack + Celestia setup for this repository, please visit the [Optimism & Celestia guides and tutorials](https://docs.celestia.org/developers/intro-to-op-stack/) to get started. In this repository you'll find numerous core components of the OP Stack, the decentralized software stack maintained by the Optimism Collective that powers Optimism and forms the backbone of blockchains like [OP Mainnet](https://explorer.optimism.io/) and [Base](https://base.org). The OP Stack is designed to be aggressively open-source — you are welcome to explore, modify, and extend this code. +The OP Stack powers Optimism, an Ethereum L2 blockchain, and forms the technical foundation for the [the Optimism Collective](https://app.optimism.io/announcement)—a group committed to the **impact=profit** principle. This principle rewards individuals for their positive contributions to the collective. + +Optimism addresses critical coordination failures in the crypto ecosystem, such as funding public goods and infrastructure. The OP Stack focuses on creating a shared, open-source system for developing new L2 blockchains within the proposed Superchain ecosystem, promoting collaboration and preventing redundant efforts. + +As Optimism evolves, the OP Stack will adapt, encompassing components ranging from blockchain infrastructure to governance systems. This software suite aims to simplify L2 blockchain creation while supporting the growth and development of the Optimism ecosystem. + +## What is Celestia? + +Celestia is a modular consensus and data network, built to enable anyone to easily deploy their own blockchain with minimal overhead. + +Celestia is a minimal blockchain that only orders and publishes transactions and does not execute them. By decoupling the consensus and application execution layers, Celestia modularizes the blockchain technology stack and unlocks new possibilities for decentralized application builders. Lean more at [Celestia.org](https://celestia.org). + +## Maintenance + +The maintenance guide for this repository can be found in the Wiki tab of the repository or [here](https://github.com/celestiaorg/optimism/wiki). ## Documentation @@ -40,6 +59,22 @@ In this repository you'll find numerous core components of the OP Stack, the dec ## Specification Detailed specifications for the OP Stack can be found within the [OP Stack Specs](https://github.com/ethereum-optimism/specs) repository. +If you're interested in the technical details of how Optimism works, refer to the [Optimism Protocol Specification](https://github.com/ethereum-optimism/specs). +If you want to build on top of Celestia, take a look at the documentation at [docs.celestia.org](https://docs.celestia.org). + +If you want to learn more about the OP Stack, check out the documentation at [stack.optimism.io](https://stack.optimism.io/). + +## Community + +### Optimism + +General discussion happens most frequently on the [Optimism discord](https://discord.gg/optimism). +Governance discussion can also be found on the [Optimism Governance Forum](https://gov.optimism.io/). + +### Celestia + +General discussion happens most frequently on the [Celestia discord](https://discord.com/invite/YsnTPcSfWQ). +Other discussions can be found on the [Celestia forum](https://forum.celestia.org). ## Community @@ -53,12 +88,52 @@ The OP Stack is a collaborative project. By collaborating on free, open software [CONTRIBUTING.md](./CONTRIBUTING.md) contains a detailed explanation of the contributing process for this repository. Make sure to use the [Developer Quick Start](./CONTRIBUTING.md#development-quick-start) to properly set up your development environment. [Good First Issues](https://github.com/ethereum-optimism/optimism/issues?q=is:open+is:issue+label:D-good-first-issue) are a great place to look for tasks to tackle if you're not sure where to start. +Read through [CONTRIBUTING.md](./CONTRIBUTING.md) for a general overview of our contribution process. +Use the [Developer Quick Start](./CONTRIBUTING.md#development-quick-start) to get your development environment set up to start working on the Optimism Monorepo. +Then check out the list of [Good First Issues](https://github.com/ethereum-optimism/optimism/issues?q=is:open+is:issue+label:D-good-first-issue) to find something fun to work on! +Typo fixes are welcome; however, please create a single commit with all of the typo fixes & batch as many fixes together in a PR as possible. Spammy PRs will be closed. -## Security Policy and Vulnerability Reporting +## e2e testing Please refer to the canonical [Security Policy](https://github.com/ethereum-optimism/.github/blob/master/SECURITY.md) document for detailed information about how to report vulnerabilities in this codebase. Bounty hunters are encouraged to check out the [Optimism Immunefi bug bounty program](https://immunefi.com/bounty/optimism/). The Optimism Immunefi program offers up to $2,000,042 for in-scope critical vulnerabilities. +This repository has updated end-to-end tests in the `op-e2e` package to work with +Celestia as the data availability (DA) layer. + +Currently, the tests assume a working [Celestia devnet](https://github.com/rollkit/local-celestia-devnet) running locally: + +```bash +docker run -p 26650:26650 ghcr.io/rollkit/local-celestia-devnet:v0.12.7 +``` + +The e2e tests can be triggered with: + +```bash +cd $HOME/optimism +cd op-e2e +OP_E2E_DISABLE_PARALLEL=true OP_E2E_CANNON_ENABLED=false OP_NODE_DA_RPC=localhost:26650 OP_BATCHER_DA_RPC=localhost:26650 make test +``` + +## Bridging + +If you have the OP Stack + Celestia setup running, you can test out bridging from the L1 +to the L2. + +To do this, first navigate to the `packages/contracts-bedrock` directory and create a +`.env` file with the following contents: + +```bash +L1_PROVIDER_URL=http://localhost:8545 +L2_PROVIDER_URL=http://localhost:9545 +PRIVATE_KEY=bf7604d9d3a1c7748642b1b7b05c2bd219c9faa91458b370f85e5a40f3b03af7 +``` + +Then, run the following from the same directory: + +```bash +npx hardhat deposit --network devnetL1 --l1-provider-url http://localhost:8545 --l2-provider-url http://localhost:9545 --amount-eth --to
+``` ## Directory Structure diff --git a/bedrock-devnet/devnet/__init__.py b/bedrock-devnet/devnet/__init__.py index 70243dc790c8..400923876197 100644 --- a/bedrock-devnet/devnet/__init__.py +++ b/bedrock-devnet/devnet/__init__.py @@ -287,6 +287,10 @@ def devnet_deploy(paths): log.info('Bringing up `op-node`, `op-proposer` and `op-batcher`.') run_command(['docker', 'compose', 'up', '-d', 'op-node', 'op-proposer', 'op-batcher', 'artifact-server'], cwd=paths.ops_bedrock_dir, env=docker_env) + log.info('Bringing up DA') + run_command(['docker', 'compose', 'up', '-d', 'da'], cwd=paths.ops_bedrock_dir, env=docker_env) + wait_up(26650) + # Optionally bring up op-challenger. if not DEVNET_L2OO: log.info('Bringing up `op-challenger`.') @@ -363,10 +367,11 @@ def run_command_preset(command: CommandPreset): return proc.returncode -def run_command(args, check=True, shell=False, cwd=None, env=None, timeout=None): +def run_command(args, check=True, shell=False, cwd=None, env=None, timeout=None, capture_output=False): env = env if env else {} return subprocess.run( args, + capture_output=capture_output, check=check, shell=shell, env={ diff --git a/docs/op-stack/src/assets/docs/understand/Celestia-logo-color-color.svg b/docs/op-stack/src/assets/docs/understand/Celestia-logo-color-color.svg new file mode 100644 index 000000000000..6d22cb408925 --- /dev/null +++ b/docs/op-stack/src/assets/docs/understand/Celestia-logo-color-color.svg @@ -0,0 +1 @@ +Celestia-logo-color-black \ No newline at end of file diff --git a/go.mod b/go.mod index 7478f4a18528..8cddae9d1f98 100644 --- a/go.mod +++ b/go.mod @@ -44,6 +44,7 @@ require ( github.com/pkg/profile v1.7.0 github.com/prometheus/client_golang v1.20.5 github.com/protolambda/ctxlock v0.1.0 + github.com/rollkit/go-da v0.5.0 github.com/stretchr/testify v1.9.0 github.com/urfave/cli/v2 v2.27.5 golang.org/x/crypto v0.28.0 @@ -97,6 +98,7 @@ require ( github.com/fatih/color v1.16.0 // indirect github.com/felixge/fgprof v0.9.3 // indirect github.com/ferranbt/fastssz v0.1.2 // indirect + github.com/filecoin-project/go-jsonrpc v0.3.1 // indirect github.com/flynn/noise v1.1.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08 // indirect @@ -112,6 +114,7 @@ require ( github.com/gofrs/flock v0.8.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect + github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/gopacket v1.1.19 // indirect github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect @@ -229,6 +232,7 @@ require ( github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect go.etcd.io/bbolt v1.3.5 // indirect + go.opencensus.io v0.22.3 // indirect go.uber.org/dig v1.18.0 // indirect go.uber.org/fx v1.22.2 // indirect go.uber.org/mock v0.4.0 // indirect @@ -239,10 +243,10 @@ require ( golang.org/x/sys v0.26.0 // indirect golang.org/x/text v0.19.0 // indirect golang.org/x/tools v0.24.0 // indirect - google.golang.org/genproto v0.0.0-20230726155614-23370e0ffb3e // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 // indirect - google.golang.org/grpc v1.57.1 // indirect + golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect + google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 7b74fd565d0f..b8002793ed16 100644 --- a/go.sum +++ b/go.sum @@ -196,6 +196,8 @@ github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g= github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw= github.com/ferranbt/fastssz v0.1.2 h1:Dky6dXlngF6Qjc+EfDipAkE83N5I5DE68bY6O0VLNPk= github.com/ferranbt/fastssz v0.1.2/go.mod h1:X5UPrE2u1UJjxHA8X54u04SBwdAQjG2sFtWs39YxyWs= +github.com/filecoin-project/go-jsonrpc v0.3.1 h1:qwvAUc5VwAkooquKJmfz9R2+F8znhiqcNHYjEp/NM10= +github.com/filecoin-project/go-jsonrpc v0.3.1/go.mod h1:jBSvPTl8V1N7gSTuCR4bis8wnQnIjHbRPpROol6iQKM= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v1.1.0 h1:KjPQoQCEFdZDiP03phOvGi11+SVVhBG2wOWAorLsstg= github.com/flynn/noise v1.1.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= @@ -257,6 +259,8 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -700,6 +704,8 @@ github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rollkit/go-da v0.5.0 h1:sQpZricNS+2TLx3HMjNWhtRfqtvVC/U4pWHpfUz3eN4= +github.com/rollkit/go-da v0.5.0/go.mod h1:VsUeAoPvKl4Y8wWguu/VibscYiFFePkkrvZWyTjZHww= github.com/rs/cors v1.11.0 h1:0B9GE/r9Bc2UxRMMtymBkHTenPkHDv0CW4Y98GBY+po= github.com/rs/cors v1.11.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= @@ -805,6 +811,8 @@ github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQ go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= +go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/dig v1.18.0 h1:imUL1UiY0Mg4bqbFfsRQO5G4CGRBec/ZujWTvSVp3pw= @@ -851,6 +859,7 @@ golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa/go.mod h1:akd2r19cwCdwSwWeId golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -924,6 +933,7 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1027,6 +1037,7 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df h1:5Pf6pFKu98ODmgnpvkJ3kFUOQGGLIzLIkbzUHp47618= golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= @@ -1040,18 +1051,20 @@ google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoA google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20230726155614-23370e0ffb3e h1:xIXmWJ303kJCuogpj0bHq+dcjcZHU+XFyc1I0Yl9cRg= -google.golang.org/genproto v0.0.0-20230726155614-23370e0ffb3e/go.mod h1:0ggbjUrZYpy1q+ANUS30SEoGZ53cdfwtbuG7Ptgy108= -google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 h1:FmF5cCW94Ij59cfpoLiwTgodWmm60eEV0CjlsVg2fuw= -google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 h1:eSaPbMR4T7WfH9FvABk36NBMacoTUKdWCvV0dx+KfOg= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5/go.mod h1:zBEcrKX2ZOcEkHWxBPAIvYUWOKKMIhYcmNiUIu2ji3I= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 h1:SeZZZx0cP0fqUyA+oRzP9k7cSwJlvDFiROO72uwD6i0= +google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97/go.mod h1:t1VqOqqvce95G3hIDCT5FeO3YUc6Q4Oe24L/+rNMxRk= +google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97 h1:W18sezcAYs+3tDZX4F80yctqa12jcP1PUS2gQu1zTPU= +google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97/go.mod h1:iargEX0SFPm3xcfMI0d1domjg0ZF4Aa0p2awqyxhvF0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 h1:6GQBEOdGkX6MMTLT9V+TjtIRZCw9VPD5Z+yHY9wMgS0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97/go.mod h1:v7nGkzlmW8P3n/bKmWBn2WpBjpOEx8Q6gMueudAmKfY= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.57.1 h1:upNTNqv0ES+2ZOOqACwVtS3Il8M12/+Hz41RCPzAjQg= -google.golang.org/grpc v1.57.1/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU= +google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/op-batcher/batcher/config.go b/op-batcher/batcher/config.go index ac8bad7791a7..6a4e539a6d1d 100644 --- a/op-batcher/batcher/config.go +++ b/op-batcher/batcher/config.go @@ -11,6 +11,7 @@ import ( altda "github.com/ethereum-optimism/optimism/op-alt-da" "github.com/ethereum-optimism/optimism/op-batcher/compressor" "github.com/ethereum-optimism/optimism/op-batcher/flags" + celestia "github.com/ethereum-optimism/optimism/op-celestia" "github.com/ethereum-optimism/optimism/op-node/rollup/derive" "github.com/ethereum-optimism/optimism/op-service/eth" oplog "github.com/ethereum-optimism/optimism/op-service/log" @@ -106,6 +107,7 @@ type CLIConfig struct { PprofConfig oppprof.CLIConfig RPC oprpc.CLIConfig AltDA altda.CLIConfig + DaConfig celestia.CLIConfig } func (c *CLIConfig) Check() error { @@ -161,6 +163,9 @@ func (c *CLIConfig) Check() error { if err := c.RPC.Check(); err != nil { return err } + if err := c.DaConfig.Check(); err != nil { + return err + } return nil } @@ -195,5 +200,6 @@ func NewConfig(ctx *cli.Context) *CLIConfig { PprofConfig: oppprof.ReadCLIConfig(ctx), RPC: oprpc.ReadCLIConfig(ctx), AltDA: altda.ReadCLIConfig(ctx), + DaConfig: celestia.ReadCLIConfig(ctx), } } diff --git a/op-batcher/batcher/driver.go b/op-batcher/batcher/driver.go index b31dd29e2e05..0b7eb8d5d2e8 100644 --- a/op-batcher/batcher/driver.go +++ b/op-batcher/batcher/driver.go @@ -2,6 +2,7 @@ package batcher import ( "context" + "encoding/hex" "errors" "fmt" "io" @@ -12,6 +13,7 @@ import ( altda "github.com/ethereum-optimism/optimism/op-alt-da" "github.com/ethereum-optimism/optimism/op-batcher/metrics" + celestia "github.com/ethereum-optimism/optimism/op-celestia" "github.com/ethereum-optimism/optimism/op-node/rollup" "github.com/ethereum-optimism/optimism/op-node/rollup/derive" "github.com/ethereum-optimism/optimism/op-service/dial" @@ -86,6 +88,7 @@ type DriverSetup struct { ChannelConfig ChannelConfigProvider AltDA *altda.DAClient ChannelOutFactory ChannelOutFactory + DAClient *celestia.DAClient } // BatchSubmitter encapsulates a service responsible for submitting L2 tx @@ -757,6 +760,15 @@ func (l *BatchSubmitter) blobTxCandidate(data txData) (*txmgr.TxCandidate, error func (l *BatchSubmitter) calldataTxCandidate(data []byte) *txmgr.TxCandidate { l.Log.Info("Building Calldata transaction candidate", "size", len(data)) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Duration(l.RollupConfig.BlockTime)*time.Second) + ids, _, err := l.DAClient.Client.Submit(ctx, [][]byte{data}, -1) + cancel() + if err == nil && len(ids) == 1 { + l.Log.Info("celestia: blob successfully submitted", "id", hex.EncodeToString(ids[0])) + data = append([]byte{celestia.DerivationVersionCelestia}, ids[0]...) + } else { + l.Log.Info("celestia: blob submission failed; falling back to eth", "err", err) + } return &txmgr.TxCandidate{ To: &l.RollupConfig.BatchInboxAddress, TxData: data, diff --git a/op-batcher/batcher/service.go b/op-batcher/batcher/service.go index 90a85cc4ee48..173a21a658b7 100644 --- a/op-batcher/batcher/service.go +++ b/op-batcher/batcher/service.go @@ -16,6 +16,7 @@ import ( "github.com/ethereum-optimism/optimism/op-batcher/flags" "github.com/ethereum-optimism/optimism/op-batcher/metrics" "github.com/ethereum-optimism/optimism/op-batcher/rpc" + celestia "github.com/ethereum-optimism/optimism/op-celestia" "github.com/ethereum-optimism/optimism/op-node/chaincfg" "github.com/ethereum-optimism/optimism/op-node/params" "github.com/ethereum-optimism/optimism/op-node/rollup" @@ -73,6 +74,7 @@ type BatcherService struct { stopped atomic.Bool NotSubmittingOnStart bool + DAClient *celestia.DAClient } type DriverSetupOption func(setup *DriverSetup) @@ -124,6 +126,13 @@ func (bs *BatcherService) initFromCLIConfig(ctx context.Context, version string, if err := bs.initPProf(cfg); err != nil { return fmt.Errorf("failed to init profiling: %w", err) } + // init before driver + if err := bs.initAltDA(cfg); err != nil { + return fmt.Errorf("failed to init AltDA: %w", err) + } + if err := bs.initDA(cfg); err != nil { + return fmt.Errorf("failed to start da server: %w", err) + } bs.initDriver(opts...) if err := bs.initRPCServer(cfg); err != nil { return fmt.Errorf("failed to start RPC server: %w", err) @@ -328,6 +337,7 @@ func (bs *BatcherService) initDriver(opts ...DriverSetupOption) { EndpointProvider: bs.EndpointProvider, ChannelConfig: bs.ChannelConfig, AltDA: bs.AltDA, + DAClient: bs.DAClient, } for _, opt := range opts { opt(&ds) @@ -366,6 +376,15 @@ func (bs *BatcherService) initAltDA(cfg *CLIConfig) error { return nil } +func (bs *BatcherService) initDA(cfg *CLIConfig) error { + client, err := celestia.NewDAClient(cfg.DaConfig.DaRpc) + if err != nil { + return err + } + bs.DAClient = client + return nil +} + // Start runs once upon start of the batcher lifecycle, // and starts batch-submission work if the batcher is configured to start submit data on startup. func (bs *BatcherService) Start(_ context.Context) error { diff --git a/op-batcher/flags/flags.go b/op-batcher/flags/flags.go index 3fe66f33981b..5a1303a94b0a 100644 --- a/op-batcher/flags/flags.go +++ b/op-batcher/flags/flags.go @@ -10,6 +10,7 @@ import ( altda "github.com/ethereum-optimism/optimism/op-alt-da" "github.com/ethereum-optimism/optimism/op-batcher/compressor" + celestia "github.com/ethereum-optimism/optimism/op-celestia" "github.com/ethereum-optimism/optimism/op-node/rollup/derive" opservice "github.com/ethereum-optimism/optimism/op-service" openum "github.com/ethereum-optimism/optimism/op-service/enum" @@ -43,7 +44,6 @@ var ( Usage: "HTTP provider URL for Rollup node. A comma-separated list enables the active L2 endpoint provider. Such a list needs to match the number of l2-eth-rpcs provided.", EnvVars: prefixEnvVars("ROLLUP_RPC"), } - // Optional flags SubSafetyMarginFlag = &cli.Uint64Flag{ Name: "sub-safety-margin", Usage: "The batcher tx submission safety margin (in #L1-blocks) to subtract " + @@ -193,6 +193,7 @@ func init() { optionalFlags = append(optionalFlags, oppprof.CLIFlags(EnvVarPrefix)...) optionalFlags = append(optionalFlags, txmgr.CLIFlags(EnvVarPrefix)...) optionalFlags = append(optionalFlags, altda.CLIFlags(EnvVarPrefix, "")...) + optionalFlags = append(optionalFlags, celestia.CLIFlags(EnvVarPrefix)...) Flags = append(requiredFlags, optionalFlags...) } diff --git a/op-celestia/cli.go b/op-celestia/cli.go new file mode 100644 index 000000000000..7ea11148d094 --- /dev/null +++ b/op-celestia/cli.go @@ -0,0 +1,94 @@ +package celestia + +import ( + "errors" + "fmt" + "net" + + "github.com/urfave/cli/v2" + + opservice "github.com/ethereum-optimism/optimism/op-service" +) + +const ( + DaRpcFlagName = "da.rpc" +) + +var ( + defaultDaRpc = "localhost:26650" + + ErrInvalidPort = errors.New("invalid port") +) + +func Check(address string) error { + _, port, err := net.SplitHostPort(address) + if err != nil { + return err + } + + if port == "" { + return ErrInvalidPort + } + + _, err = net.LookupPort("tcp", port) + if err != nil { + return err + } + + return nil +} + +func CLIFlags(envPrefix string) []cli.Flag { + return []cli.Flag{ + &cli.StringFlag{ + Name: DaRpcFlagName, + Usage: "dial address of data availability grpc client", + Value: defaultDaRpc, + EnvVars: opservice.PrefixEnvVar(envPrefix, "DA_RPC"), + }, + } +} + +type Config struct { + DaRpc string +} + +func (c Config) Check() error { + if c.DaRpc == "" { + c.DaRpc = defaultDaRpc + } + + if err := Check(c.DaRpc); err != nil { + return fmt.Errorf("invalid da rpc: %w", err) + } + + return nil +} + +type CLIConfig struct { + DaRpc string +} + +func (c CLIConfig) Check() error { + if c.DaRpc == "" { + c.DaRpc = defaultDaRpc + } + + if err := Check(c.DaRpc); err != nil { + return fmt.Errorf("invalid da rpc: %w", err) + } + + return nil +} + +func NewCLIConfig() CLIConfig { + return CLIConfig{ + DaRpc: defaultDaRpc, + } +} + +func ReadCLIConfig(ctx *cli.Context) CLIConfig { + return CLIConfig{ + DaRpc: ctx.String(DaRpcFlagName), + } +} diff --git a/op-celestia/da.go b/op-celestia/da.go new file mode 100644 index 000000000000..feecb3a82fad --- /dev/null +++ b/op-celestia/da.go @@ -0,0 +1,10 @@ +package celestia + +// DerivationVersionCelestia is a byte marker for celestia references submitted +// to the batch inbox address as calldata. +// Mnemonic 0xce = celestia +// version 0xce references are encoded as: +// [8]byte block height ++ [32]byte commitment +// in little-endian encoding. +// see: https://github.com/rollkit/celestia-da/blob/1f2df375fd2fcc59e425a50f7eb950daa5382ef0/celestia.go#L141-L160 +const DerivationVersionCelestia = 0xce diff --git a/op-celestia/da_client.go b/op-celestia/da_client.go new file mode 100644 index 000000000000..14637176cb61 --- /dev/null +++ b/op-celestia/da_client.go @@ -0,0 +1,26 @@ +package celestia + +import ( + "time" + + "github.com/rollkit/go-da/proxy" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +type DAClient struct { + Client *proxy.Client + GetTimeout time.Duration +} + +func NewDAClient(rpc string) (*DAClient, error) { + client := proxy.NewClient() + err := client.Start(rpc, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + return nil, err + } + return &DAClient{ + Client: client, + GetTimeout: time.Minute, + }, nil +} diff --git a/op-e2e/system/e2esys/setup.go b/op-e2e/system/e2esys/setup.go index fc557aa1a932..1927a526f634 100644 --- a/op-e2e/system/e2esys/setup.go +++ b/op-e2e/system/e2esys/setup.go @@ -43,6 +43,7 @@ import ( altda "github.com/ethereum-optimism/optimism/op-alt-da" bss "github.com/ethereum-optimism/optimism/op-batcher/batcher" batcherFlags "github.com/ethereum-optimism/optimism/op-batcher/flags" + celestia "github.com/ethereum-optimism/optimism/op-celestia" "github.com/ethereum-optimism/optimism/op-chain-ops/genesis" "github.com/ethereum-optimism/optimism/op-e2e/config" "github.com/ethereum-optimism/optimism/op-e2e/e2eutils" @@ -151,6 +152,7 @@ func DefaultSystemConfig(t testing.TB, opts ...SystemConfigOpt) SystemConfig { RuntimeConfigReloadInterval: time.Minute * 10, ConfigPersistence: &rollupNode.DisabledConfigPersistence{}, Sync: sync.Config{SyncMode: sync.CLSync}, + DaConfig: celestia.Config{DaRpc: "localhost:26650"}, }, RoleVerif: { Driver: driver.Config{ @@ -167,6 +169,7 @@ func DefaultSystemConfig(t testing.TB, opts ...SystemConfigOpt) SystemConfig { RuntimeConfigReloadInterval: time.Minute * 10, ConfigPersistence: &rollupNode.DisabledConfigPersistence{}, Sync: sync.Config{SyncMode: sync.CLSync}, + DaConfig: celestia.Config{DaRpc: "localhost:26650"}, }, }, Loggers: map[string]log.Logger{ @@ -904,6 +907,7 @@ func (cfg SystemConfig) Start(t *testing.T, startOpts ...StartOption) (*System, DataAvailabilityType: sys.Cfg.DataAvailabilityType, CompressionAlgo: compressionAlgo, AltDA: batcherAltDACLIConfig, + DaConfig: celestia.CLIConfig{DaRpc: "localhost:26650"}, } // Batch Submitter batcher, err := bss.BatcherServiceFromCLIConfig(context.Background(), "0.0.1", batcherCLIConfig, sys.Cfg.Loggers["batcher"]) diff --git a/op-node/flags/flags.go b/op-node/flags/flags.go index 54334c150296..8cf85a49c3a1 100644 --- a/op-node/flags/flags.go +++ b/op-node/flags/flags.go @@ -7,6 +7,7 @@ import ( "github.com/urfave/cli/v2" altda "github.com/ethereum-optimism/optimism/op-alt-da" + celestia "github.com/ethereum-optimism/optimism/op-celestia" "github.com/ethereum-optimism/optimism/op-node/rollup/engine" "github.com/ethereum-optimism/optimism/op-node/rollup/sync" openum "github.com/ethereum-optimism/optimism/op-service/enum" @@ -441,6 +442,7 @@ func init() { optionalFlags = append(optionalFlags, DeprecatedFlags...) optionalFlags = append(optionalFlags, opflags.CLIFlags(EnvVarPrefix, RollupCategory)...) optionalFlags = append(optionalFlags, altda.CLIFlags(EnvVarPrefix, AltDACategory)...) + optionalFlags = append(optionalFlags, celestia.CLIFlags(EnvVarPrefix)...) Flags = append(requiredFlags, optionalFlags...) } diff --git a/op-node/node/config.go b/op-node/node/config.go index a78b55853aa2..248349158977 100644 --- a/op-node/node/config.go +++ b/op-node/node/config.go @@ -8,6 +8,7 @@ import ( "time" altda "github.com/ethereum-optimism/optimism/op-alt-da" + celestia "github.com/ethereum-optimism/optimism/op-celestia" "github.com/ethereum-optimism/optimism/op-node/flags" "github.com/ethereum-optimism/optimism/op-node/p2p" "github.com/ethereum-optimism/optimism/op-node/rollup" @@ -74,6 +75,8 @@ type Config struct { // AltDA config AltDA altda.CLIConfig + + DaConfig celestia.Config } type RPCConfig struct { @@ -177,6 +180,9 @@ func (cfg *Config) Check() error { if cfg.AltDA.Enabled { log.Warn("Alt-DA Mode is a Beta feature of the MIT licensed OP Stack. While it has received initial review from core contributors, it is still undergoing testing, and may have bugs or other issues.") } + if err := cfg.DaConfig.Check(); err != nil { + return fmt.Errorf("da config error: %w", err) + } return nil } diff --git a/op-node/node/node.go b/op-node/node/node.go index 298c98aa2b18..7c38df251e9e 100644 --- a/op-node/node/node.go +++ b/op-node/node/node.go @@ -139,6 +139,9 @@ func (n *OpNode) init(ctx context.Context, cfg *Config) error { if err := n.initL2(ctx, cfg); err != nil { return fmt.Errorf("failed to init L2: %w", err) } + if err := n.initDA(ctx, cfg); err != nil { + return fmt.Errorf("failed to init da: %w", err) + } if err := n.initRuntimeConfig(ctx, cfg); err != nil { // depends on L2, to signal initial runtime values to return fmt.Errorf("failed to init the runtime config: %w", err) } @@ -378,6 +381,10 @@ func (n *OpNode) initL1BeaconAPI(ctx context.Context, cfg *Config) error { } } +func (n *OpNode) initDA(ctx context.Context, cfg *Config) error { + return driver.SetDAClient(cfg.DaConfig) +} + func (n *OpNode) initL2(ctx context.Context, cfg *Config) error { rpcClient, rpcCfg, err := cfg.L2.Setup(ctx, n.log, &cfg.Rollup) if err != nil { diff --git a/op-node/rollup/derive/blob_data_source.go b/op-node/rollup/derive/blob_data_source.go index 2c4626941b8b..95e7248464dd 100644 --- a/op-node/rollup/derive/blob_data_source.go +++ b/op-node/rollup/derive/blob_data_source.go @@ -86,7 +86,10 @@ func (ds *BlobDataSource) open(ctx context.Context) ([]blobOrCalldata, error) { return nil, NewTemporaryError(fmt.Errorf("failed to open blob data source: %w", err)) } - data, hashes := dataAndHashesFromTxs(txs, &ds.dsCfg, ds.batcherAddr, ds.log) + data, hashes, err := dataAndHashesFromTxs(txs, &ds.dsCfg, ds.batcherAddr, ds.log) + if err != nil { + return nil, err + } if len(hashes) == 0 { // there are no blobs to fetch so we can return immediately @@ -115,11 +118,12 @@ func (ds *BlobDataSource) open(ctx context.Context) ([]blobOrCalldata, error) { // dataAndHashesFromTxs extracts calldata and datahashes from the input transactions and returns them. It // creates a placeholder blobOrCalldata element for each returned blob hash that must be populated // by fillBlobPointers after blob bodies are retrieved. -func dataAndHashesFromTxs(txs types.Transactions, config *DataSourceConfig, batcherAddr common.Address, logger log.Logger) ([]blobOrCalldata, []eth.IndexedBlobHash) { +func dataAndHashesFromTxs(txs types.Transactions, config *DataSourceConfig, batcherAddr common.Address, logger log.Logger) ([]blobOrCalldata, []eth.IndexedBlobHash, error) { data := []blobOrCalldata{} var hashes []eth.IndexedBlobHash blobIndex := 0 // index of each blob in the block's blob sidecar for _, tx := range txs { + logger := log.New("tx", tx.Hash()) // skip any non-batcher transactions if !isValidBatchTx(tx, config.l1Signer, config.batchInboxAddress, batcherAddr, logger) { blobIndex += len(tx.BlobHashes()) @@ -127,8 +131,15 @@ func dataAndHashesFromTxs(txs types.Transactions, config *DataSourceConfig, batc } // handle non-blob batcher transactions by extracting their calldata if tx.Type() != types.BlobTxType { - calldata := eth.Data(tx.Data()) - data = append(data, blobOrCalldata{nil, &calldata}) + calldata, err := DataFromEVMTransactions(*config, batcherAddr, types.Transactions{tx}, logger) + if err != nil { + return nil, nil, err + } + if len(calldata) == 0 { + log.Warn("celestia: skipping empty calldata") + continue + } + data = append(data, blobOrCalldata{nil, &calldata[0]}) continue } // handle blob batcher transactions by extracting their blob hashes, ignoring any calldata. @@ -145,7 +156,7 @@ func dataAndHashesFromTxs(txs types.Transactions, config *DataSourceConfig, batc blobIndex += 1 } } - return data, hashes + return data, hashes, nil } // fillBlobPointers goes back through the data array and fills in the pointers to the fetched blob diff --git a/op-node/rollup/derive/blob_data_source_test.go b/op-node/rollup/derive/blob_data_source_test.go index e5e31dc957b9..885fbdf0684d 100644 --- a/op-node/rollup/derive/blob_data_source_test.go +++ b/op-node/rollup/derive/blob_data_source_test.go @@ -45,7 +45,8 @@ func TestDataAndHashesFromTxs(t *testing.T) { } calldataTx, _ := types.SignNewTx(privateKey, signer, txData) txs := types.Transactions{calldataTx} - data, blobHashes := dataAndHashesFromTxs(txs, &config, batcherAddr, logger) + data, blobHashes, err := dataAndHashesFromTxs(txs, &config, batcherAddr, logger) + require.NoError(t, err) require.Equal(t, 1, len(data)) require.Equal(t, 0, len(blobHashes)) @@ -60,14 +61,16 @@ func TestDataAndHashesFromTxs(t *testing.T) { } blobTx, _ := types.SignNewTx(privateKey, signer, blobTxData) txs = types.Transactions{blobTx} - data, blobHashes = dataAndHashesFromTxs(txs, &config, batcherAddr, logger) + data, blobHashes, err = dataAndHashesFromTxs(txs, &config, batcherAddr, logger) + require.NoError(t, err) require.Equal(t, 1, len(data)) require.Equal(t, 1, len(blobHashes)) require.Nil(t, data[0].calldata) // try again with both the blob & calldata transactions and make sure both are picked up txs = types.Transactions{blobTx, calldataTx} - data, blobHashes = dataAndHashesFromTxs(txs, &config, batcherAddr, logger) + data, blobHashes, err = dataAndHashesFromTxs(txs, &config, batcherAddr, logger) + require.NoError(t, err) require.Equal(t, 2, len(data)) require.Equal(t, 1, len(blobHashes)) require.NotNil(t, data[1].calldata) @@ -75,7 +78,8 @@ func TestDataAndHashesFromTxs(t *testing.T) { // make sure blob tx to the batch inbox is ignored if not signed by the batcher blobTx, _ = types.SignNewTx(testutils.RandomKey(), signer, blobTxData) txs = types.Transactions{blobTx} - data, blobHashes = dataAndHashesFromTxs(txs, &config, batcherAddr, logger) + data, blobHashes, err = dataAndHashesFromTxs(txs, &config, batcherAddr, logger) + require.NoError(t, err) require.Equal(t, 0, len(data)) require.Equal(t, 0, len(blobHashes)) @@ -84,7 +88,8 @@ func TestDataAndHashesFromTxs(t *testing.T) { blobTxData.To = testutils.RandomAddress(rng) blobTx, _ = types.SignNewTx(privateKey, signer, blobTxData) txs = types.Transactions{blobTx} - data, blobHashes = dataAndHashesFromTxs(txs, &config, batcherAddr, logger) + data, blobHashes, err = dataAndHashesFromTxs(txs, &config, batcherAddr, logger) + require.NoError(t, err) require.Equal(t, 0, len(data)) require.Equal(t, 0, len(blobHashes)) } diff --git a/op-node/rollup/derive/calldata_source.go b/op-node/rollup/derive/calldata_source.go index 0e8147261e93..7fe415ae7cf6 100644 --- a/op-node/rollup/derive/calldata_source.go +++ b/op-node/rollup/derive/calldata_source.go @@ -2,6 +2,7 @@ package derive import ( "context" + "encoding/hex" "errors" "fmt" "io" @@ -11,9 +12,20 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" + celestia "github.com/ethereum-optimism/optimism/op-celestia" "github.com/ethereum-optimism/optimism/op-service/eth" ) +var daClient *celestia.DAClient + +func SetDAClient(c *celestia.DAClient) error { + if daClient != nil { + return errors.New("da client already configured") + } + daClient = c + return nil +} + // CalldataSource is a fault tolerant approach to fetching data. // The constructor will never fail & it will instead re-attempt the fetcher // at a later point. @@ -32,7 +44,7 @@ type CalldataSource struct { // NewCalldataSource creates a new calldata source. It suppresses errors in fetching the L1 block if they occur. // If there is an error, it will attempt to fetch the result on the next call to `Next`. -func NewCalldataSource(ctx context.Context, log log.Logger, dsCfg DataSourceConfig, fetcher L1TransactionFetcher, ref eth.L1BlockRef, batcherAddr common.Address) DataIter { +func NewCalldataSource(ctx context.Context, log log.Logger, dsCfg DataSourceConfig, fetcher L1TransactionFetcher, ref eth.L1BlockRef, batcherAddr common.Address) (DataIter, error) { _, txs, err := fetcher.InfoAndTxsByHash(ctx, ref.Hash) if err != nil { return &CalldataSource{ @@ -42,12 +54,23 @@ func NewCalldataSource(ctx context.Context, log log.Logger, dsCfg DataSourceConf fetcher: fetcher, log: log, batcherAddr: batcherAddr, - } + }, nil + } + data, err := DataFromEVMTransactions(dsCfg, batcherAddr, txs, log.New("origin", ref)) + if err != nil { + return &CalldataSource{ + open: false, + ref: ref, + dsCfg: dsCfg, + fetcher: fetcher, + log: log, + batcherAddr: batcherAddr, + }, err } return &CalldataSource{ open: true, - data: DataFromEVMTransactions(dsCfg, batcherAddr, txs, log.New("origin", ref)), - } + data: data, + }, nil } // Next returns the next piece of data if it has it. If the constructor failed, this @@ -57,7 +80,11 @@ func (ds *CalldataSource) Next(ctx context.Context) (eth.Data, error) { if !ds.open { if _, txs, err := ds.fetcher.InfoAndTxsByHash(ctx, ds.ref.Hash); err == nil { ds.open = true - ds.data = DataFromEVMTransactions(ds.dsCfg, ds.batcherAddr, txs, ds.log) + ds.data, err = DataFromEVMTransactions(ds.dsCfg, ds.batcherAddr, txs, ds.log) + if err != nil { + // already wrapped + return nil, err + } } else if errors.Is(err, ethereum.NotFound) { return nil, NewResetError(fmt.Errorf("failed to open calldata source: %w", err)) } else { @@ -76,12 +103,38 @@ func (ds *CalldataSource) Next(ctx context.Context) (eth.Data, error) { // DataFromEVMTransactions filters all of the transactions and returns the calldata from transactions // that are sent to the batch inbox address from the batch sender address. // This will return an empty array if no valid transactions are found. -func DataFromEVMTransactions(dsCfg DataSourceConfig, batcherAddr common.Address, txs types.Transactions, log log.Logger) []eth.Data { +func DataFromEVMTransactions(dsCfg DataSourceConfig, batcherAddr common.Address, txs types.Transactions, log log.Logger) ([]eth.Data, error) { out := []eth.Data{} for _, tx := range txs { if isValidBatchTx(tx, dsCfg.l1Signer, dsCfg.batchInboxAddress, batcherAddr, log) { - out = append(out, tx.Data()) + data := tx.Data() + switch len(data) { + case 0: + out = append(out, data) + default: + switch data[0] { + case celestia.DerivationVersionCelestia: + log.Info("celestia: blob request", "id", hex.EncodeToString(tx.Data())) + ctx, cancel := context.WithTimeout(context.Background(), daClient.GetTimeout) + blobs, err := daClient.Client.Get(ctx, [][]byte{data[1:]}) + cancel() + if err != nil { + return nil, NewResetError(fmt.Errorf("celestia: failed to resolve frame: %w", err)) + } + if len(blobs) != 1 { + log.Warn("celestia: unexpected length for blobs", "expected", 1, "got", len(blobs)) + if len(blobs) == 0 { + log.Warn("celestia: skipping empty blobs") + continue + } + } + out = append(out, blobs[0]) + default: + out = append(out, data) + log.Info("celestia: using eth fallback") + } + } } } - return out + return out, nil } diff --git a/op-node/rollup/derive/calldata_source_test.go b/op-node/rollup/derive/calldata_source_test.go index 01b2616cca3f..7dc17c6aa19f 100644 --- a/op-node/rollup/derive/calldata_source_test.go +++ b/op-node/rollup/derive/calldata_source_test.go @@ -121,8 +121,10 @@ func TestDataFromEVMTransactions(t *testing.T) { } } - out := DataFromEVMTransactions(DataSourceConfig{cfg.L1Signer(), cfg.BatchInboxAddress, false}, batcherAddr, txs, testlog.Logger(t, log.LevelCrit)) + out, err := DataFromEVMTransactions(DataSourceConfig{cfg.L1Signer(), cfg.BatchInboxAddress, false}, batcherAddr, txs, testlog.Logger(t, log.LevelCrit)) + require.NoError(t, err) require.ElementsMatch(t, expectedData, out) + require.NoError(t, err) } } diff --git a/op-node/rollup/derive/data_source.go b/op-node/rollup/derive/data_source.go index 8d064a7cdb8c..50f71467f232 100644 --- a/op-node/rollup/derive/data_source.go +++ b/op-node/rollup/derive/data_source.go @@ -68,13 +68,17 @@ func (ds *DataSourceFactory) OpenData(ctx context.Context, ref eth.L1BlockRef, b // Creates a data iterator from blob or calldata source so we can forward it to the altDA source // if enabled as it still requires an L1 data source for fetching input commmitments. var src DataIter + var err error if ds.ecotoneTime != nil && ref.Time >= *ds.ecotoneTime { if ds.blobsFetcher == nil { return nil, fmt.Errorf("ecotone upgrade active but beacon endpoint not configured") } src = NewBlobDataSource(ctx, ds.log, ds.dsCfg, ds.fetcher, ds.blobsFetcher, ref, batcherAddr) } else { - src = NewCalldataSource(ctx, ds.log, ds.dsCfg, ds.fetcher, ref, batcherAddr) + src, err = NewCalldataSource(ctx, ds.log, ds.dsCfg, ds.fetcher, ref, batcherAddr) + if err != nil { + return src, err + } } if ds.dsCfg.altDAEnabled { // altDA([calldata | blobdata](l1Ref)) -> data diff --git a/op-node/rollup/driver/da.go b/op-node/rollup/driver/da.go new file mode 100644 index 000000000000..89d715c408c9 --- /dev/null +++ b/op-node/rollup/driver/da.go @@ -0,0 +1,14 @@ +package driver + +import ( + celestia "github.com/ethereum-optimism/optimism/op-celestia" + "github.com/ethereum-optimism/optimism/op-node/rollup/derive" +) + +func SetDAClient(cfg celestia.Config) error { + client, err := celestia.NewDAClient(cfg.DaRpc) + if err != nil { + return err + } + return derive.SetDAClient(client) +} diff --git a/op-node/service.go b/op-node/service.go index b24e2a638335..a3583d39a61d 100644 --- a/op-node/service.go +++ b/op-node/service.go @@ -18,6 +18,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/urfave/cli/v2" + celestia "github.com/ethereum-optimism/optimism/op-celestia" "github.com/ethereum-optimism/optimism/op-node/flags" "github.com/ethereum-optimism/optimism/op-node/node" p2pcli "github.com/ethereum-optimism/optimism/op-node/p2p/cli" @@ -112,7 +113,8 @@ func NewConfig(ctx *cli.Context, log log.Logger) (*node.Config, error) { ConductorRpc: ctx.String(flags.ConductorRpcFlag.Name), ConductorRpcTimeout: ctx.Duration(flags.ConductorRpcTimeoutFlag.Name), - AltDA: altda.ReadCLIConfig(ctx), + AltDA: altda.ReadCLIConfig(ctx), + DaConfig: celestia.Config(celestia.ReadCLIConfig(ctx)), } if err := cfg.LoadPersisted(log); err != nil { diff --git a/ops-bedrock/docker-compose.yml b/ops-bedrock/docker-compose.yml index adcaea8f4d1b..2c84b0ffa8f8 100644 --- a/ops-bedrock/docker-compose.yml +++ b/ops-bedrock/docker-compose.yml @@ -14,6 +14,18 @@ volumes: services: + da: + image: ghcr.io/rollkit/local-celestia-devnet:v0.12.7 + ports: + - "26650:26650" + environment: + CELESTIA_NAMESPACE: 000008e5f679bf7116cb + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:26659/header/1"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s l1: build: @@ -94,6 +106,7 @@ services: - l1-bn - l1-vc - l2 + - da build: context: ../ dockerfile: ops/docker/op-stack-go/Dockerfile @@ -102,9 +115,7 @@ services: command: > op-node --l1=ws://l1:8546 - --l1.beacon=http://l1-bn:5052 - --l1.epoch-poll-interval=12s - --l1.http-poll-interval=6s + --l1.beacon.ignore --l2=http://l2:8551 --l2.jwt-secret=/config/jwt-secret.txt --sequencer.enabled @@ -129,6 +140,8 @@ services: --altda.enabled=${ALTDA_ENABLED} --altda.da-service=${ALTDA_SERVICE} --altda.da-server=http://da-server:3100 + environment: + OP_NODE_DA_RPC: "da:26650" ports: - "7545:8545" - "9003:9003" @@ -177,6 +190,8 @@ services: - l1 - l2 - op-node + da: + condition: service_healthy build: context: ../ dockerfile: ops/docker/op-stack-go/Dockerfile @@ -190,7 +205,8 @@ services: OP_BATCHER_L1_ETH_RPC: http://l1:8545 OP_BATCHER_L2_ETH_RPC: http://l2:8545 OP_BATCHER_ROLLUP_RPC: http://op-node:8545 - OP_BATCHER_MAX_CHANNEL_DURATION: 2 + OFFLINE_GAS_ESTIMATION: null + OP_BATCHER_MAX_CHANNEL_DURATION: 1 OP_BATCHER_SUB_SAFETY_MARGIN: 4 # SWS is 15, ChannelTimeout is 40 OP_BATCHER_POLL_INTERVAL: 1s OP_BATCHER_NUM_CONFIRMATIONS: 1 @@ -203,7 +219,9 @@ services: OP_BATCHER_ALTDA_ENABLED: "${ALTDA_ENABLED}" OP_BATCHER_ALTDA_DA_SERVICE: "${ALTDA_SERVICE}" OP_BATCHER_ALTDA_DA_SERVER: "http://da-server:3100" - OP_BATCHER_DATA_AVAILABILITY_TYPE: "${DA_TYPE}" + OP_BATCHER_DA_RPC: "http://da:26658" + OP_BATCHER_DA_AUTH_TOKEN: "${CELESTIA_NODE_AUTH_TOKEN}" + OP_BATCHER_DA_NAMESPACE: "00000000000000000000000000000000000000000008e5f679bf7116cb" op-challenger: depends_on: diff --git a/ops/docker/op-stack-go/Dockerfile.dockerignore b/ops/docker/op-stack-go/Dockerfile.dockerignore index 1c0841df1f33..c26cffc94db1 100644 --- a/ops/docker/op-stack-go/Dockerfile.dockerignore +++ b/ops/docker/op-stack-go/Dockerfile.dockerignore @@ -5,6 +5,7 @@ !/cannon !/op-batcher !/op-bootnode +!/op-celestia !/op-chain-ops !/op-challenger !/packages/contracts-bedrock/snapshots From 87899983fe94f18e99424423f91ae288c442f155 Mon Sep 17 00:00:00 2001 From: Javed Khan Date: Mon, 25 Mar 2024 15:45:17 -0700 Subject: [PATCH 02/11] da: upgrade to go-da v0.5.0 --- bedrock-devnet/devnet/__init__.py | 14 ++-- go.mod | 1 + op-batcher/batcher/driver.go | 2 +- op-batcher/batcher/service.go | 2 +- op-celestia/cli.go | 91 +++++++++--------------- op-celestia/da_client.go | 17 +++-- op-e2e/system/e2esys/setup.go | 4 +- op-node/node/config.go | 2 +- op-node/rollup/derive/calldata_source.go | 2 +- op-node/rollup/driver/da.go | 4 +- op-node/service.go | 2 +- ops-bedrock/docker-compose.yml | 13 ++-- 12 files changed, 69 insertions(+), 85 deletions(-) diff --git a/bedrock-devnet/devnet/__init__.py b/bedrock-devnet/devnet/__init__.py index 400923876197..22c9f5fb4326 100644 --- a/bedrock-devnet/devnet/__init__.py +++ b/bedrock-devnet/devnet/__init__.py @@ -283,14 +283,20 @@ def devnet_deploy(paths): docker_env['ALTDA_GENERIC_DA'] = 'false' docker_env['ALTDA_SERVICE'] = 'false' + log.info('Bringing up DA') + run_command(['docker', 'compose', 'up', '-d', 'da'], cwd=paths.ops_bedrock_dir, env=docker_env) + wait_up(26658) + time.sleep(15) + celestia_node_auth_token = run_command([ + 'docker', 'compose', 'exec', '--no-TTY', 'da', 'celestia', 'bridge', 'auth', 'admin', '--node.store', '/home/celestia/bridge' + ], cwd=paths.ops_bedrock_dir, env=docker_env, capture_output=True).stdout.decode().strip() + docker_env['CELESTIA_NODE_AUTH_TOKEN'] = celestia_node_auth_token + print('CELESTIA_NODE_AUTH_TOKEN: ', celestia_node_auth_token) + # Bring up the rest of the services. log.info('Bringing up `op-node`, `op-proposer` and `op-batcher`.') run_command(['docker', 'compose', 'up', '-d', 'op-node', 'op-proposer', 'op-batcher', 'artifact-server'], cwd=paths.ops_bedrock_dir, env=docker_env) - log.info('Bringing up DA') - run_command(['docker', 'compose', 'up', '-d', 'da'], cwd=paths.ops_bedrock_dir, env=docker_env) - wait_up(26650) - # Optionally bring up op-challenger. if not DEVNET_L2OO: log.info('Bringing up `op-challenger`.') diff --git a/go.mod b/go.mod index 8cddae9d1f98..a2b90b4441e1 100644 --- a/go.mod +++ b/go.mod @@ -247,6 +247,7 @@ require ( google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect + google.golang.org/grpc v1.60.1 // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/op-batcher/batcher/driver.go b/op-batcher/batcher/driver.go index 0b7eb8d5d2e8..8b08631466ef 100644 --- a/op-batcher/batcher/driver.go +++ b/op-batcher/batcher/driver.go @@ -761,7 +761,7 @@ func (l *BatchSubmitter) blobTxCandidate(data txData) (*txmgr.TxCandidate, error func (l *BatchSubmitter) calldataTxCandidate(data []byte) *txmgr.TxCandidate { l.Log.Info("Building Calldata transaction candidate", "size", len(data)) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Duration(l.RollupConfig.BlockTime)*time.Second) - ids, _, err := l.DAClient.Client.Submit(ctx, [][]byte{data}, -1) + ids, err := l.DAClient.Client.Submit(ctx, [][]byte{data}, -1, l.DAClient.Namespace) cancel() if err == nil && len(ids) == 1 { l.Log.Info("celestia: blob successfully submitted", "id", hex.EncodeToString(ids[0])) diff --git a/op-batcher/batcher/service.go b/op-batcher/batcher/service.go index 173a21a658b7..9cd2caaf392f 100644 --- a/op-batcher/batcher/service.go +++ b/op-batcher/batcher/service.go @@ -377,7 +377,7 @@ func (bs *BatcherService) initAltDA(cfg *CLIConfig) error { } func (bs *BatcherService) initDA(cfg *CLIConfig) error { - client, err := celestia.NewDAClient(cfg.DaConfig.DaRpc) + client, err := celestia.NewDAClient(cfg.DaConfig.Rpc, cfg.DaConfig.AuthToken, cfg.DaConfig.Namespace) if err != nil { return err } diff --git a/op-celestia/cli.go b/op-celestia/cli.go index 7ea11148d094..4c2e19145d9d 100644 --- a/op-celestia/cli.go +++ b/op-celestia/cli.go @@ -1,94 +1,67 @@ package celestia import ( - "errors" - "fmt" - "net" - "github.com/urfave/cli/v2" opservice "github.com/ethereum-optimism/optimism/op-service" ) const ( - DaRpcFlagName = "da.rpc" -) - -var ( - defaultDaRpc = "localhost:26650" - - ErrInvalidPort = errors.New("invalid port") + // RPCFlagName defines the flag for the rpc url + RPCFlagName = "da.rpc" + // AuthTokenFlagName defines the flag for the auth token + AuthTokenFlagName = "da.auth_token" + // NamespaceFlagName defines the flag for the namespace + NamespaceFlagName = "da.namespace" + + // NamespaceSize is the size of the hex encoded namespace string + NamespaceSize = 58 + + // defaultRPC is the default rpc dial address + defaultRPC = "grpc://localhost:26650" ) -func Check(address string) error { - _, port, err := net.SplitHostPort(address) - if err != nil { - return err - } - - if port == "" { - return ErrInvalidPort - } - - _, err = net.LookupPort("tcp", port) - if err != nil { - return err - } - - return nil -} - func CLIFlags(envPrefix string) []cli.Flag { return []cli.Flag{ &cli.StringFlag{ - Name: DaRpcFlagName, - Usage: "dial address of data availability grpc client", - Value: defaultDaRpc, + Name: RPCFlagName, + Usage: "dial address of the data availability rpc client; supports grpc, http, https", + Value: defaultRPC, EnvVars: opservice.PrefixEnvVar(envPrefix, "DA_RPC"), }, + &cli.StringFlag{ + Name: AuthTokenFlagName, + Usage: "authentication token of the data availability client", + EnvVars: opservice.PrefixEnvVar(envPrefix, "DA_AUTH_TOKEN"), + }, + &cli.StringFlag{ + Name: NamespaceFlagName, + Usage: "namespace of the data availability client", + EnvVars: opservice.PrefixEnvVar(envPrefix, "DA_NAMESPACE"), + }, } } -type Config struct { - DaRpc string -} - -func (c Config) Check() error { - if c.DaRpc == "" { - c.DaRpc = defaultDaRpc - } - - if err := Check(c.DaRpc); err != nil { - return fmt.Errorf("invalid da rpc: %w", err) - } - - return nil -} - type CLIConfig struct { - DaRpc string + Rpc string + AuthToken string + Namespace string } func (c CLIConfig) Check() error { - if c.DaRpc == "" { - c.DaRpc = defaultDaRpc - } - - if err := Check(c.DaRpc); err != nil { - return fmt.Errorf("invalid da rpc: %w", err) - } - return nil } func NewCLIConfig() CLIConfig { return CLIConfig{ - DaRpc: defaultDaRpc, + Rpc: defaultRPC, } } func ReadCLIConfig(ctx *cli.Context) CLIConfig { return CLIConfig{ - DaRpc: ctx.String(DaRpcFlagName), + Rpc: ctx.String(RPCFlagName), + AuthToken: ctx.String(AuthTokenFlagName), + Namespace: ctx.String(NamespaceFlagName), } } diff --git a/op-celestia/da_client.go b/op-celestia/da_client.go index 14637176cb61..8cf70df37644 100644 --- a/op-celestia/da_client.go +++ b/op-celestia/da_client.go @@ -1,26 +1,31 @@ package celestia import ( + "encoding/hex" "time" + "github.com/rollkit/go-da" "github.com/rollkit/go-da/proxy" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" ) type DAClient struct { - Client *proxy.Client + Client da.DA GetTimeout time.Duration + Namespace da.Namespace } -func NewDAClient(rpc string) (*DAClient, error) { - client := proxy.NewClient() - err := client.Start(rpc, grpc.WithTransportCredentials(insecure.NewCredentials())) +func NewDAClient(rpc, token, namespace string) (*DAClient, error) { + client, err := proxy.NewClient(rpc, token) + if err != nil { + return nil, err + } + ns, err := hex.DecodeString(namespace) if err != nil { return nil, err } return &DAClient{ Client: client, GetTimeout: time.Minute, + Namespace: ns, }, nil } diff --git a/op-e2e/system/e2esys/setup.go b/op-e2e/system/e2esys/setup.go index 1927a526f634..b39346b14cff 100644 --- a/op-e2e/system/e2esys/setup.go +++ b/op-e2e/system/e2esys/setup.go @@ -152,7 +152,7 @@ func DefaultSystemConfig(t testing.TB, opts ...SystemConfigOpt) SystemConfig { RuntimeConfigReloadInterval: time.Minute * 10, ConfigPersistence: &rollupNode.DisabledConfigPersistence{}, Sync: sync.Config{SyncMode: sync.CLSync}, - DaConfig: celestia.Config{DaRpc: "localhost:26650"}, + DaConfig: celestia.CLIConfig{Rpc: "grpc://localhost:26650"}, }, RoleVerif: { Driver: driver.Config{ @@ -169,7 +169,7 @@ func DefaultSystemConfig(t testing.TB, opts ...SystemConfigOpt) SystemConfig { RuntimeConfigReloadInterval: time.Minute * 10, ConfigPersistence: &rollupNode.DisabledConfigPersistence{}, Sync: sync.Config{SyncMode: sync.CLSync}, - DaConfig: celestia.Config{DaRpc: "localhost:26650"}, + DaConfig: celestia.CLIConfig{Rpc: "grpc://localhost:26650"}, }, }, Loggers: map[string]log.Logger{ diff --git a/op-node/node/config.go b/op-node/node/config.go index 248349158977..31770295f21d 100644 --- a/op-node/node/config.go +++ b/op-node/node/config.go @@ -76,7 +76,7 @@ type Config struct { // AltDA config AltDA altda.CLIConfig - DaConfig celestia.Config + DaConfig celestia.CLIConfig } type RPCConfig struct { diff --git a/op-node/rollup/derive/calldata_source.go b/op-node/rollup/derive/calldata_source.go index 7fe415ae7cf6..ab95b06d54a8 100644 --- a/op-node/rollup/derive/calldata_source.go +++ b/op-node/rollup/derive/calldata_source.go @@ -116,7 +116,7 @@ func DataFromEVMTransactions(dsCfg DataSourceConfig, batcherAddr common.Address, case celestia.DerivationVersionCelestia: log.Info("celestia: blob request", "id", hex.EncodeToString(tx.Data())) ctx, cancel := context.WithTimeout(context.Background(), daClient.GetTimeout) - blobs, err := daClient.Client.Get(ctx, [][]byte{data[1:]}) + blobs, err := daClient.Client.Get(ctx, [][]byte{data[1:]}, daClient.Namespace) cancel() if err != nil { return nil, NewResetError(fmt.Errorf("celestia: failed to resolve frame: %w", err)) diff --git a/op-node/rollup/driver/da.go b/op-node/rollup/driver/da.go index 89d715c408c9..dc5e5f3ceac0 100644 --- a/op-node/rollup/driver/da.go +++ b/op-node/rollup/driver/da.go @@ -5,8 +5,8 @@ import ( "github.com/ethereum-optimism/optimism/op-node/rollup/derive" ) -func SetDAClient(cfg celestia.Config) error { - client, err := celestia.NewDAClient(cfg.DaRpc) +func SetDAClient(cfg celestia.CLIConfig) error { + client, err := celestia.NewDAClient(cfg.Rpc, cfg.AuthToken, cfg.Namespace) if err != nil { return err } diff --git a/op-node/service.go b/op-node/service.go index a3583d39a61d..a2add986385f 100644 --- a/op-node/service.go +++ b/op-node/service.go @@ -114,7 +114,7 @@ func NewConfig(ctx *cli.Context, log log.Logger) (*node.Config, error) { ConductorRpcTimeout: ctx.Duration(flags.ConductorRpcTimeoutFlag.Name), AltDA: altda.ReadCLIConfig(ctx), - DaConfig: celestia.Config(celestia.ReadCLIConfig(ctx)), + DaConfig: celestia.ReadCLIConfig(ctx), } if err := cfg.LoadPersisted(log); err != nil { diff --git a/ops-bedrock/docker-compose.yml b/ops-bedrock/docker-compose.yml index 2c84b0ffa8f8..c1902e972734 100644 --- a/ops-bedrock/docker-compose.yml +++ b/ops-bedrock/docker-compose.yml @@ -15,11 +15,9 @@ volumes: services: da: - image: ghcr.io/rollkit/local-celestia-devnet:v0.12.7 + image: ghcr.io/rollkit/local-celestia-devnet:v0.13.1 ports: - - "26650:26650" - environment: - CELESTIA_NAMESPACE: 000008e5f679bf7116cb + - "26658:26658" healthcheck: test: ["CMD", "curl", "-f", "http://localhost:26659/header/1"] interval: 10s @@ -141,7 +139,9 @@ services: --altda.da-service=${ALTDA_SERVICE} --altda.da-server=http://da-server:3100 environment: - OP_NODE_DA_RPC: "da:26650" + OP_NODE_DA_RPC: "http://da:26658" + OP_NODE_DA_AUTH_TOKEN: "${CELESTIA_NODE_AUTH_TOKEN}" + OP_NODE_DA_NAMESPACE: "00000000000000000000000000000000000000000008e5f679bf7116cb" ports: - "7545:8545" - "9003:9003" @@ -190,8 +190,7 @@ services: - l1 - l2 - op-node - da: - condition: service_healthy + - da build: context: ../ dockerfile: ops/docker/op-stack-go/Dockerfile From a915368b4af78c12c686673d0eaa1af3c86fae02 Mon Sep 17 00:00:00 2001 From: Emiliano Bonassi Date: Thu, 2 May 2024 19:32:46 +0200 Subject: [PATCH 03/11] feat(op-batcher): support disable eth fallback --- op-batcher/batcher/driver.go | 19 +++++++++++++++++-- op-batcher/batcher/service.go | 2 +- op-celestia/cli.go | 23 ++++++++++++++++------- op-celestia/da_client.go | 16 +++++++++------- op-node/rollup/driver/da.go | 2 +- 5 files changed, 44 insertions(+), 18 deletions(-) diff --git a/op-batcher/batcher/driver.go b/op-batcher/batcher/driver.go index 8b08631466ef..57f4faf03e57 100644 --- a/op-batcher/batcher/driver.go +++ b/op-batcher/batcher/driver.go @@ -721,7 +721,10 @@ func (l *BatchSubmitter) sendTransaction(txdata txData, queue *txmgr.Queue[txRef if nf := len(txdata.frames); nf != 1 { l.Log.Crit("Unexpected number of frames in calldata tx", "num_frames", nf) } - candidate = l.calldataTxCandidate(txdata.CallData()) + candidate, err = l.celestiaTxCandidate(txdata.CallData()) + if err != nil { + return err + } } l.sendTx(txdata, false, candidate, queue, receiptsCh) @@ -759,6 +762,14 @@ func (l *BatchSubmitter) blobTxCandidate(data txData) (*txmgr.TxCandidate, error } func (l *BatchSubmitter) calldataTxCandidate(data []byte) *txmgr.TxCandidate { + l.Log.Info("Building Calldata transaction candidate", "size", len(data)) + return &txmgr.TxCandidate{ + To: &l.RollupConfig.BatchInboxAddress, + TxData: data, + } +} + +func (l *BatchSubmitter) celestiaTxCandidate(data []byte) (*txmgr.TxCandidate, error) { l.Log.Info("Building Calldata transaction candidate", "size", len(data)) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Duration(l.RollupConfig.BlockTime)*time.Second) ids, err := l.DAClient.Client.Submit(ctx, [][]byte{data}, -1, l.DAClient.Namespace) @@ -767,12 +778,16 @@ func (l *BatchSubmitter) calldataTxCandidate(data []byte) *txmgr.TxCandidate { l.Log.Info("celestia: blob successfully submitted", "id", hex.EncodeToString(ids[0])) data = append([]byte{celestia.DerivationVersionCelestia}, ids[0]...) } else { + if l.DAClient.EthFallbackDisabled { + return nil, fmt.Errorf("celestia: blob submission failed; eth fallback disabled: %w", err) + } + l.Log.Info("celestia: blob submission failed; falling back to eth", "err", err) } return &txmgr.TxCandidate{ To: &l.RollupConfig.BatchInboxAddress, TxData: data, - } + }, nil } func (l *BatchSubmitter) handleReceipt(r txmgr.TxReceipt[txRef]) { diff --git a/op-batcher/batcher/service.go b/op-batcher/batcher/service.go index 9cd2caaf392f..66f5260d9216 100644 --- a/op-batcher/batcher/service.go +++ b/op-batcher/batcher/service.go @@ -377,7 +377,7 @@ func (bs *BatcherService) initAltDA(cfg *CLIConfig) error { } func (bs *BatcherService) initDA(cfg *CLIConfig) error { - client, err := celestia.NewDAClient(cfg.DaConfig.Rpc, cfg.DaConfig.AuthToken, cfg.DaConfig.Namespace) + client, err := celestia.NewDAClient(cfg.DaConfig.Rpc, cfg.DaConfig.AuthToken, cfg.DaConfig.Namespace, cfg.DaConfig.EthFallbackDisabled) if err != nil { return err } diff --git a/op-celestia/cli.go b/op-celestia/cli.go index 4c2e19145d9d..40550ba5cf4f 100644 --- a/op-celestia/cli.go +++ b/op-celestia/cli.go @@ -8,11 +8,13 @@ import ( const ( // RPCFlagName defines the flag for the rpc url - RPCFlagName = "da.rpc" + RPCFlagName = "da.rpc" // AuthTokenFlagName defines the flag for the auth token AuthTokenFlagName = "da.auth_token" // NamespaceFlagName defines the flag for the namespace NamespaceFlagName = "da.namespace" + // EthFallbackDisabledFlagName defines the flag for disabling eth fallback + EthFallbackDisabledFlagName = "da.eth_fallback_disabled" // NamespaceSize is the size of the hex encoded namespace string NamespaceSize = 58 @@ -39,13 +41,19 @@ func CLIFlags(envPrefix string) []cli.Flag { Usage: "namespace of the data availability client", EnvVars: opservice.PrefixEnvVar(envPrefix, "DA_NAMESPACE"), }, + &cli.BoolFlag{ + Name: EthFallbackDisabledFlagName, + Usage: "disable eth fallback", + EnvVars: opservice.PrefixEnvVar(envPrefix, "ETH_FALLBACK_DISABLED"), + }, } } type CLIConfig struct { - Rpc string - AuthToken string - Namespace string + Rpc string + AuthToken string + Namespace string + EthFallbackDisabled bool } func (c CLIConfig) Check() error { @@ -60,8 +68,9 @@ func NewCLIConfig() CLIConfig { func ReadCLIConfig(ctx *cli.Context) CLIConfig { return CLIConfig{ - Rpc: ctx.String(RPCFlagName), - AuthToken: ctx.String(AuthTokenFlagName), - Namespace: ctx.String(NamespaceFlagName), + Rpc: ctx.String(RPCFlagName), + AuthToken: ctx.String(AuthTokenFlagName), + Namespace: ctx.String(NamespaceFlagName), + EthFallbackDisabled: ctx.Bool(EthFallbackDisabledFlagName), } } diff --git a/op-celestia/da_client.go b/op-celestia/da_client.go index 8cf70df37644..d3d407a10b94 100644 --- a/op-celestia/da_client.go +++ b/op-celestia/da_client.go @@ -9,12 +9,13 @@ import ( ) type DAClient struct { - Client da.DA - GetTimeout time.Duration - Namespace da.Namespace + Client da.DA + GetTimeout time.Duration + Namespace da.Namespace + EthFallbackDisabled bool } -func NewDAClient(rpc, token, namespace string) (*DAClient, error) { +func NewDAClient(rpc, token, namespace string, ethFallbackDisabled bool) (*DAClient, error) { client, err := proxy.NewClient(rpc, token) if err != nil { return nil, err @@ -24,8 +25,9 @@ func NewDAClient(rpc, token, namespace string) (*DAClient, error) { return nil, err } return &DAClient{ - Client: client, - GetTimeout: time.Minute, - Namespace: ns, + Client: client, + GetTimeout: time.Minute, + Namespace: ns, + EthFallbackDisabled: ethFallbackDisabled, }, nil } diff --git a/op-node/rollup/driver/da.go b/op-node/rollup/driver/da.go index dc5e5f3ceac0..cacef1afd0c0 100644 --- a/op-node/rollup/driver/da.go +++ b/op-node/rollup/driver/da.go @@ -6,7 +6,7 @@ import ( ) func SetDAClient(cfg celestia.CLIConfig) error { - client, err := celestia.NewDAClient(cfg.Rpc, cfg.AuthToken, cfg.Namespace) + client, err := celestia.NewDAClient(cfg.Rpc, cfg.AuthToken, cfg.Namespace, false) if err != nil { return err } From aaa287155c2d1fd4caf39b6afdcac01c4d1f31f2 Mon Sep 17 00:00:00 2001 From: Emiliano Bonassi Date: Wed, 8 May 2024 16:12:10 +0200 Subject: [PATCH 04/11] chore(op-batcher): refactor EthFallbackDisabledFlagName --- op-celestia/cli.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/op-celestia/cli.go b/op-celestia/cli.go index 40550ba5cf4f..bf00f83bb877 100644 --- a/op-celestia/cli.go +++ b/op-celestia/cli.go @@ -44,7 +44,7 @@ func CLIFlags(envPrefix string) []cli.Flag { &cli.BoolFlag{ Name: EthFallbackDisabledFlagName, Usage: "disable eth fallback", - EnvVars: opservice.PrefixEnvVar(envPrefix, "ETH_FALLBACK_DISABLED"), + EnvVars: opservice.PrefixEnvVar(envPrefix, "DA_ETH_FALLBACK_DISABLED"), }, } } From 9b5471f733a973f5c27c937776d14006c2764813 Mon Sep 17 00:00:00 2001 From: Javed Khan Date: Mon, 13 May 2024 15:07:53 -0700 Subject: [PATCH 05/11] da: use env auth token if set --- bedrock-devnet/devnet/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/bedrock-devnet/devnet/__init__.py b/bedrock-devnet/devnet/__init__.py index 22c9f5fb4326..0bc614872428 100644 --- a/bedrock-devnet/devnet/__init__.py +++ b/bedrock-devnet/devnet/__init__.py @@ -287,9 +287,11 @@ def devnet_deploy(paths): run_command(['docker', 'compose', 'up', '-d', 'da'], cwd=paths.ops_bedrock_dir, env=docker_env) wait_up(26658) time.sleep(15) - celestia_node_auth_token = run_command([ - 'docker', 'compose', 'exec', '--no-TTY', 'da', 'celestia', 'bridge', 'auth', 'admin', '--node.store', '/home/celestia/bridge' - ], cwd=paths.ops_bedrock_dir, env=docker_env, capture_output=True).stdout.decode().strip() + celestia_node_auth_token = os.getenv("CELESTIA_NODE_AUTH_TOKEN") + if not celestia_node_auth_token: + celestia_node_auth_token = run_command([ + 'docker', 'compose', 'exec', '--no-TTY', 'da', 'celestia', 'bridge', 'auth', 'admin', '--node.store', '/home/celestia/bridge' + ], cwd=paths.ops_bedrock_dir, env=docker_env, capture_output=True).stdout.decode().strip() docker_env['CELESTIA_NODE_AUTH_TOKEN'] = celestia_node_auth_token print('CELESTIA_NODE_AUTH_TOKEN: ', celestia_node_auth_token) From 78638d54e28f55146336ec71755f8630ae3758e2 Mon Sep 17 00:00:00 2001 From: Emiliano Bonassi Date: Fri, 10 May 2024 10:04:04 +0200 Subject: [PATCH 06/11] feat: enable multi frame txs and frame size setting for calldata txs --- op-batcher/batcher/channel_config.go | 3 +++ op-batcher/batcher/config.go | 8 ++++++++ op-batcher/batcher/driver.go | 4 ++-- op-batcher/batcher/service.go | 11 +++++++++++ op-batcher/batcher/test_batch_submitter.go | 2 +- op-batcher/flags/flags.go | 14 ++++++++++++++ op-e2e/system/e2esys/setup.go | 2 +- 7 files changed, 40 insertions(+), 4 deletions(-) diff --git a/op-batcher/batcher/channel_config.go b/op-batcher/batcher/channel_config.go index 45dc1d4dcfa4..3ecdca91fad2 100644 --- a/op-batcher/batcher/channel_config.go +++ b/op-batcher/batcher/channel_config.go @@ -33,6 +33,9 @@ type ChannelConfig struct { // A value of 0 disables a maximum. MaxBlocksPerSpanBatch int + // MultiFrameTxs controls whether to put all frames of a channel inside a single tx. + MultiFrameTxs bool + // Target number of frames to create per channel. // For blob transactions, this controls the number of blobs to target adding // to each blob tx. diff --git a/op-batcher/batcher/config.go b/op-batcher/batcher/config.go index 6a4e539a6d1d..4bf8f5fd4a85 100644 --- a/op-batcher/batcher/config.go +++ b/op-batcher/batcher/config.go @@ -61,6 +61,12 @@ type CLIConfig struct { // Maximum number of blocks to add to a span batch. Default is 0 - no maximum. MaxBlocksPerSpanBatch int + // MaxFrameSize is the maximum size of a frame in a batch tx. + MaxFrameSize uint64 + + // MultiFrameTxs controls whether to put all frames of a channel inside a single tx. + MultiFrameTxs bool + // The target number of frames to create per channel. Controls number of blobs // per blob tx, if using Blob DA. TargetNumFrames int @@ -184,6 +190,8 @@ func NewConfig(ctx *cli.Context) *CLIConfig { MaxChannelDuration: ctx.Uint64(flags.MaxChannelDurationFlag.Name), MaxL1TxSize: ctx.Uint64(flags.MaxL1TxSizeBytesFlag.Name), MaxBlocksPerSpanBatch: ctx.Int(flags.MaxBlocksPerSpanBatch.Name), + MaxFrameSize: ctx.Uint64(flags.MaxFrameSizeFlag.Name), + MultiFrameTxs: ctx.Bool(flags.MultiFrameTxsFlag.Name), TargetNumFrames: ctx.Int(flags.TargetNumFramesFlag.Name), ApproxComprRatio: ctx.Float64(flags.ApproxComprRatioFlag.Name), Compressor: ctx.String(flags.CompressorFlag.Name), diff --git a/op-batcher/batcher/driver.go b/op-batcher/batcher/driver.go index 57f4faf03e57..f0c629f79678 100644 --- a/op-batcher/batcher/driver.go +++ b/op-batcher/batcher/driver.go @@ -648,7 +648,7 @@ func (l *BatchSubmitter) cancelBlockingTx(queue *txmgr.Queue[txRef], receiptsCh var candidate *txmgr.TxCandidate var err error if isBlockedBlob { - candidate = l.calldataTxCandidate([]byte{}) + candidate, _ = l.calldataTxCandidate([]byte{}) } else if candidate, err = l.blobTxCandidate(emptyTxData); err != nil { panic(err) // this error should not happen } @@ -718,7 +718,7 @@ func (l *BatchSubmitter) sendTransaction(txdata txData, queue *txmgr.Queue[txRef } } else { // sanity check - if nf := len(txdata.frames); nf != 1 { + if nf := len(txdata.frames); nf > l.ChannelConfig.ChannelConfig().TargetNumFrames { l.Log.Crit("Unexpected number of frames in calldata tx", "num_frames", nf) } candidate, err = l.celestiaTxCandidate(txdata.CallData()) diff --git a/op-batcher/batcher/service.go b/op-batcher/batcher/service.go index 66f5260d9216..beab878c2d27 100644 --- a/op-batcher/batcher/service.go +++ b/op-batcher/batcher/service.go @@ -212,11 +212,22 @@ func (bs *BatcherService) initChannelConfig(cfg *CLIConfig) error { MaxChannelDuration: cfg.MaxChannelDuration, MaxFrameSize: cfg.MaxL1TxSize - 1, // account for version byte prefix; reset for blobs MaxBlocksPerSpanBatch: cfg.MaxBlocksPerSpanBatch, + MultiFrameTxs: cfg.MultiFrameTxs, TargetNumFrames: cfg.TargetNumFrames, SubSafetyMargin: cfg.SubSafetyMargin, BatchType: cfg.BatchType, } + // override max frame size if set + if cfg.MaxFrameSize > 0 { + cc.MaxFrameSize = cfg.MaxFrameSize + } + + // enable multi-frame txs if set + if cfg.MultiFrameTxs { + cc.MultiFrameTxs = true + } + switch cfg.DataAvailabilityType { case flags.BlobsType, flags.AutoType: if !cfg.TestUseMaxTxSizeForBlobs { diff --git a/op-batcher/batcher/test_batch_submitter.go b/op-batcher/batcher/test_batch_submitter.go index 9ff5ca69796f..a9719002913a 100644 --- a/op-batcher/batcher/test_batch_submitter.go +++ b/op-batcher/batcher/test_batch_submitter.go @@ -29,7 +29,7 @@ func (l *TestBatchSubmitter) JamTxPool(ctx context.Context) error { var err error cc := l.state.cfgProvider.ChannelConfig() if cc.UseBlobs { - candidate = l.calldataTxCandidate([]byte{}) + candidate, _ = l.calldataTxCandidate([]byte{}) } else if candidate, err = l.blobTxCandidate(emptyTxData); err != nil { return err } diff --git a/op-batcher/flags/flags.go b/op-batcher/flags/flags.go index 5a1303a94b0a..10f2278b8759 100644 --- a/op-batcher/flags/flags.go +++ b/op-batcher/flags/flags.go @@ -81,6 +81,18 @@ var ( Usage: "Maximum number of blocks to add to a span batch. Default is 0 - no maximum.", EnvVars: prefixEnvVars("MAX_BLOCKS_PER_SPAN_BATCH"), } + MaxFrameSizeFlag = &cli.Uint64Flag{ + Name: "max-frame-size-bytes", + Usage: "The maximum size of a frame. 0 to use default value (120k-1)", + Value: 0, + EnvVars: prefixEnvVars("MAX_FRAME_SIZE_BYTES"), + } + MultiFrameTxsFlag = &cli.BoolFlag{ + Name: "multi-frame-txs", + Usage: "Whether to put all frames of a channel inside a single tx. Ignored for blobs, where true will be used.", + Value: false, + EnvVars: prefixEnvVars("MULTI_FRAME_TXS"), + } TargetNumFramesFlag = &cli.IntFlag{ Name: "target-num-frames", Usage: "The target number of frames to create per channel. Controls number of blobs per blob tx, if using Blob DA.", @@ -175,6 +187,8 @@ var optionalFlags = []cli.Flag{ MaxChannelDurationFlag, MaxL1TxSizeBytesFlag, MaxBlocksPerSpanBatch, + MaxFrameSizeFlag, + MultiFrameTxsFlag, TargetNumFramesFlag, ApproxComprRatioFlag, CompressorFlag, diff --git a/op-e2e/system/e2esys/setup.go b/op-e2e/system/e2esys/setup.go index b39346b14cff..cfa0373341c4 100644 --- a/op-e2e/system/e2esys/setup.go +++ b/op-e2e/system/e2esys/setup.go @@ -907,7 +907,7 @@ func (cfg SystemConfig) Start(t *testing.T, startOpts ...StartOption) (*System, DataAvailabilityType: sys.Cfg.DataAvailabilityType, CompressionAlgo: compressionAlgo, AltDA: batcherAltDACLIConfig, - DaConfig: celestia.CLIConfig{DaRpc: "localhost:26650"}, + DaConfig: celestia.CLIConfig{Rpc: "localhost:26650"}, } // Batch Submitter batcher, err := bss.BatcherServiceFromCLIConfig(context.Background(), "0.0.1", batcherCLIConfig, sys.Cfg.Loggers["batcher"]) From b33ab9eb8c3898c18e1daa1ebf16c7bd401d9299 Mon Sep 17 00:00:00 2001 From: Emiliano Bonassi Date: Tue, 14 May 2024 18:19:05 -0700 Subject: [PATCH 07/11] chore: add log for multiframetxs setting --- op-batcher/batcher/service.go | 1 + 1 file changed, 1 insertion(+) diff --git a/op-batcher/batcher/service.go b/op-batcher/batcher/service.go index beab878c2d27..ffae31c1b5c1 100644 --- a/op-batcher/batcher/service.go +++ b/op-batcher/batcher/service.go @@ -264,6 +264,7 @@ func (bs *BatcherService) initChannelConfig(cfg *CLIConfig) error { bs.Log.Info("Initialized channel-config", "da_type", cfg.DataAvailabilityType, "use_alt_da", bs.UseAltDA, + "multi_frame_txs", cc.MultiFrameTxs, "max_frame_size", cc.MaxFrameSize, "target_num_frames", cc.TargetNumFrames, "compressor", cc.CompressorConfig.Kind, From 0e9f287643b836f549fff97ad9daebab5f4a2780 Mon Sep 17 00:00:00 2001 From: Javed Khan Date: Tue, 2 Jul 2024 16:27:27 -0700 Subject: [PATCH 08/11] da: try using blobdata for eth fallback --- op-batcher/batcher/driver.go | 47 ++++++++++++++------- op-batcher/batcher/service.go | 2 +- op-batcher/batcher/test_batch_submitter.go | 2 +- op-celestia/cli.go | 49 ++++++++++++++++++---- op-celestia/da_client.go | 22 ++++++---- op-node/rollup/driver/da.go | 13 +++++- 6 files changed, 98 insertions(+), 37 deletions(-) diff --git a/op-batcher/batcher/driver.go b/op-batcher/batcher/driver.go index f0c629f79678..5877cbe1423a 100644 --- a/op-batcher/batcher/driver.go +++ b/op-batcher/batcher/driver.go @@ -648,7 +648,7 @@ func (l *BatchSubmitter) cancelBlockingTx(queue *txmgr.Queue[txRef], receiptsCh var candidate *txmgr.TxCandidate var err error if isBlockedBlob { - candidate, _ = l.calldataTxCandidate([]byte{}) + candidate = l.calldataTxCandidate([]byte{}) } else if candidate, err = l.blobTxCandidate(emptyTxData); err != nil { panic(err) // this error should not happen } @@ -694,6 +694,20 @@ func (l *BatchSubmitter) publishToAltDAAndL1(txdata txData, queue *txmgr.Queue[t } } +// fallbackTxCandidate creates a fallback tx candidate for the given txdata. +func (l *BatchSubmitter) fallbackTxCandidate(txdata txData) (*txmgr.TxCandidate, error) { + switch l.DAClient.FallbackMode { + case celestia.FallbackModeBlobData: + return l.blobTxCandidate(txdata) + case celestia.FallbackModeCallData: + return l.calldataTxCandidate(txdata.CallData()), nil + case celestia.FallbackModeDisabled: + return nil, fmt.Errorf("celestia: fallback disabled") + default: + return nil, fmt.Errorf("celestia: unknown fallback mode: %s", l.DAClient.FallbackMode) + } +} + // sendTransaction creates & queues for sending a transaction to the batch inbox address with the given `txData`. // This call will block if the txmgr queue is at the max-pending limit. // The method will block if the queue's MaxPendingTransactions is exceeded. @@ -723,7 +737,13 @@ func (l *BatchSubmitter) sendTransaction(txdata txData, queue *txmgr.Queue[txRef } candidate, err = l.celestiaTxCandidate(txdata.CallData()) if err != nil { - return err + l.Log.Error("celestia: blob submission failed", "err", err) + candidate, err = l.fallbackTxCandidate(txdata) + if err != nil { + l.Log.Error("celestia: fallback failed", "err", err) + l.recordFailedTx(txdata.ID(), err) + return nil + } } } @@ -770,24 +790,19 @@ func (l *BatchSubmitter) calldataTxCandidate(data []byte) *txmgr.TxCandidate { } func (l *BatchSubmitter) celestiaTxCandidate(data []byte) (*txmgr.TxCandidate, error) { - l.Log.Info("Building Calldata transaction candidate", "size", len(data)) + l.Log.Info("Building Celestia transaction candidate", "size", len(data)) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Duration(l.RollupConfig.BlockTime)*time.Second) ids, err := l.DAClient.Client.Submit(ctx, [][]byte{data}, -1, l.DAClient.Namespace) cancel() - if err == nil && len(ids) == 1 { - l.Log.Info("celestia: blob successfully submitted", "id", hex.EncodeToString(ids[0])) - data = append([]byte{celestia.DerivationVersionCelestia}, ids[0]...) - } else { - if l.DAClient.EthFallbackDisabled { - return nil, fmt.Errorf("celestia: blob submission failed; eth fallback disabled: %w", err) - } - - l.Log.Info("celestia: blob submission failed; falling back to eth", "err", err) + if err != nil { + return nil, err } - return &txmgr.TxCandidate{ - To: &l.RollupConfig.BatchInboxAddress, - TxData: data, - }, nil + if len(ids) != 1 { + return nil, fmt.Errorf("celestia: expected 1 id, got %d", len(ids)) + } + l.Log.Info("celestia: blob successfully submitted", "id", hex.EncodeToString(ids[0])) + data = append([]byte{celestia.DerivationVersionCelestia}, ids[0]...) + return l.calldataTxCandidate(data), nil } func (l *BatchSubmitter) handleReceipt(r txmgr.TxReceipt[txRef]) { diff --git a/op-batcher/batcher/service.go b/op-batcher/batcher/service.go index ffae31c1b5c1..f19539ddc224 100644 --- a/op-batcher/batcher/service.go +++ b/op-batcher/batcher/service.go @@ -389,7 +389,7 @@ func (bs *BatcherService) initAltDA(cfg *CLIConfig) error { } func (bs *BatcherService) initDA(cfg *CLIConfig) error { - client, err := celestia.NewDAClient(cfg.DaConfig.Rpc, cfg.DaConfig.AuthToken, cfg.DaConfig.Namespace, cfg.DaConfig.EthFallbackDisabled) + client, err := celestia.NewDAClient(cfg.DaConfig.Rpc, cfg.DaConfig.AuthToken, cfg.DaConfig.Namespace, cfg.DaConfig.FallbackMode) if err != nil { return err } diff --git a/op-batcher/batcher/test_batch_submitter.go b/op-batcher/batcher/test_batch_submitter.go index a9719002913a..9ff5ca69796f 100644 --- a/op-batcher/batcher/test_batch_submitter.go +++ b/op-batcher/batcher/test_batch_submitter.go @@ -29,7 +29,7 @@ func (l *TestBatchSubmitter) JamTxPool(ctx context.Context) error { var err error cc := l.state.cfgProvider.ChannelConfig() if cc.UseBlobs { - candidate, _ = l.calldataTxCandidate([]byte{}) + candidate = l.calldataTxCandidate([]byte{}) } else if candidate, err = l.blobTxCandidate(emptyTxData); err != nil { return err } diff --git a/op-celestia/cli.go b/op-celestia/cli.go index bf00f83bb877..c0ab92606c44 100644 --- a/op-celestia/cli.go +++ b/op-celestia/cli.go @@ -1,11 +1,22 @@ package celestia import ( + "fmt" + "github.com/urfave/cli/v2" opservice "github.com/ethereum-optimism/optimism/op-service" ) +const ( + // FallbackModeDisabled is the fallback mode disabled + FallbackModeDisabled = "disabled" + // FallbackModeBlobData is the fallback mode blob data + FallbackModeBlobData = "blobdata" + // FallbackModeCallData is the fallback mode call data + FallbackModeCallData = "calldata" +) + const ( // RPCFlagName defines the flag for the rpc url RPCFlagName = "da.rpc" @@ -15,6 +26,8 @@ const ( NamespaceFlagName = "da.namespace" // EthFallbackDisabledFlagName defines the flag for disabling eth fallback EthFallbackDisabledFlagName = "da.eth_fallback_disabled" + // FallbackModeFlagName defines the flag for fallback mode + FallbackModeFlagName = "da.fallback_mode" // NamespaceSize is the size of the hex encoded namespace string NamespaceSize = 58 @@ -43,17 +56,35 @@ func CLIFlags(envPrefix string) []cli.Flag { }, &cli.BoolFlag{ Name: EthFallbackDisabledFlagName, - Usage: "disable eth fallback", + Usage: "disable eth fallback (deprecated, use FallbackModeFlag instead)", EnvVars: opservice.PrefixEnvVar(envPrefix, "DA_ETH_FALLBACK_DISABLED"), + Action: func(c *cli.Context, e bool) error { + if e { + return c.Set(FallbackModeFlagName, FallbackModeDisabled) + } + return nil + }, + }, + &cli.StringFlag{ + Name: FallbackModeFlagName, + Usage: fmt.Sprintf("fallback mode; must be one of: %s, %s or %s", FallbackModeDisabled, FallbackModeBlobData, FallbackModeCallData), + EnvVars: opservice.PrefixEnvVar(envPrefix, "DA_FALLBACK_MODE"), + Value: FallbackModeCallData, + Action: func(c *cli.Context, s string) error { + if s != FallbackModeDisabled && s != FallbackModeBlobData && s != FallbackModeCallData { + return fmt.Errorf("invalid fallback mode: %s; must be one of: %s, %s or %s", s, FallbackModeDisabled, FallbackModeBlobData, FallbackModeCallData) + } + return nil + }, }, } } type CLIConfig struct { - Rpc string - AuthToken string - Namespace string - EthFallbackDisabled bool + Rpc string + AuthToken string + Namespace string + FallbackMode string } func (c CLIConfig) Check() error { @@ -68,9 +99,9 @@ func NewCLIConfig() CLIConfig { func ReadCLIConfig(ctx *cli.Context) CLIConfig { return CLIConfig{ - Rpc: ctx.String(RPCFlagName), - AuthToken: ctx.String(AuthTokenFlagName), - Namespace: ctx.String(NamespaceFlagName), - EthFallbackDisabled: ctx.Bool(EthFallbackDisabledFlagName), + Rpc: ctx.String(RPCFlagName), + AuthToken: ctx.String(AuthTokenFlagName), + Namespace: ctx.String(NamespaceFlagName), + FallbackMode: ctx.String(FallbackModeFlagName), } } diff --git a/op-celestia/da_client.go b/op-celestia/da_client.go index d3d407a10b94..f5c14fe50377 100644 --- a/op-celestia/da_client.go +++ b/op-celestia/da_client.go @@ -2,6 +2,7 @@ package celestia import ( "encoding/hex" + "fmt" "time" "github.com/rollkit/go-da" @@ -9,13 +10,13 @@ import ( ) type DAClient struct { - Client da.DA - GetTimeout time.Duration - Namespace da.Namespace - EthFallbackDisabled bool + Client da.DA + GetTimeout time.Duration + Namespace da.Namespace + FallbackMode string } -func NewDAClient(rpc, token, namespace string, ethFallbackDisabled bool) (*DAClient, error) { +func NewDAClient(rpc, token, namespace, fallbackMode string) (*DAClient, error) { client, err := proxy.NewClient(rpc, token) if err != nil { return nil, err @@ -24,10 +25,13 @@ func NewDAClient(rpc, token, namespace string, ethFallbackDisabled bool) (*DACli if err != nil { return nil, err } + if fallbackMode != "disabled" && fallbackMode != "blobdata" && fallbackMode != "calldata" { + return nil, fmt.Errorf("celestia: unknown fallback mode: %s", fallbackMode) + } return &DAClient{ - Client: client, - GetTimeout: time.Minute, - Namespace: ns, - EthFallbackDisabled: ethFallbackDisabled, + Client: client, + GetTimeout: time.Minute, + Namespace: ns, + FallbackMode: fallbackMode, }, nil } diff --git a/op-node/rollup/driver/da.go b/op-node/rollup/driver/da.go index cacef1afd0c0..a8e29000e714 100644 --- a/op-node/rollup/driver/da.go +++ b/op-node/rollup/driver/da.go @@ -6,7 +6,18 @@ import ( ) func SetDAClient(cfg celestia.CLIConfig) error { - client, err := celestia.NewDAClient(cfg.Rpc, cfg.AuthToken, cfg.Namespace, false) + // NOTE: we always read using blob_data_source.go + // If the transaction has calldata, based on the prefix byte. + // - If the prefix byte is 0xce + // - We interpret the calldata as a celestia reference and fetch + // the data from celestia. + // - Otherwise, we use the calldata fallback mode. + // If the transaction has blobs, we use blobdata fallback mode. + // See dataAndHashesFromTxs and DataFromEVMTransactions + // The read path always operates in the most permissive mode and is + // independent of the fallback mode. + // Therefore the configuration value for FallbackMode passed here does not matter. + client, err := celestia.NewDAClient(cfg.Rpc, cfg.AuthToken, cfg.Namespace, cfg.FallbackMode) if err != nil { return err } From eb8649a7eabb59d61de6ebb9f2192de52b5bee6c Mon Sep 17 00:00:00 2001 From: Javed Khan Date: Wed, 4 Sep 2024 09:05:05 -0700 Subject: [PATCH 09/11] da: add gas price flag (#451) --- op-batcher/batcher/driver.go | 2 +- op-batcher/batcher/service.go | 2 +- op-celestia/cli.go | 13 +++++++++++++ op-celestia/da_client.go | 4 +++- op-node/rollup/driver/da.go | 2 +- 5 files changed, 19 insertions(+), 4 deletions(-) diff --git a/op-batcher/batcher/driver.go b/op-batcher/batcher/driver.go index 5877cbe1423a..1327aa9f5d6a 100644 --- a/op-batcher/batcher/driver.go +++ b/op-batcher/batcher/driver.go @@ -792,7 +792,7 @@ func (l *BatchSubmitter) calldataTxCandidate(data []byte) *txmgr.TxCandidate { func (l *BatchSubmitter) celestiaTxCandidate(data []byte) (*txmgr.TxCandidate, error) { l.Log.Info("Building Celestia transaction candidate", "size", len(data)) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Duration(l.RollupConfig.BlockTime)*time.Second) - ids, err := l.DAClient.Client.Submit(ctx, [][]byte{data}, -1, l.DAClient.Namespace) + ids, err := l.DAClient.Client.Submit(ctx, [][]byte{data}, l.DAClient.GasPrice, l.DAClient.Namespace) cancel() if err != nil { return nil, err diff --git a/op-batcher/batcher/service.go b/op-batcher/batcher/service.go index f19539ddc224..f93519a2174c 100644 --- a/op-batcher/batcher/service.go +++ b/op-batcher/batcher/service.go @@ -389,7 +389,7 @@ func (bs *BatcherService) initAltDA(cfg *CLIConfig) error { } func (bs *BatcherService) initDA(cfg *CLIConfig) error { - client, err := celestia.NewDAClient(cfg.DaConfig.Rpc, cfg.DaConfig.AuthToken, cfg.DaConfig.Namespace, cfg.DaConfig.FallbackMode) + client, err := celestia.NewDAClient(cfg.DaConfig.Rpc, cfg.DaConfig.AuthToken, cfg.DaConfig.Namespace, cfg.DaConfig.FallbackMode, cfg.DaConfig.GasPrice) if err != nil { return err } diff --git a/op-celestia/cli.go b/op-celestia/cli.go index c0ab92606c44..33b2f1767552 100644 --- a/op-celestia/cli.go +++ b/op-celestia/cli.go @@ -28,12 +28,17 @@ const ( EthFallbackDisabledFlagName = "da.eth_fallback_disabled" // FallbackModeFlagName defines the flag for fallback mode FallbackModeFlagName = "da.fallback_mode" + // GasPriceFlagName defines the flag for gas price + GasPriceFlagName = "da.gas_price" // NamespaceSize is the size of the hex encoded namespace string NamespaceSize = 58 // defaultRPC is the default rpc dial address defaultRPC = "grpc://localhost:26650" + + // defaultGasPrice is the default gas price + defaultGasPrice = -1 ) func CLIFlags(envPrefix string) []cli.Flag { @@ -77,6 +82,12 @@ func CLIFlags(envPrefix string) []cli.Flag { return nil }, }, + &cli.Float64Flag{ + Name: GasPriceFlagName, + Usage: "gas price of the data availability client", + Value: defaultGasPrice, + EnvVars: opservice.PrefixEnvVar(envPrefix, "DA_GAS_PRICE"), + }, } } @@ -85,6 +96,7 @@ type CLIConfig struct { AuthToken string Namespace string FallbackMode string + GasPrice float64 } func (c CLIConfig) Check() error { @@ -103,5 +115,6 @@ func ReadCLIConfig(ctx *cli.Context) CLIConfig { AuthToken: ctx.String(AuthTokenFlagName), Namespace: ctx.String(NamespaceFlagName), FallbackMode: ctx.String(FallbackModeFlagName), + GasPrice: ctx.Float64(GasPriceFlagName), } } diff --git a/op-celestia/da_client.go b/op-celestia/da_client.go index f5c14fe50377..b15940ca335d 100644 --- a/op-celestia/da_client.go +++ b/op-celestia/da_client.go @@ -14,9 +14,10 @@ type DAClient struct { GetTimeout time.Duration Namespace da.Namespace FallbackMode string + GasPrice float64 } -func NewDAClient(rpc, token, namespace, fallbackMode string) (*DAClient, error) { +func NewDAClient(rpc, token, namespace, fallbackMode string, gasPrice float64) (*DAClient, error) { client, err := proxy.NewClient(rpc, token) if err != nil { return nil, err @@ -33,5 +34,6 @@ func NewDAClient(rpc, token, namespace, fallbackMode string) (*DAClient, error) GetTimeout: time.Minute, Namespace: ns, FallbackMode: fallbackMode, + GasPrice: gasPrice, }, nil } diff --git a/op-node/rollup/driver/da.go b/op-node/rollup/driver/da.go index a8e29000e714..6fa2cd750456 100644 --- a/op-node/rollup/driver/da.go +++ b/op-node/rollup/driver/da.go @@ -17,7 +17,7 @@ func SetDAClient(cfg celestia.CLIConfig) error { // The read path always operates in the most permissive mode and is // independent of the fallback mode. // Therefore the configuration value for FallbackMode passed here does not matter. - client, err := celestia.NewDAClient(cfg.Rpc, cfg.AuthToken, cfg.Namespace, cfg.FallbackMode) + client, err := celestia.NewDAClient(cfg.Rpc, cfg.AuthToken, cfg.Namespace, cfg.FallbackMode, cfg.GasPrice) if err != nil { return err } From 3303b89524abb18af02d56fa0e2655d34b41b857 Mon Sep 17 00:00:00 2001 From: Javed Khan Date: Tue, 22 Oct 2024 14:10:40 -0700 Subject: [PATCH 10/11] da: reuse useblobs for multiframetxs (#452) --- op-batcher/batcher/channel_config.go | 3 --- op-batcher/batcher/driver.go | 40 +++++++++++++--------------- op-batcher/batcher/service.go | 5 ++-- 3 files changed, 20 insertions(+), 28 deletions(-) diff --git a/op-batcher/batcher/channel_config.go b/op-batcher/batcher/channel_config.go index 3ecdca91fad2..45dc1d4dcfa4 100644 --- a/op-batcher/batcher/channel_config.go +++ b/op-batcher/batcher/channel_config.go @@ -33,9 +33,6 @@ type ChannelConfig struct { // A value of 0 disables a maximum. MaxBlocksPerSpanBatch int - // MultiFrameTxs controls whether to put all frames of a channel inside a single tx. - MultiFrameTxs bool - // Target number of frames to create per channel. // For blob transactions, this controls the number of blobs to target adding // to each blob tx. diff --git a/op-batcher/batcher/driver.go b/op-batcher/batcher/driver.go index 1327aa9f5d6a..1dfb699590b9 100644 --- a/op-batcher/batcher/driver.go +++ b/op-batcher/batcher/driver.go @@ -721,31 +721,27 @@ func (l *BatchSubmitter) sendTransaction(txdata txData, queue *txmgr.Queue[txRef return nil } - var candidate *txmgr.TxCandidate - if txdata.asBlob { - if candidate, err = l.blobTxCandidate(txdata); err != nil { - // We could potentially fall through and try a calldata tx instead, but this would - // likely result in the chain spending more in gas fees than it is tuned for, so best - // to just fail. We do not expect this error to trigger unless there is a serious bug - // or configuration issue. - return fmt.Errorf("could not create blob tx candidate: %w", err) - } - } else { - // sanity check - if nf := len(txdata.frames); nf > l.ChannelConfig.ChannelConfig().TargetNumFrames { - l.Log.Crit("Unexpected number of frames in calldata tx", "num_frames", nf) - } - candidate, err = l.celestiaTxCandidate(txdata.CallData()) + // force celestia tx candidate, multiframe is set by UseBlobs which is not affected + txdata.asBlob = false + // sanity check + if nf := len(txdata.frames); nf > l.ChannelConfig.ChannelConfig().TargetNumFrames { + l.Log.Crit("Unexpected number of frames in calldata tx", "num_frames", nf) + } + candidate, err := l.celestiaTxCandidate(txdata.CallData()) + if err != nil { + l.Log.Error("celestia: blob submission failed", "err", err) + candidate, err = l.fallbackTxCandidate(txdata) if err != nil { - l.Log.Error("celestia: blob submission failed", "err", err) - candidate, err = l.fallbackTxCandidate(txdata) - if err != nil { - l.Log.Error("celestia: fallback failed", "err", err) - l.recordFailedTx(txdata.ID(), err) - return nil - } + l.Log.Error("celestia: fallback failed", "err", err) + l.recordFailedTx(txdata.ID(), err) + return nil } } + // restore asBlob for cancellation in case of blobdata fallback + if len(candidate.Blobs) > 0 { + txdata.asBlob = true + } + l.Log.Info("tx candidate", "ID", txdata.ID(), "len(txdata.frames)", len(txdata.frames), "txdata.asBlob", txdata.asBlob) l.sendTx(txdata, false, candidate, queue, receiptsCh) return nil diff --git a/op-batcher/batcher/service.go b/op-batcher/batcher/service.go index f93519a2174c..db1389c44ee7 100644 --- a/op-batcher/batcher/service.go +++ b/op-batcher/batcher/service.go @@ -212,7 +212,6 @@ func (bs *BatcherService) initChannelConfig(cfg *CLIConfig) error { MaxChannelDuration: cfg.MaxChannelDuration, MaxFrameSize: cfg.MaxL1TxSize - 1, // account for version byte prefix; reset for blobs MaxBlocksPerSpanBatch: cfg.MaxBlocksPerSpanBatch, - MultiFrameTxs: cfg.MultiFrameTxs, TargetNumFrames: cfg.TargetNumFrames, SubSafetyMargin: cfg.SubSafetyMargin, BatchType: cfg.BatchType, @@ -225,7 +224,7 @@ func (bs *BatcherService) initChannelConfig(cfg *CLIConfig) error { // enable multi-frame txs if set if cfg.MultiFrameTxs { - cc.MultiFrameTxs = true + cc.UseBlobs = true } switch cfg.DataAvailabilityType { @@ -264,7 +263,7 @@ func (bs *BatcherService) initChannelConfig(cfg *CLIConfig) error { bs.Log.Info("Initialized channel-config", "da_type", cfg.DataAvailabilityType, "use_alt_da", bs.UseAltDA, - "multi_frame_txs", cc.MultiFrameTxs, + "use_blobs", cc.UseBlobs, "max_frame_size", cc.MaxFrameSize, "target_num_frames", cc.TargetNumFrames, "compressor", cc.CompressorConfig.Kind, From 09fed70de5b2ddc55f8e7ad18f4484a07118632c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 23 Oct 2024 01:13:29 +0000 Subject: [PATCH 11/11] dependabot(gomod): bump github.com/andybalholm/brotli Bumps [github.com/andybalholm/brotli](https://github.com/andybalholm/brotli) from 1.1.0 to 1.1.1. - [Commits](https://github.com/andybalholm/brotli/compare/v1.1.0...v1.1.1) --- updated-dependencies: - dependency-name: github.com/andybalholm/brotli dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a2b90b4441e1..2f6e4745dc6d 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ toolchain go1.22.7 require ( github.com/BurntSushi/toml v1.4.0 - github.com/andybalholm/brotli v1.1.0 + github.com/andybalholm/brotli v1.1.1 github.com/btcsuite/btcd v0.24.2 github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 github.com/cockroachdb/pebble v1.1.2 diff --git a/go.sum b/go.sum index b8002793ed16..31f02a21b37f 100644 --- a/go.sum +++ b/go.sum @@ -33,8 +33,8 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/allegro/bigcache v1.2.1 h1:hg1sY1raCwic3Vnsvje6TT7/pnZba83LeFck5NrFKSc= github.com/allegro/bigcache v1.2.1/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= -github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= -github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= github.com/armon/go-metrics v0.3.8/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= @@ -802,6 +802,8 @@ github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofm github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=