diff --git a/.env.example.holesky b/.env.example.holesky index b8ea4a94..24067f39 100644 --- a/.env.example.holesky +++ b/.env.example.holesky @@ -56,7 +56,7 @@ EIGENDA_PROXY_EIGENDA_SERVICE_MANAGER_ADDR=0xD4A7E1Bd8015057293f0D0A557088c28694 # EIGENDA_PROXY_LOG_FORMAT=text # The lowest log level that will be output -# EIGENDA_PROXY_LOG_LEVEL=INFO +# EIGENDA_PROXY_LOG_LEVEL=DEBUG # Show pid in the log # EIGENDA_PROXY_LOG_PID=false @@ -81,9 +81,40 @@ EIGENDA_PROXY_EIGENDA_SERVICE_MANAGER_ADDR=0xD4A7E1Bd8015057293f0D0A557088c28694 # access key secret for S3 storage # EIGENDA_PROXY_S3_ACCESS_KEY_SECRET= - + # bucket name for S3 storage # EIGENDA_PROXY_S3_BUCKET= - + # endpoint for S3 storage # EIGENDA_PROXY_S3_ENDPOINT + +# same as EIGENDA_PROXY_MAX_BLOB_LENGTH but caps size when we use proxy +# with WVM perm storage enabled Maximum size at this moment is 100kb. +# WVM_EIGENDA_PROXY_MAX_BLOB_LENGTH=99Kb +# WVM secondary storage related environment variables +# NOTE: 8mb or (8388608 bytes) is a maximum allowed encoded blob size + +# Set to true to enable WVM chain as a secondary storage +# EIGENDA_PROXY_WEAVE_VM_ENABLED= + +# WVM rpc endpoint +# EIGENDA_PROXY_WEAVE_VM_ENDPOINT=https://testnet-rpc.wvm.dev/ + +# WVM chain id +# EIGENDA_PROXY_WEAVE_VM_CHAIN_ID=9496 + +# WVM HTTP requests operations timeout +# EIGENDA_PROXY_WEAVE_VM_TIMEOUT=5s + +# WVM web3signer endpoint +# EIGENDA_PROXY_WEAVE_VM_WEB3_SIGNER_ENDPOINT= + +# WVM private key in case you don't use web3signer, not recommended +# EIGENDA_PROXY_WEAVE_VM_PRIV_KEY_HEX= + +# WVM paths to TLS related files in case you use web3signer with TLS enabled +# EIGENDA_PROXY_WEAVE_VM_WEB3_SIGNER_TLS_CERT_FILE= + +# EIGENDA_PROXY_WEAVE_VM_WEB3_SIGNER_TLS_KEY_FILE= + +# EIGENDA_PROXY_WEAVE_VM_WEB3_SIGNER_TLS_CA_CERT_FILE= diff --git a/.env.example.mainnet b/.env.example.mainnet index 8b7536b3..39a7ed26 100644 --- a/.env.example.mainnet +++ b/.env.example.mainnet @@ -10,7 +10,7 @@ EIGENDA_PROXY_EIGENDA_ETH_RPC= EIGENDA_PROXY_EIGENDA_DISPERSER_RPC=disperser.eigenda.xyz:443 # The deployed EigenDA service manager address. The list can be found here: https://github.com/Layr-Labs/eigenlayer-middleware/?tab=readme-ov-file#current-mainnet-deployment -EIGENDA_PROXY_EIGENDA_SERVICE_MANAGER_ADDR=0x870679E138bCdf293b7Ff14dD44b70FC97e12fc0 +EIGENDA_PROXY_EIGENDA_SERVICE_MANAGER_ADDR=0x870679E138bCdf293b7Ff14dD44b70FC97e12fc0 # Custom quorum IDs for writing blobs. Should not include default quorums 0 or 1. # EIGENDA_PROXY_EIGENDA_CUSTOM_QUORUM_IDS= @@ -85,4 +85,8 @@ EIGENDA_PROXY_EIGENDA_SERVICE_MANAGER_ADDR=0x870679E138bCdf293b7Ff14dD44b70FC97e # EIGENDA_PROXY_S3_BUCKET= # endpoint for S3 storage -# EIGENDA_PROXY_S3_ENDPOINT \ No newline at end of file +# EIGENDA_PROXY_S3_ENDPOINT + +# same as EIGENDA_PROXY_MAX_BLOB_LENGTH but caps size when we use proxy +# with WVM perm storage enabled Maximum size at this moment is 100kb. +# WVM_EIGENDA_PROXY_MAX_BLOB_LENGTH=99Kb \ No newline at end of file diff --git a/LICENSE b/LICENSE index 103adc74..47329152 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2024 Teddy Knox +Copyright (c) 2024 WVM Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md index ac039a7c..233181f1 100644 --- a/README.md +++ b/README.md @@ -19,61 +19,71 @@ Features: * Performs DA certificate verification during retrieval to ensure that data represented by bad DA certificates do not become part of the canonical chain. * Compatibility with Optimism's alt-da commitment type with eigenda backend. * Compatibility with Optimism's keccak-256 commitment type with S3 storage. +* Blobs permanent backup storage option via WeaveVM. In order to disperse to the EigenDA network in production, or at high throughput on testnet, please register your authentication ethereum address through [this form](https://forms.gle/3QRNTYhSMacVFNcU8). Your EigenDA authentication keypair address should not be associated with any funds anywhere. ## Configuration Options -| Option | Default Value | Environment Variable | Description | -| ------------------------------------------- | ------------------------------- | -------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `--addr` | `"127.0.0.1"` | `$EIGENDA_PROXY_ADDR` | Server listening address | -| `--eigenda.cache-path` | `"resources/SRSTables/"` | `$EIGENDA_PROXY_EIGENDA_TARGET_CACHE_PATH` | Directory path to SRS tables for caching. | -| `--eigenda.custom-quorum-ids` | | `$EIGENDA_PROXY_EIGENDA_CUSTOM_QUORUM_IDS` | Custom quorum IDs for writing blobs. Should not include default quorums 0 or 1. | -| `--eigenda.disable-point-verification-mode` | `false` | `$EIGENDA_PROXY_EIGENDA_DISABLE_POINT_VERIFICATION_MODE` | Disable point verification mode. This mode performs IFFT on data before writing and FFT on data after reading. Disabling requires supplying the entire blob for verification against the KZG commitment. | -| `--eigenda.disable-tls` | `false` | `$EIGENDA_PROXY_EIGENDA_GRPC_DISABLE_TLS` | Disable TLS for gRPC communication with the EigenDA disperser. Default is false. | -| `--eigenda.cert-verification-disabled` | `false` | `$EIGENDA_PROXY_EIGENDA_CERT_VERIFICATION_DISABLED` | Whether to verify certificates received from EigenDA disperser. | -| `--eigenda.disperser-rpc` | | `$EIGENDA_PROXY_EIGENDA_DISPERSER_RPC` | RPC endpoint of the EigenDA disperser. | -| `--eigenda.svc-manager-addr` | | `$EIGENDA_PROXY_EIGENDA_SERVICE_MANAGER_ADDR` | The deployed EigenDA service manager address. The list can be found here: https://github.com/Layr-Labs/eigenlayer-middleware/?tab=readme-ov-file#current-mainnet-deployment | -| `--eigenda.eth-confirmation-depth` | `-1` | `$EIGENDA_PROXY_EIGENDA_ETH_CONFIRMATION_DEPTH` | The number of Ethereum blocks of confirmation that the DA bridging transaction must have before it is assumed by the proxy to be final. If set negative the proxy will always wait for blob finalization. | -| `--eigenda.eth-rpc` | | `$EIGENDA_PROXY_EIGENDA_ETH_RPC` | JSON RPC node endpoint for the Ethereum network used for finalizing DA blobs. See available list here: https://docs.eigenlayer.xyz/eigenda/networks/ | -| `--eigenda.g1-path` | `"resources/g1.point"` | `$EIGENDA_PROXY_EIGENDA_TARGET_KZG_G1_PATH` | Directory path to g1.point file. | -| `--eigenda.g2-power-of-2-path` | `"resources/g2.point.powerOf2"` | `$EIGENDA_PROXY_EIGENDA_TARGET_KZG_G2_POWER_OF_2_PATH` | Directory path to g2.point.powerOf2 file. | -| `--eigenda.max-blob-length` | `"16MiB"` | `$EIGENDA_PROXY_EIGENDA_MAX_BLOB_LENGTH` | Maximum blob length to be written or read from EigenDA. Determines the number of SRS points loaded into memory for KZG commitments. Example units: '30MiB', '4Kb', '30MB'. Maximum size slightly exceeds 1GB. | -| `--eigenda.put-blob-encoding-version` | `0` | `$EIGENDA_PROXY_EIGENDA_PUT_BLOB_ENCODING_VERSION` | Blob encoding version to use when writing blobs from the high-level interface. | -| `--eigenda.response-timeout` | `60s` | `$EIGENDA_PROXY_EIGENDA_RESPONSE_TIMEOUT` | Total time to wait for a response from the EigenDA disperser. Default is 60 seconds. | -| `--eigenda.signer-private-key-hex` | | `$EIGENDA_PROXY_EIGENDA_SIGNER_PRIVATE_KEY_HEX` | Hex-encoded signer private key. This key should not be associated with an Ethereum address holding any funds. | -| `--eigenda.status-query-retry-interval` | `5s` | `$EIGENDA_PROXY_EIGENDA_STATUS_QUERY_INTERVAL` | Interval between retries when awaiting network blob finalization. Default is 5 seconds. | -| `--eigenda.status-query-timeout` | `30m0s` | `$EIGENDA_PROXY_EIGENDA_STATUS_QUERY_TIMEOUT` | Duration to wait for a blob to finalize after being sent for dispersal. Default is 30 minutes. | -| `--log.color` | `false` | `$EIGENDA_PROXY_LOG_COLOR` | Color the log output if in terminal mode. | -| `--log.format` | `text` | `$EIGENDA_PROXY_LOG_FORMAT` | Format the log output. Supported formats: 'text', 'terminal', 'logfmt', 'json', 'json-pretty'. | -| `--log.level` | `INFO` | `$EIGENDA_PROXY_LOG_LEVEL` | The lowest log level that will be output. | -| `--log.pid` | `false` | `$EIGENDA_PROXY_LOG_PID` | Show pid in the log. | -| `--memstore.enabled` | `false` | `$EIGENDA_PROXY_MEMSTORE_ENABLED` | Whether to use mem-store for DA logic. | -| `--memstore.expiration` | `25m0s` | `$EIGENDA_PROXY_MEMSTORE_EXPIRATION` | Duration that a mem-store blob/commitment pair are allowed to live. | -| `--memstore.put-latency` | `0` | `$EIGENDA_PROXY_MEMSTORE_PUT_LATENCY` | Artificial latency added for memstore backend to mimic EigenDA's dispersal latency. | -| `--memstore.get-latency` | `0` | `$EIGENDA_PROXY_MEMSTORE_GET_LATENCY` | Artificial latency added for memstore backend to mimic EigenDA's retrieval latency. | -| `--metrics.addr` | `"0.0.0.0"` | `$EIGENDA_PROXY_METRICS_ADDR` | Metrics listening address. | -| `--metrics.enabled` | `false` | `$EIGENDA_PROXY_METRICS_ENABLED` | Enable the metrics server. | -| `--metrics.port` | `7300` | `$EIGENDA_PROXY_METRICS_PORT` | Metrics listening port. | -| `--port` | `3100` | `$EIGENDA_PROXY_PORT` | Server listening port. | -| `--s3.credential-type` | | `$EIGENDA_PROXY_S3_CREDENTIAL_TYPE` | Static or iam. | -| `--s3.access-key-id` | | `$EIGENDA_PROXY_S3_ACCESS_KEY_ID` | Access key id for S3 storage. | -| `--s3.access-key-id` | | `$EIGENDA_PROXY_S3_ACCESS_KEY_ID` | Access key id for S3 storage. | -| `--s3.access-key-secret` | | `$EIGENDA_PROXY_S3_ACCESS_KEY_SECRET` | Access key secret for S3 storage. | -| `--s3.bucket` | | `$EIGENDA_PROXY_S3_BUCKET` | Bucket name for S3 storage. | -| `--s3.path` | | `$EIGENDA_PROXY_S3_PATH` | Bucket path for S3 storage. | -| `--s3.endpoint` | | `$EIGENDA_PROXY_S3_ENDPOINT` | Endpoint for S3 storage. | -| `--s3.enable-tls` | | `$EIGENDA_PROXY_S3_ENABLE_TLS` | Enable TLS connection to S3 endpoint. | -| `--storage.fallback-targets` | `[]` | `$EIGENDA_PROXY_STORAGE_FALLBACK_TARGETS` | Fall back backend targets. Supports S3. | Backup storage locations to read from in the event of eigenda retrieval failure. | -| `--storage.cache-targets` | `[]` | `$EIGENDA_PROXY_STORAGE_CACHE_TARGETS` | Caching targets. Supports S3. | Caches data to backend targets after dispersing to DA, retrieved from before trying read from EigenDA. | -| `--storage.concurrent-write-threads` | `0` | `$EIGENDA_PROXY_STORAGE_CONCURRENT_WRITE_THREADS` | Number of threads spun-up for async secondary storage insertions. (<=0) denotes single threaded insertions where (>0) indicates decoupled writes. | -| `--s3.timeout` | `5s` | `$EIGENDA_PROXY_S3_TIMEOUT` | timeout for S3 storage operations (e.g. get, put) | -| `--redis.db` | `0` | `$EIGENDA_PROXY_REDIS_DB` | redis database to use after connecting to server | -| `--redis.endpoint` | `""` | `$EIGENDA_PROXY_REDIS_ENDPOINT` | redis endpoint url | -| `--redis.password` | `""` | `$EIGENDA_PROXY_REDIS_PASSWORD` | redis password | -| `--redis.eviction` | `24h0m0s` | `$EIGENDA_PROXY_REDIS_EVICTION` | entry eviction/expiration time | -| `--help, -h` | `false` | | Show help. | -| `--version, -v` | `false` | | Print the version. | +| Option | Default Value | Environment Variable | Description | +|--------|---------------|----------------------|-------------| +| `--addr` | `"127.0.0.1"` | `$EIGENDA_PROXY_ADDR` | Server listening address | +| `--eigenda.cache-path` | `"resources/SRSTables/"` | `$EIGENDA_PROXY_EIGENDA_TARGET_CACHE_PATH` | Directory path to SRS tables for caching. | +| `--eigenda.custom-quorum-ids` | | `$EIGENDA_PROXY_EIGENDA_CUSTOM_QUORUM_IDS` | Custom quorum IDs for writing blobs. Should not include default quorums 0 or 1. | +| `--eigenda.disable-point-verification-mode` | `false` | `$EIGENDA_PROXY_EIGENDA_DISABLE_POINT_VERIFICATION_MODE` | Disable point verification mode. This mode performs IFFT on data before writing and FFT on data after reading. Disabling requires supplying the entire blob for verification against the KZG commitment. | +| `--eigenda.disable-tls` | `false` | `$EIGENDA_PROXY_EIGENDA_GRPC_DISABLE_TLS` | Disable TLS for gRPC communication with the EigenDA disperser. Default is false. | +| --eigenda.cert-verification-disabled | `false` | `$EIGENDA_PROXY_EIGENDA_CERT_VERIFICATION_DISABLED` | Whether to verify certificates received from EigenDA disperser. | +| `--eigenda.disperser-rpc` | | `$EIGENDA_PROXY_EIGENDA_DISPERSER_RPC` | RPC endpoint of the EigenDA disperser. | +| `--eigenda.svc-manager-addr` | | `$EIGENDA_PROXY_EIGENDA_SERVICE_MANAGER_ADDR` | The deployed EigenDA service manager address. The list can be found here: | +| `--eigenda.eth-confirmation-depth` | `-1` | `$EIGENDA_PROXY_EIGENDA_ETH_CONFIRMATION_DEPTH` | The number of Ethereum blocks of confirmation that the DA bridging transaction must have before it is assumed by the proxy to be final. If set negative the proxy will always wait for blob finalization. | +| `--eigenda.eth-rpc` | | `$EIGENDA_PROXY_EIGENDA_ETH_RPC` | JSON RPC node endpoint for the Ethereum network used for finalizing DA blobs. See available list here: | +| `--eigenda.g1-path` | `"resources/g1.point"` | `$EIGENDA_PROXY_EIGENDA_TARGET_KZG_G1_PATH` | Directory path to g1.point file. | +| `--eigenda.g2-power-of-2-path` | `"resources/g2.point.powerOf2"` | `$EIGENDA_PROXY_EIGENDA_TARGET_KZG_G2_POWER_OF_2_PATH` | Directory path to g2.point.powerOf2 file. | +| `--eigenda.max-blob-length` | `"16MiB"` | `$EIGENDA_PROXY_EIGENDA_MAX_BLOB_LENGTH` | Maximum blob length to be written or read from EigenDA. Determines the number of SRS points loaded into memory for KZG commitments. Example units: '30MiB', '4Kb', '30MB'. Maximum size slightly exceeds 1GB. | +| `--eigenda.put-blob-encoding-version` | `0` | `$EIGENDA_PROXY_EIGENDA_PUT_BLOB_ENCODING_VERSION` | Blob encoding version to use when writing blobs from the high-level interface. | +| `--eigenda.response-timeout` | `60s` | `$EIGENDA_PROXY_EIGENDA_RESPONSE_TIMEOUT` | Total time to wait for a response from the EigenDA disperser. Default is 60 seconds. | +| `--eigenda.signer-private-key-hex` | | `$EIGENDA_PROXY_EIGENDA_SIGNER_PRIVATE_KEY_HEX` | Hex-encoded signer private key. This key should not be associated with an Ethereum address holding any funds. | +| `--eigenda.status-query-retry-interval` | `5s` | `$EIGENDA_PROXY_EIGENDA_STATUS_QUERY_INTERVAL` | Interval between retries when awaiting network blob finalization. Default is 5 seconds. | +| `--eigenda.status-query-timeout` | `30m0s` | `$EIGENDA_PROXY_EIGENDA_STATUS_QUERY_TIMEOUT` | Duration to wait for a blob to finalize after being sent for dispersal. Default is 30 minutes. | +| `--log.color` | `false` | `$EIGENDA_PROXY_LOG_COLOR` | Color the log output if in terminal mode. | +| `--log.format` | `text` | `$EIGENDA_PROXY_LOG_FORMAT` | Format the log output. Supported formats: 'text', 'terminal', 'logfmt', 'json', 'json-pretty'. | +| `--log.level` | `INFO` | `$EIGENDA_PROXY_LOG_LEVEL` | The lowest log level that will be output. | +| `--log.pid` | `false` | `$EIGENDA_PROXY_LOG_PID` | Show pid in the log. | +| `--memstore.enabled` | `false` | `$EIGENDA_PROXY_MEMSTORE_ENABLED` | Whether to use mem-store for DA logic. | +| `--memstore.expiration` | `25m0s` | `$EIGENDA_PROXY_MEMSTORE_EXPIRATION` | Duration that a mem-store blob/commitment pair are allowed to live. | +| `--memstore.put-latency` | `0` | `$EIGENDA_PROXY_MEMSTORE_PUT_LATENCY` | Artificial latency added for memstore backend to mimic EigenDA's dispersal latency. | +| `--memstore.get-latency` | `0` | `$EIGENDA_PROXY_MEMSTORE_GET_LATENCY` | Artificial latency added for memstore backend to mimic EigenDA's retrieval latency. | +| `--metrics.addr` | `"0.0.0.0"` | `$EIGENDA_PROXY_METRICS_ADDR` | Metrics listening address. | +| `--metrics.enabled` | `false` | `$EIGENDA_PROXY_METRICS_ENABLED` | Enable the metrics server. | +| `--metrics.port` | `7300` | `$EIGENDA_PROXY_METRICS_PORT` | Metrics listening port. | +| `--port` | `3100` | `$EIGENDA_PROXY_PORT` | Server listening port. | +| `--s3.credential-type` | | `$EIGENDA_PROXY_S3_CREDENTIAL_TYPE` | Static or iam. | +| `--s3.access-key-id` | | `$EIGENDA_PROXY_S3_ACCESS_KEY_ID` | Access key id for S3 storage. | +| `--s3.access-key-id` | | `$EIGENDA_PROXY_S3_ACCESS_KEY_ID` | Access key id for S3 storage. | +| `--s3.access-key-secret` | | `$EIGENDA_PROXY_S3_ACCESS_KEY_SECRET` | Access key secret for S3 storage. | +| `--s3.bucket` | | `$EIGENDA_PROXY_S3_BUCKET` | Bucket name for S3 storage. | +| `--s3.path` | | `$EIGENDA_PROXY_S3_PATH` | Bucket path for S3 storage. | +| `--s3.endpoint` | | `$EIGENDA_PROXY_S3_ENDPOINT` | Endpoint for S3 storage. | +| `--s3.enable-tls` | | `$EIGENDA_PROXY_S3_ENABLE_TLS` | Enable TLS connection to S3 endpoint. | +| `--storage.fallback-targets` | `[]` | `$EIGENDA_PROXY_STORAGE_FALLBACK_TARGETS` | Fall back backend targets. Supports S3. | Backup storage locations to read from in the event of eigenda retrieval failure. | +| `--storage.cache-targets` | `[]` | `$EIGENDA_PROXY_STORAGE_CACHE_TARGETS` | Caching targets. Supports S3. | Caches data to backend targets after dispersing to DA, retrieved from before trying read from EigenDA. | +| `--storage.concurrent-write-threads` | `0` | `$EIGENDA_PROXY_STORAGE_CONCURRENT_WRITE_THREADS` | Number of threads spun-up for async secondary storage insertions. (<=0) denotes single threaded insertions where (>0) indicates decoupled writes. | +| `--s3.timeout` | `5s` | `$EIGENDA_PROXY_S3_TIMEOUT` | timeout for S3 storage operations (e.g. get, put) | +| `--redis.db` | `0` | `$EIGENDA_PROXY_REDIS_DB` | redis database to use after connecting to server | +| `--redis.endpoint` | `""` | `$EIGENDA_PROXY_REDIS_ENDPOINT` | redis endpoint url | +| `--redis.password` | `""` | `$EIGENDA_PROXY_REDIS_PASSWORD` | redis password | +| `--redis.eviction` | `24h0m0s` | `$EIGENDA_PROXY_REDIS_EVICTION` | entry eviction/expiration time | +| `--help, -h` | `false` | | Show help. | +| `--version, -v` | `false` | | Print the version. | +| `--weavevm.enabled` | | `$EIGENDA_PROXY_WEAVE_VM_ENABLED` | Set to true to enable WeaveVM chain as a secondary storage. | +| `--weavevm.endpoint` | | `$EIGENDA_PROXY_WEAVE_VM_ENDPOINT` | Ednpoint of WeaveVM rpc node. | +| `--weavevm.chain_id` | | `$EIGENDA_PROXY_WEAVE_VM_CHAIN_ID` | Chain id of weaveVM network. Right now there is only Alphanet 9496 but in future can vary. | +| `--weavevm.timeout` | | `$EIGENDA_PROXY_WEAVE_VM_TIMEOUT` | WeaveVM HTTP requests operations timeout. | +| `--weavevm.private_key_hex` | | `$EIGENDA_PROXY_WEAVE_VM_PRIV_KEY_HEX` | WeaveVM signer private key. | +| `--weavevm.web3_signer_endpoint` | | `$EIGENDA_PROXY_WEAVE_VM_WEB3_SIGNER_ENDPOINT` | WeaveVM web3signer endpoint. | +| `--weavevm.web3_signer_tls_cert_file` | | `$EIGENDA_PROXY_WEAVE_VM_WEB3_SIGNER_TLS_CERT_FILE` | WeaveVM web3 signer path to TLS cert. | +| `--weavevm.web3_signer_tls_key_file` | | `$EIGENDA_PROXY_WEAVE_VM_WEB3_SIGNER_TLS_KEY_FILE` | WeaveVM web3 signer path to TLS key. | +| `--weavevm.web3_signer_tls_ca_cert_file` | | `$EIGENDA_PROXY_WEAVE_VM_WEB3_SIGNER_TLS_CA_CERT_FILE` | WeaveVM web3 signer path to CA cert.endpoint. | ### Certificate verification @@ -86,7 +96,6 @@ In order for the EigenDA Proxy to avoid a trust assumption on the EigenDA disper To target this feature, use the CLI flags `--eigenda-svc-manager-addr`, `--eigenda-eth-rpc`. - #### Soft Confirmations An optional `--eigenda-eth-confirmation-depth` flag can be provided to specify a number of ETH block confirmations to wait before verifying the blob certificate. This allows for blobs to be accredited upon `confirmation` versus waiting (e.g, 25-30m) for `finalization`. The following integer expressions are supported: @@ -99,13 +108,14 @@ An optional `--eigenda-eth-confirmation-depth` flag can be provided to specify a An ephemeral memory store backend can be used for faster feedback testing when testing rollup integrations. To target this feature, use the CLI flags `--memstore.enabled`, `--memstore.expiration`. ### Asynchronous Secondary Insertions + An optional `--routing.concurrent-write-routines` flag can be provided to enable asynchronous processing for secondary writes - allowing for more efficient dispersals in the presence of a hefty secondary routing layer. This flag specifies the number of write routines spun-up with supported thread counts in range `[1, 100)`. ### Storage Fallback -An optional storage fallback CLI flag `--routing.fallback-targets` can be leveraged to ensure resiliency when **reading**. When enabled, a blob is persisted to a fallback target after being successfully dispersed. Fallback targets use the keccak256 hash of the existing EigenDA commitment as their key, for succinctness. In the event that blobs cannot be read from EigenDA, they will then be retrieved in linear order from the provided fallback targets. + +An optional storage fallback CLI flag `--routing.fallback-targets` can be leveraged to ensure resiliency when **reading**. When enabled, a blob is persisted to a fallback target after being successfully dispersed. Fallback targets use the keccak256 hash of the existing EigenDA commitment as their key, for succinctness. In the event that blobs cannot be read from EigenDA, they will then be retrieved in linear order from the provided fallback targets. ### Storage Caching -An optional storage caching CLI flag `--routing.cache-targets` can be leveraged to ensure less redundancy and more optimal reading. When enabled, a blob is persisted to each cache target after being successfully dispersed using the keccak256 hash of the existing EigenDA commitment for the fallback target key. This ensure second order keys are succinct. Upon a blob retrieval request, the cached targets are first referenced to read the blob data before referring to EigenDA. ### Failover Signals In the event that the EigenDA disperser or network is down, the proxy will return a 503 (Service Unavailable) status code as a response to POST requests, which rollup batchers can use to failover and start submitting blobs to the L1 chain instead. For more info, see our failover designs for [op-stack](https://github.com/ethereum-optimism/specs/issues/434) and for [arbitrum](https://hackmd.io/@epociask/SJUyIZlZkx). @@ -175,10 +185,13 @@ We also provide network-specific example env configuration files in `.env.exampl Container can be built via running `make docker-build`. ## Commitment Schemas + Currently, there are two commitment modes supported with unique encoding schemas for each. The `version byte` is shared for all modes and denotes which version of the EigenDA certificate is being used/requested. The following versions are currently supported: + * `0x0`: V0 certificate type (i.e, dispersal blob info struct with verification against service manager) ### Optimism Commitment Mode + For `alt-da` clients running on Optimism, the following commitment schema is supported: ``` @@ -213,6 +226,7 @@ The `raw commitment` is an RLP-encoded [EigenDA certificate](https://github.com/ Unit tests can be ran via invoking `make test`. ### Integration + End-to-end (E2E) tests can be ran via `make e2e-test`. ### Holesky @@ -223,15 +237,50 @@ A holesky integration test can be ran using `make holesky-test` to assert proper An E2E test exists which spins up a local OP sequencer instance using the [op-e2e](https://github.com/ethereum-optimism/optimism/tree/develop/op-e2e) framework for asserting correct interaction behaviors with batch submission and state derivation. These tests can be ran via `make optimism-test`. -## Metrics +## WeaveVM: Secondary Blobs Storage + +[WeaveVM](https://wvm.dev) is a sovereign EVM rollup that uses Arweave for permanent ledger archival and data storage. In this scope, WeaveVM provides an EVM gateway/interface for EigenDA blobs on Arweave's Permaweb, removing the need for trust assumptions and relying on centralized third party services to sync historical data and provides a "pay once, save forever" data storage feature for EigenDA blobs. + +### Key Details -To the see list of available metrics, run `./bin/eigenda-proxy doc metrics` +* WeaveVM provides a gateway for Arweave's permanent with its own (WeaveVM) high data throughput of the permanently stored data into EigenDA. +* Current maximum encoded blob size is 8 MB (8_388_608 bytes). +* ***WeaveVM currently operating in public testnet (Alphanet) - not recommended to use it in production environment.*** -To quickly set up monitoring dashboard, add eigenda-proxy metrics endpoint to a reachable prometheus server config as a scrape target, add prometheus datasource to Grafana to, and import the existing [Grafana dashboard JSON file](./grafana_dashboard.json) +### Prerequisites and Resources +1. Review the configuration parameters table and `.env` file settings for the Holesky network. +2. Obtain test tWVM tokens through our [faucet](https://wvm.dev/faucet) for testing purposes. +3. Monitor your transactions using the [WeaveVM explorer](https://explorer.wvm.dev). +4. For the most up-to-date guidelines and detailed instructions on using the EigenDA-WeaveVM integration, please refer to our [official documentation](https://docs.wvm.dev/da-integrations/weavevm-eigenda-proxy-server) + +### Example of Booting EigenDA proxy with WeaveVM as a Secondary Storage + +You may follow the general path to boot EigenDA proxy server and just provide additional WeaveVM-related CLI config. Set `weavevm.private-key-hex` with your WeaveVM private key (you may also use web3signer as a remote signer for WeaveVM transactions -- please check the configuration params and WeaveVM docs). + +```log +./bin/eigenda-proxy \ + --addr 127.0.0.1 \ + --port 3100 \ + --eigenda.disperser-rpc disperser-holesky.eigenda.xyz:443 \ + --eigenda.signer-private-key-hex $PRIVATE_KEY \ + --eigenda.max-blob-length 8MiB \ + --eigenda.eth-rpc https://ethereum-holesky-rpc.publicnode.com \ + --eigenda.svc-manager-addr 0xD4A7E1Bd8015057293f0D0A557088c286942e84b \ + --weavevm.endpoint https://testnet-rpc.wvm.dev/ \ + --weavevm.chain_id 9496 \ + --weavevm.enabled \ + --weavevm.private_key_hex $WVM_PRIV_KEY \ + --storage.fallback-targets weavevm \ + --storage.concurrent-write-routines 2 +``` ## Resources * [op-stack](https://github.com/ethereum-optimism/optimism) * [Alt-DA spec](https://specs.optimism.io/experimental/alt-da.html) * [eigen da](https://github.com/Layr-Labs/eigenda) + +## License + +This repository is licensed under the [MIT License](./LICENSE) diff --git a/common/store.go b/common/store.go index 2d508538..5491cb7f 100644 --- a/common/store.go +++ b/common/store.go @@ -16,12 +16,12 @@ const ( RedisBackendType UnknownBackendType -) -var ( - ErrProxyOversizedBlob = fmt.Errorf("encoded blob is larger than max blob size") + WeaveVMBackendType ) +var ErrProxyOversizedBlob = fmt.Errorf("encoded blob is larger than max blob size") + func (b BackendType) String() string { switch b { case EigenDABackendType: @@ -32,6 +32,8 @@ func (b BackendType) String() string { return "S3" case RedisBackendType: return "Redis" + case WeaveVMBackendType: + return "WeaveVM" case UnknownBackendType: fallthrough default: @@ -51,6 +53,8 @@ func StringToBackendType(s string) BackendType { return S3BackendType case "redis": return RedisBackendType + case "weavevm": + return WeaveVMBackendType case "unknown": fallthrough default: diff --git a/e2e/main_test.go b/e2e/main_test.go index 0af174fb..e536a42a 100644 --- a/e2e/main_test.go +++ b/e2e/main_test.go @@ -24,6 +24,7 @@ import ( var ( runTestnetIntegrationTests bool // holesky tests runIntegrationTests bool // memstore tests + runWeaveVMTests bool ) // ParseEnv ... reads testing cfg fields. Go test flags don't work for this library due to the dependency on Optimism's E2E framework @@ -34,6 +35,7 @@ func ParseEnv() { if runIntegrationTests && runTestnetIntegrationTests { panic("only one of INTEGRATION=true or TESTNET=true env var can be set") } + runWeaveVMTests = os.Getenv("EIGENDA_PROXY_WEAVE_VM_PRIV_KEY_HEX") != "" } // TestMain ... run main controller @@ -80,7 +82,6 @@ func requireStandardClientSetGet(t *testing.T, ts e2e.TestSuite, blob []byte) { preimage, err := daClient.GetData(ts.Ctx, blobInfo) require.NoError(t, err) require.Equal(t, blob, preimage) - } // requireOPClientSetGet ... ensures that alt-da client can disperse and read a blob @@ -93,5 +94,4 @@ func requireOPClientSetGet(t *testing.T, ts e2e.TestSuite, blob []byte, precompu preimage, err := daClient.GetInput(ts.Ctx, commit) require.NoError(t, err) require.Equal(t, blob, preimage) - } diff --git a/e2e/server_test.go b/e2e/server_test.go index 380da2bf..2a281b2b 100644 --- a/e2e/server_test.go +++ b/e2e/server_test.go @@ -37,7 +37,6 @@ this test asserts that the data can be posted/read to EigenDA with a concurrent S3 backend configured */ func TestOptimismClientWithGenericCommitment(t *testing.T) { - if !runIntegrationTests && !runTestnetIntegrationTests { t.Skip("Skipping test as INTEGRATION or TESTNET env var not set") } @@ -124,11 +123,10 @@ func TestProxyCachingWithRedis(t *testing.T) { } /* - Ensure that fallback location is read from when EigenDA blob is not available. - This is done by setting the memstore expiration time to 1ms and waiting for the blob to expire - before attempting to read it. +Ensure that fallback location is read from when EigenDA blob is not available. +This is done by setting the memstore expiration time to 1ms and waiting for the blob to expire +before attempting to read it. */ - func TestProxyReadFallback(t *testing.T) { // test can't be ran against holesky since read failure case can't be manually triggered if !runIntegrationTests || runTestnetIntegrationTests { @@ -166,3 +164,49 @@ func TestProxyReadFallback(t *testing.T) { requireWriteReadSecondary(t, ts.Metrics.SecondaryRequestsTotal, common.S3BackendType) requireDispersalRetrievalEigenDA(t, ts.Metrics.HTTPServerRequestsTotal, commitments.Standard) } + +/* +Tests fallback when weaveVM secondary backend is used. +Works only when EIGENDA_PROXY_WEAVE_VM_PRIV_KEY is set +*/ +func TestProxyReadFallbackOnWvm(t *testing.T) { + if !runWeaveVMTests { + t.Skip("Skipping test as EIGENDA_PROXY_WEAVE_VM_PRIV_KEY has not been set") + } + + // test can't be ran against holesky since read failure case can't be manually triggered + if !runIntegrationTests || runTestnetIntegrationTests { + t.Skip("Skipping test as INTEGRATION env var not set") + } + + t.Parallel() + + // setup server with WeaveVM as a fallback option + testCfg := e2e.TestConfig(useMemory()) + testCfg.UseWeaveVMFallback = true + // ensure that blob memstore eviction times result in near immediate activation + testCfg.Expiration = time.Millisecond * 1 + + tsConfig := e2e.TestSuiteConfig(testCfg) + ts, kill := e2e.CreateTestSuite(tsConfig) + defer kill() + + cfg := &client.Config{ + URL: ts.Address(), + } + daClient := client.New(cfg) + expectedBlob := e2e.RandBytes(1_000_000) + t.Log("Setting input data on proxy server...") + blobInfo, err := daClient.SetData(ts.Ctx, expectedBlob) + require.NoError(t, err) + + time.Sleep(1 * time.Second) + t.Log("Getting input data from proxy server...") + actualBlob, err := daClient.GetData(ts.Ctx, blobInfo) + require.NoError(t, err) + require.Equal(t, expectedBlob, actualBlob) + + requireStandardClientSetGet(t, ts, e2e.RandBytes(1_000_000)) + requireWriteReadSecondary(t, ts.Metrics.SecondaryRequestsTotal, common.WeaveVMBackendType) + requireDispersalRetrievalEigenDA(t, ts.Metrics.HTTPServerRequestsTotal, commitments.Standard) +} diff --git a/e2e/setup.go b/e2e/setup.go index 7ddd5041..12a105d0 100644 --- a/e2e/setup.go +++ b/e2e/setup.go @@ -15,6 +15,8 @@ import ( "github.com/Layr-Labs/eigenda-proxy/store/generated_key/memstore" "github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/redis" "github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/s3" + weavevm "github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/weave_vm/types" + "github.com/Layr-Labs/eigenda-proxy/verify" "github.com/Layr-Labs/eigenda/api/clients" "github.com/Layr-Labs/eigenda/encoding/kzg" @@ -113,6 +115,7 @@ type Cfg struct { UseS3Caching bool UseRedisCaching bool UseS3Fallback bool + UseWeaveVMFallback bool } func TestConfig(useMemory bool) *Cfg { @@ -123,6 +126,7 @@ func TestConfig(useMemory bool) *Cfg { UseS3Caching: false, UseRedisCaching: false, UseS3Fallback: false, + UseWeaveVMFallback: false, WriteThreadCount: 0, } } @@ -158,6 +162,21 @@ func createS3Config(eigendaCfg server.Config) server.CLIConfig { } } +func createWeaveVMConfig(eigendaCfg server.Config) server.CLIConfig { + pkHex := os.Getenv("EIGENDA_PROXY_WEAVE_VM_PRIV_KEY_HEX") + eigendaCfg.StorageConfig.WeaveVMConfig = weavevm.Config{ + Enabled: true, + Endpoint: "https://testnet-rpc.wvm.dev/", + ChainID: 9496, + // set higher than 5s in e2e tests + Timeout: 10 * time.Second, + PrivateKeyHex: pkHex, + } + return server.CLIConfig{ + EigenDAConfig: eigendaCfg, + } +} + func TestSuiteConfig(testCfg *Cfg) server.CLIConfig { // load signer key from environment pk := os.Getenv(privateKey) @@ -236,6 +255,10 @@ func TestSuiteConfig(testCfg *Cfg) server.CLIConfig { eigendaCfg.StorageConfig.FallbackTargets = []string{"S3"} cfg = createS3Config(eigendaCfg) + case testCfg.UseWeaveVMFallback: + eigendaCfg.StorageConfig.FallbackTargets = []string{"weavevm"} + cfg = createWeaveVMConfig(eigendaCfg) + case testCfg.UseRedisCaching: eigendaCfg.StorageConfig.CacheTargets = []string{"redis"} cfg = createRedisConfig(eigendaCfg) @@ -272,7 +295,6 @@ func CreateTestSuite(testSuiteCfg server.CLIConfig) (TestSuite, func()) { log, m, ) - if err != nil { panic(err) } @@ -339,7 +361,7 @@ func createS3Bucket(bucketName string) { } func RandStr(n int) string { - var letterRunes = []rune("abcdefghijklmnopqrstuvwxyz") + letterRunes := []rune("abcdefghijklmnopqrstuvwxyz") b := make([]rune, n) for i := range b { b[i] = letterRunes[rand.Intn(len(letterRunes))] diff --git a/flags/flags.go b/flags/flags.go index 103afeab..cebeea1e 100644 --- a/flags/flags.go +++ b/flags/flags.go @@ -6,6 +6,7 @@ import ( "github.com/Layr-Labs/eigenda-proxy/store/generated_key/memstore" "github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/redis" "github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/s3" + weavevm "github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/weave_vm" "github.com/Layr-Labs/eigenda-proxy/verify" "github.com/urfave/cli/v2" @@ -24,6 +25,7 @@ const ( S3Category = "S3 Cache/Fallback" VerifierCategory = "KZG and Cert Verifier" VerifierDeprecatedCategory = "DEPRECATED VERIFIER FLAGS -- THESE WILL BE REMOVED IN V2.0.0" + WeaveVMCategory = "WeaveVM Fallback/Perm Storage option" ) const ( @@ -73,4 +75,5 @@ func init() { Flags = append(Flags, memstore.CLIFlags(EnvVarPrefix, MemstoreFlagsCategory)...) Flags = append(Flags, verify.CLIFlags(EnvVarPrefix, VerifierCategory)...) Flags = append(Flags, verify.DeprecatedCLIFlags(EnvVarPrefix, VerifierDeprecatedCategory)...) + Flags = append(Flags, weavevm.CLIFlags(EnvVarPrefix, WeaveVMCategory)...) } diff --git a/go.mod b/go.mod index c77ad910..93bc3be6 100644 --- a/go.mod +++ b/go.mod @@ -211,6 +211,7 @@ require ( github.com/opencontainers/image-spec v1.1.0 // indirect github.com/opencontainers/runtime-spec v1.2.0 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect + github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 // indirect github.com/pingcap/errors v0.11.4 // indirect diff --git a/go.sum b/go.sum index c6077c9c..44c289e9 100644 --- a/go.sum +++ b/go.sum @@ -694,6 +694,8 @@ github.com/ory/dockertest/v3 v3.10.0 h1:4K3z2VMe8Woe++invjaTB7VRyQXQy5UY+loujO4a github.com/ory/dockertest/v3 v3.10.0/go.mod h1:nr57ZbRWMqfsdGdFNLHz5jjNdDb7VVFnzAeW1n5N1Lg= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= +github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM= diff --git a/server/load_store.go b/server/load_store.go index cb75a6ae..d155e632 100644 --- a/server/load_store.go +++ b/server/load_store.go @@ -11,6 +11,7 @@ import ( "github.com/Layr-Labs/eigenda-proxy/store/generated_key/memstore" "github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/redis" "github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/s3" + weavevm "github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/weave_vm" "github.com/Layr-Labs/eigenda-proxy/verify" "github.com/Layr-Labs/eigenda/api/clients" "github.com/ethereum/go-ethereum/log" @@ -19,7 +20,7 @@ import ( // TODO - create structured abstraction for dependency injection vs. overloading stateless functions // populateTargets ... creates a list of storage backends based on the provided target strings -func populateTargets(targets []string, s3 common.PrecomputedKeyStore, redis *redis.Store) []common.PrecomputedKeyStore { +func populateTargets(targets []string, s3 common.PrecomputedKeyStore, redis *redis.Store, weaveVM common.PrecomputedKeyStore) []common.PrecomputedKeyStore { stores := make([]common.PrecomputedKeyStore, len(targets)) for i, f := range targets { @@ -38,6 +39,12 @@ func populateTargets(targets []string, s3 common.PrecomputedKeyStore, redis *red } stores[i] = s3 + case common.WeaveVMBackendType: + if weaveVM == nil { + panic(fmt.Sprintf("WeaveVM backend is not configured but specified in targets: %s", f)) + } + stores[i] = weaveVM + case common.EigenDABackendType, common.MemoryBackendType: panic(fmt.Sprintf("Invalid target for fallback: %s", f)) @@ -58,6 +65,7 @@ func LoadStoreManager(ctx context.Context, cfg CLIConfig, log log.Logger, m metr var err error var s3Store *s3.Store var redisStore *redis.Store + var weaveVMStore *weavevm.Store if cfg.EigenDAConfig.StorageConfig.S3Config.Bucket != "" && cfg.EigenDAConfig.StorageConfig.S3Config.Endpoint != "" { log.Info("Using S3 backend") @@ -67,6 +75,16 @@ func LoadStoreManager(ctx context.Context, cfg CLIConfig, log log.Logger, m metr } } + if cfg.EigenDAConfig.StorageConfig.WeaveVMConfig.Enabled { + if cfg.EigenDAConfig.StorageConfig.WeaveVMConfig.Endpoint != "" { + log.Info("Using WeaveVM backend") + weaveVMStore, err = weavevm.NewStore(&cfg.EigenDAConfig.StorageConfig.WeaveVMConfig, log) + if err != nil { + return nil, fmt.Errorf("failed to create WeaveVM store: %w", err) + } + } + } + if cfg.EigenDAConfig.StorageConfig.RedisConfig.Endpoint != "" { log.Info("Using Redis backend") // create Redis backend store @@ -122,8 +140,8 @@ func LoadStoreManager(ctx context.Context, cfg CLIConfig, log log.Logger, m metr } // create secondary storage router - fallbacks := populateTargets(cfg.EigenDAConfig.StorageConfig.FallbackTargets, s3Store, redisStore) - caches := populateTargets(cfg.EigenDAConfig.StorageConfig.CacheTargets, s3Store, redisStore) + fallbacks := populateTargets(cfg.EigenDAConfig.StorageConfig.FallbackTargets, s3Store, redisStore, weaveVMStore) + caches := populateTargets(cfg.EigenDAConfig.StorageConfig.CacheTargets, s3Store, redisStore, weaveVMStore) secondary := store.NewSecondaryManager(log, m, caches, fallbacks) if secondary.Enabled() { // only spin-up go routines if secondary storage is enabled @@ -135,6 +153,6 @@ func LoadStoreManager(ctx context.Context, cfg CLIConfig, log log.Logger, m metr } } - log.Info("Creating storage router", "eigenda backend type", eigenDA != nil, "s3 backend type", s3Store != nil) + log.Info("Creating storage router", "eigenda backend type", eigenDA != nil, "s3 backend type", s3Store != nil, "weaveVM backend type", weaveVMStore != nil) return store.NewManager(eigenDA, s3Store, log, secondary) } diff --git a/store/cli.go b/store/cli.go index 7801ff12..06872ce0 100644 --- a/store/cli.go +++ b/store/cli.go @@ -3,6 +3,7 @@ package store import ( "github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/redis" "github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/s3" + weavevm "github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/weave_vm" "github.com/urfave/cli/v2" ) @@ -55,5 +56,6 @@ func ReadConfig(ctx *cli.Context) Config { CacheTargets: ctx.StringSlice(CacheTargetsFlagName), RedisConfig: redis.ReadConfig(ctx), S3Config: s3.ReadConfig(ctx), + WeaveVMConfig: weavevm.ReadConfig(ctx), } } diff --git a/store/generated_key/eigenda/eigenda.go b/store/generated_key/eigenda/eigenda.go index b67ce1d2..343a4890 100644 --- a/store/generated_key/eigenda/eigenda.go +++ b/store/generated_key/eigenda/eigenda.go @@ -73,6 +73,7 @@ func (e Store) Put(ctx context.Context, value []byte) ([]byte, error) { if err != nil { return nil, fmt.Errorf("EigenDA client failed to re-encode blob: %w", err) } + // WVM: check that the data is lower than 100kb - Set it in configs via proxy config // TODO: We should move this length check inside PutBlob if uint64(len(encodedBlob)) > e.cfg.MaxBlobSizeBytes { return nil, fmt.Errorf("%w: blob length %d, max blob size %d", common.ErrProxyOversizedBlob, len(value), e.cfg.MaxBlobSizeBytes) @@ -166,3 +167,51 @@ func (e Store) Verify(ctx context.Context, key []byte, value []byte) error { // verify DA certificate against EigenDA's batch metadata that's bridged to Ethereum return e.verifier.VerifyCert(ctx, &cert) } + +// GetWvmTxHashByCommitment uses commitment to get wvm tx hash from the internal map(temprorary hack) +// and returns it to the caller + +/* TODO: make it nice +func (e EigenDAStore) GetWvmTxHashByCommitment(ctx context.Context, key []byte) (string, error) { + e.log.Info("try get wvm tx hash using provided commitment") + var cert verify.Certificate + err := rlp.DecodeBytes(key, &cert) + if err != nil { + return "", fmt.Errorf("failed to decode DA cert to RLP format: %w", err) + } + + wvmTxHash, err := e.wvmClient.GetWvmTxHashByCommitment(ctx, &cert) + if err != nil { + return "", err + } + + return wvmTxHash, nil +} + +func (e EigenDAStore) GetBlobFromWvm(ctx context.Context, key []byte) ([]byte, error) { + var cert verify.Certificate + err := rlp.DecodeBytes(key, &cert) + if err != nil { + return nil, fmt.Errorf("failed to decode DA cert to RLP format: %w", err) + } + + wvmTxHash, err := e.wvmClient.GetWvmTxHashByCommitment(ctx, &cert) + if err != nil { + return nil, err + } + + e.log.Info("found wvm tx hash using provided commitment", "provided key", commitmentKey(cert.BlobVerificationProof.BatchId, cert.BlobVerificationProof.BlobIndex)) + + wvmDecodedBlob, err := e.wvmClient.GetBlobFromWvm(ctx, wvmTxHash) + if err != nil { + return nil, fmt.Errorf("failed to get eigenda blob from wvm: %w", err) + } + + decodedData, err := e.client.Codec.DecodeBlob(wvmDecodedBlob) + if err != nil { + return nil, fmt.Errorf("error decoding eigen blob: %w", err) + } + + return decodedData, nil +} +*/ diff --git a/store/generated_key/memstore/memstore_test.go b/store/generated_key/memstore/memstore_test.go index a8030515..4929f2b9 100644 --- a/store/generated_key/memstore/memstore_test.go +++ b/store/generated_key/memstore/memstore_test.go @@ -93,7 +93,6 @@ func TestExpiration(t *testing.T) { _, err = ms.Get(ctx, key) require.Error(t, err) - } func TestLatency(t *testing.T) { @@ -125,5 +124,4 @@ func TestLatency(t *testing.T) { _, err = ms.Get(ctx, key) require.NoError(t, err) require.GreaterOrEqual(t, time.Since(timeBeforeGet), getLatency) - } diff --git a/store/precomputed_key/s3/s3.go b/store/precomputed_key/s3/s3.go index fc468192..51f3eac3 100644 --- a/store/precomputed_key/s3/s3.go +++ b/store/precomputed_key/s3/s3.go @@ -37,16 +37,18 @@ func StringToCredentialType(s string) CredentialType { var _ common.PrecomputedKeyStore = (*Store)(nil) -type CredentialType string -type Config struct { - CredentialType CredentialType - Endpoint string - EnableTLS bool - AccessKeyID string - AccessKeySecret string - Bucket string - Path string -} +type ( + CredentialType string + Config struct { + CredentialType CredentialType + Endpoint string + EnableTLS bool + AccessKeyID string + AccessKeySecret string + Bucket string + Path string + } +) // Custom MarshalJSON function to control what gets included in the JSON output // TODO: Probably best would be to separate config from secrets everywhere. diff --git a/store/precomputed_key/weave_vm/cli.go b/store/precomputed_key/weave_vm/cli.go new file mode 100644 index 00000000..47ee662f --- /dev/null +++ b/store/precomputed_key/weave_vm/cli.go @@ -0,0 +1,107 @@ +package weave_vm + +import ( + "time" + + weaveVMtypes "github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/weave_vm/types" + "github.com/urfave/cli/v2" +) + +var ( + EnabledFlagName = withFlagPrefix("enabled") + EndpointFlagName = withFlagPrefix("endpoint") + ChainIDFlagName = withFlagPrefix("chain_id") + TimeoutFlagName = withFlagPrefix("timeout") + + PrivateKeyHexFlagName = withFlagPrefix("private_key_hex") + Web3SignerEndpointFlagName = withFlagPrefix("web3_signer_endpoint") + Web3SignerTLSCertFileFlagName = withFlagPrefix("web3_signer_tls_cert_file") + Web3SignerTLSKeyFileFlagName = withFlagPrefix("web3_signer_tls_key_file") + Web3SignerTLSCACertFileFlagName = withFlagPrefix("web3_signer_tls_ca_cert_file") +) + +func withFlagPrefix(s string) string { + return "weavevm." + s +} + +func withEnvPrefix(envPrefix, s string) []string { + return []string{envPrefix + "_WEAVE_VM_" + s} +} + +// CLIFlags ... used for WeaveVM backend configuration +// category is used to group the flags in the help output (see https://cli.urfave.org/v2/examples/flags/#grouping) +func CLIFlags(envPrefix, category string) []cli.Flag { + return []cli.Flag{ + &cli.BoolFlag{ + Name: EnabledFlagName, + Value: false, + Usage: "enable WeaveVM perm storage", + EnvVars: withEnvPrefix(envPrefix, "ENABLED"), + Category: category, + }, + &cli.StringFlag{ + Name: EndpointFlagName, + Usage: "endpoint for WeaveVM chain rpc", + EnvVars: withEnvPrefix(envPrefix, "ENDPOINT"), + Category: category, + }, + &cli.StringFlag{ + Name: ChainIDFlagName, + Usage: "chain ID of WeaveVM chain", + EnvVars: withEnvPrefix(envPrefix, "CHAIN_ID"), + Category: category, + }, + &cli.StringFlag{ + Name: PrivateKeyHexFlagName, + Usage: "private key hex of WeaveVM chain", + EnvVars: withEnvPrefix(envPrefix, "PRIVATE_KEY_HEX"), + Category: category, + }, + &cli.StringFlag{ + Name: Web3SignerEndpointFlagName, + Usage: "web3signer endpoint", + EnvVars: withEnvPrefix(envPrefix, "WEB3_SIGNER_ENDPOINT"), + Category: category, + }, + &cli.StringFlag{ + Name: Web3SignerTLSCertFileFlagName, + Usage: "web3signer tls cert file", + EnvVars: withEnvPrefix(envPrefix, "WEB3_SIGNER_TLS_CERT_FILE"), + Category: category, + }, + &cli.StringFlag{ + Name: Web3SignerTLSKeyFileFlagName, + Usage: "web3signer tls key file", + EnvVars: withEnvPrefix(envPrefix, "WEB3_SIGNER_TLS_KEY_FILE"), + Category: category, + }, + &cli.StringFlag{ + Name: Web3SignerTLSCACertFileFlagName, + Usage: "web3signer tls ca cert file", + EnvVars: withEnvPrefix(envPrefix, "WEB3_SIGNER_TLS_CERT_FILE"), + Category: category, + }, + &cli.DurationFlag{ + Name: TimeoutFlagName, + Usage: "timeout for WeaveVM HTTP requests operations (e.g. get, put)", + Value: 5 * time.Second, + EnvVars: withEnvPrefix(envPrefix, "TIMEOUT"), + Category: category, + }, + } +} + +func ReadConfig(ctx *cli.Context) weaveVMtypes.Config { + return weaveVMtypes.Config{ + Endpoint: ctx.String(EndpointFlagName), + ChainID: ctx.Int64(ChainIDFlagName), + Enabled: ctx.Bool(EnabledFlagName), + Timeout: ctx.Duration(TimeoutFlagName), + + PrivateKeyHex: ctx.String(PrivateKeyHexFlagName), + Web3SignerEndpoint: ctx.String(Web3SignerEndpointFlagName), + Web3SignerTLSCertFile: ctx.String(Web3SignerTLSCertFileFlagName), + Web3SignerTLSKeyFile: ctx.String(Web3SignerTLSKeyFileFlagName), + Web3SignerTLSCACertFile: ctx.String(Web3SignerTLSCACertFileFlagName), + } +} diff --git a/store/precomputed_key/weave_vm/rpc/wvm_rpc_client.go b/store/precomputed_key/weave_vm/rpc/wvm_rpc_client.go new file mode 100644 index 00000000..da3795aa --- /dev/null +++ b/store/precomputed_key/weave_vm/rpc/wvm_rpc_client.go @@ -0,0 +1,241 @@ +package rpc + +import ( + "context" + "encoding/hex" + "encoding/json" + "fmt" + "reflect" + "strconv" + "strings" + "time" + + weaveVMtypes "github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/weave_vm/types" + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + ethtypes "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" +) + +type Signer interface { + GetAccount(ctx context.Context) (common.Address, error) + SignTransaction(ctx context.Context, signData *weaveVMtypes.SignData) (string, error) +} + +// WeaveVM RPC client +type RPCClient struct { + log log.Logger + client *ethclient.Client + chainID int64 + signer Signer +} + +func NewWvmRPCClient(log log.Logger, cfg *weaveVMtypes.Config, signer Signer) (*RPCClient, error) { + client, err := ethclient.Dial(cfg.Endpoint) + if err != nil { + return nil, fmt.Errorf("failed to connect to the WeaveVM client: %w", err) + } + + ethRPCClient := &RPCClient{ + log: log, + client: client, + chainID: cfg.ChainID, + signer: signer, + } + + return ethRPCClient, nil +} + +func (rpc *RPCClient) SendTransaction(ctx context.Context, to string, data []byte) (string, error) { + gas, err := rpc.estimateGas(ctx, to, data) + if err != nil { + return "", fmt.Errorf("failed to store data in weaveVM: failed estimate gas: %w", err) + } + + weaveVMRawTx, err := rpc.createRawTransaction(ctx, to, string(data), gas) + if err != nil { + return "", fmt.Errorf("failed to store data in weaveVM: failed create transaction: %w", err) + } + + weaveVMTxHash, err := rpc.sendRawTransaction(ctx, weaveVMRawTx) + if err != nil { + return "", fmt.Errorf("failed to store data in weaveVM: failed to send transaction: %w", err) + } + + return weaveVMTxHash, nil +} + +// estimateGas tries estimates the suggested amount of gas that required to execute a given transaction. +func (rpc *RPCClient) estimateGas(ctx context.Context, to string, data []byte) (uint64, error) { + var ( + toAddr = common.HexToAddress(to) + bytesData []byte + err error + ) + + fromAddress, err := rpc.signer.GetAccount(ctx) + if err != nil { + return 0, fmt.Errorf("failed to estimate gas, no signer: %w", err) + } + + var encoded string + if string(data) != "" { + if ok := strings.HasPrefix(string(data), "0x"); !ok { + encoded = hexutil.Encode(data) + } + + bytesData, err = hexutil.Decode(encoded) + if err != nil { + return 0, err + } + } + + msg := ethereum.CallMsg{ + From: fromAddress, + To: &toAddr, + Gas: 0x00, + Data: bytesData, + } + + gas, err := rpc.client.EstimateGas(ctx, msg) + if err != nil { + return 0, err + } + + rpc.log.Debug("weaveVM: estimated tx gas price", "price", gas) + + return gas, nil +} + +// createRawTransaction creates a raw EIP-1559 transaction and returns it as a hex string. +func (rpc *RPCClient) createRawTransaction(ctx context.Context, to string, data string, gasLimit uint64) (string, error) { + baseFee, err := rpc.client.SuggestGasPrice(ctx) + if err != nil { + return "", err + } + + fromAddress, err := rpc.signer.GetAccount(ctx) + if err != nil { + return "", fmt.Errorf("failed to get an account from signer: %w", err) + } + nonce, err := rpc.client.PendingNonceAt(ctx, fromAddress) + if err != nil { + return "", err + } + + signData := weaveVMtypes.SignData{To: to, Data: data, GasLimit: gasLimit, GasFeeCap: baseFee, Nonce: nonce} + return rpc.signer.SignTransaction(ctx, &signData) +} + +func (rpc *RPCClient) sendRawTransaction(ctx context.Context, signedTxHex string) (string, error) { + var err error + var signedTxBytes []byte + + if strings.HasPrefix(signedTxHex, "0x") { + signedTxBytes, err = hexutil.Decode(signedTxHex) + if err != nil { + return "", fmt.Errorf("failed to decode signed transaction: %w", err) + } + } else { + signedTxBytes, err = hex.DecodeString(signedTxHex) + if err != nil { + return "", fmt.Errorf("failed to decode signed transaction: %w", err) + } + } + + tx := new(ethtypes.Transaction) + err = tx.UnmarshalBinary(signedTxBytes) + if err != nil { + err = rlp.DecodeBytes(signedTxBytes, tx) + if err != nil { + return "", fmt.Errorf("failed to parse signed transaction: %w", err) + } + } + + err = rpc.client.SendTransaction(ctx, tx) + if err != nil { + return "", err + } + + rpc.log.Info("weaveVM: successfully sent transaction", "tx hash", tx.Hash().String()) + + err = rpc.logReceipt(tx) + if err != nil { + rpc.log.Error("failed to log sent transaction receipt", "error", err) + } + + return tx.Hash().String(), nil +} + +func (rpc *RPCClient) logReceipt(tx *ethtypes.Transaction) error { + var txDetails Transaction + txBytes, err := tx.MarshalJSON() + if err != nil { + return err + } + if err := json.Unmarshal(txBytes, &txDetails); err != nil { + return err + } + + txDetails.TransactionTime = tx.Time().Format(time.RFC822) + txDetails.TransactionCost = tx.Cost().String() + + convertFields := []string{"Nonce", "MaxPriorityFeePerGas", "MaxFeePerGas", "Value", "Type", "Gas"} + for _, field := range convertFields { + if err := convertHexField(&txDetails, field); err != nil { + return err + } + } + + txJSON, err := json.MarshalIndent(txDetails, "", "\t") + if err != nil { + return err + } + + rpc.log.Debug("weaveVM: transaction receipt", "tx receipt", string(txJSON)) + return nil +} + +// Transaction represents the structure of the transaction JSON. +type Transaction struct { + Type string `json:"type"` + ChainID string `json:"chainId"` + Nonce string `json:"nonce"` + To string `json:"to"` + Gas string `json:"gas"` + GasPrice string `json:"gasPrice,omitempty"` + MaxPriorityFeePerGas string `json:"maxPriorityFeePerGas"` + MaxFeePerGas string `json:"maxFeePerGas"` + Value string `json:"value"` + Input string `json:"input"` + AccessList []string `json:"accessList"` + V string `json:"v"` + R string `json:"r"` + S string `json:"s"` + YParity string `json:"yParity"` + Hash string `json:"hash"` + TransactionTime string `json:"transactionTime,omitempty"` + TransactionCost string `json:"transactionCost,omitempty"` +} + +func convertHexField(tx *Transaction, field string) error { + typeOfTx := reflect.TypeOf(*tx) + txValue := reflect.ValueOf(tx).Elem() + hexStr := txValue.FieldByName(field).String() + intValue, err := strconv.ParseUint(hexStr[2:], 16, 64) + if err != nil { + return err + } + + decimalStr := strconv.FormatUint(intValue, 10) + _, ok := typeOfTx.FieldByName(field) + if !ok { + return fmt.Errorf("field %s does not exist in Transaction struct", field) + } + txValue.FieldByName(field).SetString(decimalStr) + + return nil +} diff --git a/store/precomputed_key/weave_vm/signer/private_key_signer.go b/store/precomputed_key/weave_vm/signer/private_key_signer.go new file mode 100644 index 00000000..5d2d3848 --- /dev/null +++ b/store/precomputed_key/weave_vm/signer/private_key_signer.go @@ -0,0 +1,116 @@ +package signer + +import ( + "bytes" + "context" + "crypto/ecdsa" + "encoding/hex" + "fmt" + "math/big" + "strings" + + weaveVMtypes "github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/weave_vm/types" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + ethtypes "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/log" +) + +type PrivateKeySigner struct { + privateKey string + chainID int64 + + log log.Logger +} + +func NewPrivateKeySigner(privateKey string, log log.Logger, chainID int64) *PrivateKeySigner { + return &PrivateKeySigner{privateKey: privateKey, log: log, chainID: chainID} +} + +func (pks *PrivateKeySigner) SignTransaction(_ context.Context, signData *weaveVMtypes.SignData) (string, error) { + return pks.signTxWithPrivateKey(signData.To, signData.Data, signData.GasFeeCap, signData.GasLimit, signData.Nonce) +} + +func (pks *PrivateKeySigner) signTxWithPrivateKey(to string, data string, gasFeeCap *big.Int, gasLimit uint64, nonce uint64) (string, error) { + pks.log.Info("sign transaction using private key") + + // Prepare data payload. + var hexData string + if strings.HasPrefix(data, "0x") { + hexData = data + } else { + hexData = hexutil.Encode([]byte(data)) + } + bytesData, err := hexutil.Decode(hexData) + if err != nil { + return "", err + } + + toAddr := common.HexToAddress(to) + txData := ethtypes.DynamicFeeTx{ + ChainID: big.NewInt(pks.chainID), + Nonce: nonce, + GasTipCap: big.NewInt(0), + GasFeeCap: gasFeeCap, + Gas: gasLimit, + To: &toAddr, + Data: bytesData, + } + tx := ethtypes.NewTx(&txData) + + pKeyBytes, err := hexutil.Decode("0x" + pks.privateKey) + if err != nil { + return "", err + } + // Convert the private key bytes to an ECDSA private key. + ecdsaPrivateKey, err := crypto.ToECDSA(pKeyBytes) + if err != nil { + return "", err + } + + signedTx, err := ethtypes.SignTx(tx, ethtypes.LatestSignerForChainID(big.NewInt(pks.chainID)), ecdsaPrivateKey) + if err != nil { + return "", err + } + + // Encode the signed transaction into RLP (Recursive Length Prefix) format for transmission. + var buf bytes.Buffer + err = signedTx.EncodeRLP(&buf) + if err != nil { + return "", err + } + + // Return the RLP-encoded transaction as a hexadecimal string. + rawTxRLPHex := hex.EncodeToString(buf.Bytes()) + + return rawTxRLPHex, nil +} + +func (pks *PrivateKeySigner) GetAccount(_ context.Context) (common.Address, error) { + account, _, err := pks.getAddressFromPrivateKey() + return account, err +} + +func (pks *PrivateKeySigner) getAddressFromPrivateKey() (common.Address, *ecdsa.PrivateKey, error) { + // Getting public address from private key + pKeyBytes, err := hexutil.Decode("0x" + pks.privateKey) + if err != nil { + return common.Address{}, nil, err + } + // Convert the private key bytes to an ECDSA private key. + ecdsaPrivateKey, err := crypto.ToECDSA(pKeyBytes) + if err != nil { + return common.Address{}, nil, err + } + // Extract the public key from the ECDSA private key. + publicKey := ecdsaPrivateKey.Public() + publicKeyECDSA, ok := publicKey.(*ecdsa.PublicKey) + if !ok { + return common.Address{}, nil, fmt.Errorf("error casting public key to ECDSA") + } + + // Compute the Ethereum address of the signer from the public key. + fromAddress := crypto.PubkeyToAddress(*publicKeyECDSA) + return fromAddress, ecdsaPrivateKey, nil +} diff --git a/store/precomputed_key/weave_vm/signer/web3_signer_client.go b/store/precomputed_key/weave_vm/signer/web3_signer_client.go new file mode 100644 index 00000000..b6af4fa3 --- /dev/null +++ b/store/precomputed_key/weave_vm/signer/web3_signer_client.go @@ -0,0 +1,253 @@ +package signer + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "io" + "math/big" + "net" + "net/http" + "os" + "strings" + + weaveVMtypes "github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/weave_vm/types" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/log" +) + +type Web3SignerClient struct { + chainID int64 + endpoint string + log log.Logger + client *http.Client +} + +func NewWeb3SignerClient(cfg *weaveVMtypes.Config, log log.Logger) (*Web3SignerClient, error) { + transport := &http.Transport{ + DialContext: (&net.Dialer{ + Timeout: cfg.Timeout, + }).DialContext, + TLSHandshakeTimeout: cfg.Timeout, + } + + // Configure TLS if cert and key files are provided + if cfg.Web3SignerTLSCertFile != "" && cfg.Web3SignerTLSKeyFile != "" { + err := configureTransportTLS(transport, cfg) + if err != nil { + return nil, err + } + } + + client := &http.Client{ + Transport: transport, + Timeout: cfg.Timeout, + } + + return &Web3SignerClient{ + endpoint: cfg.Web3SignerEndpoint, + log: log, + client: client, + chainID: cfg.ChainID, + }, nil +} + +func configureTransportTLS(transport *http.Transport, cfg *weaveVMtypes.Config) error { + tlsConfig := &tls.Config{ + MinVersion: tls.VersionTLS12, + } + + cert, err := tls.LoadX509KeyPair(cfg.Web3SignerTLSCertFile, cfg.Web3SignerTLSKeyFile) + if err != nil { + return fmt.Errorf("failed to load client certificate: %w", err) + } + tlsConfig.Certificates = []tls.Certificate{cert} + + // Load CA certificate if provided + if cfg.Web3SignerTLSCACertFile != "" { + caCert, err := os.ReadFile(cfg.Web3SignerTLSCACertFile) + if err != nil { + return fmt.Errorf("failed to read CA certificate: %w", err) + } + + caCertPool := x509.NewCertPool() + if !caCertPool.AppendCertsFromPEM(caCert) { + return fmt.Errorf("failed to parse CA certificate") + } + + tlsConfig.RootCAs = caCertPool + } else { + // If no CA cert provided, skip verification + log.Warn("No CA certificate provided, TLS verification will be skipped", + "endpoint", cfg.Web3SignerEndpoint) + tlsConfig.InsecureSkipVerify = true + } + + transport.TLSClientConfig = tlsConfig + log.Info("TLS configuration enabled for Web3Signer client", + "cert_file", cfg.Web3SignerTLSCertFile, + "key_file", cfg.Web3SignerTLSKeyFile, + "ca_file", cfg.Web3SignerTLSCACertFile) + + return nil +} + +func (web3s *Web3SignerClient) SignTransaction(ctx context.Context, signData *weaveVMtypes.SignData) (string, error) { + return web3s.signTxWithWeb3Signer(ctx, signData.To, signData.Data, signData.GasFeeCap, signData.GasLimit, signData.Nonce) +} + +func (web3s *Web3SignerClient) signTxWithWeb3Signer(ctx context.Context, to string, data string, gasFeeCap *big.Int, gasLimit, nonce uint64) (string, error) { + web3s.log.Info("sign transaction using web3signer") + + fromAddress, err := web3s.GetAccount(ctx) + if err != nil { + return "", fmt.Errorf("failed to get an account from web3signer rpc: %w", err) + } + + toAddr := common.HexToAddress(to) + // Prepare data payload. + var hexData string + if strings.HasPrefix(data, "0x") { + hexData = data + } else { + hexData = hexutil.Encode([]byte(data)) + } + bytesData, err := hexutil.Decode(hexData) + if err != nil { + return "", err + } + + // Prepare transaction for Web3Signer + tx := map[string]interface{}{ + "from": fromAddress.String(), + "to": toAddr.String(), + "gas": fmt.Sprintf("0x%x", gasLimit), + "maxFeePerGas": fmt.Sprintf("0x%x", gasFeeCap), + "maxPriorityFeePerGas": "0x0", + "value": "0x0", + "data": bytesData, + "nonce": fmt.Sprintf("0x%x", nonce), + "chainId": fmt.Sprintf("0x%x", web3s.chainID), + } + + // Sign transaction using Web3Signer + signedTx, err := web3s.signTransaction(ctx, tx) + if err != nil { + return "", fmt.Errorf("failed to sign transaction with web3signer: %w", err) + } + + return signedTx, nil +} + +func (web3s *Web3SignerClient) GetAccount(ctx context.Context) (common.Address, error) { + accounts, err := web3s.getAccounts(ctx) + if err != nil { + return common.Address{}, err + } + + return accounts[0], nil +} + +// GetAccount retrieves the account addresses from Web3Signer +func (web3s *Web3SignerClient) getAccounts(ctx context.Context) ([]common.Address, error) { + request := weaveVMtypes.SignerJSONRPCRequest{ + Version: "2.0", + Method: "eth_accounts", + Params: []interface{}{}, + ID: 1, + } + + response, err := web3s.doRequest(ctx, &request) + if err != nil { + return nil, fmt.Errorf("web3signer request failed: %w", err) + } + + // Parse the raw JSON result into a string array + var result []string + if err := json.Unmarshal(response.Result, &result); err != nil { + return nil, fmt.Errorf("failed to unmarshal addresses: %w", err) + } + + if len(result) == 0 { + return nil, fmt.Errorf("web3signer returned no accounts") + } + + addresses := make([]common.Address, len(result)) + for i, address := range result { + if !common.IsHexAddress(address) { + return nil, fmt.Errorf("invalid address format received: %s", address) + } + addresses[i] = common.HexToAddress(address) + } + + return addresses, nil +} + +// SignTransaction signs a transaction using Web3Signer +func (web3s *Web3SignerClient) signTransaction(ctx context.Context, tx interface{}) (string, error) { + request := weaveVMtypes.SignerJSONRPCRequest{ + Version: "2.0", + Method: "eth_signTransaction", + Params: []interface{}{tx}, + ID: 1, + } + + response, err := web3s.doRequest(ctx, &request) + if err != nil { + return "", fmt.Errorf("web3signer request failed: %w", err) + } + + // Parse the raw JSON result into a string + var signedTx string + if err := json.Unmarshal(response.Result, &signedTx); err != nil { + return "", fmt.Errorf("failed to unmarshal signed transaction: %w", err) + } + + if signedTx == "" { + return "", fmt.Errorf("web3signer returned empty signature") + } + + return signedTx, nil +} + +// doRequest performs the HTTP request to the Web3Signer endpoint +func (web3s *Web3SignerClient) doRequest(ctx context.Context, request *weaveVMtypes.SignerJSONRPCRequest) (*weaveVMtypes.SignerJSONRPCResponse, error) { + reqBody, err := json.Marshal(request) + if err != nil { + return nil, fmt.Errorf("failed to marshal request: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, web3s.endpoint, bytes.NewReader(reqBody)) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + + resp, err := web3s.client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to send request: %w", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + + var jsonRPCResponse weaveVMtypes.SignerJSONRPCResponse + if err := json.Unmarshal(body, &jsonRPCResponse); err != nil { + return nil, fmt.Errorf("failed to unmarshal response: %w", err) + } + + if jsonRPCResponse.Error != nil { + return nil, fmt.Errorf("web3signer error: %s (code: %d)", + jsonRPCResponse.Error.Message, + jsonRPCResponse.Error.Code) + } + + return &jsonRPCResponse, nil +} diff --git a/store/precomputed_key/weave_vm/types/signer.go b/store/precomputed_key/weave_vm/types/signer.go new file mode 100644 index 00000000..d98a6cfb --- /dev/null +++ b/store/precomputed_key/weave_vm/types/signer.go @@ -0,0 +1,33 @@ +package types + +import ( + "encoding/json" + "math/big" +) + +type SignData struct { + To string + Data string + GasLimit uint64 + GasFeeCap *big.Int + Nonce uint64 +} + +type SignerJSONRPCRequest struct { + Version string `json:"jsonrpc"` + Method string `json:"method"` + Params interface{} `json:"params"` + ID int64 `json:"id"` +} + +type SignerJSONRPCResponse struct { + Version string `json:"jsonrpc"` + Result json.RawMessage `json:"result"` + Error *SignerJSONRPCError `json:"error,omitempty"` + ID int64 `json:"id"` +} + +type SignerJSONRPCError struct { + Code int `json:"code"` + Message string `json:"message"` +} diff --git a/store/precomputed_key/weave_vm/types/types.go b/store/precomputed_key/weave_vm/types/types.go new file mode 100644 index 00000000..1a6f1a93 --- /dev/null +++ b/store/precomputed_key/weave_vm/types/types.go @@ -0,0 +1,36 @@ +package types + +import ( + "time" +) + +const ( + ArchivePoolAddress = "0x0000000000000000000000000000000000000000" // the data settling address, a unified standard across WeaveVM archiving services + WeaveVMMaxTransactionSize = 8_388_608 +) + +// Config...WeaveVM client configuration +type Config struct { + Enabled bool + // RPC endpoint of WeaveVM chain + Endpoint string + // WeaveVM chain id + ChainID int64 + // Timeout on WeaveVM calls in seconds + Timeout time.Duration + + // WeaveVm Private Key + PrivateKeyHex string + // Web3Signer configuration + Web3SignerEndpoint string + Web3SignerTLSCertFile string + Web3SignerTLSKeyFile string + Web3SignerTLSCACertFile string +} + +type RetrieverResponse struct { + ArweaveBlockHash string `json:"arweave_block_hash"` + Calldata string `json:"calldata"` + WarDecodedCalldata string `json:"war_decoded_calldata"` + WvmBlockHash string `json:"wvm_block_hash"` +} diff --git a/store/precomputed_key/weave_vm/wvm.go b/store/precomputed_key/weave_vm/wvm.go new file mode 100644 index 00000000..f3fc9d08 --- /dev/null +++ b/store/precomputed_key/weave_vm/wvm.go @@ -0,0 +1,210 @@ +package weave_vm + +import ( + "bytes" + "context" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/Layr-Labs/eigenda-proxy/common" + rpc "github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/weave_vm/rpc" + signer "github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/weave_vm/signer" + weaveVMtypes "github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/weave_vm/types" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/log" + cache "github.com/patrickmn/go-cache" +) + +type WeaveVM interface { + SendTransaction(ctx context.Context, to string, data []byte) (string, error) +} + +// Store...wraps weaveVM client, ethclient and concurrent internal cache +type Store struct { + client WeaveVM + log log.Logger + txCache *cache.Cache + cfg *weaveVMtypes.Config +} + +func NewStore(cfg *weaveVMtypes.Config, log log.Logger) (*Store, error) { + store := &Store{cfg: cfg, log: log, txCache: cache.New(24*15*time.Hour, 24*time.Hour)} + + if cfg.Web3SignerEndpoint != "" { + web3signer, err := signer.NewWeb3SignerClient(cfg, log) + if err != nil { + return nil, fmt.Errorf("failed to initialize web3signer client: %w", err) + } + client, err := rpc.NewWvmRPCClient(log, cfg, web3signer) + if err != nil { + return nil, fmt.Errorf("failed to initialize rpc client for weaveVM chain: %w", err) + } + store.client = client + return store, nil + } + + // Use PrivateKey signer + if cfg.PrivateKeyHex == "" { + return nil, fmt.Errorf("weaveVM private key is empty and weaveVM web3 signer is empty") + } + privateKeySigner := signer.NewPrivateKeySigner(cfg.PrivateKeyHex, log, cfg.ChainID) + client, err := rpc.NewWvmRPCClient(log, cfg, privateKeySigner) + if err != nil { + return nil, fmt.Errorf("failed to initialize rpc client for weaveVM chain: %w", err) + } + + store.client = client + + return store, nil +} + +func (weaveVM *Store) BackendType() common.BackendType { + return common.WeaveVMBackendType +} + +func (weaveVM *Store) Verify(_ context.Context, key []byte, value []byte) error { + h := crypto.Keccak256Hash(value) + if !bytes.Equal(h[:], key) { + return fmt.Errorf("key does not match value, expected: %s got: %s", hex.EncodeToString(key), h.Hex()) + } + + return nil +} + +func (weaveVM *Store) Put(ctx context.Context, key []byte, value []byte) error { + ctx, cancel := context.WithTimeout(ctx, weaveVM.cfg.Timeout) + defer cancel() + + weaveVMTxHash, err := weaveVM.client.SendTransaction(ctx, weaveVMtypes.ArchivePoolAddress, value) + if err != nil { + return fmt.Errorf("failed to send weaveVM transaction: %w", err) + } + + weaveVM.txCache.Set(string(key), weaveVMTxHash, cache.DefaultExpiration) + + weaveVM.log.Debug("weaveVM backend: save weaveVM tx hash - batch_id:blob_index in internal storage", + "tx hash", weaveVMTxHash, "provided key", string(key)) + + return nil +} + +func (weaveVM *Store) Get(ctx context.Context, key []byte) ([]byte, error) { + ctx, cancel := context.WithTimeout(ctx, weaveVM.cfg.Timeout) + defer cancel() + + weaveVMTxHash, err := weaveVM.getWvmTxHashByCommitment(key) + if err != nil { + return nil, err + } + + weaveVM.log.Info("weaveVM backend: found weaveVM tx hash using provided commitment key", "provided key", string(key)) + data, err := weaveVM.getFromGateway(ctx, weaveVMTxHash) + if err != nil { + return nil, fmt.Errorf("failed to get eigenda blob from weaveVM: %w", err) + } + + return data, nil +} + +// GetWvmTxHashByCommitment uses commitment to get weaveVM tx hash from the internal map(temprorary hack) +// and returns it to the caller +func (weaveVM *Store) getWvmTxHashByCommitment(key []byte) (string, error) { + weaveVMTxHash, found := weaveVM.txCache.Get(string(key)) + if !found { + weaveVM.log.Info("weaveVM backend: tx hash using provided commitment NOT FOUND", "provided key", string(key)) + return "", fmt.Errorf("weaveVM backend: tx hash for provided commitment not found") + } + + return weaveVMTxHash.(string), nil +} + +const weaveVMGatewayURL = "https://gateway.wvm.dev/v1/calldata/%s" + +// Modified get function with improved error handling +func (weaveVM *Store) getFromGateway(ctx context.Context, weaveVMTxHash string) ([]byte, error) { + type WvmRetrieverResponse struct { + ArweaveBlockHash string `json:"arweave_block_hash"` + Calldata string `json:"calldata"` + WarDecodedCalldata string `json:"war_decoded_calldata"` + WvmBlockHash string `json:"weaveVM_block_hash"` + } + + r, err := http.NewRequestWithContext(ctx, http.MethodGet, + fmt.Sprintf(weaveVMGatewayURL, + weaveVMTxHash), nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + r.Header.Set("Accept", "application/json") + client := &http.Client{ + Timeout: weaveVM.cfg.Timeout, + } + + weaveVM.log.Debug("sending request to WeaveVM data retriever", + "url", r.URL.String(), + "headers", r.Header) + + resp, err := client.Do(r) + if err != nil { + return nil, fmt.Errorf("failed to call weaveVM-data-retriever: %w", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + + if err := validateResponse(resp, body); err != nil { + weaveVM.log.Error("invalid response from WeaveVM data retriever", + "status", resp.Status, + "content_type", resp.Header.Get("Content-Type"), + "body", string(body)) + return nil, fmt.Errorf("invalid response: %w", err) + } + + var weaveVMData WvmRetrieverResponse + if err = json.Unmarshal(body, &weaveVMData); err != nil { + weaveVM.log.Error("failed to unmarshal response", + "error", err, + "body", string(body), + "content_type", resp.Header.Get("Content-Type")) + return nil, fmt.Errorf("failed to unmarshal response: %w, body: %s", err, string(body)) + } + + weaveVM.log.Info("weaveVM backend: get data from weaveVM", + "arweave_block_hash", weaveVMData.ArweaveBlockHash, + "weaveVM_block_hash", weaveVMData.WvmBlockHash, + "calldata_length", len(weaveVMData.Calldata)) + + calldataBlob, err := hexutil.Decode(weaveVMData.Calldata) + if err != nil { + return nil, fmt.Errorf("failed to decode calldata: %w", err) + } + + if len(calldataBlob) == 0 { + return nil, fmt.Errorf("decoded blob has length zero") + } + + return calldataBlob, nil +} + +func validateResponse(resp *http.Response, body []byte) error { + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("unexpected status code: %d, body: %s", resp.StatusCode, string(body)) + } + + contentType := resp.Header.Get("Content-Type") + if !strings.Contains(contentType, "application/json") { + return fmt.Errorf("unexpected content type: %s, body: %s", contentType, string(body)) + } + + return nil +} diff --git a/store/store.go b/store/store.go index 9821fce3..f8defb34 100644 --- a/store/store.go +++ b/store/store.go @@ -6,6 +6,8 @@ import ( "github.com/Layr-Labs/eigenda-proxy/common" "github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/redis" "github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/s3" + weavevm "github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/weave_vm/types" + "github.com/Layr-Labs/eigenda-proxy/verify" ) type Config struct { @@ -14,8 +16,9 @@ type Config struct { CacheTargets []string // secondary storage cfgs - RedisConfig redis.Config - S3Config s3.Config + RedisConfig redis.Config + S3Config s3.Config + WeaveVMConfig weavevm.Config } // checkTargets ... verifies that a backend target slice is constructed correctly @@ -52,6 +55,21 @@ func (cfg *Config) Check() error { return fmt.Errorf("redis password is set, but endpoint is not") } + // NOTE: we take the MaxBlobLengthBytes from verify package as it is done in memstore + // 8 MiB in bytes is 8_388_608 + if cfg.WeaveVMConfig.Enabled && verify.MaxBlobLengthBytes > weavevm.WeaveVMMaxTransactionSize { + return fmt.Errorf("current max blob size with weavevm secondary backend enabled is 8MiB") + } + if cfg.WeaveVMConfig.Enabled && (cfg.WeaveVMConfig.Endpoint == "" || cfg.WeaveVMConfig.ChainID == 0) { + return fmt.Errorf("weavevm secondary backend enabled: endpoint or chain id has not been provided") + } + if cfg.WeaveVMConfig.Enabled && (cfg.WeaveVMConfig.PrivateKeyHex == "" && cfg.WeaveVMConfig.Web3SignerEndpoint == "") { + return fmt.Errorf("weavevm secondary backend enabled: both private key and web3 signer endpoints are empty") + } + if cfg.WeaveVMConfig.Enabled && (cfg.WeaveVMConfig.PrivateKeyHex != "" && cfg.WeaveVMConfig.Web3SignerEndpoint != "") { + return fmt.Errorf("weavevm secondary backend is enabled: please provide either a private key or a Web3Signer endpoint as your signing method") + } + err := cfg.checkTargets(cfg.FallbackTargets) if err != nil { return err diff --git a/store/store_test.go b/store/store_test.go index 0f214aa3..ab7b1ad5 100644 --- a/store/store_test.go +++ b/store/store_test.go @@ -7,6 +7,8 @@ import ( "github.com/Layr-Labs/eigenda-proxy/store" "github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/redis" "github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/s3" + weavevm "github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/weave_vm/types" + "github.com/stretchr/testify/require" ) @@ -26,6 +28,13 @@ func validCfg() *store.Config { AccessKeyID: "access-key-id", AccessKeySecret: "access-key-secret", }, + WeaveVMConfig: weavevm.Config{ + Enabled: true, + Endpoint: "https://testnet-rpc.wvm.dev/", + ChainID: 9496, + Web3SignerEndpoint: "http://localhost:9000", + PrivateKeyHex: "", + }, } } @@ -112,4 +121,13 @@ func TestConfigVerification(t *testing.T) { err := cfg.Check() require.Error(t, err) }) + + t.Run("BadWeaveVMConfiguration", func(t *testing.T) { + cfg := validCfg() + cfg.WeaveVMConfig.Endpoint = "" + cfg.WeaveVMConfig.Enabled = true + + err := cfg.Check() + require.Error(t, err) + }) }