feat: Integrate WeaveVM as a secondary backend in EigenDA proxy

.env.example.holesky

@@ -56,7 +56,7 @@ EIGENDA_PROXY_EIGENDA_SERVICE_MANAGER_ADDR=0xD4A7E1Bd8015057293f0D0A557088c28694
 # EIGENDA_PROXY_LOG_FORMAT=text
 
 # The lowest log level that will be output
-# EIGENDA_PROXY_LOG_LEVEL=INFO
+# EIGENDA_PROXY_LOG_LEVEL=DEBUG
 
 # Show pid in the log
 # EIGENDA_PROXY_LOG_PID=false
@@ -81,9 +81,40 @@ EIGENDA_PROXY_EIGENDA_SERVICE_MANAGER_ADDR=0xD4A7E1Bd8015057293f0D0A557088c28694
 
 # access key secret for S3 storage
 # EIGENDA_PROXY_S3_ACCESS_KEY_SECRET=
-          
+
 # bucket name for S3 storage
 # EIGENDA_PROXY_S3_BUCKET=
-          
+
 # endpoint for S3 storage
 # EIGENDA_PROXY_S3_ENDPOINT
+
+# same as EIGENDA_PROXY_MAX_BLOB_LENGTH but caps size when we use proxy
+# with WVM perm storage enabled Maximum size at this moment is 100kb.
+# WVM_EIGENDA_PROXY_MAX_BLOB_LENGTH=99Kb
+# WVM secondary storage related environment variables
+# NOTE: 8mb or (8388608 bytes) is a maximum allowed encoded blob size
+
+# Set to true to enable WVM chain as a secondary storage
+# EIGENDA_PROXY_WEAVE_VM_ENABLED=
+
+# WVM rpc endpoint
+# EIGENDA_PROXY_WEAVE_VM_ENDPOINT=https://testnet-rpc.wvm.dev/
+
+# WVM chain id
+# EIGENDA_PROXY_WEAVE_VM_CHAIN_ID=9496
+
+# WVM HTTP requests operations timeout
+# EIGENDA_PROXY_WEAVE_VM_TIMEOUT=5s
+
+# WVM web3signer endpoint
+# EIGENDA_PROXY_WEAVE_VM_WEB3_SIGNER_ENDPOINT=
+
+# WVM private key in case you don't use web3signer, not recommended
+# EIGENDA_PROXY_WEAVE_VM_PRIV_KEY_HEX=
+
+# WVM paths to TLS related files in case you use web3signer with TLS enabled
+# EIGENDA_PROXY_WEAVE_VM_WEB3_SIGNER_TLS_CERT_FILE=
+
+# EIGENDA_PROXY_WEAVE_VM_WEB3_SIGNER_TLS_KEY_FILE=
+
+# EIGENDA_PROXY_WEAVE_VM_WEB3_SIGNER_TLS_CA_CERT_FILE=

.env.example.mainnet

@@ -10,7 +10,7 @@ EIGENDA_PROXY_EIGENDA_ETH_RPC=
 EIGENDA_PROXY_EIGENDA_DISPERSER_RPC=disperser.eigenda.xyz:443
 
 # The deployed EigenDA service manager address. The list can be found here: https://github.com/Layr-Labs/eigenlayer-middleware/?tab=readme-ov-file#current-mainnet-deployment
-EIGENDA_PROXY_EIGENDA_SERVICE_MANAGER_ADDR=0x870679E138bCdf293b7Ff14dD44b70FC97e12fc0 
+EIGENDA_PROXY_EIGENDA_SERVICE_MANAGER_ADDR=0x870679E138bCdf293b7Ff14dD44b70FC97e12fc0
 
 # Custom quorum IDs for writing blobs. Should not include default quorums 0 or 1.
 # EIGENDA_PROXY_EIGENDA_CUSTOM_QUORUM_IDS=
@@ -85,4 +85,8 @@ EIGENDA_PROXY_EIGENDA_SERVICE_MANAGER_ADDR=0x870679E138bCdf293b7Ff14dD44b70FC97e
 # EIGENDA_PROXY_S3_BUCKET=
 
 # endpoint for S3 storage
-# EIGENDA_PROXY_S3_ENDPOINT
+# EIGENDA_PROXY_S3_ENDPOINT
+
+# same as EIGENDA_PROXY_MAX_BLOB_LENGTH but caps size when we use proxy
+# with WVM perm storage enabled Maximum size at this moment is 100kb.
+# WVM_EIGENDA_PROXY_MAX_BLOB_LENGTH=99Kb

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2024 Teddy Knox
+Copyright (c) 2024 WVM
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+SOFTWARE.

common/store.go

@@ -16,12 +16,12 @@ const (
 	RedisBackendType
 
 	UnknownBackendType
-)
 
-var (
-	ErrProxyOversizedBlob = fmt.Errorf("encoded blob is larger than max blob size")
+	WeaveVMBackendType
 )
 
+var ErrProxyOversizedBlob = fmt.Errorf("encoded blob is larger than max blob size")
+
 func (b BackendType) String() string {
 	switch b {
 	case EigenDABackendType:
@@ -32,6 +32,8 @@ func (b BackendType) String() string {
 		return "S3"
 	case RedisBackendType:
 		return "Redis"
+	case WeaveVMBackendType:
+		return "WeaveVM"
 	case UnknownBackendType:
 		fallthrough
 	default:
@@ -51,6 +53,8 @@ func StringToBackendType(s string) BackendType {
 		return S3BackendType
 	case "redis":
 		return RedisBackendType
+	case "weavevm":
+		return WeaveVMBackendType
 	case "unknown":
 		fallthrough
 	default:

e2e/main_test.go

@@ -24,6 +24,7 @@ import (
 var (
 	runTestnetIntegrationTests bool // holesky tests
 	runIntegrationTests        bool // memstore tests
+	runWeaveVMTests            bool
 )
 
 // ParseEnv ... reads testing cfg fields. Go test flags don't work for this library due to the dependency on Optimism's E2E framework
@@ -34,6 +35,7 @@ func ParseEnv() {
 	if runIntegrationTests && runTestnetIntegrationTests {
 		panic("only one of INTEGRATION=true or TESTNET=true env var can be set")
 	}
+	runWeaveVMTests = os.Getenv("EIGENDA_PROXY_WEAVE_VM_PRIV_KEY_HEX") != ""
 }
 
 // TestMain ... run main controller
@@ -80,7 +82,6 @@ func requireStandardClientSetGet(t *testing.T, ts e2e.TestSuite, blob []byte) {
 	preimage, err := daClient.GetData(ts.Ctx, blobInfo)
 	require.NoError(t, err)
 	require.Equal(t, blob, preimage)
-
 }
 
 // requireOPClientSetGet ... ensures that alt-da client can disperse and read a blob
@@ -93,5 +94,4 @@ func requireOPClientSetGet(t *testing.T, ts e2e.TestSuite, blob []byte, precompu
 	preimage, err := daClient.GetInput(ts.Ctx, commit)
 	require.NoError(t, err)
 	require.Equal(t, blob, preimage)
-
 }

e2e/server_test.go

@@ -37,7 +37,6 @@ this test asserts that the data can be posted/read to EigenDA
 with a concurrent S3 backend configured
 */
 func TestOptimismClientWithGenericCommitment(t *testing.T) {
-
 	if !runIntegrationTests && !runTestnetIntegrationTests {
 		t.Skip("Skipping test as INTEGRATION or TESTNET env var not set")
 	}
@@ -124,11 +123,10 @@ func TestProxyCachingWithRedis(t *testing.T) {
 }
 
 /*
-	Ensure that fallback location is read from when EigenDA blob is not available.
-	This is done by setting the memstore expiration time to 1ms and waiting for the blob to expire
-	before attempting to read it.
+Ensure that fallback location is read from when EigenDA blob is not available.
+This is done by setting the memstore expiration time to 1ms and waiting for the blob to expire
+before attempting to read it.
 */
-
 func TestProxyReadFallback(t *testing.T) {
 	// test can't be ran against holesky since read failure case can't be manually triggered
 	if !runIntegrationTests || runTestnetIntegrationTests {
@@ -166,3 +164,49 @@ func TestProxyReadFallback(t *testing.T) {
 	requireWriteReadSecondary(t, ts.Metrics.SecondaryRequestsTotal, common.S3BackendType)
 	requireDispersalRetrievalEigenDA(t, ts.Metrics.HTTPServerRequestsTotal, commitments.Standard)
 }
+
+/*
+Tests fallback when weaveVM secondary backend is used.
+Works only when EIGENDA_PROXY_WEAVE_VM_PRIV_KEY is set
+*/
+func TestProxyReadFallbackOnWvm(t *testing.T) {
+	if !runWeaveVMTests {
+		t.Skip("Skipping test as EIGENDA_PROXY_WEAVE_VM_PRIV_KEY has not been set")
+	}
+
+	// test can't be ran against holesky since read failure case can't be manually triggered
+	if !runIntegrationTests || runTestnetIntegrationTests {
+		t.Skip("Skipping test as INTEGRATION env var not set")
+	}
+
+	t.Parallel()
+
+	// setup server with WeaveVM as a fallback option
+	testCfg := e2e.TestConfig(useMemory())
+	testCfg.UseWeaveVMFallback = true
+	// ensure that blob memstore eviction times result in near immediate activation
+	testCfg.Expiration = time.Millisecond * 1
+
+	tsConfig := e2e.TestSuiteConfig(testCfg)
+	ts, kill := e2e.CreateTestSuite(tsConfig)
+	defer kill()
+
+	cfg := &client.Config{
+		URL: ts.Address(),
+	}
+	daClient := client.New(cfg)
+	expectedBlob := e2e.RandBytes(1_000_000)
+	t.Log("Setting input data on proxy server...")
+	blobInfo, err := daClient.SetData(ts.Ctx, expectedBlob)
+	require.NoError(t, err)
+
+	time.Sleep(1 * time.Second)
+	t.Log("Getting input data from proxy server...")
+	actualBlob, err := daClient.GetData(ts.Ctx, blobInfo)
+	require.NoError(t, err)
+	require.Equal(t, expectedBlob, actualBlob)
+
+	requireStandardClientSetGet(t, ts, e2e.RandBytes(1_000_000))
+	requireWriteReadSecondary(t, ts.Metrics.SecondaryRequestsTotal, common.WeaveVMBackendType)
+	requireDispersalRetrievalEigenDA(t, ts.Metrics.HTTPServerRequestsTotal, commitments.Standard)
+}

e2e/setup.go

@@ -15,6 +15,8 @@ import (
 	"github.com/Layr-Labs/eigenda-proxy/store/generated_key/memstore"
 	"github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/redis"
 	"github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/s3"
+	weavevm "github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/weave_vm/types"
+
 	"github.com/Layr-Labs/eigenda-proxy/verify"
 	"github.com/Layr-Labs/eigenda/api/clients"
 	"github.com/Layr-Labs/eigenda/encoding/kzg"
@@ -113,6 +115,7 @@ type Cfg struct {
 	UseS3Caching       bool
 	UseRedisCaching    bool
 	UseS3Fallback      bool
+	UseWeaveVMFallback bool
 }
 
 func TestConfig(useMemory bool) *Cfg {
@@ -123,6 +126,7 @@ func TestConfig(useMemory bool) *Cfg {
 		UseS3Caching:       false,
 		UseRedisCaching:    false,
 		UseS3Fallback:      false,
+		UseWeaveVMFallback: false,
 		WriteThreadCount:   0,
 	}
 }
@@ -158,6 +162,21 @@ func createS3Config(eigendaCfg server.Config) server.CLIConfig {
 	}
 }
 
+func createWeaveVMConfig(eigendaCfg server.Config) server.CLIConfig {
+	pkHex := os.Getenv("EIGENDA_PROXY_WEAVE_VM_PRIV_KEY_HEX")
+	eigendaCfg.StorageConfig.WeaveVMConfig = weavevm.Config{
+		Enabled:  true,
+		Endpoint: "https://testnet-rpc.wvm.dev/",
+		ChainID:  9496,
+		// set higher than 5s in e2e tests
+		Timeout:       10 * time.Second,
+		PrivateKeyHex: pkHex,
+	}
+	return server.CLIConfig{
+		EigenDAConfig: eigendaCfg,
+	}
+}
+
 func TestSuiteConfig(testCfg *Cfg) server.CLIConfig {
 	// load signer key from environment
 	pk := os.Getenv(privateKey)
@@ -236,6 +255,10 @@ func TestSuiteConfig(testCfg *Cfg) server.CLIConfig {
 		eigendaCfg.StorageConfig.FallbackTargets = []string{"S3"}
 		cfg = createS3Config(eigendaCfg)
 
+	case testCfg.UseWeaveVMFallback:
+		eigendaCfg.StorageConfig.FallbackTargets = []string{"weavevm"}
+		cfg = createWeaveVMConfig(eigendaCfg)
+
 	case testCfg.UseRedisCaching:
 		eigendaCfg.StorageConfig.CacheTargets = []string{"redis"}
 		cfg = createRedisConfig(eigendaCfg)
@@ -272,7 +295,6 @@ func CreateTestSuite(testSuiteCfg server.CLIConfig) (TestSuite, func()) {
 		log,
 		m,
 	)
-
 	if err != nil {
 		panic(err)
 	}
@@ -339,7 +361,7 @@ func createS3Bucket(bucketName string) {
 }
 
 func RandStr(n int) string {
-	var letterRunes = []rune("abcdefghijklmnopqrstuvwxyz")
+	letterRunes := []rune("abcdefghijklmnopqrstuvwxyz")
 	b := make([]rune, n)
 	for i := range b {
 		b[i] = letterRunes[rand.Intn(len(letterRunes))]

flags/flags.go

@@ -6,6 +6,7 @@ import (
 	"github.com/Layr-Labs/eigenda-proxy/store/generated_key/memstore"
 	"github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/redis"
 	"github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/s3"
+	weavevm "github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/weave_vm"
 	"github.com/Layr-Labs/eigenda-proxy/verify"
 	"github.com/urfave/cli/v2"
 
@@ -24,6 +25,7 @@ const (
 	S3Category                 = "S3 Cache/Fallback"
 	VerifierCategory           = "KZG and Cert Verifier"
 	VerifierDeprecatedCategory = "DEPRECATED VERIFIER FLAGS -- THESE WILL BE REMOVED IN V2.0.0"
+	WeaveVMCategory            = "WeaveVM Fallback/Perm Storage option"
 )
 
 const (
@@ -73,4 +75,5 @@ func init() {
 	Flags = append(Flags, memstore.CLIFlags(EnvVarPrefix, MemstoreFlagsCategory)...)
 	Flags = append(Flags, verify.CLIFlags(EnvVarPrefix, VerifierCategory)...)
 	Flags = append(Flags, verify.DeprecatedCLIFlags(EnvVarPrefix, VerifierDeprecatedCategory)...)
+	Flags = append(Flags, weavevm.CLIFlags(EnvVarPrefix, WeaveVMCategory)...)
 }

go.mod

@@ -211,6 +211,7 @@ require (
 	github.com/opencontainers/image-spec v1.1.0 // indirect
 	github.com/opencontainers/runtime-spec v1.2.0 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
+	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
 	github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 // indirect
 	github.com/pingcap/errors v0.11.4 // indirect

go.sum

@@ -694,6 +694,8 @@ github.com/ory/dockertest/v3 v3.10.0 h1:4K3z2VMe8Woe++invjaTB7VRyQXQy5UY+loujO4a
 github.com/ory/dockertest/v3 v3.10.0/go.mod h1:nr57ZbRWMqfsdGdFNLHz5jjNdDb7VVFnzAeW1n5N1Lg=
 github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
 github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM=