diff --git a/cmd/substreams/remote-build.go b/cmd/substreams/remote-build.go new file mode 100644 index 00000000..b357efa5 --- /dev/null +++ b/cmd/substreams/remote-build.go @@ -0,0 +1,68 @@ +package main + +import ( + "context" + "fmt" + "os" + + "github.com/spf13/cobra" + "github.com/streamingfast/cli" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + + pbbuild "github.com/streamingfast/substreams/remotebuild/pb/sf/remotebuild/v1" +) + +func init() { + rootCmd.AddCommand(remoteBuildCmd) +} + +var remoteBuildCmd = &cobra.Command{ + Use: "remote-build ", + Short: "Send request to remote build server to build and package a substreams", + Long: cli.Dedent(` + Call the remote build server to build and package a substream. + + Examples: substreams remote-build https://substreams-remotebuild-endpoint:443 my-substream.zip + `), + RunE: remoteBuildE, + Args: cobra.ExactArgs(2), + SilenceUsage: true, +} + +func remoteBuildE(cmd *cobra.Command, args []string) error { + remoteBuildURL := args[0] + filepath := args[2] + + fmt.Println("Connecting to remote build server...") + conn, err := grpc.NewClient(remoteBuildURL, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + return fmt.Errorf("failed to connect: %w", err) + } + + fmt.Println("Reading file...") + b, err := os.ReadFile(filepath) + if err != nil { + return fmt.Errorf("failed to read file: %w", err) + } + + client := pbbuild.NewBuildServiceClient(conn) + buildResponse, err := client.Build(context.Background(), &pbbuild.BuildRequest{ + SourceCode: b, + Env: []string{"ENV1=test", ""}, + CollectPattern: "substreams.spkg", + }) + if err != nil { + return fmt.Errorf("failed to build: %w", err) + } + + for _, artifact := range buildResponse.Artifacts { + fmt.Printf("Writing file %s...\n", artifact.Filename) + err = os.WriteFile(artifact.Filename, artifact.Content, 0644) + if err != nil { + return fmt.Errorf("failed to write file: %w", err) + } + } + + return nil +} diff --git a/go.mod b/go.mod index 76620076..06092089 100644 --- a/go.mod +++ b/go.mod @@ -64,6 +64,7 @@ require ( github.com/streamingfast/shutter v1.5.0 github.com/streamingfast/substreams-sdk-go v0.0.0-20240110154316-5fb21a7a330b github.com/streamingfast/substreams-sink-sql v1.0.1-0.20231127153906-acf5f3e34330 + github.com/streamingfast/substreams/remotebuild v0.0.0-20240603152110-02be178a87b0 github.com/test-go/testify v1.1.4 github.com/tetratelabs/wazero v1.7.1 github.com/tidwall/pretty v1.2.1 @@ -71,7 +72,7 @@ require ( go.opentelemetry.io/otel v1.24.0 go.opentelemetry.io/otel/trace v1.24.0 go.uber.org/atomic v1.10.0 - golang.org/x/mod v0.12.0 + golang.org/x/mod v0.17.0 golang.org/x/net v0.22.0 golang.org/x/oauth2 v0.18.0 google.golang.org/grpc v1.64.0 @@ -82,7 +83,6 @@ require ( connectrpc.com/grpchealth v1.3.0 // indirect connectrpc.com/otelconnect v0.7.0 // indirect github.com/bits-and-blooms/bitset v1.12.0 // indirect - github.com/bmatcuk/doublestar v1.3.4 // indirect github.com/bobg/go-generics/v2 v2.1.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/google/s2a-go v0.1.7 // indirect @@ -91,7 +91,6 @@ require ( github.com/mattn/go-sqlite3 v1.14.16 // indirect github.com/mschoch/smat v0.2.0 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect - github.com/saracen/zipextra v0.0.0-20220303013732-0187cb0159ea // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect go.uber.org/goleak v1.3.0 // indirect golang.org/x/time v0.5.0 // indirect @@ -122,7 +121,6 @@ require ( github.com/benbjohnson/clock v1.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blendle/zapdriver v1.3.2-0.20200203083823-9200777f8a3d // indirect - github.com/bmatcuk/doublestar/v4 v4.6.1 github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect github.com/bufbuild/protocompile v0.4.0 // indirect github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect @@ -191,7 +189,6 @@ require ( github.com/prometheus/common v0.44.0 // indirect github.com/prometheus/procfs v0.11.0 // indirect github.com/rivo/uniseg v0.4.4 // indirect - github.com/saracen/fastzip v0.1.11 github.com/sethvargo/go-retry v0.2.3 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/afero v1.10.0 // indirect @@ -218,8 +215,8 @@ require ( go.opentelemetry.io/otel/sdk v1.23.1 // indirect go.uber.org/multierr v1.10.0 // indirect golang.org/x/crypto v0.21.0 // indirect - golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 - golang.org/x/sync v0.6.0 // indirect + golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc + golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect diff --git a/go.sum b/go.sum index 38e46bac..e741980a 100644 --- a/go.sum +++ b/go.sum @@ -146,10 +146,6 @@ github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6 github.com/blendle/zapdriver v1.3.1/go.mod h1:mdXfREi6u5MArG4j9fewC+FGnXaBR+T4Ox4J2u4eHCc= github.com/blendle/zapdriver v1.3.2-0.20200203083823-9200777f8a3d h1:fSlGu5ePbkjBidXuj2O5j9EcYrVB5Cr6/wdkYyDgxZk= github.com/blendle/zapdriver v1.3.2-0.20200203083823-9200777f8a3d/go.mod h1:yCBkgASmKHgUOFjK9h1sOytUVgA+JkQjqj3xYP4AdWY= -github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0= -github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE= -github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwNy7PA4I= -github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/bobg/go-generics/v2 v2.1.1 h1:4rN9upY6Xm4TASSMeH+NzUghgO4h/SbNrQphIjRd/R0= github.com/bobg/go-generics/v2 v2.1.1/go.mod h1:iPMSRVFlzkJSYOCXQ0n92RA3Vxw0RBv2E8j9ZODXgHk= github.com/btcsuite/btcd/btcec/v2 v2.2.0 h1:fzn1qaOt32TuLjFlkzYSsBC35Q3KUjT1SwPxiMSCF5k= @@ -545,10 +541,6 @@ github.com/rs/cors v1.10.0 h1:62NOS1h+r8p1mW6FM0FSB0exioXLhd/sh15KpjWBZ+8= github.com/rs/cors v1.10.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sahilm/fuzzy v0.1.0/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y= -github.com/saracen/fastzip v0.1.11 h1:NnExbTEJbya7148cov09BCxwfur9tQ5BQ1QyQH6XleA= -github.com/saracen/fastzip v0.1.11/go.mod h1:/lN5BiU451/OZMS+hfhVsSDj/RNrxYmO9EYxCtMrFrY= -github.com/saracen/zipextra v0.0.0-20220303013732-0187cb0159ea h1:8czYLkvzZRE+AElIQeDffQdgR+CC3wKEFILYU/1PeX4= -github.com/saracen/zipextra v0.0.0-20220303013732-0187cb0159ea/go.mod h1:hnzuad9d2wdd3z8fC6UouHQK5qZxqv3F/E6MMzXc7q0= github.com/schollz/closestmatch v2.1.0+incompatible h1:Uel2GXEpJqOWBrlyI+oY9LTiyyjYS17cCYRqP13/SHk= github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g= github.com/sethvargo/go-retry v0.2.3 h1:oYlgvIvsju3jNbottWABtbnoLC+GDtLdBHxKWxQm/iU= @@ -616,6 +608,8 @@ github.com/streamingfast/substreams-sdk-go v0.0.0-20240110154316-5fb21a7a330b h1 github.com/streamingfast/substreams-sdk-go v0.0.0-20240110154316-5fb21a7a330b/go.mod h1:TJXOmIAPY+FJvosBoMAiQeZzi32QiBXzCTSgVsss9Oo= github.com/streamingfast/substreams-sink-sql v1.0.1-0.20231127153906-acf5f3e34330 h1:svW/I3N8vW2JqJwE/cWXiULOEZi6eslxA60xHVqDaXk= github.com/streamingfast/substreams-sink-sql v1.0.1-0.20231127153906-acf5f3e34330/go.mod h1:4Zd5Re1SrhXDnO3VJh/FJcn64SyZPqAv6gFPjbyqMYI= +github.com/streamingfast/substreams/remotebuild v0.0.0-20240603152110-02be178a87b0 h1:JtorDjvHXSuGIVhNF8b1h3N4akImTcM4XwoV3KoVODE= +github.com/streamingfast/substreams/remotebuild v0.0.0-20240603152110-02be178a87b0/go.mod h1:BoDRKoaVY3lZHql/wQtSm0ohcm1VZrM77snOFpzMCsE= github.com/streamingfast/validator v0.0.0-20210812013448-b9da5752ce14 h1:/HCMj7plThWPFXsj/O8f+zDIAtXciwP4rK6tb+9UvPQ= github.com/streamingfast/validator v0.0.0-20210812013448-b9da5752ce14/go.mod h1:t4h97mWfTs6v0zjEFuGDOoW5wLtu9+ttegIx99i7gsM= github.com/streamingfast/wasmtime-go/v4 v4.0.0-freemem3 h1:raJHR0JWgYiSyX0vZ3leRK/TkNcn4ZUGTf+d64g48KQ= @@ -744,8 +738,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw= -golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc h1:O9NuF4s+E/PvMIy+9IUZB9znFwUIXEWSstNjek6VpVg= +golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -772,8 +766,8 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -849,8 +843,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=