From 3d5f799429b9b0b47d325c26f596af143d4c9f00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Duchesneau?= Date: Fri, 22 Dec 2023 14:36:10 -0500 Subject: [PATCH] prepare release 1.3.0, fix service-deploy parameter override --- cmd/substreams/info.go | 3 +++ cmd/substreams/service-deploy.go | 30 ++++++++++++++++++++++-------- docs/release-notes/change-log.md | 13 ++++++++----- info/info.go | 15 ++++++++++----- 4 files changed, 43 insertions(+), 18 deletions(-) diff --git a/cmd/substreams/info.go b/cmd/substreams/info.go index ded73e11e..30eae0d13 100644 --- a/cmd/substreams/info.go +++ b/cmd/substreams/info.go @@ -80,6 +80,9 @@ func runInfo(cmd *cobra.Command, args []string) error { fmt.Println("Name:", mod.Name) fmt.Println("Initial block:", mod.InitialBlock) fmt.Println("Kind:", mod.Kind) + for _, input := range mod.Inputs { + fmt.Printf("Input: %s: %s\n", input.Type, input.Name) + } switch mod.Kind { case "map": diff --git a/cmd/substreams/service-deploy.go b/cmd/substreams/service-deploy.go index 7c43dc3ff..b9709744e 100644 --- a/cmd/substreams/service-deploy.go +++ b/cmd/substreams/service-deploy.go @@ -16,7 +16,9 @@ import ( func init() { serviceCmd.AddCommand(deployCmd) - deployCmd.Flags().StringArrayP("parameters", "p", []string{}, "Parameters to pass to the substreams") + deployCmd.Flags().StringArray("deployment-params", []string{}, "Extra parameters to pass to the deployment endpoint") + deployCmd.Flags().StringArrayP("params", "p", []string{}, "Parameters to pass to the substreams (ex: module2=key1=valX&key2=valY)") + deployCmd.Flags().StringP("network", "n", "", "Network to deploy to (overrides the 'network' field in the manifest)") deployCmd.Flags().Bool("prod", false, "Enable production mode (default: false)") } @@ -37,7 +39,19 @@ func deployE(cmd *cobra.Command, args []string) error { file := args[0] - reader, err := manifest.NewReader(file) + paramsString := sflags.MustGetStringArray(cmd, "params") + params, err := manifest.ParseParams(paramsString) + if err != nil { + return fmt.Errorf("parsing params: %w", err) + } + + network := sflags.MustGetString(cmd, "network") + readerOptions := []manifest.Option{ + manifest.WithOverrideNetwork(network), + manifest.WithParams(params), + } + + reader, err := manifest.NewReader(file, readerOptions...) if err != nil { return err } @@ -53,10 +67,10 @@ func deployE(cmd *cobra.Command, args []string) error { return fmt.Errorf("cannot deploy package %q: it does not specify a sink_module", file) } - //request parameters - // fmt.Println("request parameters") + pkg.Networks = nil // we don't want to send this to the server, so it does not apply network values again, possibly losing the overriden params + paramsMap := make(map[string]string) - for _, param := range mustGetStringArray(cmd, "parameters") { + for _, param := range mustGetStringArray(cmd, "deployment-params") { parts := strings.SplitN(param, "=", 2) if len(parts) != 2 { return fmt.Errorf("invalid parameter format: %q", param) @@ -64,9 +78,9 @@ func deployE(cmd *cobra.Command, args []string) error { paramsMap[parts[0]] = parts[1] } - params := []*pbsinksvc.Parameter{} + deployParams := []*pbsinksvc.Parameter{} for k, v := range paramsMap { - params = append(params, &pbsinksvc.Parameter{ + deployParams = append(deployParams, &pbsinksvc.Parameter{ Key: k, Value: v, }) @@ -74,7 +88,7 @@ func deployE(cmd *cobra.Command, args []string) error { req := connect.NewRequest(&pbsinksvc.DeployRequest{ SubstreamsPackage: pkg, - Parameters: params, + Parameters: deployParams, DevelopmentMode: !sflags.MustGetBool(cmd, "prod"), }) if err := addHeaders(cmd, req); err != nil { diff --git a/docs/release-notes/change-log.md b/docs/release-notes/change-log.md index 4271d48c1..3d46dc484 100644 --- a/docs/release-notes/change-log.md +++ b/docs/release-notes/change-log.md @@ -9,7 +9,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Unreleased +## v1.3.0 ### Highlights @@ -17,25 +17,28 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * [BREAKING CHANGE] Remove the support for the `deriveFrom` overrides. The `imports`, along with the new `networks` feature, should provide a better mechanism to cover the use cases that `deriveFrom` tried to address. {% hint style="info" %} -> These changes are all handled in the substreams CLI. The Substreams server endpoints do not need to be upgraded to support it. +> These changes are all handled in the substreams CLI, applying the necessary changes to the package before sending the requests. The Substreams server endpoints do not need to be upgraded to support it. {% endhint %} ### Added * Added `networks` field at the top level of the manifest definition, with `initialBlock` and `params` overrides for each module. See the substreams.yaml.example file in the repository or https://substreams.streamingfast.io/reference-and-specs/manifests for more details and example usage. * The networks `params` and `initialBlock`` overrides for the chosen network are applied to the module directly before being sent to the server. All network configurations are kept when packing an .spkg file. -* Added the `--network` flag for choosing the network on `run` and `gui` command. Default behavior is to use the one defined as `network` in the manifest. +* Added the `--network` flag for choosing the network on `run`, `gui` and `alpha service deploy` commands. Default behavior is to use the one defined as `network` in the manifest. * Added the `--endpoint` flag to `substreams alpha service serve` to specify substreams endpoint to connect to * Added endpoints for Antelope chains +* Command 'substreams info' now shows the params ### Removed * Removed the handling of the `DeriveFrom` keyword in manifest, this override feature is going away. * Removed the `--skip-package-validation`` option only on run/gui/inspect/info -### Fixed +### Changed -* The `substreams gui` command no longer reads the .spkg multiple times with different behavior during its process. +* Added the `--params` flag to `alpha service deploy` to apply per-module parameters to the substreams before pushing it. +* Renamed the `--parameters` flag to `--deployment-params` in `alpha service deploy`, to clarify the intent of those parameters (given to the endpoint, not applied to the substreams modules) +* Small improvement on `substreams gui` command: no longer reads the .spkg multiple times with different behavior during its process. ## v1.2.0 diff --git a/info/info.go b/info/info.go index 8aab10720..cdb8d6f21 100644 --- a/info/info.go +++ b/info/info.go @@ -88,13 +88,15 @@ func Basic(pkg *pbsubstreams.Package, graph *manifest.ModuleGraph) (*BasicInfo, } manifestInfo := &BasicInfo{ - Name: name, - Network: pkg.Network, - Version: version, - Image: pkg.Image, - Networks: make(map[string]*manifest.NetworkParams), + Name: name, + Network: pkg.Network, + Version: version, + Image: pkg.Image, } + if pkg.Networks != nil { + manifestInfo.Networks = make(map[string]*manifest.NetworkParams) + } for k, v := range pkg.Networks { params := &manifest.NetworkParams{} @@ -167,6 +169,9 @@ func Basic(pkg *pbsubstreams.Package, graph *manifest.ModuleGraph) (*BasicInfo, if v.Store.Mode > 0 { inputInfo.Mode = strPtr(v.Store.Mode.Pretty()) } + case *pbsubstreams.Module_Input_Params_: + inputInfo.Type = "params" + inputInfo.Name = input.GetParams().Value default: inputInfo.Type = "unknown" inputInfo.Name = "unknown"