Skip to content

Commit

Permalink
prepare release 1.3.0, fix service-deploy parameter override
Browse files Browse the repository at this point in the history
  • Loading branch information
sduchesneau committed Dec 22, 2023
1 parent d377089 commit 3d5f799
Show file tree
Hide file tree
Showing 4 changed files with 43 additions and 18 deletions.
3 changes: 3 additions & 0 deletions cmd/substreams/info.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,9 @@ func runInfo(cmd *cobra.Command, args []string) error {
fmt.Println("Name:", mod.Name)
fmt.Println("Initial block:", mod.InitialBlock)
fmt.Println("Kind:", mod.Kind)
for _, input := range mod.Inputs {
fmt.Printf("Input: %s: %s\n", input.Type, input.Name)
}

switch mod.Kind {
case "map":
Expand Down
30 changes: 22 additions & 8 deletions cmd/substreams/service-deploy.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,9 @@ import (

func init() {
serviceCmd.AddCommand(deployCmd)
deployCmd.Flags().StringArrayP("parameters", "p", []string{}, "Parameters to pass to the substreams")
deployCmd.Flags().StringArray("deployment-params", []string{}, "Extra parameters to pass to the deployment endpoint")
deployCmd.Flags().StringArrayP("params", "p", []string{}, "Parameters to pass to the substreams (ex: module2=key1=valX&key2=valY)")
deployCmd.Flags().StringP("network", "n", "", "Network to deploy to (overrides the 'network' field in the manifest)")
deployCmd.Flags().Bool("prod", false, "Enable production mode (default: false)")
}

Expand All @@ -37,7 +39,19 @@ func deployE(cmd *cobra.Command, args []string) error {

file := args[0]

reader, err := manifest.NewReader(file)
paramsString := sflags.MustGetStringArray(cmd, "params")
params, err := manifest.ParseParams(paramsString)
if err != nil {
return fmt.Errorf("parsing params: %w", err)
}

network := sflags.MustGetString(cmd, "network")
readerOptions := []manifest.Option{
manifest.WithOverrideNetwork(network),
manifest.WithParams(params),
}

reader, err := manifest.NewReader(file, readerOptions...)
if err != nil {
return err
}
Expand All @@ -53,28 +67,28 @@ func deployE(cmd *cobra.Command, args []string) error {
return fmt.Errorf("cannot deploy package %q: it does not specify a sink_module", file)
}

//request parameters
// fmt.Println("request parameters")
pkg.Networks = nil // we don't want to send this to the server, so it does not apply network values again, possibly losing the overriden params

paramsMap := make(map[string]string)
for _, param := range mustGetStringArray(cmd, "parameters") {
for _, param := range mustGetStringArray(cmd, "deployment-params") {
parts := strings.SplitN(param, "=", 2)
if len(parts) != 2 {
return fmt.Errorf("invalid parameter format: %q", param)
}
paramsMap[parts[0]] = parts[1]
}

params := []*pbsinksvc.Parameter{}
deployParams := []*pbsinksvc.Parameter{}
for k, v := range paramsMap {
params = append(params, &pbsinksvc.Parameter{
deployParams = append(deployParams, &pbsinksvc.Parameter{
Key: k,
Value: v,
})
}

req := connect.NewRequest(&pbsinksvc.DeployRequest{
SubstreamsPackage: pkg,
Parameters: params,
Parameters: deployParams,
DevelopmentMode: !sflags.MustGetBool(cmd, "prod"),
})
if err := addHeaders(cmd, req); err != nil {
Expand Down
13 changes: 8 additions & 5 deletions docs/release-notes/change-log.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,33 +9,36 @@ All notable changes to this project will be documented in this file.

The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## Unreleased
## v1.3.0

### Highlights

* Support new `networks` configuration block in `substreams.yaml` to override modules' *params* and *initial_block*. Network can be specified at run-time, avoiding the need for separate spkg files for each chain.
* [BREAKING CHANGE] Remove the support for the `deriveFrom` overrides. The `imports`, along with the new `networks` feature, should provide a better mechanism to cover the use cases that `deriveFrom` tried to address.

{% hint style="info" %}
> These changes are all handled in the substreams CLI. The Substreams server endpoints do not need to be upgraded to support it.
> These changes are all handled in the substreams CLI, applying the necessary changes to the package before sending the requests. The Substreams server endpoints do not need to be upgraded to support it.
{% endhint %}

### Added

* Added `networks` field at the top level of the manifest definition, with `initialBlock` and `params` overrides for each module. See the substreams.yaml.example file in the repository or https://substreams.streamingfast.io/reference-and-specs/manifests for more details and example usage.
* The networks `params` and `initialBlock`` overrides for the chosen network are applied to the module directly before being sent to the server. All network configurations are kept when packing an .spkg file.
* Added the `--network` flag for choosing the network on `run` and `gui` command. Default behavior is to use the one defined as `network` in the manifest.
* Added the `--network` flag for choosing the network on `run`, `gui` and `alpha service deploy` commands. Default behavior is to use the one defined as `network` in the manifest.
* Added the `--endpoint` flag to `substreams alpha service serve` to specify substreams endpoint to connect to
* Added endpoints for Antelope chains
* Command 'substreams info' now shows the params

### Removed

* Removed the handling of the `DeriveFrom` keyword in manifest, this override feature is going away.
* Removed the `--skip-package-validation`` option only on run/gui/inspect/info

### Fixed
### Changed

* The `substreams gui` command no longer reads the .spkg multiple times with different behavior during its process.
* Added the `--params` flag to `alpha service deploy` to apply per-module parameters to the substreams before pushing it.
* Renamed the `--parameters` flag to `--deployment-params` in `alpha service deploy`, to clarify the intent of those parameters (given to the endpoint, not applied to the substreams modules)
* Small improvement on `substreams gui` command: no longer reads the .spkg multiple times with different behavior during its process.

## v1.2.0

Expand Down
15 changes: 10 additions & 5 deletions info/info.go
Original file line number Diff line number Diff line change
Expand Up @@ -88,13 +88,15 @@ func Basic(pkg *pbsubstreams.Package, graph *manifest.ModuleGraph) (*BasicInfo,
}

manifestInfo := &BasicInfo{
Name: name,
Network: pkg.Network,
Version: version,
Image: pkg.Image,
Networks: make(map[string]*manifest.NetworkParams),
Name: name,
Network: pkg.Network,
Version: version,
Image: pkg.Image,
}

if pkg.Networks != nil {
manifestInfo.Networks = make(map[string]*manifest.NetworkParams)
}
for k, v := range pkg.Networks {
params := &manifest.NetworkParams{}

Expand Down Expand Up @@ -167,6 +169,9 @@ func Basic(pkg *pbsubstreams.Package, graph *manifest.ModuleGraph) (*BasicInfo,
if v.Store.Mode > 0 {
inputInfo.Mode = strPtr(v.Store.Mode.Pretty())
}
case *pbsubstreams.Module_Input_Params_:
inputInfo.Type = "params"
inputInfo.Name = input.GetParams().Value
default:
inputInfo.Type = "unknown"
inputInfo.Name = "unknown"
Expand Down

0 comments on commit 3d5f799

Please sign in to comment.