Skip to content

Commit

Permalink
Update to substreams-sink 0.3.4
Browse files Browse the repository at this point in the history
  • Loading branch information
@maoueh
maoueh committed Mar 19, 2024
1 parent c54d45e commit cbdab1b
Show file tree
Hide file tree
Showing 5 changed files with 117 additions and 115 deletions.
6 changes: 6 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## v1.1.3

* Fixed spurious error reporting when the sinker is terminating or has been canceled.

* Updated `substreams` dependency to latest version `v1.3.7`.

## v1.1.2

* Improved `substreams stream stats` log line but now using `substreams_sink_progress_message_total_processed_blocks` for `progress_block_rate` replacing the `progress_msg_rate` which wasn't meaningful anymore (and broken because the metric was never updated).
Expand Down
10 changes: 9 additions & 1 deletion cmd/substreams-sink-noop/head_tracker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,24 +16,28 @@ import (
"github.com/streamingfast/dgrpc"
"github.com/streamingfast/shutter"
sinknoop "github.com/streamingfast/substreams-sink-noop"
"github.com/streamingfast/substreams/client"
pbsubstreamsrpc "github.com/streamingfast/substreams/pb/sf/substreams/rpc/v2"
"go.uber.org/zap"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/metadata"
)

type HeadTracker struct {
*shutter.Shutter
client pbsubstreamsrpc.StreamClient
callOpts []grpc.CallOption
headers client.Headers
value AtomicValue[bstream.BlockRef]
}

func NewHeadTracker(client pbsubstreamsrpc.StreamClient, callOpts []grpc.CallOption) *HeadTracker {
func NewHeadTracker(client pbsubstreamsrpc.StreamClient, callOpts []grpc.CallOption, headers client.Headers) *HeadTracker {
return &HeadTracker{
Shutter: shutter.New(),
client: client,
callOpts: callOpts,
headers: headers,
}
}

Expand Down Expand Up @@ -110,6 +114,10 @@ func (s *HeadTracker) streamHead(ctx context.Context, activeCursor string) (stri
return activeCursor, receivedMessage, fmt.Errorf("add head tracker manifest to substreams request: %w", err)
}

if s.headers.IsSet() {
ctx = metadata.AppendToOutgoingContext(ctx, s.headers.ToArray()...)
}

stream, err := s.client.Blocks(ctx, req, s.callOpts...)
if err != nil {
return activeCursor, receivedMessage, retryable(fmt.Errorf("call sf.substreams.rpc.v2.Stream/Blocks: %w", err))
Expand Down
4 changes: 2 additions & 2 deletions cmd/substreams-sink-noop/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,11 +106,11 @@ func run(cmd *cobra.Command, args []string) error {
zap.String("manage_listen_addr", apiListenAddr),
)

headTrackerClient, headTrackerConnClose, headTrackerCallOpts, err := client.NewSubstreamsClient(sinker.ClientConfig())
headTrackerClient, headTrackerConnClose, headTrackerCallOpts, headTrackerHeaders, err := client.NewSubstreamsClient(sinker.ClientConfig())
cli.NoError(err, "Unable to create head tracker client")
defer headTrackerConnClose()

headFetcher := NewHeadTracker(headTrackerClient, headTrackerCallOpts)
headFetcher := NewHeadTracker(headTrackerClient, headTrackerCallOpts, headTrackerHeaders)
app.OnTerminating(func(_ error) { headFetcher.Close() })
headFetcher.OnTerminated(func(err error) { app.Shutdown(err) })

Expand Down
65 changes: 32 additions & 33 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,26 +13,26 @@ require (
github.com/streamingfast/bstream v0.0.2-0.20231121211820-e45c1b42f472
github.com/streamingfast/cli v0.0.4-0.20230825151644-8cc84512cd80
github.com/streamingfast/derr v0.0.0-20230515163924-8570aaa43fe1
github.com/streamingfast/dgrpc v0.0.0-20230929132851-893fc52687fa
github.com/streamingfast/dgrpc v0.0.0-20240219152146-57bb131c39ca
github.com/streamingfast/dmetrics v0.0.0-20240214191810-524a5c58fbaa
github.com/streamingfast/logging v0.0.0-20230608130331-f22c91403091
github.com/streamingfast/pbgo v0.0.6-0.20231120172814-537d034aad5e
github.com/streamingfast/shutter v1.5.0
github.com/streamingfast/substreams v1.3.0
github.com/streamingfast/substreams-sink v0.3.3
github.com/streamingfast/substreams v1.3.7
github.com/streamingfast/substreams-sink v0.3.4
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.44.0
go.uber.org/zap v1.26.0
golang.org/x/oauth2 v0.10.0
google.golang.org/grpc v1.58.0
google.golang.org/protobuf v1.31.0
golang.org/x/oauth2 v0.15.0
google.golang.org/grpc v1.61.0
google.golang.org/protobuf v1.32.0
gopkg.in/yaml.v3 v3.0.1
)

require (
cloud.google.com/go v0.110.4 // indirect
cloud.google.com/go/compute v1.21.0 // indirect
cloud.google.com/go v0.111.0 // indirect
cloud.google.com/go/compute v1.23.3 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/iam v1.1.1 // indirect
cloud.google.com/go/iam v1.1.5 // indirect
cloud.google.com/go/storage v1.30.1 // indirect
github.com/Azure/azure-pipeline-go v0.2.3 // indirect
github.com/Azure/azure-storage-blob-go v0.14.0 // indirect
Expand All @@ -45,30 +45,29 @@ require (
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/chzyer/readline v1.5.0 // indirect
github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe // indirect
github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 // indirect
github.com/cncf/xds/go v0.0.0-20231109132714-523115ebc101 // indirect
github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
github.com/envoyproxy/go-control-plane v0.11.1 // indirect
github.com/envoyproxy/protoc-gen-validate v1.0.2 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/go-logr/logr v1.2.4 // indirect
github.com/go-logr/logr v1.4.1 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/s2a-go v0.1.4 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
github.com/googleapis/gax-go/v2 v2.11.0 // indirect
github.com/google/s2a-go v0.1.7 // indirect
github.com/google/uuid v1.4.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
github.com/googleapis/gax-go/v2 v2.12.0 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/ipfs/boxo v0.8.0 // indirect
github.com/ipfs/go-cid v0.4.0 // indirect
github.com/ipfs/go-ipfs-api v0.6.0 // indirect
github.com/jhump/protoreflect v1.14.0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/klauspost/compress v1.16.5 // indirect
github.com/klauspost/compress v1.16.6 // indirect
github.com/klauspost/cpuid/v2 v2.2.3 // indirect
github.com/libp2p/go-buffer-pool v0.1.0 // indirect
github.com/libp2p/go-flow-metrics v0.1.0 // indirect
Expand Down Expand Up @@ -106,33 +105,33 @@ require (
github.com/spf13/cast v1.5.0 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/streamingfast/dbin v0.9.1-0.20231117225723-59790c798e2c // indirect
github.com/streamingfast/dstore v0.1.1-0.20230620124109-3924b3b36c77 // indirect
github.com/streamingfast/dstore v0.1.1-0.20240215171730-493ad5a0f537 // indirect
github.com/streamingfast/opaque v0.0.0-20210811180740-0c01d37ea308 // indirect
github.com/stretchr/testify v1.8.4 // indirect
github.com/subosito/gotenv v1.4.2 // indirect
github.com/teris-io/shortid v0.0.0-20171029131806-771a37caa5cf // indirect
github.com/whyrusleeping/tar-utils v0.0.0-20180509141711-8c6c8ba81d5c // indirect
github.com/yourbasic/graph v0.0.0-20210606180040-8ecfec1c2869 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/otel v1.18.0 // indirect
go.opentelemetry.io/otel/metric v1.18.0 // indirect
go.opentelemetry.io/otel/trace v1.18.0 // indirect
go.opentelemetry.io/otel v1.23.1 // indirect
go.opentelemetry.io/otel/metric v1.23.1 // indirect
go.opentelemetry.io/otel/trace v1.23.1 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/multierr v1.10.0 // indirect
golang.org/x/crypto v0.14.0 // indirect
golang.org/x/crypto v0.19.0 // indirect
golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 // indirect
golang.org/x/mod v0.12.0 // indirect
golang.org/x/net v0.17.0 // indirect
golang.org/x/sync v0.3.0 // indirect
golang.org/x/sys v0.13.0 // indirect
golang.org/x/term v0.13.0 // indirect
golang.org/x/text v0.13.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/api v0.126.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect
golang.org/x/net v0.21.0 // indirect
golang.org/x/sync v0.5.0 // indirect
golang.org/x/sys v0.17.0 // indirect
golang.org/x/term v0.17.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/time v0.5.0 // indirect
google.golang.org/api v0.152.0 // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/genproto v0.0.0-20231212172506-995d672761c0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240102182953-50ed04b92917 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
lukechampine.com/blake3 v1.1.7 // indirect
Expand Down
Loading

0 comments on commit cbdab1b

Please sign in to comment.