From d55c5c8cd7e1b371067496f6dca479c442cf61b1 Mon Sep 17 00:00:00 2001 From: Matthieu Vachon Date: Fri, 19 May 2023 14:07:20 -0400 Subject: [PATCH] Bug fix, flag rename and Docker images --- .sfreleaser | 9 +----- CHANGELOG.md | 10 ++++++ cmd/substreams-sink-kv/inject.go | 15 +++++++-- cmd/substreams-sink-kv/serve.go | 3 +- ...ckerfile.goreleaser => Dockerfile.builder} | 0 devel/docker/Dockerfile.runtime | 26 ++++++++++++++++ devel/docker/push.sh | 31 ++++++++++++++++--- 7 files changed, 76 insertions(+), 18 deletions(-) rename devel/docker/{Dockerfile.goreleaser => Dockerfile.builder} (100%) create mode 100644 devel/docker/Dockerfile.runtime diff --git a/.sfreleaser b/.sfreleaser index c469317..7606a95 100755 --- a/.sfreleaser +++ b/.sfreleaser @@ -2,12 +2,5 @@ global: language: golang variant: application release: - # You need a specialized goreleaser-cross image that contains WasmEdge (include/lib(s)) - # for each supported platform. For now, we do not publish this image to a public - # Docker registry. So you need to build it manually with - # - # ./devel/docker/push.sh - # - # It will build the image locally, then you will be able to release. - goreleaser-docker-image: goreleaser-cross-wasmedge:v1.20.3 + goreleaser-docker-image: ghcr.io/streamingfast/substreams-sink-kv-builder:v1.20.3 upload-substreams-spkg: substreams.yaml \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index c41a971..5d66cfe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## v2.1.2 + +### Changed + +* _Deprecation_ The flag `substreams-sink-kv inject --listen-addr=...` must now be used like `substreams-sink-kv inject --server-listen-addr=...` + +### Fixed + +* Added back missing flag `--server-api-prefix` to define API prefix when server mode is active. + ## v2.1.1 ### Added diff --git a/cmd/substreams-sink-kv/inject.go b/cmd/substreams-sink-kv/inject.go index 4eb9a47..9c82af8 100644 --- a/cmd/substreams-sink-kv/inject.go +++ b/cmd/substreams-sink-kv/inject.go @@ -6,7 +6,6 @@ import ( "github.com/spf13/cobra" "github.com/spf13/pflag" - "github.com/spf13/viper" "github.com/streamingfast/cli" . "github.com/streamingfast/cli" "github.com/streamingfast/cli/sflags" @@ -25,8 +24,12 @@ var injectCmd = Command(injectRunE, sink.AddFlagsToSet(flags) flags.Int("flush-interval", 1000, "When in catch up mode, flush every N blocks") - flags.String("listen-addr", "", "Launch query server on this address") flags.String("module", "", "An explicit module to sink, if not provided, expecting the Substreams manifest to defined 'sink' configuration") + flags.String("server-listen-addr", "", "Launch query server on this address") + flags.String("server-api-prefix", "", "Launch query server with this API prefix so the URl to query is /") + + flags.String("listen-addr", "", "Launch query server on this address") + flags.Lookup("listen-addr").Deprecated = "use --server-listen-addr instead" }), Description(` Fills a KV store from a Substreams output and optionally runs a server. Authentication @@ -108,7 +111,13 @@ func injectRunE(cmd *cobra.Command, args []string) error { kvSinker.Run(ctx) }() - if listenAddr := viper.GetString("inject-listen-addr"); listenAddr != "" { + listenAddr, provided := sflags.MustGetStringProvided(cmd, "server-listen-addr") + if !provided { + // Fallback to deprecated flag + listenAddr = sflags.MustGetString(cmd, "listen-addr") + } + + if listenAddr != "" { zlog.Info("setting up query server", zap.String("dsn", dsn), zap.String("listen_addr", listenAddr)) server, err := setupServer(cmd, sink.Package(), kvDB) if err != nil { diff --git a/cmd/substreams-sink-kv/serve.go b/cmd/substreams-sink-kv/serve.go index 50bced4..5811198 100644 --- a/cmd/substreams-sink-kv/serve.go +++ b/cmd/substreams-sink-kv/serve.go @@ -30,7 +30,6 @@ var serveCmd = Command(serveRunE, Flags(func(flags *pflag.FlagSet) { flags.String("listen-addr", ":7878", "Listen via GRPC Connect-Web on this address") flags.Bool("listen-ssl-self-signed", false, "Listen with an HTTPS server (with self-signed certificate)") - flags.String("api-prefix", "", "Prefix that will be added to the Connect Web routes") }), Description(` Test @@ -128,7 +127,7 @@ func setupServer(cmd *cobra.Command, pkg *pbsubstreams.Package, kvDB *db.DB) (se return nil, fmt.Errorf("find proto file descriptor: %w", err) } - config, err := wasmquery.NewServiceConfig(fileDesc, wasmServ.GrpcService, sflags.MustGetString(cmd, "api-prefix")) + config, err := wasmquery.NewServiceConfig(fileDesc, wasmServ.GrpcService, sflags.MustGetString(cmd, "server-api-prefix")) if err != nil { return nil, fmt.Errorf("failed to setup grpc config: %w", err) } diff --git a/devel/docker/Dockerfile.goreleaser b/devel/docker/Dockerfile.builder similarity index 100% rename from devel/docker/Dockerfile.goreleaser rename to devel/docker/Dockerfile.builder diff --git a/devel/docker/Dockerfile.runtime b/devel/docker/Dockerfile.runtime new file mode 100644 index 0000000..0106b77 --- /dev/null +++ b/devel/docker/Dockerfile.runtime @@ -0,0 +1,26 @@ +FROM ubuntu:20.04 + +ARG TARGETOS +ARG TARGETARCH +ARG VERSION + +RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ + apt-get -y install -y \ + ca-certificates libssl1.1 vim htop iotop sysstat \ + dstat strace lsof curl wget jq tzdata && \ + rm /etc/localtime && \ + ln -snf /usr/share/zoneinfo/America/Montreal /etc/localtime && \ + dpkg-reconfigure -f noninteractive tzdata && \ + apt-get -y install -y git && \ + rm -rf /var/cache/apt /var/lib/apt/lists/* + +RUN cd /root &&\ + curl -sSf https://raw.githubusercontent.com/WasmEdge/WasmEdge/master/utils/install.sh | bash -s -- --version 0.11.2 + +RUN cd /root &&\ + wget -O substreams-sink-kv.tar.gz https://github.com/streamingfast/substreams-sink-kv/releases/download/$VERSION/substreams-sink-kv_linux_x86_64.tar.gz &&\ + tar -xzvf substreams-sink-kv.tar.gz &&\ + cp -R substreams-sink-kv /usr/local/bin/ &&\ + rm -rf LICENSE README.md + +ENTRYPOINT [ "/usr/local/bin/substreams-sink-kv" ] diff --git a/devel/docker/push.sh b/devel/docker/push.sh index 3e2909c..10b8f6f 100755 --- a/devel/docker/push.sh +++ b/devel/docker/push.sh @@ -2,23 +2,44 @@ ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +tag_host="ghcr.io/streamingfast/" + main() { pushd "$ROOT" &> /dev/null - while getopts "h" opt; do + while getopts "hl" opt; do case $opt in h) usage && exit 0;; + l) tag_host="";; \?) usage_error "Invalid option: -$OPTARG";; esac done shift $((OPTIND-1)) docker_file="$1"; shift - if [[ $docker_file == "" ]]; then - docker_file="Dockerfile.goreleaser" + if [[ $# -ge 1 ]]; then + version="$1"; shift + fi + + if [[ ! -f "$docker_file" ]]; then + usage_error "Provided Dockerfile '$docker_file' does not exist" + fi + + if [[ "$version" == "" ]]; then + version=`git -c 'versionsort.suffix=-' ls-remote --exit-code --refs --sort='version:refname' --tags origin '*.*.*' | tail -n1 | tr "\t" " " | cut -d ' ' -f 2 | sed 's|refs/tags/||g'` + fi + + name="`printf $docker_file | sed 's/Dockerfile.//g'`" + + if [[ "$name" == "runtime" ]]; then + build_args="--build-arg='VERSION=$version' --platform=linux/amd64 -t '${tag_host}substreams-sink-kv:$version'" + elif [[ "$name" == "builder" ]]; then + build_args="--build-arg='VERSION=$version' --platform=linux/amd64,linux/arm64 -t ${tag_host}substreams-sink-kv-builder:v1.20.3" + else + usage_error "Unknown Dockerfile '$docker_file'" fi - docker build -f "$docker_file" . -t goreleaser-cross-wasmedge:v1.20.3 + docker buildx build -f "$docker_file" . --push $build_args } usage_error() { @@ -32,7 +53,7 @@ usage_error() { } usage() { - echo "usage: push.sh " + echo "usage: push.sh []" echo "" echo "Builds and push the following Dockefile with the correct tags." echo ""