Merge pull request #6 from streamingfast/codegen-cmd

Codegen overhaul

.github/workflows/docker.yml

@@ -7,6 +7,7 @@ on:
     branches:
       - develop
       - feature/*
+      - codegen-cmd
 
 env:
   REGISTRY: ghcr.io

.github/workflows/main.yaml

@@ -0,0 +1,59 @@
+name: Integration tests
+
+on:
+  push:
+    branches:
+      - develop
+      - codegen-cmd
+  pull_request:
+    branches:
+      - "**"
+
+jobs:
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+    if: "${{ !startsWith(github.event.head_commit.message, 'GitBook: [#') }}"
+
+    env:
+      TEST_LOCAL_CODEGEN: "true"
+      RUN_INTEGRATION_TESTS: "true"
+      CODEGEN_MAINNET_API_KEY: ${{ secrets.CODEGEN_MAINNET_API_KEY }}
+      REGISTRY: ghcr.io
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v3
+
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ">=1.23.0"
+          cache: true
+
+      - name: Cache Go modules
+        uses: actions/cache@v3
+        with:
+          # In order:
+          # * Module download cache
+          # * Build cache (Linux)
+          # * Build cache (Mac)
+          path: |
+            ~/go/pkg/mod
+            ~/.cache/go-build
+            ~/Library/Caches/go-build
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Run Tests
+        run: go test -v ./...
+
+      - name: Build
+        run: go build ./...

.gitignore

@@ -6,4 +6,5 @@
 .envrc
 /sink-data
 .idea
-*.log
+*.logmytest
+/*.log

Dockerfile

@@ -1,5 +1,5 @@
 # syntax=docker/dockerfile:1
-FROM --platform=$BUILDPLATFORM golang:1.22-alpine AS builder
+FROM --platform=$BUILDPLATFORM golang:1.23-alpine AS builder
 
 WORKDIR /src
 

README.md

@@ -2,31 +2,24 @@
 
 ## Usage
 
-Run the `substreams-codegen` backend:
+Use the codegen.substreams.dev and codegen-staging.substreams.dev endpoints.
 
 ```bash
-# remove the DEBUG var if you want info level logs
-DEBUG=.* go run ./cmd/substreams-codegen api --http-listen-addr "*:9000"
-```
-
-(Optionally) Run the remotebuild server locally:
-
-```bash
-GENERATOR_KEEP_FILES=true go run -v ./cmd/remotebuild
+substreams init --codegen-endpoint https://localhost:9000
 ```
 
-Use the `substreams` cli to initialize your Substreams.
+## Develop
 
 ```bash
-substreams init --discovery-endpoint http://localhost:9000
+DEBUG=.* go run ./cmd/substreams-codegen api --http-listen-addr "*:9000"
 ```
 
 ## Principles
 
 You write a `Conversation` (or `Convo` for short) struct.
 
-- This function embed or has a `state` variable with the state you want to build.
-  - This _state_ is what gets serialized to JSON and sent to the client. It must be a usable interface, so pick your names wisely.
+- This function embed or has a `State` variable with the state you want to build.
+  - This _State_ is what gets serialized to JSON and sent to the client. It must be a usable interface, so pick your names wisely.
   - Anything that is exported will be shared with the client, and come back during hydration (if a connection gets interrupted, or the user kept their `generator.json` state, and wants to simply rebuild).
 - It has an `Update()` method that is registered into the `loop.EventLoop` by the `codegen` framework.
   - This function can mutate state, and it is the only method who can mutate state.
@@ -37,8 +30,8 @@ You write a `Conversation` (or `Convo` for short) struct.
 
 The code generation:
 
-- Any `gotmpl` files will go through templating, and be passed the _state_ struct as a single parameter.
-- The _state_ struct should have helper methods to allow getting data from the state
+- Any `gotmpl` files will go through templating, and be passed the _State_ struct as a single parameter.
+- The _State_ struct should have helper methods to allow getting data from the state
 
 ## Some notes on popular contracts
 

buf.gen.yaml

@@ -14,7 +14,7 @@ plugins:
       - Msf/codegen/conversation/v1/conversation.proto=github.com/streamingfast/substreams-codegen/pb/sf/codegen/conversation/v1;pbconvo
       - Msf/codegen/remotebuild/v1/remotebuild.proto=github.com/streamingfast/substreams-codegen/pb/sf/codegen/remotebuild/v1;pbbuild
 
-  - plugin: buf.build/grpc/go
+  - plugin: buf.build/grpc/go:v1.4.0
     out: pb
     opt:
       - paths=source_relative

common-templates/.gitignore.gotmpl

@@ -0,0 +1,16 @@
+This template holds chunks of common .gitignore
+
+{{ define "gitignore_usage" -}}
+# substreams auth file
+.substreams.env
+
+# Compiled source files
+target/
+
+# Sink data when running any sinker
+sink-data/
+
+# The spkg packed by the subtreams cli
+*.spkg
+{{- end }}
+

common-templates/README.md.gotmpl

@@ -0,0 +1,12 @@
+This template holds chunks of common README.md
+
+{{ define "readme_usage" -}}
+## Usage
+
+```bash
+substreams build
+substreams auth
+substreams gui
+```
+{{- end }}
+

common.go

@@ -0,0 +1,41 @@
+package codegen
+
+import (
+	"embed"
+	"io/fs"
+	"text/template"
+
+	"github.com/bmatcuk/doublestar/v4"
+)
+
+//go:embed common-templates/*.gotmpl
+var commonTemplatesFS embed.FS
+
+var commonTemplates *template.Template
+
+func init() {
+	var err error
+	commonTemplates, err = parseCommonTemplates()
+	if err != nil {
+		panic(err)
+	}
+}
+func parseCommonTemplates() (*template.Template, error) {
+	t := template.New("").Funcs(templateFuncs)
+	filenames, err := doublestar.Glob(commonTemplatesFS, "**/*.gotmpl")
+	if err != nil {
+		return nil, err
+	}
+
+	for _, filename := range filenames {
+		b, err := fs.ReadFile(commonTemplatesFS, filename)
+		if err != nil {
+			return nil, err
+		}
+		_, err = t.New(filename).Parse(string(b))
+		if err != nil {
+			return nil, err
+		}
+	}
+	return t, nil
+}