switching to local protobuf plugins in CI (#1762)

working around frequent errors with protobuf generation with Buf remote
plugins

.github/actions/genprotos/action.yml

@@ -0,0 +1,36 @@
+name: 'PeerDB genprotos'
+description: 'Install buf with local plugins, generate protos and cache'
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/checkout@v4
+    - name: check cache
+      id: cache
+      uses: actions/cache@v4
+      with:
+        path: |
+          ./flow/generated/protos
+          ./nexus/pt/src/gen
+          ./ui/grpc_generated
+        key: ${{ runner.os }}-build-genprotos-${{ hashFiles('./protos/peers.proto', './protos/flow.proto', './protos/route.proto') }}
+
+    - if: steps.cache.outputs.cache-hit != 'true'
+      uses: actions/setup-go@v5
+      with:
+        go-version: '1.22.3'
+        cache: false
+    - if: steps.cache.outputs.cache-hit != 'true'
+      uses: bufbuild/[email protected]
+    - if: steps.cache.outputs.cache-hit != 'true'
+      uses: dtolnay/rust-toolchain@stable
+    - if: steps.cache.outputs.cache-hit != 'true'
+      name: install protobuf plugins and generate protos
+      shell: bash
+      run: |
+        go install google.golang.org/protobuf/cmd/[email protected]
+        go install google.golang.org/grpc/cmd/[email protected]
+        go install github.com/grpc-ecosystem/grpc-gateway/v2/[email protected]
+        npm install ts-proto
+        cargo install protoc-gen-prost protoc-gen-tonic protoc-gen-prost-serde
+        PATH="$(go env GOPATH)/bin:$HOME/.cargo/bin:$PATH" buf generate --template buf.gen-local.yaml --debug protos
+

.github/workflows/ci.yml

@@ -30,10 +30,9 @@ jobs:
           --health-retries 5
     steps:
       - uses: actions/checkout@v4
-        with:
-          submodules: recursive
 
-      - uses: dtolnay/rust-toolchain@stable
+      - name: generate or hydrate protos
+        uses: ./.github/actions/genprotos
 
       - name: install system dependencies
         run: |
@@ -60,11 +59,6 @@ jobs:
         with:
           workspaces: nexus
 
-      - uses: bufbuild/[email protected]
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-
-      - run: buf generate protos
 
       - name: cargo check
         run: cargo check

.github/workflows/cleanup.yml

@@ -14,7 +14,7 @@ jobs:
 
       - uses: actions/setup-go@v5
         with:
-          go-version: "1.22"
+          go-version: '1.22.3'
           cache-dependency-path: e2e_cleanup/go.sum
 
       - name: download go modules

.github/workflows/codeql-analysis.yml

@@ -36,11 +36,8 @@ jobs:
     - name: Checkout repository
       uses: actions/checkout@v4
 
-    - uses: bufbuild/[email protected]
-      with:
-        github_token: ${{ secrets.GITHUB_TOKEN }}
-
-    - run: buf generate protos
+    - name: generate or hydrate protos
+      uses: ./.github/actions/genprotos
 
     - name: install lib-geos
       run: |

.github/workflows/customer-docker.yml

@@ -18,16 +18,10 @@ jobs:
       contents: read
       packages: write
     steps:
-      - name: checkout
-        uses: actions/checkout@v4
-        with:
-          submodules: recursive
-
-      - uses: bufbuild/[email protected]
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: actions/checkout@v4
 
-      - run: buf generate protos
+      - name: generate or hydrate protos
+        uses: ./.github/actions/genprotos
 
       - uses: depot/setup-action@v1
 

.github/workflows/dev-docker.yml

@@ -17,16 +17,10 @@ jobs:
       contents: read
       packages: write
     steps:
-      - name: checkout
-        uses: actions/checkout@v4
-        with:
-          submodules: recursive
-
-      - uses: bufbuild/[email protected]
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: actions/checkout@v4
 
-      - run: buf generate protos
+      - name: generate or hydrate protos
+        uses: ./.github/actions/genprotos
 
       - uses: depot/setup-action@v1
 

.github/workflows/flow.yml

@@ -32,18 +32,14 @@ jobs:
           xpack.security.enrollment.enabled: false
 
     steps:
-      - name: checkout sources
-        uses: actions/checkout@v4
+      - uses: actions/checkout@v4
 
-      - uses: bufbuild/[email protected]
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-
-      - run: buf generate protos
+      - name: generate or hydrate protos
+        uses: ./.github/actions/genprotos
 
       - uses: actions/setup-go@v5
         with:
-          go-version: "1.22"
+          go-version: '1.22.3'
           cache-dependency-path: flow/go.sum
 
       - name: install lib-geos

.github/workflows/golang-lint.yml

@@ -14,17 +14,17 @@ jobs:
     runs-on: [ubicloud-standard-4-ubuntu-2204-arm]
     steps:
       - uses: actions/checkout@v4
-      - uses: bufbuild/[email protected]
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-      - run: buf generate protos
+
+      - name: generate or hydrate protos
+        uses: ./.github/actions/genprotos
+
       - name: install lib-geos
         run: |
           sudo apt-get update
           sudo apt-get install libgeos-dev
       - uses: actions/setup-go@v5
         with:
-          go-version: "1.22"
+          go-version: '1.22.3'
           cache: false
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v6

.github/workflows/rust-lint.yml

@@ -16,20 +16,10 @@ jobs:
         runner: [ubicloud-standard-4-ubuntu-2204-arm]
     runs-on: ${{ matrix.runner }}
     steps:
-      - name: checkout
-        uses: actions/checkout@v4
-        with:
-          submodules: recursive
+      - uses: actions/checkout@v4
 
-      - uses: bufbuild/[email protected]
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-
-      - run: buf generate protos
-
-      - uses: dtolnay/rust-toolchain@stable
-        with:
-          components: clippy
+      - name: generate or hydrate protos
+        uses: ./.github/actions/genprotos
 
       - name: clippy
         run: cargo clippy -- -D warnings

.github/workflows/stable-docker.yml

@@ -15,16 +15,10 @@ jobs:
       contents: read
       packages: write
     steps:
-      - name: checkout
-        uses: actions/checkout@v4
-        with:
-          submodules: recursive
-
-      - uses: bufbuild/[email protected]
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: actions/checkout@v4
 
-      - run: buf generate protos
+      - name: generate or hydrate protos
+        uses: ./.github/actions/genprotos
 
       - uses: depot/setup-action@v1
 

.github/workflows/ui-build.yml

@@ -18,11 +18,8 @@ jobs:
       - name: checkout
         uses: actions/checkout@v4
 
-      - uses: bufbuild/[email protected]
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-
-      - run: buf generate protos
+      - name: generate or hydrate protos
+        uses: ./.github/actions/genprotos
 
       - name: Install Node.js dependencies
         working-directory: ui

.github/workflows/ui-lint.yml

@@ -22,11 +22,8 @@ jobs:
       - name: checkout
         uses: actions/checkout@v4
 
-      - uses: bufbuild/[email protected]
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-
-      - run: buf generate protos
+      - name: generate or hydrate protos
+        uses: ./.github/actions/genprotos
 
       - name: Install Node.js dependencies
         working-directory: ui

README.md

@@ -20,7 +20,7 @@ We support different modes of streaming - log based (CDC), cursor based (timesta
 ## Get started
 
 ```bash
-git clone --recursive [email protected]:PeerDB-io/peerdb.git
+git clone [email protected]:PeerDB-io/peerdb.git
 cd peerdb
 
 # Run docker containers: postgres as catalog, temporal, PeerDB server, PeerDB flow API + workers, PeerDB UI

buf.gen-local.yaml

@@ -0,0 +1,37 @@
+version: v1
+managed:
+  enabled: true
+  go_package_prefix:
+    default: generated/protos
+    except:
+      - buf.build/googleapis/googleapis
+plugins:
+  - plugin: go
+    out: flow/generated/protos
+    opt: paths=source_relative
+  - plugin: go-grpc
+    out: flow/generated/protos
+    opt:
+      - paths=source_relative
+  - plugin: prost
+    out: nexus/pt/src/gen
+    opt:
+      - compile_well_known_types
+      - extern_path=.google.protobuf=::pbjson_types
+  - plugin: tonic
+    out: nexus/pt/src/gen
+  - plugin: prost-serde
+    out: nexus/pt/src/gen
+    opt:
+      - ignore_unknown_fields=true
+  - name: ts
+    out: ui/grpc_generated
+    strategy: all
+    path: ./node_modules/ts-proto/protoc-gen-ts_proto
+    opt:
+      - esModuleInterop=true
+      - outputServices=grpc-js
+  - plugin: grpc-gateway
+    out: flow/generated/protos
+    opt:
+      - paths=source_relative