diff --git a/.github/actions/genprotos/action.yml b/.github/actions/genprotos/action.yml new file mode 100644 index 0000000000..abedf058b6 --- /dev/null +++ b/.github/actions/genprotos/action.yml @@ -0,0 +1,36 @@ +name: 'PeerDB genprotos' +description: 'Install buf with local plugins, generate protos and cache' +runs: + using: "composite" + steps: + - uses: actions/checkout@v4 + - name: check cache + id: cache + uses: actions/cache@v4 + with: + path: | + ./flow/generated/protos + ./nexus/pt/src/gen + ./ui/grpc_generated + key: ${{ runner.os }}-build-genprotos-${{ hashFiles('./protos/peers.proto', './protos/flow.proto', './protos/route.proto') }} + + - if: steps.cache.outputs.cache-hit != 'true' + uses: actions/setup-go@v5 + with: + go-version: '1.22.3' + cache: false + - if: steps.cache.outputs.cache-hit != 'true' + uses: bufbuild/buf-setup-action@v1.29.0-1 + - if: steps.cache.outputs.cache-hit != 'true' + uses: dtolnay/rust-toolchain@stable + - if: steps.cache.outputs.cache-hit != 'true' + name: install protobuf plugins and generate protos + shell: bash + run: | + go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34 + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.3 + go install github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway@v2.20.0 + npm install ts-proto + cargo install protoc-gen-prost protoc-gen-tonic protoc-gen-prost-serde + PATH="$(go env GOPATH)/bin:$HOME/.cargo/bin:$PATH" buf generate --template buf.gen-local.yaml --debug protos + diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e77f61d7a..09a9c9e6c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,10 +30,9 @@ jobs: --health-retries 5 steps: - uses: actions/checkout@v4 - with: - submodules: recursive - - uses: dtolnay/rust-toolchain@stable + - name: generate or hydrate protos + uses: ./.github/actions/genprotos - name: install system dependencies run: | @@ -60,11 +59,6 @@ jobs: with: workspaces: nexus - - uses: bufbuild/buf-setup-action@v1.29.0-1 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - - - run: buf generate protos - name: cargo check run: cargo check diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml index c4b42c2851..c6f5e95f23 100644 --- a/.github/workflows/cleanup.yml +++ b/.github/workflows/cleanup.yml @@ -14,7 +14,7 @@ jobs: - uses: actions/setup-go@v5 with: - go-version: "1.22" + go-version: '1.22.3' cache-dependency-path: e2e_cleanup/go.sum - name: download go modules diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 08ae2e2f1b..5de1d92c40 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -36,11 +36,8 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - - uses: bufbuild/buf-setup-action@v1.29.0-1 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - - - run: buf generate protos + - name: generate or hydrate protos + uses: ./.github/actions/genprotos - name: install lib-geos run: | diff --git a/.github/workflows/customer-docker.yml b/.github/workflows/customer-docker.yml index 9e2d1ceea7..8278ec3d27 100644 --- a/.github/workflows/customer-docker.yml +++ b/.github/workflows/customer-docker.yml @@ -18,16 +18,10 @@ jobs: contents: read packages: write steps: - - name: checkout - uses: actions/checkout@v4 - with: - submodules: recursive - - - uses: bufbuild/buf-setup-action@v1.29.0-1 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} + - uses: actions/checkout@v4 - - run: buf generate protos + - name: generate or hydrate protos + uses: ./.github/actions/genprotos - uses: depot/setup-action@v1 diff --git a/.github/workflows/dev-docker.yml b/.github/workflows/dev-docker.yml index 418ee0fb41..6011ec4ab4 100644 --- a/.github/workflows/dev-docker.yml +++ b/.github/workflows/dev-docker.yml @@ -17,16 +17,10 @@ jobs: contents: read packages: write steps: - - name: checkout - uses: actions/checkout@v4 - with: - submodules: recursive - - - uses: bufbuild/buf-setup-action@v1.29.0-1 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} + - uses: actions/checkout@v4 - - run: buf generate protos + - name: generate or hydrate protos + uses: ./.github/actions/genprotos - uses: depot/setup-action@v1 diff --git a/.github/workflows/flow.yml b/.github/workflows/flow.yml index 10a04df0b2..fb1774a769 100644 --- a/.github/workflows/flow.yml +++ b/.github/workflows/flow.yml @@ -32,18 +32,14 @@ jobs: xpack.security.enrollment.enabled: false steps: - - name: checkout sources - uses: actions/checkout@v4 + - uses: actions/checkout@v4 - - uses: bufbuild/buf-setup-action@v1.29.0-1 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - - - run: buf generate protos + - name: generate or hydrate protos + uses: ./.github/actions/genprotos - uses: actions/setup-go@v5 with: - go-version: "1.22" + go-version: '1.22.3' cache-dependency-path: flow/go.sum - name: install lib-geos diff --git a/.github/workflows/golang-lint.yml b/.github/workflows/golang-lint.yml index 1482a9fb3c..9eaf42c110 100644 --- a/.github/workflows/golang-lint.yml +++ b/.github/workflows/golang-lint.yml @@ -14,17 +14,17 @@ jobs: runs-on: [ubicloud-standard-4-ubuntu-2204-arm] steps: - uses: actions/checkout@v4 - - uses: bufbuild/buf-setup-action@v1.29.0-1 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - - run: buf generate protos + + - name: generate or hydrate protos + uses: ./.github/actions/genprotos + - name: install lib-geos run: | sudo apt-get update sudo apt-get install libgeos-dev - uses: actions/setup-go@v5 with: - go-version: "1.22" + go-version: '1.22.3' cache: false - name: golangci-lint uses: golangci/golangci-lint-action@v6 diff --git a/.github/workflows/rust-lint.yml b/.github/workflows/rust-lint.yml index 5bae41631d..b9e43c1a24 100644 --- a/.github/workflows/rust-lint.yml +++ b/.github/workflows/rust-lint.yml @@ -16,20 +16,10 @@ jobs: runner: [ubicloud-standard-4-ubuntu-2204-arm] runs-on: ${{ matrix.runner }} steps: - - name: checkout - uses: actions/checkout@v4 - with: - submodules: recursive + - uses: actions/checkout@v4 - - uses: bufbuild/buf-setup-action@v1.29.0-1 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - - - run: buf generate protos - - - uses: dtolnay/rust-toolchain@stable - with: - components: clippy + - name: generate or hydrate protos + uses: ./.github/actions/genprotos - name: clippy run: cargo clippy -- -D warnings diff --git a/.github/workflows/stable-docker.yml b/.github/workflows/stable-docker.yml index a2300bee23..9eabbcfb28 100644 --- a/.github/workflows/stable-docker.yml +++ b/.github/workflows/stable-docker.yml @@ -15,16 +15,10 @@ jobs: contents: read packages: write steps: - - name: checkout - uses: actions/checkout@v4 - with: - submodules: recursive - - - uses: bufbuild/buf-setup-action@v1.29.0-1 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} + - uses: actions/checkout@v4 - - run: buf generate protos + - name: generate or hydrate protos + uses: ./.github/actions/genprotos - uses: depot/setup-action@v1 diff --git a/.github/workflows/ui-build.yml b/.github/workflows/ui-build.yml index e825186f54..feea1ffda5 100644 --- a/.github/workflows/ui-build.yml +++ b/.github/workflows/ui-build.yml @@ -18,11 +18,8 @@ jobs: - name: checkout uses: actions/checkout@v4 - - uses: bufbuild/buf-setup-action@v1.29.0-1 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - - - run: buf generate protos + - name: generate or hydrate protos + uses: ./.github/actions/genprotos - name: Install Node.js dependencies working-directory: ui diff --git a/.github/workflows/ui-lint.yml b/.github/workflows/ui-lint.yml index 66948554bd..31e2340ffb 100644 --- a/.github/workflows/ui-lint.yml +++ b/.github/workflows/ui-lint.yml @@ -22,11 +22,8 @@ jobs: - name: checkout uses: actions/checkout@v4 - - uses: bufbuild/buf-setup-action@v1.29.0-1 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - - - run: buf generate protos + - name: generate or hydrate protos + uses: ./.github/actions/genprotos - name: Install Node.js dependencies working-directory: ui diff --git a/README.md b/README.md index 3ff1041321..007778b2aa 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ We support different modes of streaming - log based (CDC), cursor based (timesta ## Get started ```bash -git clone --recursive git@github.com:PeerDB-io/peerdb.git +git clone git@github.com:PeerDB-io/peerdb.git cd peerdb # Run docker containers: postgres as catalog, temporal, PeerDB server, PeerDB flow API + workers, PeerDB UI diff --git a/buf.gen-local.yaml b/buf.gen-local.yaml new file mode 100644 index 0000000000..04b2638369 --- /dev/null +++ b/buf.gen-local.yaml @@ -0,0 +1,37 @@ +version: v1 +managed: + enabled: true + go_package_prefix: + default: generated/protos + except: + - buf.build/googleapis/googleapis +plugins: + - plugin: go + out: flow/generated/protos + opt: paths=source_relative + - plugin: go-grpc + out: flow/generated/protos + opt: + - paths=source_relative + - plugin: prost + out: nexus/pt/src/gen + opt: + - compile_well_known_types + - extern_path=.google.protobuf=::pbjson_types + - plugin: tonic + out: nexus/pt/src/gen + - plugin: prost-serde + out: nexus/pt/src/gen + opt: + - ignore_unknown_fields=true + - name: ts + out: ui/grpc_generated + strategy: all + path: ./node_modules/ts-proto/protoc-gen-ts_proto + opt: + - esModuleInterop=true + - outputServices=grpc-js + - plugin: grpc-gateway + out: flow/generated/protos + opt: + - paths=source_relative