Merge branch 'main' into snyk-upgrade-fa80e3a072e8abf83e3a81d8d332c561

.github/workflows/ci.yml

@@ -4,13 +4,15 @@ on:
   push:
     branches: [main, release/*]
   pull_request:
-    branches: [main]
+    branches:
+      - "main"
+      - "release/*"
 
 jobs:
   build:
     strategy:
       matrix:
-        runner: [ubuntu-latest, ubicloud]
+        runner: [ubicloud-standard-8]
     runs-on: ${{ matrix.runner }}
     timeout-minutes: 30
     services:
@@ -31,7 +33,6 @@ jobs:
       - uses: actions/checkout@v3
         with:
           submodules: recursive
-          token: ${{ secrets.SUBMODULE_CHECKOUT }}
 
       - uses: dtolnay/rust-toolchain@stable
 
@@ -46,15 +47,15 @@ jobs:
         with:
           name: "bq_service_account.json"
           json: ${{ secrets.GCP_GH_CI_PKEY }}
-          dir: 'nexus/server/tests/assets/'
+          dir: "nexus/server/tests/assets/"
 
       - name: setup snowflake credentials
         id: sf-credentials
         uses: jsdaniell/[email protected]
         with:
           name: "snowflake_creds.json"
           json: ${{ secrets.SNOWFLAKE_GH_CI_PKEY }}
-          dir: 'nexus/server/tests/assets/'
+          dir: "nexus/server/tests/assets/"
 
       - name: cargo check
         run: cargo check

.github/workflows/customer-docker.yml

@@ -0,0 +1,52 @@
+name: Customer Docker images
+
+on:
+  push:
+    branches:
+     - 'customer-*'
+  pull_request:
+    branches:
+     - 'customer-*'
+
+jobs:
+  docker-build:
+    strategy:
+      matrix:
+        runner: [ubicloud]
+    runs-on: ${{ matrix.runner }}
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - uses: depot/setup-action@v1
+
+      - name: Login to GitHub Container Registry
+        uses: docker/[email protected]
+        with:
+          registry: ghcr.io
+          username: ${{github.actor}}
+          password: ${{secrets.GITHUB_TOKEN}}
+
+      - name: Set Short Commit Hash
+        id: vars
+        run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+
+      - name: extract branch name suffix for customer
+        id: branch
+        run: |
+          echo "::set-output name=branch::$(echo $GITHUB_REF | sed -e 's/.*customer-//')"
+
+      - name: Build (optionally publish) PeerDB Images
+        uses: depot/bake-action@v1
+        with:
+          token: ${{ secrets.DEPOT_TOKEN }}
+          files: ./docker-bake.hcl
+          push: ${{ startsWith(github.ref, 'refs/heads/customer-') }}
+        env:
+          SHA_SHORT: ${{ steps.branch.outputs.branch }}-${{ steps.vars.outputs.sha_short }}
+          TAG: latest-${{ steps.branch.outputs.branch }}

.github/workflows/dev-docker.yml

@@ -10,7 +10,7 @@ jobs:
   docker-build:
     strategy:
       matrix:
-        runner: [ubuntu-latest, ubicloud]
+        runner: [ubicloud]
     runs-on: ${{ matrix.runner }}
     permissions:
       contents: read
@@ -20,7 +20,6 @@ jobs:
         uses: actions/checkout@v3
         with:
           submodules: recursive
-          token: ${{ secrets.SUBMODULE_CHECKOUT }}
 
       - uses: depot/setup-action@v1
 
@@ -35,46 +34,12 @@ jobs:
         id: vars
         run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
 
-      - name: Build (optionally publish) PeerDB Dev Image
-        uses: depot/build-push-action@v1
+      - name: Build (optionally publish) PeerDB Images
+        uses: depot/bake-action@v1
         with:
           token: ${{ secrets.DEPOT_TOKEN }}
-          platforms: linux/amd64,linux/arm64
-          context: .
-          file: stacks/nexus.Dockerfile
+          files: ./docker-bake.hcl
           push: ${{ github.ref == 'refs/heads/main' }}
-          tags: |
-            ghcr.io/peerdb-io/peerdb-server:dev-${{ steps.vars.outputs.sha_short }}
-
-      - name: Build (optionally publish) Flow API Dev Image
-        uses: depot/build-push-action@v1
-        with:
-          token: ${{ secrets.DEPOT_TOKEN }}
-          platforms: linux/amd64,linux/arm64
-          context: .
-          file: stacks/flow-api.Dockerfile
-          push: ${{ github.ref == 'refs/heads/main' }}
-          tags: |
-            ghcr.io/peerdb-io/flow-api:dev-${{ steps.vars.outputs.sha_short }}
-
-      - name: Build (optionally publish) Flow Worker Dev Image
-        uses: depot/build-push-action@v1
-        with:
-          token: ${{ secrets.DEPOT_TOKEN }}
-          platforms: linux/amd64,linux/arm64
-          context: .
-          file: stacks/flow-worker.Dockerfile
-          push: ${{ github.ref == 'refs/heads/main' }}
-          tags: |
-            ghcr.io/peerdb-io/flow-worker:dev-${{ steps.vars.outputs.sha_short }}
-
-      - name: Build (optionally publish) Flow Snapshot Worker Dev Image
-        uses: depot/build-push-action@v1
-        with:
-          token: ${{ secrets.DEPOT_TOKEN }}
-          platforms: linux/amd64,linux/arm64
-          context: .
-          file: stacks/flow-snapshot-worker.Dockerfile
-          push: ${{ github.ref == 'refs/heads/main' }}
-          tags: |
-            ghcr.io/peerdb-io/flow-snapshot-worker:dev-${{ steps.vars.outputs.sha_short }}
+        env:
+          SHA_SHORT: dev-${{ steps.vars.outputs.sha_short }}
+          TAG: latest-dev

.github/workflows/flow.yml

@@ -2,7 +2,8 @@ name: Flow build and test
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - "main"
   push:
     branches: [main]
 
@@ -15,7 +16,7 @@ jobs:
     timeout-minutes: 30
     services:
       pg_cdc:
-        image: postgres:15.4-alpine
+        image: postgis/postgis:15-3.4-alpine
         ports:
           - 7132:5432
         env:
@@ -34,12 +35,17 @@ jobs:
 
       - uses: actions/setup-go@v3
         with:
-          go-version: '>=1.19.0'
+          go-version: ">=1.19.0"
 
       - name: install gotestsum
         run: |
           go install gotest.tools/gotestsum@latest
-
+      
+      - name: install lib-geos
+        run: |
+          sudo apt-get update
+          sudo apt-get install libgeos-dev
+      
       - name: download go modules
         run: |
           go mod download
@@ -58,6 +64,20 @@ jobs:
         with:
           name: "snowflake_creds.json"
           json: ${{ secrets.SNOWFLAKE_GH_CI_PKEY }}
+
+      - name: setup S3 credentials
+        id: s3-credentials
+        uses: jsdaniell/[email protected]
+        with:
+          name: "s3_creds.json"
+          json: ${{ secrets.S3_CREDS }}
+
+      - name: setup GCS credentials
+        id: gcs-credentials
+        uses: jsdaniell/[email protected]
+        with:
+          name: "gcs_creds.json"
+          json: ${{ secrets.GCS_CREDS }}
 
       - name: create hstore extension and increase logical replication limits
         run: |
@@ -73,14 +93,16 @@ jobs:
 
       - name: run tests
         run: |
-          gotestsum --format testname -- -p 4 ./... -timeout 1200s
+          gotestsum --format testname -- -p 4 ./... -timeout 2400s
         working-directory: ./flow
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AWS_REGION: ${{ secrets.AWS_REGION }}
           TEST_BQ_CREDS: ${{ github.workspace }}/bq_service_account.json
           TEST_SF_CREDS: ${{ github.workspace }}/snowflake_creds.json
+          TEST_S3_CREDS: ${{ github.workspace }}/s3_creds.json
+          TEST_GCS_CREDS: ${{ github.workspace }}/gcs_creds.json
           AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
           AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
           AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}

.github/workflows/golang-lint.yml

@@ -1,6 +1,9 @@
 name: GolangCI-Lint
 
-on: [pull_request]
+on:
+  pull_request:
+    branches:
+      - "main"
 
 jobs:
   golangci-lint:
@@ -10,14 +13,13 @@ jobs:
       pull-requests: write
     strategy:
       matrix:
-        runner: [ubuntu-latest, ubicloud]
+        runner: [ubicloud]
     runs-on: ${{ matrix.runner }}
     steps:
       - name: checkout
         uses: actions/checkout@v3
         with:
           submodules: recursive
-          token: ${{ secrets.SUBMODULE_CHECKOUT }}
 
       - name: golangci-lint
         uses: reviewdog/action-golangci-lint@v2

.github/workflows/rust-lint.yml

@@ -1,5 +1,10 @@
 name: clippy-action
-on: [pull_request]
+
+on:
+  pull_request:
+    branches:
+      - "main"
+
 jobs:
   clippy:
     permissions:
@@ -8,23 +13,18 @@ jobs:
       pull-requests: write
     strategy:
       matrix:
-        runner: [ubuntu-latest, ubicloud]
+        runner: [ubicloud-standard-4]
     runs-on: ${{ matrix.runner }}
     steps:
-    - name: checkout
-      uses: actions/checkout@v3
-      with:
-        submodules: recursive
-        token: ${{ secrets.SUBMODULE_CHECKOUT }}
+      - name: checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
 
-    - uses: dtolnay/rust-toolchain@stable
-      with:
-        components: clippy
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          components: clippy
 
-    - uses: giraffate/clippy-action@v1
-      with:
-        reporter: 'github-pr-review'
-        github_token: ${{ secrets.GITHUB_TOKEN }}
-        workdir: ./nexus
-      env:
-        REVIEWDOG_TOKEN: ${{ secrets.REVIEWDOG_TOKEN }}
+      - name: clippy
+        run: cargo clippy -- -D warnings
+        working-directory: ./nexus