Merge branch 'main' into stable

.editorconfig

@@ -0,0 +1,15 @@
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.rs]
+indent_style = space
+indent_size = 4
+
+[{package.json,*.yml,*.yaml,*.proto}]
+indent_style = space
+indent_size = 2

.github/workflows/ci.yml

@@ -10,7 +10,7 @@ jobs:
   build:
     strategy:
       matrix:
-        runner: [ubicloud-standard-8-ubuntu-2204-arm]
+        runner: [ubicloud-standard-2-ubuntu-2204-arm]
     runs-on: ${{ matrix.runner }}
     timeout-minutes: 30
     services:
@@ -41,20 +41,30 @@ jobs:
 
       - name: setup gcp service account
         id: gcp-service-account
-        uses: jsdaniell/[email protected].2
+        uses: jsdaniell/[email protected].3
         with:
           name: "bq_service_account.json"
           json: ${{ secrets.GCP_GH_CI_PKEY }}
           dir: "nexus/server/tests/assets/"
 
       - name: setup snowflake credentials
         id: sf-credentials
-        uses: jsdaniell/[email protected].2
+        uses: jsdaniell/[email protected].3
         with:
           name: "snowflake_creds.json"
           json: ${{ secrets.SNOWFLAKE_GH_CI_PKEY }}
           dir: "nexus/server/tests/assets/"
 
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: nexus
+
+      - uses: bufbuild/[email protected]
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - run: buf generate protos
+
       - name: cargo check
         run: cargo check
         working-directory: ./nexus

.github/workflows/cleanup.yml

@@ -0,0 +1,73 @@
+name: Flow cleanup
+
+on:
+  schedule:
+  - cron: "0 0 * * *"
+
+jobs:
+  flow_cleanup:
+    runs-on: ubicloud-standard-2-ubuntu-2204-arm
+    timeout-minutes: 60
+    steps:
+      - name: checkout sources
+        uses: actions/checkout@v4
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version: "1.22"
+          cache-dependency-path: e2e_cleanup/go.sum
+
+      - name: download go modules
+        run: |
+          go mod download
+        working-directory: ./e2e_cleanup
+
+      - name: setup gcp service account
+        id: gcp-service-account
+        uses: jsdaniell/[email protected]
+        with:
+          name: "bq_service_account.json"
+          json: ${{ secrets.GCP_GH_CI_PKEY }}
+
+      - name: setup snowflake credentials
+        id: sf-credentials
+        uses: jsdaniell/[email protected]
+        with:
+          name: "snowflake_creds.json"
+          json: ${{ secrets.SNOWFLAKE_GH_CI_PKEY }}
+
+      - name: setup S3 credentials
+        id: s3-credentials
+        uses: jsdaniell/[email protected]
+        with:
+          name: "s3_creds.json"
+          json: ${{ secrets.S3_CREDS }}
+
+      - name: setup GCS credentials
+        id: gcs-credentials
+        uses: jsdaniell/[email protected]
+        with:
+          name: "gcs_creds.json"
+          json: ${{ secrets.GCS_CREDS }}
+
+      - name: run cleanup
+        run: go run main.go
+        working-directory: ./e2e_cleanup
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_REGION: ${{ secrets.AWS_REGION }}
+          TEST_BQ_CREDS: ${{ github.workspace }}/bq_service_account.json
+          TEST_SF_CREDS: ${{ github.workspace }}/snowflake_creds.json
+          TEST_S3_CREDS: ${{ github.workspace }}/s3_creds.json
+          TEST_GCS_CREDS: ${{ github.workspace }}/gcs_creds.json
+          AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
+          AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
+          AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
+          AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          ENABLE_SQLSERVER_TESTS: true
+          SQLSERVER_HOST: ${{ secrets.SQLSERVER_HOST }}
+          SQLSERVER_PORT: ${{ secrets.SQLSERVER_PORT }}
+          SQLSERVER_USER: ${{ secrets.SQLSERVER_USER }}
+          SQLSERVER_PASSWORD: ${{ secrets.SQLSERVER_PASSWORD }}
+          SQLSERVER_DB: ${{ secrets.SQLSERVER_DB }}

.github/workflows/customer-docker.yml

@@ -12,7 +12,7 @@ jobs:
   docker-build:
     strategy:
       matrix:
-        runner: [ubicloud-standard-2-ubuntu-2204-arm]
+        runner: [ubuntu-latest]
     runs-on: ${{ matrix.runner }}
     permissions:
       contents: read
@@ -23,6 +23,12 @@ jobs:
         with:
           submodules: recursive
 
+      - uses: bufbuild/[email protected]
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - run: buf generate protos
+
       - uses: depot/setup-action@v1
 
       - name: Login to GitHub Container Registry

.github/workflows/dev-docker.yml

@@ -10,6 +10,7 @@ jobs:
   docker-build:
     strategy:
       matrix:
+        # ubuntu-latest leverages larger GH runner pool & completes in ~30s instead of ~3m
         runner: [ubuntu-latest]
     runs-on: ${{ matrix.runner }}
     permissions:
@@ -21,6 +22,12 @@ jobs:
         with:
           submodules: recursive
 
+      - uses: bufbuild/[email protected]
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - run: buf generate protos
+
       - uses: depot/setup-action@v1
 
       - name: Login to GitHub Container Registry

.github/workflows/flow.yml

@@ -10,92 +10,93 @@ jobs:
   flow_test:
     strategy:
       matrix:
-        runner: [ubicloud-standard-8-ubuntu-2204-arm]
+        runner: [ubicloud-standard-16-ubuntu-2204-arm]
     runs-on: ${{ matrix.runner }}
     timeout-minutes: 30
     services:
-      pg_cdc:
+      catalog:
         image: imresamu/postgis:15-3.4-alpine
         ports:
-          - 7132:5432
+          - 5432:5432
         env:
-          POSTGRES_USER: postgres
+          PGUSER: postgres
           POSTGRES_PASSWORD: postgres
           POSTGRES_DB: postgres
-        options: >-
-          --name pg_cdc
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
+
     steps:
       - name: checkout sources
         uses: actions/checkout@v4
 
-      - uses: actions/setup-go@v4
+      - uses: bufbuild/buf-setup-[email protected]
         with:
-          go-version: ">=1.21.0"
-          cache-dependency-path: flow/go.sum
+          github_token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: install gotestsum
-        run: |
-          go install gotest.tools/gotestsum@latest
+      - run: buf generate protos
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version: "1.22"
+          cache-dependency-path: flow/go.sum
 
       - name: install lib-geos
         run: |
           sudo apt-get update
           sudo apt-get install libgeos-dev
 
-      - name: download go modules
-        run: |
-          go mod download
+      - run: go mod download
         working-directory: ./flow
 
       - name: setup gcp service account
         id: gcp-service-account
-        uses: jsdaniell/[email protected].2
+        uses: jsdaniell/[email protected].3
         with:
           name: "bq_service_account.json"
           json: ${{ secrets.GCP_GH_CI_PKEY }}
 
       - name: setup snowflake credentials
         id: sf-credentials
-        uses: jsdaniell/[email protected].2
+        uses: jsdaniell/[email protected].3
         with:
           name: "snowflake_creds.json"
           json: ${{ secrets.SNOWFLAKE_GH_CI_PKEY }}
 
       - name: setup S3 credentials
         id: s3-credentials
-        uses: jsdaniell/[email protected].2
+        uses: jsdaniell/[email protected].3
         with:
           name: "s3_creds.json"
           json: ${{ secrets.S3_CREDS }}
 
       - name: setup GCS credentials
         id: gcs-credentials
-        uses: jsdaniell/[email protected].2
+        uses: jsdaniell/[email protected].3
         with:
           name: "gcs_creds.json"
           json: ${{ secrets.GCS_CREDS }}
 
       - name: create hstore extension, increase logical replication limits, and setup catalog database
         run: >
-          docker exec pg_cdc psql -h localhost -p 5432 -U postgres -c "CREATE EXTENSION hstore;"
+          docker exec "${{ job.services.catalog.id }}" psql -U postgres -c "CREATE EXTENSION hstore;"
           -c "ALTER SYSTEM SET wal_level=logical;"
           -c "ALTER SYSTEM SET max_replication_slots=192;"
           -c "ALTER SYSTEM SET max_wal_senders=256;"
           -c "ALTER SYSTEM SET max_connections=2048;" &&
-          (cat ../nexus/catalog/migrations/V{?,??}__* | docker exec -i pg_cdc psql -h localhost -p 5432 -U postgres) &&
-          docker restart pg_cdc
-        working-directory: ./flow
+          (cat ./nexus/catalog/migrations/V{?,??}__* | docker exec -i "${{ job.services.catalog.id }}" psql -U postgres) &&
+          docker restart "${{ job.services.catalog.id }}"
         env:
-          PG_CDC: empty
           PGPASSWORD: postgres
 
+      - name: Install Temporal CLI
+        uses: temporalio/setup-temporal@v0
+
       - name: run tests
         run: |
-          gotestsum --format testname -- -p 8 ./... -timeout 1200s
+          temporal server start-dev --namespace default --headless &
+          go build -ldflags="-s -w" -o peer-flow
+          temporal operator search-attribute create --name MirrorName --type Text --namespace default
+          ./peer-flow worker &
+          ./peer-flow snapshot-worker &
+          go test -p 32 ./... -timeout 900s
         working-directory: ./flow
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -116,8 +117,7 @@ jobs:
           SQLSERVER_PASSWORD: ${{ secrets.SQLSERVER_PASSWORD }}
           SQLSERVER_DB: ${{ secrets.SQLSERVER_DB }}
           PEERDB_CATALOG_HOST: localhost
-          PEERDB_CATALOG_PORT: 7132
+          PEERDB_CATALOG_PORT: 5432
           PEERDB_CATALOG_USER: postgres
           PEERDB_CATALOG_PASSWORD: postgres
           PEERDB_CATALOG_DATABASE: postgres
-          PEERDB_CDC_IDLE_TIMEOUT_SECONDS: 10

.github/workflows/golang-lint.yml

@@ -5,29 +5,30 @@ on:
     branches: [main]
     paths: [flow/**]
 
+permissions:
+  contents: read
+
 jobs:
-  golangci-lint:
-    permissions:
-      checks: write
-      contents: read
-      pull-requests: write
-    strategy:
-      matrix:
-        runner: [ubicloud-standard-4-ubuntu-2204-arm]
-    runs-on: ${{ matrix.runner }}
+  golangci:
+    name: lint
+    runs-on: [ubicloud-standard-4-ubuntu-2204-arm]
     steps:
-      - name: checkout
-        uses: actions/checkout@v4
+      - uses: actions/checkout@v4
+      - uses: bufbuild/[email protected]
         with:
-          submodules: recursive
-
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+      - run: buf generate protos
+      - name: install lib-geos
+        run: |
+          sudo apt-get update
+          sudo apt-get install libgeos-dev
+      - uses: actions/setup-go@v5
+        with:
+          go-version: "1.22"
+          cache: false
       - name: golangci-lint
-        uses: reviewdog/action-golangci-lint@v2
+        uses: golangci/golangci-lint-action@v4
         with:
-          workdir: ./flow
-          reporter: github-pr-review
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          golangci_lint_flags: "--timeout 10m"
-          fail_on_error: true
-        env:
-          REVIEWDOG_TOKEN: ${{ secrets.REVIEWDOG_TOKEN }}
+          version: v1.56
+          working-directory: ./flow
+          args: --timeout=10m

.github/workflows/rust-lint.yml

@@ -21,6 +21,12 @@ jobs:
         with:
           submodules: recursive
 
+      - uses: bufbuild/[email protected]
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - run: buf generate protos
+
       - uses: dtolnay/rust-toolchain@stable
         with:
           components: clippy

.github/workflows/stable-docker.yml

@@ -20,6 +20,12 @@ jobs:
         with:
           submodules: recursive
 
+      - uses: bufbuild/[email protected]
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - run: buf generate protos
+
       - uses: depot/setup-action@v1
 
       - name: Login to GitHub Container Registry

.github/workflows/ui-build.yml

@@ -18,6 +18,12 @@ jobs:
       - name: checkout
         uses: actions/checkout@v4
 
+      - uses: bufbuild/[email protected]
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - run: buf generate protos
+
       - name: Install Node.js dependencies
         working-directory: ui
         run: npm ci