Merge branch 'main' into better-error

PeerDB-io · Sep 15, 2023 · 3f211e2 · 3f211e2
2 parents eb4d6f5 + 21e3dd8
commit 3f211e2
Show file tree

Hide file tree

Showing 318 changed files with 34,913 additions and 5,022 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -4,6 +4,11 @@ nexus/target
 thirdparty/**/*
 thirdparty
 
+ui/node_modules/**/*
+ui/node_modules
+ui/.next/**/*
+ui/.next
+
 .git
 .gitignore
 .github

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -8,7 +8,10 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        runner: [ubuntu-latest, ubicloud]
+    runs-on: ${{ matrix.runner }}
     timeout-minutes: 30
     services:
       catalog_peer:
@@ -36,7 +39,7 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install -y protobuf-compiler libssl-dev pkg-config build-essential
-      
+
       - name: setup gcp service account
         id: gcp-service-account
         uses: jsdaniell/[email protected]

diff --git a/.github/workflows/dev-debian.yml b/.github/workflows/dev-debian.yml
@@ -9,7 +9,10 @@ on:
 jobs:
   release:
     name: build and release
-    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        runner: [ubuntu-latest, ubicloud]
+    runs-on: ${{ matrix.runner }}
     steps:
       - name: checkout sources
         uses: actions/checkout@v3

diff --git a/.github/workflows/dev-docker.yml b/.github/workflows/dev-docker.yml
@@ -8,7 +8,10 @@ on:
 
 jobs:
   docker-build:
-    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        runner: [ubuntu-latest, ubicloud]
+    runs-on: ${{ matrix.runner }}
     permissions:
       contents: read
       packages: write
@@ -36,6 +39,7 @@ jobs:
         uses: depot/build-push-action@v1
         with:
           token: ${{ secrets.DEPOT_TOKEN }}
+          platforms: linux/amd64,linux/arm64
           context: .
           file: stacks/nexus.Dockerfile
           push: ${{ github.ref == 'refs/heads/main' }}
@@ -46,6 +50,7 @@ jobs:
         uses: depot/build-push-action@v1
         with:
           token: ${{ secrets.DEPOT_TOKEN }}
+          platforms: linux/amd64,linux/arm64
           context: .
           file: stacks/flow-api.Dockerfile
           push: ${{ github.ref == 'refs/heads/main' }}
@@ -56,6 +61,7 @@ jobs:
         uses: depot/build-push-action@v1
         with:
           token: ${{ secrets.DEPOT_TOKEN }}
+          platforms: linux/amd64,linux/arm64
           context: .
           file: stacks/flow-worker.Dockerfile
           push: ${{ github.ref == 'refs/heads/main' }}
@@ -66,6 +72,7 @@ jobs:
         uses: depot/build-push-action@v1
         with:
           token: ${{ secrets.DEPOT_TOKEN }}
+          platforms: linux/amd64,linux/arm64
           context: .
           file: stacks/flow-snapshot-worker.Dockerfile
           push: ${{ github.ref == 'refs/heads/main' }}

diff --git a/.github/workflows/flow.yml b/.github/workflows/flow.yml
@@ -1,25 +1,29 @@
 name: Flow build and test
 
 on:
-  push:
-    branches: [main]
   pull_request:
     branches: [main]
+  push:
+    branches: [main]
 
 jobs:
   flow_test:
-    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        runner: [ubuntu-latest, ubicloud]
+    runs-on: ${{ matrix.runner }}
     timeout-minutes: 30
     services:
       pg_cdc:
-        image: debezium/postgres:14-alpine
+        image: postgres:15.4-alpine
         ports:
           - 7132:5432
         env:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
           POSTGRES_DB: postgres
         options: >-
+          --name pg_cdc
           --health-cmd pg_isready
           --health-interval 10s
           --health-timeout 5s
@@ -55,18 +59,21 @@ jobs:
           name: "snowflake_creds.json"
           json: ${{ secrets.SNOWFLAKE_GH_CI_PKEY }}
 
-      - name: create hstore extension
+      - name: create hstore extension and increase logical replication limits
         run: |
-          sudo apt-get update
-          sudo apt-get install -y postgresql-client
-          psql -h localhost -p 7132 -U postgres -c "CREATE EXTENSION hstore;"
+          docker exec pg_cdc psql -h localhost -p 5432 -U postgres -c "CREATE EXTENSION hstore;"
+          docker exec pg_cdc psql -h localhost -p 5432 -U postgres -c "ALTER SYSTEM SET wal_level=logical;"
+          docker exec pg_cdc psql -h localhost -p 5432 -U postgres -c "ALTER SYSTEM SET max_replication_slots=100;"
+          docker exec pg_cdc psql -h localhost -p 5432 -U postgres -c "ALTER SYSTEM SET max_wal_senders=100;"
+          docker restart pg_cdc
         working-directory: ./flow
         env:
+          PG_CDC: empty
           PGPASSWORD: postgres
 
       - name: run tests
         run: |
-          gotestsum --format testname -- -p 1 ./... -timeout 1200s
+          gotestsum --format testname -- -p 4 ./... -timeout 1200s
         working-directory: ./flow
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}

diff --git a/.github/workflows/golang-lint.yml b/.github/workflows/golang-lint.yml
@@ -8,7 +8,10 @@ jobs:
       checks: write
       contents: read
       pull-requests: write
-    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        runner: [ubuntu-latest, ubicloud]
+    runs-on: ${{ matrix.runner }}
     steps:
       - name: checkout
         uses: actions/checkout@v3

diff --git a/.github/workflows/rust-lint.yml b/.github/workflows/rust-lint.yml
@@ -6,7 +6,10 @@ jobs:
       checks: write
       contents: read
       pull-requests: write
-    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        runner: [ubuntu-latest, ubicloud]
+    runs-on: ${{ matrix.runner }}
     steps:
     - name: checkout
       uses: actions/checkout@v3

diff --git a/.github/workflows/stable-debian.yml b/.github/workflows/stable-debian.yml
@@ -8,7 +8,10 @@ on:
 jobs:
   release:
     name: build and release
-    runs-on: ubuntu-latest-64-core
+    strategy:
+      matrix:
+        runner: [ubuntu-latest, ubicloud]
+    runs-on: ${{ matrix.runner }}
     permissions:
       contents: write
     steps:

diff --git a/.github/workflows/stable-docker.yml b/.github/workflows/stable-docker.yml
@@ -7,7 +7,10 @@ on:
 
 jobs:
   docker-build:
-    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        runner: [ubuntu-latest, ubicloud]
+    runs-on: ${{ matrix.runner }}
     permissions:
       contents: read
       packages: write
@@ -30,6 +33,7 @@ jobs:
       - name: Publish PeerDB Stable Image
         uses: depot/build-push-action@v1
         with:
+          platforms: linux/amd64,linux/arm64
           token: ${{ secrets.DEPOT_TOKEN }}
           context: .
           file: stacks/nexus.Dockerfile
@@ -40,6 +44,7 @@ jobs:
       - name: Publish Flow API Image
         uses: depot/build-push-action@v1
         with:
+          platforms: linux/amd64,linux/arm64
           token: ${{ secrets.DEPOT_TOKEN }}
           context: .
           file: stacks/flow-api.Dockerfile
@@ -50,6 +55,7 @@ jobs:
       - name: Publish Flow Worker Stable Image
         uses: depot/build-push-action@v1
         with:
+          platforms: linux/amd64,linux/arm64
           token: ${{ secrets.DEPOT_TOKEN }}
           context: .
           file: stacks/flow-worker.Dockerfile
@@ -60,6 +66,7 @@ jobs:
       - name: Publish Flow Snapshot Worker Stable Image
         uses: depot/build-push-action@v1
         with:
+          platforms: linux/amd64,linux/arm64
           token: ${{ secrets.DEPOT_TOKEN }}
           context: .
           file: stacks/flow-snapshot-worker.Dockerfile

diff --git a/buf.gen.yaml b/buf.gen.yaml
@@ -22,3 +22,8 @@ plugins:
     out: nexus/pt/src
     opt:
       - ignore_unknown_fields=true
+  - plugin: buf.build/community/stephenh-ts-proto:v1.156.8
+    out: ui/grpc_generated
+    opt:
+      - esModuleInterop=true
+      - outputServices=grpc-js
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -12,12 +12,13 @@ x-flow-worker-env: &flow-worker-env
   AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-""}
   AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-""}
   AWS_REGION: ${AWS_REGION:-""}
-# enables worker profiling using Go's pprof
-  ENABLE_PROFILING: true
-# enables exporting of mirror metrics to Prometheus for visualization using Grafana
-  ENABLE_METRICS: true
-# enables exporting of mirror metrics to Catalog in the PEERDB_STATS schema.
-  ENABLE_STATS: true
+  # enables worker profiling using Go's pprof
+  ENABLE_PROFILING: "true"
+  # enables exporting of mirror metrics to Prometheus for visualization using Grafana
+  ENABLE_METRICS: "true"
+  # enables exporting of mirror metrics to Catalog in the PEERDB_STATS schema.
+  ENABLE_STATS: "true"
+  PYROSCOPE_SERVER_ADDRESS: http://pyroscope:4040
 
 services:
   catalog:
@@ -60,6 +61,12 @@ services:
     labels:
       kompose.volume.type: configMap
 
+  pyroscope:
+    container_name: pyroscope
+    image: grafana/pyroscope:latest
+    ports:
+      - 4040:4040
+
   temporal-admin-tools:
     container_name: temporal-admin-tools
     depends_on:
@@ -83,6 +90,7 @@ services:
     environment:
       - TEMPORAL_ADDRESS=temporal:7233
       - TEMPORAL_CORS_ORIGINS=http://localhost:3000
+      - TEMPORAL_CSRF_COOKIE_INSECURE=true
     image: temporalio/ui:2.17.2
     ports:
       - 8085:8080
@@ -95,6 +103,7 @@ services:
     ports:
       - 8112:8112
     environment:
+      <<: [*catalog-config]
       TEMPORAL_HOST_PORT: temporal:7233
     depends_on:
       temporal-admin-tools:
@@ -118,7 +127,6 @@ services:
       dockerfile: stacks/flow-worker.Dockerfile
     environment:
       <<: [*catalog-config, *flow-worker-env]
-      PROFILING_SERVER: 0.0.0.0:6060
       METRICS_SERVER: 0.0.0.0:6061
     ports:
       - 6060:6060
@@ -134,7 +142,6 @@ services:
       dockerfile: stacks/flow-worker.Dockerfile
     environment:
       <<: [*catalog-config, *flow-worker-env]
-      PROFILING_SERVER: 0.0.0.0:6062
       METRICS_SERVER: 0.0.0.0:6063
     ports:
       - 6062:6062
@@ -153,7 +160,6 @@ services:
       dockerfile: stacks/flow-worker.Dockerfile
     environment:
       <<: [*catalog-config, *flow-worker-env]
-      PROFILING_SERVER: 0.0.0.0:6064
       METRICS_SERVER: 0.0.0.0:6065
     ports:
       - 6064:6064
@@ -210,6 +216,17 @@ services:
       - multi-metrics
       - metrics
 
+  peerdb_ui:
+    container_name: peerdb_ui
+    build:
+      context: .
+      dockerfile: stacks/ui.Dockerfile
+    ports:
+      - 3000:3000
+    environment:
+      <<: *catalog-config
+      PEERDB_FLOW_SERVER_ADDRESS: flow_api:8112
+
 volumes:
   pgdata:
   prometheusdata: