Skip to content

Commit

Permalink
Merge branch 'main' into stable
Browse files Browse the repository at this point in the history
  • Loading branch information
Amogh-Bharadwaj committed Oct 16, 2023
2 parents 424708e + 0957cfb commit 8ee07ff
Show file tree
Hide file tree
Showing 420 changed files with 55,447 additions and 11,109 deletions.
5 changes: 5 additions & 0 deletions .dockerignore
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,11 @@ nexus/target
thirdparty/**/*
thirdparty

ui/node_modules/**/*
ui/node_modules
ui/.next/**/*
ui/.next

.git
.gitignore
.github
Expand Down
16 changes: 10 additions & 6 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,16 @@ on:
push:
branches: [main, release/*]
pull_request:
branches: [main]
branches:
- "main"
- "release/*"

jobs:
build:
runs-on: ubuntu-latest
strategy:
matrix:
runner: [ubicloud-standard-8]
runs-on: ${{ matrix.runner }}
timeout-minutes: 30
services:
catalog_peer:
Expand All @@ -28,30 +33,29 @@ jobs:
- uses: actions/checkout@v3
with:
submodules: recursive
token: ${{ secrets.SUBMODULE_CHECKOUT }}

- uses: dtolnay/rust-toolchain@stable

- name: install system dependencies
run: |
sudo apt-get update
sudo apt-get install -y protobuf-compiler libssl-dev pkg-config build-essential
- name: setup gcp service account
id: gcp-service-account
uses: jsdaniell/[email protected]
with:
name: "bq_service_account.json"
json: ${{ secrets.GCP_GH_CI_PKEY }}
dir: 'nexus/server/tests/assets/'
dir: "nexus/server/tests/assets/"

- name: setup snowflake credentials
id: sf-credentials
uses: jsdaniell/[email protected]
with:
name: "snowflake_creds.json"
json: ${{ secrets.SNOWFLAKE_GH_CI_PKEY }}
dir: 'nexus/server/tests/assets/'
dir: "nexus/server/tests/assets/"

- name: cargo check
run: cargo check
Expand Down
45 changes: 0 additions & 45 deletions .github/workflows/dev-debian.yml

This file was deleted.

30 changes: 25 additions & 5 deletions .github/workflows/dev-docker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,13 @@ name: Dev Docker Images
on:
push:
branches: [main]
pull_request:
branches: [main]

jobs:
docker-build:
runs-on: ubuntu-latest
strategy:
matrix:
runner: [ubicloud]
runs-on: ${{ matrix.runner }}
permissions:
contents: read
packages: write
Expand All @@ -17,7 +18,6 @@ jobs:
uses: actions/checkout@v3
with:
submodules: recursive
token: ${{ secrets.SUBMODULE_CHECKOUT }}

- uses: depot/setup-action@v1

Expand All @@ -36,38 +36,58 @@ jobs:
uses: depot/build-push-action@v1
with:
token: ${{ secrets.DEPOT_TOKEN }}
platforms: linux/amd64,linux/arm64
context: .
file: stacks/nexus.Dockerfile
file: stacks/peerdb-server.Dockerfile
push: ${{ github.ref == 'refs/heads/main' }}
tags: |
ghcr.io/peerdb-io/peerdb-server:dev-${{ steps.vars.outputs.sha_short }}
ghcr.io/peerdb-io/peerdb-server:latest-dev
- name: Build (optionally publish) Flow API Dev Image
uses: depot/build-push-action@v1
with:
token: ${{ secrets.DEPOT_TOKEN }}
platforms: linux/amd64,linux/arm64
context: .
file: stacks/flow-api.Dockerfile
push: ${{ github.ref == 'refs/heads/main' }}
tags: |
ghcr.io/peerdb-io/flow-api:dev-${{ steps.vars.outputs.sha_short }}
ghcr.io/peerdb-io/flow-api:latest-dev
- name: Build (optionally publish) Flow Worker Dev Image
uses: depot/build-push-action@v1
with:
token: ${{ secrets.DEPOT_TOKEN }}
platforms: linux/amd64,linux/arm64
context: .
file: stacks/flow-worker.Dockerfile
push: ${{ github.ref == 'refs/heads/main' }}
tags: |
ghcr.io/peerdb-io/flow-worker:dev-${{ steps.vars.outputs.sha_short }}
ghcr.io/peerdb-io/flow-worker:latest-dev
- name: Build (optionally publish) Flow Snapshot Worker Dev Image
uses: depot/build-push-action@v1
with:
token: ${{ secrets.DEPOT_TOKEN }}
platforms: linux/amd64,linux/arm64
context: .
file: stacks/flow-snapshot-worker.Dockerfile
push: ${{ github.ref == 'refs/heads/main' }}
tags: |
ghcr.io/peerdb-io/flow-snapshot-worker:dev-${{ steps.vars.outputs.sha_short }}
ghcr.io/peerdb-io/flow-snapshot-worker:latest-dev
- name: Build (optionally publish) PeerDB UI Dev Image
uses: depot/build-push-action@v1
with:
token: ${{ secrets.DEPOT_TOKEN }}
platforms: linux/amd64,linux/arm64
context: .
file: stacks/peerdb-ui.Dockerfile
push: ${{ github.ref == 'refs/heads/main' }}
tags: |
ghcr.io/peerdb-io/peerdb-ui:dev-${{ steps.vars.outputs.sha_short }}
ghcr.io/peerdb-io/peerdb-ui:latest-dev
44 changes: 34 additions & 10 deletions .github/workflows/flow.yml
Original file line number Diff line number Diff line change
@@ -1,25 +1,30 @@
name: Flow build and test

on:
push:
branches: [main]
pull_request:
branches:
- "main"
push:
branches: [main]

jobs:
flow_test:
runs-on: ubuntu-latest
strategy:
matrix:
runner: [ubicloud-standard-8]
runs-on: ${{ matrix.runner }}
timeout-minutes: 30
services:
pg_cdc:
image: debezium/postgres:14-alpine
image: postgres:15.4-alpine
ports:
- 7132:5432
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: postgres
options: >-
--name pg_cdc
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
Expand All @@ -30,7 +35,7 @@ jobs:

- uses: actions/setup-go@v3
with:
go-version: '>=1.19.0'
go-version: ">=1.19.0"

- name: install gotestsum
run: |
Expand All @@ -54,26 +59,45 @@ jobs:
with:
name: "snowflake_creds.json"
json: ${{ secrets.SNOWFLAKE_GH_CI_PKEY }}

- name: setup S3 credentials
id: s3-credentials
uses: jsdaniell/[email protected]
with:
name: "s3_creds.json"
json: ${{ secrets.S3_CREDS }}

- name: setup GCS credentials
id: gcs-credentials
uses: jsdaniell/[email protected]
with:
name: "gcs_creds.json"
json: ${{ secrets.GCS_CREDS }}

- name: create hstore extension
- name: create hstore extension and increase logical replication limits
run: |
sudo apt-get update
sudo apt-get install -y postgresql-client
psql -h localhost -p 7132 -U postgres -c "CREATE EXTENSION hstore;"
docker exec pg_cdc psql -h localhost -p 5432 -U postgres -c "CREATE EXTENSION hstore;"
docker exec pg_cdc psql -h localhost -p 5432 -U postgres -c "ALTER SYSTEM SET wal_level=logical;"
docker exec pg_cdc psql -h localhost -p 5432 -U postgres -c "ALTER SYSTEM SET max_replication_slots=100;"
docker exec pg_cdc psql -h localhost -p 5432 -U postgres -c "ALTER SYSTEM SET max_wal_senders=100;"
docker restart pg_cdc
working-directory: ./flow
env:
PG_CDC: empty
PGPASSWORD: postgres

- name: run tests
run: |
gotestsum --format testname -- -p 1 ./... -timeout 1200s
gotestsum --format testname -- -p 4 ./... -timeout 2400s
working-directory: ./flow
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ secrets.AWS_REGION }}
TEST_BQ_CREDS: ${{ github.workspace }}/bq_service_account.json
TEST_SF_CREDS: ${{ github.workspace }}/snowflake_creds.json
TEST_S3_CREDS: ${{ github.workspace }}/s3_creds.json
TEST_GCS_CREDS: ${{ github.workspace }}/gcs_creds.json
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
Expand Down
11 changes: 8 additions & 3 deletions .github/workflows/golang-lint.yml
Original file line number Diff line number Diff line change
@@ -1,20 +1,25 @@
name: GolangCI-Lint

on: [pull_request]
on:
pull_request:
branches:
- "main"

jobs:
golangci-lint:
permissions:
checks: write
contents: read
pull-requests: write
runs-on: ubuntu-latest
strategy:
matrix:
runner: [ubicloud]
runs-on: ${{ matrix.runner }}
steps:
- name: checkout
uses: actions/checkout@v3
with:
submodules: recursive
token: ${{ secrets.SUBMODULE_CHECKOUT }}

- name: golangci-lint
uses: reviewdog/action-golangci-lint@v2
Expand Down
37 changes: 20 additions & 17 deletions .github/workflows/rust-lint.yml
Original file line number Diff line number Diff line change
@@ -1,27 +1,30 @@
name: clippy-action
on: [pull_request]

on:
pull_request:
branches:
- "main"

jobs:
clippy:
permissions:
checks: write
contents: read
pull-requests: write
runs-on: ubuntu-latest
strategy:
matrix:
runner: [ubicloud-standard-4]
runs-on: ${{ matrix.runner }}
steps:
- name: checkout
uses: actions/checkout@v3
with:
submodules: recursive
token: ${{ secrets.SUBMODULE_CHECKOUT }}
- name: checkout
uses: actions/checkout@v3
with:
submodules: recursive

- uses: dtolnay/rust-toolchain@stable
with:
components: clippy
- uses: dtolnay/rust-toolchain@stable
with:
components: clippy

- uses: giraffate/clippy-action@v1
with:
reporter: 'github-pr-review'
github_token: ${{ secrets.GITHUB_TOKEN }}
workdir: ./nexus
env:
REVIEWDOG_TOKEN: ${{ secrets.REVIEWDOG_TOKEN }}
- name: clippy
run: cargo clippy -- -D warnings
working-directory: ./nexus
Loading

0 comments on commit 8ee07ff

Please sign in to comment.