From 311dbe2874ddd51b25e4ffbf97ad6826091d2e8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philip=20Dub=C3=A9?= Date: Tue, 27 Feb 2024 21:54:18 +0000 Subject: [PATCH] flow ci: try running temporal, getting out of sdk/testsuite --- .github/workflows/flow.yml | 42 +++++------------ docker-compose-ci.yml | 97 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 29 deletions(-) create mode 100644 docker-compose-ci.yml diff --git a/.github/workflows/flow.yml b/.github/workflows/flow.yml index 0a398c8de4..ea2a754893 100644 --- a/.github/workflows/flow.yml +++ b/.github/workflows/flow.yml @@ -13,21 +13,6 @@ jobs: runner: [ubicloud-standard-16-ubuntu-2204-arm] runs-on: ${{ matrix.runner }} timeout-minutes: 30 - services: - pg_cdc: - image: imresamu/postgis:15-3.4-alpine - ports: - - 7132:5432 - env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: postgres - options: >- - --name pg_cdc - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 steps: - name: checkout sources uses: actions/checkout@v4 @@ -43,18 +28,14 @@ jobs: go-version: "1.22" cache-dependency-path: flow/go.sum - - name: install gotestsum - run: | - go install gotest.tools/gotestsum@latest + - run: go install gotest.tools/gotestsum@latest - name: install lib-geos run: | sudo apt-get update sudo apt-get install libgeos-dev - - name: download go modules - run: | - go mod download + - run: go mod download working-directory: ./flow - name: setup gcp service account @@ -85,23 +66,26 @@ jobs: name: "gcs_creds.json" json: ${{ secrets.GCS_CREDS }} + - run: docker compose -f docker-compose-ci.yml up -d + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} + - name: create hstore extension, increase logical replication limits, and setup catalog database run: > - docker exec pg_cdc psql -h localhost -p 5432 -U postgres -c "CREATE EXTENSION hstore;" + docker exec catalog psql -U postgres -c "CREATE EXTENSION hstore;" -c "ALTER SYSTEM SET wal_level=logical;" -c "ALTER SYSTEM SET max_replication_slots=192;" -c "ALTER SYSTEM SET max_wal_senders=256;" -c "ALTER SYSTEM SET max_connections=2048;" && - (cat ../nexus/catalog/migrations/V{?,??}__* | docker exec -i pg_cdc psql -h localhost -p 5432 -U postgres) && - docker restart pg_cdc - working-directory: ./flow + (cat ./nexus/catalog/migrations/V{?,??}__* | docker exec -i catalog psql -U postgres) && + docker restart catalog env: - PG_CDC: empty PGPASSWORD: postgres - name: run tests - run: | - gotestsum --format testname -- -p 24 ./... -timeout 1200s + run: gotestsum --format testname -- -p 60 ./... -timeout 1200s working-directory: ./flow env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} @@ -122,7 +106,7 @@ jobs: SQLSERVER_PASSWORD: ${{ secrets.SQLSERVER_PASSWORD }} SQLSERVER_DB: ${{ secrets.SQLSERVER_DB }} PEERDB_CATALOG_HOST: localhost - PEERDB_CATALOG_PORT: 7132 + PEERDB_CATALOG_PORT: 9901 PEERDB_CATALOG_USER: postgres PEERDB_CATALOG_PASSWORD: postgres PEERDB_CATALOG_DATABASE: postgres diff --git a/docker-compose-ci.yml b/docker-compose-ci.yml new file mode 100644 index 0000000000..d5a8fffa8d --- /dev/null +++ b/docker-compose-ci.yml @@ -0,0 +1,97 @@ +version: "3.9" + +x-catalog-config: &catalog-config + PEERDB_CATALOG_HOST: catalog + PEERDB_CATALOG_PORT: 5432 + PEERDB_CATALOG_USER: postgres + PEERDB_CATALOG_PASSWORD: postgres + PEERDB_CATALOG_DATABASE: postgres + +x-flow-worker-env: &flow-worker-env + TEMPORAL_HOST_PORT: temporal:7233 + TEMPORAL_CLIENT_CERT: + TEMPORAL_CLIENT_KEY: + PEERDB_TEMPORAL_NAMESPACE: default + # For GCS, these will be your HMAC keys instead + # For more information: + # https://cloud.google.com/storage/docs/authentication/managing-hmackeys + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-} + # For GCS, set this to "auto" without the quotes + AWS_REGION: ${AWS_REGION:-} + # For GCS, set this as: https://storage.googleapis.com + AWS_ENDPOINT: ${AWS_ENDPOINT:-} + +services: + catalog: + container_name: catalog + image: imresamu/postgis:15-3.4-alpine + ports: + - 9901:5432 + environment: + PGUSER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + extra_hosts: + - "host.docker.internal:host-gateway" + healthcheck: + test: ["CMD", "pg_isready", "-d", "postgres", "-U", "postgres"] + interval: 10s + timeout: 30s + retries: 5 + start_period: 60s + + temporal: + container_name: temporal + depends_on: + catalog: + condition: service_healthy + environment: + - DB=postgresql + - DB_PORT=5432 + - POSTGRES_USER=postgres + - POSTGRES_PWD=postgres + - POSTGRES_SEEDS=catalog + - DYNAMIC_CONFIG_FILE_PATH=config/dynamicconfig/development-sql.yaml + image: temporalio/auto-setup:1.22 + ports: + - 7233:7233 + volumes: + - ./temporal-dynamicconfig:/etc/temporal/config/dynamicconfig + labels: + kompose.volume.type: configMap + + flow-api: + container_name: flow_api + build: + context: . + dockerfile: stacks/flow.Dockerfile + target: flow-api + args: + PEERDB_VERSION_SHA_SHORT: ${PEERDB_VERSION_SHA_SHORT:-} + ports: + - 8112:8112 + - 8113:8113 + environment: + <<: [*catalog-config, *flow-worker-env] + depends_on: temporal + + flow-snapshot-worker: + container_name: flow-snapshot-worker + build: + context: . + dockerfile: stacks/flow.Dockerfile + target: flow-snapshot-worker + environment: + <<: [*catalog-config, *flow-worker-env] + depends_on: temporal + + flow-worker: + container_name: flow-worker + build: + context: . + dockerfile: stacks/flow.Dockerfile + target: flow-worker + environment: + <<: [*catalog-config, *flow-worker-env] + depends_on: temporal