flow.yml

name: Flow build and test

on:
  pull_request:
    branches: [main]
  push:
    branches: [main]

jobs:
  flow_test:
    strategy:
      matrix:
        runner: [ubuntu-latest, ubicloud]
    runs-on: ${{ matrix.runner }}
    timeout-minutes: 30
    services:
      pg_cdc:
        image: postgres:15.4-alpine
        ports:
          - 7132:5432
        env:
          POSTGRES_USER: postgres
          POSTGRES_PASSWORD: postgres
          POSTGRES_DB: postgres
        options: >-
          --name pg_cdc
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
    steps:
      - name: checkout sources
        uses: actions/checkout@v3

      - uses: actions/setup-go@v3
        with:
          go-version: '>=1.19.0'

      - name: install gotestsum
        run: |
          go install gotest.tools/gotestsum@latest

      - name: download go modules
        run: |
          go mod download
        working-directory: ./flow

      - name: setup gcp service account
        id: gcp-service-account
        uses: jsdaniell/create-json@v1.2.2
        with:
          name: "bq_service_account.json"
          json: ${{ secrets.GCP_GH_CI_PKEY }}

      - name: setup snowflake credentials
        id: sf-credentials
        uses: jsdaniell/create-json@v1.2.2
        with:
          name: "snowflake_creds.json"
          json: ${{ secrets.SNOWFLAKE_GH_CI_PKEY }}

      - name: create hstore extension and increase logical replication limits
        run: |
          docker exec pg_cdc psql -h localhost -p 5432 -U postgres -c "CREATE EXTENSION hstore;"
          docker exec pg_cdc psql -h localhost -p 5432 -U postgres -c "ALTER SYSTEM SET wal_level=logical;"
          docker exec pg_cdc psql -h localhost -p 5432 -U postgres -c "ALTER SYSTEM SET max_replication_slots=100;"
          docker exec pg_cdc psql -h localhost -p 5432 -U postgres -c "ALTER SYSTEM SET max_wal_senders=100;"
          docker restart pg_cdc
        working-directory: ./flow
        env:
          PG_CDC: empty
          PGPASSWORD: postgres

      - name: run tests
        run: |
          gotestsum --format testname -- -p 4 ./... -timeout 1200s
        working-directory: ./flow
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          AWS_REGION: ${{ secrets.AWS_REGION }}
          TEST_BQ_CREDS: ${{ github.workspace }}/bq_service_account.json
          TEST_SF_CREDS: ${{ github.workspace }}/snowflake_creds.json
          AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
          AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
          AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
          AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
          ENABLE_SQLSERVER_TESTS: true
          SQLSERVER_HOST: ${{ secrets.SQLSERVER_HOST }}
          SQLSERVER_PORT: ${{ secrets.SQLSERVER_PORT }}
          SQLSERVER_USER: ${{ secrets.SQLSERVER_USER }}
          SQLSERVER_PASSWORD: ${{ secrets.SQLSERVER_PASSWORD }}
          SQLSERVER_DB: ${{ secrets.SQLSERVER_DB }}