-
Notifications
You must be signed in to change notification settings - Fork 97
110 lines (99 loc) · 3.59 KB
/
flow.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
name: Flow build and test
on:
pull_request:
branches:
- "main"
push:
branches: [main]
jobs:
flow_test:
strategy:
matrix:
runner: [ubicloud-standard-8]
runs-on: ${{ matrix.runner }}
timeout-minutes: 30
services:
pg_cdc:
image: postgres:15.4-alpine
ports:
- 7132:5432
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: postgres
options: >-
--name pg_cdc
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- name: checkout sources
uses: actions/checkout@v3
- uses: actions/setup-go@v3
with:
go-version: ">=1.19.0"
- name: install gotestsum
run: |
go install gotest.tools/gotestsum@latest
- name: download go modules
run: |
go mod download
working-directory: ./flow
- name: setup gcp service account
id: gcp-service-account
uses: jsdaniell/create-json@v1.2.2
with:
name: "bq_service_account.json"
json: ${{ secrets.GCP_GH_CI_PKEY }}
- name: setup snowflake credentials
id: sf-credentials
uses: jsdaniell/create-json@v1.2.2
with:
name: "snowflake_creds.json"
json: ${{ secrets.SNOWFLAKE_GH_CI_PKEY }}
- name: setup S3 credentials
id: s3-credentials
uses: jsdaniell/create-json@v1.2.2
with:
name: "s3_creds.json"
json: ${{ secrets.S3_CREDS }}
- name: setup GCS credentials
id: gcs-credentials
uses: jsdaniell/create-json@v1.2.2
with:
name: "gcs_creds.json"
json: ${{ secrets.GCS_CREDS }}
- name: create hstore extension and increase logical replication limits
run: |
docker exec pg_cdc psql -h localhost -p 5432 -U postgres -c "CREATE EXTENSION hstore;"
docker exec pg_cdc psql -h localhost -p 5432 -U postgres -c "ALTER SYSTEM SET wal_level=logical;"
docker exec pg_cdc psql -h localhost -p 5432 -U postgres -c "ALTER SYSTEM SET max_replication_slots=100;"
docker exec pg_cdc psql -h localhost -p 5432 -U postgres -c "ALTER SYSTEM SET max_wal_senders=100;"
docker restart pg_cdc
working-directory: ./flow
env:
PG_CDC: empty
PGPASSWORD: postgres
- name: run tests
run: |
gotestsum --format testname -- -p 4 ./... -timeout 2400s
working-directory: ./flow
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ secrets.AWS_REGION }}
TEST_BQ_CREDS: ${{ github.workspace }}/bq_service_account.json
TEST_SF_CREDS: ${{ github.workspace }}/snowflake_creds.json
TEST_S3_CREDS: ${{ github.workspace }}/s3_creds.json
TEST_GCS_CREDS: ${{ github.workspace }}/gcs_creds.json
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
ENABLE_SQLSERVER_TESTS: true
SQLSERVER_HOST: ${{ secrets.SQLSERVER_HOST }}
SQLSERVER_PORT: ${{ secrets.SQLSERVER_PORT }}
SQLSERVER_USER: ${{ secrets.SQLSERVER_USER }}
SQLSERVER_PASSWORD: ${{ secrets.SQLSERVER_PASSWORD }}
SQLSERVER_DB: ${{ secrets.SQLSERVER_DB }}