optimizing flow tests by separating and parallelizing test suites (#356)

1. Split each connector into its own test suite, and rewrote them to
allow running multiple test suites in parallel.
2. Better separation on source PG instance [shared by all test suites of
a run] and destination BQ and SF instances [shared by all runs].
3. some code cleanup and fixing some tests up.
4. Switched to using a vanilla PG instance, but configured with higher
logical replication limits.

.github/workflows/flow.yml

@@ -1,25 +1,26 @@
 name: Flow build and test
 
 on:
-  push:
-    branches: [main]
   pull_request:
     branches: [main]
+  push:
+    branches: [main]
 
 jobs:
   flow_test:
     runs-on: ubuntu-latest
     timeout-minutes: 30
     services:
       pg_cdc:
-        image: debezium/postgres:14-alpine
+        image: postgres:15.4-alpine
         ports:
           - 7132:5432
         env:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
           POSTGRES_DB: postgres
         options: >-
+          --name pg_cdc
           --health-cmd pg_isready
           --health-interval 10s
           --health-timeout 5s
@@ -55,18 +56,21 @@ jobs:
           name: "snowflake_creds.json"
           json: ${{ secrets.SNOWFLAKE_GH_CI_PKEY }}
 
-      - name: create hstore extension
+      - name: create hstore extension and increase logical replication limits
         run: |
-          sudo apt-get update
-          sudo apt-get install -y postgresql-client
-          psql -h localhost -p 7132 -U postgres -c "CREATE EXTENSION hstore;"
+          docker exec pg_cdc psql -h localhost -p 5432 -U postgres -c "CREATE EXTENSION hstore;"
+          docker exec pg_cdc psql -h localhost -p 5432 -U postgres -c "ALTER SYSTEM SET wal_level=logical;"          
+          docker exec pg_cdc psql -h localhost -p 5432 -U postgres -c "ALTER SYSTEM SET max_replication_slots=100;"
+          docker exec pg_cdc psql -h localhost -p 5432 -U postgres -c "ALTER SYSTEM SET max_wal_senders=100;"
+          docker restart pg_cdc         
         working-directory: ./flow
         env:
+          PG_CDC: 
           PGPASSWORD: postgres
 
       - name: run tests
         run: |
-          gotestsum --format testname -- -p 1 ./... -timeout 1200s
+          gotestsum --format testname -- -p 4 ./... -timeout 1200s
         working-directory: ./flow
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}

flow/e2e/bigquery_helper.go → flow/e2e/bigquery/bigquery_helper.go

@@ -1,4 +1,4 @@
-package e2e
+package e2e_bigquery
 
 import (
 	"context"
@@ -12,6 +12,7 @@ import (
 	"cloud.google.com/go/bigquery"
 	"cloud.google.com/go/civil"
 	peer_bq "github.com/PeerDB-io/peer-flow/connectors/bigquery"
+	"github.com/PeerDB-io/peer-flow/e2e"
 	"github.com/PeerDB-io/peer-flow/generated/protos"
 	"github.com/PeerDB-io/peer-flow/model"
 	"github.com/PeerDB-io/peer-flow/model/qvalue"
@@ -45,7 +46,7 @@ func NewBigQueryTestHelper() (*BigQueryTestHelper, error) {
 		return nil, fmt.Errorf("TEST_BQ_CREDS env var not set")
 	}
 
-	content, err := readFileToBytes(jsonPath)
+	content, err := e2e.ReadFileToBytes(jsonPath)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read file: %w", err)
 	}