From 806c7da8ab1c10d6fe5488acc7ee989d1cc9f6ae Mon Sep 17 00:00:00 2001 From: Andy Pfister Date: Tue, 22 Oct 2024 16:51:01 +0200 Subject: [PATCH] Test pgautoupgrade container using AdventureWorks database --- .gitignore | 3 +++ README.md | 11 ++++----- test.sh | 27 ++++++++++++++------ test/.env | 6 ----- test/docker-compose-pg10.yml | 45 ---------------------------------- test/docker-compose-pg11.yml | 45 ---------------------------------- test/docker-compose-pg12.yml | 45 ---------------------------------- test/docker-compose-pg13.yml | 45 ---------------------------------- test/docker-compose-pg14.yml | 45 ---------------------------------- test/docker-compose-pg15.yml | 45 ---------------------------------- test/docker-compose-pg16.yml | 45 ---------------------------------- test/docker-compose-pg9.5.yml | 45 ---------------------------------- test/docker-compose-pg9.6.yml | 45 ---------------------------------- test/docker-compose-pgauto.yml | 45 ---------------------------------- 14 files changed, 28 insertions(+), 469 deletions(-) diff --git a/.gitignore b/.gitignore index 586767f..7f734ad 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ pgstuff test/postgres-data .idea + +test/AdventureWorks.sql +test/AdventureWorks.tar.xz diff --git a/README.md b/README.md index 6478e8b..f79a88b 100644 --- a/README.md +++ b/README.md @@ -133,15 +133,14 @@ To run the tests, use: $ make test ``` -The test script creates an initial PostgreSQL database for -Redash using an older PG version, then starts Redash using -the above "automatic updating" PostgreSQL image to update -the database to the latest PostgreSQL version. +The test script imports the AdventureWorks database (ported from Microsoft +land) into an older PG version, then starts the pgautoupgrade container to +update the database to the latest PostgreSQL version. It then checks that the database files were indeed updated to the newest PostgreSQL release, and outputs an obvious SUCCESS/FAILURE message for that loop. The test runs in a loop, testing (in sequence) upgrades from -PostgreSQL versions 9.5, 9.6, 10.x, 11.x, 12.x, 13.x, 14.x -and 15.x. +PostgreSQL versions 9.5, 9.6, 10.x, 11.x, 12.x, 13.x, 14.x, 15.x., 16.x and +17.x. diff --git a/test.sh b/test.sh index 4ed757f..edb786e 100755 --- a/test.sh +++ b/test.sh @@ -25,6 +25,18 @@ test_down() { docker compose -f test/docker-compose-pgauto.yml down } +import_adventure_works() { + # Create the PostgreSQL database using a specific version of PostgreSQL + docker compose -f "docker-compose-pg${VERSION}.yml" up -d --wait + + # Create a database and import AdventureWorks + docker compose -f "docker-compose-pg${VERSION}.yml" exec postgres createdb -U postgres AdventureWorks + cat AdventureWorks.sql | docker compose -f "docker-compose-pg${VERSION}.yml" exec --no-TTY postgres psql -U postgres -d AdventureWorks > /dev/null 2>&1 + + # Shutdown the existing version + docker compose -f "docker-compose-pg${VERSION}.yml" down -v +} + test_run() { VERSION=$1 TARGET=$2 @@ -36,10 +48,9 @@ test_run() { sudo rm -rf postgres-data fi - # Create the PostgreSQL database using a specific version of PostgreSQL - docker compose -f "docker-compose-pg${VERSION}.yml" run --rm server create_db + import_adventure_works - # Start Redash normally, using an "autoupdate" version of PostgreSQL + # Start pgautoupgrade container TARGET_TAG="${TARGET}-${FLAVOR}" docker compose -f docker-compose-pgauto.yml up --wait -d # Verify the PostgreSQL data files are now the target version @@ -60,10 +71,7 @@ test_run() { ## ## Tests for one shot mode ## - banner '-' "Testing 'one shot' automatic upgrade mode for PostgreSQL ${VERSION} to ${TARGET}" - - # Create the PostgreSQL database using a specific version of PostgreSQL - docker compose -f "docker-compose-pg${VERSION}.yml" run --rm server create_db + import_adventure_works # Shut down all of the containers docker compose -f "docker-compose-pg${VERSION}.yml" down --remove-orphans @@ -102,6 +110,11 @@ fi # Change into the test directory cd test || exit 1 +echo "Downloading and extracting the AdventureWork database ..." +rm -rf AdventureWorks.tar.xz AdventureWorks.sql +curl -L -o AdventureWorks.tar.xz "https://github.com/pgautoupgrade/AdventureWorks-for-Postgres/raw/refs/heads/master/AdventureWorks.tar.xz" +tar -xf AdventureWorks.tar.xz + for version in "${PG_VERSIONS[@]}"; do # Only test if the version is less than the latest version if [[ $(echo "$version < $PGTARGET" | bc) -eq 1 ]]; then diff --git a/test/.env b/test/.env index ea5e2ba..0dee66a 100644 --- a/test/.env +++ b/test/.env @@ -1,8 +1,2 @@ # The values below are useful for testing this repo with -#REDASH_LOG_LEVEL=INFO -REDASH_COOKIE_SECRET=HKuj2aZx7vRQXghnegXY0tDZWPS493F5 -REDASH_SECRET_KEY=SHGVxI1Yecfz9k7xjwMnF0hCtwh04dFk -PYTHONUNBUFFERED=0 -REDASH_REDIS_URL=redis://redis:6379/0 POSTGRES_PASSWORD=FmTKs5vX52ufKR1rd8tn4MoSP7zvCJwb -REDASH_DATABASE_URL=postgresql://postgres:FmTKs5vX52ufKR1rd8tn4MoSP7zvCJwb@postgres/postgres diff --git a/test/docker-compose-pg10.yml b/test/docker-compose-pg10.yml index 1517d4e..8b93c49 100644 --- a/test/docker-compose-pg10.yml +++ b/test/docker-compose-pg10.yml @@ -1,43 +1,4 @@ -x-redash-service: &redash-service - image: redash/redash:10.1.0.b50633 - depends_on: - redis: - condition: service_healthy - postgres: - condition: service_healthy - env_file: .env - restart: always services: - server: - <<: *redash-service - command: server - ports: - - "127.0.0.1:5000:5000" - environment: - REDASH_WEB_WORKERS: 22 - scheduler: - <<: *redash-service - command: scheduler - scheduled_worker: - <<: *redash-service - command: worker - environment: - QUEUES: "scheduled_queries,schemas" - WORKERS_COUNT: 2 - adhoc_worker: - <<: *redash-service - command: worker - environment: - QUEUES: "queries" - WORKERS_COUNT: 2 - redis: - image: redis:7-alpine - restart: always - healthcheck: - test: redis-cli -h 127.0.0.1 ping | grep -q PONG - interval: 10s - timeout: 5s - retries: 5 postgres: image: postgres:10-alpine env_file: .env @@ -49,9 +10,3 @@ services: interval: 10s timeout: 5s retries: 5 - worker: - <<: *redash-service - command: worker - environment: - QUEUES: "periodic emails default" - WORKERS_COUNT: 2 diff --git a/test/docker-compose-pg11.yml b/test/docker-compose-pg11.yml index 3037db5..4cf733c 100644 --- a/test/docker-compose-pg11.yml +++ b/test/docker-compose-pg11.yml @@ -1,43 +1,4 @@ -x-redash-service: &redash-service - image: redash/redash:10.1.0.b50633 - depends_on: - redis: - condition: service_healthy - postgres: - condition: service_healthy - env_file: .env - restart: always services: - server: - <<: *redash-service - command: server - ports: - - "127.0.0.1:5000:5000" - environment: - REDASH_WEB_WORKERS: 22 - scheduler: - <<: *redash-service - command: scheduler - scheduled_worker: - <<: *redash-service - command: worker - environment: - QUEUES: "scheduled_queries,schemas" - WORKERS_COUNT: 2 - adhoc_worker: - <<: *redash-service - command: worker - environment: - QUEUES: "queries" - WORKERS_COUNT: 2 - redis: - image: redis:7-alpine - restart: always - healthcheck: - test: redis-cli -h 127.0.0.1 ping | grep -q PONG - interval: 10s - timeout: 5s - retries: 5 postgres: image: postgres:11-alpine env_file: .env @@ -49,9 +10,3 @@ services: interval: 10s timeout: 5s retries: 5 - worker: - <<: *redash-service - command: worker - environment: - QUEUES: "periodic emails default" - WORKERS_COUNT: 2 diff --git a/test/docker-compose-pg12.yml b/test/docker-compose-pg12.yml index f8a7459..f961eb2 100644 --- a/test/docker-compose-pg12.yml +++ b/test/docker-compose-pg12.yml @@ -1,43 +1,4 @@ -x-redash-service: &redash-service - image: redash/redash:10.1.0.b50633 - depends_on: - redis: - condition: service_healthy - postgres: - condition: service_healthy - env_file: .env - restart: always services: - server: - <<: *redash-service - command: server - ports: - - "127.0.0.1:5000:5000" - environment: - REDASH_WEB_WORKERS: 22 - scheduler: - <<: *redash-service - command: scheduler - scheduled_worker: - <<: *redash-service - command: worker - environment: - QUEUES: "scheduled_queries,schemas" - WORKERS_COUNT: 2 - adhoc_worker: - <<: *redash-service - command: worker - environment: - QUEUES: "queries" - WORKERS_COUNT: 2 - redis: - image: redis:7-alpine - restart: always - healthcheck: - test: redis-cli -h 127.0.0.1 ping | grep -q PONG - interval: 10s - timeout: 5s - retries: 5 postgres: image: postgres:12-alpine env_file: .env @@ -49,9 +10,3 @@ services: interval: 10s timeout: 5s retries: 5 - worker: - <<: *redash-service - command: worker - environment: - QUEUES: "periodic emails default" - WORKERS_COUNT: 2 diff --git a/test/docker-compose-pg13.yml b/test/docker-compose-pg13.yml index d854e70..fa18588 100644 --- a/test/docker-compose-pg13.yml +++ b/test/docker-compose-pg13.yml @@ -1,43 +1,4 @@ -x-redash-service: &redash-service - image: redash/redash:10.1.0.b50633 - depends_on: - redis: - condition: service_healthy - postgres: - condition: service_healthy - env_file: .env - restart: always services: - server: - <<: *redash-service - command: server - ports: - - "127.0.0.1:5000:5000" - environment: - REDASH_WEB_WORKERS: 22 - scheduler: - <<: *redash-service - command: scheduler - scheduled_worker: - <<: *redash-service - command: worker - environment: - QUEUES: "scheduled_queries,schemas" - WORKERS_COUNT: 2 - adhoc_worker: - <<: *redash-service - command: worker - environment: - QUEUES: "queries" - WORKERS_COUNT: 2 - redis: - image: redis:7-alpine - restart: always - healthcheck: - test: redis-cli -h 127.0.0.1 ping | grep -q PONG - interval: 10s - timeout: 5s - retries: 5 postgres: image: postgres:13-alpine env_file: .env @@ -49,9 +10,3 @@ services: interval: 10s timeout: 5s retries: 5 - worker: - <<: *redash-service - command: worker - environment: - QUEUES: "periodic emails default" - WORKERS_COUNT: 2 diff --git a/test/docker-compose-pg14.yml b/test/docker-compose-pg14.yml index 94d982a..ffc9294 100644 --- a/test/docker-compose-pg14.yml +++ b/test/docker-compose-pg14.yml @@ -1,43 +1,4 @@ -x-redash-service: &redash-service - image: redash/redash:10.1.0.b50633 - depends_on: - redis: - condition: service_healthy - postgres: - condition: service_healthy - env_file: .env - restart: always services: - server: - <<: *redash-service - command: server - ports: - - "127.0.0.1:5000:5000" - environment: - REDASH_WEB_WORKERS: 22 - scheduler: - <<: *redash-service - command: scheduler - scheduled_worker: - <<: *redash-service - command: worker - environment: - QUEUES: "scheduled_queries,schemas" - WORKERS_COUNT: 2 - adhoc_worker: - <<: *redash-service - command: worker - environment: - QUEUES: "queries" - WORKERS_COUNT: 2 - redis: - image: redis:7-alpine - restart: always - healthcheck: - test: redis-cli -h 127.0.0.1 ping | grep -q PONG - interval: 10s - timeout: 5s - retries: 5 postgres: image: postgres:14-alpine env_file: .env @@ -49,9 +10,3 @@ services: interval: 10s timeout: 5s retries: 5 - worker: - <<: *redash-service - command: worker - environment: - QUEUES: "periodic emails default" - WORKERS_COUNT: 2 diff --git a/test/docker-compose-pg15.yml b/test/docker-compose-pg15.yml index 0cc83d2..18db87c 100644 --- a/test/docker-compose-pg15.yml +++ b/test/docker-compose-pg15.yml @@ -1,43 +1,4 @@ -x-redash-service: &redash-service - image: redash/redash:10.1.0.b50633 - depends_on: - redis: - condition: service_healthy - postgres: - condition: service_healthy - env_file: .env - restart: always services: - server: - <<: *redash-service - command: server - ports: - - "127.0.0.1:5000:5000" - environment: - REDASH_WEB_WORKERS: 22 - scheduler: - <<: *redash-service - command: scheduler - scheduled_worker: - <<: *redash-service - command: worker - environment: - QUEUES: "scheduled_queries,schemas" - WORKERS_COUNT: 2 - adhoc_worker: - <<: *redash-service - command: worker - environment: - QUEUES: "queries" - WORKERS_COUNT: 2 - redis: - image: redis:7-alpine - restart: always - healthcheck: - test: redis-cli -h 127.0.0.1 ping | grep -q PONG - interval: 10s - timeout: 5s - retries: 5 postgres: image: postgres:15-alpine env_file: .env @@ -49,9 +10,3 @@ services: interval: 10s timeout: 5s retries: 5 - worker: - <<: *redash-service - command: worker - environment: - QUEUES: "periodic emails default" - WORKERS_COUNT: 2 diff --git a/test/docker-compose-pg16.yml b/test/docker-compose-pg16.yml index 96e0bed..4e4404b 100644 --- a/test/docker-compose-pg16.yml +++ b/test/docker-compose-pg16.yml @@ -1,43 +1,4 @@ -x-redash-service: &redash-service - image: redash/redash:10.1.0.b50633 - depends_on: - redis: - condition: service_healthy - postgres: - condition: service_healthy - env_file: .env - restart: always services: - server: - <<: *redash-service - command: server - ports: - - "127.0.0.1:5000:5000" - environment: - REDASH_WEB_WORKERS: 22 - scheduler: - <<: *redash-service - command: scheduler - scheduled_worker: - <<: *redash-service - command: worker - environment: - QUEUES: "scheduled_queries,schemas" - WORKERS_COUNT: 2 - adhoc_worker: - <<: *redash-service - command: worker - environment: - QUEUES: "queries" - WORKERS_COUNT: 2 - redis: - image: redis:7-alpine - restart: always - healthcheck: - test: redis-cli -h 127.0.0.1 ping | grep -q PONG - interval: 10s - timeout: 5s - retries: 5 postgres: image: postgres:16-alpine env_file: .env @@ -49,9 +10,3 @@ services: interval: 10s timeout: 5s retries: 5 - worker: - <<: *redash-service - command: worker - environment: - QUEUES: "periodic emails default" - WORKERS_COUNT: 2 diff --git a/test/docker-compose-pg9.5.yml b/test/docker-compose-pg9.5.yml index 92ad396..29ca60c 100644 --- a/test/docker-compose-pg9.5.yml +++ b/test/docker-compose-pg9.5.yml @@ -1,43 +1,4 @@ -x-redash-service: &redash-service - image: redash/redash:10.1.0.b50633 - depends_on: - redis: - condition: service_healthy - postgres: - condition: service_healthy - env_file: .env - restart: always services: - server: - <<: *redash-service - command: server - ports: - - "127.0.0.1:5000:5000" - environment: - REDASH_WEB_WORKERS: 22 - scheduler: - <<: *redash-service - command: scheduler - scheduled_worker: - <<: *redash-service - command: worker - environment: - QUEUES: "scheduled_queries,schemas" - WORKERS_COUNT: 2 - adhoc_worker: - <<: *redash-service - command: worker - environment: - QUEUES: "queries" - WORKERS_COUNT: 2 - redis: - image: redis:7-alpine - restart: always - healthcheck: - test: redis-cli -h 127.0.0.1 ping | grep -q PONG - interval: 10s - timeout: 5s - retries: 5 postgres: image: postgres:9.5-alpine env_file: .env @@ -49,9 +10,3 @@ services: interval: 10s timeout: 5s retries: 5 - worker: - <<: *redash-service - command: worker - environment: - QUEUES: "periodic emails default" - WORKERS_COUNT: 2 diff --git a/test/docker-compose-pg9.6.yml b/test/docker-compose-pg9.6.yml index 228e5f6..e97a746 100644 --- a/test/docker-compose-pg9.6.yml +++ b/test/docker-compose-pg9.6.yml @@ -1,43 +1,4 @@ -x-redash-service: &redash-service - image: redash/redash:10.1.0.b50633 - depends_on: - redis: - condition: service_healthy - postgres: - condition: service_healthy - env_file: .env - restart: always services: - server: - <<: *redash-service - command: server - ports: - - "127.0.0.1:5000:5000" - environment: - REDASH_WEB_WORKERS: 22 - scheduler: - <<: *redash-service - command: scheduler - scheduled_worker: - <<: *redash-service - command: worker - environment: - QUEUES: "scheduled_queries,schemas" - WORKERS_COUNT: 2 - adhoc_worker: - <<: *redash-service - command: worker - environment: - QUEUES: "queries" - WORKERS_COUNT: 2 - redis: - image: redis:7-alpine - restart: always - healthcheck: - test: redis-cli -h 127.0.0.1 ping | grep -q PONG - interval: 10s - timeout: 5s - retries: 5 postgres: image: postgres:9.6-alpine env_file: .env @@ -49,9 +10,3 @@ services: interval: 10s timeout: 5s retries: 5 - worker: - <<: *redash-service - command: worker - environment: - QUEUES: "periodic emails default" - WORKERS_COUNT: 2 diff --git a/test/docker-compose-pgauto.yml b/test/docker-compose-pgauto.yml index f3cb600..ad30c8d 100644 --- a/test/docker-compose-pgauto.yml +++ b/test/docker-compose-pgauto.yml @@ -1,43 +1,4 @@ -x-redash-service: &redash-service - image: redash/redash:10.1.0.b50633 - depends_on: - redis: - condition: service_healthy - postgres: - condition: service_healthy - env_file: .env - restart: always services: - server: - <<: *redash-service - command: server - ports: - - "127.0.0.1:5000:5000" - environment: - REDASH_WEB_WORKERS: 22 - scheduler: - <<: *redash-service - command: scheduler - scheduled_worker: - <<: *redash-service - command: worker - environment: - QUEUES: "scheduled_queries,schemas" - WORKERS_COUNT: 2 - adhoc_worker: - <<: *redash-service - command: worker - environment: - QUEUES: "queries" - WORKERS_COUNT: 2 - redis: - image: redis:7-alpine - restart: always - healthcheck: - test: redis-cli -h 127.0.0.1 ping | grep -q PONG - interval: 10s - timeout: 5s - retries: 5 postgres: image: pgautoupgrade/pgautoupgrade:${TARGET_TAG:-dev} env_file: .env @@ -49,9 +10,3 @@ services: interval: 10s timeout: 5s retries: 5 - worker: - <<: *redash-service - command: worker - environment: - QUEUES: "periodic emails default" - WORKERS_COUNT: 2