From 88e69ba9cd5dbb89673e170111b2ec454576cb55 Mon Sep 17 00:00:00 2001 From: Tiago Garcia Date: Fri, 17 Dec 2021 14:34:29 +0000 Subject: [PATCH 1/2] Minor improvements to assist with native execution and db setup loading --- Makefile | 9 +++++++-- api/apps/api/config/default.json | 2 -- api/apps/api/src/utils/config.utils.spec.ts | 8 ++++++++ api/apps/geoprocessing/config/default.json | 2 -- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 57c8662fc7..60f7543c26 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,14 @@ .PHONY: start # Read values as needed from .env +# +# Optionally pass 'environment=' to load env vars from '.env-something' instead +# Useful to separate i.e. dev from testing databases +# # If using the same variables in recipes that need to use a dotenv file other # than .env, remember to check that no values from .env are being used # inadvertently. -ENVFILE := $(if $(environment), .env-test-e2e, .env) +ENVFILE := $(if $(environment), .env-$(environment), .env) ifneq (,$(wildcard $(ENVFILE))) include $(ENVFILE) export @@ -103,7 +107,7 @@ seed-geoapi-init-data: sed -e "s/\$$feature_id/$$featureid/g" api/apps/api/test/fixtures/features/$${table_name}.sql | docker-compose $(DOCKER_COMPOSE_FILE) exec -T $(GEO_DB_INSTANCE) psql -U "${GEO_POSTGRES_USER}"; \ done; -# need notebook service to execute a expecific notebook. this requires a full geodb +# need notebook service to execute a specific notebook. this requires a full geodb generate-geo-test-data: extract-geo-test-data docker-compose --project-name ${COMPOSE_PROJECT_NAME} -f ./data/docker-compose.yml exec marxan-science-notebooks papermill --progress-bar --log-output work/notebooks/Lab/convert_csv_sql.ipynb /dev/null mv -f -u -Z data/data/processed/test-wdpa-data.sql api/apps/api/test/fixtures/test-wdpa-data.sql @@ -236,4 +240,5 @@ native-seed-geoapi-init-data: done; native-seed-api-with-test-data: native-db-migrate native-seed-api-init-data | native-seed-geoapi-init-data + @echo "seeding db with testing project and scenarios" psql -U "${API_POSTGRES_USER}" -h "${API_POSTGRES_HOST}" ${API_POSTGRES_DB} < api/apps/api/test/fixtures/test-data.sql diff --git a/api/apps/api/config/default.json b/api/apps/api/config/default.json index 808fd8b82d..df1779d309 100644 --- a/api/apps/api/config/default.json +++ b/api/apps/api/config/default.json @@ -17,7 +17,6 @@ "concurrency": 50 }, "postgresApi": { - "url": null, "port": 5432, "host": null, "username": null, @@ -27,7 +26,6 @@ "logging": "error" }, "postgresGeoApi": { - "url": null, "port": 5432, "host": null, "username": null, diff --git a/api/apps/api/src/utils/config.utils.spec.ts b/api/apps/api/src/utils/config.utils.spec.ts index ef194c0b95..a07de976a9 100644 --- a/api/apps/api/src/utils/config.utils.spec.ts +++ b/api/apps/api/src/utils/config.utils.spec.ts @@ -8,6 +8,14 @@ process.env.NETWORK_CORS_ORIGINS = extraCorsOrigin; import { AppConfig } from './config.utils'; describe('AppConfig', () => { + beforeAll(() => { + if (process.env.NODE_CONFIG_DIR !== 'apps/api/config') { + throw Error( + `Running the test suite with NODE_CONFIG_DIR=${process.env.NODE_CONFIG_DIR}, which may cause this test to fail. Please use NODE_CONFIG_DIR=apps/api/config.`, + ); + } + }); + describe('getFromArrayAndParsedString', () => { // Expected full result from `network.cors.origins`. If updating the default // list in `config`, relevant tests should break and this list should be diff --git a/api/apps/geoprocessing/config/default.json b/api/apps/geoprocessing/config/default.json index e2c24b99f7..499f6d325e 100644 --- a/api/apps/geoprocessing/config/default.json +++ b/api/apps/geoprocessing/config/default.json @@ -10,7 +10,6 @@ } }, "postgresApi": { - "url": null, "port": 5432, "host": null, "username": null, @@ -20,7 +19,6 @@ "logging": "error" }, "postgresGeoApi": { - "url": null, "port": 5432, "host": null, "username": null, From 225d12f807187632b1ee48a4f39bace8aac1e0bf Mon Sep 17 00:00:00 2001 From: Tiago Garcia Date: Mon, 20 Dec 2021 11:34:47 +0000 Subject: [PATCH 2/2] WIP - further sanitize Makefile and env vars --- .github/workflows/api-e2e-tests.yml | 2 +- Makefile | 28 ++++------ docker-compose-test-e2e.ci.yml | 85 ----------------------------- docker-compose-test-e2e.local.yml | 16 ------ docker-compose-test-e2e.yml | 2 + docker-compose.yml | 17 +++--- env.default | 1 + 7 files changed, 22 insertions(+), 129 deletions(-) delete mode 100644 docker-compose-test-e2e.ci.yml delete mode 100644 docker-compose-test-e2e.local.yml diff --git a/.github/workflows/api-e2e-tests.yml b/.github/workflows/api-e2e-tests.yml index 9bb52febee..5be93afedb 100644 --- a/.github/workflows/api-e2e-tests.yml +++ b/.github/workflows/api-e2e-tests.yml @@ -33,7 +33,7 @@ jobs: - name: Checkout envs run: docker-compose -f docker-compose-test-e2e.yml -f docker-compose-test-e2e.ci.yml config - name: Run CI tests via make task - run: make run-test-e2e-ci + run: make test-e2e-backend test-api-unit: runs-on: ubuntu-18.04 timeout-minutes: 10 diff --git a/Makefile b/Makefile index 60f7543c26..c7ae7e80e8 100644 --- a/Makefile +++ b/Makefile @@ -2,24 +2,24 @@ # Read values as needed from .env # -# Optionally pass 'environment=' to load env vars from '.env-something' instead +# Optionally pass 'environment=' to load env vars from '.env.something' instead # Useful to separate i.e. dev from testing databases # # If using the same variables in recipes that need to use a dotenv file other # than .env, remember to check that no values from .env are being used # inadvertently. -ENVFILE := $(if $(environment), .env-$(environment), .env) +ENVFILE := $(if $(environment),.env.$(environment),.env) ifneq (,$(wildcard $(ENVFILE))) include $(ENVFILE) + DOCKER_COMPOSE_FILE := --env-file $(ENVFILE) export endif -CIENV := $(if $(filter $(environment), ci), -f docker-compose-test-e2e.ci.yml , -f docker-compose-test-e2e.local.yml) -API_DB_INSTANCE := $(if $(environment), test-e2e-postgresql-api, postgresql-api) -GEO_DB_INSTANCE := $(if $(environment), test-e2e-postgresql-geo-api, postgresql-geo-api) -REDIS_INSTANCE := $(if $(environment), test-e2e-redis, redis) +API_DB_INSTANCE := $(if $(filter $(IS_DOCKER_E2E_TESTS_RUNTIME), true), test-e2e-postgresql-api, postgresql-api) +GEO_DB_INSTANCE := $(if $(filter $(IS_DOCKER_E2E_TESTS_RUNTIME), true), test-e2e-postgresql-geo-api, postgresql-geo-api) +REDIS_INSTANCE := $(if $(filter $(IS_DOCKER_E2E_TESTS_RUNTIME), true), test-e2e-redis, redis) -DOCKER_COMPOSE_FILE := $(if $(environment), -f docker-compose-test-e2e.yml $(CIENV), -f docker-compose.yml ) +DOCKER_COMPOSE_FILE := $(DOCKER_COMPOSE_FILE) $(if $(filter $(IS_DOCKER_E2E_TESTS_RUNTIME), true), -f docker-compose-test-e2e.yml, -f docker-compose.yml ) DOCKER_CLEAN_VOLUMES := $(if $(environment), , \ docker volume rm -f marxan-cloud_marxan-cloud-postgresql-api-data && \ docker volume rm -f marxan-cloud_marxan-cloud-postgresql-geo-data && \ @@ -33,7 +33,6 @@ NC :=\033[0m # No Color test-commands: @echo $(ENVFILE) @echo $(DOCKER_COMPOSE_FILE) - @echo $(CIENV) @echo $(API_POSTGRES_DB) @echo $(GEO_POSTGRES_USER) @@ -128,8 +127,7 @@ test-start-services: clean-slate docker-compose $(DOCKER_COMPOSE_FILE) exec -T api ./apps/api/entrypoint.sh run-migrations-for-e2e-tests && \ docker-compose $(DOCKER_COMPOSE_FILE) exec -T geoprocessing ./apps/geoprocessing/entrypoint.sh run-migrations-for-e2e-tests -seed-dbs-e2e: test-start-services - $(MAKE) seed-api-with-test-data +seed-dbs-e2e: test-start-services seed-api-with-test-data test-e2e-api: seed-dbs-e2e docker-compose $(DOCKER_COMPOSE_FILE) exec -T api ./apps/api/entrypoint.sh test-e2e @@ -138,13 +136,7 @@ test-e2e-geoprocessing: seed-dbs-e2e docker-compose $(DOCKER_COMPOSE_FILE) exec -T geoprocessing ./apps/geoprocessing/entrypoint.sh test-e2e test-e2e-backend: test-e2e-api test-e2e-geoprocessing - $(MAKE) test-clean-slate - -run-test-e2e-local: - $(MAKE) --keep-going test-e2e-backend environment=local - -run-test-e2e-ci: - $(MAKE) --keep-going test-e2e-backend environment=ci + $(MAKE) test-clean-slate $(ARGS) setup-test-unit-backend: # build API and geoprocessing containers @@ -161,7 +153,7 @@ test-unit-geo: test-unit-backend: setup-test-unit-backend test-unit-api test-unit-geo run-test-unit: - $(MAKE) --keep-going test-unit-backend + $(MAKE) --keep-going test-unit-backend $(ARGS) dump-geodb-data: docker-compose exec -T postgresql-geo-api pg_dump -T migrations -a -U "${GEO_POSTGRES_USER}" -F t ${GEO_POSTGRES_DB} | gzip > data/data/processed/db_dumps/geo_db-$$(date +%Y-%m-%d).tar.gz diff --git a/docker-compose-test-e2e.ci.yml b/docker-compose-test-e2e.ci.yml deleted file mode 100644 index 96ce4d8a81..0000000000 --- a/docker-compose-test-e2e.ci.yml +++ /dev/null @@ -1,85 +0,0 @@ -version: "3.6" -services: - api: - environment: - - API_SERVICE_PORT - - API_SERVICE_URL - - API_AUTH_JWT_SECRET - - API_RUN_MIGRATIONS_ON_STARTUP - - API_LOGGING_MUTE_ALL - - API_POSTGRES_USER - - API_POSTGRES_PASSWORD - - API_POSTGRES_DB - - POSTGRES_API_SERVICE_PORT - - APP_SERVICE_PORT - - GEOPROCESSING_SERVICE_PORT - - GEOPROCESSING_RUN_MIGRATIONS_ON_STARTUP - - GEO_POSTGRES_USER - - GEO_POSTGRES_PASSWORD - - GEO_POSTGRES_DB - - POSTGRES_GEO_SERVICE_PORT - - REDIS_API_SERVICE_PORT - - API_AUTH_X_API_KEY - - geoprocessing: - environment: - - API_SERVICE_PORT - - API_SERVICE_URL - - API_AUTH_JWT_SECRET - - API_RUN_MIGRATIONS_ON_STARTUP - - API_LOGGING_MUTE_ALL - - API_POSTGRES_USER - - API_POSTGRES_PASSWORD - - API_POSTGRES_DB - - POSTGRES_API_SERVICE_PORT - - APP_SERVICE_PORT - - GEOPROCESSING_SERVICE_PORT - - GEOPROCESSING_RUN_MIGRATIONS_ON_STARTUP - - GEO_POSTGRES_USER - - GEO_POSTGRES_PASSWORD - - GEO_POSTGRES_DB - - POSTGRES_GEO_SERVICE_PORT - - REDIS_API_SERVICE_PORT - - API_AUTH_X_API_KEY - - test-e2e-postgresql-api: - environment: - - API_SERVICE_PORT - - API_SERVICE_URL - - API_AUTH_JWT_SECRET - - API_RUN_MIGRATIONS_ON_STARTUP - - API_POSTGRES_USER - - API_POSTGRES_PASSWORD - - API_POSTGRES_DB - - POSTGRES_API_SERVICE_PORT - - APP_SERVICE_PORT - - GEOPROCESSING_SERVICE_PORT - - GEOPROCESSING_RUN_MIGRATIONS_ON_STARTUP - - GEO_POSTGRES_USER - - GEO_POSTGRES_PASSWORD - - GEO_POSTGRES_DB - - POSTGRES_GEO_SERVICE_PORT - - REDIS_API_SERVICE_PORT - - test-e2e-postgresql-geo-api: - environment: - - API_SERVICE_PORT - - API_SERVICE_URL - - API_AUTH_JWT_SECRET - - API_RUN_MIGRATIONS_ON_STARTUP - - API_POSTGRES_USER - - API_POSTGRES_PASSWORD - - API_POSTGRES_DB - - POSTGRES_API_SERVICE_PORT - - APP_SERVICE_PORT - - GEOPROCESSING_SERVICE_PORT - - GEOPROCESSING_RUN_MIGRATIONS_ON_STARTUP - - GEO_POSTGRES_USER - - GEO_POSTGRES_PASSWORD - - GEO_POSTGRES_DB - - POSTGRES_GEO_SERVICE_PORT - - REDIS_API_SERVICE_PORT - - test-e2e-redis: - environment: - - REDIS_API_SERVICE_PORT diff --git a/docker-compose-test-e2e.local.yml b/docker-compose-test-e2e.local.yml deleted file mode 100644 index f378807ded..0000000000 --- a/docker-compose-test-e2e.local.yml +++ /dev/null @@ -1,16 +0,0 @@ -version: "3.8" -services: - api: - env_file: .env-test-e2e - - geoprocessing: - env_file: .env-test-e2e - - test-e2e-postgresql-api: - env_file: .env-test-e2e - - test-e2e-postgresql-geo-api: - env_file: .env-test-e2e - - test-e2e-redis: - env_file: .env-test-e2e diff --git a/docker-compose-test-e2e.yml b/docker-compose-test-e2e.yml index 659dba4f79..3aa2c4c062 100644 --- a/docker-compose-test-e2e.yml +++ b/docker-compose-test-e2e.yml @@ -18,6 +18,7 @@ services: - ./api/apps:/opt/marxan-api/apps - ./api/libs:/opt/marxan-api/libs - marxan-cloud-backend-e2e-temp-storage:/tmp/storage + env_file: ${ENVFILE} environment: - NODE_PATH=src - NODE_ENV=test @@ -54,6 +55,7 @@ services: - ./api/apps:/opt/marxan-geoprocessing/apps - ./api/libs:/opt/marxan-geoprocessing/libs - marxan-cloud-backend-e2e-temp-storage:/tmp/storage + env_file: ${ENVFILE} environment: - NODE_PATH=src - NODE_ENV=test diff --git a/docker-compose.yml b/docker-compose.yml index ce1bc77e78..f1b9db03df 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,16 +17,16 @@ services: - ./api/apps:/opt/marxan-api/apps - ./api/libs:/opt/marxan-api/libs - marxan-cloud-backend-temp-storage:/tmp/storage - env_file: .env + env_file: ${ENVFILE} environment: - NODE_PATH=src - NODE_ENV=development - - API_POSTGRES_HOST=marxan-postgresql-api + - API_POSTGRES_HOST=postgresql-api - API_POSTGRES_USER - API_POSTGRES_PASSWORD - API_POSTGRES_PORT=5432 + - GEO_POSTGRES_HOST=postgresql-geo-api - API_POSTGRES_DB - - GEO_POSTGRES_HOST=marxan-postgresql-geo-api - GEO_POSTGRES_USER - GEO_POSTGRES_PASSWORD - GEO_POSTGRES_PORT=5432 @@ -49,7 +49,7 @@ services: volumes: - ./app/src:/opt/marxan-app/src - ./app/test:/opt/marxan-app/test - env_file: .env + env_file: ${ENVFILE} environment: - NODE_PATH=src - NODE_ENV=development @@ -70,16 +70,16 @@ services: - ./api/apps:/opt/marxan-geoprocessing/apps - ./api/libs:/opt/marxan-geoprocessing/libs - marxan-cloud-backend-temp-storage:/tmp/storage - env_file: .env + env_file: ${ENVFILE} environment: - NODE_PATH=src - NODE_ENV=development - - API_POSTGRES_HOST=marxan-postgresql-api + - API_POSTGRES_HOST=postgresql-api - API_POSTGRES_USER - API_POSTGRES_PASSWORD - API_POSTGRES_PORT=5432 - API_POSTGRES_DB - - GEO_POSTGRES_HOST=marxan-postgresql-geo-api + - GEO_POSTGRES_HOST=postgresql-geo-api - GEO_POSTGRES_USER - GEO_POSTGRES_PASSWORD - GEO_POSTGRES_PORT=5432 @@ -176,8 +176,7 @@ services: image: apache/airflow:2.0.0 restart: on-failure entrypoint: ['sh','./scripts/entrypoint.sh'] - env_file: - - .env + env_file: ${ENVFILE} environment: - AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgresql-airflow/airflow - AIRFLOW__CORE__EXECUTOR=LocalExecutor diff --git a/env.default b/env.default index 2a7e3090b5..efec3dc11e 100644 --- a/env.default +++ b/env.default @@ -32,3 +32,4 @@ APPLICATION_BASE_URL=http://localhost:3000 PASSWORD_RESET_TOKEN_PREFIX=/auth/reset-password?token= PASSWORD_RESET_EXPIRATION=1800000 SIGNUP_CONFIRMATION_TOKEN_PREFIX=/auth/sign-up-confirmation?token= +IS_DOCKER_E2E_TESTS_RUNTIME=false