From a8439c05d24025d76e376ffc19a01a86ed34faab Mon Sep 17 00:00:00 2001 From: Ivan Ogasawara Date: Wed, 25 May 2022 17:23:07 -0300 Subject: [PATCH] chore(docker): add postgresql service properly --- .env.tpl | 10 ++++++-- .github/workflows/main.yaml | 14 +++++++---- Data_Collection/CRON_scripts/config.py | 8 +++---- Makefile | 27 +++++++++++++++++++--- docker/compose-dev.yaml | 24 ------------------- docker/compose-prod.yaml | 6 ----- docker/{compose-base.yaml => compose.yaml} | 3 +++ docker/postgresql/dev/prepare-db.sh | 19 +++++++++++++++ docker/postgresql/dump.sh | 19 +++++++++++++++ docker/postgresql/prepare-db.sh | 21 ----------------- docker/postgresql/restore.sh | 18 +++++++++++++++ docker/sql/{ => dev}/database.sql | 0 docker/sql/{ => dev}/epigraphhub.sql | 0 docker/sql/{ => dev}/privatehub.sql | 0 14 files changed, 104 insertions(+), 65 deletions(-) delete mode 100644 docker/compose-dev.yaml delete mode 100644 docker/compose-prod.yaml rename docker/{compose-base.yaml => compose.yaml} (97%) create mode 100644 docker/postgresql/dev/prepare-db.sh create mode 100755 docker/postgresql/dump.sh delete mode 100644 docker/postgresql/prepare-db.sh create mode 100755 docker/postgresql/restore.sh rename docker/sql/{ => dev}/database.sql (100%) rename docker/sql/{ => dev}/epigraphhub.sql (100%) rename docker/sql/{ => dev}/privatehub.sql (100%) diff --git a/.env.tpl b/.env.tpl index 14dcfc2e..28440164 100644 --- a/.env.tpl +++ b/.env.tpl @@ -2,14 +2,20 @@ ANSIBLE_VAULT_KEY=${ANSIBLE_VAULT_KEY} ENV=${ENV} FLOWER_PORT=${FLOWER_PORT} FLOWER_PASSWORD=${FLOWER_PASSWORD} +HOST_POSTGRES_DUMP=${HOST_POSTGRES_DUMP} MAPBOX_API_KEY=${MAPBOX_API_KEY} +POSTGRES_DUMP_HOST=${POSTGRES_DUMP_HOST} +POSTGRES_DUMP_CONTAINER=${POSTGRES_DUMP_CONTAINER} POSTGRES_HOST=${POSTGRES_HOST} POSTGRES_PORT=${POSTGRES_PORT} POSTGRES_USER=${POSTGRES_USER} POSTGRES_PASSWORD=${POSTGRES_PASSWORD} POSTGRES_DB=${POSTGRES_DB} -POSTGRES_DB_PRIVATE=${POSTGRES_DB_PRIVATE} -POSTGRES_DB_SANDBOX=${POSTGRES_DB_SANDBOX} +POSTGRES_EPIGRAPH_USER=${POSTGRES_EPIGRAPH_USER} +POSTGRES_EPIGRAPH_PASSWORD=${POSTGRES_EPIGRAPH_PASSWORD} +POSTGRES_EPIGRAPH_DB=${POSTGRES_EPIGRAPH_DB} +POSTGRES_EPIGRAPH_DB_PRIVATE=${POSTGRES_DB_EPIGRAPH_PRIVATE} +POSTGRES_EPIGRAPH_DB_SANDBOX=${POSTGRES_DB_EPIGRAPH_SANDBOX} RECAPTCHA_PUBLIC_KEY=${RECAPTCHA_PUBLIC_KEY} RECAPTCHA_PRIVATE_KEY=${RECAPTCHA_PRIVATE_KEY} REDIS_HOST=${REDIS_HOST} diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 9d13dd23..4a9c8f38 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -10,13 +10,17 @@ env: ENV: dev FLOWER_PORT: 28888 FLOWER_PASSWORD: flowerpass + POSTGRES_DUMP_HOST: /tmp/dump POSTGRES_HOST: epigraphhub-db POSTGRES_PORT: 25432 - POSTGRES_USER: dev_epigraph - POSTGRES_PASSWORD: dev_epigraph - POSTGRES_DB: dev_epigraphhub - POSTGRES_DB_PRIVATE: dev_privatehub - POSTGRES_DB_SANDBOX: dev_sandbox + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + POSTGRES_EPIGRAPH_USER: dev_epigraph + POSTGRES_EPIGRAPH_PASSWORD: dev_epigraph + POSTGRES_EPIGRAPH_DB: dev_epigraphhub + POSTGRES_EPIGRAPH_DB_PRIVATE: dev_privatehub + POSTGRES_EPIGRAPH_DB_SANDBOX: dev_sandbox REDIS_HOST: epigraphhub-redis REDIS_PORT: 6379 REDIS_PASSWORD: redispass diff --git a/Data_Collection/CRON_scripts/config.py b/Data_Collection/CRON_scripts/config.py index 868adebe..2dada98b 100644 --- a/Data_Collection/CRON_scripts/config.py +++ b/Data_Collection/CRON_scripts/config.py @@ -8,10 +8,10 @@ DB_HOST = os.environ.get("POSTGRES_HOST") DB_PORT = os.environ.get("POSTGRES_PORT") -DB_USER = os.environ.get("POSTGRES_USER") -DB_PASSWORD = os.environ.get("POSTGRES_PASSWORD") -DB_NAME = os.environ.get("POSTGRES_DB") -DB_NAME_PRIVATE = os.environ.get("POSTGRES_DB_PRIVATE") +DB_USER = os.environ.get("POSTGRES_EPIGRAPH_USER") +DB_PASSWORD = os.environ.get("POSTGRES_EPIGRAPH_PASSWORD") +DB_NAME = os.environ.get("POSTGRES_EPIGRAPH_DB") +DB_NAME_PRIVATE = os.environ.get("POSTGRES_EPIGRAPH_DB_PRIVATE") DB_URI = ( f"postgresql://{DB_USER}:{DB_PASSWORD}" diff --git a/Makefile b/Makefile index 3414debe..1d0c16f4 100644 --- a/Makefile +++ b/Makefile @@ -4,12 +4,12 @@ SERVICE:=epigraphhub-superset ENV:=dev CONSOLE:=bash CRON:= +FILE:= DOCKER=docker-compose \ --env-file .env \ --project-name eph-$(ENV) \ - --file docker/compose-base.yaml \ - --file docker/compose-$(ENV).yaml + --file docker/compose.yaml # DOCKER @@ -30,6 +30,11 @@ docker-stop: $(DOCKER) stop ${SERVICES} +.PHONY:docker-down +docker-down: + $(DOCKER) down --volumes + + .PHONY:docker-restart docker-restart: docker-stop docker-start echo "[II] Docker services restarted!" @@ -38,6 +43,10 @@ docker-restart: docker-stop docker-start docker-logs-follow: $(DOCKER) logs --follow --tail 300 ${SERVICES} +.PHONY:docker-logs-follow +docker-logs-follow: + $(DOCKER) logs --follow --tail 100 ${SERVICES} + .PHONY:docker-logs docker-logs: $(DOCKER) logs --tail 300 ${SERVICES} @@ -60,7 +69,7 @@ docker-wait-all: docker-dev-prepare-db: # used for development $(DOCKER) exec -T epigraphhub-superset \ - bash /opt/EpiGraphHub/docker/postgresql/prepare-db.sh + bash /opt/EpiGraphHub/docker/postgresql/dev/prepare-db.sh .PHONY:docker-run-cron @@ -100,6 +109,18 @@ docker-run-bash: $(DOCKER) run --rm ${SERVICE} bash +.PHONY:docker-db-dump +docker-db-dump: + $(DOCKER) exec -T epigraphhub-superset bash \ + /opt/EpiGraphHub/docker/postgresql/dump.sh + + +.PHONY:docker-db-restore +docker-db-restore: + $(DOCKER) exec -T epigraphhub-superset bash \ + /opt/EpiGraphHub/docker/postgresql/restore.sh ${FILE} + + # conda .ONESHELL: diff --git a/docker/compose-dev.yaml b/docker/compose-dev.yaml deleted file mode 100644 index 9308974c..00000000 --- a/docker/compose-dev.yaml +++ /dev/null @@ -1,24 +0,0 @@ -version: '3.4' -services: - - epigraphhub-superset: - depends_on: - - epigraphhub-db - - epigraphhub-celery - - epigraphhub-redis - - epigraphhub-flower - - epigraphhub-db: - environment: - # overwrite .env default variables - - POSTGRES_USER=postgres - - POSTGRES_PASSWORD=postgres - - POSTGRES_DB=postgres - ports: - - ${POSTGRES_PORT}:25432 - healthcheck: - test: ["CMD-SHELL", "pg_isready -p ${POSTGRES_PORT} -U postgres"] - interval: 10s - timeout: 5s - retries: 10 - command: -p 25432 diff --git a/docker/compose-prod.yaml b/docker/compose-prod.yaml deleted file mode 100644 index 1ddfab2b..00000000 --- a/docker/compose-prod.yaml +++ /dev/null @@ -1,6 +0,0 @@ -version: '3.4' -services: - - epigraphhub-superset: - depends_on: - - epigraphhub-celery diff --git a/docker/compose-base.yaml b/docker/compose.yaml similarity index 97% rename from docker/compose-base.yaml rename to docker/compose.yaml index fba63974..6c98d705 100644 --- a/docker/compose-base.yaml +++ b/docker/compose.yaml @@ -38,9 +38,11 @@ services: volumes: - ..:/opt/EpiGraphHub - ${SUPERSET_DB_PATH_DIR_HOST}:/opt/data/superset/ + - ${POSTGRES_DUMP_HOST}:${POSTGRES_DUMP_CONTAINER} ports: - ${SUPERSET_PORT}:8088 depends_on: + - epigraphhub-db - epigraphhub-redis - epigraphhub-celery - epigraphhub-celery-beat @@ -167,3 +169,4 @@ services: volumes: redis: + pgdata: diff --git a/docker/postgresql/dev/prepare-db.sh b/docker/postgresql/dev/prepare-db.sh new file mode 100644 index 00000000..250a4150 --- /dev/null +++ b/docker/postgresql/dev/prepare-db.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +# ================================= +# NOTE: USE IT JUST FOR DEVELOPMENT +# ================================= + +set -ex + +DOCKER_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd ../.. && pwd )" +PSQL_CONNECTION="postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}" + +echo "[II] CREATE DATABASE" +psql "${PSQL_CONNECTION}/postgres" < ${DOCKER_DIR}/sql/dev/database.sql + +echo "[II] LOAD EPIGRAPHHUB DUMP" +psql "${PSQL_CONNECTION}/${POSTGRES_EPIGRAPH_DB}" < ${DOCKER_DIR}/sql/dev/epigraphhub.sql + +echo "[II] LOAD PRIVATE DUMP" +psql "${PSQL_CONNECTION}/${POSTGRES_EPIGRAPH_DB_PRIVATE}" < ${DOCKER_DIR}/sql/dev/privatehub.sql diff --git a/docker/postgresql/dump.sh b/docker/postgresql/dump.sh new file mode 100755 index 00000000..63f5990d --- /dev/null +++ b/docker/postgresql/dump.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +PROJECT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd ../.. && pwd )" + +if [ -f ${PROJECT_DIR}/.env ]; then + # Load Environment Variables + export $(cat ${PROJECT_DIR}/.env | grep -v '#' | sed 's/\r$//' | awk '/=/ {print $1}' ) +fi + +POSTGRES_DUMP_HOST=${POSTGRES_DUMP_HOST:-/tmp/dump} +mkdir -p ${POSTGRES_DUMP_HOST} + +set -ex + +PGPASSWORD=${POSTGRES_PASSWORD} pg_dumpall \ + --host ${POSTGRES_HOST} \ + --port ${POSTGRES_PORT} \ + --user ${POSTGRES_USER} \ + > ${POSTGRES_DUMP_HOST}/$(date --iso-8601).dump diff --git a/docker/postgresql/prepare-db.sh b/docker/postgresql/prepare-db.sh deleted file mode 100644 index 536fc9cd..00000000 --- a/docker/postgresql/prepare-db.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash - -# ================================= -# NOTE: USE IT JUST FOR DEVELOPMENT -# ================================= - -set -ex - -DOCKER_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )" - -echo "[II] CREATE DATABASE" -psql "postgresql://postgres:postgres@$POSTGRES_HOST:$POSTGRES_PORT/postgres" \ - < ${DOCKER_DIR}/sql/database.sql - -echo "[II] LOAD EPIGRAPHHUB DUMP" -psql "postgresql://postgres:postgres@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB" \ - < ${DOCKER_DIR}/sql/epigraphhub.sql - -echo "[II] LOAD PRIVATE DUMP" -psql "postgresql://postgres:postgres@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB_PRIVATE" \ - < ${DOCKER_DIR}/sql/privatehub.sql diff --git a/docker/postgresql/restore.sh b/docker/postgresql/restore.sh new file mode 100755 index 00000000..2668fe5c --- /dev/null +++ b/docker/postgresql/restore.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +PROJECT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd ../.. && pwd )" + +if [ -f ${PROJECT_DIR}/.env ]; then + # Load Environment Variables + export $(cat ${PROJECT_DIR}/.env | grep -v '#' | sed 's/\r$//' | awk '/=/ {print $1}' ) +fi + +POSTGRES_DUMP_HOST=${POSTGRES_DUMP_HOST:-/tmp/dump} + +set -ex + +PGPASSWORD=${POSTGRES_PASSWORD} psql \ + --host ${POSTGRES_HOST} \ + --port ${POSTGRES_PORT} \ + --user ${POSTGRES_USER} \ + -f ${POSTGRES_DUMP_HOST}/${1} ${POSTGRES_DB} diff --git a/docker/sql/database.sql b/docker/sql/dev/database.sql similarity index 100% rename from docker/sql/database.sql rename to docker/sql/dev/database.sql diff --git a/docker/sql/epigraphhub.sql b/docker/sql/dev/epigraphhub.sql similarity index 100% rename from docker/sql/epigraphhub.sql rename to docker/sql/dev/epigraphhub.sql diff --git a/docker/sql/privatehub.sql b/docker/sql/dev/privatehub.sql similarity index 100% rename from docker/sql/privatehub.sql rename to docker/sql/dev/privatehub.sql