From 560b97b6d0af6fd78f5dc26d21e6267abc840bec Mon Sep 17 00:00:00 2001 From: Penelope Yong Date: Tue, 2 Jul 2024 15:09:00 +0800 Subject: [PATCH] Add bash script --- Dockerfile | 20 +++++--------------- deploy.sh | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 15 deletions(-) create mode 100755 deploy.sh diff --git a/Dockerfile b/Dockerfile index d487005..a575a39 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,23 +29,13 @@ COPY ./README.md ./README.md RUN python -m pip install --upgrade pip setuptools \ && python -m pip install . -# Set env variables -ENV IGNORE_EXPERIMENTAL_WARNINGS=1 -RUN mkdir persist -ENV DAGSTER_HOME=/popgetter-prod/persist -ENV DAGSTER_MODULE_NAME=popgetter -ENV ENV=prod -ENV AZURE_STORAGE_ACCOUNT=popgetter -ENV AZURE_CONTAINER=prod -# This doesn't work -# ENV AZURE_DIRECTORY=$(python -c 'import popgetter; print(popgetter.__version__)' 2>/dev/null) -ENV AZURE_DIRECTORY=0.1.0 -ENV POPGETTER_COUNTRIES=bel,gb_nir - # TODO: Not the safest! ARG SAS_TOKEN RUN [ -z "$SAS_TOKEN" ] && echo "SAS_TOKEN build arg is required" && exit 1 || true ENV SAS_TOKEN=$SAS_TOKEN -ENTRYPOINT ["/bin/bash"] -CMD ["-c", "python -m popgetter.run all"] +ARG POPGETTER_COUNTRIES +ENV POPGETTER_COUNTRIES=$POPGETTER_COUNTRIES + +COPY ./deploy.sh ./deploy.sh +ENTRYPOINT ["./deploy.sh"] diff --git a/deploy.sh b/deploy.sh new file mode 100755 index 0000000..6ae543b --- /dev/null +++ b/deploy.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -e + +if [ -z "$ENV" ]; then + echo "ENV environment variable not set; must be either 'dev' or 'prod'" + exit 1 +fi + +if [ -z "$POPGETTER_COUNTRIES" ]; then + echo "POPGETTER_COUNTRIES environment variable not set; must be comma-separated list of country IDs" + exit 1 +fi + +export IGNORE_EXPERIMENTAL_WARNINGS=1 +export DAGSTER_MODULE_NAME=popgetter +export DAGSTER_HOME=$(mktemp -d) +touch $DAGSTER_HOME/dagster.yaml # Silences Dagster warnings + +echo "Relevant environment variables:" +echo " - POPGETTER_COUNTRIES: $POPGETTER_COUNTRIES" +echo " - ENV: $ENV" +if [ $ENV == "prod" ]; then + export AZURE_STORAGE_ACCOUNT=popgetter + export AZURE_CONTAINER=prod + export AZURE_DIRECTORY=$(python -c 'import popgetter; print(popgetter.__version__)' 2>/dev/null) + if [ -z "$SAS_TOKEN" ]; then + echo "SAS_TOKEN environment variable not set; it is required for Azure deployments" + exit 1 + else + echo " - SAS_TOKEN: (exists)" + fi + echo " - AZURE_STORAGE_ACCOUNT: $AZURE_STORAGE_ACCOUNT" + echo " - AZURE_CONTAINER: $AZURE_CONTAINER" + echo " - AZURE_DIRECTORY: $AZURE_DIRECTORY" +fi + +echo "Generating popgetter data. This may take a while." +python -m popgetter.run all