From 77bd4e3b5c57ee8335f6fe0088a253bbd81cd06e Mon Sep 17 00:00:00 2001 From: Michal Kleszcz Date: Mon, 16 Sep 2019 17:27:38 +0200 Subject: [PATCH 1/4] Sync backend with boilerplate --- .env.example | 7 - .gitignore | 3 +- .travis.yml | 7 +- Dockerfile | 10 +- Pipfile | 3 + Pipfile.lock | 111 ++++++++++++- docker-compose-ci.yml | 35 ++++ docker-compose.yml | 26 --- restauth/settings.py | 22 ++- scripts/dev/install_localstack_fixtures.sh | 18 +++ scripts/dev/run-backend.sh | 21 +++ scripts/dev/run_tests.sh | 26 +++ scripts/dev/wait_for_postgres.py | 48 ++++++ scripts/run-backend.sh | 6 + test.env | 19 +++ uwsgi.ini | 4 + wait-for-it.sh | 177 --------------------- 17 files changed, 318 insertions(+), 225 deletions(-) delete mode 100644 .env.example create mode 100644 docker-compose-ci.yml delete mode 100644 docker-compose.yml mode change 100644 => 100755 restauth/settings.py create mode 100755 scripts/dev/install_localstack_fixtures.sh create mode 100755 scripts/dev/run-backend.sh create mode 100755 scripts/dev/run_tests.sh create mode 100644 scripts/dev/wait_for_postgres.py create mode 100755 scripts/run-backend.sh create mode 100644 test.env create mode 100644 uwsgi.ini delete mode 100755 wait-for-it.sh diff --git a/.env.example b/.env.example deleted file mode 100644 index d3abe65..0000000 --- a/.env.example +++ /dev/null @@ -1,7 +0,0 @@ -DJANGO_SETTINGS_MODULE=restauth.settings - -DEBUG=on -ALLOWED_HOSTS=* -SECRET_KEY=t6xm-!%va(qgw@)*4+sbnzyhk_&gc#s1r6!qy*=a9%u06rumy& - -DATABASE_URL=psql://postgres:postgres@db/postgres diff --git a/.gitignore b/.gitignore index ad53d06..5f2123d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,5 +2,4 @@ __pycache__/ *.py[cod] .idea/ db.sqlite3 -.pytest_cache/ -.env \ No newline at end of file +.pytest_cache/ \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index c572eb7..fc3d74e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,7 @@ language: python python: - "3.6" -install: - - pip install pipenv - - pipenv install --dev +services: + - docker script: - - pipenv run pytest \ No newline at end of file + - BACKEND_IMAGE=backend docker-compose -f docker-compose-ci.yml run backend /app/scripts/dev/run_tests.sh \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 523a696..b47342b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,12 +9,14 @@ RUN apk update \ && apk add postgresql-dev \ && apk add build-base linux-headers pcre-dev +RUN pip install awscli + EXPOSE 3031 -WORKDIR /code -COPY Pipfile Pipfile.lock ./ +WORKDIR /app +COPY Pipfile Pipfile.lock /app/ RUN pip install --upgrade pip RUN pip install pipenv RUN pipenv install --system --dev -COPY . . +COPY . /app -CMD ["./run-backend.sh"] +CMD ["/app/scripts/run-backend.sh"] diff --git a/Pipfile b/Pipfile index d7ae4c0..226be8e 100644 --- a/Pipfile +++ b/Pipfile @@ -9,6 +9,9 @@ djangorestframework = "~=3.10" djangorestframework-jwt = "~=1.11" django-hashid-field = "~=2.1" dj-database-url = "*" +uwsgi = "*" +boto3 = "==1.9.93" +"psycopg2-binary" = "*" [dev-packages] factory_boy = "==2.10.0" diff --git a/Pipfile.lock b/Pipfile.lock index a876ad6..73845de 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "ee2d14bfc26dd04ca3fa93fd8406d00a6c4cc4827a670ed223c575121fb3d406" + "sha256": "fc6c62cb0f15629ad6503e135a9e4b99cc15a6cbd8312e2b279ba4b2df3b4fa7" }, "pipfile-spec": 6, "requires": { @@ -16,6 +16,21 @@ ] }, "default": { + "boto3": { + "hashes": [ + "sha256:0bed0db8c10b88b3daa042adaa1fb6c3262caed39d28086e8548015405c71744", + "sha256:70e71e0192a68f65754ab9d2a335be3c6856a1e8a15f3bd6263ea12e2f442bc7" + ], + "index": "pypi", + "version": "==1.9.93" + }, + "botocore": { + "hashes": [ + "sha256:7c391c46cf1f8d6c04758f84b51337a64136077e4a160eced87551fdcc051669", + "sha256:c9148df92ba21a90ea32f2c7f185c31b1c8b8e48417d0ba8cad02b9b3336c09a" + ], + "version": "==1.12.228" + }, "dj-database-url": { "hashes": [ "sha256:4aeaeb1f573c74835b0686a2b46b85990571159ffc21aa57ecd4d1e1cb334163", @@ -56,12 +71,61 @@ "index": "pypi", "version": "==1.11.0" }, + "docutils": { + "hashes": [ + "sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0", + "sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827", + "sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99" + ], + "version": "==0.15.2" + }, "hashids": { "hashes": [ "sha256:6539b892a426e75747a9c0ad69409e9566f9c21b79310fc3424b5b6726f28da6" ], "version": "==1.2.0" }, + "jmespath": { + "hashes": [ + "sha256:3720a4b1bd659dd2eecad0666459b9788813e032b83e7ba58578e48254e0a0e6", + "sha256:bde2aef6f44302dfb30320115b17d030798de8c4110e28d5cf6cf91a7a31074c" + ], + "version": "==0.9.4" + }, + "psycopg2-binary": { + "hashes": [ + "sha256:080c72714784989474f97be9ab0ddf7b2ad2984527e77f2909fcd04d4df53809", + "sha256:110457be80b63ff4915febb06faa7be002b93a76e5ba19bf3f27636a2ef58598", + "sha256:171352a03b22fc099f15103959b52ee77d9a27e028895d7e5fde127aa8e3bac5", + "sha256:19d013e7b0817087517a4b3cab39c084d78898369e5c46258aab7be4f233d6a1", + "sha256:249b6b21ae4eb0f7b8423b330aa80fab5f821b9ffc3f7561a5e2fd6bb142cf5d", + "sha256:2ac0731d2d84b05c7bb39e85b7e123c3a0acd4cda631d8d542802c88deb9e87e", + "sha256:2b6d561193f0dc3f50acfb22dd52ea8c8dfbc64bcafe3938b5f209cc17cb6f00", + "sha256:2bd23e242e954214944481124755cbefe7c2cf563b1a54cd8d196d502f2578bf", + "sha256:3e1239242ca60b3725e65ab2f13765fc199b03af9eaf1b5572f0e97bdcee5b43", + "sha256:3eb70bb697abbe86b1d2b1316370c02ba320bfd1e9e35cf3b9566a855ea8e4e5", + "sha256:51a2fc7e94b98bd1bb5d4570936f24fc2b0541b63eccadf8fdea266db8ad2f70", + "sha256:52f1bdafdc764b7447e393ed39bb263eccb12bfda25a4ac06d82e3a9056251f6", + "sha256:5b3581319a3951f1e866f4f6c5e42023db0fae0284273b82e97dfd32c51985cd", + "sha256:63c1b66e3b2a3a336288e4bcec499e0dc310cd1dceaed1c46fa7419764c68877", + "sha256:8123a99f24ecee469e5c1339427bcdb2a33920a18bb5c0d58b7c13f3b0298ba3", + "sha256:85e699fcabe7f817c0f0a412d4e7c6627e00c412b418da7666ff353f38e30f67", + "sha256:8dbff4557bbef963697583366400822387cccf794ccb001f1f2307ed21854c68", + "sha256:908d21d08d6b81f1b7e056bbf40b2f77f8c499ab29e64ec5113052819ef1c89b", + "sha256:af39d0237b17d0a5a5f638e9dffb34013ce2b1d41441fd30283e42b22d16858a", + "sha256:af51bb9f055a3f4af0187149a8f60c9d516cf7d5565b3dac53358796a8fb2a5b", + "sha256:b2ecac57eb49e461e86c092761e6b8e1fd9654dbaaddf71a076dcc869f7014e2", + "sha256:cd37cc170678a4609becb26b53a2bc1edea65177be70c48dd7b39a1149cabd6e", + "sha256:d17e3054b17e1a6cb8c1140f76310f6ede811e75b7a9d461922d2c72973f583e", + "sha256:d305313c5a9695f40c46294d4315ed3a07c7d2b55e48a9010dad7db7a66c8b7f", + "sha256:dd0ef0eb1f7dd18a3f4187226e226a7284bda6af5671937a221766e6ef1ee88f", + "sha256:e1adff53b56db9905db48a972fb89370ad5736e0450b96f91bcf99cadd96cfd7", + "sha256:f0d43828003c82dbc9269de87aa449e9896077a71954fbbb10a614c017e65737", + "sha256:f78e8b487de4d92640105c1389e5b90be3496b1d75c90a666edd8737cc2dbab7" + ], + "index": "pypi", + "version": "==2.8.3" + }, "pyjwt": { "hashes": [ "sha256:5c6eca3c2940464d106b99ba83b00c6add741c9becaec087fb7ccdefea71350e", @@ -69,6 +133,14 @@ ], "version": "==1.7.1" }, + "python-dateutil": { + "hashes": [ + "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", + "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e" + ], + "markers": "python_version >= '2.7'", + "version": "==2.8.0" + }, "pytz": { "hashes": [ "sha256:26c0b32e437e54a18161324a2fca3c4b9846b74a8dccddd843113109e1116b32", @@ -76,12 +148,41 @@ ], "version": "==2019.2" }, + "s3transfer": { + "hashes": [ + "sha256:6efc926738a3cd576c2a79725fed9afde92378aa5c6a957e3af010cb019fac9d", + "sha256:b780f2411b824cb541dbcd2c713d0cb61c7d1bcadae204cdddda2b35cef493ba" + ], + "version": "==0.2.1" + }, + "six": { + "hashes": [ + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + ], + "version": "==1.12.0" + }, "sqlparse": { "hashes": [ "sha256:40afe6b8d4b1117e7dff5504d7a8ce07d9a1b15aeeade8a2d10f130a834f8177", "sha256:7c3dca29c022744e95b547e867cee89f4fce4373f3549ccd8797d8eb52cdb873" ], "version": "==0.3.0" + }, + "urllib3": { + "hashes": [ + "sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1", + "sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232" + ], + "markers": "python_version >= '3.4'", + "version": "==1.25.3" + }, + "uwsgi": { + "hashes": [ + "sha256:4972ac538800fb2d421027f49b4a1869b66048839507ccf0aa2fda792d99f583" + ], + "index": "pypi", + "version": "==2.0.18" } }, "develop": { @@ -94,10 +195,10 @@ }, "certifi": { "hashes": [ - "sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939", - "sha256:945e3ba63a0b9f577b1395204e13c3a231f9bc0223888be653286534e5873695" + "sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50", + "sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef" ], - "version": "==2019.6.16" + "version": "==2019.9.11" }, "chardet": { "hashes": [ @@ -259,6 +360,7 @@ "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e" ], + "markers": "python_version >= '2.7'", "version": "==2.8.0" }, "requests": { @@ -312,6 +414,7 @@ "sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1", "sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232" ], + "markers": "python_version >= '3.4'", "version": "==1.25.3" } } diff --git a/docker-compose-ci.yml b/docker-compose-ci.yml new file mode 100644 index 0000000..f673cca --- /dev/null +++ b/docker-compose-ci.yml @@ -0,0 +1,35 @@ +version: '3' + +services: + db: + image: postgres:9.6.8 + ports: + - "5432:5432" + env_file: test.env + + localstack: + image: localstack/localstack:0.10.2 + ports: + - "4567-4584:4567-4584" + environment: + - SERVICES=secretsmanager + - DEBUG=${DEBUG- } + - DATA_DIR=${DATA_DIR- } + - PORT_WEB_UI=${PORT_WEB_UI- } + - LAMBDA_EXECUTOR=${LAMBDA_EXECUTOR- } + - LAMBDA_REMOTE_DOCKER=false + - KINESIS_ERROR_PROBABILITY=${KINESIS_ERROR_PROBABILITY- } + - DOCKER_HOST=unix:///var/run/docker.sock + - AWS_EXECUTION_ENV=True + - AWS_DEFAULT_REGION=us-east-1 + - AWS_ACCESS_KEY_ID=foo + - AWS_SECRET_ACCESS_KEY=bar + + backend: + image: ${BACKEND_IMAGE} + depends_on: + - db + - localstack + restart: on-failure + env_file: test.env + command: ./scripts/dev/run_tests.sh diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 08d7579..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,26 +0,0 @@ -version: '3' - -services: - db: - image: postgres:9.6 - ports: - - "5432:5432" - volumes: - - restauth_backend:/var/lib/postgresql/data - - dev: - build: . - image: restauth_backend - command: "./wait-for-it.sh db:5432 -- python manage.py runserver 0:8000" - volumes: - - .:/code - ports: - - "8000:8000" - depends_on: - - "db" - restart: on-failure - env_file: .env - -volumes: - restauth_backend: - external: true diff --git a/restauth/settings.py b/restauth/settings.py old mode 100644 new mode 100755 index 6cc0b2f..0a3bb5f --- a/restauth/settings.py +++ b/restauth/settings.py @@ -1,9 +1,19 @@ import os +import json import dj_database_url +import boto3 # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +secrets_manager = boto3.client( + 'secretsmanager', endpoint_url=os.environ.get('SECRET_MANAGER_ENDPOINT_URL', None)) + +db_secret_arn = os.environ['DB_SECRET_ARN'] + +db_secret_value = secrets_manager.get_secret_value(SecretId=db_secret_arn) +# contains host, username, password and port +db_connection_config = json.loads(db_secret_value.get('SecretString')) # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/ @@ -64,7 +74,17 @@ # https://docs.djangoproject.com/en/1.11/ref/settings/#databases DATABASES = { - 'default': dj_database_url.parse('sqlite:///db.sqlite3'), + "default": { + "ENGINE": "django.db.backends.postgresql", + "NAME": os.getenv("POSTGRES_DB"), + "USER": db_connection_config.get('username'), + "PASSWORD": db_connection_config.get('password'), + "HOST": db_connection_config.get('host'), + # Persistent connections avoid the overhead of re-establishing a connection + # to the database in each request + "CONN_MAX_AGE": int(os.getenv("POSTGRES_CONN_MAX_AGE", '60')), + "PORT": db_connection_config.get('port'), + } } diff --git a/scripts/dev/install_localstack_fixtures.sh b/scripts/dev/install_localstack_fixtures.sh new file mode 100755 index 0000000..0066994 --- /dev/null +++ b/scripts/dev/install_localstack_fixtures.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -e + +function wait_for_secretsmanager { + until aws --no-sign-request --endpoint-url="$SECRET_MANAGER_ENDPOINT_URL" secretsmanager list-secrets; do + >&2 echo "Secretsmanager is unavailable - sleeping" + sleep 1 + done +} + +function install_db_secret { + SECRET_STRING="{\"host\": \"db\", \"username\": \"$POSTGRES_USER\", \"password\": \"$POSTGRES_PASSWORD\", \"port\": $POSTGRES_PORT}" + + aws --endpoint-url="$SECRET_MANAGER_ENDPOINT_URL" secretsmanager create-secret \ + --name "$DB_SECRET_ARN" \ + --secret-string "$SECRET_STRING" +} diff --git a/scripts/dev/run-backend.sh b/scripts/dev/run-backend.sh new file mode 100755 index 0000000..94da8d7 --- /dev/null +++ b/scripts/dev/run-backend.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -e + +. $(dirname "$0")/install_localstack_fixtures.sh + +# wait untill secretsmanager become ready +wait_for_secretsmanager +echo "Secrets manager is up" + +# install all localstack fixtures +{ + install_db_secret && + echo "DB secrets set" +} || { + echo "DB secrets NOT set" +} + +python ./scripts/dev/wait_for_postgres.py && + ./manage.py migrate && + ./manage.py runserver 0.0.0.0:8000 diff --git a/scripts/dev/run_tests.sh b/scripts/dev/run_tests.sh new file mode 100755 index 0000000..7b9a810 --- /dev/null +++ b/scripts/dev/run_tests.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +set -e + +. $(dirname "$0")/install_localstack_fixtures.sh + +# wait untill secretsmanager become ready +wait_for_secretsmanager +echo "Secrets manager is up" + +# install all localstack fixtures +{ + install_db_secret && + echo "DB secrets set" +} || { + echo "DB secrets NOT set" +} + +echo "Waiting for DB" +python ./scripts/dev/wait_for_postgres.py + +echo "Run migrations" +/app/manage.py makemigrations --dry-run --check || { echo "ERROR: there were changes in the models, but migration listed above have not been created and are not saved in version control"; exit 1; } + +echo "Run pytest" +pytest --maxfail=1 --junitxml=/test-results/report.xml \ No newline at end of file diff --git a/scripts/dev/wait_for_postgres.py b/scripts/dev/wait_for_postgres.py new file mode 100644 index 0000000..9663f56 --- /dev/null +++ b/scripts/dev/wait_for_postgres.py @@ -0,0 +1,48 @@ +import os +import logging +from time import time, sleep +import psycopg2 +import boto3 +import json + +secrets_manager = boto3.client( + 'secretsmanager', endpoint_url=os.environ.get('SECRET_MANAGER_ENDPOINT_URL', None)) + +db_secret_arn = os.environ['DB_SECRET_ARN'] + +db_secret_value = secrets_manager.get_secret_value(SecretId=db_secret_arn) +# contains host, username, password and port +db_connection_config = json.loads(db_secret_value.get('SecretString')) + +check_timeout = os.getenv("POSTGRES_CHECK_TIMEOUT", 30) +check_interval = os.getenv("POSTGRES_CHECK_INTERVAL", 1) +interval_unit = "second" if check_interval == 1 else "seconds" +config = { + "dbname": os.getenv("POSTGRES_DB", "postgres"), + "user": db_connection_config.get('username'), + "password": db_connection_config.get('password'), + "host": db_connection_config.get('host'), +} + +start_time = time() +logger = logging.getLogger() +logger.setLevel(logging.INFO) +logger.addHandler(logging.StreamHandler()) + + +def pg_isready(host, user, password, dbname): + while time() - start_time < check_timeout: + try: + conn = psycopg2.connect(**vars()) + logger.info("Postgres is ready! ✨ 💅") + conn.close() + return True + except psycopg2.OperationalError: + logger.info(f"Postgres isn't ready. Waiting for {check_interval} {interval_unit}...") + sleep(check_interval) + + logger.error(f"We could not connect to Postgres within {check_timeout} seconds.") + return False + + +pg_isready(**config) diff --git a/scripts/run-backend.sh b/scripts/run-backend.sh new file mode 100755 index 0000000..c65f633 --- /dev/null +++ b/scripts/run-backend.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -e + +python manage.py migrate --no-input +uwsgi --ini uwsgi.ini diff --git a/test.env b/test.env new file mode 100644 index 0000000..1694c41 --- /dev/null +++ b/test.env @@ -0,0 +1,19 @@ +DEBUG=on + +POSTGRES_PASSWORD=test-app-secret +POSTGRES_DB=test-app +POSTGRES_USER=test-user +POSTGRES_HOST=db +POSTGRES_PORT=5432 +POSTGRES_CONN_MAX_AGE=60 + +DB_SECRET_ARN=local-db-secret + +AWS_DEFAULT_REGION=us-east-1 +AWS_ACCESS_KEY_ID=foo +AWS_SECRET_ACCESS_KEY=bar + +SECRET_MANAGER_ENDPOINT_URL=http://localstack:4584 + +# Sentry +SENTRY_DNS= diff --git a/uwsgi.ini b/uwsgi.ini new file mode 100644 index 0000000..2dae235 --- /dev/null +++ b/uwsgi.ini @@ -0,0 +1,4 @@ +[uwsgi] +http-socket = :3031 +protocol = uwsgi +module = restauth.wsgi:application diff --git a/wait-for-it.sh b/wait-for-it.sh deleted file mode 100755 index 5b3a6f9..0000000 --- a/wait-for-it.sh +++ /dev/null @@ -1,177 +0,0 @@ -#!/usr/bin/env bash -# Use this script to test if a given TCP host/port are available - -cmdname=$(basename $0) - -echoerr() { if [[ $QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } - -usage() -{ - cat << USAGE >&2 -Usage: - $cmdname host:port [-s] [-t timeout] [-- command args] - -h HOST | --host=HOST Host or IP under test - -p PORT | --port=PORT TCP port under test - Alternatively, you specify the host and port as host:port - -s | --strict Only execute subcommand if the test succeeds - -q | --quiet Don't output any status messages - -t TIMEOUT | --timeout=TIMEOUT - Timeout in seconds, zero for no timeout - -- COMMAND ARGS Execute command with args after the test finishes -USAGE - exit 1 -} - -wait_for() -{ - if [[ $TIMEOUT -gt 0 ]]; then - echoerr "$cmdname: waiting $TIMEOUT seconds for $HOST:$PORT" - else - echoerr "$cmdname: waiting for $HOST:$PORT without a timeout" - fi - start_ts=$(date +%s) - while : - do - if [[ $ISBUSY -eq 1 ]]; then - nc -z $HOST $PORT - result=$? - else - (echo > /dev/tcp/$HOST/$PORT) >/dev/null 2>&1 - result=$? - fi - if [[ $result -eq 0 ]]; then - end_ts=$(date +%s) - echoerr "$cmdname: $HOST:$PORT is available after $((end_ts - start_ts)) seconds" - break - fi - sleep 1 - done - return $result -} - -wait_for_wrapper() -{ - # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 - if [[ $QUIET -eq 1 ]]; then - timeout $BUSYTIMEFLAG $TIMEOUT $0 --quiet --child --host=$HOST --port=$PORT --timeout=$TIMEOUT & - else - timeout $BUSYTIMEFLAG $TIMEOUT $0 --child --host=$HOST --port=$PORT --timeout=$TIMEOUT & - fi - PID=$! - trap "kill -INT -$PID" INT - wait $PID - RESULT=$? - if [[ $RESULT -ne 0 ]]; then - echoerr "$cmdname: timeout occurred after waiting $TIMEOUT seconds for $HOST:$PORT" - fi - return $RESULT -} - -# process arguments -while [[ $# -gt 0 ]] -do - case "$1" in - *:* ) - hostport=(${1//:/ }) - HOST=${hostport[0]} - PORT=${hostport[1]} - shift 1 - ;; - --child) - CHILD=1 - shift 1 - ;; - -q | --quiet) - QUIET=1 - shift 1 - ;; - -s | --strict) - STRICT=1 - shift 1 - ;; - -h) - HOST="$2" - if [[ $HOST == "" ]]; then break; fi - shift 2 - ;; - --host=*) - HOST="${1#*=}" - shift 1 - ;; - -p) - PORT="$2" - if [[ $PORT == "" ]]; then break; fi - shift 2 - ;; - --port=*) - PORT="${1#*=}" - shift 1 - ;; - -t) - TIMEOUT="$2" - if [[ $TIMEOUT == "" ]]; then break; fi - shift 2 - ;; - --timeout=*) - TIMEOUT="${1#*=}" - shift 1 - ;; - --) - shift - CLI=("$@") - break - ;; - --help) - usage - ;; - *) - echoerr "Unknown argument: $1" - usage - ;; - esac -done - -if [[ "$HOST" == "" || "$PORT" == "" ]]; then - echoerr "Error: you need to provide a host and port to test." - usage -fi - -TIMEOUT=${TIMEOUT:-15} -STRICT=${STRICT:-0} -CHILD=${CHILD:-0} -QUIET=${QUIET:-0} - -# check to see if timeout is from busybox? -# check to see if timeout is from busybox? -TIMEOUT_PATH=$(realpath $(which timeout)) -if [[ $TIMEOUT_PATH =~ "busybox" ]]; then - ISBUSY=1 - BUSYTIMEFLAG="-t" -else - ISBUSY=0 - BUSYTIMEFLAG="" -fi - -if [[ $CHILD -gt 0 ]]; then - wait_for - RESULT=$? - exit $RESULT -else - if [[ $TIMEOUT -gt 0 ]]; then - wait_for_wrapper - RESULT=$? - else - wait_for - RESULT=$? - fi -fi - -if [[ $CLI != "" ]]; then - if [[ $RESULT -ne 0 && $STRICT -eq 1 ]]; then - echoerr "$cmdname: strict mode, refusing to execute subprocess" - exit $RESULT - fi - exec "${CLI[@]}" -else - exit $RESULT -fi \ No newline at end of file From a4f85cfda7bbf31b5eb27230bbd5ed3b5f9ad864 Mon Sep 17 00:00:00 2001 From: Michal Kleszcz Date: Tue, 17 Sep 2019 12:04:16 +0200 Subject: [PATCH 2/4] Fix display BE errors --- Dockerfile | 3 ++- scripts/dev/run-backend.sh | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b47342b..9cf79c1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,8 @@ RUN apk update \ # psycopg2 dependencies && apk add --virtual build-deps gcc python3-dev musl-dev \ && apk add postgresql-dev \ - && apk add build-base linux-headers pcre-dev + && apk add build-base linux-headers pcre-dev \ + && apk add gettext RUN pip install awscli diff --git a/scripts/dev/run-backend.sh b/scripts/dev/run-backend.sh index 94da8d7..965ec34 100755 --- a/scripts/dev/run-backend.sh +++ b/scripts/dev/run-backend.sh @@ -17,5 +17,6 @@ echo "Secrets manager is up" } python ./scripts/dev/wait_for_postgres.py && + ./manage.py compilemessages && ./manage.py migrate && ./manage.py runserver 0.0.0.0:8000 From 87e26e68615a407e1b339a17464e06e0809c3608 Mon Sep 17 00:00:00 2001 From: Michal Kleszcz Date: Wed, 18 Sep 2019 12:56:42 +0200 Subject: [PATCH 3/4] Fix invalid django.mo file --- locale/error_codes/LC_MESSAGES/django.mo | Bin 441 -> 527 bytes 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 locale/error_codes/LC_MESSAGES/django.mo diff --git a/locale/error_codes/LC_MESSAGES/django.mo b/locale/error_codes/LC_MESSAGES/django.mo old mode 100644 new mode 100755 index 9f3d31eb34fdaa5b285751b3626918f600c51b05..7728399bcb400cebf7b036c4cf6ef0301a9d70fc GIT binary patch delta 167 zcmdnV+|LqvPl#nI0}wC+u?!HK05K~N`v5TrBml8E5GMk$HV{_=u>&Im!%`s42gJ{S zY)K&f6-WaS0}~K4Pjpq$3e8JQ%1Koy$ydn9PglsyQz*|Y$xtXL$}h`INlj5mE=o;F h%`3@F%qiZO9>ge>T2z!@6rY(_mY9>75|1Ry004ZtB>n&Z delta 81 zcmeBY*~#pBPl#nI0}wC)u?!IF05K~N+W;{L1OTx(5C;OW10w@NCXnU>;#EL45P|hE MObk`oxHNzf09v~UcmMzZ From 84e9987041b3acbe045209a87203c2990e10a119 Mon Sep 17 00:00:00 2001 From: Michal Kleszcz Date: Wed, 18 Sep 2019 12:57:32 +0200 Subject: [PATCH 4/4] Add migrations checks, separate migration run script --- .travis.yml | 1 + docker-compose-ci.yml | 2 +- restauth/settings.py | 4 ++-- restauth/urls.py | 5 ++++- restauth/views.py | 29 ++++++++++++++++++++++++++++- scripts/dev/run_tests.sh | 2 +- scripts/run-backend.sh | 1 - scripts/run-migrations.sh | 5 +++++ test.env | 4 ++++ 9 files changed, 46 insertions(+), 7 deletions(-) mode change 100644 => 100755 restauth/urls.py mode change 100644 => 100755 restauth/views.py create mode 100755 scripts/run-migrations.sh diff --git a/.travis.yml b/.travis.yml index fc3d74e..6e957b2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,4 +4,5 @@ python: services: - docker script: + - docker build -t backend . - BACKEND_IMAGE=backend docker-compose -f docker-compose-ci.yml run backend /app/scripts/dev/run_tests.sh \ No newline at end of file diff --git a/docker-compose-ci.yml b/docker-compose-ci.yml index f673cca..375a2a1 100644 --- a/docker-compose-ci.yml +++ b/docker-compose-ci.yml @@ -4,7 +4,7 @@ services: db: image: postgres:9.6.8 ports: - - "5432:5432" + - "5432" env_file: test.env localstack: diff --git a/restauth/settings.py b/restauth/settings.py index 0a3bb5f..ecbb741 100755 --- a/restauth/settings.py +++ b/restauth/settings.py @@ -18,9 +18,9 @@ # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/ -SECRET_KEY = os.environ.get('SECRET_KEY') +SECRET_KEY = os.environ.get('DJANGO_SECRET_KEY') DEBUG = os.environ.get('DEBUG') -ALLOWED_HOSTS = list(filter(lambda s: len(s) > 0, map(str.strip, os.environ.get('ALLOWED_HOSTS', '').split(',')))) +ALLOWED_HOSTS = list(filter(lambda s: len(s) > 0, map(str.strip, os.environ.get('ALLOWED_HOSTS', '*').split(',')))) # Application definition diff --git a/restauth/urls.py b/restauth/urls.py old mode 100644 new mode 100755 index 0c23fff..7fcac72 --- a/restauth/urls.py +++ b/restauth/urls.py @@ -25,5 +25,8 @@ url(r'^doc/', get_swagger_view(title='Documentation')), path('auth/', include(user_patterns)), - path('password-reset/', include(password_reset_patterns)) + path('password-reset/', include(password_reset_patterns)), + + path('', views.HomeView.as_view(), name='home'), + path('status/elb', views.HealthCheckView.as_view(), name='elb_status'), ] diff --git a/restauth/views.py b/restauth/views.py old mode 100644 new mode 100755 index dbbab8c..c56d959 --- a/restauth/views.py +++ b/restauth/views.py @@ -1,6 +1,11 @@ from rest_framework import generics from rest_framework import permissions +from rest_framework import views +from rest_framework import response +from rest_framework import status from rest_framework.throttling import AnonRateThrottle +from django.db import DEFAULT_DB_ALIAS, connections +from django.db.migrations.executor import MigrationExecutor from . import serializers @@ -51,4 +56,26 @@ class PasswordResetConfirmationView(generics.CreateAPIView): Set new password, it requires to provide the new password to set. """ permission_classes = (permissions.AllowAny,) - serializer_class = serializers.PasswordResetConfirmationSerializer \ No newline at end of file + serializer_class = serializers.PasswordResetConfirmationSerializer + + +class HomeView(views.APIView): + permission_classes = [] + + def get(self, request): + return response.Response(status=status.HTTP_200_OK) + + +class HealthCheckView(views.APIView): + authentication_classes = () + permission_classes = (permissions.AllowAny,) + + @staticmethod + def get(request): + executor = MigrationExecutor(connections[DEFAULT_DB_ALIAS]) + plan = executor.migration_plan(executor.loader.graph.leaf_nodes()) + + if plan: + return response.Response(status=status.HTTP_503_SERVICE_UNAVAILABLE) + + return response.Response(status=status.HTTP_200_OK) diff --git a/scripts/dev/run_tests.sh b/scripts/dev/run_tests.sh index 7b9a810..07ef4dd 100755 --- a/scripts/dev/run_tests.sh +++ b/scripts/dev/run_tests.sh @@ -20,7 +20,7 @@ echo "Waiting for DB" python ./scripts/dev/wait_for_postgres.py echo "Run migrations" -/app/manage.py makemigrations --dry-run --check || { echo "ERROR: there were changes in the models, but migration listed above have not been created and are not saved in version control"; exit 1; } +python /app/manage.py makemigrations --dry-run --check || { echo "ERROR: there were changes in the models, but migration listed above have not been created and are not saved in version control"; exit 1; } echo "Run pytest" pytest --maxfail=1 --junitxml=/test-results/report.xml \ No newline at end of file diff --git a/scripts/run-backend.sh b/scripts/run-backend.sh index c65f633..187b797 100755 --- a/scripts/run-backend.sh +++ b/scripts/run-backend.sh @@ -2,5 +2,4 @@ set -e -python manage.py migrate --no-input uwsgi --ini uwsgi.ini diff --git a/scripts/run-migrations.sh b/scripts/run-migrations.sh new file mode 100755 index 0000000..a0c1ffd --- /dev/null +++ b/scripts/run-migrations.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -e + +python manage.py migrate --no-input diff --git a/test.env b/test.env index 1694c41..963434e 100644 --- a/test.env +++ b/test.env @@ -1,5 +1,9 @@ DEBUG=on +DJANGO_SECRET_KEY=secret-key + +ALLOWED_HOSTS=* + POSTGRES_PASSWORD=test-app-secret POSTGRES_DB=test-app POSTGRES_USER=test-user