chore: bring staging changes to production

.travis.yml

@@ -1,5 +1,5 @@
-sudo: required
-language: minimal
+os: linux
+language: shell
 
 git:
   depth: 2
@@ -10,32 +10,37 @@ branches:
   - devel
 
 services:
-  - docker
+- docker
 
 env:
   global:
-    - IMAGE_REPO=gcr.io/dd-decaf-cfbf6/iam
-    - IMAGE_TAG=travis-ci-test
-    - IMAGE=${IMAGE_REPO}:${IMAGE_TAG}
+  - IMAGE=gcr.io/dd-decaf-cfbf6/iam
+  - BRANCH=${TRAVIS_BRANCH}
+  - BUILD_COMMIT=${TRAVIS_COMMIT}
+  - SHORT_COMMIT=${TRAVIS_COMMIT:0:7}
+  - BUILD_DATE=$(date -u +%Y-%m-%d)
+  - BUILD_TAG=${BRANCH}_${BUILD_DATE}_${SHORT_COMMIT}
+
+before_install:
+- make setup
 
 install:
-  - docker build -t ${IMAGE} .
-  - make setup
+- make build
+- make build-travis
+- make post-build
+- make start
 
 script:
-  - make flake8
-  - make isort
-  - make license
-  - make safety
-  - make test-travis
+- make style
+- make safety
+# Run the tests and report coverage (see https://docs.codecov.io/docs/testing-with-docker).
+- docker-compose exec -e ENVIRONMENT=testing web pytest --cov=iam --cov-report=term --cov-report=xml
+- bash <(curl -s https://codecov.io/bash)
 
 before_deploy:
-  - ./scripts/install_gcloud.sh
-  - ./scripts/install_kubectl.sh
-  - docker tag ${IMAGE} ${IMAGE_REPO}:${TRAVIS_COMMIT::12}
-  - docker tag ${IMAGE} ${IMAGE_REPO}:${TRAVIS_BRANCH}
-  - docker push ${IMAGE_REPO}:${TRAVIS_COMMIT::12}
-  - docker push ${IMAGE_REPO}:${TRAVIS_BRANCH}
+- ./scripts/install_gcloud.sh
+- ./scripts/install_kubectl.sh
+- if [ "${TRAVIS_PULL_REQUEST}" = "false" ]; then make push; fi
 
 deploy:
   provider: script

Dockerfile

@@ -1,29 +1,54 @@
-FROM dddecaf/postgres-base:master
+# Copyright (c) 2018-2020 Novo Nordisk Foundation Center for Biosustainability,
+# Technical University of Denmark.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
-ENV APP_USER=giraffe
+ARG BASE_TAG=alpine
 
-ARG UID=1000
-ARG GID=1000
+FROM dddecaf/postgres-base:${BASE_TAG}
 
-ARG CWD=/app
-ENV PYTHONPATH=${CWD}/src
-WORKDIR "${CWD}"
+ARG BASE_TAG=alpine
+ARG BUILD_COMMIT
+
+LABEL dk.dtu.biosustain.iam.alpine.vendor="Novo Nordisk Foundation \
+Center for Biosustainability, Technical University of Denmark"
+LABEL maintainer="[email protected]"
+LABEL dk.dtu.biosustain.iam.alpine.build.base-tag="${BASE_TAG}"
+LABEL dk.dtu.biosustain.iam.alpine.build.commit="${BUILD_COMMIT}"
+
+ARG CWD="/app"
 
-RUN addgroup -g "${GID}" -S "${APP_USER}" && \
-    adduser -u "${UID}" -G "${APP_USER}" -S "${APP_USER}"
+ENV PYTHONPATH="${CWD}/src"
+
+WORKDIR "${CWD}"
 
 # Install openssh to be able to generate rsa keys
 RUN apk add --update --no-cache openssh
 
-# Install python dependencies
-COPY requirements ./requirements
+COPY requirements ./requirements/
+
 RUN set -eux \
     # build-base is required to build grpcio->firebase-admin
     && apk add --no-cache --virtual .build-deps build-base linux-headers \
-    && pip-sync requirements/requirements.txt \
+    && pip install -r requirements/requirements.txt \
+    && rm -rf /root/.cache/pip \
     # Remove build dependencies to reduce layer size.
     && apk del .build-deps
 
-# Install the codebase
-COPY . "${CWD}/"
-RUN chown -R "${APP_USER}:${APP_USER}" "${CWD}"
+COPY . ./
+
+RUN chown -R "${APP_USER}:${APP_USER}" .
+
+EXPOSE 8000
+
+CMD ["gunicorn", "-c", "gunicorn.py", "iam.wsgi:app"]

LATEST_BASE_TAG

@@ -0,0 +1 @@
+ alpine_2020-05-26_97a4608

Makefile

@@ -1,33 +1,81 @@
-.PHONY: setup network keypair databases lock build start qa style test \
-		test-travis flake8 isort isort-save license stop clean logs safety
-SHELL:=/bin/bash
+.PHONY: setup post-build lock own build push start qa style safety test qc stop \
+	clean logs
 
+################################################################################
+# Variables                                                                    #
+################################################################################
+
+IMAGE ?= gcr.io/dd-decaf-cfbf6/iam
+BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD)
+BUILD_COMMIT ?= $(shell git rev-parse HEAD)
+SHORT_COMMIT ?= $(shell git rev-parse --short HEAD)
+BUILD_DATE ?= $(shell date -u +%Y-%m-%d)
+BUILD_TAG ?= ${BRANCH}_${BUILD_DATE}_${SHORT_COMMIT}
 
 #################################################################################
 # COMMANDS                                                                      #
 #################################################################################
 
-## Run all initialization targets. You must only run this once.
-setup: network keypair databases
-
 ## Create the docker bridge network if necessary.
 network:
 	docker network inspect DD-DeCaF >/dev/null 2>&1 || \
 		docker network create DD-DeCaF
 
-## Build local docker images.
+## Run all initialization targets.
+setup: network
+
+## Generate the compiled requirements files.
+lock:
+	docker pull dddecaf/tag-spy:latest
+	$(eval LATEST_BASE_TAG := $(shell docker run --rm dddecaf/tag-spy:latest tag-spy dddecaf/postgres-base alpine))
+	$(file >LATEST_BASE_TAG, $(LATEST_BASE_TAG))
+	$(eval COMPILER_TAG := $(subst alpine,alpine-compiler,$(LATEST_BASE_TAG)))
+	$(info ************************************************************)
+	$(info * Compiling service dependencies on the basis of:)
+	$(info * dddecaf/postgres-base:$(COMPILER_TAG))
+	$(info ************************************************************)
+	docker pull dddecaf/postgres-base:$(COMPILER_TAG)
+	docker run --rm --mount \
+		"source=$(CURDIR)/requirements,target=/opt/requirements,type=bind" \
+		dddecaf/postgres-base:$(COMPILER_TAG) \
+		pip-compile --allow-unsafe --verbose --generate-hashes --upgrade \
+		/opt/requirements/requirements.in
+
+## Change file ownership from root to local user.
+own:
+	sudo chown "$(shell id --user --name):$(shell id --group --name)" .
+
+## Build the Docker image for deployment.
+build-travis:
+	$(eval LATEST_BASE_TAG := $(shell cat LATEST_BASE_TAG))
+	$(info ************************************************************)
+	$(info * Building the service on the basis of:)
+	$(info * dddecaf/postgres-base:$(LATEST_BASE_TAG))
+	$(info * Today is $(shell date -u +%Y-%m-%d).)
+	$(info * Please re-run `make lock` if you want to check for and)
+	$(info * depend on a later version.)
+	$(info ************************************************************)
+	docker pull dddecaf/postgres-base:$(LATEST_BASE_TAG)
+	docker build \
+		--build-arg BASE_TAG=$(LATEST_BASE_TAG) \
+		--build-arg BUILD_COMMIT=$(BUILD_COMMIT) \
+		--tag $(IMAGE):$(BRANCH) \
+		--tag $(IMAGE):$(BUILD_TAG) \
+		.
+
+## Build the local docker-compose image.
 build:
-	docker-compose build
+	$(eval LATEST_BASE_TAG := $(shell cat LATEST_BASE_TAG))
+	BASE_TAG=$(LATEST_BASE_TAG) docker-compose build
 
-## Recompile requirements and store pinned dependencies with hashes.
-pip-compile:
-	docker run --rm -v `pwd`/requirements:/build dddecaf/postgres-base:compiler \
-		pip-compile --generate-hashes --upgrade \
-		--output-file /build/requirements.txt /build/requirements.in
+## Push local Docker images to their registries.
+push:
+	docker push $(IMAGE):$(BRANCH)
+	docker push $(IMAGE):$(BUILD_TAG)
 
 ## Start all services in the background.
 start:
-	docker-compose up -d
+	docker-compose up --force-recreate -d
 
 ## Create RSA keypair used for signing JWTs.
 keypair:
@@ -43,45 +91,51 @@ databases:
 	docker-compose stop
 	# note: not migrating iam_test db; tests will create and tear down tables
 
-## Run all QA targets.
-qa: style safety test
-
-## Run all style related targets.
-style: flake8 isort license
+## Run all post-build initialization targets. You must only run this once.
+post-build: keypair databases
 
-## Run flake8.
-flake8:
-	docker-compose run --rm web flake8 src/iam tests
+## Apply all quality assurance (QA) tools.
+qa:
+	docker-compose exec -e ENVIRONMENT=testing web \
+		isort --recursive src tests
+	docker-compose exec -e ENVIRONMENT=testing web \
+		black src tests
 
-## Check Python package import order.
 isort:
-	docker-compose run --rm web isort --check-only --recursive src/iam tests
+	docker-compose exec -e ENVIRONMENT=testing web \
+		isort --check-only --diff --recursive src tests
+
+black:
+	docker-compose exec -e ENVIRONMENT=testing web \
+		black --check --diff src tests
 
-## Sort imports and write changes to files.
-isort-save:
-	docker-compose run --rm web isort --recursive src/iam tests
+flake8:
+	docker-compose exec -e ENVIRONMENT=testing web \
+		flake8 src tests
 
-## Verify source code license headers.
 license:
-	./scripts/verify_license_headers.sh src/iam tests
+	docker-compose exec -e ENVIRONMENT=testing web \
+		./scripts/verify_license_headers.sh src tests
 
-## Check for known vulnerabilities in python dependencies.
+## Run all style checks.
+style: isort black flake8 license
+
+## Check installed dependencies for vulnerabilities.
 safety:
-	docker-compose run --rm web safety check
+	docker-compose exec -e ENVIRONMENT=testing web \
+		safety check --full-report
 
-## Run the tests.
+## Run the test suite.
 test:
-	docker-compose run --rm -e ENVIRONMENT=testing web pytest --cov=src/iam
-
-## Run the tests and report coverage (see https://docs.codecov.io/docs/testing-with-docker).
-shared := /tmp/coverage
-test-travis:
-	mkdir "$(shared)"
-	docker-compose run --rm -e ENVIRONMENT=testing \
-	    -v "$(shared):$(shared)" web \
-	    pytest --cov-report xml:$(shared)/coverage.xml --cov-report term \
-	    --cov=src/iam
-	bash <(curl -s https://codecov.io/bash) -f "$(shared)/coverage.xml"
+	docker-compose exec -e ENVIRONMENT=testing web \
+		pytest --cov=iam --cov-report=term
+
+## Run all quality control (QC) tools.
+qc: style safety test
+
+## Check the gunicorn configuration.
+gunicorn:
+	docker-compose run --rm web gunicorn --check-config -c gunicorn.py iam.wsgi:app
 
 ## Stop all services.
 stop:
@@ -95,13 +149,14 @@ clean:
 logs:
 	docker-compose logs --tail="all" -f
 
-#################################################################################
-# Self Documenting Commands                                                     #
-#################################################################################
+################################################################################
+# Self Documenting Commands                                                    #
+################################################################################
 
 .DEFAULT_GOAL := show-help
 
-# Inspired by <http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html>
+# Inspired by
+# <http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html>
 # sed script explained:
 # /^##/:
 # 	* save line in hold space
@@ -154,4 +209,5 @@ show-help:
 		} \
 		printf "\n"; \
 	}' \
-	| more $(shell test $(shell uname) = Darwin && echo '--no-init --raw-control-chars')
+	| more $(shell test $(shell uname) = Darwin \
+	&& echo '--no-init --raw-control-chars')

deployment/production/deployment.yml

@@ -201,7 +201,6 @@ spec:
           readOnly: true
         - name: prometheus-client
           mountPath: /prometheus-client
-        command: ["gunicorn", "-c", "gunicorn.py", "iam.wsgi:app"]
         readinessProbe:
           httpGet:
             path: /iam/healthz

deployment/staging/deployment.yml

@@ -201,7 +201,6 @@ spec:
           readOnly: true
         - name: prometheus-client
           mountPath: /prometheus-client
-        command: ["gunicorn", "-c", "gunicorn.py", "iam.wsgi:app"]
         readinessProbe:
           httpGet:
             path: /iam/healthz

docker-compose.yml

@@ -4,7 +4,10 @@ services:
     build:
       context: .
       dockerfile: Dockerfile
-    image: gcr.io/dd-decaf-cfbf6/iam:${IMAGE_TAG:-latest}
+      args:
+        - BASE_TAG=${BASE_TAG:-alpine}
+        - BUILD_COMMIT=${BUILD_COMMIT:-unknown}
+    image: gcr.io/dd-decaf-cfbf6/iam:${BUILD_TAG:-latest}
     networks:
       default:
       DD-DeCaF:
@@ -20,9 +23,9 @@ services:
       - postgres
     environment:
       - ENVIRONMENT=${ENVIRONMENT:-development}
-      - SCRIPT_NAME=${SCRIPT_NAME}
       - FLASK_APP=src/iam/wsgi.py
-      - ALLOWED_ORIGINS=${ALLOWED_ORIGINS:-*}
+      - SCRIPT_NAME=${SCRIPT_NAME}
+      - ALLOWED_ORIGINS=${ALLOWED_ORIGINS:-http://localhost:4200}
       - SENTRY_DSN=${SENTRY_DSN}
       - BASIC_AUTH_USERNAME=${BASIC_AUTH_USERNAME:-admin}
       - BASIC_AUTH_PASSWORD=${BASIC_AUTH_PASSWORD}
@@ -41,7 +44,6 @@ services:
       - FIREBASE_PRIVATE_KEY=${FIREBASE_PRIVATE_KEY}
       - prometheus_multiproc_dir=/prometheus-client
       - SENDGRID_API_KEY=${SENDGRID_API_KEY}
-    command: gunicorn -c gunicorn.py iam.wsgi:app
 
   postgres:
     image: postgres:9.6-alpine

pyproject.toml

@@ -0,0 +1,3 @@
+[tool.black]
+line-length = 80
+python-version = ['py36']