.gitlab-ci.yml

# Copyright (C) 2019 Magenta ApS, http://magenta.dk.
# Contact: info@magenta.dk.
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

################################################################################
# Changes to this file requires approval from Labs. Please add a person from   #
# Labs as required approval to your MR if you have any changes.                #
################################################################################

# For `docker push` to work, the following envionment variables have to set in
# the Gitlab UI.
# RELEASE_REGISTRY_USER
# RELEASE_REGISTRY_PASSWORD

# For git push of documentation to https://github.com/magenta-aps/os2bos-docs
# the private key needs to be set in the environment variable:
# DOCS_SSH_KEY

include:
  - project: 'labs/salt-automation'
    ref: 'master'
    file:
      - '/gitlab-ci-templates/common/rules.v1.yml'
      - '/gitlab-ci-templates/common/config-updater-meta.v1.yml'
      - '/gitlab-ci-templates/common/saltbert.v1.yml'

stages:
  - lint
  # build stage is not blocked by lint stage, as we have a DAG:
  # https://docs.gitlab.com/ee/ci/directed_acyclic_graph/index.html
  - build
  - test
  - release
  - deploy


variables:
  # Project variables
  RELEASE_REGISTRY: docker.io
  RELEASE_REGISTRY_IMAGE: index.docker.io/magentaaps/bevillingsplatform

  IMAGE_SHA: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA}
  IMAGE_DEV: ${CI_REGISTRY_IMAGE}:dev

  IMAGE: magentaaps/bevillingsplatform
  IMAGE_TAG: ${IMAGE}:${CI_COMMIT_TAG}
  IMAGE_LATEST: ${IMAGE}:latest



# Lint stage
#############

.lint-default: &lint-default
  stage: lint
  needs: []
  image: python:3.10
  tags:
    - docker

Lint Python:
  <<: *lint-default
  before_script:
    - pip3 install -r backend/requirements-test.txt
  script:
    - cd backend
    - black --check --diff .
    - flake8 .
    - pydocstyle .

Lint shell scripts:
  <<: *lint-default
  image: koalaman/shellcheck-alpine:latest
  before_script:
    - apk update
    - apk add git
  script:
    - git ls-files -c --exclude='*.sh' --ignored | xargs shellcheck

Lint Dockerfiles:
  <<: *lint-default
  image: hadolint/hadolint:latest-debian
  before_script:
    - apt-get -y update
    - apt-get -y install --no-install-recommends git
  script:
    - git ls-files -c --exclude='Dockerfile*' --ignored | xargs --max-lines=1 hadolint


# Build stage
#############

Build docs:
  stage: build
  needs: []
  image: python:3.10
  tags:
    - docker
  before_script:
    - pip3 install -r backend/requirements-test.txt
    - pip3 install -r backend/requirements.txt
  script:
    - sphinx-build docs/source doc-build
  artifacts:
    paths:
      - doc-build

Build app image:
  stage: build
  needs: []
  dependencies: []
  image:
    name: gcr.io/kaniko-project/executor:debug-v0.16.0
    entrypoint: [""]
  tags:
    - docker
  script:
    # Login to GitLab registry
    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
    - /kaniko/executor
      --cache
      --context=$CI_PROJECT_DIR
      --dockerfile=$CI_PROJECT_DIR/docker/Dockerfile
      --destination=${IMAGE_SHA}

# Test stage
############
.test-default: &test-default
  stage: test
  needs: ["Build app image"]
  dependencies: []

Unittest:
  <<: *test-default
  variables:
    DATABASE_NAME: bev
    DATABASE_USER: bev
    DATABASE_PASSWORD: bev
    BEV_USER_CONFIG_PATH: $CI_PROJECT_DIR/dev-environment/dev-settings.ini
    COVERAGE_FILE: $CI_PROJECT_DIR/.coverage
  image: ${IMAGE_SHA}
  tags:
    - docker
  services:
    - name: magentaaps/postgres-os2bos:1-10.14-test
      alias: db
  script:
    - cd /code/backend &&
      pytest
      --cov=core
      --cov=bevillingsplatform
      --cov=core.management.commands
      --junitxml $CI_PROJECT_DIR/junit.xml
      --cov-report html:$CI_PROJECT_DIR/coverage-html
      --cov-report term
      --cov-fail-under=100
      -p no:cacheprovider
  coverage: '/TOTAL.*\s+(\d+%)$/'
  artifacts:
    when: always
    paths:
      - $CI_PROJECT_DIR/coverage-html
    reports:
      junit: $CI_PROJECT_DIR/junit.xml

FrontendTest:
  <<: *test-default
  variables:
    FF_NETWORK_PER_BUILD: 1
    DATABASE_NAME: bev
    DATABASE_USER: bev
    DATABASE_PASSWORD: bev
    BEV_USER_CONFIG_PATH: /code/dev-environment/ci-settings.ini
    SIMPLESAMLPHP_BASEURLPATH: http://idp:8080/simplesaml
    SIMPLESAMLPHP_SP_ENTITY_ID: http://bev:5000
    SIMPLESAMLPHP_SP_ASSERTION_CONSUMER_SERVICE: http://bev:5000/api/saml2_auth/acs/
  image:
    name: testcafe/testcafe:latest
    entrypoint: ["/bin/sh", "-c"]
  tags:
    - docker
    - high-mem
  services:
    - name: magentaaps/postgres-os2bos:1-10.14-test
      alias: db
    - name: ${IMAGE_SHA}
      alias: bev
    - name: magentalabs/os2bos-idp-test:2.0.1
      alias: idp
  before_script:
    - npm --version
    - cd frontend-tests
    - npm install
  script:
    - /opt/testcafe/docker/testcafe-docker.sh chromium:headless tests/*.js --baseurl http://bev:5000 --cache --cs node_modules/axe-core/axe.min.js --screenshots $CI_PROJECT_DIR/artifacts --video $CI_PROJECT_DIR/artifacts --video-options pathPattern='${DATE}_${TIME}/${FIXTURE} - ${TEST}.mp4' --assertion-timeout 10000 --selector-timeout 10000 --pageRequestTimeout 10000 --reporter spec,xunit:$CI_PROJECT_DIR/testcafe.xml
  artifacts:
    when: always
    paths:
      - $CI_PROJECT_DIR/artifacts
    reports:
      junit: $CI_PROJECT_DIR/testcafe.xml


# Release stage
###############

.release-default: &release-default
  stage: release
  needs: ["Unittest", "FrontendTest"]
  image: alpine:3.13
  variables:
    GIT_STRATEGY: none # We do not need the source code
  dependencies: []
  tags:
    - docker
  before_script:
    # install skopeo
    - apk add skopeo=1.2.1-r0

Release dev:
  <<: *release-default
  only:
    refs:
      - develop
  script:
    - skopeo copy
      --src-creds=${CI_REGISTRY_USER}:${CI_BUILD_TOKEN}
      --dest-creds=${CI_REGISTRY_USER}:${CI_BUILD_TOKEN}
      "docker://${IMAGE_SHA}"
      "docker://${IMAGE_DEV}"


Release rc:
  <<: *release-default
  only:
    variables:
      # Matches all of SemVer 2.0.0
      - $CI_COMMIT_TAG =~ /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$/
  script:
    - skopeo copy
      --src-creds=${CI_REGISTRY_USER}:${CI_BUILD_TOKEN}
      --dest-creds=${RELEASE_REGISTRY_USER}:${RELEASE_REGISTRY_PASSWORD}
      "docker://${IMAGE_SHA}"
      "docker://${IMAGE_TAG}"

Release master:
  <<: *release-default
  only:
    variables:
      # Matches <version core> from SemVer 2.0.0 BNF grammar. Ex. 2.3.4, but not 2.3.4-rc
      - $CI_COMMIT_TAG =~ /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/
  script:
    - skopeo copy
      --src-creds=${CI_REGISTRY_USER}:${CI_BUILD_TOKEN}
      --dest-creds=${RELEASE_REGISTRY_USER}:${RELEASE_REGISTRY_PASSWORD}
      "docker://${IMAGE_SHA}"
      "docker://${IMAGE_TAG}"
    - skopeo copy
      --src-creds=${CI_REGISTRY_USER}:${CI_BUILD_TOKEN}
      --dest-creds=${RELEASE_REGISTRY_USER}:${RELEASE_REGISTRY_PASSWORD}
      "docker://${IMAGE_SHA}"
      "docker://${IMAGE_LATEST}"

Publish database docs:
  stage: release
  needs: ["Build app image"]
  image: magentalabs/schemaspy:1.0.0
  tags:
    - docker
  only:
    variables:
      # Matches <version core> from SemVer 2.0.0 BNF grammar. Ex. 2.3.4, but not 2.3.4-rc
      - $CI_COMMIT_TAG =~ /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/
  variables:
    FF_NETWORK_PER_BUILD: 1
    GIT_STRATEGY: none # We do not need the source code
    BEV_USER_CONFIG_PATH: /code/dev-environment/ci-settings.ini
    DATABASE_NAME: bev
    DATABASE_USER: bev
    DATABASE_PASSWORD: bev
    DATABASE_HOST: db
    DESCRIPTION: os2bos/$CI_COMMIT_TAG
  services:
    - name: magentaaps/postgres-os2bos:1-10.14-test
      alias: db
    - name: ${IMAGE_SHA} # we need to run the app container to run the migrations
  before_script:
    - git clone git@github.com:magenta-aps/os2bos-docs.git /os2bos-docs/
    - cp -a /er_html/* /os2bos-docs/
    - cd /os2bos-docs
    - git config user.name "deployment bot"
    - git config user.email "labs@magenta.dk"
  script:
    - git add .
    - git commit -m "updated docs to ${IMAGE_TAG}"
    - git push origin master


# Deploy stage
##############

.deploy-default: &deploy-default
  stage: deploy
  services: []
  dependencies: []
  tags:
    - deploy


Deploy test:
  # Automatic deployment of branch develop to cburtest.magenta.dk
  <<: *deploy-default
  needs: ["Release dev"]
  only:
    refs:
      - develop
  variables:
    DEPLOYMENT_NAMESPACE: cbur.magenta.test
    SALTAPI_URL: https://ctrldev1.magenta-aps.dk/api/
    SALTAPI_EAUTH: file
    GIT_STRATEGY: none
  script:
    - echo Attempting to call deployment
    - pepper --client runner --fail-any state.orchestrate ${DEPLOYMENT_NAMESPACE}
  environment:
    name: test
    url: https://cburtest.magenta.dk

Deploy dev:
  # Manuel deployment job to cburdev.magenta.dk
  <<: *deploy-default
  needs: ["Unittest", "FrontendTest"]
  except:
    refs:
      - master
      - develop
  when: manual
  variables:
    DEPLOYMENT_NAMESPACE: cbur.magenta.dev
    SALTAPI_URL: https://ctrldev1.magenta-aps.dk/api/
    SALTAPI_EAUTH: file
    GIT_STRATEGY: none
    PILLAR_OVERRIDE: "{\"OS2BOS_APPLICATION_DOCKER_IMAGE\":\"${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA}\"}"
  script:
    - echo deploying ${PILLAR_OVERRIDE}
    - pepper --client runner --fail-any state.orchestrate ${DEPLOYMENT_NAMESPACE} pillar=$(echo $PILLAR_OVERRIDE)
  environment:
    name: dev
    url: https://cburdev.magenta.dk


Update staging salt config:
  extends:
    - .rules:semver-all
    - .config:endpoint-with-tag
  needs: ["Release rc"]
  when: manual
  variables:
    ENDPOINT: os2bos/salt/ballerup/update-staging
    CONFIG_UPDATER_USERNAME: os2bos

Deploy tag to staging:
  <<: *deploy-default
  extends:
    - .rules:semver-all
    - .saltbert
  needs: ["Update staging salt config"]
  when: manual
  variables:
    SALTBERT_ORCHESTRATION: cbur.ballerup.staging
    SALTBERT_URL: https://ctrl1.magenta-aps.dk
    SALTBERT_PASSWORD: $SALTBERT_STAGING_PASSWORD
  environment:
    name: ballerup-staging
    url: https://test-bevilling.balk.dk

Update production salt config:
  extends:
    - .rules:semver-core
    - .config:endpoint-with-tag
  needs: ["Release master"]
  when: manual
  variables:
    ENDPOINT: os2bos/salt/ballerup/update-production
    CONFIG_UPDATER_USERNAME: os2bos


Deploy tag to production:
  <<: *deploy-default
  extends:
    - .rules:semver-core
    - .saltbert
  needs: ["Update production salt config"]
  when: manual
  variables:
    SALTBERT_ORCHESTRATION: cbur.ballerup.production
    SALTBERT_URL: https://ctrl1.magenta-aps.dk
    SALTBERT_PASSWORD: $SALTBERT_PROD_PASSWORD
  environment:
    name: ballerup-production
    url: https://bevilling.balk.dk