.gitlab-ci.yml

stages:
  - build
  - test
  - deploy

variables:
  POSTGRES_DB: postgres
  POSTGRES_USER: postgres
  POSTGRES_PASSWORD: ""
  POSTGRES_HOST_AUTH_METHOD: trust

test:
  stage: test
  needs: []
  image: python:3.13-slim-bookworm
  services:
    - name: postgres:15
      alias: postgres-test
      variables:
        POSTGRES_HOST: postgres-test
  variables:
    SAMPLEDB_SQLALCHEMY_DATABASE_URI: "postgresql+psycopg2://postgres:@postgres-test:5432/postgres"
  tags:
    - kubernetes-executor
  script:
    - apt-get update
    # set up Python 3
    - DEBIAN_FRONTEND=noninteractive apt-get install -y git python3-pip postgresql-client chromium libpangocairo-1.0-0 gettext
    - mkdir ${CI_PROJECT_DIR}/test_files/
    - for i in 0 1 2 3; do
        psql -h "postgres" -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "CREATE DATABASE testdb_$i;";
        mkdir ${CI_PROJECT_DIR}/test_files/$i/;
      done
    # install dependencies
    - python3 -m pip install -r requirements.txt
    - python3 -m pip install -r requirements.test.txt
    # build translations once instead of with each test
    - python3 -m sampledb build_translations
    # set up configuration values for testing
    - export SAMPLEDB_BUILD_TRANSLATIONS=
    # run tests and gather coverage data
    - python3 -m pytest -x -s -n=4 --cov=sampledb/ --cov-report=term --cov-report=xml --junitxml=pytest.xml tests
    - coverage xml
  coverage: '/^TOTAL.*\s+(\d+\%)$/'
  artifacts:
    reports:
      coverage_report:
        coverage_format: cobertura
        path: coverage.xml
      junit: pytest.xml

analyze:
  stage: test
  needs: []
  image: python:3.13-slim-bookworm
  script:
    - apt-get update
    # set up Python 3
    - apt-get install -y git python3-pip
    # install static analysis packages and library stub files
    - python3 -m pip install -r requirements.analyze.txt
    - python3 -m pycodestyle --ignore=E402,E501,W504,W601 sampledb
    - python3 -m pyflakes sampledb
    # install requirements to allow pylint and mypy to import them
    - python3 -m pip install -r requirements.txt
    - python3 -m pylint --score no --disable=R0401,R0801,R0902,R0903,R0911,R0912,R0913,R0914,R0915,R0916,R0917,R1702,R1704,R1705,R1711,R1720,R1721,R1723,R1724,R1730,R1731,C0103,C0112,C0114,C0115,C0116,C0123,C0301,C0302,C0413,C0415,W0212,W0511,W1202,W0603,W0613,W0621,W0622,W0703,W0707,E1101,E0213,E1136  --load-plugins=pylint.extensions.docparams sampledb/
    - MYPYPATH=./stubs python3 -m mypy --strict --ignore-missing-imports --follow-imports=silent sampledb/

analyze-js:
  stage: test
  needs: []
  image: node:lts
  script:
    # install static analysis packages
    - npm install -g jshint semistandard
    # run JSHint and semistandard for some JavaScript files
    - jshint sampledb/static/sampledb/js/*.js sampledb/static/sampledb/js/*/*.js sampledb/static/sampledb/js/*/*/*.js
    - semistandard sampledb/static/sampledb/js/*.js sampledb/static/sampledb/js/*/*.js sampledb/static/sampledb/js/*/*/*.js

.template-check-migrations:
  stage: test
  needs: []
  image: python:3.13-slim-bookworm
  script:
    # define environment variables needed for SampleDB demo
    - export SAMPLEDB_CONTACT_EMAIL=example@example.org
    - export SAMPLEDB_MAIL_SERVER=example.org
    - export SAMPLEDB_MAIL_SENDER=example@example.org
    # set up system dependencies
    - apt-get update
    - apt-get install -y git python3-pip libpython3-dev libpangocairo-1.0-0 gettext postgresql-client libcairo2-dev pkg-config python3-dev
    # set up database without upgrade
    - python3 -m pip install -r requirements.txt
    - python3 -m sampledb list_actions
    # create backup for comparison
    - pg_dump -h $POSTGRES_HOST -U postgres --schema-only postgres > postgres_without_upgrade.sql
    # re-create database to ensure it is entirely empty
    - dropdb -h $POSTGRES_HOST -U postgres postgres
    - createdb -h $POSTGRES_HOST -U postgres postgres
    # check out stable version
    - git remote add upstream https://github.com/sciapp/sampledb.git || git remote set-url upstream https://github.com/sciapp/sampledb.git
    - git fetch upstream master
    - git checkout FETCH_HEAD
    # set up database with stable version
    - python3 -m pip install -r requirements.txt
    - python3 -m sampledb list_actions
    # check out current version again
    - git checkout $CI_COMMIT_SHA
    # upgrade database using migrations
    - python3 -m pip install -r requirements.txt
    - python3 -m sampledb list_actions
    # create backup for comparison
    - pg_dump -h $POSTGRES_HOST -U postgres --schema-only postgres > postgres_with_upgrade_from_master.sql
    # ensure both backups yield identical backups
    - diff --color=always --context postgres_without_upgrade.sql postgres_with_upgrade_from_master.sql
    # re-create database to ensure it is entirely empty
    - dropdb -h $POSTGRES_HOST -U postgres postgres
    - createdb -h $POSTGRES_HOST -U postgres postgres
    # check out development version
    - git fetch upstream develop
    - git checkout FETCH_HEAD
    # do not check migrating from develop to this commit if this commit is not a descendant of develop
    - git merge-base --is-ancestor HEAD $CI_COMMIT_SHA || exit 0
    # set up database with development version
    - python3 -m pip install -r requirements.txt
    - python3 -m sampledb list_actions
    # check out current version again
    - git checkout $CI_COMMIT_SHA
    # upgrade database using migrations
    - python3 -m pip install -r requirements.txt
    - python3 -m sampledb list_actions
    # create backup for comparison
    - pg_dump -h $POSTGRES_HOST -U postgres --schema-only postgres > postgres_with_upgrade_from_develop.sql
    # ensure both backups yield identical backups
    - diff --color=always --context postgres_without_upgrade.sql postgres_with_upgrade_from_develop.sql

check-migrations-postgres-11:
  extends: .template-check-migrations
  services:
    - name: postgres:11
      alias: postgres11
      variables:
        POSTGRES_HOST: postgres11
  variables:
    POSTGRES_HOST: postgres11
    SAMPLEDB_SQLALCHEMY_DATABASE_URI: "postgresql+psycopg2://postgres:@postgres11:5432/postgres"

check-migrations-postgres-12:
  extends: .template-check-migrations
  services:
    - name: postgres:12
      alias: postgres12
      variables:
        POSTGRES_HOST: postgres12
  variables:
    POSTGRES_HOST: postgres12
    SAMPLEDB_SQLALCHEMY_DATABASE_URI: "postgresql+psycopg2://postgres:@postgres12:5432/postgres"

check-migrations-postgres-13:
  extends: .template-check-migrations
  services:
    - name: postgres:13
      alias: postgres13
      variables:
        POSTGRES_HOST: postgres13
  variables:
    POSTGRES_HOST: postgres13
    SAMPLEDB_SQLALCHEMY_DATABASE_URI: "postgresql+psycopg2://postgres:@postgres13:5432/postgres"

check-migrations-postgres-14:
  extends: .template-check-migrations
  services:
    - name: postgres:14
      alias: postgres14
      variables:
        POSTGRES_HOST: postgres14
  variables:
    POSTGRES_HOST: postgres14
    SAMPLEDB_SQLALCHEMY_DATABASE_URI: "postgresql+psycopg2://postgres:@postgres14:5432/postgres"

check-migrations-postgres-15:
  extends: .template-check-migrations
  services:
    - name: postgres:15
      alias: postgres15
      variables:
        POSTGRES_HOST: postgres15
  variables:
    POSTGRES_HOST: postgres15
    SAMPLEDB_SQLALCHEMY_DATABASE_URI: "postgresql+psycopg2://postgres:@postgres15:5432/postgres"

check-translations:
  stage: test
  needs: []
  image: python:3.13-slim-bookworm
  script:
    - apt-get update
    # set up Python 3
    - apt-get install -y git python3-pip
    # install Flask-Babel
    - python3 -m pip install flask-babel
    # create a copy of the current extracted messages
    - cp sampledb/translations/de/LC_MESSAGES/extracted_messages.po extracted_messages_current_de.po
    # ensure there are no fuzzy translations
    - grep -zqv fuzzy extracted_messages_current_de.po
    # ensure there are no disabled translations
    - grep -zqv "#~ msgstr" extracted_messages_current_de.po
    # ensure there are no empty translations
    - "grep -zqvP 'msgstr \"\"\\n\\n' extracted_messages_current_de.po"
    # extract and update messages
    - pybabel extract --no-location -F babel.cfg -k lazy_gettext -o sampledb/messages.pot sampledb
    - pybabel update -i sampledb/messages.pot -d sampledb/translations -D extracted_messages
    - cp sampledb/translations/de/LC_MESSAGES/extracted_messages.po extracted_messages_generated_de.po
    # remove lines 1-19 containing the header comment and translation information
    - sed '1,19d' extracted_messages_current_de.po > extracted_messages_current_de_without_header.po
    - sed '1,19d' extracted_messages_generated_de.po > extracted_messages_generated_de_without_header.po
    # compare current and generated messages to ensure there are no changes missing in the current messages
    - diff --color=always extracted_messages_current_de_without_header.po extracted_messages_generated_de_without_header.po

documentation:
  stage: test
  needs: []
  image: python:3.13-slim-bookworm
  services:
    - name: postgres:15
      alias: postgres-documentation
      variables:
        POSTGRES_HOST: postgres-documentation
  variables:
    SAMPLEDB_SQLALCHEMY_DATABASE_URI: "postgresql+psycopg2://postgres:@postgres-documentation:5432/postgres"
  tags:
    - kubernetes-executor
  script:
    - apt-get update
    # set up Python 3
    - apt-get install -y git python3-pip libpython3-dev chromium libpangocairo-1.0-0 gettext
    # install dependencies
    - python3 -m pip install -r requirements.txt
    - python3 -m pip install -r requirements.documentation.txt
    # generate documentation images using current version
    - python3 docs/utils/generate_images.py
    # build documentation
    - python3 -m sphinx -b html -t iffSamples docs/ build_documentation/
  artifacts:
    paths:
    - build_documentation

build:
  stage: build
  image: docker:stable
  tags:
    - privileged-executor
  script:
    - apk add git
    - export VERSION=`git describe`
    - docker build --build-arg SAMPLEDB_VERSION="$VERSION" -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA .
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
    - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
    - if echo "$CI_COMMIT_TAG" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+$'; then
        export VERSION=`echo "$CI_COMMIT_TAG" | sed 's/^v//'`;
        docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:$VERSION;
        docker push $CI_REGISTRY_IMAGE:$VERSION;
      fi

test-container-status:
  stage: test
  needs: ["build"]
  image: docker:stable
  tags:
    - privileged-executor
  variables:
    CONTAINER_NAME: sampledb-$CI_COMMIT_SHA
  script:
    - docker run
        -d
        --name $CONTAINER_NAME-postgres
        -e POSTGRES_DB=postgres
        -e POSTGRES_USER=postgres
        -e POSTGRES_PASSWORD=postgres
        postgres:15
    - sleep 5 # allow container to start
    - docker ps
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
    - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
    - docker run
        -d
        --link $CONTAINER_NAME-postgres
        --name $CONTAINER_NAME
        -v $CI_PROJECT_DIR/files:/home/sampledb/files:rw
        -e SAMPLEDB_MAIL_SERVER=example.org
        -e SAMPLEDB_MAIL_SENDER=example@example.org
        -e SAMPLEDB_CONTACT_EMAIL=example@example.org
        -e SAMPLEDB_SQLALCHEMY_DATABASE_URI="postgresql+psycopg2://postgres:postgres@$CONTAINER_NAME-postgres:5432/postgres"
        $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
    - sleep 5 # allow container to start
    - docker ps
    - docker logs $CONTAINER_NAME
    - docker ps | grep $CONTAINER_NAME | grep -E '(healthy|starting)'
    - sleep 90 # 60s health start period plus 30s buffer
    - docker logs $CONTAINER_NAME
    - docker ps | grep $CONTAINER_NAME | grep '(healthy)'
  after_script:
    - docker stop $CONTAINER_NAME || true
    - docker rm -f $CONTAINER_NAME || true
    - docker stop $CONTAINER_NAME-postgres || true
    - docker rm -f $CONTAINER_NAME-postgres || true

pages:
  stage: deploy
  image: busybox
  only:
    - master
    - tags
    - develop
  script:
    - mv build_documentation public
  artifacts:
    paths:
    - public
    expire_in: 1 day

deploy-to-dev:
  stage: deploy
  image: debian:bookworm
  environment: staging
  only:
    - develop@Scientific-IT-Systems/SampleDB
  script:
    - apt-get update
    - apt-get install -y ssh
    # set up SSH for access to deployment server
    - mkdir -p --mode=700 ~/.ssh/
    - echo "$DEPLOYMENT_PRIVATE_KEY" > ~/.ssh/deployment_key
    - chmod 400 ~/.ssh/deployment_key
    - echo "$SSH_SERVER_HOSTKEYS" > ~/.ssh/known_hosts
    - chmod 400 ~/.ssh/known_hosts
    # actual deployment is handled via authorized_keys command
    - ssh -i ~/.ssh/deployment_key iffregistry@iffweb.iff.kfa-juelich.de "$CI_COMMIT_SHA"

deploy-to-github:
  stage: deploy
  image: debian:bookworm
  variables:
    GIT_STRATEGY: none
  only:
    - branches@Scientific-IT-Systems/SampleDB
    - tags@Scientific-IT-Systems/SampleDB
  script:
    - apt-get update
    - apt-get install -y git
    # set up SSH for access to GitHub
    - mkdir -p --mode=700 ~/.ssh/
    - echo "$SCIAPP_SAMPLEDB_PRIVATE_KEY" > ~/.ssh/id_rsa
    - chmod 400 ~/.ssh/id_rsa
    - echo "github.com $GITHUB_HOST_KEY" >> ~/.ssh/known_hosts
    - chmod 400 ~/.ssh/known_hosts
    - git clone --mirror "$CI_REPOSITORY_URL" repo
    - cd repo && git push --mirror git@github.com:sciapp/sampledb.git && cd -

deploy-to-github-container-registry:
  stage: deploy
  image: docker:stable
  tags:
    - privileged-executor
  only:
    - tags@Scientific-IT-Systems/SampleDB
    - develop@Scientific-IT-Systems/SampleDB
  script:
  - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
  - echo $CR_PAT | docker login ghcr.io --username USERNAME --password-stdin
  - if [ "$CI_COMMIT_REF_NAME" = "develop" ]; then
      docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA ghcr.io/sciapp/sampledb:develop;
      docker push ghcr.io/sciapp/sampledb:develop;
    fi
  - if echo "$CI_COMMIT_TAG" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+$'; then
      export VERSION=`echo "$CI_COMMIT_TAG" | sed 's/^v//'`;
      docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA ghcr.io/sciapp/sampledb:$VERSION;
      docker push ghcr.io/sciapp/sampledb:$VERSION;
      docker tag sciapp/sampledb:$VERSION ghcr.io/sciapp/sampledb:latest;
      docker push ghcr.io/sciapp/sampledb:latest;
    fi

deploy-to-dockerhub:
  stage: deploy
  image: docker:stable
  tags:
    - privileged-executor
  only:
    - tags@Scientific-IT-Systems/SampleDB
    - develop@Scientific-IT-Systems/SampleDB
  script:
  - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
  - cat $DOCKERHUB_PASSWORD_FILE | docker login --username $DOCKERHUB_USERNAME --password-stdin
  - if [ "$CI_COMMIT_REF_NAME" = "develop" ]; then
      docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA sciapp/sampledb:develop;
      docker push sciapp/sampledb:develop;
    fi
  - if echo "$CI_COMMIT_TAG" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+$'; then
      export VERSION=`echo "$CI_COMMIT_TAG" | sed 's/^v//'`;
      docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA sciapp/sampledb:$VERSION;
      docker push sciapp/sampledb:$VERSION;
      docker tag sciapp/sampledb:$VERSION sciapp/sampledb:latest;
      docker push sciapp/sampledb:latest;
    fi

deploy-to-pypi:
  stage: deploy
  image: python:3.13-slim-bookworm
  only:
    - tags@Scientific-IT-Systems/SampleDB
  script:
    - python3 -m pip install setuptools wheel twine
    - python3 setup.py sdist
    - if echo "$CI_COMMIT_TAG" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+$'; then
        export VERSION=`echo "$CI_COMMIT_TAG" | sed 's/^v//'`;
        python3 -m twine upload dist/sampledb-$VERSION.tar.gz;
      fi