.gitlab-ci.yml

# == Summary of Baserow's CI workflow:
#
# This file contains the gitlab CI job definitions that build and test Baserow
# automatically.
#
# === Overview of how Baserow uses git branches
#
# * `develop` is the branch we merge newly developed features onto from feature
#   branches.
# * a feature branch is a branch made starting off `develop` containing a specific
#   new feature, when finished it will be merged back onto `develop`.
# * `master` is the branch which contains official releases of Baserow, to do so we
#   periodically merge the latest changes from `develop` onto `master` and then tag
#   that new master commit with a git tag containing the version (1.8.2 etc).
#
# === How new version of Baserow is released to Dockerhub
#
# A. Create an MR from develop to master and merge it.
# B. Wait for the merge commit pipeline succeed on master which will build and test the
#    images.
# C. Tag the merge commit in the GitLab GUI with the git tag being the Baserow version
#    (1.8.2, 1.0, etc).
# D. GitLab will make a new pipeline for the tag which will push the images built in
#    step B to Dockerhub. If step B failed or has not completed yet then this pipeline
#    will fail and not push anything.
#
# === What GitLab CI steps are configured to run and when
#
# See below for the high level summary of the steps GitLab will run to build, test and
# release Baserow images in various scenarios depending on the branches involved.
#
# ==== On the master branch - When MR Merged/commit pushed/branch made
#
#   1. The backend and web-frontend dev images will be built and pushed to the
#      gitlab ci image repo.
#      1. A `{image_dev}:ci-latest-$CI_COMMIT_SHA` image is pushed for the next stages.
#      2. A `{image_dev}:ci-latest-$BRANCH_NAME` image is pushed to cache future runs.
#   2. The pushed `ci-latest-$CI_COMMIT_SHA` images will be tested and linted. If a
#      previously successful test/lint run is found for the same/prev commit AND no
#      files have changed which could possibly change the result this is skipped.
#   3. Cached from the `ci-latest-$CI_COMMIT_SHA` image the non-dev images will be built
#      and then both the dev and non-dev images will be with tagged marking them as
#      tested and pushed to the gitlab ci repo.
#   4. Trigger a pipeline in any downstream repos that depend on this one.
#
# ==== On the develop branch - When MR Merged/new commit pushed
#
#   The build and testing steps 1, 2 and 3 from above are run first and then:
#   4. Push the tested images from step 3 to the Dockerhub repo under the
#      `develop-latest` tag.
#   5. Trigger a pipeline in any downstream repos that depend on this one.
#
# ==== On feature branches - When MR Merged/new commit pushed
#
#   The build and testing steps 1, 2 and 3 from above are run.
#
# ===== On the latest commit on master - When a Git tag is created
#
#   This is done when we have merged the latest changes from develop on master, and we
#   want to release them as a new version of Baserow. GitLab will automatically detect
#   the new git tag and only do the following:
#
#   1. Push the images built from step 3 above (or fail if they don't exist) to the
#      Dockerhub repo with the tags:
#      1. `latest`
#      2. `${git tag}`
#
# ==== Older commit on master - When a Git tag created
#
#   1. Push the images built from step 3 above (or fail if they don't exist) to the
#      Dockerhub repo with the tags:
#      1. `${git tag}`
#
# ==== Any non-master commit - When a Git tag created
#
#   1. Fail as only master commits should be tagged/released.
#
# == Cleanup
#
# Images with tags starting with `ci-latest` or `ci-tested` (made in steps 1. and 3.)
# will be deleted after they are 7 days old by a job that runs daily at 11AM CET.
#
# == Docker Layer Caching and its Security implications.
#
# The build jobs defined in .gitlab/ci_includes/jobs.yml use docker BUILD_KIT enabled
# image caching to:
# 1. Cache docker image builds between different pipelines and branches.
# 2. Cache docker image builds between the build and build-final stages in a single
#    pipeline.
#
# By using BuildKit and multi-stage docker builds we are able to build and store images
# which can then be pulled and used as a cache to build new images quickly from.
#
# === When are docker builds cached between different pipelines and branches?
#
# On branches other than master:
# 1. A build job first tries to find the latest image built on that branch
#    (registry.gitlab.com/bramw/baserow/ci/IMAGE_NAME:ci-latest-BRANCH_NAME)
#    to use as a build cache.
# 2. If no latest image is found then the build job will try use the latest ci dev image
#    build on the develop branch:
#    (registry.gitlab.com/bramw/baserow/ci/IMAGE_NAME:ci-latest-develop)
# 3. Otherwise, the build job will run the build from scratch building all layers.
# 4. Once the build job finishes it will push a new ci-latest-BRANCH_NAME image for
#    future pipelines to cache from. This image will be built with
#    BUILDKIT_INLINE_CACHE=1 ensuring all of its intermediate layers can be cached from.
#
# On master:
# 1. The latest develop ci image will be used as the build cache.
# 2. Otherwise, no build caching will happen.
#
# === When are docker builds cached on the same pipeline and how?
#
# 1. The initial build stage jobs will build and push a ci image (specifically a docker
#    image built with `--target dev`, this means it will build the `dev` stage in the
#    Dockerfile). This image will be built with BUILDKIT_INLINE_CACHE=1 ensuring all of
#    its intermediate layers can be cached from.
# 2. This image will be used for testing etc if required.
# 3. Finally, in the build-final stage we build the non dev images. We cache these
#    images from two sources:
#    1. The dev ci image built by the previous build stage. This will contain all
#       intermediate layers so the non-dev build should re-use cached layers for all
#       docker layers shared by the dev and non dev stages.
#    2. The latest non-dev ci image built by first a previous pipeline on this branch
#       or if not found then the latest non-dev ci image built on develop. On master
#       similarly to the first build stage we only check develop.
#
# === Security implications of docker image caching
#
# This article does a great job explaining why docker layer caching can cause security
# issues: https://pythonspeed.com/articles/docker-cache-insecure-images/ . But
# fundamentally if you cache the FROM base_image and RUN apt upgrade && apt update
# stages docker won't ever re-run these, even if the base image has changed OR there
# have been security fixes published for the packages.
#
# === Periodic full rebuilds on develop
#
# To get around the security implications of docker image layer caching we have a
# daily ci pipeline scheduled job on develop (https://gitlab.com/bramw/baserow/-/pipeline_schedules)
# which sets TRIGGER_FULL_IMAGE_REBUILD=yes as a pipeline variable. This forces all
# the build stages to build their docker images from scratch pulling any updated base
# images.
#
# This pipeline rebuilds all the `registry.gitlab.com/bramw/baserow/ci/IMAGE_NAME:ci-latest-develop`
# images used for build caching on other branches, develop itself and on master to have
# the latest security updates.
#
# === Why does master cache from develop and not use its own ci-latest cache images?
#
# 1. Master might not have any pipelines run for weeks between releases meaning:
#   a. If it had its own ci-latest cached images they would get cleaned up before they
#      could be used
#   b. If they weren't cleaned up their layers might be massively out of date and weeks
#      old.
# 2. Ok then why not have a periodic job to rebuild on master?
#   a. We are already periodically rebuilding on develop, why do the same work twice
#      if we can just cache from develop.
#   b. Master might start randomly breaking if breaking changes appear in the base
#      layers that get rebuilt. It's much more preferable that only develop breaks
#      and we fix any issues there before they hit master.
# 3. Why not just always rebuild from scratch on master with no docker build caching?
#   a. This makes the release process slower
#   b. If a base image or package change occurs between the time we finish testing our
#      develop images and when we merge develop into master, the images are master
#      might completely break as a result. So now we would have to worry about
#      this potential source of issues as an extra step for every release.
#   c. We are essentially testing entirely different images from the ones being deployed
#      if we just test on develop and master does a full rebuild.
# 4. By having develop being the only place where we do the full rebuilds, it means we:
#    a. Test those rebuilt base layers on all the feature branches and during any
#       develop testing.
#    b. We CD from develop to staging and so these rebuilds are automatically deployed
#       and tested by that also.
#    c. Only have one source of these rebuilt layers, which we test on develop and then
#       re-use on master knowing they are safe.
#

include: '/.gitlab/ci_includes/jobs.yml'

stages:
  - build
  - test
  - build-final
  - publish

variables:
  # Visit https://gitlab.com/bramw/baserow/-/pipelines/new select your branch
  # and click run to be able to run a new pipeline where you can manually control these
  # variables.
  TRIGGER_FULL_IMAGE_REBUILD:
    value: "no"
    description: "If set to yes then all images will re-pull their base images and rebuild entirely from scratch with no caching."
  ENABLE_JOB_SKIPPING:
    value: "false"
    description: "If set to true then tests and lints will skipped when safe to do so."
  ENABLE_COVERAGE:
    value: "true"
    description: "If set to false then tests will not generate coverage or testing reports used by gitlab to show nicer MRs."
  ENABLE_RELEASES:
    value: "false"
    description: "If true then on master and develop release images will be pushed automatically"
  BUILD_ALL_IN_ONE:
    value: "false"
    description: "If true then regardless of what branch we are on the all in one image will be built."
  # An image repo which is used for storing and passing images between ci pipeline jobs
  # and also speeding up ci builds by caching from the latest ci image when building.
  CI_IMAGE_REPO: $CI_REGISTRY_IMAGE/ci
  # Any images with tags prefixed with the two variables below will be cleaned up automatically
  # by our gitlab cleanup job:
  # (https://gitlab.com/bramw/baserow/-/settings/packages_and_registries).
  #
  # ## Note:
  #   These cleanup tag prefixes are needed as gitlab only supports cleanup by defining
  #   a regex that matches tags, so we can't do cleanup differently based on image name
  #   or repo...
  #
  # IMPORTANT: UPDATE GITLAB CONTAINER REPO CLEANUP JOB REGEX IF YOU CHANGE THIS
  CLEANUP_JOB_CI_TAG_PREFIX: ci-latest-
  # IMPORTANT: UPDATE GITLAB CONTAINER REPO CLEANUP JOB REGEX IF YOU CHANGE THIS
  TESTED_IMAGE_PREFIX: ci-tested-
  BACKEND_IMAGE_NAME: backend
  BACKEND_DEV_IMAGE_NAME: backend_dev
  WEBFRONTEND_IMAGE_NAME: web-frontend
  ALLINONE_IMAGE_NAME: baserow
  CLOUDRON_IMAGE_NAME: cloudron
  HEROKU_IMAGE_NAME: heroku
  WEBFRONTEND_DEV_IMAGE_NAME: web-frontend_dev
  BACKEND_CI_DEV_IMAGE: $CI_IMAGE_REPO/$BACKEND_DEV_IMAGE_NAME:$CLEANUP_JOB_CI_TAG_PREFIX$CI_COMMIT_SHORT_SHA
  WEBFRONTEND_CI_DEV_IMAGE: $CI_IMAGE_REPO/$WEBFRONTEND_DEV_IMAGE_NAME:$CLEANUP_JOB_CI_TAG_PREFIX$CI_COMMIT_SHORT_SHA
  # Once images are tested they will publish under these names to ensure that any
  # tag only runs of the pipeline can never publish untested images.
  TESTED_BACKEND_CI_IMAGE: $CI_IMAGE_REPO/$BACKEND_IMAGE_NAME:$TESTED_IMAGE_PREFIX$CI_COMMIT_SHORT_SHA
  TESTED_WEBFRONTEND_CI_IMAGE: $CI_IMAGE_REPO/$WEBFRONTEND_IMAGE_NAME:$TESTED_IMAGE_PREFIX$CI_COMMIT_SHORT_SHA
  TESTED_BACKEND_CI_DEV_IMAGE: $CI_IMAGE_REPO/$BACKEND_DEV_IMAGE_NAME:$TESTED_IMAGE_PREFIX$CI_COMMIT_SHORT_SHA
  TESTED_WEBFRONTEND_CI_DEV_IMAGE: $CI_IMAGE_REPO/$WEBFRONTEND_DEV_IMAGE_NAME:$TESTED_IMAGE_PREFIX$CI_COMMIT_SHORT_SHA
  TESTED_ALLINONE_CI_IMAGE: $CI_IMAGE_REPO/$ALLINONE_IMAGE_NAME:$TESTED_IMAGE_PREFIX$CI_COMMIT_SHORT_SHA
  TESTED_CLOUDRON_CI_IMAGE: $CI_IMAGE_REPO/$CLOUDRON_IMAGE_NAME:$TESTED_IMAGE_PREFIX$CI_COMMIT_SHORT_SHA
  TESTED_HEROKU_CI_IMAGE: $CI_IMAGE_REPO/$HEROKU_IMAGE_NAME:$TESTED_IMAGE_PREFIX$CI_COMMIT_SHORT_SHA
  # Used to tag the latest images on $DEVELOP_BRANCH_NAME
  DEVELOP_LATEST_TAG: develop-latest
  # Names of important branches used to decide when to run certain jobs.
  MASTER_BRANCH_NAME: master
  DEVELOP_BRANCH_NAME: develop
  # The locations of the various dockerfiles to build.
  BACKEND_DOCKERFILE_PATH: $CI_PROJECT_DIR/backend/Dockerfile
  WEBFRONTEND_DOCKERFILE_PATH: $CI_PROJECT_DIR/web-frontend/Dockerfile
  ALLINONE_DOCKERFILE_PATH: $CI_PROJECT_DIR/deploy/all-in-one/Dockerfile
  CLOUDRON_DOCKERFILE_PATH: $CI_PROJECT_DIR/deploy/cloudron/Dockerfile
  HEROKU_DOCKERFILE_PATH: $CI_PROJECT_DIR/heroku.Dockerfile
  # The image path for the helper CI util image that will be built and pushed to.
  CI_UTIL_IMAGE: $CI_REGISTRY/bramw/baserow/ci/ci_util_image:latest

# ==================================== CI UTIL ====================================

# A simple util image used by the other jobs containing some helper tools like git, jq,
# coverage etc.
build-ci-util-image:
  image: docker:20.10.12
  stage: build
  services:
    - docker:20.10.12-dind
  variables:
    DOCKER_BUILDKIT: 1
    DOCKER_HOST: tcp://docker:2376
    DOCKER_TLS_CERTDIR: "/certs"
  before_script:
    - echo -n $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
  script:
    - cd .gitlab/ci_util_image
    - docker build -t $CI_UTIL_IMAGE .
    - docker push $CI_UTIL_IMAGE
  # Only trigger this job manually to prevent it running every single time a new branch
  # is made. See https://gitlab.com/gitlab-org/gitlab/-/issues/11427
  when:
    manual
  # We can't use the newer rules: syntax due to https://gitlab.com/gitlab-org/gitlab/-/issues/34756
  only:
    changes:
      - .gitlab/ci_util_image/*
  except:
    refs:
      # When a pipeline is triggered by an upstream project we don't want to rebuild.
      - pipelines
      # When a pipeline is triggered by a git commit tag we don't want to rebuild.
      - tags

# ==================================== BACKEND ====================================

# If pipeline not triggered by tag :
# - Builds the backend dev image and stores in ci repo for next stages.
build-backend-image:
  extends: .build-baserow-image
  variables:
    DEV_IMAGE_NAME: $BACKEND_DEV_IMAGE_NAME
    DOCKERFILE_PATH: $BACKEND_DOCKERFILE_PATH

# If pipeline not triggered by tag:
# - Runs the backend lint if changes to the backend, otherwise skips.
backend-lint:
  extends:
    - .docker-image-test-stage
    - .skippable-job
  variables:
    RUN_WHEN_CHANGES_MADE_IN: "backend/ premium/backend/ enterprise/backend/"
  script:
    - docker run --rm $BACKEND_CI_DEV_IMAGE lint
  needs:
    - job: build-backend-image

# If pipeline not triggered by tag:
# - Runs the backend startup check if changes to the backend, otherwise skips.
backend-check-startup:
  variables:
    RUN_WHEN_CHANGES_MADE_IN: "backend/ premium/backend/ enterprise/backend/"
    POSTGRES_USER: "baserow"
    POSTGRES_PASSWORD: "baserow"
    POSTGRES_DB: "baserow"
  extends:
    - .docker-image-test-stage
    - .skippable-job
  needs:
    - job: build-backend-image
  services:
    - docker:20.10.12-dind
    - name: postgres:11
      alias: db
  script:
    - DB_IP=$(cat /etc/hosts | awk '{if ($2 == "db") print $1;}')
    - ping -w 2 $DB_IP
    - |
      docker run -e DATABASE_USER=baserow \
          -e DATABASE_NAME=baserow \
          -e DATABASE_HOST=db \
          -e DATABASE_PASSWORD=baserow \
          --rm \
          --add-host="db:$DB_IP" \
          $BACKEND_CI_DEV_IMAGE ci-check-startup;

# If pipeline not triggered by tag:
# - Runs the backend tests (the first 1/3) if changes to the backend, otherwise skips.
# - Generates coverage db's and stores as artifact for later coverage merge and report
backend-test-group-1:
  extends:
    - .docker-image-test-stage
    - .skippable-job
  services:
    - docker:20.10.12-dind
    - name: postgres:11
      alias: db
  variables:
    DOCKER_HOST: tcp://docker:2376
    DOCKER_TLS_CERTDIR: "/certs"
    POSTGRES_USER: baserow
    POSTGRES_PASSWORD: baserow
    POSTGRES_DB: baserow
    PYTEST_SPLIT_GROUP: 1
    RUN_WHEN_CHANGES_MADE_IN: "backend/ premium/backend/ enterprise/backend/"
    DOWNLOAD_AND_UNPACK_ARTIFACTS_ON_SKIP: 'true'
  script:
    - DB_IP=$(cat /etc/hosts | awk '{if ($2 == "db") print $1;}')
    - ping -w 2 $DB_IP
    - mkdir -p reports
    - TEST_TYPE=$([[ "$ENABLE_COVERAGE" = "true" ]] && echo "ci-test" || echo "test")
    - PYTEST_EXTRA_ARGS=$([[ "$RUN_MORNING_TESTS" = "true" ]] && echo '--run-once-per-day-in-ci' || echo "")
    - |
      docker run \
        -e PYTEST_SPLITS=4 \
        -e PYTEST_SPLIT_GROUP=$PYTEST_SPLIT_GROUP \
        --name=baserow_backend_test_container \
        --add-host="db:$DB_IP" \
        $BACKEND_CI_DEV_IMAGE $TEST_TYPE $PYTEST_EXTRA_ARGS;
    - docker cp baserow_backend_test_container:/baserow/backend/reports .
    - docker rm baserow_backend_test_container
    - |
      if [[ $PYTEST_SPLIT_GROUP = 1 ]]; then
        docker run -e DATABASE_USER=baserow \
          -e DATABASE_NAME=baserow \
          -e DATABASE_HOST=db \
          -e DATABASE_PASSWORD=baserow \
          --rm \
          --add-host="db:$DB_IP" \
          $BACKEND_CI_DEV_IMAGE ci-check-startup;
      fi
  needs:
    - job: build-backend-image
  artifacts:
    name: "$CI_JOB_NAME-reports"
    paths:
      - reports/
    reports:
      junit: reports/report.xml

# Create 2 more separate groups to parallelize pytest by using separate groups to
# decrease overall build time. Pytest xdist doesn't help as the gitlab saas runners only
# have a single virtual core so `pytest -n 2+` slows things down.
backend-test-group-2:
  extends: backend-test-group-1
  variables:
    PYTEST_SPLIT_GROUP: 2

backend-test-group-3:
  extends: backend-test-group-1
  variables:
    PYTEST_SPLIT_GROUP: 3

backend-test-group-4:
  extends: backend-test-group-1
  variables:
    PYTEST_SPLIT_GROUP: 4

# Collects together all the separate backend coverage databases from previous jobs and
# combines them to generate a single report for gitlab to use. GitLab itself does not
# correctly merge these if you just add them all separately into artifacts->reports->
# cobertura.
collect-backend-coverage:
  image: $CI_UTIL_IMAGE
  stage: build-final
  interruptible: true
  only:
    variables:
      - $ENABLE_COVERAGE == "true"
  # Prevent rebuilds when tagging as all we want to do is tag and push
  except:
    refs:
      - tags
  # Depend on the `reports` artifacts from the previous jobs
  needs:
    - job: backend-test-group-1
      artifacts: true
    - job: backend-test-group-2
      artifacts: true
    - job: backend-test-group-3
      artifacts: true
    - job: backend-test-group-4
      artifacts: true
  script:
    - . /baserow/venv/bin/activate
    # The reports artifacts will be extracted before the script runs into reports by
    # gitlab
    - cp reports/.coverage.* .
    - export COVERAGE_RCFILE=backend/.coveragerc
    - coverage combine
    - coverage report
    - coverage xml -o coverage.xml
  artifacts:
    reports:
      coverage_report:
        coverage_format: cobertura
        path: coverage.xml
  coverage: '/^TOTAL.+?(\d+\%)$/'

# If pipeline not triggered by tag:
# - Build and store non-dev images in CI repo under the `ci-tested` tag so we know
#   those images have passed the tests.
build-final-backend-image:
  extends: .build-final-baserow-image
  needs:
    - job: backend-check-startup
    - job: backend-test-group-1
    - job: backend-test-group-2
    - job: backend-test-group-3
    - job: backend-test-group-4
    - job: backend-lint
  variables:
    IMAGE_NAME: $BACKEND_IMAGE_NAME
    DEV_IMAGE_NAME: $BACKEND_DEV_IMAGE_NAME
    DOCKERFILE_PATH: $BACKEND_DOCKERFILE_PATH

# ==================================== WEB-FRONTEND ====================================

# If pipeline not triggered by tag:
# - Builds the web-frontend dev image and stores in ci repo for next stages.
build-web-frontend-image:
  extends: .build-baserow-image
  variables:
    DEV_IMAGE_NAME: $WEBFRONTEND_DEV_IMAGE_NAME
    DOCKERFILE_PATH: $WEBFRONTEND_DOCKERFILE_PATH

# If pipeline not triggered by tag:
# - Runs eslint and stylelint if the web-frontend code has changed, otherwise skips.
web-frontend-lint:
  extends:
    - .docker-image-test-stage
    - .skippable-job
  needs:
    - job: build-web-frontend-image
  variables:
    RUN_WHEN_CHANGES_MADE_IN: "web-frontend/ premium/web-frontend/ enterprise/web-frontend/"
  script:
    - docker run --rm $WEBFRONTEND_CI_DEV_IMAGE lint

# If pipeline not triggered by tag:
# - Runs the web-frontend tests if the web-frontend has changed, otherwise skips.
# - Generates coverage and testing reports
web-frontend-test:
  extends:
    - .docker-image-test-stage
    - .skippable-job
  variables:
    RUN_WHEN_CHANGES_MADE_IN: "web-frontend/ premium/web-frontend/ enterprise/web-frontend/"
    DOWNLOAD_AND_UNPACK_ARTIFACTS_ON_SKIP: 'true'
  needs:
    - job: build-web-frontend-image
  script:
    - mkdir reports/ -p
    - TEST_TYPE=$([[ "$ENABLE_COVERAGE" = "true" ]] && echo "ci-test" || echo "test")
    - |
      docker run --name=webfrontend_test $WEBFRONTEND_CI_DEV_IMAGE $TEST_TYPE \
        | tee reports/stdout.txt;
    - docker cp webfrontend_test:/baserow/reports .
    - docker rm webfrontend_test
  artifacts:
    paths:
      - reports/
    reports:
      junit: reports/junit.xml
      coverage_report:
        coverage_format: cobertura
        path: coverage.xml
  coverage: '/Lines\s*:\s*(\d+.?\d*)%/'

zapier-integration-test:
  extends:
    - .docker-image-test-stage
    - .skippable-job
  variables:
    RUN_WHEN_CHANGES_MADE_IN: "integrations/zapier"
  script:
    - cd integrations/zapier
    - yarn install
    - yarn run zapier test

# If pipeline not triggered by tag:
# - Build and store non-dev images in CI repo under the `ci-tested` tag so we know
#   those images have passed the tests.
build-final-web-frontend-image:
  extends: .build-final-baserow-image
  needs:
    - job: web-frontend-test
    - job: web-frontend-lint
  variables:
    IMAGE_NAME: $WEBFRONTEND_IMAGE_NAME
    DEV_IMAGE_NAME: $WEBFRONTEND_DEV_IMAGE_NAME
    DOCKERFILE_PATH: $WEBFRONTEND_DOCKERFILE_PATH


# ================================== ALL IN ONES ======================================
# If pipeline not triggered by tag:
# - Build and store the all-in-one image in CI repo under the `ci-tested` tag so we know
#   those images have passed the tests.
build-all-in-one-image:
  extends: .build-final-baserow-image
  needs:
    - job: build-final-web-frontend-image
    - job: build-final-backend-image
  only:
    variables:
      - $CI_COMMIT_BRANCH == $MASTER_BRANCH_NAME
      - $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
      - $BUILD_ALL_IN_ONE == "true"
  variables:
    IMAGE_NAME: $ALLINONE_IMAGE_NAME
    DOCKERFILE_PATH: $ALLINONE_DOCKERFILE_PATH
    BUILD_FROM_BACKEND_IMAGE: $TESTED_BACKEND_CI_IMAGE
    BUILD_FROM_WEBFRONTEND_IMAGE: $TESTED_WEBFRONTEND_CI_IMAGE

# If pipeline not triggered by tag:
# - Build and store cloudron image in CI repo under the `ci-tested` tag so we know
#   those images have passed the tests.
build-cloudron-image:
  extends: .build-final-baserow-image
  needs:
    - job: build-all-in-one-image
  only:
    variables:
      - $CI_COMMIT_BRANCH == $MASTER_BRANCH_NAME
      - $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
      - $BUILD_ALL_IN_ONE == "true"
  variables:
    IMAGE_NAME: $CLOUDRON_IMAGE_NAME
    DOCKERFILE_PATH: $CLOUDRON_DOCKERFILE_PATH
    BUILD_FROM_IMAGE: $TESTED_ALLINONE_CI_IMAGE

# We build the heroku image to simply test it builds and ensure we can have a smoke
# test in the future.
#
# If pipeline not triggered by tag:
# - Build and store heroku image in CI repo under the `ci-tested` tag so we know
#   those images have passed the tests.
build-heroku-image:
  extends: .build-final-baserow-image
  needs:
    - job: build-all-in-one-image
  only:
    variables:
      - $CI_COMMIT_BRANCH == $MASTER_BRANCH_NAME
      - $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
      - $BUILD_ALL_IN_ONE == "true"
  variables:
    IMAGE_NAME: $HEROKU_IMAGE_NAME
    DOCKERFILE_PATH: $HEROKU_DOCKERFILE_PATH
    BUILD_FROM_IMAGE: $TESTED_ALLINONE_CI_IMAGE

# ================================== TRIGGER SAAS =====================================

# Triggers a special pipeline in dependant project and passes various variables to it.
# Only on master and develop.
trigger-saas-build:
  stage: publish
  inherit:
    variables:
      - CI_COMMIT_BRANCH
      - TESTED_BACKEND_CI_IMAGE
      - TESTED_WEBFRONTEND_CI_IMAGE
      - CI_COMMIT_SHA
      - CI_COMMIT_SHORT_SHA
      - DEVELOP_BRANCH_NAME
      - MASTER_BRANCH_NAME
      - CI_IMAGE_REPO
      - BACKEND_IMAGE_NAME
      - WEBFRONTEND_IMAGE_NAME
      - TESTED_IMAGE_PREFIX
  variables:
    UPSTREAM_SHA: $CI_COMMIT_SHA
    UPSTREAM_SHORT_SHA: $CI_COMMIT_SHORT_SHA
  only:
    variables:
      - $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
  allow_failure: true
  trigger:
    project: bramw/baserow-saas
    branch: $CI_COMMIT_BRANCH

# ================================== PUSHING BACKEND ==================================

# Push baserow/backend:develop-latest
publish-backend-develop-latest-image:
  extends: .publish-baserow-image
  only:
    variables:
      - $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
  variables:
    SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $DEVELOP_BRANCH_NAME
    SOURCE_IMAGE: $TESTED_BACKEND_CI_IMAGE
    TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$BACKEND_IMAGE_NAME:$DEVELOP_LATEST_TAG"
    TARGET_REGISTRY: $RELEASE_REGISTRY
    TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
    TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER

# Push baserow/backend:$VERSION_GIT_TAG
publish-backend-release-tagged-image:
  extends: .publish-baserow-image
  only:
    refs:
      - tags
  dependencies: []
  variables:
    SKIP_IF_TAG_NOT_ON_BRANCH: $MASTER_BRANCH_NAME
    SOURCE_IMAGE: $TESTED_BACKEND_CI_IMAGE
    TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$BACKEND_IMAGE_NAME:$CI_COMMIT_TAG"
    TARGET_REGISTRY: $RELEASE_REGISTRY
    TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
    TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER

# Push baserow/backend:latest
publish-backend-latest-release-image:
  extends: .publish-baserow-image
  only:
    refs:
      - tags
  dependencies: []
  variables:
    SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $MASTER_BRANCH_NAME
    SKIP_IF_TAG_NOT_ON_BRANCH: $MASTER_BRANCH_NAME
    SOURCE_IMAGE: $TESTED_BACKEND_CI_IMAGE
    TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$BACKEND_IMAGE_NAME:latest"
    TARGET_REGISTRY: $RELEASE_REGISTRY
    TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
    TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER

# ================================ PUSHING WEB-FRONTEND ===============================

# Push baserow/web-frontend:develop-latest
publish-webfrontend-develop-latest-image:
  extends: .publish-baserow-image
  only:
    variables:
      - $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
  dependencies: []
  variables:
    SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $DEVELOP_BRANCH_NAME
    SOURCE_IMAGE: $TESTED_WEBFRONTEND_CI_IMAGE
    TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$WEBFRONTEND_IMAGE_NAME:$DEVELOP_LATEST_TAG"
    TARGET_REGISTRY: $RELEASE_REGISTRY
    TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
    TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER

# Push baserow/web-frontend:$VERSION_GIT_TAG
publish-webfrontend-release-tagged-image:
  extends: .publish-baserow-image
  only:
    refs:
      - tags
  dependencies: []
  variables:
    SKIP_IF_TAG_NOT_ON_BRANCH: $MASTER_BRANCH_NAME
    SOURCE_IMAGE: $TESTED_WEBFRONTEND_CI_IMAGE
    TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$WEBFRONTEND_IMAGE_NAME:$CI_COMMIT_TAG"
    TARGET_REGISTRY: $RELEASE_REGISTRY
    TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
    TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER

# Push baserow/web-frontend:latest
publish-webfrontend-latest-release-image:
  extends: .publish-baserow-image
  only:
    refs:
      - tags
  dependencies: []
  variables:
    SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $MASTER_BRANCH_NAME
    SKIP_IF_TAG_NOT_ON_BRANCH: $MASTER_BRANCH_NAME
    SOURCE_IMAGE: $TESTED_WEBFRONTEND_CI_IMAGE
    TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$WEBFRONTEND_IMAGE_NAME:latest"
    TARGET_REGISTRY: $RELEASE_REGISTRY
    TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
    TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER

# ================================ PUSHING ALL-IN-ONE ===============================

# Push baserow/baserow:develop-latest
publish-allinone-develop-latest-image:
  extends: .publish-baserow-image
  only:
    variables:
      - $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
  dependencies: []
  variables:
    SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $DEVELOP_BRANCH_NAME
    SOURCE_IMAGE: $TESTED_ALLINONE_CI_IMAGE
    TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$ALLINONE_IMAGE_NAME:$DEVELOP_LATEST_TAG"
    TARGET_REGISTRY: $RELEASE_REGISTRY
    TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
    TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER

# Push baserow/baserow:$VERSION_GIT_TAG
publish-allinone-release-tagged-image:
  extends: .publish-baserow-image
  only:
    refs:
      - tags
  dependencies: []
  variables:
    SKIP_IF_TAG_NOT_ON_BRANCH: $MASTER_BRANCH_NAME
    SOURCE_IMAGE: $TESTED_ALLINONE_CI_IMAGE
    TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$ALLINONE_IMAGE_NAME:$CI_COMMIT_TAG"
    TARGET_REGISTRY: $RELEASE_REGISTRY
    TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
    TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER

# Push baserow/baserow:latest
publish-allinone-latest-release-image:
  extends: .publish-baserow-image
  only:
    refs:
      - tags
  dependencies: []
  variables:
    SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $MASTER_BRANCH_NAME
    SKIP_IF_TAG_NOT_ON_BRANCH: $MASTER_BRANCH_NAME
    SOURCE_IMAGE: $TESTED_ALLINONE_CI_IMAGE
    TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$ALLINONE_IMAGE_NAME:latest"
    TARGET_REGISTRY: $RELEASE_REGISTRY
    TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
    TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER

# Push baserow/cloudron:develop-latest
publish-cloudron-develop-latest-image:
  extends: .publish-baserow-image
  only:
    variables:
      - $CI_COMMIT_BRANCH == $DEVELOP_BRANCH_NAME
  dependencies: []
  variables:
    SKIP_IF_NOT_LATEST_COMMIT_ON_BRANCH: $DEVELOP_BRANCH_NAME
    SOURCE_IMAGE: $TESTED_CLOUDRON_CI_IMAGE
    TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$CLOUDRON_IMAGE_NAME:$DEVELOP_LATEST_TAG"
    TARGET_REGISTRY: $RELEASE_REGISTRY
    TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
    TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER

# Push baserow/cloudron:$VERSION_GIT_TAG
publish-cloudron-release-tagged-image:
  extends: .publish-baserow-image
  only:
    refs:
      - tags
  dependencies: []
  variables:
    SKIP_IF_TAG_NOT_ON_BRANCH: $MASTER_BRANCH_NAME
    SOURCE_IMAGE: $TESTED_CLOUDRON_CI_IMAGE
    TARGET_IMAGE: "$RELEASE_IMAGE_REPO/$CLOUDRON_IMAGE_NAME:$CI_COMMIT_TAG"
    TARGET_REGISTRY: $RELEASE_REGISTRY
    TARGET_REGISTRY_PASSWORD: $RELEASE_REGISTRY_PASSWORD
    TARGET_REGISTRY_USER: $RELEASE_REGISTRY_USER

# We don't push any heroku images as Heroku itself will build and use heroku.Dockerfile

# ================================ MISC LINTS ===============================

docker-file-hadolint:
  extends:
    - .docker-image-test-stage
    - .skippable-job
  dependencies: []
  variables:
    RUN_WHEN_CHANGES_MADE_IN: "Dockerfile"
  script:
    - mkdir -p reports
    # Ignore the version pinning as we want security upgrades ASAP in our docker images.
    - |
      docker run --rm -i -v "$(pwd)":/opt/hadolint/. -w /opt/hadolint \
        hadolint/hadolint:2.9.3-debian \
        hadolint --ignore DL3008 -f gitlab_codeclimate \
        backend/Dockerfile \
        web-frontend/Dockerfile \
        heroku.Dockerfile \
        deploy/*/Dockerfile > reports/hadolint.json
  artifacts:
    name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG"
    expire_in: 1 day
    when: always
    reports:
      codequality:
        - "reports/*"
    paths:
      - "reports/*"

mjml-compiled-check:
  extends:
    - .docker-image-test-stage
    - .skippable-job
  variables:
    RUN_WHEN_CHANGES_MADE_IN: ".eta"
  dependencies: []
  script:
    - cd backend/email_compiler
    - yarn install
    - yarn run compile
    - |
      git diff --exit-code || \
        (echo "Uncompiled changes found to mjml email templates, run the compiler in backend/email_compiler/ and committed the changes" && exit 1)