diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..d0d8f9c9c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,202 @@ +name: Continuous Integration + +on: + push: + branches: [main, master] + pull_request: + +jobs: + lints: + name: Run linters + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + checks: write + pull-requests: write + contents: read + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Cache pre-commit + uses: actions/cache@v3 + with: + path: ~/.cache/pre-commit + key: pre-commit-3|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }} + + - name: Install pre-commit + run: pip3 install pre-commit + + - name: Run pre-commit checks + run: pre-commit run --all-files --show-diff-on-failure --color always + + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + scan-type: "fs" + ignore-unfixed: true + exit-code: 0 # change if you want to fail build on vulnerabilities + severity: "CRITICAL,HIGH,MEDIUM" + format: "table" + output: "trivy-scanning-results.txt" + + - name: Format trivy message + run: | + echo "Trivy scanning results." >> trivy.txt + cat trivy-scanning-results.txt >> trivy.txt + + - name: Add trivy report to PR + uses: thollander/actions-comment-pull-request@v2 + continue-on-error: true + if: ${{ github.event_name == 'pull_request' }} + with: + filePath: trivy.txt + reactions: "" + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + comment_tag: trivy + + - name: Create venv + run: . ./setup_dev_env.sh + + - name: Check licenses + run: | + source venv/bin/activate + ./check_licenses.sh + + - name: Generate pip freeze + run: | + source venv/bin/activate + pip freeze > requirements-freeze.txt + + - name: Publish Artefacts + uses: actions/upload-artifact@v3 + if: always() + continue-on-error: true + with: + name: results + path: | + requirements-freeze.txt + licenses.txt + trivy-scanning-results.txt + retention-days: 30 + + - name: Publish Test Report + uses: actions/upload-artifact@v3 + if: always() + continue-on-error: true + with: + name: test-report + path: report.xml + retention-days: 10 + + - name: Validate package build + run: | + source venv/bin/activate + python -m pip install -U build + for dir in packages/*/; do python -m build "$dir"; done + + - name: Publish Package + uses: actions/upload-artifact@v3 + continue-on-error: true + if: success() + with: + name: packages + path: dist/** + retention-days: 3 + + tests: + name: Run tests + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + checks: write + pull-requests: write + contents: write # required for advanced coverage reporting (to keep branch) + strategy: + fail-fast: false # do not stop all jobs if one fails + matrix: + include: + - python-version: "3.10" + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Cache Dependencies + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements-dev.txt') }}-${{ hashFiles('**/setup.cfg') }}-${{ hashFiles('**/pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Install Dependencies + run: . ./setup_dev_env.sh + + - name: Run Tests With Coverage + run: | + # run with coverage to not execute tests twice + coverage run -m pytest -v -p no:warnings --junitxml=report.xml + coverage report + coverage xml + + - name: Test Report + uses: mikepenz/action-junit-report@v4 + continue-on-error: true + if: always() + with: + report_paths: 'report.xml' + + - name: Publish Test Report + uses: actions/upload-artifact@v3 + continue-on-error: true + if: always() + with: + name: test-report + path: report.xml + retention-days: 10 + + # simpler version for code coverage reporting + # - name: Produce Coverage report + # uses: 5monkeys/cobertura-action@v13 + # continue-on-error: true + # with: + # path: coverage.xml + # minimum_coverage: 70 + # fail_below_threshold: false + + # more complex version for better coverage reporting + - name: Produce the coverage report + uses: insightsengineering/coverage-action@v2 + continue-on-error: true + with: + # Path to the Cobertura XML report. + path: coverage.xml + # Minimum total coverage, if you want to the + # workflow to enforce it as a standard. + # This has no effect if the `fail` arg is set to `false`. + threshold: 60 + # Fail the workflow if the minimum code coverage + # reuqirements are not satisfied. + fail: false + # Publish the rendered output as a PR comment + publish: true + # Create a coverage diff report. + diff: true + # Branch to diff against. + # Compare the current coverage to the coverage + # determined on this branch. + diff-branch: ${{ github.event.repository.default_branch }} + # make report togglable + togglable-report: true + # This is where the coverage reports for the + # `diff-branch` are stored. + # Branch is created if it doesn't already exist'. + diff-storage: _xml_coverage_reports + # A custom title that can be added to the code + # coverage summary in the PR comment. + coverage-summary-title: "Code Coverage Summary" diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 7211ad5d3..000000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,201 +0,0 @@ -default: - tags: - - all-ds - -variables: - PYTHON_DOCKER_IMAGE: python:3.10-buster - DOCKER_REGISTRY: $CI_REGISTRY/$CI_PROJECT_PATH - PRECOMMIT_IMAGE: $DOCKER_REGISTRY/precommit - -# run CI on default branch or MR only -workflow: - rules: - - if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH" - - if: "$CI_MERGE_REQUEST_IID" - -stages: - - preparation - - lint - - tests - - pages - - package - - deploy - -### DOCKER PREPARATION ######################################## -# Here we only rebuild docker image if the commit -# changed either dockerfile or precommit configuration -"precommit docker": - stage: preparation - image: docker:23 - interruptible: true - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" || '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' - changes: - - docker/precommit/Dockerfile - - .pre-commit-config.yaml - services: - - docker:dind - before_script: - - echo -n $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY - script: - - set -e - - docker pull $PRECOMMIT_IMAGE:latest || true - - > - docker build -t $PRECOMMIT_IMAGE:dev-$CI_COMMIT_SHA - --pull - --cache-from $PRECOMMIT_IMAGE:latest - --label "org.opencontainers.image.title=$CI_PROJECT_TITLE" - --label "org.opencontainers.image.url=$CI_PROJECT_URL" - --label "org.opencontainers.image.version=$CI_COMMIT_REF_NAME" - --label "org.opencontainers.image.created=$CI_JOB_STARTED_AT" - --label "org.opencontainers.image.revision=$CI_COMMIT_SHA" - -f docker/precommit/Dockerfile . - - docker push $PRECOMMIT_IMAGE:dev-$CI_COMMIT_SHA - -# Here, the goal is to tag the "master" branch of previous step as "latest" -"Publish precommit docker": - stage: preparation - image: docker:23 - rules: - - if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH" - changes: - - docker/precommit/Dockerfile - - .pre-commit-config.yaml - when: always - services: - - docker:dind - before_script: - - echo -n $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY - needs: ["precommit docker"] - variables: - GIT_STRATEGY: none - script: - # Because we have no guarantee that this job will be picked up by the same runner - # that built the image in the previous step, we pull it again locally - - docker pull $PRECOMMIT_IMAGE:dev-$CI_COMMIT_SHA - - docker tag $PRECOMMIT_IMAGE:dev-$CI_COMMIT_SHA $PRECOMMIT_IMAGE:latest - - docker push $PRECOMMIT_IMAGE:latest - -######################################### - -# Jobs to enforce code quality and style consistency. - -# Running linters and autoformatters with docker image. -# NOTE: using latest is not best practice but necessary -# for automation and bootstrapping a new project. -# Consider to use stable commit SHA from main branch instead -# and manual changes to reduce surprises. - -# To overcome issue that latest might not be present -# and assume that this either new project / done in -# atomic merge request and is not main branch -lint: - image: $PRECOMMIT_IMAGE:latest - stage: lint - interruptible: true - script: - - pre-commit run --all-files - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" || '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' - changes: - - docker/precommit/Dockerfile - - .pre-commit-config.yaml - when: never - - if: $CI_PIPELINE_SOURCE == "merge_request_event" || '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' - when: always - -# this runs whenever docker is prebuilt -# instead of standard lint job -lint-precommit-changed: - image: $PRECOMMIT_IMAGE:dev-$CI_COMMIT_SHA - stage: lint - interruptible: true - script: - - pre-commit run --all-files - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - changes: - - docker/precommit/Dockerfile - - .pre-commit-config.yaml - when: always - - if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH" - when: never - -############################################# - -# Running linters and autoformatters without precommit docker image. -# You can remove preparation stage and use this job instead: -# lint: -# image: $PYTHON_DOCKER_IMAGE -# stage: lint -# interruptible: true -# before_script: -# - pip install pre-commit -# - pre-commit install -# script: -# - pre-commit run --all-files - -########################################### - -# Job to run pytest with code coverage + license check -# TODO make installing bump2version below conditional on cookiecutter parameter choice -tests: - image: $PYTHON_DOCKER_IMAGE - stage: tests - interruptible: true - before_script: - - . setup_dev_env.sh - - pip install build - script: - # fused check license here as it would take more time to do it in different place. - - ./check_licenses.sh - # test package building - - for dir in packages/*/; do python -m build "$dir"; done - # generate pip freeze - - pip freeze > requirements-freeze.txt - - cat requirements-freeze.txt - # run with coverage to not execute tests twice - - coverage run -m pytest -v -p no:warnings --junitxml=report.xml - - coverage report - - coverage xml - coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/' - artifacts: - when: always - paths: - - licenses.txt - - requirements-freeze.txt - reports: - junit: "report.xml" - coverage_report: - coverage_format: cobertura - path: coverage.xml - expire_in: "30 days" - -# Job for scanning for vulnerabilities: -"trivy security scan": - image: $PYTHON_DOCKER_IMAGE - stage: tests - interruptible: true - before_script: - - export TRIVY_VERSION=$(wget -qO - "https://api.github.com/repos/aquasecurity/trivy/releases/latest" | grep '"tag_name":' | sed -E 's/.*"v([^"]+)".*/\1/') - - echo $TRIVY_VERSION - - wget --no-verbose https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz -O - | tar -zxvf - - allow_failure: true - script: - # Build report - - ./trivy fs --exit-code 0 --cache-dir .trivycache/ --no-progress --format template --template "@contrib/gitlab.tpl" -o gl-scanning-report.json ./ - # Build html report - - ./trivy fs --exit-code 0 --cache-dir .trivycache/ --no-progress --format template --template "@contrib/html.tpl" -o gl-scanning-report.html ./ - # Print report - - ./trivy fs --exit-code 0 --cache-dir .trivycache/ --no-progress ./ - # Fail on severe vulnerabilities - - ./trivy fs --exit-code 1 --cache-dir .trivycache/ --severity CRITICAL --no-progress ./ - cache: - paths: - - .trivycache/ - artifacts: - paths: - - gl-scanning-report.html - reports: - dependency_scanning: gl-scanning-report.json - expire_in: never