diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c7b5353f..ad551177 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,11 +13,12 @@ jobs: permissions: checks: write pull-requests: write + contents: read steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: - python-version: "3.11" + python-version: "3.8" - name: Cache pre-commit uses: actions/cache@v3 @@ -60,7 +61,9 @@ jobs: run: . ./setup_dev_env.sh - name: Check licenses - run: ./check_licenses.sh + run: | + source venv/bin/activate + ./check_licenses.sh - name: Generate pip freeze run: | @@ -115,7 +118,7 @@ jobs: fail-fast: false # do not stop all jobs if one fails matrix: include: - - python-version: "3.11" + - python-version: "3.8" steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 298ce311..be7f555d 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -6,8 +6,7 @@ on: jobs: pages: - runs-on: ubuntu-latest - container: python:3.11 + runs-on: gcr.io/google.com/cloudsdktool/google-cloud-cli:latest permissions: contents: write steps: @@ -27,10 +26,6 @@ jobs: run: . ./setup_dev_env.sh - name: Build docs - run: ./build_docs.sh - - - name: Deploy - uses: peaceiris/actions-gh-pages@v3 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: ./public + run: | + source venv/bin/activate + ./.github/scripts/build_docs.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 21230ce0..00000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,291 +0,0 @@ -default: - tags: - - all-ds - -variables: - PYTHON_DOCKER_IMAGE: python:3.8-buster - DOCKER_REGISTRY: $CI_REGISTRY/$CI_PROJECT_PATH - PRECOMMIT_IMAGE: $DOCKER_REGISTRY/precommit - TESTS_IMAGE: $DOCKER_REGISTRY/tests - -# run CI on default branch or MR only -workflow: - rules: - - if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH" - - if: "$CI_MERGE_REQUEST_IID" - -stages: - - preparation - - lint - - tests - - benchmark - - docs - - package - - deploy - -### DOCKER PREPARATION ######################################## -# Here we only rebuild docker image if the commit -# changed either dockerfile or precommit configuration -"precommit docker": - stage: preparation - image: docker:23 - interruptible: true - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" || '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' - changes: - - docker/precommit/Dockerfile - - .pre-commit-config.yaml - services: - - docker:dind - before_script: - - echo -n $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY - script: - - set -e - - docker pull $PRECOMMIT_IMAGE:latest || true - - > - docker build -t $PRECOMMIT_IMAGE:dev-$CI_COMMIT_SHA - --pull - --cache-from $PRECOMMIT_IMAGE:latest - --label "org.opencontainers.image.title=$CI_PROJECT_TITLE" - --label "org.opencontainers.image.url=$CI_PROJECT_URL" - --label "org.opencontainers.image.version=$CI_COMMIT_REF_NAME" - --label "org.opencontainers.image.created=$CI_JOB_STARTED_AT" - --label "org.opencontainers.image.revision=$CI_COMMIT_SHA" - -f docker/precommit/Dockerfile . - - docker push $PRECOMMIT_IMAGE:dev-$CI_COMMIT_SHA - -# Here, the goal is to tag the "master" branch of previous step as "latest" -"Publish precommit docker": - stage: preparation - image: docker:23 - rules: - - if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH" - changes: - - docker/precommit/Dockerfile - - .pre-commit-config.yaml - when: always - services: - - docker:dind - before_script: - - echo -n $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY - needs: ["precommit docker"] - variables: - GIT_STRATEGY: none - script: - # Because we have no guarantee that this job will be picked up by the same runner - # that built the image in the previous step, we pull it again locally - - docker pull $PRECOMMIT_IMAGE:dev-$CI_COMMIT_SHA - - docker tag $PRECOMMIT_IMAGE:dev-$CI_COMMIT_SHA $PRECOMMIT_IMAGE:latest - - docker push $PRECOMMIT_IMAGE:latest - -"tests docker": - stage: preparation - image: docker:23 - interruptible: true - services: - - docker:dind - before_script: - - echo -n $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY - script: - - set -e - - docker pull $TESTS_IMAGE:latest || true - - > - docker build -t $TESTS_IMAGE:dev-$CI_COMMIT_SHA - --pull - --cache-from $TESTS_IMAGE:latest - --label "org.opencontainers.image.title=$CI_PROJECT_TITLE" - --label "org.opencontainers.image.url=$CI_PROJECT_URL" - --label "org.opencontainers.image.version=$CI_COMMIT_REF_NAME" - --label "org.opencontainers.image.created=$CI_JOB_STARTED_AT" - --label "org.opencontainers.image.revision=$CI_COMMIT_SHA" - -f docker/tests/Dockerfile . - - docker push $TESTS_IMAGE:dev-$CI_COMMIT_SHA - -######################################### - -"Deploy terraform": - image: - name: google/cloud-sdk:latest - entrypoint: [""] - stage: deploy - script: - - chmod +x .gitlab/ci/terraform_apply.sh - - sh -x .gitlab/ci/terraform_apply.sh dev - rules: - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - -######################################### - -# Jobs to enforce code quality and style consistency. - -# Running linters and autoformatters with docker image. -# NOTE: using latest is not best practice but necessary -# for automation and bootstrapping a new project. -# Consider to use stable commit SHA from main branch instead -# and manual changes to reduce surprises. - -# To overcome issue that latest might not be present -# and assume that this either new project / done in -# atomic merge request and is not main branch -lint: - image: $PRECOMMIT_IMAGE:latest - stage: lint - interruptible: true - script: - - pre-commit run --all-files - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" || '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' - changes: - - docker/precommit/Dockerfile - - .pre-commit-config.yaml - when: never - - if: $CI_PIPELINE_SOURCE == "merge_request_event" || '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' - when: always - -# this runs whenever docker is prebuilt -# instead of standard lint job -lint-precommit-changed: - image: $PRECOMMIT_IMAGE:dev-$CI_COMMIT_SHA - stage: lint - interruptible: true - script: - - pre-commit run --all-files - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - changes: - - docker/precommit/Dockerfile - - .pre-commit-config.yaml - when: always - - if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH" - when: never - -############################################# - -# Running linters and autoformatters without precommit docker image. -# You can remove preparation stage and use this job instead: -# lint: -# image: $PYTHON_DOCKER_IMAGE -# stage: lint -# interruptible: true -# before_script: -# - pip install pre-commit -# - pre-commit install -# script: -# - pre-commit run --all-files - -########################################### - -# Job to run pytest with code coverage + license check -tests: - image: $TESTS_IMAGE:dev-$CI_COMMIT_SHA - stage: tests - interruptible: true - script: - # fused check license here as it would take more time to do it in different place. - - ./check_licenses.sh - # test package building - - echo ${CI_PIPELINE_IID} >> src/dbally/VERSION - - python -m build - # generate pip freeze - - pip freeze > requirements-freeze.txt - - cat requirements-freeze.txt - # test type hints - - mypy . - # run with coverage to not execute tests twice - - coverage run -m pytest -v -p no:warnings --junitxml=report.xml tests/ - - coverage report - - coverage xml - coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/' - artifacts: - when: always - paths: - - licenses.txt - - requirements-freeze.txt - - dist/* - reports: - junit: "report.xml" - coverage_report: - coverage_format: cobertura - path: coverage.xml - expire_in: "30 days" - -# Job for running benchmark -benchmark: - image: $TESTS_IMAGE:dev-$CI_COMMIT_SHA - stage: benchmark - interruptible: true - script: - - PYTHONPATH=benchmark/ python benchmark/dbally_benchmark/evaluate.py neptune.log=True - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" || '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' - when: manual - -# Job for scanning for vulnerabilities: -"trivy security scan": - image: $PYTHON_DOCKER_IMAGE - stage: tests - interruptible: true - before_script: - - export TRIVY_VERSION=$(wget -qO - "https://api.github.com/repos/aquasecurity/trivy/releases/latest" | grep '"tag_name":' | sed -E 's/.*"v([^"]+)".*/\1/') - - echo $TRIVY_VERSION - - wget --no-verbose https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz -O - | tar -zxvf - - allow_failure: true - script: - # Build report - - ./trivy fs --exit-code 0 --cache-dir .trivycache/ --no-progress --format template --template "@contrib/gitlab.tpl" -o gl-scanning-report.json ./ - # Build html report - - ./trivy fs --exit-code 0 --cache-dir .trivycache/ --no-progress --format template --template "@contrib/html.tpl" -o gl-scanning-report.html ./ - # Print report - - ./trivy fs --exit-code 0 --cache-dir .trivycache/ --no-progress ./ - # Fail on severe vulnerabilities - - ./trivy fs --exit-code 1 --cache-dir .trivycache/ --severity CRITICAL --no-progress ./ - cache: - paths: - - .trivycache/ - artifacts: - paths: - - gl-scanning-report.html - reports: - dependency_scanning: gl-scanning-report.json - expire_in: never - -############################################## - -# Job for building & deploying to pip registry -# version builded from tests job. -"Deploy test dev package to PIP registry": - image: $TESTS_IMAGE:dev-$CI_COMMIT_SHA - stage: package - needs: [tests] - environment: testing - when: manual - allow_failure: true - script: - - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi dist/* - -# Job to build and deploy release version - note: cannot deploy two same versions! -"Release package to PIP registry": - image: $TESTS_IMAGE:dev-$CI_COMMIT_SHA - stage: deploy - environment: release - when: manual - allow_failure: true - needs: [tests] - script: - - bump2version --verbose release - - python -m build - - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --repository-url ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi dist/* - rules: - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - -############################################## - -# Job to build mkdocs docs -docs: - image: gcr.io/google.com/cloudsdktool/google-cloud-cli:latest - stage: docs - interruptible: true - script: - - ./.gitlab/ci/build_docs.sh - rules: - - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'