chore: trigger release process

.github/workflows/artifact.yml

@@ -34,7 +34,9 @@ jobs:
         run: "gcloud info"
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/[email protected]
+        with:
+          image: tonistiigi/binfmt:qemu-v7.0.0-28
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3

.github/workflows/release.yaml

@@ -2,7 +2,10 @@ name: Release
 
 "on":
   push:
-    branches: ["main", "release/**", "dev"]
+    branches:
+      - main
+
+  workflow_dispatch:
 
 concurrency:
   group: deploy
@@ -13,19 +16,15 @@ env:
 
 jobs:
   release:
+    # Ensure the workflow can be run only from main & dev branches!
+    if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' }}
     runs-on: ubuntu-latest
-    concurrency: release
     outputs:
       released: ${{ steps.semrelease.outputs.released }}
     permissions:
-      # NOTE: this enables trusted publishing.
-      # See https://github.com/pypa/gh-action-pypi-publish/tree/release/v1#trusted-publishing
-      # and https://blog.pypi.org/posts/2023-04-20-introducing-trusted-publishers/
       id-token: write
       contents: write
-
     steps:
-      # NOTE: commits using GITHUB_TOKEN does not trigger workflows
       - uses: actions/create-github-app-token@v1
         id: trigger-token
         with:
@@ -34,77 +33,62 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+          ref: ${{ github.ref_name }}
           repository: opentargets/gentropy
           token: ${{ secrets.GITHUB_TOKEN }}
           persist-credentials: false
-
-      - name: Python Semantic Release
+      - uses: python-semantic-release/[email protected]
         id: semrelease
-        uses: python-semantic-release/[email protected]
         with:
           github_token: ${{ steps.trigger-token.outputs.token }}
-
-      - name: Publish package to GitHub Release
-        uses: python-semantic-release/upload-to-gh-release@main
-        # NOTE: semrelease output is a string, so we need to compare it to a string
-        if: steps.semrelease.outputs.released == 'true'
+      - uses: python-semantic-release/[email protected]
+        if: ${{ steps.semrelease.outputs.released }} == 'true'
         with:
-          # NOTE: allow to start the workflow when push action on tag gets executed
-          # requires using GH_APP to authenitcate, otherwise push authorised with
-          # the GITHUB_TOKEN does not trigger the tag artifact workflow.
-          # see https://github.com/actions/create-github-app-token
-          github_token: ${{ secrets.GITHUB_TOKEN }}
+          github_token: ${{ steps.trigger-token.outputs.token }}
           tag: ${{ steps.semrelease.outputs.tag }}
-
-      - name: Store the distribution packages
+      - uses: actions/upload-artifact@v4
         if: steps.semrelease.outputs.released == 'true'
-        uses: actions/upload-artifact@v4
         with:
           name: python-package-distributions
           path: dist/
 
-  publish-to-pypi:
+  publish-to-testpypi:
+    name: Publish 📦 in TestPyPI
     needs: release
-    name: Publish 📦 in PyPI
     if: github.ref == 'refs/heads/main' && needs.release.outputs.released == 'true'
     runs-on: ubuntu-latest
     environment:
-      name: pypi
-      url: https://pypi.org/p/gentropy
+      name: testpypi
+      url: https://test.pypi.org/p/gentropy
     permissions:
       id-token: write # IMPORTANT: mandatory for trusted publishing
     steps:
-      - name: Download all the dists
-        uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v4
         with:
           name: python-package-distributions
           path: dist/
-      - name: Publish distribution 📦 to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
+      - uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/
 
-  publish-to-testpypi:
-    name: Publish 📦 in TestPyPI
-    needs: release
+  publish-to-pypi:
+    needs:
+      - release
+      - publish-to-testpypi
+    name: Publish 📦 in PyPI
     if: github.ref == 'refs/heads/main' && needs.release.outputs.released == 'true'
     runs-on: ubuntu-latest
-
     environment:
-      name: testpypi
-      url: https://test.pypi.org/p/gentropy
-
+      name: pypi
+      url: https://pypi.org/p/gentropy
     permissions:
       id-token: write # IMPORTANT: mandatory for trusted publishing
-
     steps:
-      - name: Download all the dists
-        uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v4
         with:
           name: python-package-distributions
           path: dist/
-      - name: Publish distribution 📦 to TestPyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          repository-url: https://test.pypi.org/legacy/
+      - uses: pypa/gh-action-pypi-publish@release/v1
 
   documentation:
     needs: release
@@ -115,23 +99,10 @@ jobs:
         with:
           fetch-depth: 0
           token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Set up Python
-        uses: actions/setup-python@v4
+      - uses: actions/setup-python@v4
         with:
           python-version: ${{ env.PYTHON_VERSION_DEFAULT }}
-      - name: Install uv
-        uses: astral-sh/setup-uv@v5
-      - name: Load cached venv
-        id: cached-dependencies
-        uses: actions/cache@v4
-        with:
-          path: .venv
-          key: |
-            venv-${{ runner.os }}-\
-            ${{ env.PYTHON_VERSION_DEFAULT }}-\
-            ${{ hashFiles('**/uv.lock') }}
-      - name: Install dependencies
-        if: steps.cached-dependencies.outputs.cache-hit != 'true'
-        run: uv sync --group docs
+      - uses: astral-sh/setup-uv@v5
+      - run: uv sync --group docs
       - name: Publish docs
         run: uv run mkdocs gh-deploy --force

.pre-commit-config.yaml

@@ -5,7 +5,7 @@ ci:
   autofix_commit_msg: "chore: pre-commit auto fixes [...]"
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.7.4
+    rev: v0.9.6
     hooks:
       - id: ruff
         args:
@@ -57,14 +57,14 @@ repos:
         exclude: "CHANGELOG.md"
 
   - repo: https://github.com/alessandrojcm/commitlint-pre-commit-hook
-    rev: v9.18.0
+    rev: v9.21.0
     hooks:
       - id: commitlint
         additional_dependencies: ["@commitlint/[email protected]"]
         stages: [commit-msg]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: "v1.13.0"
+    rev: "v1.15.0"
     hooks:
       - id: mypy
         args:
@@ -97,10 +97,10 @@ repos:
       - id: beautysh
 
   - repo: https://github.com/jsh9/pydoclint
-    rev: 0.5.9
+    rev: 0.6.2
     hooks:
       - id: pydoclint
   - repo: https://github.com/astral-sh/uv-pre-commit
-    rev: 0.5.22
+    rev: 0.6.1
     hooks:
       - id: uv-lock

Makefile

@@ -4,12 +4,13 @@ REGION ?= europe-west1
 APP_NAME ?= $$(cat pyproject.toml | grep -m 1 "name" | cut -d" " -f3 | sed  's/"//g')
 PACKAGE_VERSION ?= $(shell grep -m 1 'version = ' pyproject.toml | sed 's/version = "\(.*\)"/\1/')
 USER_SAFE ?= $(shell echo $(USER) | tr '[:upper:]' '[:lower:]')
+CLUSTER_TIMEOUT ?= 60m
 # NOTE: git rev-parse will always return the HEAD if it sits in the tag,
 # this way we can distinguish the tag vs branch name
 ifeq ($(shell git rev-parse --abbrev-ref HEAD),HEAD)
-	REF := $(shell git describe --exact-match --tags)
+	REF ?= $(shell git describe --exact-match --tags)
 else
-	REF := $(shell git rev-parse --abbrev-ref HEAD)
+	REF ?= $(shell git rev-parse --abbrev-ref HEAD)
 endif
 
 CLEAN_PACKAGE_VERSION := $(shell echo "$(PACKAGE_VERSION)" | tr -cd '[:alnum:]')
@@ -54,8 +55,8 @@ sync-gentropy-cli-script: ## Synchronize the gentropy cli script
 	@gcloud storage cp src/gentropy/cli.py ${BUCKET_NAME}/cli.py
 
 create-dev-cluster: sync-cluster-init-script sync-gentropy-cli-script ## Spin up a simple dataproc cluster with all dependencies for development purposes
-	@echo "Making sure the branch is in sync with remote, so cluster can install gentropy dev version..."
-	@./utils/clean_status.sh || (echo "ERROR: Commit and push or stash local changes, to have up to date cluster"; exit 1)
+	@echo "Making sure the cluster can reference to ${REF} branch to install gentropy..."
+	@./utils/clean_status.sh ${REF} || (echo "ERROR: Commit and push local changes, to have up to date cluster"; exit 1)
 	@echo "Creating Dataproc Dev Cluster"
 	gcloud config set project ${PROJECT_ID}
 	gcloud dataproc clusters create "ot-genetics-dev-${CLEAN_PACKAGE_VERSION}-$(USER_SAFE)" \
@@ -72,7 +73,7 @@ create-dev-cluster: sync-cluster-init-script sync-gentropy-cli-script ## Spin up
 		--optional-components=JUPYTER \
 		--enable-component-gateway \
 		--labels team=open-targets,subteam=gentropy,created_by=${USER_SAFE},environment=development, \
-		--max-idle=60m
+		--max-idle=${CLUSTER_TIMEOUT}
 
 update-dev-cluster: build ## Reinstalls the package on the dev-cluster
 	@echo "Updating Dataproc Dev Cluster"

docs/development/contributing.md

@@ -84,3 +84,27 @@ For more details on each of these steps, see the sections below.
 ### Support for python versions
 
 As of version 2.1.X gentropy supports multiple python versions. To ensure compatibility with all supported versions, unit tests are run for each of the minor python release from 3.10 to 3.12. Make sure your changes are compatible with all supported versions.
+
+### Development process
+
+The development follows simplified Git Flow process that includes usage of
+
+- `dev` (development branch)
+- `feature` branches
+- `main` (production branch)
+
+The development starts with creating new `feature` branch based on the `dev` branch. Once the feature is ready, the Pull Request for the `dev` branch is created and CI/CD Checks are performed to ensure that the code is compliant with the project conventions. Once the PR is approved, the feature branch is merged into the `dev` branch.
+
+#### Development releases
+
+One can create the dev release tagged by `vX.Y.Z-dev.V` tag. This release will not trigger the CI/CD pipeline to publish the package to the PyPi repository. The release is done by triggering the `Release` GitHub action.
+
+#### Production releases
+
+Once per week, the `Trigger PR for release` github action creates a Pull Request from `dev` to `main` branch, when the PR is approved, the `Release` GitHub action is triggered to create a production release tagged by `vX.Y.Z` tag. This release triggers the CI/CD pipeline to publish the package to the _TestPyPi_ repository. If it is successful, then the actual deployment to the _PyPI_ repository is done. The deployment to the PyPi repository must be verified by the gentropy maintainer.
+
+Below you can find a simplified diagram of the development process.
+
+<div align="center">
+  <img width="800" height="400" src="../../assets/imgs/development-flow.png" alt="development process">
+</div>

docs/development/troubleshooting.md

@@ -42,19 +42,45 @@ This can be resolved by adding the follow line to your `~/.zshrc`:
 
 ## Creating development dataproc cluster (OT users only)
 
-To start dataproc cluster in the development mode run
+!!! info "Requirements"
+
+    To create the cluster, you need to auth to the google cloud
+
+    ```bash
+    gcloud auth login
+    ```
+
+To start dataproc cluster in the development mode run.
 
 ```bash
-make create-dev-cluster
+make create-dev-cluster REF=dev
 ```
 
-!!! note "Tip"
-This command will work, provided you have fully commited and pushed all your changes to the remote repository.
+`REF` - remote branch available at the [gentropy repository](https://github.com/opentargets/gentropy)
+
+During cluster [initialization actions](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/init-actions#important_considerations_and_guidelines) the `utils/install_dependencies_on_cluster.sh` script is run, that installs `gentropy` package from the remote repository by using VCS support, hence it does not require the **gentropy package whl artifact** to be prepared in the Google Cloud Storage before the make command can be run.
+
+Check details how to make a package installable by VCS in [pip documentation](https://pip.pypa.io/en/stable/topics/vcs-support/).
+
+!!! note "How `create-dev-cluster` works"
+
+    This command will work, provided you have done one of:
+
+    - run `make create-dev-cluster REF=dev`, since the REF is requested, the cluster will attempt to install it from the remote repository.
+    - run `make create-dev-cluster` without specifying the REF or specifying REF that points to your local branch will request branch name you are checkout on your local repository, if any changes are pending locally, the cluster can not be created, it requires stashing or pushing the changes to the remote.
+
+    The command will create a new dataproc cluster with the following configuration:
+
+    - package installed from the requested **REF** (for example `dev` or `feature/xxx`)
+    - uv installed in the cluster (to speed up the installation and dependency resolution process)
+    - cli script to run gentropy steps
+
+!!! tip "Dataproc cluster timeout"
 
-The command will create a new dataproc cluster with the following configuration:
+    By default the cluster will **delete itself** when running for **60 minutes after the last submitted job to the cluster was successfully completed** (running jobs interactively via Jupyter or Jupyter lab is not treated as submitted job). To preserve the cluster for arbitrary period (**for instance when the cluster is used only for interactive jobs**) increase the cluster timeout:
 
-- package installed from the current branch you are checkout on (for example `dev` or `feature/xxx`)
-- uv installed in the cluster (to speed up the installation and dependency resolution process)
-- cli script to run gentropy steps
+    ```bash
+    make create-dev-cluster CLUSTER_TIMEOUT=1d REF=dev # 60m 1h 1d (by default 60m)
+    ```
 
-This process requires gentropy to be installable by git repository - see VCS support in [pip documentation](https://pip.pypa.io/en/stable/topics/vcs-support/).
+    For the reference on timeout format check [gcloud documentation](https://cloud.google.com/sdk/gcloud/reference/dataproc/clusters/create#--max-idle)

docs/python_api/datasources/ukb_ppp_eur/_ukb_ppp_eur.md

@@ -1,7 +1,8 @@
 ---
-title: UKB-PPP (EUR)
+title: UK Biobank Pharma Proteomics Project (UKB-PPP) (EUR)
 ---
 
 The UKB-PPP is a collaboration between the UK Biobank (UKB) and thirteen biopharmaceutical companies characterising the plasma proteomic profiles of 54,219 UKB participants.
 
-The original data is available at https://www.synapse.org/#!Synapse:syn51364943/. The associated paper is https://www.nature.com/articles/s41586-023-06592-6.
+The original data is available here: https://www.synapse.org/Synapse:syn51364943/wiki/622119.
+The associated paper is https://www.nature.com/articles/s41586-023-06592-6.

pyproject.toml

@@ -22,7 +22,7 @@ dependencies = [
   "skops (>=0.11.0, <0.12.0)",
   "shap (>=0.46, <0.47)",
   "matplotlib (>=3.10.0, <3.11.0)",
-  "google-cloud-secret-manager (>=2.12.6, <2.13.0)",
+  "google-cloud-secret-manager (>=2.12.6, <2.24.0)",
   "google-cloud-storage (>=2.14.0, <3.1.0)",
 ]
 classifiers = [
@@ -92,28 +92,20 @@ packages = ["src/gentropy"]
 match = "(main|master)"
 prerelease = false
 
-[tool.semantic_release.branches."release"]
-match = "release/*"
+[tool.semantic_release.branches.dev]
+match = "dev"
 prerelease = true
 prerelease_token = "rc"
 
 [tool.semantic_release.publish]
 dist_glob_patterns = ["dist/*"]
 upload_to_vcs_release = true
 
-[tool.semantic_release.changelog]
+[tool.semantic-release.changelog.default_templates]
 changelog_file = "CHANGELOG.md"
-exclude_commit_patterns = ["chore\\(release\\):"]
-
-[tool.semantic_release.branches."step"]
-match = "(build|chore|ci|docs|feat|fix|perf|style|refactor|test)"
-prerelease = true
-prerelease_token = "alpha"
 
-[tool.semantic_release.branches."dev"]
-match = "dev"
-prerelease = true
-prerelease_token = "dev"
+[tool.semantic-release.changelog]
+exclude_commit_patterns = ["chore\\(release\\):"]
 
 [build-system]
 requires = ["hatchling"]

src/gentropy/assets/schemas/amino_acid_variants.json

@@ -14,7 +14,7 @@
     },
     {
       "metadata": {},
-      "name": "inSilicoPredictors",
+      "name": "variantEffect",
       "nullable": true,
       "type": {
         "containsNull": true,