chore: deprecate older version of cellxgene schema (#172)

## Reason for Change - automate the cellxgene schema versions deprecation process. - #170 ## Changes - add deprecation warning to API when older version of the cellxgene schema are used. - add logic to all_ontology_generator.py to remove expired ontology files from repo, expired cellxgene schema versions from ontology_info.json, and deprecate the previous cellxgene schema version if a new version exists. ## Testing steps - updated unit tests - Need to verify that files will be removed by the GHA. ## Notes for Reviewer
chanzuckerberg · Apr 6, 2024 · 186e762 · 186e762
1 parent c0aec23
commit 186e762
Show file tree

Hide file tree

Showing 9 changed files with 340 additions and 110 deletions.
diff --git a/.github/workflows/generate_all_ontology.yml b/.github/workflows/generate_all_ontology.yml
@@ -41,6 +41,7 @@ jobs:
         run: |
           python3 ./tools/ontology-builder/src/all_ontology_generator.py
           git add ./ontology-assets/*.json.gz
+          git add ./ontology-assets/ontology_info.json
       - name: Commit
         run: |
           git commit -m "AUTO: update ontologies"

diff --git a/README.md b/README.md
@@ -16,3 +16,8 @@ This project adheres to the Contributor Covenant [code of conduct](https://githu
 ## Reporting Security Issues
 
 If you believe you have found a security issue, please responsibly disclose by contacting us at [[email protected]](mailto:[email protected]).
+
+## Updating to a new Cellxgene Schema Version
+
+1. Update the [ontology_info.json](./ontology-assets/ontology_info.json) file with the new schema version
+2. Leave the older versions in the file for backward compatibility. They will be deprecated and removed automatically after 6 months. That process is handled in [deprecate_previous_cellxgene_schema_versions](https://github.com/chanzuckerberg/cellxgene-ontology/blob/main/tools/ontology-builder/src/all_ontology_generator.py#L311-L311).
diff --git a/api/python/src/cellxgene_ontology_guide/supported_versions.py b/api/python/src/cellxgene_ontology_guide/supported_versions.py
@@ -2,6 +2,8 @@
 import gzip
 import json
 import os
+import warnings
+from datetime import datetime
 from typing import Any, Dict, List, Optional
 
 from semantic_version import Version
@@ -63,8 +65,16 @@ def __init__(self, version: Optional[str] = None):
             raise ValueError(f"Schema version {version} is not supported in this package version.")
 
         self.version = version
-        self.supported_ontologies = ontology_info[version]
+        self.supported_ontologies = ontology_info[version]["ontologies"]
         self.ontology_file_names: Dict[str, str] = {}
+        self.deprecated_on = ontology_info[version].get("deprecated_on")
+        if self.deprecated_on:
+            parsed_date = datetime.strptime(self.deprecated_on, "%Y-%m-%d")
+            warnings.warn(
+                f"Schema version {version} is deprecated as of {parsed_date}. It will be removed in a future version.",
+                DeprecationWarning,
+                stacklevel=1,
+            )
 
     def ontology(self, name: str) -> Any:
         """Return the ontology terms for the given ontology name. Load from the file cache if available.

diff --git a/api/python/tests/test_ontology_parser.py b/api/python/tests/test_ontology_parser.py
@@ -43,7 +43,9 @@ def ontology_dict():
 @pytest.fixture
 def mock_CXGSchema(ontology_dict, mock_load_supported_versions, mock_load_ontology_file):
     mock_load_supported_versions.return_value = {
-        "v5.0.0": {"CL": {"version": "2024-01-01", "source": "http://example.com", "filename": "cl.owl"}}
+        "v5.0.0": {
+            "ontologies": {"CL": {"version": "2024-01-01", "source": "http://example.com", "filename": "cl.owl"}}
+        }
     }
     cxg_schema = CXGSchema()
     cxg_schema.ontology_file_names = {"CL": "CL-ontology-2024-01-01.json.gz"}

diff --git a/api/python/tests/test_supported_versions.py b/api/python/tests/test_supported_versions.py
@@ -17,7 +17,9 @@
 @pytest.fixture
 def initialized_CXGSchemaInfo(mock_load_supported_versions):
     mock_load_supported_versions.return_value = {
-        "v5.0.0": {"CL": {"version": "v2024-01-01", "source": "http://example.com", "filename": "cl.owl"}}
+        "v5.0.0": {
+            "ontologies": {"CL": {"version": "v2024-01-01", "source": "http://example.com", "filename": "cl.owl"}}
+        }
     }
     return CXGSchema()
 
@@ -71,18 +73,29 @@ def test__load_supported_versions__OK(tmpdir):
 
 class TestCXGSchema:
     def test__init__defaults(self, mock_load_supported_versions):
-        support_versions = {"v5.0.0": "current version", "v0.0.1": "old version"}
+        support_versions = {"v5.0.0": {"ontologies": {}}, "v0.0.1": {"ontologies": {}}}
         mock_load_supported_versions.return_value = support_versions
         cxgs = CXGSchema()
         assert cxgs.version == "v5.0.0"
-        assert cxgs.supported_ontologies == support_versions["v5.0.0"]
+        assert cxgs.supported_ontologies == support_versions["v5.0.0"]["ontologies"]
 
     def test__init__specific_version(self, mock_load_supported_versions):
-        support_versions = {"v5.0.0": "current version", "v0.0.1": "old version"}
+        support_versions = {"v5.0.0": {"ontologies": {}}, "v0.0.1": {"ontologies": {}}}
         mock_load_supported_versions.return_value = support_versions
         cxgs = CXGSchema(version="v0.0.1")
         assert cxgs.version == "v0.0.1"
-        assert cxgs.supported_ontologies == support_versions["v0.0.1"]
+        assert cxgs.supported_ontologies == support_versions["v0.0.1"]["ontologies"]
+
+    def test__init__deprecated_version(self, mock_load_supported_versions):
+        support_versions = {"v5.0.0": {"ontologies": {}}, "v0.0.1": {"ontologies": {}, "deprecated_on": "2024-01-01"}}
+        mock_load_supported_versions.return_value = support_versions
+        # catch the deprecation warning
+        with pytest.warns(DeprecationWarning) as record:
+            CXGSchema(version="v0.0.1")
+        warning = record.pop()
+        assert warning.message.args[0] == (
+            "Schema version v0.0.1 is deprecated as of 2024-01-01 00:00:00. It will be removed in a " "future version."
+        )
 
     def test__init__unsupported_version(self, mock_load_supported_versions):
         mock_load_supported_versions.return_value = {}

diff --git a/artifact-schemas/ontology_info_schema.json b/artifact-schemas/ontology_info_schema.json
@@ -8,34 +8,46 @@
       "description": "The version of CellxGene schema that maps to this set of ontology versions",
       "type": "object",
       "properties": {
-        "CL": {
-          "$ref": "#/definitions/ontologyEntry"
-        },
-        "EFO": {
-          "$ref": "#/definitions/ontologyEntry"
-        },
-        "HANCESTRO": {
-          "$ref": "#/definitions/ontologyEntry"
-        },
-        "HsapDv": {
-          "$ref": "#/definitions/ontologyEntry"
-        },
-        "MONDO": {
-          "$ref": "#/definitions/ontologyEntry"
-        },
-        "MmusDv": {
-          "$ref": "#/definitions/ontologyEntry"
-        },
-        "NCBITaxon": {
-          "$ref": "#/definitions/ontologyEntry"
-        },
-        "UBERON": {
-          "$ref": "#/definitions/ontologyEntry"
+        "deprecated_on": {
+          "type": "string",
+          "description": "The date this version was deprecated. The format of the date is YYYY-MM-DD. If this is the current verison then this field will be empty."
         },
-        "PATO": {
-          "$ref": "#/definitions/ontologyEntry"
+        "ontologies": {
+          "type": "object",
+          "properties": {
+            "CL": {
+              "$ref": "#/definitions/ontologyEntry"
+            },
+            "EFO": {
+              "$ref": "#/definitions/ontologyEntry"
+            },
+            "HANCESTRO": {
+              "$ref": "#/definitions/ontologyEntry"
+            },
+            "HsapDv": {
+              "$ref": "#/definitions/ontologyEntry"
+            },
+            "MONDO": {
+              "$ref": "#/definitions/ontologyEntry"
+            },
+            "MmusDv": {
+              "$ref": "#/definitions/ontologyEntry"
+            },
+            "NCBITaxon": {
+              "$ref": "#/definitions/ontologyEntry"
+            },
+            "UBERON": {
+              "$ref": "#/definitions/ontologyEntry"
+            },
+            "PATO": {
+              "$ref": "#/definitions/ontologyEntry"
+            }
+          }
         }
-      }
+      },
+      "required": [
+        "ontologies"
+      ]
     }
   },
   "additionalProperties": false,

diff --git a/ontology-assets/ontology_info.json b/ontology-assets/ontology_info.json
@@ -1,49 +1,51 @@
 {
   "v5.0.0": {
-    "CL": {
-      "version": "v2024-01-04",
-      "source": "https://github.com/obophenotype/cell-ontology/releases/download",
-      "filename": "cl.owl"
-    },
-    "EFO": {
-      "version": "v3.62.0",
-      "source": "https://github.com/EBISPOT/efo/releases/download",
-      "filename": "efo.owl"
-    },
-    "HANCESTRO": {
-      "version": "3.0",
-      "source": "https://github.com/EBISPOT/hancestro/raw",
-      "filename": "hancestro.owl"
-    },
-    "HsapDv": {
-      "version": "11",
-      "source": "http://aber-owl.net/media/ontologies/HSAPDV",
-      "filename": "hsapdv.owl"
-    },
-    "MONDO": {
-      "version": "v2024-01-03",
-      "source": "https://github.com/monarch-initiative/mondo/releases/download",
-      "filename": "mondo.owl"
-    },
-    "MmusDv": {
-      "version": "9",
-      "source": "http://aber-owl.net/media/ontologies/MMUSDV",
-      "filename": "mmusdv.owl"
-    },
-    "NCBITaxon": {
-      "version": "v2023-06-20",
-      "source": "https://github.com/obophenotype/ncbitaxon/releases/download",
-      "filename": "ncbitaxon.owl.gz"
-    },
-    "UBERON": {
-      "version": "v2024-01-18",
-      "source": "https://github.com/obophenotype/uberon/releases/download",
-      "filename": "uberon.owl"
-    },
-    "PATO": {
-      "version": "v2023-05-18",
-      "source": "https://github.com/pato-ontology/pato/raw",
-      "filename": "pato.owl"
+    "ontologies": {
+      "CL": {
+        "version": "v2024-01-04",
+        "source": "https://github.com/obophenotype/cell-ontology/releases/download",
+        "filename": "cl.owl"
+      },
+      "EFO": {
+        "version": "v3.62.0",
+        "source": "https://github.com/EBISPOT/efo/releases/download",
+        "filename": "efo.owl"
+      },
+      "HANCESTRO": {
+        "version": "3.0",
+        "source": "https://github.com/EBISPOT/hancestro/raw",
+        "filename": "hancestro.owl"
+      },
+      "HsapDv": {
+        "version": "11",
+        "source": "http://aber-owl.net/media/ontologies/HSAPDV",
+        "filename": "hsapdv.owl"
+      },
+      "MONDO": {
+        "version": "v2024-01-03",
+        "source": "https://github.com/monarch-initiative/mondo/releases/download",
+        "filename": "mondo.owl"
+      },
+      "MmusDv": {
+        "version": "9",
+        "source": "http://aber-owl.net/media/ontologies/MMUSDV",
+        "filename": "mmusdv.owl"
+      },
+      "NCBITaxon": {
+        "version": "v2023-06-20",
+        "source": "https://github.com/obophenotype/ncbitaxon/releases/download",
+        "filename": "ncbitaxon.owl.gz"
+      },
+      "UBERON": {
+        "version": "v2024-01-18",
+        "source": "https://github.com/obophenotype/uberon/releases/download",
+        "filename": "uberon.owl"
+      },
+      "PATO": {
+        "version": "v2023-05-18",
+        "source": "https://github.com/pato-ontology/pato/raw",
+        "filename": "pato.owl"
+      }
     }
   }
 }