Merge pull request #8452 from OpenMined/freeze-protocol-version

Freeze protocol version

.github/workflows/cd-syft.yml

@@ -410,7 +410,7 @@ jobs:
           author_name: ${{ secrets.OM_BOT_NAME }}
           author_email: ${{ secrets.OM_BOT_EMAIL }}
           message: "[syft]bump version"
-          add: "['.bumpversion.cfg', 'VERSION', 'packages/grid/VERSION','packages/syft/PYPI.md', 'packages/grid/devspace.yaml', 'packages/syft/src/syft/VERSION', 'packages/syft/setup.cfg', 'packages/grid/frontend/package.json', 'packages/syft/src/syft/__init__.py', 'packages/hagrid/hagrid/manifest_template.yml', 'packages/grid/helm/syft/Chart.yaml','packages/grid/helm/repo', 'packages/hagrid/hagrid/deps.py', 'packages/grid/podman/podman-kube/podman-syft-kube.yaml' ,'packages/grid/podman/podman-kube/podman-syft-kube-config.yaml', 'packages/syftcli/manifest.yml', 'packages/syft/src/syft/protocol/protocol_version.json', 'packages/grid/backend/worker_cpu.dockerfile','packages/grid/helm/syft/values.yaml','packages/grid/helm/syft']"
+          add: "['.bumpversion.cfg', 'VERSION', 'packages/grid/VERSION','packages/syft/PYPI.md', 'packages/grid/devspace.yaml', 'packages/syft/src/syft/VERSION', 'packages/syft/setup.cfg', 'packages/grid/frontend/package.json', 'packages/syft/src/syft/__init__.py', 'packages/hagrid/hagrid/manifest_template.yml', 'packages/grid/helm/syft/Chart.yaml','packages/grid/helm/repo', 'packages/hagrid/hagrid/deps.py', 'packages/grid/podman/podman-kube/podman-syft-kube.yaml' ,'packages/grid/podman/podman-kube/podman-syft-kube-config.yaml', 'packages/syftcli/manifest.yml', 'packages/syft/src/syft/protocol/protocol_version.json', 'packages/syft/src/syft/protocol/releases/', 'packages/grid/backend/worker_cpu.dockerfile','packages/grid/helm/syft/values.yaml','packages/grid/helm/syft']"
 
       - name: Changes to commit to Syft Repo during stable release
         if: needs.merge-docker-images.outputs.release_tag == 'latest'
@@ -419,7 +419,7 @@ jobs:
           author_name: ${{ secrets.OM_BOT_NAME }}
           author_email: ${{ secrets.OM_BOT_EMAIL }}
           message: "[syft] bump protocol version"
-          add: "['packages/syft/src/syft/protocol/protocol_version.json']"
+          add: "['packages/syft/src/syft/protocol/protocol_version.json', 'packages/syft/src/syft/protocol/releases/']"
 
       - name: Scheduled Build and Publish
         if: github.event_name == 'schedule'

packages/syft/src/syft/protocol/data_protocol.py

@@ -9,16 +9,19 @@
 import re
 from typing import Any
 from typing import Dict
+from typing import List
 from typing import Optional
 from typing import Tuple
 from typing import Type
 from typing import Union
 
 # third party
+from packaging.version import parse
 from result import OkErr
 from result import Result
 
 # relative
+from .. import __version__
 from ..serde.recursive import TYPE_BANK
 from ..service.response import SyftError
 from ..service.response import SyftException
@@ -30,29 +33,33 @@
 PROTOCOL_TYPE = Union[str, int]
 
 
-def natural_key(key: PROTOCOL_TYPE) -> list[int]:
+def natural_key(key: PROTOCOL_TYPE) -> List[int]:
     """Define key for natural ordering of strings."""
     if isinstance(key, int):
         key = str(key)
     return [int(s) if s.isdigit() else s for s in re.split("(\d+)", key)]
 
 
-def sort_dict_naturally(d: dict) -> dict:
+def sort_dict_naturally(d: Dict) -> Dict:
     """Sort dictionary by keys in natural order."""
     return {k: d[k] for k in sorted(d.keys(), key=natural_key)}
 
 
-def data_protocol_file_name():
+def data_protocol_file_name() -> str:
     return PROTOCOL_STATE_FILENAME
 
 
-def data_protocol_dir():
-    return os.path.abspath(str(Path(__file__).parent))
+def data_protocol_dir() -> Path:
+    return Path(os.path.abspath(str(Path(__file__).parent)))
+
+
+def protocol_release_dir() -> Path:
+    return data_protocol_dir() / "releases"
 
 
 class DataProtocol:
     def __init__(self, filename: str) -> None:
-        self.file_path = Path(data_protocol_dir()) / filename
+        self.file_path = data_protocol_dir() / filename
         self.load_state()
 
     def load_state(self) -> None:
@@ -78,13 +85,34 @@ def _calculate_object_hash(klass: Type[SyftBaseObject]) -> str:
 
         return hashlib.sha256(json.dumps(obj_meta_info).encode()).hexdigest()
 
-    def read_history(self) -> Dict:
+    @staticmethod
+    def read_json(file_path: Path) -> Dict:
         try:
-            return json.loads(self.file_path.read_text())
+            return json.loads(file_path.read_text())
         except Exception:
             return {}
 
-    def save_history(self, history: dict) -> None:
+    def read_history(self) -> Dict:
+        protocol_history = self.read_json(self.file_path)
+
+        for version in protocol_history.keys():
+            if version == "dev":
+                continue
+            release_version_path = (
+                protocol_release_dir() / protocol_history[version]["release_name"]
+            )
+            released_version = self.read_json(file_path=release_version_path)
+            protocol_history[version] = released_version.get(version, {})
+
+        return protocol_history
+
+    def save_history(self, history: Dict) -> None:
+        for file_path in protocol_release_dir().iterdir():
+            for version in self.read_json(file_path):
+                # Skip adding file if the version is not part of the history
+                if version not in history.keys():
+                    continue
+                history[version] = {"release_name": file_path.name}
         self.file_path.write_text(json.dumps(history, indent=2) + "\n")
 
     @property
@@ -136,7 +164,7 @@ def build_state(self, stop_key: Optional[str] = None) -> dict:
                 return state_dict
         return state_dict
 
-    def diff_state(self, state: dict) -> tuple[dict, dict]:
+    def diff_state(self, state: Dict) -> tuple[Dict, Dict]:
         compare_dict = defaultdict(dict)  # what versions are in the latest code
         object_diff = defaultdict(dict)  # diff in latest code with saved json
         for k in TYPE_BANK:
@@ -274,6 +302,7 @@ def bump_protocol_version(self) -> Result[SyftSuccess, SyftError]:
 
         keys = self.protocol_history.keys()
         if "dev" not in keys:
+            self.validate_release()
             print("You can't bump the protocol if there are no staged changes.")
             return SyftError(
                 message="Failed to bump version as there are no staged changes."
@@ -287,11 +316,110 @@ def bump_protocol_version(self) -> Result[SyftSuccess, SyftError]:
 
         next_highest_protocol = highest_protocol + 1
         self.protocol_history[str(next_highest_protocol)] = self.protocol_history["dev"]
+        self.freeze_release(self.protocol_history, str(next_highest_protocol))
         del self.protocol_history["dev"]
         self.save_history(self.protocol_history)
         self.load_state()
         return SyftSuccess(message=f"Protocol Updated to {next_highest_protocol}")
 
+    @staticmethod
+    def freeze_release(protocol_history: Dict, latest_protocol: str) -> None:
+        """Freezes latest release as a separate release file."""
+
+        # Get release history
+        release_history = protocol_history[latest_protocol]
+
+        # Create new file for the version
+        syft_version = parse(__version__)
+        release_file_name = f"{syft_version.public}.json"
+        release_file = protocol_release_dir() / release_file_name
+
+        # Save the new released version
+        release_file.write_text(
+            json.dumps({latest_protocol: release_history}, indent=2)
+        )
+
+    def validate_release(self) -> None:
+        """Validate if latest release name is consistent with syft version"""
+        # Read the protocol history
+        protocol_history = self.read_json(self.file_path)
+        sorted_protocol_versions = sorted(protocol_history.keys(), key=natural_key)
+
+        # Grab the latest protocol
+        latest_protocol = (
+            sorted_protocol_versions[-1] if len(sorted_protocol_versions) > 0 else None
+        )
+
+        # Skip validation if latest protocol is dev
+        if latest_protocol is None or latest_protocol == "dev":
+            return
+
+        # Get filename of the latest protocol
+        release_name = protocol_history[latest_protocol]["release_name"]
+        # Extract syft version from release name
+        protocol_syft_version = parse(release_name.split(".json")[0])
+        current_syft_version = parse(__version__)
+
+        # If base syft version in latest protocol version is not same as current syft version
+        # Skip updating the release name
+        if protocol_syft_version.base_version != current_syft_version.base_version:
+            return
+
+        # Update release name to latest beta, stable or post based on current syft version
+        print(
+            f"Current release {release_name} will be updated to {current_syft_version}"
+        )
+
+        # Get latest protocol file path
+        latest_protocol_fp: Path = protocol_release_dir() / release_name
+
+        # New protocol file path
+        new_protocol_file_path = (
+            protocol_release_dir() / f"{current_syft_version.public}.json"
+        )
+
+        # Update older file path to newer file path
+        latest_protocol_fp.rename(new_protocol_file_path)
+        protocol_history[latest_protocol][
+            "release_name"
+        ] = f"{current_syft_version}.json"
+
+        # Save history
+        self.file_path.write_text(json.dumps(protocol_history, indent=2) + "\n")
+
+        # Reload protocol
+        self.read_history()
+
+    def revert_latest_protocol(self) -> Result[SyftSuccess, SyftError]:
+        """Revert latest protocol changes to dev"""
+
+        # Get current protocol history
+        protocol_history = self.read_json(self.file_path)
+
+        # Get latest released protocol
+        sorted_protocol_versions = sorted(protocol_history.keys(), key=natural_key)
+        latest_protocol = (
+            sorted_protocol_versions[-1] if len(sorted_protocol_versions) > 0 else None
+        )
+
+        # If current protocol is dev, skip revert
+        if latest_protocol is None or latest_protocol == "dev":
+            return SyftError(message="Revert skipped !! Already running dev protocol.")
+
+        # Read the current released protocol
+        release_name = protocol_history[latest_protocol]["release_name"]
+        protocol_file_path: Path = protocol_release_dir() / release_name
+
+        released_protocol = self.read_json(protocol_file_path)
+        protocol_history["dev"] = released_protocol[latest_protocol]
+
+        # Delete the current released protocol
+        protocol_history.pop(latest_protocol)
+        protocol_file_path.unlink()
+
+        # Save history
+        self.save_history(protocol_history)
+
     def check_protocol(self) -> Result[SyftSuccess, SyftError]:
         if len(self.diff) != 0:
             return SyftError(message="Protocol Changes Unstaged")
@@ -338,7 +466,7 @@ def has_dev(self) -> bool:
         return False
 
 
-def get_data_protocol():
+def get_data_protocol() -> DataProtocol:
     return DataProtocol(filename=data_protocol_file_name())
 
 
@@ -357,7 +485,7 @@ def check_or_stage_protocol() -> Result[SyftSuccess, SyftError]:
     return data_protocol.check_or_stage_protocol()
 
 
-def debox_arg_and_migrate(arg: Any, protocol_state: dict):
+def debox_arg_and_migrate(arg: Any, protocol_state: dict) -> Any:
     """Debox the argument based on whether it is iterable or single entity."""
     constructor = None
     extra_args = []