diff --git a/.github/workflows/cd-syft.yml b/.github/workflows/cd-syft.yml index f57a95a5f44..23436403538 100644 --- a/.github/workflows/cd-syft.yml +++ b/.github/workflows/cd-syft.yml @@ -86,13 +86,18 @@ jobs: tox -e syft.build.helm tox -e syft.package.helm + - name: Check and Bump Protocol Version + if: github.event_name == 'schedule' + run: | + tox -e syft.protocol.check + - name: Commit changes to Syft uses: EndBug/add-and-commit@v9 with: author_name: ${{ secrets.OM_BOT_NAME }} author_email: ${{ secrets.OM_BOT_EMAIL }} message: "[syft]bump version" - add: "['.bumpversion.cfg', 'VERSION', 'packages/grid/VERSION', 'packages/grid/devspace.yaml', 'packages/syft/src/syft/VERSION', 'packages/syft/setup.cfg', 'packages/grid/frontend/package.json', 'packages/syft/src/syft/__init__.py', 'packages/hagrid/hagrid/manifest_template.yml', 'packages/grid/helm/syft/Chart.yaml', 'packages/grid/helm/repo', 'packages/hagrid/hagrid/deps.py', 'packages/grid/podman/podman-kube/podman-syft-kube.yaml' , 'packages/syftcli/manifest.yml']" + add: "['.bumpversion.cfg', 'VERSION', 'packages/grid/VERSION', 'packages/grid/devspace.yaml', 'packages/syft/src/syft/VERSION', 'packages/syft/setup.cfg', 'packages/grid/frontend/package.json', 'packages/syft/src/syft/__init__.py', 'packages/hagrid/hagrid/manifest_template.yml', 'packages/grid/helm/syft/Chart.yaml', 'packages/grid/helm/repo', 'packages/hagrid/hagrid/deps.py', 'packages/grid/podman/podman-kube/podman-syft-kube.yaml' , 'packages/syftcli/manifest.yml', 'packages/syft/src/syft/protocol/protocol_version.json']" - name: Scheduled Build and Publish if: github.event_name == 'schedule' diff --git a/.github/workflows/post-merge-cleanup-notebooks.yml b/.github/workflows/post-merge-tasks.yml similarity index 63% rename from .github/workflows/post-merge-cleanup-notebooks.yml rename to .github/workflows/post-merge-tasks.yml index 8b7f07ae4c3..860b71b1442 100644 --- a/.github/workflows/post-merge-cleanup-notebooks.yml +++ b/.github/workflows/post-merge-tasks.yml @@ -1,4 +1,4 @@ -name: Post Merge - Cleanup Notebooks +name: Post Merge Tasks on: workflow_call: @@ -7,7 +7,6 @@ on: branches: - dev - main - - "0.8" jobs: post-merge-cleanup-notebooks: @@ -22,11 +21,17 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Commit changes to remove notebooks + - name: Check and Bump Protocol Version + if: github.event_name == 'schedule' + run: | + tox -e syft.protocol.check + + - name: Commit changes to bump protocol and remove notebooks uses: EndBug/add-and-commit@v9 with: author_name: ${{ secrets.OM_BOT_NAME }} author_email: ${{ secrets.OM_BOT_EMAIL }} - message: "cleanup notebooks" + message: "bump protocol and remove notebooks" remove: "-r notebooks/Experimental/" + add: "['packages/syft/src/syft/protocol/protocol_version.json']" commit: "-a" diff --git a/.github/workflows/pr-tests-linting.yml b/.github/workflows/pr-tests-linting.yml index 99dda76878c..41ec391eff9 100644 --- a/.github/workflows/pr-tests-linting.yml +++ b/.github/workflows/pr-tests-linting.yml @@ -27,9 +27,9 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Upgrade pip + - name: Install pip packages run: | - python -m pip install --upgrade --user pip + python -m pip install --upgrade --user pip tox - name: Get pip cache dir id: pip-cache @@ -47,3 +47,7 @@ jobs: ${{ runner.os }}-pip-py${{ matrix.python-version }}- - uses: pre-commit/action@v3.0.0 + + - name: Check Protocol Version + run: | + tox -e syft.protocol.check diff --git a/notebooks/Experimental/Data Migration.ipynb b/notebooks/Experimental/Data Migration.ipynb new file mode 100644 index 00000000000..42c8a9a551d --- /dev/null +++ b/notebooks/Experimental/Data Migration.ipynb @@ -0,0 +1,999 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "id": "a679e5fb", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "kj/filesystem-disk-unix.c++:1703: warning: PWD environment variable doesn't match current directory; pwd = /home/shubham/PySyft\n" + ] + } + ], + "source": [ + "import syft as sy\n", + "from syft.types.datetime import DateTime\n", + "from syft.service.metadata.node_metadata import NodeMetadata, NodeMetadataV2" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "27cd4d0f", + "metadata": {}, + "outputs": [], + "source": [ + "from syft.types.syft_object import SyftMigrationRegistry\n", + "from syft.service.metadata.migrations import *" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0f062500", + "metadata": {}, + "outputs": [], + "source": [ + "__migration_version_registry__ = {\n", + " \"canonical_name\": {version_number: \"klass_name\"}\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "7ac6a650", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/html": [ + "{'SyftObject': {1: 'syft.types.syft_object.SyftObject'}, 'PartialSyftObject': {1: 'syft.types.syft_object.PartialSyftObject'}, 'NodeServiceContext': {1: 'syft.service.context.NodeServiceContext'}, 'AuthedServiceContext': {1: 'syft.service.context.AuthedServiceContext'}, 'UnauthedServiceContext': {1: 'syft.service.context.UnauthedServiceContext'}, 'NodeMetadataUpdate': {1: 'syft.service.metadata.node_metadata.NodeMetadataUpdate'}, 'NodeMetadata': {1: 'syft.service.metadata.node_metadata.NodeMetadata', 2: 'syft.service.metadata.node_metadata.NodeMetadataV2'}, 'LinkedObject': {1: 'syft.store.linked_obj.LinkedObject'}, 'NodeConnection': {1: 'syft.client.connection.NodeConnection'}, 'APIEndpoint': {1: 'syft.client.api.APIEndpoint'}, 'SignedSyftAPICall': {1: 'syft.client.api.SignedSyftAPICall'}, 'SyftAPICall': {1: 'syft.client.api.SyftAPICall'}, 'SyftAPI': {1: 'syft.client.api.SyftAPI'}, 'User': {1: 'syft.service.user.user.User'}, 'UserUpdate': {1: 'syft.service.user.user.UserUpdate'}, 'UserCreate': {1: 'syft.service.user.user.UserCreate'}, 'UserSearch': {1: 'syft.service.user.user.UserSearch'}, 'UserView': {1: 'syft.service.user.user.UserView'}, 'UserViewPage': {1: 'syft.service.user.user.UserViewPage'}, 'UserPrivateKey': {1: 'syft.service.user.user.UserPrivateKey'}, 'NodeSettingsUpdate': {1: 'syft.service.settings.settings.NodeSettingsUpdate'}, 'NodeSettings': {1: 'syft.service.settings.settings.NodeSettings'}, 'HTTPConnection': {1: 'syft.client.client.HTTPConnection'}, 'PythonConnection': {1: 'syft.client.client.PythonConnection'}, 'DateTime': {1: 'syft.types.datetime.DateTime'}, 'BlobFile': {1: 'syft.types.blob_storage.BlobFile'}, 'SecureFilePathLocation': {1: 'syft.types.blob_storage.SecureFilePathLocation'}, 'SeaweedSecureFilePathLocation': {1: 'syft.types.blob_storage.SeaweedSecureFilePathLocation'}, 'BlobStorageEntry': {1: 'syft.types.blob_storage.BlobStorageEntry'}, 'BlobStorageMetadata': {1: 'syft.types.blob_storage.BlobStorageMetadata'}, 'CreateBlobStorageEntry': {1: 'syft.types.blob_storage.CreateBlobStorageEntry'}, 'BlobRetrieval': {1: 'syft.store.blob_storage.BlobRetrieval'}, 'SyftObjectRetrieval': {1: 'syft.store.blob_storage.SyftObjectRetrieval'}, 'BlobRetrievalByURL': {1: 'syft.store.blob_storage.BlobRetrievalByURL'}, 'BlobDeposit': {1: 'syft.store.blob_storage.BlobDeposit'}, 'WorkerSettings': {1: 'syft.node.worker_settings.WorkerSettings'}, 'HTTPNodeRoute': {1: 'syft.service.network.routes.HTTPNodeRoute'}, 'PythonNodeRoute': {1: 'syft.service.network.routes.PythonNodeRoute'}, 'EnclaveMetadata': {1: 'syft.client.enclave_client.EnclaveMetadata'}, 'DataSubject': {1: 'syft.service.data_subject.data_subject.DataSubject'}, 'DataSubjectCreate': {1: 'syft.service.data_subject.data_subject.DataSubjectCreate'}, 'DataSubjectMemberRelationship': {1: 'syft.service.data_subject.data_subject_member.DataSubjectMemberRelationship'}, 'Contributor': {1: 'syft.service.dataset.dataset.Contributor'}, 'MarkdownDescription': {1: 'syft.service.dataset.dataset.MarkdownDescription'}, 'Asset': {1: 'syft.service.dataset.dataset.Asset'}, 'CreateAsset': {1: 'syft.service.dataset.dataset.CreateAsset'}, 'Dataset': {1: 'syft.service.dataset.dataset.Dataset'}, 'DatasetPageView': {1: 'syft.service.dataset.dataset.DatasetPageView'}, 'CreateDataset': {1: 'syft.service.dataset.dataset.CreateDataset'}, 'ActionDataEmpty': {1: 'syft.service.action.action_data_empty.ActionDataEmpty'}, 'ActionFileData': {1: 'syft.service.action.action_data_empty.ActionFileData'}, 'Action': {1: 'syft.service.action.action_object.Action'}, 'ActionObject': {1: 'syft.service.action.action_object.ActionObject'}, 'AnyActionObject': {1: 'syft.service.action.action_object.AnyActionObject'}, 'TwinObject': {1: 'syft.types.twin_object.TwinObject'}, 'Policy': {1: 'syft.service.policy.policy.Policy'}, 'InputPolicy': {1: 'syft.service.policy.policy.InputPolicy'}, 'ExactMatch': {1: 'syft.service.policy.policy.ExactMatch'}, 'OutputHistory': {1: 'syft.service.policy.policy.OutputHistory'}, 'OutputPolicy': {1: 'syft.service.policy.policy.OutputPolicy'}, 'OutputPolicyExecuteCount': {1: 'syft.service.policy.policy.OutputPolicyExecuteCount'}, 'OutputPolicyExecuteOnce': {1: 'syft.service.policy.policy.OutputPolicyExecuteOnce'}, 'UserOutputPolicy': {1: 'syft.service.policy.policy.UserOutputPolicy'}, 'UserInputPolicy': {1: 'syft.service.policy.policy.UserInputPolicy'}, 'UserPolicy': {1: 'syft.service.policy.policy.UserPolicy'}, 'SubmitUserPolicy': {1: 'syft.service.policy.policy.SubmitUserPolicy'}, 'UserCode': {1: 'syft.service.code.user_code.UserCode'}, 'SubmitUserCode': {1: 'syft.service.code.user_code.SubmitUserCode'}, 'UserCodeExecutionResult': {1: 'syft.service.code.user_code.UserCodeExecutionResult'}, 'CodeHistory': {1: 'syft.service.code_history.code_history.CodeHistory'}, 'CodeHistoryView': {1: 'syft.service.code_history.code_history.CodeHistoryView'}, 'CodeHistoriesDict': {1: 'syft.service.code_history.code_history.CodeHistoriesDict'}, 'UsersCodeHistoriesDict': {1: 'syft.service.code_history.code_history.UsersCodeHistoriesDict'}, 'NodePeer': {1: 'syft.service.network.node_peer.NodePeer'}, 'ProxyClient': {1: 'syft.client.gateway_client.ProxyClient'}, 'CommandReport': {1: 'syft.service.vpn.vpn.CommandReport'}, 'CommandResult': {1: 'syft.service.vpn.vpn.CommandResult'}, 'VPNClientConnection': {1: 'syft.service.vpn.vpn.VPNClientConnection'}, 'HeadscaleAuthToken': {1: 'syft.service.vpn.headscale_client.HeadscaleAuthToken'}, 'TailscalePeer': {1: 'syft.service.vpn.tailscale_client.TailscalePeer'}, 'TailscaleStatus': {1: 'syft.service.vpn.tailscale_client.TailscaleStatus'}, 'OnDiskBlobDeposit': {1: 'syft.store.blob_storage.on_disk.OnDiskBlobDeposit'}, 'SeaweedFSBlobDeposit': {1: 'syft.store.blob_storage.seaweedfs.SeaweedFSBlobDeposit'}, 'NumpyArrayObject': {1: 'syft.service.action.numpy.NumpyArrayObject'}, 'NumpyScalarObject': {1: 'syft.service.action.numpy.NumpyScalarObject'}, 'NumpyBoolObject': {1: 'syft.service.action.numpy.NumpyBoolObject'}, 'PandasDataframeObject': {1: 'syft.service.action.pandas.PandasDataFrameObject'}, 'PandasSeriesObject': {1: 'syft.service.action.pandas.PandasSeriesObject'}, 'ReplyNotification': {1: 'syft.service.notification.notifications.ReplyNotification'}, 'Notification': {1: 'syft.service.notification.notifications.Notification'}, 'CreateNotification': {1: 'syft.service.notification.notifications.CreateNotification'}, 'Change': {1: 'syft.service.request.request.Change'}, 'ChangeStatus': {1: 'syft.service.request.request.ChangeStatus'}, 'ActionStoreChange': {1: 'syft.service.request.request.ActionStoreChange'}, 'Request': {1: 'syft.service.request.request.Request'}, 'RequestInfo': {1: 'syft.service.request.request.RequestInfo'}, 'RequestInfoFilter': {1: 'syft.service.request.request.RequestInfoFilter'}, 'SubmitRequest': {1: 'syft.service.request.request.SubmitRequest'}, 'ObjectMutation': {1: 'syft.service.request.request.ObjectMutation'}, 'EnumMutation': {1: 'syft.service.request.request.EnumMutation'}, 'UserCodeStatusChange': {1: 'syft.service.request.request.UserCodeStatusChange'}, 'ProjectEvent': {1: 'syft.service.project.project.ProjectEvent'}, 'ProjectEventAddObject': {1: 'syft.service.project.project.ProjectEventAddObject'}, 'ProjectEventAddLink': {1: 'syft.service.project.project.ProjectEventAddLink'}, 'ProjectSubEvent': {1: 'syft.service.project.project.ProjectSubEvent'}, 'ProjectThreadMessage': {1: 'syft.service.project.project.ProjectThreadMessage'}, 'ProjectMessage': {1: 'syft.service.project.project.ProjectMessage'}, 'ProjectRequestResponse': {1: 'syft.service.project.project.ProjectRequestResponse'}, 'ProjectRequest': {1: 'syft.service.project.project.ProjectRequest'}, 'AnswerProjectPoll': {1: 'syft.service.project.project.AnswerProjectPoll'}, 'ProjectPoll': {1: 'syft.service.project.project.ProjectMultipleChoicePoll'}, 'Project': {1: 'syft.service.project.project.Project'}, 'ProjectSubmit': {1: 'syft.service.project.project.ProjectSubmit'}, 'QueueItem': {1: 'syft.service.queue.queue_stash.QueueItem'}, 'ZMQClientConfig': {1: 'syft.service.queue.zmq_queue.ZMQClientConfig'}, 'Plan': {1: 'syft.service.action.plan.Plan'}}" + ], + "text/plain": [ + "{'SyftObject': {1: 'syft.types.syft_object.SyftObject'},\n", + " 'PartialSyftObject': {1: 'syft.types.syft_object.PartialSyftObject'},\n", + " 'NodeServiceContext': {1: 'syft.service.context.NodeServiceContext'},\n", + " 'AuthedServiceContext': {1: 'syft.service.context.AuthedServiceContext'},\n", + " 'UnauthedServiceContext': {1: 'syft.service.context.UnauthedServiceContext'},\n", + " 'NodeMetadataUpdate': {1: 'syft.service.metadata.node_metadata.NodeMetadataUpdate'},\n", + " 'NodeMetadata': {1: 'syft.service.metadata.node_metadata.NodeMetadata',\n", + " 2: 'syft.service.metadata.node_metadata.NodeMetadataV2'},\n", + " 'LinkedObject': {1: 'syft.store.linked_obj.LinkedObject'},\n", + " 'NodeConnection': {1: 'syft.client.connection.NodeConnection'},\n", + " 'APIEndpoint': {1: 'syft.client.api.APIEndpoint'},\n", + " 'SignedSyftAPICall': {1: 'syft.client.api.SignedSyftAPICall'},\n", + " 'SyftAPICall': {1: 'syft.client.api.SyftAPICall'},\n", + " 'SyftAPI': {1: 'syft.client.api.SyftAPI'},\n", + " 'User': {1: 'syft.service.user.user.User'},\n", + " 'UserUpdate': {1: 'syft.service.user.user.UserUpdate'},\n", + " 'UserCreate': {1: 'syft.service.user.user.UserCreate'},\n", + " 'UserSearch': {1: 'syft.service.user.user.UserSearch'},\n", + " 'UserView': {1: 'syft.service.user.user.UserView'},\n", + " 'UserViewPage': {1: 'syft.service.user.user.UserViewPage'},\n", + " 'UserPrivateKey': {1: 'syft.service.user.user.UserPrivateKey'},\n", + " 'NodeSettingsUpdate': {1: 'syft.service.settings.settings.NodeSettingsUpdate'},\n", + " 'NodeSettings': {1: 'syft.service.settings.settings.NodeSettings'},\n", + " 'HTTPConnection': {1: 'syft.client.client.HTTPConnection'},\n", + " 'PythonConnection': {1: 'syft.client.client.PythonConnection'},\n", + " 'DateTime': {1: 'syft.types.datetime.DateTime'},\n", + " 'BlobFile': {1: 'syft.types.blob_storage.BlobFile'},\n", + " 'SecureFilePathLocation': {1: 'syft.types.blob_storage.SecureFilePathLocation'},\n", + " 'SeaweedSecureFilePathLocation': {1: 'syft.types.blob_storage.SeaweedSecureFilePathLocation'},\n", + " 'BlobStorageEntry': {1: 'syft.types.blob_storage.BlobStorageEntry'},\n", + " 'BlobStorageMetadata': {1: 'syft.types.blob_storage.BlobStorageMetadata'},\n", + " 'CreateBlobStorageEntry': {1: 'syft.types.blob_storage.CreateBlobStorageEntry'},\n", + " 'BlobRetrieval': {1: 'syft.store.blob_storage.BlobRetrieval'},\n", + " 'SyftObjectRetrieval': {1: 'syft.store.blob_storage.SyftObjectRetrieval'},\n", + " 'BlobRetrievalByURL': {1: 'syft.store.blob_storage.BlobRetrievalByURL'},\n", + " 'BlobDeposit': {1: 'syft.store.blob_storage.BlobDeposit'},\n", + " 'WorkerSettings': {1: 'syft.node.worker_settings.WorkerSettings'},\n", + " 'HTTPNodeRoute': {1: 'syft.service.network.routes.HTTPNodeRoute'},\n", + " 'PythonNodeRoute': {1: 'syft.service.network.routes.PythonNodeRoute'},\n", + " 'EnclaveMetadata': {1: 'syft.client.enclave_client.EnclaveMetadata'},\n", + " 'DataSubject': {1: 'syft.service.data_subject.data_subject.DataSubject'},\n", + " 'DataSubjectCreate': {1: 'syft.service.data_subject.data_subject.DataSubjectCreate'},\n", + " 'DataSubjectMemberRelationship': {1: 'syft.service.data_subject.data_subject_member.DataSubjectMemberRelationship'},\n", + " 'Contributor': {1: 'syft.service.dataset.dataset.Contributor'},\n", + " 'MarkdownDescription': {1: 'syft.service.dataset.dataset.MarkdownDescription'},\n", + " 'Asset': {1: 'syft.service.dataset.dataset.Asset'},\n", + " 'CreateAsset': {1: 'syft.service.dataset.dataset.CreateAsset'},\n", + " 'Dataset': {1: 'syft.service.dataset.dataset.Dataset'},\n", + " 'DatasetPageView': {1: 'syft.service.dataset.dataset.DatasetPageView'},\n", + " 'CreateDataset': {1: 'syft.service.dataset.dataset.CreateDataset'},\n", + " 'ActionDataEmpty': {1: 'syft.service.action.action_data_empty.ActionDataEmpty'},\n", + " 'ActionFileData': {1: 'syft.service.action.action_data_empty.ActionFileData'},\n", + " 'Action': {1: 'syft.service.action.action_object.Action'},\n", + " 'ActionObject': {1: 'syft.service.action.action_object.ActionObject'},\n", + " 'AnyActionObject': {1: 'syft.service.action.action_object.AnyActionObject'},\n", + " 'TwinObject': {1: 'syft.types.twin_object.TwinObject'},\n", + " 'Policy': {1: 'syft.service.policy.policy.Policy'},\n", + " 'InputPolicy': {1: 'syft.service.policy.policy.InputPolicy'},\n", + " 'ExactMatch': {1: 'syft.service.policy.policy.ExactMatch'},\n", + " 'OutputHistory': {1: 'syft.service.policy.policy.OutputHistory'},\n", + " 'OutputPolicy': {1: 'syft.service.policy.policy.OutputPolicy'},\n", + " 'OutputPolicyExecuteCount': {1: 'syft.service.policy.policy.OutputPolicyExecuteCount'},\n", + " 'OutputPolicyExecuteOnce': {1: 'syft.service.policy.policy.OutputPolicyExecuteOnce'},\n", + " 'UserOutputPolicy': {1: 'syft.service.policy.policy.UserOutputPolicy'},\n", + " 'UserInputPolicy': {1: 'syft.service.policy.policy.UserInputPolicy'},\n", + " 'UserPolicy': {1: 'syft.service.policy.policy.UserPolicy'},\n", + " 'SubmitUserPolicy': {1: 'syft.service.policy.policy.SubmitUserPolicy'},\n", + " 'UserCode': {1: 'syft.service.code.user_code.UserCode'},\n", + " 'SubmitUserCode': {1: 'syft.service.code.user_code.SubmitUserCode'},\n", + " 'UserCodeExecutionResult': {1: 'syft.service.code.user_code.UserCodeExecutionResult'},\n", + " 'CodeHistory': {1: 'syft.service.code_history.code_history.CodeHistory'},\n", + " 'CodeHistoryView': {1: 'syft.service.code_history.code_history.CodeHistoryView'},\n", + " 'CodeHistoriesDict': {1: 'syft.service.code_history.code_history.CodeHistoriesDict'},\n", + " 'UsersCodeHistoriesDict': {1: 'syft.service.code_history.code_history.UsersCodeHistoriesDict'},\n", + " 'NodePeer': {1: 'syft.service.network.node_peer.NodePeer'},\n", + " 'ProxyClient': {1: 'syft.client.gateway_client.ProxyClient'},\n", + " 'CommandReport': {1: 'syft.service.vpn.vpn.CommandReport'},\n", + " 'CommandResult': {1: 'syft.service.vpn.vpn.CommandResult'},\n", + " 'VPNClientConnection': {1: 'syft.service.vpn.vpn.VPNClientConnection'},\n", + " 'HeadscaleAuthToken': {1: 'syft.service.vpn.headscale_client.HeadscaleAuthToken'},\n", + " 'TailscalePeer': {1: 'syft.service.vpn.tailscale_client.TailscalePeer'},\n", + " 'TailscaleStatus': {1: 'syft.service.vpn.tailscale_client.TailscaleStatus'},\n", + " 'OnDiskBlobDeposit': {1: 'syft.store.blob_storage.on_disk.OnDiskBlobDeposit'},\n", + " 'SeaweedFSBlobDeposit': {1: 'syft.store.blob_storage.seaweedfs.SeaweedFSBlobDeposit'},\n", + " 'NumpyArrayObject': {1: 'syft.service.action.numpy.NumpyArrayObject'},\n", + " 'NumpyScalarObject': {1: 'syft.service.action.numpy.NumpyScalarObject'},\n", + " 'NumpyBoolObject': {1: 'syft.service.action.numpy.NumpyBoolObject'},\n", + " 'PandasDataframeObject': {1: 'syft.service.action.pandas.PandasDataFrameObject'},\n", + " 'PandasSeriesObject': {1: 'syft.service.action.pandas.PandasSeriesObject'},\n", + " 'ReplyNotification': {1: 'syft.service.notification.notifications.ReplyNotification'},\n", + " 'Notification': {1: 'syft.service.notification.notifications.Notification'},\n", + " 'CreateNotification': {1: 'syft.service.notification.notifications.CreateNotification'},\n", + " 'Change': {1: 'syft.service.request.request.Change'},\n", + " 'ChangeStatus': {1: 'syft.service.request.request.ChangeStatus'},\n", + " 'ActionStoreChange': {1: 'syft.service.request.request.ActionStoreChange'},\n", + " 'Request': {1: 'syft.service.request.request.Request'},\n", + " 'RequestInfo': {1: 'syft.service.request.request.RequestInfo'},\n", + " 'RequestInfoFilter': {1: 'syft.service.request.request.RequestInfoFilter'},\n", + " 'SubmitRequest': {1: 'syft.service.request.request.SubmitRequest'},\n", + " 'ObjectMutation': {1: 'syft.service.request.request.ObjectMutation'},\n", + " 'EnumMutation': {1: 'syft.service.request.request.EnumMutation'},\n", + " 'UserCodeStatusChange': {1: 'syft.service.request.request.UserCodeStatusChange'},\n", + " 'ProjectEvent': {1: 'syft.service.project.project.ProjectEvent'},\n", + " 'ProjectEventAddObject': {1: 'syft.service.project.project.ProjectEventAddObject'},\n", + " 'ProjectEventAddLink': {1: 'syft.service.project.project.ProjectEventAddLink'},\n", + " 'ProjectSubEvent': {1: 'syft.service.project.project.ProjectSubEvent'},\n", + " 'ProjectThreadMessage': {1: 'syft.service.project.project.ProjectThreadMessage'},\n", + " 'ProjectMessage': {1: 'syft.service.project.project.ProjectMessage'},\n", + " 'ProjectRequestResponse': {1: 'syft.service.project.project.ProjectRequestResponse'},\n", + " 'ProjectRequest': {1: 'syft.service.project.project.ProjectRequest'},\n", + " 'AnswerProjectPoll': {1: 'syft.service.project.project.AnswerProjectPoll'},\n", + " 'ProjectPoll': {1: 'syft.service.project.project.ProjectMultipleChoicePoll'},\n", + " 'Project': {1: 'syft.service.project.project.Project'},\n", + " 'ProjectSubmit': {1: 'syft.service.project.project.ProjectSubmit'},\n", + " 'QueueItem': {1: 'syft.service.queue.queue_stash.QueueItem'},\n", + " 'ZMQClientConfig': {1: 'syft.service.queue.zmq_queue.ZMQClientConfig'},\n", + " 'Plan': {1: 'syft.service.action.plan.Plan'}}" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "SyftMigrationRegistry.__migration_version_registry__" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "3790c6ca", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "{'NodeMetadata': {'1x2': <function generate_transform_wrapper.<locals>.wrapper at 0x7f8d69d644c0>, '2x1': <function generate_transform_wrapper.<locals>.wrapper at 0x7f8d69d64700>}}" + ], + "text/plain": [ + "{'NodeMetadata': {'1x2': .wrapper(self: syft.service.metadata.node_metadata.NodeMetadata, context: Union[syft.types.transforms.TransformContext, syft.service.context.NodeServiceContext, NoneType] = None) -> syft.service.metadata.node_metadata.NodeMetadataV2>,\n", + " '2x1': .wrapper(self: syft.service.metadata.node_metadata.NodeMetadataV2, context: Union[syft.types.transforms.TransformContext, syft.service.context.NodeServiceContext, NoneType] = None) -> syft.service.metadata.node_metadata.NodeMetadata>}}" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "SyftMigrationRegistry.__migration_transform_registry__" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "a4ec5311", + "metadata": {}, + "outputs": [], + "source": [ + "node_metadata_v1 = NodeMetadata(\n", + " name=\"OM\",\n", + " highest_object_version=2,\n", + " lowest_object_version=1,\n", + " id=sy.UID(),\n", + " verify_key=sy.SyftSigningKey.generate().verify_key,\n", + " syft_version=\"0.8.2\",\n", + " node_type=\"domain\",\n", + " signup_enabled=True,\n", + " admin_email=\"info@openmined.org\",\n", + " node_side_type=\"low_side\",\n", + " show_warnings=False,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "8bcf95b8", + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "```python\n", + "class NodeMetadata:\n", + " id: str = 55b9c387801541b4bd7e79a574ff60c4\n", + "\n", + "```" + ], + "text/plain": [ + "syft.service.metadata.node_metadata.NodeMetadata" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sy.deserialize(sy.serialize(node_metadata_v1, to_bytes=True), from_bytes=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "a007db0f", + "metadata": {}, + "outputs": [], + "source": [ + "node_metadata_v2 = NodeMetadataV2(\n", + " name=\"OM\",\n", + " highest_version=2,\n", + " lowest_version=1,\n", + " id=sy.UID(),\n", + " verify_key=sy.SyftSigningKey.generate().verify_key,\n", + " syft_version=\"0.8.2\",\n", + " node_type=\"domain\",\n", + " signup_enabled=True,\n", + " admin_email=\"info@openmined.org\",\n", + " node_side_type=\"low_side\",\n", + " show_warnings=False,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "850e094a", + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "```python\n", + "class NodeMetadataV2:\n", + " id: str = 42543d06d6624d3eba4960214d8d4b44\n", + "\n", + "```" + ], + "text/plain": [ + "syft.service.metadata.node_metadata.NodeMetadataV2" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sy.deserialize(sy.serialize(node_metadata_v2, to_bytes=True), from_bytes=True)" + ] + }, + { + "cell_type": "markdown", + "id": "c6e562a5", + "metadata": {}, + "source": [ + "### Migrating to different versions" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "793dd7b5", + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "```python\n", + "class NodeMetadataV2:\n", + " id: str = 55b9c387801541b4bd7e79a574ff60c4\n", + "\n", + "```" + ], + "text/plain": [ + "syft.service.metadata.node_metadata.NodeMetadataV2" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "node_metadata_v1.migrate_to(version=2)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "b8b4d067", + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "```python\n", + "class NodeMetadata:\n", + " id: str = 42543d06d6624d3eba4960214d8d4b44\n", + "\n", + "```" + ], + "text/plain": [ + "syft.service.metadata.node_metadata.NodeMetadata" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "node_metadata_v2.migrate_to(version=1)" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "560dd0ca", + "metadata": {}, + "outputs": [], + "source": [ + "node = sy.Orchestra.launch(\"test-domain\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "3607e5bb", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "dict_keys(['QueueItem', 'User', 'NodeSettings', 'Dataset', 'UserCode', 'Request', 'DataSubject', 'NodePeer', 'UserPolicy', 'Notification', 'DataSubjectMemberRelationship', 'Project', 'CodeHistory', 'BlobStorageEntry'])" + ] + }, + "execution_count": 33, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "node.python_node.document_store.partitions.keys()" + ] + }, + { + "cell_type": "markdown", + "id": "18e563aa", + "metadata": {}, + "source": [ + "### Migration Detection and Migrating Data\n", + "\n", + "SyftObjectMigrationState\n", + "- canonical_name:\n", + "- current_version:\n", + "\n", + "|Name | Version\n", + "----------|--------\n", + "|User |1\n", + "|Dataset |1\n", + "|Project |2\n", + "\n", + "\n", + "- UserV2\n", + "\n", + "\n", + "- Migration to latest version:\n", + " SyftObjectMigrationState -> current version\n", + " 1 -> 2\n", + " All data is migrated to version 2\n", + " update the state to SyftObjectMigrationState current version 2\n", + " \n", + "|Name |Version\n", + "----------|--------\n", + "|User |2\n", + "|Dataset |1\n", + "|Project |2\n", + "\n", + "\n", + "Automatic Migration vs Manual Migration:\n", + "\n", + "For now we can move with Automatic Migrations where\n", + "- Setting new fields to default values\n", + "- Having transforms for handling unique fields" + ] + }, + { + "cell_type": "markdown", + "id": "ed0ae09e", + "metadata": {}, + "source": [ + "### Server is running a different version from the client\n", + "\n", + "- Client -> 0.8.2 -> UserV1\n", + "- Server -> 0.8.3 -> UserV2" + ] + }, + { + "cell_type": "markdown", + "id": "8573a9e2", + "metadata": {}, + "source": [ + "```python\n", + "@service_method():\n", + "def create(\n", + " self, \n", + " context: AuthContext, \n", + " create_user: Union[CreateUserV1, CreateUserV2]\n", + ") -> Union[CreateUserV1, CreateUserV2]:\n", + " \n", + " # we just need to take of care of \n", + " # data migrations of the table only\n", + " client_version = context.client_version\n", + " server_version = context.server_version\n", + " \n", + " create_user: UserV1 = user_create.migrate_to(server_version)\n", + "\n", + " user: UserV1 = self.stash.set(create_user, context.credentials)\n", + " \n", + " # no need for any migrations from \n", + " # Viewable/Intermediate Objects like UserViewV2 to UserViewV1\n", + " return user.migrate_to(client_version)\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "dbce0550", + "metadata": {}, + "source": [ + "api.services.user.create(UserV2)\n", + "\n", + "\n", + "```markdown\n", + "client -> {“User”: [1], …..{“NodeMetadata”: [1,2]..}},\n", + "\n", + "client -> \n", + " args and kwargs:\n", + " - calculate the versions of the args and kwargs\n", + " - store that in a dict\n", + " - e.g. \n", + " kwargs = {\"user_create\": {\"UserCreate\": 1},}\n", + " return_annotation: {\"UserView\": [1]}\n", + "```\n", + "\n", + "1. Client version is behind the server version\n", + "\n", + "\n", + "2. Server version is behind the client version\n", + " - Serialization\n", + " - Migration\n", + " \n", + "Possible Solution: \n", + " - We perform the migration transform on the client itself and serialize/deserde becomes easier.\n", + " - Server sends the ServerObjectVersionMap: {“User”: [1], …..{“NodeMetadata”: [1,2]..}}\n", + " - Client identifies if server is running a lower version:\n", + " - migrate to the supported version of object on client side\n", + " e.g. UserV2 (Client) on client side -> UserV2.migrate_to(version=1) -> send the data to the server. UserV1(server)\n", + " - otherwise server takes care of the migration\n", + " \n", + "`\n", + " ServerObjectVersionMap = {“User”: [1], …..{“NodeMetadata”: [1,2]..}}\n", + "\n", + "`\n", + "\n", + "this information comes along with Metadata: {“User”: [1], …..{“NodeMetadata”: [1,2]..}} and cache it a client level." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "09741c6e", + "metadata": {}, + "outputs": [], + "source": [ + "class User:\n", + " __canonical_name__ = \"User\"\n", + " __version__ = 1\n", + " \n", + "class UserV2:\n", + " __canonical_name__ = \"User\"\n", + " __version__ = 2\n", + "\n", + "ProtocolVersionMap\n", + "{\n", + " 1: {“User”: 1, …., “NodeMetadata”: 1..},\n", + " 2: {“User”: 1, …., “NodeMetadata”: [1, 2, 3]..},\n", + " 3: {“User”: [1, 2], …., “NodeMetadata”: [1, 2, 3]..},\n", + " 4: {\"NodeMetdata\": 3},\n", + "}\n", + "\n", + "server protocol versions -> [1, 2, 3, 4]\n", + "\n", + "Client\n", + "{\n", + " 1: {“User”: 1, …..“NodeMetadata”: 1..}, -> derived from the SyftMigrationRegistry\n", + " 2: {“User”: 1, …..“NodeMetadata”: [1, 2,3]..},\n", + "}\n", + "\n", + "client protocol versions -> [1, 2]\n", + "\n", + "\n", + "communication protocol: 2 -> Highest of the intersection of server and client\n", + " \n", + "\n", + "ProtocolVersionMap\n", + "{\n", + " 4: {\"NodeMetdata\": 3},\n", + "}\n", + "\n", + "server protocol versions -> [4]\n", + "\n", + "Client\n", + "ProtocolVersionMap:\n", + "{\n", + " 1: {“User”: 1, …..“NodeMetadata”: 1..}, -> derived from the SyftMigrationRegistry\n", + " 2: {“User”: 1, …..“NodeMetadata”: 3..},\n", + "}\n", + "\n", + "client protocol versions -> [1, 2]\n", + "\n", + "\n", + "communication protocol: No intersection -> Highest of the intersection of server and client\n", + " \n", + " \n", + "SyftMigrationRegistry:\n", + " - __migration_version_registry__: {\n", + " \"User\": {\"1\": \"...\", \"2\": \"....\"},\n", + " \"NodeMetadata\": {\"1\": \"...\", \"2\": \"....\", \"3\": \"....\"}\n", + " }\n", + " \n", + "\n", + "ProtocolVersionMap.lastest_object_map_hash -> efgdfaf\n", + " - {\"User\": 2, \"NodeMetadata\": 3} -> hash -> asdasfaf\n", + "\n", + " \n", + ">> syft upgrade protocol\n", + "\n", + "\n", + "\n", + "--->\n", + "migrations/01.py\n", + " - {\n", + " \"hash\": \"sdasrafa\",\n", + " \"object_versions\": {\"User\": 1, \"NodeMetadata\": 1},\n", + " \"protocol_version\": 1\n", + " }\n", + "migrations/02.py\n", + " - {\n", + " \"depends_on\": \"01.py\"\n", + " \"hash\": \"effasfa\",\n", + " \"object_versions\": {\"User\": 1, \"NodeMetadata\": 3},\n", + " \"protocol_version\": 2\n", + " }\n", + "\n", + "protocol_state.json\n", + "1: {\n", + " \"hash\": \"effasfa\"\n", + " \"object_versions\": {\"User\": 1, \"NodeMetadata\": 1},\n", + " \n", + "},\n", + "\n", + "2: {\n", + " \"hash\": \"12dsad4\",\n", + " \"object_versions\": {\"User\": 1, \"NodeMetadata\": 3},\n", + "},\n", + "\n", + "protocol_state.json\n", + "1: {\n", + " \"hash\": \"effasfa\"\n", + " \"object_versions\": {\"User\": 1, \"NodeMetadata\": [1]},\n", + " \"supported\": True\n", + "},\n", + "2: {\n", + " \"hash\": \"12dsad4\",\n", + " \"object_versions\": {\"User\": 1, \"NodeMetadata\": [1, 2]},\n", + " \"supported\": True\n", + "},\n", + "3: {\n", + " \"hash\": \"5235sad4\",\n", + " \"object_versions\": {\"User\": [1, 2], \"NodeMetadata\": [1,2,3]},\n", + " \"supported\": False\n", + "},\n", + "4: {\n", + " \"hash\": \"5235sad4\",\n", + " \"object_versions\": {\"User\": [1], \"NodeMetadata\": [1,2,3]},\n", + " \"supported\": False\n", + "}\n", + "# 5: {\n", + "# \"hash\": \"asd23144\",\n", + "# \"object_versions\": {\"NodeMetadata\": [1, 2, 3]},\n", + "# \"stale\": False \n", + "# }\n", + "\n", + " \n", + "client: [1, 2, 3, 4]\n", + "server: [5]\n", + "communication protocol:\n", + " No intersection -> Highest of the intersection of server and client\n", + "\n", + " \n", + "- a. When we remove an Object type altogether. e.g deleted user classes.\n", + "- b. When we remove a version of the Canonical Object. e.g. deleted version 1 of the User class.\n", + "\n", + "```\n", + "client or server:\n", + " - If the version of the object being send doesn't match the version of the object in given protocol\n", + " then we migrate to the given version\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "eecfc749", + "metadata": {}, + "outputs": [ + { + "ename": "SyntaxError", + "evalue": "invalid syntax (2320814170.py, line 1)", + "output_type": "error", + "traceback": [ + "\u001b[0;36m Input \u001b[0;32mIn [1]\u001b[0;36m\u001b[0m\n\u001b[0;31m SyftMigrationRegistry:\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n" + ] + } + ], + "source": [ + "SyftMigrationRegistry:\n", + " __migration_version_registry__ : {\n", + " \"User\": {1: \"\", 2: \"\"}\n", + " }\n", + "# >> sy.reset_protocol()\n", + "# >> sy.update_protocol()\n", + "# >> sy.update_protocol()\n", + "\n", + "\n", + "protocol_state.json\n", + "1: {\n", + " \"hash\": \"effasfa\"\n", + " \"object_versions\": {\"User\": [1], \"NodeMetadata\": [1]},\n", + " \"stale\": False\n", + "},\n", + "2: {\n", + " \n", + " \"hash\": \"12dsad4\",\n", + " \"object_versions\": {\"User\": [1], \"NodeMetadata\": [1, 2]},\n", + " \"stale\": False\n", + "},\n", + "3: {\n", + " \n", + " \"hash\": \"5235sad4\",\n", + " \"object_versions\": {\"User\": [1, 2], \"NodeMetadata\": [1,2,3]},\n", + " \"stale\": True\n", + "},\n", + "4: {\n", + "\n", + " \"hash\": \"5235sad4\",\n", + " \"object_versions\": {\"User\": [3], \"NodeMetadata\": [1,2,3]},\n", + " \"stale\": False\n", + "}\n", + "5: {\n", + " \"hash\": \"13124214\",\n", + " \"object_versions\": {\"User\": [2, 3], \"NodeMetadata\": [1, 2, 3]},\n", + " \"stale\": False\n", + " \n", + "}\n", + " \n", + "# Field Stale is marked False by default. Its set to True:\n", + " - check if in the current object versions and previous object versions:\n", + " - if a version is missing for an object\n", + " - or the canonical name is altogether is missing.\n", + " \n", + "# client: [1, 2, 3, 4]\n", + "# server: [5]\n", + "# communication protocol:\n", + "# No intersection -> Highest of the intersection of server and client\n", + "\n", + "\n", + "# client: [1]\n", + "# server: [1, 2]\n", + "# latest_protocol_on_server: 2\n", + "# latest_protocol_on_client: 1\n", + "# communication protocol: [1]\n", + "\n", + "\n", + "# Checking this on both client and server\n", + "if communication protocol < latest_protocol_on_server:\n", + " migration can happen on server\n", + "else:\n", + " migration can happens on client\n", + " \n", + "Based on the communication protocol derive,\n", + " - client_version for an object\n", + " - server_version = version in the latest_protocol\n", + " \n", + "\n", + "\n", + "@service_method():\n", + "def create(\n", + " self, \n", + " context: AuthContext, \n", + " create_user: Union[CreateUserV1, CreateUserV2]\n", + ") -> Union[CreateUserV1, CreateUserV2]:\n", + "\n", + " # we just need to take of care of \n", + " # data migrations of the table only\n", + " client_version = \n", + " server_version = \n", + "\n", + " create_user: UserV1 = user_create.migrate_to(server_version)\n", + "\n", + " user: UserV1 = self.stash.set(create_user, context.credentials)\n", + "\n", + " # no need for any migrations from \n", + " # Viewable/Intermediate Objects like UserViewV2 to UserViewV1\n", + " return user.migrate_to(client_version)\n", + "\n", + "\"object_versions\": {\n", + " \"User\": {\n", + " 1: \"sadadefafa\", \n", + " 2: \"sdasd24124\",\n", + " 3: \"asdadasafhh\"\n", + " }\n", + "}\n", + "\n", + "# hash of the object:\n", + "# hash(\n", + "# \"__canonical_name__\", __version__,\n", + "# tuple(unique_keys), field_name and field type\n", + "# )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3f9b679a", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fe2adf3f", + "metadata": {}, + "outputs": [], + "source": [ + "# \n", + "\n", + "class Weird {\n", + "name: str\n", + "data: TFTensor\n", + "}\n", + "class Weird {\n", + "name: str\n", + "data: bytes\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d5190d7d", + "metadata": {}, + "outputs": [], + "source": [ + "## Increasing versions\n", + "# hash comparision during start up\n", + "# pick the highest compatible protocol\n", + "# protocol.dev.json" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "791324b5", + "metadata": {}, + "outputs": [], + "source": [ + "# add a new version and test for upgrade_protocol and validate the communication b/w client and server \n", + "# two branches\n", + "# - first branch: add a new version to one object\n", + "# - second branch: add a new version to second object\n", + "# - merge branch: generate a final protocol state\n", + "# - validate the communication b/w client and server " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4ed6932c", + "metadata": {}, + "outputs": [], + "source": [ + "SyftMigrationState:\n", + " current_version\n", + " canonical_name\n", + " protocol_version" + ] + }, + { + "cell_type": "markdown", + "id": "d8642547", + "metadata": {}, + "source": [ + "### Implementing communication protocol" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b0981778", + "metadata": {}, + "outputs": [], + "source": [ + "# Checking this on both client and server\n", + "\n", + "\n", + "# During get_api, pass communication protocol\n", + "# on the server side:\n", + "# for_user(credentials, protocol):\n", + "# for given communication_protocol\n", + "# migrate the signature of the endpoints based on communication_protocol.\n", + "\n", + "if communication protocol < latest_protocol_on_client:\n", + " migration can happens on client\n", + " # client: [1, 2]\n", + " # server: [1]\n", + " # latest_protocol_on_server: 1\n", + " # latest_protocol_on_client: 2\n", + " # communication protocol: [1]\n", + " # Iterate over the Input args and kwargs\n", + " # and downgrade the version to given communication protocol\n", + " user_v1 = user.using_protocol(version=communication_protocol)\n", + " # Recieve the result from the server and upgrade the version of the object\n", + " return user_v1.using_protocol(version=latest_protocol_on_client)\n", + "else:\n", + " migration can happen on server\n", + " # client: [1]\n", + " # server: [1, 2]\n", + " # latest_protocol_on_server: 2\n", + " # latest_protocol_on_client: 1\n", + " # communication protocol: [1]\n", + " migration_required = communication_protocol < latest_protocol_on_server\n", + " if migration_required:\n", + " user_v2 = user.using_protocol(version=latest_protocol_on_server)\n", + " # - object_version = get_object_version_for_protocol(\"User\", latest_protocol_on_server)\n", + " # get latest version of the object for given protocol\n", + " # - user.migrate_to(object_version)\n", + " ...\n", + " ...\n", + " ..\n", + " return user_v2.using_protocol(version=communication_protocol)\n", + " # - object_version = get_object_version_for_protocol(\"User\", latest_protocol_on_server)\n", + " # get latest version of the object for given protocol\n", + " # - user.migrate_to(object_version)\n", + "\n", + " \n", + " \n", + "Based on the communication protocol derive,\n", + " - client_version for an object\n", + " - server_version = version in the latest_protocol\n", + " \n", + "\n", + "@service_method():\n", + "def create(\n", + " self, \n", + " context: AuthContext, \n", + " create_user: Union[CreateUserV1, CreateUserV2]\n", + ") -> Union[CreateUserV1, CreateUserV2]:\n", + "\n", + " # we just need to take of care of \n", + " # data migrations of the table only\n", + " client_version = \n", + " server_version = \n", + "\n", + " create_user: UserV1 = user_create.migrate_to(server_version)\n", + "\n", + " user: UserV1 = self.stash.set(create_user, context.credentials)\n", + "\n", + " # no need for any migrations from \n", + " # Viewable/Intermediate Objects like UserViewV2 to UserViewV1\n", + " return user.migrate_to(client_version)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0fae05c9", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "55064bc7", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.1" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/Experimental/Untitled.ipynb b/notebooks/Experimental/Untitled.ipynb new file mode 100644 index 00000000000..9de8340e952 --- /dev/null +++ b/notebooks/Experimental/Untitled.ipynb @@ -0,0 +1,1085 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "853b5397-fd94-4bce-9ae5-ae16d9c201df", + "metadata": {}, + "outputs": [], + "source": [ + "import syft as sy\n", + "from syft.types.datetime import DateTime\n", + "from syft.service.metadata.node_metadata import NodeMetadata, NodeMetadataV1\n", + "from syft.service.metadata.node_metadata import NodeMetadataJSON" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "d82eecfc-9d40-4fbb-a868-a1d175a745db", + "metadata": {}, + "outputs": [], + "source": [ + "from syft.protocol.data_protocol import DataProtocol" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "0fa4a070-8485-4f44-8d96-f3d28ede2fb3", + "metadata": {}, + "outputs": [], + "source": [ + "from syft.service.service import BaseConfig" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "a5632f9f-a175-4e96-b7ef-7e825ccfc50c", + "metadata": {}, + "outputs": [], + "source": [ + "from syft.store.dict_document_store import DictStoreConfig" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "0b71f13e-00d9-4e7a-8b9f-b9d7f0aff1f3", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
SyftSuccess: 110 Protocol Updates Staged to dev

" + ], + "text/plain": [ + "SyftSuccess: 110 Protocol Updates Staged to dev" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sy.stage_protocol_changes()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "14959468-9727-47d1-a89c-e0902cd9f4be", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
SyftSuccess: 0 Protocol Updates Staged to dev

" + ], + "text/plain": [ + "SyftSuccess: 0 Protocol Updates Staged to dev" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sy.stage_protocol_changes()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "019568c9-8a44-4cf7-a879-cae67e808170", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
SyftSuccess: Protocol Updated to 1

" + ], + "text/plain": [ + "SyftSuccess: Protocol Updated to 1" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sy.bump_protocol_version()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7e892015-d737-4f8a-babe-d61c5ffd9f9e", + "metadata": {}, + "outputs": [], + "source": [ + "x = sy.get_data_protocol()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "06da56ae-db2a-4723-8d70-9ad8b8edc064", + "metadata": {}, + "outputs": [], + "source": [ + "x.supported_protocols" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7e064841-d1dc-4048-8477-47dc0a75eb3e", + "metadata": {}, + "outputs": [], + "source": [ + "x.state" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "56dfe0f3-74e0-40e5-af26-42a7a12971b7", + "metadata": {}, + "outputs": [], + "source": [ + "x.state[\"PartialSyftObject\"]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8c8d38a1-4f50-4607-9752-c202dd3e231b", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d795af5b-ebf6-4454-8379-194976611a3d", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "58791646-a7d6-4925-b6d3-f804b5df8e51", + "metadata": {}, + "outputs": [], + "source": [ + "import re" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8af3d1cb-fb45-4ae1-aa69-f4ebdf9ffa3d", + "metadata": {}, + "outputs": [], + "source": [ + "def natural_key(key: int | str) -> list[int]:\n", + " \"\"\"Define key for natural ordering of strings.\"\"\"\n", + " if isinstance(key, int):\n", + " key = str(key)\n", + " return [int(s) if s.isdigit() else s for s in re.split(\"(\\d+)\", key)]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d61312af-7a8e-4290-8a4d-3477b5990f29", + "metadata": {}, + "outputs": [], + "source": [ + "x = [\"dev\"]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a0d2ac8e-4640-4d19-97e0-947832078921", + "metadata": {}, + "outputs": [], + "source": [ + "sorted(x, key=natural_key)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e391bf8a-9870-4d36-872c-96969878a515", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5c7a5f3f-09b4-43d7-abfb-81fa3f779eed", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e4b6b77b-b115-4e3c-a4a0-592c035edc1f", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "faf2e3ff-dd52-4105-a8f6-518594dcd7d0", + "metadata": {}, + "outputs": [], + "source": [ + "raise Exception" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1481115f-2f91-4809-87cb-1bbea37d57cc", + "metadata": {}, + "outputs": [], + "source": [ + "import re\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b51c2dc1-8c7c-4884-b7ab-abb190816e0e", + "metadata": {}, + "outputs": [], + "source": [ + "def natural_key(key: int | str) -> list[int]:\n", + " \"\"\"Define key for natural ordering of strings.\"\"\"\n", + " if isinstance(key, int):\n", + " key = str(key)\n", + " return [int(s) if s.isdigit() else s for s in re.split(\"(\\d+)\", key)]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "af95e3b8-daf9-4b3a-8d61-d185698b37da", + "metadata": {}, + "outputs": [], + "source": [ + "a = [\"1\", \"dev\", \"2\", \"11\"]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ed9e1877-6569-47e3-acc5-e85f3e2b19a4", + "metadata": {}, + "outputs": [], + "source": [ + "b = sorted(a, key=natural_key)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bb05d920-19f1-48ee-9f7e-f5de51ade4ce", + "metadata": {}, + "outputs": [], + "source": [ + "b" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "71ed4308-8045-42f0-9900-fa2cafb6f227", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "46a024d7-90a3-4d95-ba5e-1de1e93bc4ad", + "metadata": {}, + "outputs": [], + "source": [ + "DataProtocol._calculate_object_hash(BaseConfig)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d1ac72f4-42ff-4975-b70f-fe98d79195b1", + "metadata": {}, + "outputs": [], + "source": [ + "from syft.serde.recursive import TYPE_BANK" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e78c45a3-b59a-43c5-9719-9bccb0ca75dc", + "metadata": {}, + "outputs": [], + "source": [ + "g = {\"1\":\"a\", \"22\":\"b\", \"11\":\"c\"}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7fbdf02a-f7be-4b92-8ec8-074accbf7c4f", + "metadata": {}, + "outputs": [], + "source": [ + "max(g)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b07b2cb0-d075-45c4-ab48-5716c77da5b8", + "metadata": {}, + "outputs": [], + "source": [ + "range(2, 23)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2abfbcb1-af70-4227-b978-002e1979ca17", + "metadata": {}, + "outputs": [], + "source": [ + "(\n", + "nonrecursive,\n", + "serialize,\n", + "deserialize,\n", + "attribute_list,\n", + "exclude_attrs_list,\n", + "serde_overrides,\n", + "hash_exclude_attrs,\n", + "cls,\n", + "attribute_types,\n", + "version,\n", + ") = TYPE_BANK[\"syft.service.service.BaseConfig\"]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4a8084d8-e241-4372-9686-5cb5b0286cac", + "metadata": {}, + "outputs": [], + "source": [ + "cls" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b07f2ea-adac-42c5-9ac7-a18f64fc8c05", + "metadata": {}, + "outputs": [], + "source": [ + "cls == BaseConfig" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0edc9c91-73cc-4c21-bbb7-20670e6ef292", + "metadata": {}, + "outputs": [], + "source": [ + "DataProtocol._calculate_object_hash(cls)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0710f746-74f8-4b9d-b25b-9662517816f3", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "# Exception: BaseConfig version 1 hash has changed.\n", + "\n", + "# 332964663526a4d6ddc64bc9d77aa9324000bb721520a5465229235523fe8c7d not in \n", + "\n", + "# dict_values([{'version': 1, 'hash': 'f35db90c1c3bdd993b7b1e4f28327a8f6b1d0e43115b044167ca28d5c740178a', 'action': 'add'}]). You probably need to bump the version number.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cd615d46-f6f7-4888-bf1d-1e264b380c11", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7389ca79-d161-4e87-9a27-f24a6b3a2a40", + "metadata": {}, + "outputs": [], + "source": [ + "raise Exception" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "73f5e981-30a8-43f2-b762-69954dcc7724", + "metadata": {}, + "outputs": [], + "source": [ + "x = {'PartialSyftObject': {1: 'fa2770d76f3dd904e2c1c4a78c9ba10e5ac33b4dd7a4d61faa45a22078e94aa8'}, 'NodeMetadataUpdate': {1: 'b73adf379a179f6ef9a6d5ee9477e69697a47ad25579bab46cc1fc44ec35ba04'}, 'NodeMetadata': {2: '3986c7d8b3cf9355a1fbdd99dfe3d872fc464420b91a668ea3288ee4481bab6d'}, 'LinkedObject': {1: '8a117f8bf0282b8cf525de893404dbd88cc460a5a823d83006c2e82d88846f8d'}, 'BaseConfig': {1: '32b71d31ecba9a29c82fc4ca0e6568ac5c97204501ffdfef079c93ffbaf6b278'}, 'APIEndpoint': {1: 'c88e4405839e87fdfe90f86877ef2addd7be7281d36b7891636129fc8b3c1e8c'}, 'LibEndpoint': {1: '214b81bda5fd00e7734227a4c75981d807f3741039be5b4fb007076cf2e638cc'}, 'SignedSyftAPICall': {1: '2be9b74663354b4edeef3bc75dc67dc35bf24890c8a86a53a97957d470af06b2'}, 'SyftAPICall': {1: 'fa9520d29d9df56fb9d5d2080aecfc3be14c49e7267a3e5b9fd05ad1b0828e11'}, 'SyftAPIData': {1: '16933cd10a2c45ad44826480e88e06ffbb7a7762c058c7e84da94ddc70478e7c'}, 'SyftAPI': {1: '142a9bb5a4a63d5d072fbfbdbb48ec9535f1ae51e40f9d4f59760fb807c9a189'}, 'User': {1: '21cb3659dc4ddd0b4d58c677f46feecac9f682ea36660f42845067b29b5ad8e7'}, 'UserUpdate': {1: 'f12c19dd38330f98fb2d9e0bf47e8bdcad3f6e1c085d2994d80224cf4b905984'}, 'UserCreate': {1: 'a9d7a52aaa7dcf622e317e899e1ded3023a94b86773ca16cd7d6a334fcffbe8b'}, 'UserSearch': {1: 'e697bf5b287cf29560c94c5851d8fb6ac74d2ce5c6200539a11a257bc150c75b'}, 'UserView': {1: 'fd624963af09c0e3471dfc49b2f09fafdd7521c8af804198015cc5455d7b56bc'}, 'UserViewPage': {1: 'c856f13ddc9a405b6d52482a6a273ffb038f988d589e7bb5cd68e0c8dd8668de'}, 'UserPrivateKey': {1: 'e06f237cdfd516caff0766b5f8ba4e4b2b85824c76910f1374ffce72173f8269'}, 'StoreConfig': {1: '91790b136dba4ddbb5bd8dfcd337a4a8e448cc26b338bf15bf066ba97be56924'}, 'NodeSettingsUpdate': {1: 'ae77fb9f24004635a29abd979f5efa5a75780efe4fec1773cc533ac04aa57482'}, 'NodeSettings': {1: '2735f660f23bfda8ffdf97b8ba3ab1bcdba01461b245832e9a9cb2a661ebcb74'}, 'HTTPConnection': {1: '0a6e181e67978ce45a2414be4c4c39272ca6ed38a5fe9e9877619c13dc6aafef'}, 'PythonConnection': {1: '6ab9e80d2208ce44fb6e5db8427234680f56b1ef8092be701d6d833b6e213926'}, 'DateTime': {1: '34f9942a3f75988a1de6e46c40697698f2505336cf74282f683cfd3a7d6d4ec1'}, 'BlobFile': {1: '229121eb07430f72c66281764a690270ff821a6551c036528b9e749b343bedc1'}, 'SecureFilePathLocation': {1: 'd3e6b95de5da0861922c302e9dabf443ee337b21da695d69c85bdb1e6f0ec45b'}, 'SeaweedSecureFilePathLocation': {1: '0d5c382191c63e68b90237bb4e882abea6311ff1ba645adc784ee272de5f4623'}, 'BlobStorageEntry': {1: 'e010b50076f73fb934029a583310d13c3ec7abaa93520090fae3fb16457868fc'}, 'BlobStorageMetadata': {1: 'f1d0b4085276ba5f15a8cd81553374d465317d96e0a1c427e33e2d866e362d22'}, 'CreateBlobStorageEntry': {1: '183fd3ed16b0687f01979b3b76bbb17a9b029bd39f11d46faf54b3e5205e9e2d'}, 'BlobRetrieval': {1: 'c55f486ea79e9e96c047b32987e555357248cd30f18e1a8244fefe73457e5b9e'}, 'SyftObjectRetrieval': {1: 'da96484c8e57fc060c4dba29ef4e375284720dd05f1ed2ee60e1df52450437cd'}, 'BlobRetrievalByURL': {1: '656a44e91ce560056679b459c9fd33b55a5c2b0754e455099a456074e2e14822'}, 'BlobDeposit': {1: '23a73cc9bff8e6833681e55d872121b6d54520d76f9426fd199a19eb847deea4'}, 'WorkerSettings': {1: '2fe75dd39cb6367bd9cea2c7f59e40a85bbbcfc44f518572f377ef25c3acd205'}, 'HTTPNodeRoute': {1: 'b4662c11f7487ab907caf3cadf8c33eca2e0fbd640ed1fba872c3f982b749986'}, 'PythonNodeRoute': {1: 'd8f268996e5443a248cc2eb5f4a568591e5f9e18952f8519f3f995e1f1f486e4'}, 'EnclaveMetadata': {1: '6ad19306231ebbb4d8b0c4e4cc82b881298835862a2c48f2358db47215a020ac'}, 'DataSubject': {1: '287ab306e1c4ebe0c883600ffd52dc734d08f0217b2a961afbdc6c7883bf4ccd'}, 'DataSubjectCreate': {1: '8b3487af42ba249d7cf705c7f66a09dd832c055814091def34a38133f8148158'}, 'DataSubjectMemberRelationship': {1: '6aed0e1548d6b09bfac132040f7315f49c13855a6bc147a4f1aa4ce09572b952'}, 'Contributor': {1: '3e27f1ea23cecfc3e0956743ae832f0f70ecd144f9df3f128b123e9347944afa'}, 'MarkdownDescription': {1: '506d47fa85728ad444f2fa657e39b341bc759d21a80325831b1e84926ee748f1'}, 'Asset': {1: 'f8370e8dd87df6a05bde1226c52c3ce6e7de636e6032341b977efe5e601a731d'}, 'CreateAsset': {1: 'c3a82856694adcb3c916a961776c2fa3bc90a7ccb50b8b9d42776810266ed241'}, 'Dataset': {1: 'd7a45bf9128472715e63a67192d4181672eadd8b5873d2ba96c2df3b2be749b9'}, 'DatasetPageView': {1: 'c7da1fac28f70c19d199f123b04fbd4a9c7681e3846dee0de70ea926a9440a2e'}, 'CreateDataset': {1: '6a31617de99738bc176f19397884581051443c7c7ba952c983929fed688a9d7e'}, 'ActionDataEmpty': {1: 'fc83d91ac6ba78c749a4e7e16c1aa117baaae62b5f33156ded4e5a79b9532a98'}, 'ActionFileData': {1: '47a0a5f9fb48de09885d4f9a6c5a5a05f4dd04575ea42df7dea0cab02817063f'}, 'Action': {1: '204b3c97b41f208ecb17c541af55e171675faaefa10c103b405a4077b0226a7e'}, 'AnyActionObject': {1: 'f11bd6135ba46247c06bfa8a3c6b7f2e540a1033afe0512c3359e31eb3d59388'}, 'TwinObject': {1: '8f6abd756d41f9639865c6fd55e6cc2ec6b89fd18bac3c77bf3a0502d81ca8ec'}, 'ExactMatch': {1: 'f71a495d2452190596fe435eaf59b07ba390d7895f6c808a2f87a1719227ba9c'}, 'OutputHistory': {1: '24f282dd181ecc7a05c93c02dff84dff45aa52928e2331a15ef667645e9bcf0b'}, 'OutputPolicyExecuteCount': {1: '95d198d2021dbaf9965df39a4571ad06e787684ff79bd6e8a720c47825eebd7e'}, 'OutputPolicyExecuteOnce': {1: 'b8bc1fea2e9b51b5dfc3cbd4b0a131cb2a5b1fe746b5e329395bf319b38bf9b2'}, 'UserPolicy': {1: 'ec3f8ea84e2b05ce56db8f35cff14f78569e921f566477581b3299eb6a9fa053'}, 'SubmitUserPolicy': {1: '3147a4e45270367a40ca8c4b7b502c8683200d123828720365521b90b2775794'}, 'UserCode': {1: 'ad509fccb7bb5c00971453c1f1235785f40d5d5b3eee3df1dc9edafc758c7193'}, 'SubmitUserCode': {1: 'b4a919054beb2488c7b4839d60c494400d791112adf009631dce13787cd58e78'}, 'UserCodeExecutionResult': {1: '9aab24def6616ac908ef1100b654b4dbeca1ea4cfff229c53d6b14491c795555'}, 'CodeHistory': {1: 'bbbd59801717a98691034a37c9de388c7a65db2721bd0c5c9ff0dbe8fc70be67'}, 'CodeHistoryView': {1: '142e78415da10dae739e923d39ce511496a3c7b31e8c4553a6cbb1907c126a3a'}, 'CodeHistoriesDict': {1: '453af101a1de8e37a0bfacf22077c35994718791f295956f1f06727f8d9b7fe8'}, 'UsersCodeHistoriesDict': {1: 'cf8ef92a08cabb068e4616c1700451b75ba4d511343830e3e56b65882fb784aa'}, 'NodePeer': {1: '8920d9e456fd1a13f46c0528a8fe3fec8af46440b3eb89e8d7d48ad64babee1e'}, 'CommandReport': {1: 'a81fe3d0cc5796f45e925d09b6e8132b79fe5df0f341d55b3870c109f8c1e19d'}, 'CommandResult': {1: '14b48d4a1cbc5f5ae1e5e74834e7f1002adae7b2766000ea822f180fd7cd39db'}, 'VPNClientConnection': {1: '7d44711978f930d42c06d04483abcdb1f230782a8f16f4feb5efb7b2b2093bb2'}, 'HeadscaleAuthToken': {1: '0b363503b6c611b44e33561a2a6db3f260cfd4bbc5f4245deac5052fd5149803'}, 'TailscalePeer': {1: '8ff85aa2b913a6bb206b9de0495d9f74a17f55823891da98cb6fdbe78f46a44b'}, 'TailscaleStatus': {1: 'ed262f4b9a569d9933f4a86cd2caa2ce213fc7a2319a1371f6a3cf3ccf884c8a'}, 'OnDiskBlobDeposit': {1: 'da3abda453def0d7c70c8a5dfcc3c8d00dd6822f60ddc01be3bdead4b0b5b482'}, 'SeaweedFSBlobDeposit': {1: 'bcbec5dcdc06a0c87f89a10a6a8809706f24cedd97b5f850f8b48840a1f41941'}, 'NumpyArrayObject': {1: 'd47a376401d92d47e5748e34f98ee270f8ebfd52cffbe6271b5faa8193e728c5'}, 'NumpyScalarObject': {1: '952bebb4dd3e3641c33b4ebcf2c051dbdebae5f1bf3b7b63ea89423360705411'}, 'NumpyBoolObject': {1: 'b7a231baaa4b1f519d70c5afb15b4a9b7232f1128f7fd3709c1ea8b7345f8c6c'}, 'PandasDataframeObject': {1: 'ff9d6c1884413f712d95d29190e30938b33de19e11dff9f88d9b89c51499cac5'}, 'PandasSeriesObject': {1: '69eadfe14e5a7035767d2538e2db8775da6569cf5127f58d13315c4b85e5603d'}, 'ReplyNotification': {1: 'ce1e2a6b0d618478d3b1b992e4c8605817919c88a4884ca0540e0886ecdb8215'}, 'Notification': {1: '1e5a65d91e27bf53d5b2ed0b45d9cee0cf77104b7111f99223194ceb0d0137fe'}, 'CreateNotification': {1: '6858b743ac07d853a0302dc64de0e7d852135e2564ebad325d5ff35d17c29f6f'}, 'Change': {1: '2c470ff8aa076b88105640ce79d361a9b439927e501c238fa33ac7c1c45aa2c0'}, 'ChangeStatus': {1: '7571229e92d8d52a0e90fa8856169b41045b42b50568b266823bdcea838dfb39'}, 'ActionStoreChange': {1: 'cf527995930cce09d90806713d30301493240079319bcc85e894428aee46017e'}, 'Request': {1: '340f4ac61ccbf6f566666327d6bca043dcd643e6f8e24897ef10bd6312e74995'}, 'RequestInfo': {1: 'd571708de3c187ca5840c64784d99f7bfce8f33aa2ba48f9d56b824564551654'}, 'RequestInfoFilter': {1: 'c336af8d474071eb61e5f467513753e64d4e153e12892f9c6875b235057b0f0a'}, 'SubmitRequest': {1: '1870ce541169eab04cb69d3ae88ea30dc2fcdd997b620567ca9d87936d9600cf'}, 'ObjectMutation': {1: '275d9cf180904d1e34e1f3d7e838105e843127faf5a64029a1cf85d00234b8c9'}, 'EnumMutation': {1: '3a1d1b47e0cdb5094298bce58bc9b76ecb66064459504f910c7d755eb1d5e276'}, 'UserCodeStatusChange': {1: '928dd4ceeb4858b18b806ca62b49a840f54269f7866744c9aa6edb0af9d7dfc1'}, 'SyftObjectMigrationState': {1: '194fd4dc57764d454ac763d256e3bfcd2b0040a134daf9ee0d8e5ac7ab21abbc'}, 'ProjectThreadMessage': {1: '9f8b11d603caae6d0e0f28957949dfc57c26fec9685f2c80501330b1d6bae665'}, 'ProjectMessage': {1: 'd678beafc33f7e7df7e771a82d5cba6d5a36728a033d3976b6e5998726733d27'}, 'ProjectRequestResponse': {1: '51b5a5d8cf0bde45abd2bd3a4411f93769fa542666a137ce9611d38fb48ffb4c'}, 'ProjectRequest': {1: '9eff1b3cc74c9706722363abb4062fc77c0a4f093d448b795ad662861649f111'}, 'AnswerProjectPoll': {1: 'f538a4fcae286cbc9755f51e2f2ce8809d66ce5d66f50173ef1824f89ce9b51d'}, 'ProjectPoll': {1: 'b456a699a249fd3fffe9739cdd9ec3ee8c05e59b2d9872ad9864167d78088091'}, 'Project': {1: 'bf59890e92d95b362cc7ef9c3d7fa6a1815978e02111a30cbcb047239e57d61e'}, 'ProjectSubmit': {1: '5084844056ddefcea7fc634dd9945c03ef6d030bcd8f63aa07fe11fea0a5389f'}, 'QueueItem': {1: '1d53446d5cd788120b15ea6b108a4a7abd480377370be7128f44297f8fb00b76'}, 'ZMQClientConfig': {1: 'e3153f18c9fd04cf07b844153d093c8a090baac4c99d71ecd6491961e7f1dafb'}, 'Plan': {1: '41713fc89a2cab7db592df6cd1c45e1309f86a50a8f531ddaf4052947186b0e0'}}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5db2bb72-f188-42fe-9f1c-e79150dddb0b", + "metadata": {}, + "outputs": [], + "source": [ + "import json" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f0b4e62d-548b-4ba5-97a0-ae81417c7346", + "metadata": {}, + "outputs": [], + "source": [ + "json.dumps(x)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "054fe585-2a32-4f18-af71-1a944a7e94d6", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "# Exception: BaseConfig version 1 hash has changed.\n", + "# 332964663526a4d6ddc64bc9d77aa9324000bb721520a5465229235523fe8c7d not in \n", + "# dict_values([{'version': 1, 'hash': 'f35db90c1c3bdd993b7b1e4f28327a8f6b1d0e43115b044167ca28d5c740178a', 'action': 'add'}]). You probably need to bump the version number.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4b89e304-d8f1-42ab-86c0-3552500e6853", + "metadata": {}, + "outputs": [], + "source": [ + "# k syft.service.service.ServiceConfig BaseConfig 1 332964663526a4d6ddc64bc9d77aa9324000bb721520a5465229235523fe8c7d\n", + "# got versions for canonical name BaseConfig {1: {'version': 1, 'hash': 'f35db90c1c3bdd993b7b1e4f28327a8f6b1d0e43115b044167ca28d5c740178a', 'action': 'add'}}\n", + "# is versoin in versions keys 1 dict_keys([1]) True\n", + "# is hash in hash values 332964663526a4d6ddc64bc9d77aa9324000bb721520a5465229235523fe8c7d dict_values([{'version': 1, 'hash': 'f35db90c1c3bdd993b7b1e4f28327a8f6b1d0e43115b044167ca28d5c740178a', 'action': 'add'}]) False\n", + "# else 1 dict_keys([1]) True" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b4697ff3-4781-489b-a506-759bbc9aa68a", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "de5a7d5a-6746-4d58-a5dd-f3ed1ea9a875", + "metadata": {}, + "outputs": [], + "source": [ + "import json" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6f1f75c5-caa3-43e4-a28b-64864d3aa832", + "metadata": {}, + "outputs": [], + "source": [ + "import re\n", + "\n", + "def natural_key(key: int | str) -> list[int]:\n", + " \"\"\"Define key for natural ordering of strings.\"\"\"\n", + " if isinstance(key, int):\n", + " key = str(key)\n", + " return [int(s) if s.isdigit() else s for s in re.split('(\\d+)', key)]\n", + "\n", + "def sort_dict_naturally(d: dict) -> dict:\n", + " \"\"\"Sort dictionary by keys in natural order.\"\"\"\n", + " return {k: d[k] for k in sorted(d.keys(), key=natural_key)}\n", + "\n", + "\n", + "# Example usage:\n", + "d = {\n", + " \"apple10\": \"value10\",\n", + " \"apple2\": \"value2\",\n", + " \"banana\": \"value_banana\",\n", + " \"apple1\": \"value1\",\n", + "}\n", + "\n", + "sorted_d = sort_dict_naturally(d)\n", + "print(sorted_d)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d8b3d6da-4a0f-4539-92bf-33130648c9ff", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fe79ee3a-2a0c-4e60-bc44-c1e7d1d81d3d", + "metadata": {}, + "outputs": [], + "source": [ + "from syft.types.syft_object import SyftMigrationRegistry, SyftObjectRegistry\n", + "from syft.service.metadata.migrations import *" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c0907163-d6ab-4a86-8ba9-b6f13c6a8390", + "metadata": {}, + "outputs": [], + "source": [ + "from typing import Optional" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "588beaf9-7aaf-4691-9de5-9165a92aa550", + "metadata": {}, + "outputs": [], + "source": [ + "from collections import defaultdict\n", + "def protocol_state_builder(protocol_dict: dict, stop_key: Optional[str] = None):\n", + " sorted_dict = sort_dict_naturally(protocol_dict)\n", + " state_dict = defaultdict(dict)\n", + " for k, v in sorted_dict.items():\n", + " # stop early\n", + " if stop_key == k:\n", + " return state_dict\n", + " print(\"k\", k, v)\n", + " object_versions = sorted_dict[k][\"object_versions\"]\n", + " print(\"got object versions\", object_versions)\n", + " for canonical_name, object_metadata in object_versions.items():\n", + " action = object_metadata[\"action\"]\n", + " version = object_metadata[\"version\"]\n", + " hash_str = object_metadata[\"hash\"]\n", + " print(\"canonical\", canonical_name, object_metadata)\n", + " versions = state_dict[canonical_name]\n", + " if action == \"add\" and (version in versions.keys() or hash_str in versions.values()):\n", + " raise Exception(f\"Can't add {object_metadata} already in state {versions}\")\n", + " elif action == \"remove\" and (version not in versions.keys() or hash_str not in versions.values()):\n", + " raise Exception(f\"Can't remove {object_metadata} missing from state {versions}\")\n", + " if action == \"add\":\n", + " versions[version] = hash_str\n", + " elif action == \"remove\":\n", + " del versions[version]\n", + " state_dict[canonical_name] = versions\n", + " return state_dict" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aa30fea0-1eda-4806-b74f-39ae88c68031", + "metadata": {}, + "outputs": [], + "source": [ + "a = {}\n", + "a[\"1\"] = {}\n", + "a[\"1\"][\"object_versions\"] = {}\n", + "a[\"1\"][\"object_versions\"][\"Thing\"] = {}\n", + "a[\"1\"][\"object_versions\"][\"Thing\"][\"version\"] = 1\n", + "a[\"1\"][\"object_versions\"][\"Thing\"][\"hash\"] = \"abc\"\n", + "a[\"1\"][\"object_versions\"][\"Thing\"][\"action\"] = \"add\"\n", + "\n", + "a[\"2\"] = {}\n", + "a[\"2\"][\"object_versions\"] = {}\n", + "a[\"2\"][\"object_versions\"][\"Thing\"] = {}\n", + "a[\"2\"][\"object_versions\"][\"Thing\"][\"version\"] = 2\n", + "a[\"2\"][\"object_versions\"][\"Thing\"][\"hash\"] = \"def\"\n", + "a[\"2\"][\"object_versions\"][\"Thing\"][\"action\"] = \"add\"\n", + "\n", + "a[\"3\"] = {}\n", + "a[\"3\"][\"object_versions\"] = {}\n", + "a[\"3\"][\"object_versions\"][\"Thing\"] = {}\n", + "a[\"3\"][\"object_versions\"][\"Thing\"][\"version\"] = 1\n", + "a[\"3\"][\"object_versions\"][\"Thing\"][\"hash\"] = \"abc\"\n", + "a[\"3\"][\"object_versions\"][\"Thing\"][\"action\"] = \"remove\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b19fd8f9-cbb6-4ddb-9626-3ee1ddb2b874", + "metadata": {}, + "outputs": [], + "source": [ + "a" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "77afd622-1ab3-4720-9f7e-c290db2b315a", + "metadata": {}, + "outputs": [], + "source": [ + "b = protocol_state_builder(a)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "df702f68-a5b9-4aef-be53-60fff941fe59", + "metadata": {}, + "outputs": [], + "source": [ + "b" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ac5ac02e-22d3-4655-aebb-e1d5ef182b06", + "metadata": {}, + "outputs": [], + "source": [ + "sy.serde.recursive.TYPE_BANK[\"numpy.ndarray\"]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "be941a27-fa20-4854-9b6b-99c55f1bc817", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a4085166-e092-4d53-a795-610a2b9d1a27", + "metadata": {}, + "outputs": [], + "source": [ + "# (\n", + "# nonrecursive,\n", + "# serialize,\n", + "# deserialize,\n", + "# attribute_list,\n", + "# exclude_attrs_list,\n", + "# serde_overrides,\n", + "# hash_exclude_attrs,\n", + "# cls,\n", + "# attribute_types,\n", + "# version,\n", + "# ) = sy.serde.recursive.TYPE_BANK[\"syft.service.metadata.node_metadata.NodeMetadataV2\"]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "47ceef1c-0a6f-4b20-aca1-017c0fcc0b55", + "metadata": {}, + "outputs": [], + "source": [ + "# sy.serde.recursive.TYPE_BANK.keys()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a0237c1a-536f-4780-ae40-18161d413c59", + "metadata": {}, + "outputs": [], + "source": [ + "def _calculate_object_hash(klass) -> str:\n", + " # TODO: this depends on what is marked as serde\n", + " field_data = {\n", + " field_name: repr(model_field.annotation)\n", + " for field_name, model_field in klass.__fields__.items()\n", + " }\n", + " obj_meta_info = {\n", + " \"canonical_name\": klass.__canonical_name__,\n", + " \"version\": klass.__version__,\n", + " \"unique_keys\": getattr(klass, \"__attr_unique__\", []),\n", + " \"field_data\": field_data,\n", + " }\n", + "\n", + " return hashlib.sha256(json.dumps(obj_meta_info).encode()).hexdigest()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8a1e67d5-d004-4929-8c88-fabe3735482d", + "metadata": {}, + "outputs": [], + "source": [ + "from syft.types.syft_object import SyftBaseObject\n", + "import hashlib" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fe52e359-f632-4f2b-a84c-2e60d2362ed3", + "metadata": {}, + "outputs": [], + "source": [ + "def diff_state(state: dict) -> dict:\n", + " object_diff = {}\n", + " compare_dict = {}\n", + " type_bank = sy.serde.recursive.TYPE_BANK\n", + " for k in type_bank:\n", + " (\n", + " nonrecursive,\n", + " serialize,\n", + " deserialize,\n", + " attribute_list,\n", + " exclude_attrs_list,\n", + " serde_overrides,\n", + " hash_exclude_attrs,\n", + " cls,\n", + " attribute_types,\n", + " version,\n", + " ) = type_bank[k]\n", + " if issubclass(cls, SyftBaseObject):\n", + " hash_str = _calculate_object_hash(cls)\n", + " canonical_name = cls.__canonical_name__\n", + " print(\"k\", k, canonical_name, cls, version, hash_str)\n", + "\n", + " # build this up for later\n", + " compare_dict[canonical_name] = {}\n", + " compare_dict[canonical_name][version] = hash_str\n", + "\n", + " if canonical_name not in state:\n", + " # new object so its an add\n", + " object_diff[canonical_name] = {}\n", + " object_diff[canonical_name][\"version\"] = version\n", + " object_diff[canonical_name][\"hash\"] = hash_str\n", + " object_diff[canonical_name][\"action\"] = \"add\"\n", + " continue\n", + " \n", + " versions = object_diff[canonical_name]\n", + "\n", + " if version in versions.keys() and hash_str in versions.values():\n", + " # already there so do nothing\n", + " continue\n", + " elif version in versions.keys():\n", + " raise Exception(f\"{canonical_name} {cls} version {version} hash has changed. You probably need to bump the version number.\")\n", + " else:\n", + " # new object so its an add\n", + " object_diff[canonical_name] = {}\n", + " object_diff[canonical_name][\"version\"] = version\n", + " object_diff[canonical_name][\"hash\"] = hash_str\n", + " object_diff[canonical_name][\"action\"] = \"add\"\n", + " continue\n", + "\n", + " # now check for remove actions\n", + " for canonical_name in state:\n", + " for version, hash_str in state[canonical_name].items():\n", + " if canonical_name not in compare_dict:\n", + " # missing so its a remove\n", + " object_diff[canonical_name] = {}\n", + " object_diff[canonical_name][\"version\"] = version\n", + " object_diff[canonical_name][\"hash\"] = hash_str\n", + " object_diff[canonical_name][\"action\"] = \"remove\"\n", + " continue\n", + " versions = compare_dict[canonical_name]\n", + " if version in versions.keys():\n", + " # missing so its a remove\n", + " object_diff[canonical_name] = {}\n", + " object_diff[canonical_name][\"version\"] = version\n", + " object_diff[canonical_name][\"hash\"] = hash_str\n", + " object_diff[canonical_name][\"action\"] = \"remove\"\n", + " continue\n", + " return object_diff" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "01838413-4b9c-4490-bc24-52e9e7f00c53", + "metadata": {}, + "outputs": [], + "source": [ + "g = diff_state(b)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "223e946b-65fc-4d1e-a4d5-9046f21f3d36", + "metadata": {}, + "outputs": [], + "source": [ + "g" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d1635cac-912f-49ee-9683-9b672671c811", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a2b2e8ba-232f-4fe1-8d18-b3ed41964bb0", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "01e5e780-ab38-46e0-923d-d3153e992d4e", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a0ff7367-c239-4079-9802-c98292a51150", + "metadata": {}, + "outputs": [], + "source": [ + "a[\"1\"] = 1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6c2fb95e-944a-405e-8282-116ccfeb64d6", + "metadata": {}, + "outputs": [], + "source": [ + "a[\"2\"] = 2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12dfbc65-bc40-4ae4-9443-c3efd68359fb", + "metadata": {}, + "outputs": [], + "source": [ + "a[\"11\"] = 1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "09fcd132-0bd3-4b1c-8fa2-de02de5dad94", + "metadata": {}, + "outputs": [], + "source": [ + "a[\"dev\"] = 4" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4b28d32f-0fd2-4e8b-a062-5f87352fa5d0", + "metadata": {}, + "outputs": [], + "source": [ + "a" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "df86253c-0230-4d61-b390-2a6693976e63", + "metadata": {}, + "outputs": [], + "source": [ + "y = json.dumps(a)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2cc10838-07f1-4e8e-b8b0-8247a90cef66", + "metadata": {}, + "outputs": [], + "source": [ + "y" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ec13f3af-10fb-4b1a-8f2c-c2e2a3cc940c", + "metadata": {}, + "outputs": [], + "source": [ + "d = json.loads(y)\n", + "d" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9d263c27-a68e-4c6f-a0c6-b117ee044a1e", + "metadata": {}, + "outputs": [], + "source": [ + "d" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "961f4782-1acc-4eb1-ae6e-d3427dbd16b8", + "metadata": {}, + "outputs": [], + "source": [ + "for k,v in d.items():\n", + " print(\"k\", k, v)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "03766fe6-780a-4f9d-ac77-4f2720986b2f", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b814b62e-8ebf-4bb3-a6d9-55cfed1506f7", + "metadata": {}, + "outputs": [], + "source": [ + "a[1] = \"a\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d8237a9e-1556-42bb-9380-8197a90a9e32", + "metadata": {}, + "outputs": [], + "source": [ + "sorted_d = sort_dict_naturally(a)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "047465a4-6148-4d30-9da4-21be15ff7f03", + "metadata": {}, + "outputs": [], + "source": [ + "sorted_d" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "afb57bc4-66f2-49f8-a291-96be4e71dd43", + "metadata": {}, + "outputs": [], + "source": [ + "object_versions = dict()\n", + "object_versions[\"SyftObject\"] = {\"version\": \"1\", \"hash\": \"25a574002025025cfd155e3970305293e21fdd6af9dcde176990802306cc0359\", \"action\":\"add\"}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3bdf972b-cee3-4cde-867f-73b1efb20bde", + "metadata": {}, + "outputs": [], + "source": [ + "a[\"1\"] = object_versions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b2a7ebe9-98fb-4449-a73f-3b171a2df9d5", + "metadata": {}, + "outputs": [], + "source": [ + "a" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a466dfd5-54c4-4aa4-852c-473090758725", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.1" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/packages/grid/backend/grid/core/config.py b/packages/grid/backend/grid/core/config.py index 47373b66bb9..0b9948ca961 100644 --- a/packages/grid/backend/grid/core/config.py +++ b/packages/grid/backend/grid/core/config.py @@ -111,6 +111,7 @@ def get_emails_enabled(cls, v: bool, values: Dict[str, Any]) -> bool: True if os.getenv("TEST_MODE", "false").lower() == "true" else False ) ASSOCIATION_TIMEOUT: int = 10 + DEV_MODE: bool = True if os.getenv("DEV_MODE", "false").lower() == "true" else False class Config: case_sensitive = True diff --git a/packages/grid/backend/grid/main.py b/packages/grid/backend/grid/main.py index 4fc0b35a946..8c4b34507d8 100644 --- a/packages/grid/backend/grid/main.py +++ b/packages/grid/backend/grid/main.py @@ -6,6 +6,9 @@ from fastapi.responses import JSONResponse from starlette.middleware.cors import CORSMiddleware +# syft absolute +from syft.protocol.data_protocol import stage_protocol_changes + # grid absolute from grid.api.router import api_router from grid.core.config import settings @@ -31,6 +34,12 @@ app.include_router(api_router, prefix=settings.API_V2_STR) +if settings.DEV_MODE: + print("Staging protocol changes...") + status = stage_protocol_changes() + print(status) + + # needed for Google Kubernetes Engine LoadBalancer Healthcheck @app.get( "/", diff --git a/packages/grid/worker/worker.py b/packages/grid/worker/worker.py index 3150f84669f..ea3b9d18315 100644 --- a/packages/grid/worker/worker.py +++ b/packages/grid/worker/worker.py @@ -10,11 +10,13 @@ from syft.node.domain import Domain from syft.node.enclave import Enclave from syft.node.gateway import Gateway +from syft.node.node import get_dev_mode from syft.node.node import get_enable_warnings from syft.node.node import get_node_name from syft.node.node import get_node_side_type from syft.node.node import get_node_type from syft.node.routes import make_routes +from syft.protocol.data_protocol import stage_protocol_changes worker_classes = { NodeType.DOMAIN: Domain, @@ -41,6 +43,11 @@ app = FastAPI(title="Worker") +if get_dev_mode(): + print("Staging protocol changes...") + status = stage_protocol_changes() + print(status) + @app.get("/") async def root() -> str: diff --git a/packages/hagrid/hagrid/orchestra.py b/packages/hagrid/hagrid/orchestra.py index 099bd5e1f91..31ef1ba7f09 100644 --- a/packages/hagrid/hagrid/orchestra.py +++ b/packages/hagrid/hagrid/orchestra.py @@ -26,6 +26,7 @@ # syft absolute from syft.abstract_node import NodeSideType from syft.abstract_node import NodeType + from syft.protocol.data_protocol import stage_protocol_changes from syft.service.response import SyftError except Exception: # nosec # print("Please install syft with `pip install syft`") @@ -259,6 +260,10 @@ def deploy_to_python( if hasattr(NodeType, "GATEWAY"): worker_classes[NodeType.GATEWAY] = sy.Gateway + if dev_mode: + print("Staging Protocol Changes...") + stage_protocol_changes() + if port: if port == "auto": # dont use default port to prevent port clashes in CI diff --git a/packages/syft/MANIFEST.in b/packages/syft/MANIFEST.in index 28648461b0f..8cdc899ed4f 100644 --- a/packages/syft/MANIFEST.in +++ b/packages/syft/MANIFEST.in @@ -2,3 +2,4 @@ include src/syft/VERSION include src/syft/capnp/* include src/syft/cache/* include src/syft/img/* +include src/syft/protocol/protocol_version.json diff --git a/packages/syft/src/syft/__init__.py b/packages/syft/src/syft/__init__.py index 1f95ce2cfa3..77e5d6a5ec4 100644 --- a/packages/syft/src/syft/__init__.py +++ b/packages/syft/src/syft/__init__.py @@ -33,6 +33,10 @@ from .node.server import serve_node # noqa: F401 from .node.server import serve_node as bind_worker # noqa: F401 from .node.worker import Worker # noqa: F401 +from .protocol.data_protocol import bump_protocol_version # noqa: F401 +from .protocol.data_protocol import check_or_stage_protocol # noqa: F401 +from .protocol.data_protocol import get_data_protocol # noqa: F401 +from .protocol.data_protocol import stage_protocol_changes # noqa: F401 from .serde import NOTHING # noqa: F401 from .serde.deserialize import _deserialize as deserialize # noqa: F401 from .serde.serializable import serializable # noqa: F401 diff --git a/packages/syft/src/syft/client/api.py b/packages/syft/src/syft/client/api.py index c1b2087b333..86bab84c558 100644 --- a/packages/syft/src/syft/client/api.py +++ b/packages/syft/src/syft/client/api.py @@ -4,6 +4,7 @@ # stdlib from collections import OrderedDict import inspect +from inspect import Parameter from inspect import signature import types from typing import Any @@ -14,6 +15,8 @@ from typing import Tuple from typing import Union from typing import _GenericAlias +from typing import get_args +from typing import get_origin # third party from nacl.exceptions import BadSignatureError @@ -26,6 +29,9 @@ from ..abstract_node import AbstractNode from ..node.credentials import SyftSigningKey from ..node.credentials import SyftVerifyKey +from ..protocol.data_protocol import PROTOCOL_TYPE +from ..protocol.data_protocol import get_data_protocol +from ..protocol.data_protocol import migrate_args_and_kwargs from ..serde.deserialize import _deserialize from ..serde.recursive import index_syft_by_module_name from ..serde.serializable import serializable @@ -46,6 +52,7 @@ from ..types.identity import Identity from ..types.syft_object import SYFT_OBJECT_VERSION_1 from ..types.syft_object import SyftBaseObject +from ..types.syft_object import SyftMigrationRegistry from ..types.syft_object import SyftObject from ..types.uid import LineageID from ..types.uid import UID @@ -110,6 +117,9 @@ class APIEndpoint(SyftObject): @serializable() class LibEndpoint(SyftBaseObject): + __canonical_name__ = "LibEndpoint" + __version__ = SYFT_OBJECT_VERSION_1 + # TODO: bad name, change service_path: str module_path: str @@ -206,6 +216,7 @@ def generate_remote_function( path: str, make_call: Callable, pre_kwargs: Dict[str, Any], + communication_protocol: PROTOCOL_TYPE, warning: Optional[APIEndpointWarning], ): if "blocking" in signature.parameters: @@ -219,6 +230,11 @@ def wrapper(*args, **kwargs): blocking = bool(kwargs["blocking"]) del kwargs["blocking"] + # Migrate args and kwargs to communication protocol + args, kwargs = migrate_args_and_kwargs( + to_protocol=communication_protocol, args=args, kwargs=kwargs + ) + res = validate_callable_args_and_kwargs(args, kwargs, signature) if isinstance(res, SyftError): @@ -228,6 +244,8 @@ def wrapper(*args, **kwargs): if pre_kwargs: _valid_kwargs.update(pre_kwargs) + _valid_kwargs["communication_protocol"] = communication_protocol + api_call = SyftAPICall( node_uid=node_uid, path=path, @@ -240,6 +258,11 @@ def wrapper(*args, **kwargs): if not allowed: return result = make_call(api_call=api_call) + + result, _ = migrate_args_and_kwargs( + [result], kwargs={}, to_latest_protocol=True + ) + result = result[0] return result wrapper.__ipython_inspector_signature_override__ = signature @@ -253,6 +276,7 @@ def generate_remote_lib_function( path: str, module_path: str, make_call: Callable, + communication_protocol: PROTOCOL_TYPE, pre_kwargs: Dict[str, Any], ): if "blocking" in signature.parameters: @@ -372,8 +396,80 @@ def debox_signed_syftapicall_response( return signed_result.message.data +def downgrade_signature(signature: Signature, object_versions: Dict): + migrated_parameters = [] + for _, parameter in signature.parameters.items(): + annotation = unwrap_and_migrate_annotation( + parameter.annotation, object_versions + ) + migrated_parameter = Parameter( + name=parameter.name, + default=parameter.default, + annotation=annotation, + kind=parameter.kind, + ) + migrated_parameters.append(migrated_parameter) + + migrated_return_annotation = unwrap_and_migrate_annotation( + signature.return_annotation, object_versions + ) + + try: + new_signature = Signature( + parameters=migrated_parameters, + return_annotation=migrated_return_annotation, + ) + except Exception as e: + raise e + + return new_signature + + +def unwrap_and_migrate_annotation(annotation, object_versions): + args = get_args(annotation) + origin = get_origin(annotation) + if len(args) == 0: + if ( + isinstance(annotation, type) + and issubclass(annotation, SyftBaseObject) + and annotation.__canonical_name__ in object_versions + ): + downgrade_to_version = int( + max(object_versions[annotation.__canonical_name__]) + ) + downgrade_klass_name = SyftMigrationRegistry.__migration_version_registry__[ + annotation.__canonical_name__ + ][downgrade_to_version] + new_arg = index_syft_by_module_name(downgrade_klass_name) + return new_arg + else: + return annotation + + migrated_annotations = [] + for arg in args: + migrated_annotation = unwrap_and_migrate_annotation(arg, object_versions) + migrated_annotations.append(migrated_annotation) + + migrated_annotations = tuple(migrated_annotations) + + if hasattr(annotation, "copy_with"): + return annotation.copy_with(migrated_annotations) + elif origin is not None: + return origin[migrated_annotations] + else: + return migrated_annotation[0] + + @instrument -@serializable(attrs=["endpoints", "node_uid", "node_name", "lib_endpoints"]) +@serializable( + attrs=[ + "endpoints", + "node_uid", + "node_name", + "lib_endpoints", + "communication_protocol", + ] +) class SyftAPI(SyftObject): # version __canonical_name__ = "SyftAPI" @@ -391,13 +487,16 @@ class SyftAPI(SyftObject): # serde / storage rules refresh_api_callback: Optional[Callable] = None __user_role: ServiceRole = ServiceRole.NONE + communication_protocol: PROTOCOL_TYPE # def __post_init__(self) -> None: # pass @staticmethod def for_user( - node: AbstractNode, user_verify_key: Optional[SyftVerifyKey] = None + node: AbstractNode, + communication_protocol: PROTOCOL_TYPE, + user_verify_key: Optional[SyftVerifyKey] = None, ) -> SyftAPI: # relative # TODO: Maybe there is a possibility of merging ServiceConfig and APIEndpoint @@ -414,6 +513,22 @@ def for_user( node=node, role=role, credentials=user_verify_key ) + # If server uses a higher protocol version than client, then + # signatures needs to be downgraded. + if node.current_protocol == "dev" and communication_protocol != "dev": + # We assume dev is the highest staged protocol + signature_needs_downgrade = True + else: + signature_needs_downgrade = node.current_protocol != "dev" and int( + node.current_protocol + ) > int(communication_protocol) + data_protocol = get_data_protocol() + + if signature_needs_downgrade: + object_version_for_protocol = data_protocol.get_object_versions( + communication_protocol + ) + for ( path, service_config, @@ -423,13 +538,23 @@ def for_user( if service_warning: service_warning = service_warning.message_from(warning_context) service_warning.enabled = node.enable_warnings + + signature = ( + downgrade_signature( + signature=service_config.signature, + object_versions=object_version_for_protocol, + ) + if signature_needs_downgrade + else service_config.signature + ) + endpoint = APIEndpoint( service_path=path, module_path=path, name=service_config.public_name, description="", doc_string=service_config.doc_string, - signature=service_config.signature, + signature=signature, # TODO: Migrate signature based on communication protocol has_self=False, warning=service_warning, ) @@ -476,6 +601,7 @@ def for_user( endpoints=endpoints, lib_endpoints=lib_endpoints, __user_role=role, + communication_protocol=communication_protocol, ) @property @@ -529,7 +655,7 @@ def _add_route( _self._add_submodule(_last_module, endpoint_method) def generate_endpoints(self) -> None: - def build_endpoint_tree(endpoints): + def build_endpoint_tree(endpoints, communication_protocol): api_module = APIModule(path="") for _, v in endpoints.items(): signature = v.signature @@ -544,6 +670,7 @@ def build_endpoint_tree(endpoints): self.make_call, pre_kwargs=v.pre_kwargs, warning=v.warning, + communication_protocol=communication_protocol, ) elif isinstance(v, LibEndpoint): endpoint_function = generate_remote_lib_function( @@ -554,6 +681,7 @@ def build_endpoint_tree(endpoints): v.module_path, self.make_call, pre_kwargs=v.pre_kwargs, + communication_protocol=communication_protocol, ) endpoint_function.__doc__ = v.doc_string @@ -561,8 +689,12 @@ def build_endpoint_tree(endpoints): return api_module if self.lib_endpoints is not None: - self.libs = build_endpoint_tree(self.lib_endpoints) - self.api_module = build_endpoint_tree(self.endpoints) + self.libs = build_endpoint_tree( + self.lib_endpoints, self.communication_protocol + ) + self.api_module = build_endpoint_tree( + self.endpoints, self.communication_protocol + ) @property def services(self) -> APIModule: diff --git a/packages/syft/src/syft/client/client.py b/packages/syft/src/syft/client/client.py index 38bc3e6d2ac..c6371f487a0 100644 --- a/packages/syft/src/syft/client/client.py +++ b/packages/syft/src/syft/client/client.py @@ -34,6 +34,9 @@ from ..node.credentials import SyftSigningKey from ..node.credentials import SyftVerifyKey from ..node.credentials import UserLoginCredentials +from ..protocol.data_protocol import DataProtocol +from ..protocol.data_protocol import PROTOCOL_TYPE +from ..protocol.data_protocol import get_data_protocol from ..serde.deserialize import _deserialize from ..serde.serializable import serializable from ..serde.serialize import _serialize @@ -214,14 +217,22 @@ def get_node_metadata(self, credentials: SyftSigningKey) -> NodeMetadataJSON: metadata_json = json.loads(response) return NodeMetadataJSON(**metadata_json) - def get_api(self, credentials: SyftSigningKey) -> SyftAPI: - params = {"verify_key": str(credentials.verify_key)} + def get_api( + self, credentials: SyftSigningKey, communication_protocol: int + ) -> SyftAPI: + params = { + "verify_key": str(credentials.verify_key), + "communication_protocol": communication_protocol, + } if self.proxy_target_uid: obj = forward_message_to_proxy( self.make_call, proxy_target_uid=self.proxy_target_uid, path="api", - kwargs={"credentials": credentials}, + kwargs={ + "credentials": credentials, + "communication_protocol": communication_protocol, + }, credentials=credentials, ) else: @@ -229,6 +240,7 @@ def get_api(self, credentials: SyftSigningKey) -> SyftAPI: obj = _deserialize(content, from_bytes=True) obj.connection = self obj.signing_key = credentials + obj.communication_protocol = communication_protocol if self.proxy_target_uid: obj.node_uid = self.proxy_target_uid return cast(SyftAPI, obj) @@ -333,20 +345,29 @@ def get_node_metadata(self, credentials: SyftSigningKey) -> NodeMetadataJSON: else: return self.node.metadata.to(NodeMetadataJSON) - def get_api(self, credentials: SyftSigningKey) -> SyftAPI: + def get_api( + self, credentials: SyftSigningKey, communication_protocol: int + ) -> SyftAPI: # todo: its a bit odd to identify a user by its verify key maybe? if self.proxy_target_uid: obj = forward_message_to_proxy( self.make_call, proxy_target_uid=self.proxy_target_uid, path="api", - kwargs={"credentials": credentials}, + kwargs={ + "credentials": credentials, + "communication_protocol": communication_protocol, + }, credentials=credentials, ) else: - obj = self.node.get_api(for_user=credentials.verify_key) + obj = self.node.get_api( + for_user=credentials.verify_key, + communication_protocol=communication_protocol, + ) obj.connection = self obj.signing_key = credentials + obj.communication_protocol = communication_protocol if self.proxy_target_uid: obj.node_uid = self.proxy_target_uid return obj @@ -444,6 +465,8 @@ def __init__( self.metadata = metadata self.credentials: Optional[SyftSigningKey] = credentials self._api = api + self.communication_protocol = None + self.current_protocol = None self.post_init() @@ -454,6 +477,32 @@ def post_init(self) -> None: if self.metadata is None: self._fetch_node_metadata(self.credentials) + self.communication_protocol = self._get_communication_protocol( + self.metadata.supported_protocols + ) + + def _get_communication_protocol( + self, protocols_supported_by_server: List + ) -> Union[int, str]: + data_protocol: DataProtocol = get_data_protocol() + protocols_supported_by_client: List[ + PROTOCOL_TYPE + ] = data_protocol.supported_protocols + + self.current_protocol = data_protocol.latest_version + common_protocols = set(protocols_supported_by_client).intersection( + protocols_supported_by_server + ) + + if len(common_protocols) == 0: + raise Exception( + "No common communication protocol found between the client and the server." + ) + + if "dev" in common_protocols: + return "dev" + return max(common_protocols) + def create_project( self, name: str, description: str, user_email_address: str ) -> Any: @@ -759,7 +808,10 @@ def _fetch_node_metadata(self, credentials: SyftSigningKey) -> None: self.metadata = metadata def _fetch_api(self, credentials: SyftSigningKey): - _api: SyftAPI = self.connection.get_api(credentials=credentials) + _api: SyftAPI = self.connection.get_api( + credentials=credentials, + communication_protocol=self.communication_protocol, + ) def refresh_callback(): return self._fetch_api(self.credentials) diff --git a/packages/syft/src/syft/node/node.py b/packages/syft/src/syft/node/node.py index 1b2f3a92c00..eb0e9e7d7a3 100644 --- a/packages/syft/src/syft/node/node.py +++ b/packages/syft/src/syft/node/node.py @@ -43,8 +43,12 @@ from ..client.api import debox_signed_syftapicall_response from ..exceptions.exception import PySyftException from ..external import OBLV +from ..protocol.data_protocol import PROTOCOL_TYPE +from ..protocol.data_protocol import get_data_protocol from ..serde.deserialize import _deserialize from ..serde.serialize import _serialize +from ..service.action.action_object import Action +from ..service.action.action_object import ActionObject from ..service.action.action_service import ActionService from ..service.action.action_store import DictActionStore from ..service.action.action_store import SQLiteActionStore @@ -63,6 +67,7 @@ from ..service.metadata.node_metadata import NodeMetadata from ..service.network.network_service import NetworkService from ..service.notification.notification_service import NotificationService +from ..service.object_search.migration_state_service import MigrateStateService from ..service.policy.policy_service import PolicyService from ..service.project.project_service import ProjectService from ..service.queue.queue import APICallMessageHandler @@ -91,12 +96,12 @@ from ..store.document_store import StoreConfig from ..store.sqlite_document_store import SQLiteStoreClientConfig from ..store.sqlite_document_store import SQLiteStoreConfig -from ..types.syft_object import HIGHEST_SYFT_OBJECT_VERSION -from ..types.syft_object import LOWEST_SYFT_OBJECT_VERSION +from ..types.syft_object import SYFT_OBJECT_VERSION_1 from ..types.syft_object import SyftObject from ..types.uid import UID from ..util.experimental_flags import flags from ..util.telemetry import instrument +from ..util.util import get_env from ..util.util import get_root_data_path from ..util.util import random_name from ..util.util import str_to_bool @@ -128,10 +133,6 @@ def gipc_decoder(obj_bytes): DEFAULT_ROOT_PASSWORD = "DEFAULT_ROOT_PASSWORD" # nosec -def get_env(key: str, default: Optional[Any] = None) -> Optional[str]: - return os.environ.get(key, default) - - def get_private_key_env() -> Optional[str]: return get_env(NODE_PRIVATE_KEY) @@ -287,6 +288,7 @@ def __init__( CodeHistoryService, MetadataService, BlobStorageService, + MigrateStateService, ] if services is None else services @@ -455,10 +457,58 @@ def root_client(self): root_client.api.refresh_api_callback() return root_client + def _find_pending_migrations(self): + klasses_to_be_migrated = [] + + context = AuthedServiceContext( + node=self, + credentials=self.verify_key, + role=ServiceRole.ADMIN, + ) + migration_state_service = self.get_service(MigrateStateService) + + canonical_name_version_map = [] + + # Track all object types from document store + for partition in self.document_store.partitions.values(): + object_type = partition.settings.object_type + canonical_name = object_type.__canonical_name__ + object_version = object_type.__version__ + canonical_name_version_map.append((canonical_name, object_version)) + + # Track all object types from action store + action_object_types = [Action, ActionObject] + action_object_types.extend(ActionObject.__subclasses__()) + for object_type in action_object_types: + canonical_name = object_type.__canonical_name__ + object_version = object_type.__version__ + canonical_name_version_map.append((canonical_name, object_version)) + + for canonical_name, current_version in canonical_name_version_map: + migration_state = migration_state_service.get_state(context, canonical_name) + if ( + migration_state is not None + and migration_state.current_version != migration_state.latest_version + ): + klasses_to_be_migrated.append(canonical_name) + else: + migration_state_service.register_migration_state( + context, + current_version=current_version, + canonical_name=canonical_name, + ) + + return klasses_to_be_migrated + @property def guest_client(self): return self.get_guest_client() + @property + def current_protocol(self) -> List: + data_protocol = get_data_protocol() + return data_protocol.latest_version + def get_guest_client(self, verbose: bool = True): # relative from ..client.client import PythonConnection @@ -589,6 +639,7 @@ def _construct_services(self): CodeHistoryService, MetadataService, BlobStorageService, + MigrateStateService, ] if OBLV: @@ -655,8 +706,8 @@ def metadata(self) -> NodeMetadata: name=name, id=self.id, verify_key=self.verify_key, - highest_object_version=HIGHEST_SYFT_OBJECT_VERSION, - lowest_object_version=LOWEST_SYFT_OBJECT_VERSION, + highest_version=SYFT_OBJECT_VERSION_1, + lowest_version=SYFT_OBJECT_VERSION_1, syft_version=__version__, deployed_on=deployed_on, description=description, @@ -830,8 +881,16 @@ def handle_api_call_with_unsigned_result( return item return result - def get_api(self, for_user: Optional[SyftVerifyKey] = None) -> SyftAPI: - return SyftAPI.for_user(node=self, user_verify_key=for_user) + def get_api( + self, + for_user: Optional[SyftVerifyKey] = None, + communication_protocol: Optional[PROTOCOL_TYPE] = None, + ) -> SyftAPI: + return SyftAPI.for_user( + node=self, + user_verify_key=for_user, + communication_protocol=communication_protocol, + ) def get_method_with_context( self, function: Callable, context: NodeServiceContext diff --git a/packages/syft/src/syft/node/routes.py b/packages/syft/src/syft/node/routes.py index 8bbe0df8e6d..b9f7ffc396d 100644 --- a/packages/syft/src/syft/node/routes.py +++ b/packages/syft/src/syft/node/routes.py @@ -14,6 +14,7 @@ # relative from ..abstract_node import AbstractNode +from ..protocol.data_protocol import PROTOCOL_TYPE from ..serde.deserialize import _deserialize as deserialize from ..serde.serialize import _serialize as serialize from ..service.context import NodeServiceContext @@ -68,15 +69,21 @@ def syft_metadata_capnp() -> Response: media_type="application/octet-stream", ) - def handle_syft_new_api(user_verify_key: SyftVerifyKey) -> Response: + def handle_syft_new_api( + user_verify_key: SyftVerifyKey, communication_protocol: PROTOCOL_TYPE + ) -> Response: return Response( - serialize(worker.get_api(user_verify_key), to_bytes=True), + serialize( + worker.get_api(user_verify_key, communication_protocol), to_bytes=True + ), media_type="application/octet-stream", ) # get the SyftAPI object @router.get("/api") - def syft_new_api(request: Request, verify_key: str) -> Response: + def syft_new_api( + request: Request, verify_key: str, communication_protocol: PROTOCOL_TYPE + ) -> Response: user_verify_key: SyftVerifyKey = SyftVerifyKey.from_string(verify_key) if TRACE_MODE: with trace.get_tracer(syft_new_api.__module__).start_as_current_span( @@ -84,9 +91,9 @@ def syft_new_api(request: Request, verify_key: str) -> Response: context=extract(request.headers), kind=trace.SpanKind.SERVER, ): - return handle_syft_new_api(user_verify_key) + return handle_syft_new_api(user_verify_key, communication_protocol) else: - return handle_syft_new_api(user_verify_key) + return handle_syft_new_api(user_verify_key, communication_protocol) def handle_new_api_call(data: bytes) -> Response: obj_msg = deserialize(blob=data, from_bytes=True) diff --git a/packages/syft/src/syft/protocol/data_protocol.py b/packages/syft/src/syft/protocol/data_protocol.py new file mode 100644 index 00000000000..aaa90f41c77 --- /dev/null +++ b/packages/syft/src/syft/protocol/data_protocol.py @@ -0,0 +1,401 @@ +# stdlib +from collections import defaultdict +import hashlib +import json +import os +from pathlib import Path +import re +from typing import Any +from typing import Dict +from typing import Optional +from typing import Tuple +from typing import Type +from typing import Union + +# third party +from result import OkErr +from result import Result + +# relative +from ..serde.recursive import TYPE_BANK +from ..service.response import SyftError +from ..service.response import SyftException +from ..service.response import SyftSuccess +from ..types.syft_object import SyftBaseObject + +PROTOCOL_STATE_FILENAME = "protocol_version.json" +PROTOCOL_TYPE = Union[str, int] + + +def natural_key(key: PROTOCOL_TYPE) -> list[int]: + """Define key for natural ordering of strings.""" + if isinstance(key, int): + key = str(key) + return [int(s) if s.isdigit() else s for s in re.split("(\d+)", key)] + + +def sort_dict_naturally(d: dict) -> dict: + """Sort dictionary by keys in natural order.""" + return {k: d[k] for k in sorted(d.keys(), key=natural_key)} + + +def data_protocol_file_name(): + return PROTOCOL_STATE_FILENAME + + +def data_protocol_dir(): + return os.path.abspath(str(Path(__file__).parent)) + + +class DataProtocol: + def __init__(self, filename: str) -> None: + self.file_path = Path(data_protocol_dir()) / filename + self.load_state() + + def load_state(self) -> None: + self.protocol_history = self.read_history() + self.state = self.build_state() + self.diff, self.current = self.diff_state(self.state) + self.protocol_support = self.calculate_supported_protocols() + + @staticmethod + def _calculate_object_hash(klass: Type[SyftBaseObject]) -> str: + # TODO: this depends on what is marked as serde + field_name_keys = sorted(klass.__fields__.keys()) + field_data = { + field_name: repr(klass.__fields__[field_name].annotation) + for field_name in field_name_keys + } + obj_meta_info = { + "canonical_name": klass.__canonical_name__, + "version": klass.__version__, + "unique_keys": getattr(klass, "__attr_unique__", []), + "field_data": field_data, + } + + return hashlib.sha256(json.dumps(obj_meta_info).encode()).hexdigest() + + def read_history(self) -> Dict: + return json.loads(self.file_path.read_text()) + + def save_history(self, history: dict) -> None: + self.file_path.write_text(json.dumps(history, indent=2) + "\n") + + @property + def latest_version(self) -> PROTOCOL_TYPE: + sorted_versions = sorted(self.protocol_history.keys(), key=natural_key) + if len(sorted_versions) > 0: + return sorted_versions[-1] if self.has_dev else int(sorted_versions[-1]) + return "dev" + + @staticmethod + def _hash_to_sha256(obj_dict: Dict) -> str: + return hashlib.sha256(json.dumps(obj_dict).encode()).hexdigest() + + def build_state(self, stop_key: Optional[str] = None) -> dict: + sorted_dict = sort_dict_naturally(self.protocol_history) + state_dict = defaultdict(dict) + for k, _v in sorted_dict.items(): + object_versions = sorted_dict[k]["object_versions"] + for canonical_name, versions in object_versions.items(): + for version, object_metadata in versions.items(): + action = object_metadata["action"] + version = object_metadata["version"] + hash_str = object_metadata["hash"] + state_versions = state_dict[canonical_name] + if action == "add" and ( + str(version) in state_versions.keys() + or hash_str in state_versions.values() + ): + raise Exception( + f"Can't add {object_metadata} already in state {versions}" + ) + elif action == "remove" and ( + str(version) not in state_versions.keys() + or hash_str not in state_versions.values() + ): + raise Exception( + f"Can't remove {object_metadata} missing from state {versions} for object {canonical_name}." + ) + if action == "add": + state_dict[canonical_name][str(version)] = hash_str + elif action == "remove": + del state_dict[canonical_name][str(version)] + # stop early + if stop_key == k: + return state_dict + return state_dict + + def diff_state(self, state: dict) -> tuple[dict, dict]: + compare_dict = defaultdict(dict) # what versions are in the latest code + object_diff = defaultdict(dict) # diff in latest code with saved json + for k in TYPE_BANK: + ( + nonrecursive, + serialize, + deserialize, + attribute_list, + exclude_attrs_list, + serde_overrides, + hash_exclude_attrs, + cls, + attribute_types, + version, + ) = TYPE_BANK[k] + if issubclass(cls, SyftBaseObject): + canonical_name = cls.__canonical_name__ + hash_str = DataProtocol._calculate_object_hash(cls) + + # build this up for later + compare_dict[canonical_name][str(version)] = hash_str + + if canonical_name not in state: + # new object so its an add + object_diff[canonical_name][str(version)] = {} + object_diff[canonical_name][str(version)]["version"] = version + object_diff[canonical_name][str(version)]["hash"] = hash_str + object_diff[canonical_name][str(version)]["action"] = "add" + continue + + versions = state[canonical_name] + if ( + str(version) in versions.keys() + and versions[str(version)] == hash_str + ): + # already there so do nothing + continue + elif str(version) in versions.keys(): + raise Exception( + f"{canonical_name} for class {cls.__name__} fqn {cls} " + + f"version {version} hash has changed. " + + f"{hash_str} not in {versions.values()}. " + + "Is a unique __canonical_name__ for this subclass missing? " + + "If the class has changed you will need to bump the version number." + ) + else: + # new object so its an add + object_diff[canonical_name][str(version)] = {} + object_diff[canonical_name][str(version)]["version"] = version + object_diff[canonical_name][str(version)]["hash"] = hash_str + object_diff[canonical_name][str(version)]["action"] = "add" + continue + + # now check for remove actions + for canonical_name in state: + for version, hash_str in state[canonical_name].items(): + if canonical_name not in compare_dict: + # missing so its a remove + object_diff[canonical_name][str(version)] = {} + object_diff[canonical_name][str(version)]["version"] = version + object_diff[canonical_name][str(version)]["hash"] = hash_str + object_diff[canonical_name][str(version)]["action"] = "remove" + continue + versions = compare_dict[canonical_name] + if str(version) not in versions.keys(): + # missing so its a remove + object_diff[canonical_name][str(version)] = {} + object_diff[canonical_name][str(version)]["version"] = version + object_diff[canonical_name][str(version)]["hash"] = hash_str + object_diff[canonical_name][str(version)]["action"] = "remove" + continue + return object_diff, compare_dict + + def stage_protocol_changes(self) -> Result[SyftSuccess, SyftError]: + change_count = 0 + current_history = self.protocol_history + if "dev" not in current_history: + current_history["dev"] = {} + current_history["dev"]["object_versions"] = {} + object_versions = current_history["dev"]["object_versions"] + for canonical_name, versions in self.diff.items(): + for version, version_metadata in versions.items(): + if canonical_name not in object_versions: + object_versions[canonical_name] = {} + change_count += 1 + object_versions[canonical_name][version] = version_metadata + + current_history["dev"]["object_versions"] = object_versions + + # trim empty dev + if len(current_history["dev"]["object_versions"]) == 0: + del current_history["dev"] + + self.save_history(current_history) + self.load_state() + return SyftSuccess(message=f"{change_count} Protocol Updates Staged to dev") + + def bump_protocol_version(self) -> Result[SyftSuccess, SyftError]: + if len(self.diff): + raise Exception( + "You can't bump the protocol version with unstaged changes." + ) + + keys = self.protocol_history.keys() + if "dev" not in keys: + raise Exception( + "You can't bump the protocol if there are no staged changes." + ) + + highest_protocol = 0 + for k in self.protocol_history.keys(): + if k == "dev": + continue + highest_protocol = max(highest_protocol, int(k)) + + next_highest_protocol = highest_protocol + 1 + self.protocol_history[str(next_highest_protocol)] = self.protocol_history["dev"] + del self.protocol_history["dev"] + self.save_history(self.protocol_history) + self.load_state() + return SyftSuccess(message=f"Protocol Updated to {next_highest_protocol}") + + def check_protocol(self) -> Result[SyftSuccess, SyftError]: + if len(self.diff) != 0: + return SyftError(message="Protocol Changes Unstanged") + else: + return SyftSuccess(message="Protocol Stable") + + def check_or_stage_protocol(self) -> Result[SyftSuccess, SyftError]: + if not self.check_protocol(): + self.stage_protocol_changes() + result = self.check_protocol() + return result + + @property + def supported_protocols(self) -> list[Union[int, str]]: + """Returns a list of protocol numbers that are marked as supported.""" + supported = [] + for version, is_supported in self.protocol_support.items(): + if is_supported: + if version != "dev": + version = int(version) + supported.append(version) + return supported + + def calculate_supported_protocols(self) -> dict: + protocol_supported = {} + # go through each historical protocol version + for v, version_data in self.protocol_history.items(): + # we assume its supported until we prove otherwise + protocol_supported[v] = True + # iterate through each object + for canonical_name, _ in version_data["object_versions"].items(): + if canonical_name not in self.state: + protocol_supported[v] = False + break + return protocol_supported + + def get_object_versions(self, protocol: Union[int, str]) -> list: + return self.protocol_history[str(protocol)]["object_versions"] + + @property + def has_dev(self) -> bool: + if "dev" in self.protocol_history.keys(): + return True + return False + + +def get_data_protocol(): + return DataProtocol(filename=data_protocol_file_name()) + + +def stage_protocol_changes() -> Result[SyftSuccess, SyftError]: + data_protocol = get_data_protocol() + return data_protocol.stage_protocol_changes() + + +def bump_protocol_version() -> Result[SyftSuccess, SyftError]: + data_protocol = get_data_protocol() + return data_protocol.bump_protocol_version() + + +def check_or_stage_protocol() -> Result[SyftSuccess, SyftError]: + data_protocol = get_data_protocol() + return data_protocol.check_or_stage_protocol() + + +def debox_arg_and_migrate(arg: Any, protocol_state: dict): + """Debox the argument based on whether it is iterable or single entity.""" + box_to_result_type = None + + if type(arg) in OkErr: + box_to_result_type = type(arg) + arg = arg.value + + single_entity = False + is_tuple = isinstance(arg, tuple) + + if isinstance(arg, (list, tuple)): + iterable_keys = range(len(arg)) + arg = list(arg) + elif isinstance(arg, dict): + iterable_keys = arg.keys() + else: + iterable_keys = range(1) + arg = [arg] + single_entity = True + + for key in iterable_keys: + _object = arg[key] + if isinstance(_object, SyftBaseObject): + current_version = int(_object.__version__) + migrate_to_version = int(max(protocol_state[_object.__canonical_name__])) + if current_version > migrate_to_version: # downgrade + versions = range(current_version - 1, migrate_to_version - 1, -1) + else: # upgrade + versions = range(current_version + 1, migrate_to_version + 1) + for version in versions: + _object = _object.migrate_to(version) + arg[key] = _object + + wrapped_arg = arg[0] if single_entity else arg + wrapped_arg = tuple(wrapped_arg) if is_tuple else wrapped_arg + if box_to_result_type is not None: + wrapped_arg = box_to_result_type(wrapped_arg) + + return wrapped_arg + + +def migrate_args_and_kwargs( + args: Tuple, + kwargs: Dict, + to_protocol: Optional[PROTOCOL_TYPE] = None, + to_latest_protocol: bool = False, +) -> Tuple[Tuple, Dict]: + """Migrate args and kwargs to latest version for given protocol. + + If `to_protocol` is None, then migrate to latest protocol version. + + """ + data_protocol = get_data_protocol() + + if to_protocol is None: + to_protocol = data_protocol.latest_version if to_latest_protocol else None + + if to_protocol is None: + raise SyftException(message="Protocol version missing.") + + # If latest protocol being used is equal to the protocol to be migrate + # then skip migration of the object + if to_protocol == data_protocol.latest_version: + return args, kwargs + + protocol_state = data_protocol.build_state(stop_key=str(to_protocol)) + + migrated_kwargs, migrated_args = {}, [] + + for param_name, param_val in kwargs.items(): + migrated_val = debox_arg_and_migrate( + arg=param_val, + protocol_state=protocol_state, + ) + migrated_kwargs[param_name] = migrated_val + + for arg in args: + migrated_val = debox_arg_and_migrate( + arg=arg, + protocol_state=protocol_state, + ) + migrated_args.append(migrated_val) + + return tuple(migrated_args), migrated_kwargs diff --git a/packages/syft/src/syft/protocol/protocol_version.json b/packages/syft/src/syft/protocol/protocol_version.json new file mode 100644 index 00000000000..0967ef424bc --- /dev/null +++ b/packages/syft/src/syft/protocol/protocol_version.json @@ -0,0 +1 @@ +{} diff --git a/packages/syft/src/syft/serde/recursive.py b/packages/syft/src/syft/serde/recursive.py index fa01f91c58c..19ac5170a21 100644 --- a/packages/syft/src/syft/serde/recursive.py +++ b/packages/syft/src/syft/serde/recursive.py @@ -135,6 +135,7 @@ def recursive_serde_register( attributes = set(attribute_list) if attribute_list else None attribute_types = get_types(cls, attributes) serde_overrides = getattr(cls, "__serde_overrides__", {}) + version = getattr(cls, "__version__", None) # without fqn duplicate class names overwrite serde_attributes = ( @@ -147,6 +148,7 @@ def recursive_serde_register( hash_exclude_attrs, cls, attribute_types, + version, ) TYPE_BANK[fqn] = serde_attributes @@ -203,6 +205,7 @@ def rs_object2proto(self: Any, for_hashing: bool = False) -> _DynamicStructBuild hash_exclude_attrs, cls, attribute_types, + version, ) = TYPE_BANK[fqn] if nonrecursive or is_type: @@ -304,6 +307,7 @@ def rs_proto2object(proto: _DynamicStructBuilder) -> Any: hash_exclude_attrs, cls, attribute_types, + version, ) = TYPE_BANK[proto.fullyQualifiedName] if class_type == type(None): diff --git a/packages/syft/src/syft/service/action/action_graph.py b/packages/syft/src/syft/service/action/action_graph.py index 489e51e91e5..c3e25ac098b 100644 --- a/packages/syft/src/syft/service/action/action_graph.py +++ b/packages/syft/src/syft/service/action/action_graph.py @@ -344,6 +344,8 @@ def _load_from_path(file_path: str) -> None: @serializable() class InMemoryGraphConfig(StoreConfig): + __canonical_name__ = "InMemoryGraphConfig" + store_type: Type[BaseGraphStore] = NetworkXBackingStore client_config: StoreClientConfig = InMemoryStoreClientConfig() locking_config: LockingConfig = ThreadingLockingConfig() @@ -356,6 +358,8 @@ class ActionGraphStore: @serializable() class InMemoryActionGraphStore(ActionGraphStore): + __canonical_name__ = "InMemoryActionGraphStore" + def __init__(self, store_config: StoreConfig, reset: bool = False): self.store_config: StoreConfig = store_config self.graph: Type[BaseGraphStore] = self.store_config.store_type( diff --git a/packages/syft/src/syft/service/action/action_object.py b/packages/syft/src/syft/service/action/action_object.py index a49bcedd780..f713bb69c9a 100644 --- a/packages/syft/src/syft/service/action/action_object.py +++ b/packages/syft/src/syft/service/action/action_object.py @@ -222,6 +222,9 @@ class ActionObjectPointer: class PreHookContext(SyftBaseObject): + __canonical_name__ = "PreHookContext" + __version__ = SYFT_OBJECT_VERSION_1 + """Hook context Parameters: @@ -446,6 +449,7 @@ def debox_args_and_kwargs(args: Any, kwargs: Any) -> Tuple[Any, Any]: ] +@serializable() class ActionObject(SyftObject): """Action object for remote execution.""" diff --git a/packages/syft/src/syft/service/context.py b/packages/syft/src/syft/service/context.py index 908e2a2c80c..a03bf4f99d4 100644 --- a/packages/syft/src/syft/service/context.py +++ b/packages/syft/src/syft/service/context.py @@ -57,6 +57,9 @@ class UnauthedServiceContext(NodeServiceContext): class ChangeContext(SyftBaseObject): + __canonical_name__ = "ChangeContext" + __version__ = SYFT_OBJECT_VERSION_1 + node: Optional[AbstractNode] = None approving_user_credentials: Optional[SyftVerifyKey] requesting_user_credentials: Optional[SyftVerifyKey] diff --git a/packages/syft/src/syft/service/metadata/__init__.py b/packages/syft/src/syft/service/metadata/__init__.py new file mode 100644 index 00000000000..80eb6c422b0 --- /dev/null +++ b/packages/syft/src/syft/service/metadata/__init__.py @@ -0,0 +1,2 @@ +# relative +from .migrations import * # noqa: F403 diff --git a/packages/syft/src/syft/service/metadata/migrations.py b/packages/syft/src/syft/service/metadata/migrations.py new file mode 100644 index 00000000000..dd6200b97a2 --- /dev/null +++ b/packages/syft/src/syft/service/metadata/migrations.py @@ -0,0 +1,21 @@ +# relative +from ...types.syft_migration import migrate +from ...types.transforms import rename +from .node_metadata import NodeMetadata +from .node_metadata import NodeMetadataV1 + + +@migrate(NodeMetadataV1, NodeMetadata) +def upgrade_metadata_v1_to_v2(): + return [ + rename("highest_object_version", "highest_version"), + rename("lowest_object_version", "lowest_version"), + ] + + +@migrate(NodeMetadata, NodeMetadataV1) +def downgrade_metadata_v2_to_v1(): + return [ + rename("highest_version", "highest_object_version"), + rename("lowest_version", "lowest_object_version"), + ] diff --git a/packages/syft/src/syft/service/metadata/node_metadata.py b/packages/syft/src/syft/service/metadata/node_metadata.py index d3b03e71c1a..a3b3922b36e 100644 --- a/packages/syft/src/syft/service/metadata/node_metadata.py +++ b/packages/syft/src/syft/service/metadata/node_metadata.py @@ -9,12 +9,15 @@ # third party from packaging import version from pydantic import BaseModel +from pydantic import root_validator # relative from ...abstract_node import NodeType from ...node.credentials import SyftVerifyKey +from ...protocol.data_protocol import get_data_protocol from ...serde.serializable import serializable from ...types.syft_object import SYFT_OBJECT_VERSION_1 +from ...types.syft_object import SYFT_OBJECT_VERSION_2 from ...types.syft_object import StorableObjectType from ...types.syft_object import SyftObject from ...types.transforms import convert_types @@ -60,7 +63,7 @@ class NodeMetadataUpdate(SyftObject): @serializable() -class NodeMetadata(SyftObject): +class NodeMetadataV1(SyftObject): __canonical_name__ = "NodeMetadata" __version__ = SYFT_OBJECT_VERSION_1 @@ -88,14 +91,43 @@ def check_version(self, client_version: str) -> bool: ) +@serializable() +class NodeMetadata(SyftObject): + __canonical_name__ = "NodeMetadata" + __version__ = SYFT_OBJECT_VERSION_2 + + name: str + highest_version: int + lowest_version: int + id: UID + verify_key: SyftVerifyKey + syft_version: str + node_type: NodeType = NodeType.DOMAIN + deployed_on: str = "Date" + organization: str = "OpenMined" + on_board: bool = False + description: str = "Text" + signup_enabled: bool + admin_email: str + node_side_type: str + show_warnings: bool + + def check_version(self, client_version: str) -> bool: + return check_version( + client_version=client_version, + server_version=self.syft_version, + server_name=self.name, + ) + + @serializable() class NodeMetadataJSON(BaseModel, StorableObjectType): metadata_version: int name: str id: str verify_key: str - highest_object_version: int - lowest_object_version: int + highest_object_version: Optional[int] + lowest_object_version: Optional[int] syft_version: str node_type: str = NodeType.DOMAIN.value deployed_on: str = "Date" @@ -106,6 +138,14 @@ class NodeMetadataJSON(BaseModel, StorableObjectType): admin_email: str node_side_type: str show_warnings: bool + supported_protocols: List = [] + + @root_validator(pre=True) + def add_protocol_versions(cls, values: dict) -> dict: + if "supported_protocols" not in values: + data_protocol = get_data_protocol() + values["supported_protocols"] = data_protocol.supported_protocols + return values def check_version(self, client_version: str) -> bool: return check_version( @@ -121,13 +161,17 @@ def metadata_to_json() -> List[Callable]: drop(["__canonical_name__"]), rename("__version__", "metadata_version"), convert_types(["id", "verify_key", "node_type"], str), + rename("highest_version", "highest_object_version"), + rename("lowest_version", "lowest_object_version"), ] @transform(NodeMetadataJSON, NodeMetadata) def json_to_metadata() -> List[Callable]: return [ - drop(["metadata_version"]), + drop(["metadata_version", "supported_protocols"]), convert_types(["id", "verify_key"], [UID, SyftVerifyKey]), convert_types(["node_type"], NodeType), + rename("highest_object_version", "highest_version"), + rename("lowest_object_version", "lowest_version"), ] diff --git a/packages/syft/src/syft/service/network/network_service.py b/packages/syft/src/syft/service/network/network_service.py index ee8fe02843b..aa9a4b62c9c 100644 --- a/packages/syft/src/syft/service/network/network_service.py +++ b/packages/syft/src/syft/service/network/network_service.py @@ -503,7 +503,6 @@ def http_connection_to_node_route() -> List[Callable]: def get_python_node_route(context: TransformContext) -> TransformContext: context.output["id"] = context.obj.node.id - print("Store config....", context.obj.node.blob_store_config) context.output["worker_settings"] = WorkerSettings.from_node(context.obj.node) context.output["proxy_target_uid"] = context.obj.proxy_target_uid return context diff --git a/packages/syft/src/syft/service/object_search/migration_state_service.py b/packages/syft/src/syft/service/object_search/migration_state_service.py new file mode 100644 index 00000000000..fefb28dc60c --- /dev/null +++ b/packages/syft/src/syft/service/object_search/migration_state_service.py @@ -0,0 +1,74 @@ +# stdlib +from typing import Union + +# relative +from ...serde.serializable import serializable +from ...store.document_store import DocumentStore +from ..context import AuthedServiceContext +from ..response import SyftError +from ..service import AbstractService +from ..service import service_method +from .object_migration_state import SyftMigrationStateStash +from .object_migration_state import SyftObjectMigrationState + + +@serializable() +class MigrateStateService(AbstractService): + store: DocumentStore + stash: SyftObjectMigrationState + + def __init__(self, store: DocumentStore) -> None: + self.store = store + self.stash = SyftMigrationStateStash(store=store) + + @service_method(path="migration", name="get_version") + def get_version( + self, context: AuthedServiceContext, canonical_name: str + ) -> Union[int, SyftError]: + """Search for the metadata for an object.""" + + result = self.stash.get_by_name( + canonical_name=canonical_name, credentials=context.credentials + ) + + if result.is_err(): + return SyftError(message=f"{result.err()}") + + migration_state = result.ok() + + if migration_state is None: + return SyftError( + message=f"No migration state exists for canonical name: {canonical_name}" + ) + + return migration_state.current_version + + @service_method(path="migration", name="get_state") + def get_state( + self, context: AuthedServiceContext, canonical_name: str + ) -> Union[bool, SyftError]: + result = self.stash.get_by_name( + canonical_name=canonical_name, credentials=context.credentials + ) + + if result.is_err(): + return SyftError(message=f"{result.err()}") + + return result.ok() + + @service_method(path="migration", name="register_migration_state") + def register_migration_state( + self, + context: AuthedServiceContext, + current_version: int, + canonical_name: str, + ) -> Union[SyftObjectMigrationState, SyftError]: + obj = SyftObjectMigrationState( + current_version=current_version, canonical_name=canonical_name + ) + result = self.stash.set(migration_state=obj, credentials=context.credentials) + + if result.is_err(): + return SyftError(message=f"{result.err()}") + + return result.ok() diff --git a/packages/syft/src/syft/service/object_search/object_migration_state.py b/packages/syft/src/syft/service/object_search/object_migration_state.py new file mode 100644 index 00000000000..686c1ccb8fd --- /dev/null +++ b/packages/syft/src/syft/service/object_search/object_migration_state.py @@ -0,0 +1,78 @@ +# stdlib +from typing import List +from typing import Optional + +# third party +from result import Result + +# relative +from ...node.credentials import SyftVerifyKey +from ...serde.serializable import serializable +from ...store.document_store import BaseStash +from ...store.document_store import DocumentStore +from ...store.document_store import PartitionKey +from ...store.document_store import PartitionSettings +from ...types.syft_object import SYFT_OBJECT_VERSION_1 +from ...types.syft_object import SyftMigrationRegistry +from ...types.syft_object import SyftObject +from ..action.action_permissions import ActionObjectPermission + + +@serializable() +class SyftObjectMigrationState(SyftObject): + __canonical_name__ = "SyftObjectMigrationState" + __version__ = SYFT_OBJECT_VERSION_1 + + __attr_unique__ = ["canonical_name"] + + canonical_name: str + current_version: int + + @property + def latest_version(self) -> Optional[int]: + available_versions = SyftMigrationRegistry.get_versions( + canonical_name=self.canonical_name, + ) + if available_versions is None: + return None + + return sorted(available_versions, reverse=True)[0] + + @property + def supported_versions(self) -> List: + return SyftMigrationRegistry.get_versions(self.canonical_name) + + +KlassNamePartitionKey = PartitionKey(key="canonical_name", type_=str) + + +@serializable() +class SyftMigrationStateStash(BaseStash): + object_type = SyftObjectMigrationState + settings: PartitionSettings = PartitionSettings( + name=SyftObjectMigrationState.__canonical_name__, + object_type=SyftObjectMigrationState, + ) + + def __init__(self, store: DocumentStore) -> None: + super().__init__(store=store) + + def set( + self, + credentials: SyftVerifyKey, + migration_state: SyftObjectMigrationState, + add_permissions: Optional[List[ActionObjectPermission]] = None, + ) -> Result[SyftObjectMigrationState, str]: + res = self.check_type(migration_state, self.object_type) + # we dont use and_then logic here as it is hard because of the order of the arguments + if res.is_err(): + return res + return super().set( + credentials=credentials, obj=res.ok(), add_permissions=add_permissions + ) + + def get_by_name( + self, canonical_name: str, credentials: SyftVerifyKey + ) -> Result[SyftObjectMigrationState, str]: + qks = KlassNamePartitionKey.with_obj(canonical_name) + return self.query_one(credentials=credentials, qks=qks) diff --git a/packages/syft/src/syft/service/service.py b/packages/syft/src/syft/service/service.py index 974fe2d02b6..52fb081b541 100644 --- a/packages/syft/src/syft/service/service.py +++ b/packages/syft/src/syft/service/service.py @@ -21,6 +21,7 @@ # relative from ..abstract_node import AbstractNode from ..node.credentials import SyftVerifyKey +from ..protocol.data_protocol import migrate_args_and_kwargs from ..serde.lib_permissions import CMPCRUDPermission from ..serde.lib_permissions import CMPPermission from ..serde.lib_service_registry import CMPBase @@ -32,6 +33,7 @@ from ..serde.signature import signature_remove_context from ..serde.signature import signature_remove_self from ..store.linked_obj import LinkedObject +from ..types.syft_object import SYFT_OBJECT_VERSION_1 from ..types.syft_object import SyftBaseObject from ..types.syft_object import SyftObject from ..types.syft_object import attach_attribute_to_syft_object @@ -73,6 +75,9 @@ def resolve_link( @serializable() class BaseConfig(SyftBaseObject): + __canonical_name__ = "BaseConfig" + __version__ = SYFT_OBJECT_VERSION_1 + public_path: str private_path: str public_name: str @@ -85,6 +90,7 @@ class BaseConfig(SyftBaseObject): @serializable() class ServiceConfig(BaseConfig): + __canonical_name__ = "ServiceConfig" permissions: List roles: List[ServiceRole] @@ -94,6 +100,7 @@ def has_permission(self, user_service_role: ServiceRole): @serializable() class LibConfig(BaseConfig): + __canonical_name__ = "LibConfig" permissions: Set[CMPPermission] def has_permission(self, credentials: SyftVerifyKey): @@ -329,6 +336,12 @@ def wrapper(func): input_signature = deepcopy(signature) def _decorator(self, *args, **kwargs): + communication_protocol = kwargs.pop("communication_protocol", None) + + if communication_protocol: + args, kwargs = migrate_args_and_kwargs( + args=args, kwargs=kwargs, to_latest_protocol=True + ) if autosplat is not None and len(autosplat) > 0: args, kwargs = reconstruct_args_kwargs( signature=input_signature, @@ -337,6 +350,13 @@ def _decorator(self, *args, **kwargs): kwargs=kwargs, ) result = func(self, *args, **kwargs) + if communication_protocol: + result, _ = migrate_args_and_kwargs( + args=(result,), + kwargs={}, + to_protocol=communication_protocol, + ) + result = result[0] context = kwargs.get("context", None) context = args[0] if context is None else context attrs_to_attach = { diff --git a/packages/syft/src/syft/store/dict_document_store.py b/packages/syft/src/syft/store/dict_document_store.py index b6af5e4bc0e..516a2fc85c5 100644 --- a/packages/syft/src/syft/store/dict_document_store.py +++ b/packages/syft/src/syft/store/dict_document_store.py @@ -78,6 +78,7 @@ def reset(self): @serializable() class DictStoreConfig(StoreConfig): + __canonical_name__ = "DictStoreConfig" """Dictionary-based configuration Parameters: diff --git a/packages/syft/src/syft/store/mongo_document_store.py b/packages/syft/src/syft/store/mongo_document_store.py index 05b5a94d16e..9a79aac1253 100644 --- a/packages/syft/src/syft/store/mongo_document_store.py +++ b/packages/syft/src/syft/store/mongo_document_store.py @@ -26,6 +26,7 @@ from ..service.action.action_permissions import ActionObjectWRITE from ..service.action.action_permissions import ActionPermission from ..service.response import SyftSuccess +from ..types.syft_object import SYFT_OBJECT_VERSION_1 from ..types.syft_object import StorableObjectType from ..types.syft_object import SyftBaseObject from ..types.syft_object import SyftObject @@ -47,6 +48,9 @@ @serializable() class MongoDict(SyftBaseObject): + __canonical_name__ = "MongoDict" + __version__ = SYFT_OBJECT_VERSION_1 + keys: List[Any] values: List[Any] @@ -570,6 +574,7 @@ class MongoDocumentStore(DocumentStore): @serializable() class MongoStoreConfig(StoreConfig): + __canonical_name__ = "MongoStoreConfig" """Mongo Store configuration Parameters: diff --git a/packages/syft/src/syft/store/sqlite_document_store.py b/packages/syft/src/syft/store/sqlite_document_store.py index 27195ae7ac5..856f604da09 100644 --- a/packages/syft/src/syft/store/sqlite_document_store.py +++ b/packages/syft/src/syft/store/sqlite_document_store.py @@ -386,6 +386,7 @@ def file_path(self) -> Optional[Path]: @serializable() class SQLiteStoreConfig(StoreConfig): + __canonical_name__ = "SQLiteStoreConfig" """SQLite Store config, used by SQLiteStorePartition Parameters: diff --git a/packages/syft/src/syft/types/base.py b/packages/syft/src/syft/types/base.py index de182c0f71a..bb5160aebb0 100644 --- a/packages/syft/src/syft/types/base.py +++ b/packages/syft/src/syft/types/base.py @@ -1,3 +1,5 @@ +# stdlib + # third party from pydantic import BaseModel diff --git a/packages/syft/src/syft/types/syft_migration.py b/packages/syft/src/syft/types/syft_migration.py new file mode 100644 index 00000000000..86f99320d8e --- /dev/null +++ b/packages/syft/src/syft/types/syft_migration.py @@ -0,0 +1,58 @@ +# stdlib +from typing import Callable +from typing import Optional +from typing import Union + +# relative +from .syft_object import SyftMigrationRegistry +from .transforms import generate_transform_wrapper +from .transforms import validate_klass_and_version + + +def migrate( + klass_from: Union[type, str], + klass_to: Union[type, str], + version_from: Optional[int] = None, + version_to: Optional[int] = None, +) -> Callable: + ( + klass_from_str, + version_from, + klass_to_str, + version_to, + ) = validate_klass_and_version( + klass_from=klass_from, + version_from=version_from, + klass_to=klass_to, + version_to=version_to, + ) + + if klass_from_str != klass_to_str: + raise Exception( + "Migration can only be performed across classes with same canonical name." + f"Provided args: klass_from: {klass_from_str}, klass_to: {klass_to_str}" + ) + + if version_from is None or version_to is None: + raise Exception( + "Version information missing at either of the classes." + f"{klass_from_str} has version: {version_from}, {klass_to_str} has version: {version_to}" + ) + + def decorator(function: Callable): + transforms = function() + + wrapper = generate_transform_wrapper( + klass_from=klass_from, klass_to=klass_to, transforms=transforms + ) + + SyftMigrationRegistry.register_transform( + klass_type_str=klass_from_str, + version_from=version_from, + version_to=version_to, + method=wrapper, + ) + + return function + + return decorator diff --git a/packages/syft/src/syft/types/syft_object.py b/packages/syft/src/syft/types/syft_object.py index 9aa056bc87f..49f89f4f99d 100644 --- a/packages/syft/src/syft/types/syft_object.py +++ b/packages/syft/src/syft/types/syft_object.py @@ -23,7 +23,6 @@ # third party import pandas as pd import pydantic -from pydantic import BaseModel from pydantic import EmailStr from pydantic.fields import Undefined from result import OkErr @@ -51,11 +50,6 @@ SYFT_OBJECT_VERSION_1 = 1 SYFT_OBJECT_VERSION_2 = 2 -supported_object_versions = [SYFT_OBJECT_VERSION_1, SYFT_OBJECT_VERSION_2] - -HIGHEST_SYFT_OBJECT_VERSION = max(supported_object_versions) -LOWEST_SYFT_OBJECT_VERSION = min(supported_object_versions) - # These attributes are dynamically added based on node/client # that is interaction with the SyftObject @@ -80,7 +74,7 @@ def hash(self) -> str: return self.__sha256__().hex() -class SyftBaseObject(BaseModel, SyftHashableObject): +class SyftBaseObject(pydantic.BaseModel, SyftHashableObject): class Config: arbitrary_types_allowed = True @@ -96,6 +90,9 @@ def _set_obj_location_(self, node_uid, credentials): class Context(SyftBaseObject): + __canonical_name__ = "Context" + __version__ = SYFT_OBJECT_VERSION_1 + pass @@ -179,10 +176,153 @@ def get_transform( ) +class SyftMigrationRegistry: + __migration_version_registry__: Dict[str, Dict[int, str]] = {} + __migration_transform_registry__: Dict[str, Dict[str, Callable]] = {} + + def __init_subclass__(cls, **kwargs: Any) -> None: + """ + Populate the `__migration_version_registry__` dictionary with format + __migration_version_registry__ = { + "canonical_name": {version_number: "klass_name"} + } + For example + __migration_version_registry__ = { + 'APIEndpoint': {1: 'syft.client.api.APIEndpoint'}, + 'Action': {1: 'syft.service.action.action_object.Action'}, + } + """ + super().__init_subclass__(**kwargs) + klass = type(cls) if not isinstance(cls, type) else cls + cls.register_version(klass=klass) + + @classmethod + def register_version(cls, klass: type): + if hasattr(klass, "__canonical_name__") and hasattr(klass, "__version__"): + mapping_string = klass.__canonical_name__ + klass_version = klass.__version__ + fqn = f"{klass.__module__}.{klass.__name__}" + + if ( + mapping_string in cls.__migration_version_registry__ + and not autoreload_enabled() + ): + versions = cls.__migration_version_registry__[mapping_string] + versions[klass_version] = fqn + else: + # only if the cls has not been registered do we want to register it + cls.__migration_version_registry__[mapping_string] = { + klass_version: fqn + } + + @classmethod + def get_versions(cls, canonical_name: str) -> List[int]: + available_versions: Dict = cls.__migration_version_registry__.get( + canonical_name, + {}, + ) + return list(available_versions.keys()) + + @classmethod + def register_transform( + cls, klass_type_str: str, version_from: int, version_to: int, method: Callable + ) -> None: + """ + Populate the __migration_transform_registry__ dictionary with format + __migration_version_registry__ = { + "canonical_name": {"version_from x version_to": } + } + For example + {'NodeMetadata': {'1x2': , + '2x1': }} + """ + if klass_type_str not in cls.__migration_version_registry__: + raise Exception(f"{klass_type_str} is not yet registered.") + + available_versions = cls.__migration_version_registry__[klass_type_str] + + versions_exists = ( + version_from in available_versions and version_to in available_versions + ) + + if versions_exists: + mapping_string = f"{version_from}x{version_to}" + if klass_type_str not in cls.__migration_transform_registry__: + cls.__migration_transform_registry__[klass_type_str] = {} + cls.__migration_transform_registry__[klass_type_str][ + mapping_string + ] = method + else: + raise Exception( + f"Available versions for {klass_type_str} are: {available_versions}." + f"You're trying to add a transform from version: {version_from} to version: {version_to}" + ) + + @classmethod + def get_migration( + cls, type_from: Type[SyftBaseObject], type_to: Type[SyftBaseObject] + ) -> Callable: + for type_from_mro in type_from.mro(): + if ( + issubclass(type_from_mro, SyftBaseObject) + and type_from_mro != SyftBaseObject + ): + klass_from = type_from_mro.__canonical_name__ + version_from = type_from_mro.__version__ + + for type_to_mro in type_to.mro(): + if ( + issubclass(type_to_mro, SyftBaseObject) + and type_to_mro != SyftBaseObject + ): + klass_to = type_to_mro.__canonical_name__ + version_to = type_to_mro.__version__ + + if klass_from == klass_to: + mapping_string = f"{version_from}x{version_to}" + if ( + mapping_string + in cls.__migration_transform_registry__[klass_from] + ): + return cls.__migration_transform_registry__[klass_from][ + mapping_string + ] + + @classmethod + def get_migration_for_version( + cls, type_from: Type[SyftBaseObject], version_to: int + ) -> Callable: + canonical_name = type_from.__canonical_name__ + for type_from_mro in type_from.mro(): + if ( + issubclass(type_from_mro, SyftBaseObject) + and type_from_mro != SyftBaseObject + ): + klass_from = type_from_mro.__canonical_name__ + if klass_from != canonical_name: + continue + version_from = type_from_mro.__version__ + mapping_string = f"{version_from}x{version_to}" + if ( + mapping_string + in cls.__migration_transform_registry__[ + type_from.__canonical_name__ + ] + ): + return cls.__migration_transform_registry__[klass_from][ + mapping_string + ] + + raise Exception( + f"No migration found for class type: {type_from} to " + "version: {version_to} in the migration registry." + ) + + print_type_cache = defaultdict(list) -class SyftObject(SyftBaseObject, SyftObjectRegistry): +class SyftObject(SyftBaseObject, SyftObjectRegistry, SyftMigrationRegistry): __canonical_name__ = "SyftObject" __version__ = SYFT_OBJECT_VERSION_1 @@ -458,6 +598,17 @@ def _syft_unique_keys_dict(cls) -> Dict[str, type]: def _syft_searchable_keys_dict(cls) -> Dict[str, type]: return cls._syft_keys_types_dict("__attr_searchable__") + def migrate_to(self, version: int, context: Optional[Context] = None) -> Any: + if self.__version__ != version: + migration_transform = SyftMigrationRegistry.get_migration_for_version( + type_from=type(self), version_to=version + ) + return migration_transform( + self, + context, + ) + return self + def short_qual_name(name: str) -> str: # If the name is a qualname of formax a.b.c.d we will only get d diff --git a/packages/syft/src/syft/util/util.py b/packages/syft/src/syft/util/util.py index dd992643233..9a464d8c5ef 100644 --- a/packages/syft/src/syft/util/util.py +++ b/packages/syft/src/syft/util/util.py @@ -22,6 +22,7 @@ import sys import threading import time +import types from types import ModuleType from typing import Any from typing import Callable @@ -51,6 +52,10 @@ PANDAS_DATA = f"{DATASETS_URL}/pandas_cookbook" +def get_env(key: str, default: Optional[Any] = None) -> Optional[str]: + return os.environ.get(key, default) + + def full_name_with_qualname(klass: type) -> str: """Returns the klass module name + klass qualname.""" try: @@ -865,3 +870,12 @@ def get_interpreter_module() -> str: def thread_ident() -> int: return threading.current_thread().ident + + +def set_klass_module_to_syft(klass, module_name): + if module_name not in sys.modules["syft"].__dict__: + new_module = types.ModuleType(module_name) + else: + new_module = sys.modules["syft"].__dict__[module_name] + setattr(new_module, klass.__name__, klass) + sys.modules["syft"].__dict__[module_name] = new_module diff --git a/packages/syft/tests/conftest.py b/packages/syft/tests/conftest.py index 734faf9d5a5..49d19b1049f 100644 --- a/packages/syft/tests/conftest.py +++ b/packages/syft/tests/conftest.py @@ -1,9 +1,17 @@ +# stdlib +import json +from pathlib import Path +from unittest import mock + # third party from faker import Faker import pytest # syft absolute import syft as sy +from syft.protocol.data_protocol import bump_protocol_version +from syft.protocol.data_protocol import get_data_protocol +from syft.protocol.data_protocol import stage_protocol_changes # relative from .syft.stores.store_fixtures_test import dict_action_store # noqa: F401 @@ -26,8 +34,41 @@ def faker(): return Faker() +def create_file(filepath: Path, data: dict): + with open(filepath, "w") as fp: + fp.write(json.dumps(data)) + + +def remove_file(filepath: Path): + filepath.unlink(missing_ok=True) + + +@pytest.fixture(autouse=True) +def protocol_file(): + random_name = sy.UID().to_string() + protocol_dir = sy.SYFT_PATH / "protocol" + file_path = protocol_dir / f"{random_name}.json" + dp = get_data_protocol() + create_file(filepath=file_path, data=dp.protocol_history) + yield file_path + remove_file(filepath=file_path) + + +@pytest.fixture(autouse=True) +def stage_protocol(protocol_file: Path): + with mock.patch( + "syft.protocol.data_protocol.PROTOCOL_STATE_FILENAME", + protocol_file.name, + ): + dp = get_data_protocol() + stage_protocol_changes() + bump_protocol_version() + yield dp.protocol_history + dp.save_history(dp.protocol_history) + + @pytest.fixture(autouse=True) -def worker(faker): +def worker(faker, stage_protocol): return sy.Worker.named(name=faker.name()) diff --git a/packages/syft/tests/syft/hash_test.py b/packages/syft/tests/syft/hash_test.py index 9032a64a3aa..68655836437 100644 --- a/packages/syft/tests/syft/hash_test.py +++ b/packages/syft/tests/syft/hash_test.py @@ -4,6 +4,7 @@ # syft absolute from syft.serde.serializable import serializable +from syft.types.syft_object import SYFT_OBJECT_VERSION_1 from syft.types.syft_object import SyftBaseObject from syft.types.syft_object import SyftHashableObject @@ -25,6 +26,9 @@ def __init__(self, key, value, flag=None): @serializable(attrs=["id", "data"]) class MockWrapper(SyftBaseObject, SyftHashableObject): + __canonical_name__ = "MockWrapper" + __version__ = SYFT_OBJECT_VERSION_1 + id: str data: Optional[MockObject] diff --git a/packages/syft/tests/syft/migrations/protocol_communication_test.py b/packages/syft/tests/syft/migrations/protocol_communication_test.py new file mode 100644 index 00000000000..254a2033873 --- /dev/null +++ b/packages/syft/tests/syft/migrations/protocol_communication_test.py @@ -0,0 +1,233 @@ +# stdlib +from copy import deepcopy +from typing import List +from typing import Type +from typing import Union +from unittest import mock + +# syft absolute +import syft as sy +from syft.node.worker import Worker +from syft.protocol.data_protocol import get_data_protocol +from syft.serde.recursive import TYPE_BANK +from syft.serde.serializable import serializable +from syft.service.context import AuthedServiceContext +from syft.service.response import SyftError +from syft.service.service import AbstractService +from syft.service.service import ServiceConfigRegistry +from syft.service.service import service_method +from syft.service.user.user_roles import GUEST_ROLE_LEVEL +from syft.store.document_store import BaseStash +from syft.store.document_store import DocumentStore +from syft.store.document_store import PartitionSettings +from syft.types.syft_migration import migrate +from syft.types.syft_object import SYFT_OBJECT_VERSION_1 +from syft.types.syft_object import SYFT_OBJECT_VERSION_2 +from syft.types.syft_object import SyftBaseObject +from syft.types.syft_object import SyftObject +from syft.types.transforms import convert_types +from syft.types.transforms import rename +from syft.types.uid import UID +from syft.util.util import index_syft_by_module_name + +MOCK_TYPE_BANK = deepcopy(TYPE_BANK) + + +def get_klass_version_1(): + @serializable() + class SyftMockObjectTestV1(SyftObject): + __canonical_name__ = "SyftMockObjectTest" + __version__ = SYFT_OBJECT_VERSION_1 + + id: UID + name: str + version: int + + return SyftMockObjectTestV1 + + +def get_klass_version_2(): + @serializable() + class SyftMockObjectTestV2(SyftObject): + __canonical_name__ = "SyftMockObjectTest" + __version__ = SYFT_OBJECT_VERSION_2 + + id: UID + full_name: str + version: str + + return SyftMockObjectTestV2 + + +def setup_migration_transforms(mock_klass_v1, mock_klass_v2): + @migrate(mock_klass_v1, mock_klass_v2) + def mock_v1_to_v2(): + return [rename("name", "full_name"), convert_types(["version"], str)] + + @migrate(mock_klass_v2, mock_klass_v1) + def mock_v2_to_v1(): + return [rename("full_name", "name"), convert_types(["version"], int)] + + return mock_v1_to_v2, mock_v2_to_v1 + + +def get_stash_klass(syft_object: Type[SyftBaseObject]): + @serializable() + class SyftMockObjectStash(BaseStash): + object_type = syft_object + settings: PartitionSettings = PartitionSettings( + name=object_type.__canonical_name__, + object_type=syft_object, + ) + + def __init__(self, store: DocumentStore) -> None: + super().__init__(store=store) + + return SyftMockObjectStash + + +def setup_service_method(syft_object): + stash_klass: BaseStash = get_stash_klass(syft_object=syft_object) + + @serializable() + class SyftMockObjectService(AbstractService): + store: DocumentStore + stash: stash_klass + __module__: str = "syft.test" + + def __init__(self, store: DocumentStore) -> None: + self.store = store + self.stash = stash_klass(store=store) + + @service_method( + path="dummy.syft_object", + name="get", + roles=GUEST_ROLE_LEVEL, + ) + def get( + self, context: AuthedServiceContext + ) -> Union[List[syft_object], SyftError]: + result = self.stash.get_all(context.credentials, has_permission=True) + if result.is_ok(): + return result.ok() + return SyftError(message=f"{result.err()}") + + return SyftMockObjectService + + +def setup_version_one(node_name: str): + syft_klass_version_one = get_klass_version_1() + sy.stage_protocol_changes() + sy.bump_protocol_version() + + syft_service_klass = setup_service_method( + syft_object=syft_klass_version_one, + ) + + node = sy.orchestra.launch(node_name, dev_mode=True, reset=True) + + worker: Worker = node.python_node + + worker.services.append(syft_service_klass) + worker.service_path_map[syft_service_klass.__name__.lower()] = syft_service_klass( + store=worker.document_store + ) + + return node, syft_klass_version_one + + +def setup_version_second(node_name: str, klass_version_one: type): + syft_klass_version_second = get_klass_version_2() + setup_migration_transforms(klass_version_one, syft_klass_version_second) + + sy.stage_protocol_changes() + sy.bump_protocol_version() + + syft_service_klass = setup_service_method(syft_object=syft_klass_version_second) + + node = sy.orchestra.launch(node_name, dev_mode=True) + + worker: Worker = node.python_node + + worker.services.append(syft_service_klass) + worker.service_path_map[syft_service_klass.__name__.lower()] = syft_service_klass( + store=worker.document_store + ) + + return node, syft_klass_version_second + + +def test_client_server_running_different_protocols(stage_protocol): + def patched_index_syft_by_module_name(fully_qualified_name: str): + if klass_v1.__name__ in fully_qualified_name: + return klass_v1 + elif klass_v2.__name__ in fully_qualified_name: + return klass_v2 + + return index_syft_by_module_name(fully_qualified_name) + + node_name = UID().to_string() + with mock.patch("syft.serde.recursive.TYPE_BANK", MOCK_TYPE_BANK): + with mock.patch( + "syft.protocol.data_protocol.TYPE_BANK", + MOCK_TYPE_BANK, + ): + with mock.patch( + "syft.client.api.index_syft_by_module_name", + patched_index_syft_by_module_name, + ): + # Setup mock object version one + nh1, klass_v1 = setup_version_one(node_name) + assert klass_v1.__canonical_name__ == "SyftMockObjectTest" + assert klass_v1.__name__ == "SyftMockObjectTestV1" + + nh1_client = nh1.client + assert nh1_client is not None + result_from_client_1 = nh1_client.api.services.dummy.get() + + protocol_version_with_mock_obj_v1 = get_data_protocol().latest_version + + # No data saved + assert len(result_from_client_1) == 0 + + # Setup mock object version second + nh2, klass_v2 = setup_version_second( + node_name, klass_version_one=klass_v1 + ) + + # Create a sample data in version second + sample_data = klass_v2(full_name="John", version=str(1), id=UID()) + + assert isinstance(sample_data, klass_v2) + + # Validate migrations + sample_data_v1 = sample_data.migrate_to( + version=klass_v1.__version__, + ) + assert sample_data_v1.name == sample_data.full_name + assert sample_data_v1.version == int(sample_data.version) + + # Set the sample data in version second + service_klass = nh1.python_node.get_service("SyftMockObjectService") + service_klass.stash.set( + nh1.python_node.root_client.verify_key, + sample_data, + ) + + nh2_client = nh2.client + assert nh2_client is not None + # Force communication protocol to when version object is defined + nh2_client.communication_protocol = protocol_version_with_mock_obj_v1 + # Reset api + nh2_client._api = None + + # Call the API with an older communication protocol version + result2 = nh2_client.api.services.dummy.get() + assert isinstance(result2, list) + + # Validate the data received + for data in result2: + assert isinstance(data, klass_v1) + assert data.name == sample_data.full_name + assert data.version == int(sample_data.version) + ServiceConfigRegistry.__service_config_registry__.pop("dummy.syft_object", None) diff --git a/packages/syft/tests/syft/settings/fixtures.py b/packages/syft/tests/syft/settings/fixtures.py index 1a4c6ac0faf..3d3c3abee8b 100644 --- a/packages/syft/tests/syft/settings/fixtures.py +++ b/packages/syft/tests/syft/settings/fixtures.py @@ -7,13 +7,13 @@ # syft absolute from syft.__init__ import __version__ from syft.abstract_node import NodeSideType +from syft.abstract_node import NodeType from syft.service.metadata.node_metadata import NodeMetadataJSON from syft.service.settings.settings import NodeSettings from syft.service.settings.settings import NodeSettingsUpdate from syft.service.settings.settings_service import SettingsService from syft.service.settings.settings_stash import SettingsStash -from syft.types.syft_object import HIGHEST_SYFT_OBJECT_VERSION -from syft.types.syft_object import LOWEST_SYFT_OBJECT_VERSION +from syft.types.syft_object import SYFT_OBJECT_VERSION_1 @pytest.fixture @@ -52,13 +52,14 @@ def metadata_json(faker) -> NodeMetadataJSON: name=faker.name(), id=faker.text(), verify_key=faker.text(), - highest_object_version=HIGHEST_SYFT_OBJECT_VERSION, - lowest_object_version=LOWEST_SYFT_OBJECT_VERSION, + highest_version=SYFT_OBJECT_VERSION_1, + lowest_version=SYFT_OBJECT_VERSION_1, syft_version=__version__, signup_enabled=False, admin_email="info@openmined.org", node_side_type=NodeSideType.LOW_SIDE.value, show_warnings=False, + node_type=NodeType.DOMAIN.value, ) diff --git a/packages/syft/tests/syft/settings/settings_service_test.py b/packages/syft/tests/syft/settings/settings_service_test.py index 967c00ececa..a8ba4d0da68 100644 --- a/packages/syft/tests/syft/settings/settings_service_test.py +++ b/packages/syft/tests/syft/settings/settings_service_test.py @@ -230,8 +230,8 @@ def test_settings_allow_guest_registration( mock_node_metadata = NodeMetadata( name=faker.name(), verify_key=verify_key, - highest_object_version=1, - lowest_object_version=2, + highest_version=1, + lowest_version=2, syft_version=syft.__version__, signup_enabled=False, admin_email="info@openmined.org", @@ -312,8 +312,8 @@ def get_mock_client(faker, root_client, role): mock_node_metadata = NodeMetadata( name=faker.name(), verify_key=verify_key, - highest_object_version=1, - lowest_object_version=2, + highest_version=1, + lowest_version=2, syft_version=syft.__version__, signup_enabled=False, admin_email="info@openmined.org", diff --git a/packages/syft/tests/syft/stores/store_mocks_test.py b/packages/syft/tests/syft/stores/store_mocks_test.py index bba35d928dd..38a6824cc76 100644 --- a/packages/syft/tests/syft/stores/store_mocks_test.py +++ b/packages/syft/tests/syft/stores/store_mocks_test.py @@ -52,17 +52,19 @@ class MockObjectType(SyftObject): @serializable() class MockStore(DocumentStore): + __canonical_name__ = "MockStore" pass @serializable() class MockSyftObject(SyftObject): - __canonical_name__ = UID() + __canonical_name__ = str(UID()) data: Any @serializable() class MockStoreConfig(StoreConfig): + __canonical_name__ = "MockStoreConfig" store_type: Type[DocumentStore] = MockStore db_name: str = "testing" backing_store: Type[KeyValueBackingStore] = MockKeyValueBackingStore diff --git a/tox.ini b/tox.ini index 848937f998b..5ae5f7c8f3c 100644 --- a/tox.ini +++ b/tox.ini @@ -20,6 +20,7 @@ envlist = syft.build.helm syft.package.helm syft.test.helm + syft.protocol.check syftcli.test.unit syftcli.publish syftcli.build @@ -271,9 +272,11 @@ commands = bash -c "docker volume rm test-gateway-1_tailscale-data --force || true" bash -c "docker volume rm test-gateway-1_headscale-data --force || true" - bash -c 'HAGRID_ART=$HAGRID_ART hagrid launch test_gateway_1 network to docker:9081 $HAGRID_FLAGS --no-health-checks --verbose --no-warnings' - bash -c 'HAGRID_ART=$HAGRID_ART hagrid launch test_domain_1 domain to docker:9082 $HAGRID_FLAGS --no-health-checks --enable-signup --verbose --no-warnings' - bash -c 'HAGRID_ART=$HAGRID_ART hagrid launch test_domain_2 domain to docker:9083 --headless $HAGRID_FLAGS --enable-signup --no-health-checks --verbose --no-warnings' + python -c 'import syft as sy; sy.stage_protocol_changes()' + + bash -c 'HAGRID_ART=$HAGRID_ART hagrid launch test_gateway_1 network to docker:9081 $HAGRID_FLAGS --no-health-checks --verbose --no-warnings --dev' + bash -c 'HAGRID_ART=$HAGRID_ART hagrid launch test_domain_1 domain to docker:9082 $HAGRID_FLAGS --no-health-checks --enable-signup --verbose --no-warnings --dev' + bash -c 'HAGRID_ART=$HAGRID_ART hagrid launch test_domain_2 domain to docker:9083 --headless $HAGRID_FLAGS --enable-signup --no-health-checks --verbose --no-warnings --dev' ; wait for nodes to start docker ps @@ -342,6 +345,23 @@ commands = pip install jupyter jupyterlab --upgrade jupyter lab --ip 0.0.0.0 --ServerApp.token={posargs} +[testenv:syft.protocol.check] +description = Syft Protocol Check +deps = + {[testenv:syft]deps} +changedir = {toxinidir}/packages/syft +allowlist_externals = + bash + +setenv = + BUMP = {env:BUMP:False} +commands = + bash -c "echo Using BUMP=${BUMP}" + python -c 'import syft as sy; sy.check_or_stage_protocol()' + bash -c 'if [[ "$BUMP" != "False" ]]; then \ + python -c "import syft as sy; sy.bump_protocol_version()"; \ + fi' + [testenv:syft.publish] changedir = {toxinidir}/packages/syft description = Build and Publish Syft Wheel @@ -351,7 +371,6 @@ commands = python -c 'from shutil import rmtree; rmtree("build", True); rmtree("dist", True)' python -m build . - [testenv:syft.test.security] description = Security Checks for Syft changedir = {toxinidir}/packages/syft @@ -535,6 +554,7 @@ setenv = ORCHESTRA_DEPLOYMENT_TYPE = {env:ORCHESTRA_DEPLOYMENT_TYPE:podman} NODE_PORT = {env:NODE_PORT:8080} commands = + python -c 'import syft as sy; sy.stage_protocol_changes()' bash -c "podman pod rm --force --all || true"; bash -c "podman system prune --volumes --force || true"; bash -c "podman volume rm $(podman volume ls -q)||true"; @@ -604,6 +624,7 @@ setenv = ORCHESTRA_DEPLOYMENT_TYPE = {env:ORCHESTRA_DEPLOYMENT_TYPE:k8s} NODE_PORT = {env:NODE_PORT:9082} commands = + python -c 'import syft as sy; sy.stage_protocol_changes()' k3d version ; bash -c "docker rm $(docker ps -aq) --force || true" @@ -868,6 +889,7 @@ commands = bash -c '(kubectl logs service/backend --context k3d-syft --namespace syft -f &) | grep -q "Application startup complete" || true' + ; frontend bash -c 'if [[ "$PYTEST_MODULES" == *"frontend"* ]]; then \ echo "Starting frontend"; date; \