From 9421f8270960f62a4b0c5fc85b76ba5fa74bfd41 Mon Sep 17 00:00:00 2001 From: Brendan Schell Date: Thu, 19 Sep 2024 17:44:16 -0400 Subject: [PATCH 01/28] Cherry-pick 6cdae719e711f537e720ca69528a5352797c2de4 Co-authored-by: Sameer Wagh --- .gitignore | 4 +- .../sync/000-setup-high-low-datasites.ipynb | 662 +++++---- .../sync/010-setup-high-low-datasites.ipynb | 331 +++++ .../sync/011-users-emails-passwords.ipynb | 669 +++++++++ .../sync/020-configure-api-and-sync.ipynb | 1314 +++++++++-------- .../bigquery/sync/030-ds-submit-request.ipynb | 66 +- .../sync/040-do-review-requests.ipynb | 60 +- .../bigquery/sync/050-ds-get-results.ipynb | 22 +- .../syft/util/test_helpers/email_helpers.py | 24 +- 9 files changed, 2174 insertions(+), 978 deletions(-) create mode 100644 notebooks/scenarios/bigquery/sync/010-setup-high-low-datasites.ipynb create mode 100644 notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb diff --git a/.gitignore b/.gitignore index 7285116f6ca..035ec2245f2 100644 --- a/.gitignore +++ b/.gitignore @@ -86,8 +86,8 @@ packages/grid/helm/examples/dev/migration.yaml # dynaconf settings file **/settings.yaml - +# Any temporary material created for scenarios notebooks/scenarios/bigquery/*.json - +notebooks/scenarios/bigquery/sync/*.json notebooks/tutorials/version-upgrades/*.yaml notebooks/tutorials/version-upgrades/*.blob diff --git a/notebooks/scenarios/bigquery/sync/000-setup-high-low-datasites.ipynb b/notebooks/scenarios/bigquery/sync/000-setup-high-low-datasites.ipynb index 82e69b5b829..1c6f3baec58 100644 --- a/notebooks/scenarios/bigquery/sync/000-setup-high-low-datasites.ipynb +++ b/notebooks/scenarios/bigquery/sync/000-setup-high-low-datasites.ipynb @@ -1,335 +1,331 @@ { - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# stdlib\n", - "import os\n", - "\n", - "# Testing works over 4 possibilities\n", - "# 1. (python/in-memory workers and using tox commands)\n", - "# 2. (python/in-memory workers and manually running notebooks)\n", - "# 3. (using k8s and using tox commands)\n", - "# 4. (using k8s and manually running notebooks)\n", - "# Uncomment the lines below if in the 4th possibility\n", - "\n", - "# os.environ[\"ORCHESTRA_DEPLOYMENT_TYPE\"] = \"remote\"\n", - "# os.environ[\"DEV_MODE\"] = \"True\"\n", - "# os.environ[\"TEST_EXTERNAL_REGISTRY\"] = \"k3d-registry.localhost:5800\"\n", - "# os.environ[\"CLUSTER_HTTP_PORT_HIGH\"] = \"9081\"\n", - "# os.environ[\"CLUSTER_HTTP_PORT_LOW\"] = \"9083\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# stdlib\n", - "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", - "high_port = os.environ.get(\"CLUSTER_HTTP_PORT_HIGH\", \"9081\")\n", - "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"9083\")\n", - "print(environment, high_port, low_port)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# syft absolute\n", - "import syft as sy\n", - "from syft import test_settings\n", - "from syft.util.test_helpers.worker_helpers import (\n", - " build_and_launch_worker_pool_from_docker_str,\n", - ")\n", - "from syft.util.test_helpers.worker_helpers import (\n", - " launch_worker_pool_from_docker_tag_and_registry,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Launch server & login" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "server_low = sy.orchestra.launch(\n", - " name=\"bigquery-low\",\n", - " server_side_type=\"low\",\n", - " dev_mode=True,\n", - " reset=True,\n", - " n_consumers=1,\n", - " create_producer=True,\n", - " port=low_port,\n", - ")\n", - "\n", - "server_high = sy.orchestra.launch(\n", - " name=\"bigquery-high\",\n", - " server_side_type=\"high\",\n", - " dev_mode=True,\n", - " reset=True,\n", - " n_consumers=1,\n", - " create_producer=True,\n", - " port=high_port,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# start email server here" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "low_client = server_low.login(email=\"info@openmined.org\", password=\"changethis\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "high_client = server_high.login(email=\"info@openmined.org\", password=\"changethis\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert len(high_client.worker_pools.get_all()) == 1\n", - "assert len(low_client.worker_pools.get_all()) == 1" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Setup High First\n", - "\n", - "- If using an external registery, we want to get this from the test_settings.\n", - "- We build the docker image over the base docker image in Syft\n", - "- We give a tag called worker-bigquery to our custom pool image" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "external_registry = test_settings.get(\"external_registry\", default=\"docker.io\")\n", - "\n", - "base_worker_image = high_client.images.get_all()[0]\n", - "\n", - "worker_dockerfile = f\"\"\"\n", - "FROM {str(base_worker_image.image_identifier)}\n", - "\n", - "RUN uv pip install db-dtypes google-cloud-bigquery \n", - "\n", - "\"\"\".strip()\n", - "\n", - "docker_tag = str(base_worker_image.image_identifier).replace(\n", - " \"backend\", \"worker-bigquery\"\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "worker_pool_name = \"bigquery-pool\"\n", - "custom_pool_pod_annotations = {\"bigquery-custom-pool\": \"Pod annotation for bigquery\"}\n", - "custom_pool_pod_labels = {\"bigquery-custom-pool\": \"Pod_label_for_bigquery\"}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "build_and_launch_worker_pool_from_docker_str(\n", - " environment=environment,\n", - " client=high_client,\n", - " worker_pool_name=worker_pool_name,\n", - " custom_pool_pod_annotations=custom_pool_pod_annotations,\n", - " custom_pool_pod_labels=custom_pool_pod_labels,\n", - " worker_dockerfile=worker_dockerfile,\n", - " external_registry=external_registry,\n", - " docker_tag=docker_tag,\n", - " scale_to=2,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert len(high_client.worker_pools.get_all()) == 2" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "high_client.settings.allow_guest_signup(enable=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Setup Low" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "launch_result = launch_worker_pool_from_docker_tag_and_registry(\n", - " environment=environment,\n", - " client=low_client,\n", - " worker_pool_name=worker_pool_name,\n", - " custom_pool_pod_annotations=custom_pool_pod_annotations,\n", - " custom_pool_pod_labels=custom_pool_pod_labels,\n", - " docker_tag=docker_tag,\n", - " external_registry=external_registry,\n", - " scale_to=1,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert len(low_client.worker_pools.get_all()) == 2" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Register a DS only on the low side" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "low_client.register(\n", - " email=\"data_scientist@openmined.org\",\n", - " password=\"verysecurepassword\",\n", - " password_verify=\"verysecurepassword\",\n", - " name=\"John Doe\",\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "low_client.settings.allow_guest_signup(enable=False)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert (\n", - " len(low_client.api.services.user.get_all()) == 2\n", - "), \"Only DS and Admin should be at low side\"\n", - "assert (\n", - " len(high_client.api.services.user.get_all()) == 1\n", - "), \"Only Admin should be at high side\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Close" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if environment != \"remote\":\n", - " server_high.land()\n", - " server_low.land()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# TODO close email client" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.4" - } - }, - "nbformat": 4, - "nbformat_minor": 4 + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# stdlib\n", + "import os\n", + "\n", + "# Testing works over 4 possibilities\n", + "# 1. (python/in-memory workers and using tox commands)\n", + "# 2. (python/in-memory workers and manually running notebooks)\n", + "# 3. (using k8s and using tox commands)\n", + "# 4. (using k8s and manually running notebooks)\n", + "# Uncomment the lines below if in the 4th possibility\n", + "\n", + "# os.environ[\"ORCHESTRA_DEPLOYMENT_TYPE\"] = \"remote\"\n", + "# os.environ[\"DEV_MODE\"] = \"True\"\n", + "# os.environ[\"TEST_EXTERNAL_REGISTRY\"] = \"k3d-registry.localhost:5800\"\n", + "# os.environ[\"CLUSTER_HTTP_PORT_HIGH\"] = \"9081\"\n", + "# os.environ[\"CLUSTER_HTTP_PORT_LOW\"] = \"9083\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# stdlib\n", + "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", + "high_port = os.environ.get(\"CLUSTER_HTTP_PORT_HIGH\", \"9081\")\n", + "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"9083\")\n", + "print(environment, high_port, low_port)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# syft absolute\n", + "import syft as sy\n", + "from syft import test_settings\n", + "from syft.util.test_helpers.worker_helpers import (\n", + " build_and_launch_worker_pool_from_docker_str,\n", + ")\n", + "from syft.util.test_helpers.worker_helpers import (\n", + " launch_worker_pool_from_docker_tag_and_registry,\n", + ")\n", + "from syft.util.test_helpers.email_helpers import get_email_server\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Launch server & login" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "server_low = sy.orchestra.launch(\n", + " name=\"bigquery-low\",\n", + " server_side_type=\"low\",\n", + " dev_mode=True,\n", + " reset=True,\n", + " n_consumers=1,\n", + " create_producer=True,\n", + " port=low_port,\n", + ")\n", + "\n", + "server_high = sy.orchestra.launch(\n", + " name=\"bigquery-high\",\n", + " server_side_type=\"high\",\n", + " dev_mode=True,\n", + " reset=True,\n", + " n_consumers=1,\n", + " create_producer=True,\n", + " port=high_port,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Email Server" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "email_server_low, smtp_server_low = get_email_server(reset=True, server_side_type=\"low\")\n", + "email_server_high, smtp_server_high = get_email_server(\n", + " reset=True, server_side_type=\"high\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "low_client = server_low.login(email=\"info@openmined.org\", password=\"changethis\")\n", + "high_client = server_high.login(email=\"info@openmined.org\", password=\"changethis\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(high_client.worker_pools.get_all()) == 1\n", + "assert len(low_client.worker_pools.get_all()) == 1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Setup High First\n", + "\n", + "- If using an external registery, we want to get this from the test_settings.\n", + "- We build the docker image over the base docker image in Syft\n", + "- We give a tag called worker-bigquery to our custom pool image" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "external_registry = test_settings.get(\"external_registry\", default=\"docker.io\")\n", + "\n", + "base_worker_image = high_client.images.get_all()[0]\n", + "\n", + "worker_dockerfile = f\"\"\"\n", + "FROM {str(base_worker_image.image_identifier)}\n", + "\n", + "RUN uv pip install db-dtypes google-cloud-bigquery \n", + "\n", + "\"\"\".strip()\n", + "\n", + "docker_tag = str(base_worker_image.image_identifier).replace(\n", + " \"backend\", \"worker-bigquery\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "worker_pool_name = \"bigquery-pool\"\n", + "custom_pool_pod_annotations = {\"bigquery-custom-pool\": \"Pod annotation for bigquery\"}\n", + "custom_pool_pod_labels = {\"bigquery-custom-pool\": \"Pod_label_for_bigquery\"}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "build_and_launch_worker_pool_from_docker_str(\n", + " environment=environment,\n", + " client=high_client,\n", + " worker_pool_name=worker_pool_name,\n", + " custom_pool_pod_annotations=custom_pool_pod_annotations,\n", + " custom_pool_pod_labels=custom_pool_pod_labels,\n", + " worker_dockerfile=worker_dockerfile,\n", + " external_registry=external_registry,\n", + " docker_tag=docker_tag,\n", + " scale_to=2,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(high_client.worker_pools.get_all()) == 2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "high_client.settings.allow_guest_signup(enable=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Setup Low" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "launch_result = launch_worker_pool_from_docker_tag_and_registry(\n", + " environment=environment,\n", + " client=low_client,\n", + " worker_pool_name=worker_pool_name,\n", + " custom_pool_pod_annotations=custom_pool_pod_annotations,\n", + " custom_pool_pod_labels=custom_pool_pod_labels,\n", + " docker_tag=docker_tag,\n", + " external_registry=external_registry,\n", + " scale_to=1,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(low_client.worker_pools.get_all()) == 2" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Register a DS only on the low side" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "low_client.register(\n", + " email=\"data_scientist@openmined.org\",\n", + " password=\"verysecurepassword\",\n", + " password_verify=\"verysecurepassword\",\n", + " name=\"John Doe\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "low_client.settings.allow_guest_signup(enable=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert (\n", + " len(low_client.api.services.user.get_all()) == 2\n", + "), \"Only DS and Admin should be at low side\"\n", + "assert (\n", + " len(high_client.api.services.user.get_all()) == 1\n", + "), \"Only Admin should be at high side\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Close" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if environment != \"remote\":\n", + " server_high.land()\n", + " server_low.land()\n", + " smtp_server_low.stop()\n", + " smtp_server_high.stop()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.4" + } + }, + "nbformat": 4, + "nbformat_minor": 4 } diff --git a/notebooks/scenarios/bigquery/sync/010-setup-high-low-datasites.ipynb b/notebooks/scenarios/bigquery/sync/010-setup-high-low-datasites.ipynb new file mode 100644 index 00000000000..1c6f3baec58 --- /dev/null +++ b/notebooks/scenarios/bigquery/sync/010-setup-high-low-datasites.ipynb @@ -0,0 +1,331 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# stdlib\n", + "import os\n", + "\n", + "# Testing works over 4 possibilities\n", + "# 1. (python/in-memory workers and using tox commands)\n", + "# 2. (python/in-memory workers and manually running notebooks)\n", + "# 3. (using k8s and using tox commands)\n", + "# 4. (using k8s and manually running notebooks)\n", + "# Uncomment the lines below if in the 4th possibility\n", + "\n", + "# os.environ[\"ORCHESTRA_DEPLOYMENT_TYPE\"] = \"remote\"\n", + "# os.environ[\"DEV_MODE\"] = \"True\"\n", + "# os.environ[\"TEST_EXTERNAL_REGISTRY\"] = \"k3d-registry.localhost:5800\"\n", + "# os.environ[\"CLUSTER_HTTP_PORT_HIGH\"] = \"9081\"\n", + "# os.environ[\"CLUSTER_HTTP_PORT_LOW\"] = \"9083\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# stdlib\n", + "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", + "high_port = os.environ.get(\"CLUSTER_HTTP_PORT_HIGH\", \"9081\")\n", + "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"9083\")\n", + "print(environment, high_port, low_port)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# syft absolute\n", + "import syft as sy\n", + "from syft import test_settings\n", + "from syft.util.test_helpers.worker_helpers import (\n", + " build_and_launch_worker_pool_from_docker_str,\n", + ")\n", + "from syft.util.test_helpers.worker_helpers import (\n", + " launch_worker_pool_from_docker_tag_and_registry,\n", + ")\n", + "from syft.util.test_helpers.email_helpers import get_email_server\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Launch server & login" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "server_low = sy.orchestra.launch(\n", + " name=\"bigquery-low\",\n", + " server_side_type=\"low\",\n", + " dev_mode=True,\n", + " reset=True,\n", + " n_consumers=1,\n", + " create_producer=True,\n", + " port=low_port,\n", + ")\n", + "\n", + "server_high = sy.orchestra.launch(\n", + " name=\"bigquery-high\",\n", + " server_side_type=\"high\",\n", + " dev_mode=True,\n", + " reset=True,\n", + " n_consumers=1,\n", + " create_producer=True,\n", + " port=high_port,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Email Server" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "email_server_low, smtp_server_low = get_email_server(reset=True, server_side_type=\"low\")\n", + "email_server_high, smtp_server_high = get_email_server(\n", + " reset=True, server_side_type=\"high\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "low_client = server_low.login(email=\"info@openmined.org\", password=\"changethis\")\n", + "high_client = server_high.login(email=\"info@openmined.org\", password=\"changethis\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(high_client.worker_pools.get_all()) == 1\n", + "assert len(low_client.worker_pools.get_all()) == 1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Setup High First\n", + "\n", + "- If using an external registery, we want to get this from the test_settings.\n", + "- We build the docker image over the base docker image in Syft\n", + "- We give a tag called worker-bigquery to our custom pool image" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "external_registry = test_settings.get(\"external_registry\", default=\"docker.io\")\n", + "\n", + "base_worker_image = high_client.images.get_all()[0]\n", + "\n", + "worker_dockerfile = f\"\"\"\n", + "FROM {str(base_worker_image.image_identifier)}\n", + "\n", + "RUN uv pip install db-dtypes google-cloud-bigquery \n", + "\n", + "\"\"\".strip()\n", + "\n", + "docker_tag = str(base_worker_image.image_identifier).replace(\n", + " \"backend\", \"worker-bigquery\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "worker_pool_name = \"bigquery-pool\"\n", + "custom_pool_pod_annotations = {\"bigquery-custom-pool\": \"Pod annotation for bigquery\"}\n", + "custom_pool_pod_labels = {\"bigquery-custom-pool\": \"Pod_label_for_bigquery\"}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "build_and_launch_worker_pool_from_docker_str(\n", + " environment=environment,\n", + " client=high_client,\n", + " worker_pool_name=worker_pool_name,\n", + " custom_pool_pod_annotations=custom_pool_pod_annotations,\n", + " custom_pool_pod_labels=custom_pool_pod_labels,\n", + " worker_dockerfile=worker_dockerfile,\n", + " external_registry=external_registry,\n", + " docker_tag=docker_tag,\n", + " scale_to=2,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(high_client.worker_pools.get_all()) == 2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "high_client.settings.allow_guest_signup(enable=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Setup Low" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "launch_result = launch_worker_pool_from_docker_tag_and_registry(\n", + " environment=environment,\n", + " client=low_client,\n", + " worker_pool_name=worker_pool_name,\n", + " custom_pool_pod_annotations=custom_pool_pod_annotations,\n", + " custom_pool_pod_labels=custom_pool_pod_labels,\n", + " docker_tag=docker_tag,\n", + " external_registry=external_registry,\n", + " scale_to=1,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(low_client.worker_pools.get_all()) == 2" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Register a DS only on the low side" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "low_client.register(\n", + " email=\"data_scientist@openmined.org\",\n", + " password=\"verysecurepassword\",\n", + " password_verify=\"verysecurepassword\",\n", + " name=\"John Doe\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "low_client.settings.allow_guest_signup(enable=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert (\n", + " len(low_client.api.services.user.get_all()) == 2\n", + "), \"Only DS and Admin should be at low side\"\n", + "assert (\n", + " len(high_client.api.services.user.get_all()) == 1\n", + "), \"Only Admin should be at high side\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Close" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if environment != \"remote\":\n", + " server_high.land()\n", + " server_low.land()\n", + " smtp_server_low.stop()\n", + " smtp_server_high.stop()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.4" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb b/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb new file mode 100644 index 00000000000..a36ed1257f3 --- /dev/null +++ b/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb @@ -0,0 +1,669 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "0", + "metadata": {}, + "outputs": [], + "source": [ + "# import os\n", + "\n", + "# os.environ[\"ORCHESTRA_DEPLOYMENT_TYPE\"] = \"remote\"\n", + "# os.environ[\"DEV_MODE\"] = \"True\"\n", + "# os.environ[\"TEST_EXTERNAL_REGISTRY\"] = \"k3d-registry.localhost:5800\"\n", + "# os.environ[\"CLUSTER_HTTP_PORT_HIGH\"] = \"9081\"\n", + "# os.environ[\"CLUSTER_HTTP_PORT_LOW\"] = \"9083\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "# isort: off\n", + "# stdlib\n", + "import os\n", + "\n", + "# syft absolute\n", + "import syft as sy\n", + "from syft import test_helpers # noqa: F401\n", + "\n", + "# third party\n", + "from email_helpers import SENDER\n", + "from email_helpers import create_user\n", + "from email_helpers import get_email_server\n", + "from email_helpers import make_user\n", + "from email_helpers import save_users\n", + "# isort: on" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", + "high_port = os.environ.get(\"CLUSTER_HTTP_PORT_HIGH\", \"auto\")\n", + "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"auto\")\n", + "print(environment, high_port, low_port)" + ] + }, + { + "cell_type": "markdown", + "id": "3", + "metadata": {}, + "source": [ + "# Launch server & login" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "server_low = sy.orchestra.launch(\n", + " name=\"bigquery-low\",\n", + " server_side_type=\"low\",\n", + " dev_mode=True,\n", + " n_consumers=1,\n", + " create_producer=True,\n", + " port=low_port,\n", + ")\n", + "\n", + "server_high = sy.orchestra.launch(\n", + " name=\"bigquery-high\",\n", + " server_side_type=\"high\",\n", + " dev_mode=True,\n", + " n_consumers=1,\n", + " create_producer=True,\n", + " port=high_port,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "5", + "metadata": {}, + "source": [ + "### Email Server" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "email_server_low, smtp_server_low = get_email_server(server_side_type=\"low\")\n", + "email_server_high, smtp_server_high = get_email_server(server_side_type=\"high\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "low_client = server_low.login(email=\"info@openmined.org\", password=\"changethis\")\n", + "high_client = server_high.login(email=\"info@openmined.org\", password=\"changethis\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "low_client.users" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": {}, + "outputs": [], + "source": [ + "low_client.api.services.settings.enable_notifications(\n", + " email_sender=SENDER,\n", + " email_server=\"localhost\",\n", + " email_port=\"9025\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "high_client.api.services.settings.enable_notifications(\n", + " email_sender=SENDER,\n", + " email_server=\"localhost\",\n", + " email_port=\"9026\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "11", + "metadata": {}, + "source": [ + "# Register users" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12", + "metadata": {}, + "outputs": [], + "source": [ + "num_users = int(os.environ.get(\"NUM_TEST_USERS\", 5))\n", + "print(f\"registering {num_users} users\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "13", + "metadata": {}, + "outputs": [], + "source": [ + "users = []\n", + "email_disable_index = 0\n", + "reset_password_index = 1\n", + "for i in range(num_users):\n", + " user = make_user()\n", + " user._email_server = email_server_low\n", + " create_user(low_client, user)\n", + " user.client = low_client\n", + " if email_disable_index == i:\n", + " user.email_disabled = True\n", + " if reset_password_index == i:\n", + " user.reset_password = True\n", + " users.append(user)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "14", + "metadata": {}, + "outputs": [], + "source": [ + "save_users(users)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15", + "metadata": {}, + "outputs": [], + "source": [ + "# stdlib\n", + "import asyncio\n", + "\n", + "await asyncio.sleep(5)" + ] + }, + { + "cell_type": "markdown", + "id": "16", + "metadata": {}, + "source": [ + "## Verify Emails are sent" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "17", + "metadata": {}, + "outputs": [], + "source": [ + "# everyone gets a welcome email\n", + "server_name = low_client.name\n", + "for user in users:\n", + " emails = user.emails\n", + " assert len(emails) == 1\n", + " welcome_email = user.emails[0]\n", + " assert welcome_email.email_from == SENDER\n", + " assert len(welcome_email.email_to) == 1\n", + " assert welcome_email.email_to[0] == user.email\n", + " assert f\"Welcome to {server_name}\" in welcome_email.email_content" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "18", + "metadata": {}, + "outputs": [], + "source": [ + "# one user disables notifications\n", + "# one user requests a password reset\n", + "no_email_user = None\n", + "reset_password_user = None\n", + "for user in users:\n", + " user.client = low_client # get user client\n", + " if user.email_disabled:\n", + " no_email_user = user\n", + " # disable for this user only\n", + " user.client.api.notifications.deactivate()\n", + "\n", + " if user.reset_password:\n", + " # ask admin for forgot password flow\n", + " user.client.guest().forgot_password(email=user.email)\n", + " assert \"Password Reset Requested\" in user.emails[1].email_content\n", + " reset_password_user = user" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "19", + "metadata": {}, + "outputs": [], + "source": [ + "ds0 = users[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "20", + "metadata": {}, + "outputs": [], + "source": [ + "ds0_user = ds0.client.account\n", + "ds0_user" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "21", + "metadata": {}, + "outputs": [], + "source": [ + "# with sy.raises(\n", + "# sy.SyftException(public_message=\"*tried to update user*\"\n", + "# ), show=True): this is different on k8s no idea why\n", + "with sy.raises(sy.SyftException, show=True):\n", + " ds0.client.users.update(uid=ds0_user.id, role=\"admin\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "22", + "metadata": {}, + "outputs": [], + "source": [ + "# with sy.raises(sy.SyftException(public_message=\"*tried to update user*\"), show=True):\n", + "with sy.raises(sy.SyftException, show=True):\n", + " ds0_user.update(role=\"admin\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "23", + "metadata": {}, + "outputs": [], + "source": [ + "# TODO: test disabling and re-enabling all notifications" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24", + "metadata": {}, + "outputs": [], + "source": [ + "# low_client.api.services.settings.disable_notifications()\n", + "# low_client.api.services.settings.enable_notifications()" + ] + }, + { + "cell_type": "markdown", + "id": "25", + "metadata": {}, + "source": [ + "## Test reset password" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "26", + "metadata": {}, + "outputs": [], + "source": [ + "# This is necessary as it sets the new token value in user.reset_token\n", + "token = reset_password_user.get_token()\n", + "token" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "27", + "metadata": {}, + "outputs": [], + "source": [ + "# This is necessary as it sets the new password value in user.new_password\n", + "passw = reset_password_user.make_new_password()\n", + "passw" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "28", + "metadata": {}, + "outputs": [], + "source": [ + "assert token\n", + "assert passw" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "29", + "metadata": {}, + "outputs": [], + "source": [ + "output = reset_password_user.client.guest().reset_password(\n", + " token=reset_password_user.reset_token, new_password=reset_password_user.new_password\n", + ")\n", + "output" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "30", + "metadata": {}, + "outputs": [], + "source": [ + "assert isinstance(output, sy.SyftSuccess)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "31", + "metadata": {}, + "outputs": [], + "source": [ + "# relogin\n", + "reset_password_user.relogin()\n", + "# reset_password_user.client = reset_password_user.client" + ] + }, + { + "cell_type": "markdown", + "id": "32", + "metadata": {}, + "source": [ + "## Reset password second time" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "33", + "metadata": {}, + "outputs": [], + "source": [ + "reset_password_user.client.guest().forgot_password(email=reset_password_user.email)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "34", + "metadata": {}, + "outputs": [], + "source": [ + "output = reset_password_user.client.guest().reset_password(\n", + " token=reset_password_user.get_token(),\n", + " new_password=reset_password_user.make_new_password(),\n", + ")\n", + "output" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "35", + "metadata": {}, + "outputs": [], + "source": [ + "assert isinstance(output, sy.SyftSuccess)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "36", + "metadata": {}, + "outputs": [], + "source": [ + "# print(f\"token:\\t\\t {reset_password_user.reset_token}\\n\\\n", + "# password:\\t {reset_password_user.password}\\n\\\n", + "# new password:\\t {reset_password_user.new_password}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "37", + "metadata": {}, + "outputs": [], + "source": [ + "reset_password_user.update_password()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "38", + "metadata": {}, + "outputs": [], + "source": [ + "# print(f\"token:\\t\\t {reset_password_user.reset_token}\\n\\\n", + "# password:\\t {reset_password_user.password}\\n\\\n", + "# new password:\\t {reset_password_user.new_password}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "39", + "metadata": {}, + "outputs": [], + "source": [ + "# relogin\n", + "reset_password_user.relogin()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "40", + "metadata": {}, + "outputs": [], + "source": [ + "save_users(users)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "41", + "metadata": {}, + "outputs": [], + "source": [ + "reset_password_user" + ] + }, + { + "cell_type": "markdown", + "id": "42", + "metadata": {}, + "source": [ + "## Reduce token expiration and try resetting" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "43", + "metadata": {}, + "outputs": [], + "source": [ + "# Variable is poorly named, token expiration time is in seconds and not minutes\n", + "low_client.api.services.settings.update(pwd_token_config={\"token_exp_min\": 3})\n", + "low_client.refresh()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "44", + "metadata": {}, + "outputs": [], + "source": [ + "reset_password_user.client.guest().forgot_password(email=reset_password_user.email)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "45", + "metadata": {}, + "outputs": [], + "source": [ + "# Wait 3 seconds to ensure token expires\n", + "await asyncio.sleep(3)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "46", + "metadata": {}, + "outputs": [], + "source": [ + "# This should throw a SyftError because we waited too long\n", + "output = reset_password_user.client.guest().reset_password(\n", + " token=reset_password_user.get_token(),\n", + " new_password=reset_password_user.make_new_password(),\n", + ")\n", + "output" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "47", + "metadata": {}, + "outputs": [], + "source": [ + "assert isinstance(output, sy.SyftError)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "48", + "metadata": {}, + "outputs": [], + "source": [ + "# relogin\n", + "with sy.raises(sy.SyftException, show=True):\n", + " reset_password_user.relogin()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "49", + "metadata": {}, + "outputs": [], + "source": [ + "# Set things back to the the default settings\n", + "low_client.api.services.settings.update(pwd_token_config={\"token_exp_min\": 1800})\n", + "low_client.refresh()" + ] + }, + { + "cell_type": "markdown", + "id": "50", + "metadata": {}, + "source": [ + "# Cleanup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "51", + "metadata": {}, + "outputs": [], + "source": [ + "if environment != \"remote\":\n", + " server_high.land()\n", + " server_low.land()\n", + " smtp_server_low.stop()\n", + " smtp_server_high.stop()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "52", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb b/notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb index 2f9658634e3..4da648ca984 100644 --- a/notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb +++ b/notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb @@ -1,622 +1,694 @@ { - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# stdlib\n", - "import os\n", - "\n", - "# os.environ[\"ORCHESTRA_DEPLOYMENT_TYPE\"] = \"remote\"\n", - "# os.environ[\"DEV_MODE\"] = \"True\"\n", - "# os.environ[\"TEST_EXTERNAL_REGISTRY\"] = \"k3d-registry.localhost:5800\"\n", - "# os.environ[\"CLUSTER_HTTP_PORT_HIGH\"] = \"9081\"\n", - "# os.environ[\"CLUSTER_HTTP_PORT_LOW\"] = \"9083\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# stdlib\n", - "\n", - "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", - "high_port = os.environ.get(\"CLUSTER_HTTP_PORT_HIGH\", \"9081\")\n", - "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"9083\")\n", - "print(environment, high_port, low_port)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# stdlib\n", - "\n", - "# set to use the live APIs\n", - "# import os\n", - "# os.environ[\"TEST_BIGQUERY_APIS_LIVE\"] = \"True\"\n", - "# third party\n", - "import pandas as pd\n", - "\n", - "# syft absolute\n", - "import syft as sy\n", - "from syft import test_settings\n", - "from syft.client.syncing import compare_clients\n", - "from syft.util.test_helpers.apis import make_schema\n", - "from syft.util.test_helpers.apis import make_submit_query\n", - "from syft.util.test_helpers.apis import make_test_query" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Launch server and login" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "server_low = sy.orchestra.launch(\n", - " name=\"bigquery-low\",\n", - " server_side_type=\"low\",\n", - " dev_mode=True,\n", - " n_consumers=1,\n", - " create_producer=True,\n", - " port=low_port,\n", - ")\n", - "\n", - "server_high = sy.orchestra.launch(\n", - " name=\"bigquery-high\",\n", - " server_side_type=\"high\",\n", - " dev_mode=True,\n", - " n_consumers=1,\n", - " create_producer=True,\n", - " port=high_port,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "low_client = server_low.login(email=\"info@openmined.org\", password=\"changethis\")\n", - "high_client = server_high.login(email=\"info@openmined.org\", password=\"changethis\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert len(high_client.worker_pools.get_all()) == 2\n", - "assert len(low_client.worker_pools.get_all()) == 2" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "this_worker_pool_name = \"bigquery-pool\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Load database information from test_settings" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "dataset_1 = test_settings.get(\"dataset_1\", default=\"dataset_1\")\n", - "dataset_2 = test_settings.get(\"dataset_2\", default=\"dataset_2\")\n", - "table_1 = test_settings.get(\"table_1\", default=\"table_1\")\n", - "table_2 = test_settings.get(\"table_2\", default=\"table_2\")\n", - "table_2_col_id = test_settings.get(\"table_2_col_id\", default=\"table_id\")\n", - "table_2_col_score = test_settings.get(\"table_2_col_score\", default=\"colname\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Create and test different endpoints" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "----" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Create `biquery.schema` endpoint" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "schema_function = make_schema(\n", - " settings={\n", - " \"calls_per_min\": 5,\n", - " },\n", - " worker_pool=this_worker_pool_name,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "high_client.custom_api.add(endpoint=schema_function)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "result = high_client.api.services.bigquery.schema()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert len(result) == 23" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "TODO: Note that when we do not create a job, the type of result is `syft.service.action.pandas.PandasDataFrameObject` and not pandas but the `.get()` method will get you the expected answer" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# syft absolute\n", - "from syft.service.action.pandas import PandasDataFrameObject\n", - "\n", - "# assert isinstance(result, pd.DataFrame)\n", - "assert isinstance(result, PandasDataFrameObject)\n", - "assert isinstance(result.get(), pd.DataFrame)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "____" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Create `biquery.test_query` endpoint" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "mock_func = make_test_query(\n", - " settings={\n", - " \"rate_limiter_enabled\": True,\n", - " \"calls_per_min\": 10,\n", - " }\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "private_func = make_test_query(\n", - " settings={\n", - " \"rate_limiter_enabled\": False,\n", - " }\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "new_endpoint = sy.TwinAPIEndpoint(\n", - " path=\"bigquery.test_query\",\n", - " description=\"This endpoint allows to query Bigquery storage via SQL queries.\",\n", - " private_function=private_func,\n", - " mock_function=mock_func,\n", - " worker_pool=this_worker_pool_name,\n", - ")\n", - "\n", - "high_client.custom_api.add(endpoint=new_endpoint)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Some features for updating endpoint" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Here, we update the endpoint to timeout after 100s (rather the default of 60s)\n", - "high_client.api.services.api.update(\n", - " endpoint_path=\"bigquery.test_query\", endpoint_timeout=120\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "high_client.api.services.api.update(\n", - " endpoint_path=\"bigquery.test_query\", hide_mock_definition=True\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Test the `bigquery.test_query` endpoint" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Test mock version\n", - "result = high_client.api.services.bigquery.test_query.mock(\n", - " sql_query=f\"SELECT * FROM {dataset_1}.{table_1} LIMIT 10\"\n", - ")\n", - "result" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert len(result) == 10" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Test mock version for wrong queries\n", - "with sy.raises(\n", - " sy.SyftException(public_message=\"*must be qualified with a dataset*\"), show=True\n", - "):\n", - " high_client.api.services.bigquery.test_query.mock(\n", - " sql_query=\"SELECT * FROM invalid_table LIMIT 1\"\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Test private version\n", - "result = high_client.api.services.bigquery.test_query.private(\n", - " sql_query=f\"SELECT * FROM {dataset_1}.{table_1} LIMIT 12\"\n", - ")\n", - "result" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert len(result) == 12" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "____" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Create `submit_query` endpoint" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "submit_query_function = make_submit_query(\n", - " settings={}, worker_pool=this_worker_pool_name\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "high_client.custom_api.add(endpoint=submit_query_function)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "high_client.api.services.api.update(\n", - " endpoint_path=\"bigquery.submit_query\", hide_mock_definition=True\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Testing submit query\n", - "result = high_client.api.services.bigquery.submit_query(\n", - " func_name=\"my_func\",\n", - " query=f\"SELECT * FROM {dataset_1}.{table_1} LIMIT 2\",\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert \"Query submitted\" in result\n", - "result" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "job = high_client.code.my_func(blocking=False)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "res = job.wait().get()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert isinstance(res, pd.DataFrame)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Test endpoints" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "high_client.custom_api.api_endpoints()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert len(high_client.custom_api.api_endpoints()) == 3" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert (\n", - " high_client.api.services.bigquery.test_query\n", - " and high_client.api.services.bigquery.submit_query\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Syncing" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "diff = compare_clients(\n", - " from_client=high_client, to_client=low_client, hide_usercode=False\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# TODO verify that jobs are actually filtered out\n", - "# TODO we need to think about whether its possible for the admin to create more data here that would break sync" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "widget = diff.resolve()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# TODO maybe see if non-internal method we can use or make it public\n", - "widget._share_all()\n", - "widget._sync_all()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert len(low_client.jobs.get_all()) == 0" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert len(low_client.custom_api.api_endpoints()) == 3" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert len(high_client.custom_api.api_endpoints()) == 3" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if environment != \"remote\":\n", - " server_high.land()\n", - " server_low.land()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.4" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# stdlib\n", + "import os\n", + "\n", + "# os.environ[\"ORCHESTRA_DEPLOYMENT_TYPE\"] = \"remote\"\n", + "# os.environ[\"DEV_MODE\"] = \"True\"\n", + "# os.environ[\"TEST_EXTERNAL_REGISTRY\"] = \"k3d-registry.localhost:5800\"\n", + "# os.environ[\"CLUSTER_HTTP_PORT_HIGH\"] = \"9081\"\n", + "# os.environ[\"CLUSTER_HTTP_PORT_LOW\"] = \"9083\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# stdlib\n", + "\n", + "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", + "high_port = os.environ.get(\"CLUSTER_HTTP_PORT_HIGH\", \"9081\")\n", + "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"9083\")\n", + "print(environment, high_port, low_port)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# stdlib\n", + "\n", + "# set to use the live APIs\n", + "# import os\n", + "# os.environ[\"TEST_BIGQUERY_APIS_LIVE\"] = \"True\"\n", + "# third party\n", + "import pandas as pd\n", + "\n", + "# syft absolute\n", + "import syft as sy\n", + "from syft import test_settings\n", + "from syft.client.syncing import compare_clients\n", + "from syft.util.test_helpers.apis import make_schema\n", + "from syft.util.test_helpers.apis import make_submit_query\n", + "from syft.util.test_helpers.apis import make_test_query\n", + "from syft.util.test_helpers.email_helpers import get_email_server\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Launch server and login" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "server_low = sy.orchestra.launch(\n", + " name=\"bigquery-low\",\n", + " server_side_type=\"low\",\n", + " dev_mode=True,\n", + " n_consumers=1,\n", + " create_producer=True,\n", + " port=low_port,\n", + ")\n", + "\n", + "server_high = sy.orchestra.launch(\n", + " name=\"bigquery-high\",\n", + " server_side_type=\"high\",\n", + " dev_mode=True,\n", + " n_consumers=1,\n", + " create_producer=True,\n", + " port=high_port,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Email Server" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "email_server_low, smtp_server_low = get_email_server(server_side_type=\"low\")\n", + "email_server_high, smtp_server_high = get_email_server(server_side_type=\"high\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "low_client = server_low.login(email=\"info@openmined.org\", password=\"changethis\")\n", + "high_client = server_high.login(email=\"info@openmined.org\", password=\"changethis\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(high_client.worker_pools.get_all()) == 2\n", + "assert len(low_client.worker_pools.get_all()) == 2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "this_worker_pool_name = \"bigquery-pool\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Load database information from test_settings" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dataset_1 = test_settings.get(\"dataset_1\", default=\"dataset_1\")\n", + "dataset_2 = test_settings.get(\"dataset_2\", default=\"dataset_2\")\n", + "table_1 = test_settings.get(\"table_1\", default=\"table_1\")\n", + "table_2 = test_settings.get(\"table_2\", default=\"table_2\")\n", + "table_2_col_id = test_settings.get(\"table_2_col_id\", default=\"table_id\")\n", + "table_2_col_score = test_settings.get(\"table_2_col_score\", default=\"colname\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Create and test different endpoints" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "----" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create `biquery.schema` endpoint" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "schema_function = make_schema(\n", + " settings={\n", + " \"calls_per_min\": 5,\n", + " },\n", + " worker_pool=this_worker_pool_name,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "high_client.custom_api.add(endpoint=schema_function)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "result = high_client.api.services.bigquery.schema()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(result) == 23" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "TODO: Note that when we do not create a job, the type of result is `syft.service.action.pandas.PandasDataFrameObject` and not pandas but the `.get()` method will get you the expected answer" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# syft absolute\n", + "from syft.service.action.pandas import PandasDataFrameObject\n", + "\n", + "# assert isinstance(result, pd.DataFrame)\n", + "assert isinstance(result, PandasDataFrameObject)\n", + "assert isinstance(result.get(), pd.DataFrame)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "____" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create `biquery.test_query` endpoint" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "mock_func = make_test_query(\n", + " settings={\n", + " \"rate_limiter_enabled\": True,\n", + " \"calls_per_min\": 10,\n", + " }\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "private_func = make_test_query(\n", + " settings={\n", + " \"rate_limiter_enabled\": False,\n", + " }\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "new_endpoint = sy.TwinAPIEndpoint(\n", + " path=\"bigquery.test_query\",\n", + " description=\"This endpoint allows to query Bigquery storage via SQL queries.\",\n", + " private_function=private_func,\n", + " mock_function=mock_func,\n", + " worker_pool=this_worker_pool_name,\n", + ")\n", + "\n", + "high_client.custom_api.add(endpoint=new_endpoint)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Some features for updating endpoint" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Here, we update the endpoint to timeout after 100s (rather the default of 60s)\n", + "high_client.api.services.api.update(\n", + " endpoint_path=\"bigquery.test_query\", endpoint_timeout=120\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "high_client.api.services.api.update(\n", + " endpoint_path=\"bigquery.test_query\", hide_mock_definition=True\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Test the `bigquery.test_query` endpoint" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Test mock version\n", + "result = high_client.api.services.bigquery.test_query.mock(\n", + " sql_query=f\"SELECT * FROM {dataset_1}.{table_1} LIMIT 10\"\n", + ")\n", + "result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(result) == 10" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Test mock version for wrong queries\n", + "with sy.raises(\n", + " sy.SyftException(public_message=\"*must be qualified with a dataset*\"), show=True\n", + "):\n", + " high_client.api.services.bigquery.test_query.mock(\n", + " sql_query=\"SELECT * FROM invalid_table LIMIT 1\"\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Test private version\n", + "result = high_client.api.services.bigquery.test_query.private(\n", + " sql_query=f\"SELECT * FROM {dataset_1}.{table_1} LIMIT 12\"\n", + ")\n", + "result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(result) == 12" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "____" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create `submit_query` endpoint" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "submit_query_function = make_submit_query(\n", + " settings={}, worker_pool=this_worker_pool_name\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "high_client.custom_api.add(endpoint=submit_query_function)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "high_client.api.services.api.update(\n", + " endpoint_path=\"bigquery.submit_query\", hide_mock_definition=True\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Testing submit query\n", + "result = high_client.api.services.bigquery.submit_query(\n", + " func_name=\"my_func\",\n", + " query=f\"SELECT * FROM {dataset_1}.{table_1} LIMIT 2\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert \"Query submitted\" in result\n", + "result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "job = high_client.code.my_func(blocking=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "res = job.wait().get()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert isinstance(res, pd.DataFrame)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Test endpoints" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "high_client.custom_api.api_endpoints()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(high_client.custom_api.api_endpoints()) == 3" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert (\n", + " high_client.api.services.bigquery.test_query\n", + " and high_client.api.services.bigquery.submit_query\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Syncing" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "diff = compare_clients(\n", + " from_client=high_client, to_client=low_client, hide_usercode=False\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# TODO verify that jobs are actually filtered out\n", + "# TODO we need to think about whether its possible for the admin to create more data here that would break sync" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "widget = diff.resolve()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# TODO maybe see if non-internal method we can use or make it public\n", + "widget._share_all()\n", + "widget._sync_all()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(low_client.jobs.get_all()) == 0" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(low_client.custom_api.api_endpoints()) == 3" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(high_client.custom_api.api_endpoints()) == 3" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Test emails" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# email_server_low.get_emails_for_user(user_email=\"info@openmined.org\")\n", + "assert len(email_server_high.get_emails_for_user(user_email=\"info@openmined.org\")) == 2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert (\n", + " \"Job Failed\"\n", + " in email_server_high.get_emails_for_user(user_email=\"info@openmined.org\")[\n", + " 0\n", + " ].email_content\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert (\n", + " \"A new request has been submitted and requires your attention\"\n", + " in email_server_high.get_emails_for_user(user_email=\"info@openmined.org\")[\n", + " 1\n", + " ].email_content\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Clean up" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if environment != \"remote\":\n", + " server_high.land()\n", + " server_low.land()\n", + " smtp_server_low.stop()\n", + " smtp_server_high.stop()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.4" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/notebooks/scenarios/bigquery/sync/030-ds-submit-request.ipynb b/notebooks/scenarios/bigquery/sync/030-ds-submit-request.ipynb index 21e8437f283..3a223714e26 100644 --- a/notebooks/scenarios/bigquery/sync/030-ds-submit-request.ipynb +++ b/notebooks/scenarios/bigquery/sync/030-ds-submit-request.ipynb @@ -34,12 +34,16 @@ "metadata": {}, "outputs": [], "source": [ - "# third party\n", - "import pandas as pd\n", - "\n", + "# isort: off\n", "# syft absolute\n", "import syft as sy\n", - "from syft import test_settings" + "from syft import test_settings\n", + "from syft import test_helpers # noqa: F401\n", + "\n", + "# third party\n", + "import pandas as pd\n", + "from email_helpers import get_email_server\n", + "# isort: on" ] }, { @@ -65,6 +69,15 @@ ")" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "email_server_low, smtp_server_low = get_email_server(server_side_type=\"low\")" + ] + }, { "cell_type": "code", "execution_count": null, @@ -273,7 +286,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Shutdown" + "# Test emails" ] }, { @@ -282,7 +295,45 @@ "metadata": {}, "outputs": [], "source": [ - "# todo: add email server shutdown" + "assert len(email_server_low.get_emails_for_user(user_email=\"info@openmined.org\")) == 1\n", + "assert (\n", + " len(email_server_low.get_emails_for_user(user_email=ds_client.account.email)) == 1\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert (\n", + " \"A new request has been submitted and requires your attention\"\n", + " in email_server_low.get_emails_for_user(user_email=\"info@openmined.org\")[\n", + " 0\n", + " ].email_content\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert (\n", + " \"We regret to inform you that your function job\"\n", + " in email_server_low.get_emails_for_user(user_email=ds_client.account.email)[\n", + " 0\n", + " ].email_content\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Shutdown" ] }, { @@ -292,7 +343,8 @@ "outputs": [], "source": [ "if environment != \"remote\":\n", - " server_low.land()" + " server_low.land()\n", + " smtp_server_low.stop()" ] }, { diff --git a/notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb b/notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb index e9b43ae3eb1..2dae55e4abb 100644 --- a/notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb +++ b/notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb @@ -6,15 +6,19 @@ "metadata": {}, "outputs": [], "source": [ + "# isort: off\n", "# stdlib\n", "import os\n", "\n", - "# third party\n", - "import pandas as pd\n", - "\n", "# syft absolute\n", "import syft as sy\n", "from syft.client.syncing import compare_clients\n", + "from syft import test_helpers # noqa: F401\n", + "\n", + "# third party\n", + "import pandas as pd\n", + "from email_helpers import get_email_server\n", + "# isort: on\n", "\n", "# os.environ[\"ORCHESTRA_DEPLOYMENT_TYPE\"] = \"remote\"\n", "# os.environ[\"CLUSTER_HTTP_PORT_HIGH\"] = \"9081\"\n", @@ -64,6 +68,16 @@ ")" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "email_server_low, smtp_server_low = get_email_server(server_side_type=\"low\")\n", + "email_server_high, smtp_server_high = get_email_server(server_side_type=\"high\")" + ] + }, { "cell_type": "code", "execution_count": null, @@ -266,6 +280,42 @@ "assert \"approved\" in str(user_request.status).lower()" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Test email" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Should this user be notified about approval? Right now it seems that it is not.\n", + "assert (\n", + " len(email_server_low.get_emails_for_user(user_email=\"data_scientist@openmined.org\"))\n", + " == 1\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Similarly, after syncing, should the admin be notified about anything?" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Shutdown" + ] + }, { "cell_type": "code", "execution_count": null, @@ -274,7 +324,9 @@ "source": [ "if environment != \"remote\":\n", " server_high.land()\n", - " server_low.land()" + " server_low.land()\n", + " smtp_server_low.stop()\n", + " smtp_server_high.stop()" ] }, { diff --git a/notebooks/scenarios/bigquery/sync/050-ds-get-results.ipynb b/notebooks/scenarios/bigquery/sync/050-ds-get-results.ipynb index 531f0c8f410..ebeb59a9608 100644 --- a/notebooks/scenarios/bigquery/sync/050-ds-get-results.ipynb +++ b/notebooks/scenarios/bigquery/sync/050-ds-get-results.ipynb @@ -6,14 +6,18 @@ "metadata": {}, "outputs": [], "source": [ + "# isort: off\n", "# stdlib\n", "import os\n", "\n", - "# third party\n", - "import pandas as pd\n", - "\n", "# syft absolute\n", "import syft as sy\n", + "from syft import test_helpers # noqa: F401\n", + "\n", + "# third party\n", + "import pandas as pd\n", + "from email_helpers import get_email_server\n", + "# isort: on\n", "\n", "# os.environ[\"ORCHESTRA_DEPLOYMENT_TYPE\"] = \"remote\"\n", "# os.environ[\"CLUSTER_HTTP_PORT_LOW\"] = \"9083\"" @@ -53,6 +57,15 @@ ")" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "email_server_low, smtp_server_low = get_email_server(server_side_type=\"low\")" + ] + }, { "cell_type": "code", "execution_count": null, @@ -114,7 +127,8 @@ "outputs": [], "source": [ "if environment != \"remote\":\n", - " server_low.land()" + " server_low.land()\n", + " smtp_server_low.stop()" ] }, { diff --git a/packages/syft/src/syft/util/test_helpers/email_helpers.py b/packages/syft/src/syft/util/test_helpers/email_helpers.py index ddfee82fef3..094c30645a3 100644 --- a/packages/syft/src/syft/util/test_helpers/email_helpers.py +++ b/packages/syft/src/syft/util/test_helpers/email_helpers.py @@ -218,9 +218,9 @@ def user_exists(root_client, email: str) -> bool: class SMTPTestServer: - def __init__(self, email_server): - self.port = 9025 - self.hostname = "0.0.0.0" # nosec: B104 + def __init__(self, email_server, port=9025): + self.port = port + self.hostname = "0.0.0.0" self._stop_event = asyncio.Event() # Simple email handler class @@ -251,7 +251,7 @@ async def handle_DATA(self, server, session, envelope): print(f"> Error initializing SMTPTestServer Controller: {e}") def start(self): - print(f"> Starting SMTPTestServer on: {self.hostname}:{self.port}") + # print(f"> Starting SMTPTestServer on: {self.hostname}:{self.port}") asyncio.create_task(self.async_loop()) async def async_loop(self): @@ -314,11 +314,21 @@ def run_with_timeout(self, condition_func, *args, **kwargs): return result -def get_email_server(reset=False): - email_server = EmailServer() +def get_email_server(reset=False, server_side_type="high"): + if server_side_type == "high": + email_file = "./emails_high.json" + port = 9026 + elif server_side_type == "low": + email_file = "./emails_low.json" + port = 9025 + else: + email_file = "./emails.json" + port = 9025 + + email_server = EmailServer(filepath=email_file) if reset: email_server.reset_emails() - smtp_server = SMTPTestServer(email_server) + smtp_server = SMTPTestServer(email_server, port=port) smtp_server.start() return email_server, smtp_server From 54dd7a95c08d5189d52cdd3a462fc04c37daa315 Mon Sep 17 00:00:00 2001 From: Sameer Wagh Date: Wed, 18 Sep 2024 15:33:30 -0400 Subject: [PATCH 02/28] email server landing --- .../bigquery/sync/010-setup-high-low-datasites.ipynb | 2 +- .../scenarios/bigquery/sync/011-users-emails-passwords.ipynb | 4 ++-- notebooks/scenarios/bigquery/sync/030-ds-submit-request.ipynb | 2 +- .../scenarios/bigquery/sync/040-do-review-requests.ipynb | 4 ++-- notebooks/scenarios/bigquery/sync/050-ds-get-results.ipynb | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/notebooks/scenarios/bigquery/sync/010-setup-high-low-datasites.ipynb b/notebooks/scenarios/bigquery/sync/010-setup-high-low-datasites.ipynb index 1c6f3baec58..0990280fdc2 100644 --- a/notebooks/scenarios/bigquery/sync/010-setup-high-low-datasites.ipynb +++ b/notebooks/scenarios/bigquery/sync/010-setup-high-low-datasites.ipynb @@ -328,4 +328,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} +} \ No newline at end of file diff --git a/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb b/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb index a36ed1257f3..fc52c3ba391 100644 --- a/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb +++ b/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb @@ -632,8 +632,8 @@ "if environment != \"remote\":\n", " server_high.land()\n", " server_low.land()\n", - " smtp_server_low.stop()\n", - " smtp_server_high.stop()" + "smtp_server_low.stop()\n", + "smtp_server_high.stop()" ] }, { diff --git a/notebooks/scenarios/bigquery/sync/030-ds-submit-request.ipynb b/notebooks/scenarios/bigquery/sync/030-ds-submit-request.ipynb index 3a223714e26..8d64892206a 100644 --- a/notebooks/scenarios/bigquery/sync/030-ds-submit-request.ipynb +++ b/notebooks/scenarios/bigquery/sync/030-ds-submit-request.ipynb @@ -344,7 +344,7 @@ "source": [ "if environment != \"remote\":\n", " server_low.land()\n", - " smtp_server_low.stop()" + "smtp_server_low.stop()" ] }, { diff --git a/notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb b/notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb index 2dae55e4abb..7ffec4c22d1 100644 --- a/notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb +++ b/notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb @@ -325,8 +325,8 @@ "if environment != \"remote\":\n", " server_high.land()\n", " server_low.land()\n", - " smtp_server_low.stop()\n", - " smtp_server_high.stop()" + "smtp_server_low.stop()\n", + "smtp_server_high.stop()" ] }, { diff --git a/notebooks/scenarios/bigquery/sync/050-ds-get-results.ipynb b/notebooks/scenarios/bigquery/sync/050-ds-get-results.ipynb index ebeb59a9608..e8432254ebe 100644 --- a/notebooks/scenarios/bigquery/sync/050-ds-get-results.ipynb +++ b/notebooks/scenarios/bigquery/sync/050-ds-get-results.ipynb @@ -128,7 +128,7 @@ "source": [ "if environment != \"remote\":\n", " server_low.land()\n", - " smtp_server_low.stop()" + "smtp_server_low.stop()" ] }, { From 63ff65fe6e9267c98a203753871ea0daf0e3d2bd Mon Sep 17 00:00:00 2001 From: Brendan Schell Date: Thu, 19 Sep 2024 17:53:13 -0400 Subject: [PATCH 03/28] More changes to cherry-pick email branch in Co-authored-by: Sameer Wagh --- .../sync/000-setup-high-low-datasites.ipynb | 4 +- .../sync/001-scale-delete-worker-pools.ipynb | 37 +- .../sync/010-setup-high-low-datasites.ipynb | 331 ------------------ .../sync/020-configure-api-and-sync.ipynb | 6 +- .../syft/util/test_helpers/email_helpers.py | 2 +- 5 files changed, 22 insertions(+), 358 deletions(-) delete mode 100644 notebooks/scenarios/bigquery/sync/010-setup-high-low-datasites.ipynb diff --git a/notebooks/scenarios/bigquery/sync/000-setup-high-low-datasites.ipynb b/notebooks/scenarios/bigquery/sync/000-setup-high-low-datasites.ipynb index 1c6f3baec58..3fe3966ba5d 100644 --- a/notebooks/scenarios/bigquery/sync/000-setup-high-low-datasites.ipynb +++ b/notebooks/scenarios/bigquery/sync/000-setup-high-low-datasites.ipynb @@ -302,8 +302,8 @@ "if environment != \"remote\":\n", " server_high.land()\n", " server_low.land()\n", - " smtp_server_low.stop()\n", - " smtp_server_high.stop()" + "smtp_server_low.stop()\n", + "smtp_server_high.stop()" ] } ], diff --git a/notebooks/scenarios/bigquery/sync/001-scale-delete-worker-pools.ipynb b/notebooks/scenarios/bigquery/sync/001-scale-delete-worker-pools.ipynb index d2ca74cd56f..6ea1af97242 100644 --- a/notebooks/scenarios/bigquery/sync/001-scale-delete-worker-pools.ipynb +++ b/notebooks/scenarios/bigquery/sync/001-scale-delete-worker-pools.ipynb @@ -79,28 +79,31 @@ { "cell_type": "code", "execution_count": null, - "id": "5", + "id": "6", "metadata": {}, "outputs": [], "source": [ - "server_low = sy.orchestra.launch(\n", - " name=\"bigquery-low\",\n", - " server_side_type=\"low\",\n", - " dev_mode=True,\n", - " n_consumers=1,\n", - " create_producer=True,\n", - " port=low_port,\n", + "email_server_low, smtp_server_low = get_email_server(server_side_type=\"low\")\n", + "email_server_high, smtp_server_high = get_email_server(\n", + " server_side_type=\"high\"\n", ")" ] }, { "cell_type": "code", "execution_count": null, - "id": "6", + "id": "5", "metadata": {}, "outputs": [], "source": [ - "email_server, smtp_server = get_email_server(reset=True)" + "server_low = sy.orchestra.launch(\n", + " name=\"bigquery-low\",\n", + " server_side_type=\"low\",\n", + " dev_mode=True,\n", + " n_consumers=1,\n", + " create_producer=True,\n", + " port=low_port,\n", + ")" ] }, { @@ -328,16 +331,6 @@ ")" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "26", - "metadata": {}, - "outputs": [], - "source": [ - "smtp_server.stop()" - ] - }, { "cell_type": "code", "execution_count": null, @@ -346,7 +339,9 @@ "outputs": [], "source": [ "if environment != \"remote\":\n", - " server_low.land()" + " server_low.land()\n", + "smtp_server_low.stop()\n", + "smtp_server_high.stop()" ] } ], diff --git a/notebooks/scenarios/bigquery/sync/010-setup-high-low-datasites.ipynb b/notebooks/scenarios/bigquery/sync/010-setup-high-low-datasites.ipynb deleted file mode 100644 index 0990280fdc2..00000000000 --- a/notebooks/scenarios/bigquery/sync/010-setup-high-low-datasites.ipynb +++ /dev/null @@ -1,331 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# stdlib\n", - "import os\n", - "\n", - "# Testing works over 4 possibilities\n", - "# 1. (python/in-memory workers and using tox commands)\n", - "# 2. (python/in-memory workers and manually running notebooks)\n", - "# 3. (using k8s and using tox commands)\n", - "# 4. (using k8s and manually running notebooks)\n", - "# Uncomment the lines below if in the 4th possibility\n", - "\n", - "# os.environ[\"ORCHESTRA_DEPLOYMENT_TYPE\"] = \"remote\"\n", - "# os.environ[\"DEV_MODE\"] = \"True\"\n", - "# os.environ[\"TEST_EXTERNAL_REGISTRY\"] = \"k3d-registry.localhost:5800\"\n", - "# os.environ[\"CLUSTER_HTTP_PORT_HIGH\"] = \"9081\"\n", - "# os.environ[\"CLUSTER_HTTP_PORT_LOW\"] = \"9083\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# stdlib\n", - "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", - "high_port = os.environ.get(\"CLUSTER_HTTP_PORT_HIGH\", \"9081\")\n", - "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"9083\")\n", - "print(environment, high_port, low_port)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# syft absolute\n", - "import syft as sy\n", - "from syft import test_settings\n", - "from syft.util.test_helpers.worker_helpers import (\n", - " build_and_launch_worker_pool_from_docker_str,\n", - ")\n", - "from syft.util.test_helpers.worker_helpers import (\n", - " launch_worker_pool_from_docker_tag_and_registry,\n", - ")\n", - "from syft.util.test_helpers.email_helpers import get_email_server\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Launch server & login" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "server_low = sy.orchestra.launch(\n", - " name=\"bigquery-low\",\n", - " server_side_type=\"low\",\n", - " dev_mode=True,\n", - " reset=True,\n", - " n_consumers=1,\n", - " create_producer=True,\n", - " port=low_port,\n", - ")\n", - "\n", - "server_high = sy.orchestra.launch(\n", - " name=\"bigquery-high\",\n", - " server_side_type=\"high\",\n", - " dev_mode=True,\n", - " reset=True,\n", - " n_consumers=1,\n", - " create_producer=True,\n", - " port=high_port,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Email Server" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "email_server_low, smtp_server_low = get_email_server(reset=True, server_side_type=\"low\")\n", - "email_server_high, smtp_server_high = get_email_server(\n", - " reset=True, server_side_type=\"high\"\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "low_client = server_low.login(email=\"info@openmined.org\", password=\"changethis\")\n", - "high_client = server_high.login(email=\"info@openmined.org\", password=\"changethis\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert len(high_client.worker_pools.get_all()) == 1\n", - "assert len(low_client.worker_pools.get_all()) == 1" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Setup High First\n", - "\n", - "- If using an external registery, we want to get this from the test_settings.\n", - "- We build the docker image over the base docker image in Syft\n", - "- We give a tag called worker-bigquery to our custom pool image" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "external_registry = test_settings.get(\"external_registry\", default=\"docker.io\")\n", - "\n", - "base_worker_image = high_client.images.get_all()[0]\n", - "\n", - "worker_dockerfile = f\"\"\"\n", - "FROM {str(base_worker_image.image_identifier)}\n", - "\n", - "RUN uv pip install db-dtypes google-cloud-bigquery \n", - "\n", - "\"\"\".strip()\n", - "\n", - "docker_tag = str(base_worker_image.image_identifier).replace(\n", - " \"backend\", \"worker-bigquery\"\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "worker_pool_name = \"bigquery-pool\"\n", - "custom_pool_pod_annotations = {\"bigquery-custom-pool\": \"Pod annotation for bigquery\"}\n", - "custom_pool_pod_labels = {\"bigquery-custom-pool\": \"Pod_label_for_bigquery\"}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "build_and_launch_worker_pool_from_docker_str(\n", - " environment=environment,\n", - " client=high_client,\n", - " worker_pool_name=worker_pool_name,\n", - " custom_pool_pod_annotations=custom_pool_pod_annotations,\n", - " custom_pool_pod_labels=custom_pool_pod_labels,\n", - " worker_dockerfile=worker_dockerfile,\n", - " external_registry=external_registry,\n", - " docker_tag=docker_tag,\n", - " scale_to=2,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert len(high_client.worker_pools.get_all()) == 2" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "high_client.settings.allow_guest_signup(enable=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Setup Low" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "launch_result = launch_worker_pool_from_docker_tag_and_registry(\n", - " environment=environment,\n", - " client=low_client,\n", - " worker_pool_name=worker_pool_name,\n", - " custom_pool_pod_annotations=custom_pool_pod_annotations,\n", - " custom_pool_pod_labels=custom_pool_pod_labels,\n", - " docker_tag=docker_tag,\n", - " external_registry=external_registry,\n", - " scale_to=1,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert len(low_client.worker_pools.get_all()) == 2" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Register a DS only on the low side" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "low_client.register(\n", - " email=\"data_scientist@openmined.org\",\n", - " password=\"verysecurepassword\",\n", - " password_verify=\"verysecurepassword\",\n", - " name=\"John Doe\",\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "low_client.settings.allow_guest_signup(enable=False)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert (\n", - " len(low_client.api.services.user.get_all()) == 2\n", - "), \"Only DS and Admin should be at low side\"\n", - "assert (\n", - " len(high_client.api.services.user.get_all()) == 1\n", - "), \"Only Admin should be at high side\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Close" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if environment != \"remote\":\n", - " server_high.land()\n", - " server_low.land()\n", - " smtp_server_low.stop()\n", - " smtp_server_high.stop()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.4" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} \ No newline at end of file diff --git a/notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb b/notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb index 4da648ca984..bdf917e83ea 100644 --- a/notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb +++ b/notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb @@ -658,8 +658,8 @@ "if environment != \"remote\":\n", " server_high.land()\n", " server_low.land()\n", - " smtp_server_low.stop()\n", - " smtp_server_high.stop()" + "smtp_server_low.stop()\n", + "smtp_server_high.stop()" ] }, { @@ -691,4 +691,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} \ No newline at end of file +} diff --git a/packages/syft/src/syft/util/test_helpers/email_helpers.py b/packages/syft/src/syft/util/test_helpers/email_helpers.py index 094c30645a3..6579ab7e5d9 100644 --- a/packages/syft/src/syft/util/test_helpers/email_helpers.py +++ b/packages/syft/src/syft/util/test_helpers/email_helpers.py @@ -314,7 +314,7 @@ def run_with_timeout(self, condition_func, *args, **kwargs): return result -def get_email_server(reset=False, server_side_type="high"): +def get_email_server(reset=False, server_side_type=""): if server_side_type == "high": email_file = "./emails_high.json" port = 9026 From 55d4d623980ea495c9ed9c8cea8b9e8fb3964975 Mon Sep 17 00:00:00 2001 From: Brendan Schell Date: Thu, 19 Sep 2024 18:41:03 -0400 Subject: [PATCH 04/28] some initial changes to submit multiple jobs --- .../sync/000-setup-high-low-datasites.ipynb | 672 ++++---- .../sync/001-scale-delete-worker-pools.ipynb | 18 +- .../sync/011-users-emails-passwords.ipynb | 17 +- .../sync/020-configure-api-and-sync.ipynb | 1391 +++++++++-------- .../bigquery/sync/021-create-jobs.ipynb | 441 ++++++ .../bigquery/sync/030-ds-submit-request.ipynb | 12 +- .../sync/040-do-review-requests.ipynb | 29 +- .../bigquery/sync/050-ds-get-results.ipynb | 25 +- .../syft/util/test_helpers/email_helpers.py | 7 +- 9 files changed, 1548 insertions(+), 1064 deletions(-) create mode 100644 notebooks/scenarios/bigquery/sync/021-create-jobs.ipynb diff --git a/notebooks/scenarios/bigquery/sync/000-setup-high-low-datasites.ipynb b/notebooks/scenarios/bigquery/sync/000-setup-high-low-datasites.ipynb index 3fe3966ba5d..1b07c9476c0 100644 --- a/notebooks/scenarios/bigquery/sync/000-setup-high-low-datasites.ipynb +++ b/notebooks/scenarios/bigquery/sync/000-setup-high-low-datasites.ipynb @@ -1,331 +1,345 @@ { - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# stdlib\n", - "import os\n", - "\n", - "# Testing works over 4 possibilities\n", - "# 1. (python/in-memory workers and using tox commands)\n", - "# 2. (python/in-memory workers and manually running notebooks)\n", - "# 3. (using k8s and using tox commands)\n", - "# 4. (using k8s and manually running notebooks)\n", - "# Uncomment the lines below if in the 4th possibility\n", - "\n", - "# os.environ[\"ORCHESTRA_DEPLOYMENT_TYPE\"] = \"remote\"\n", - "# os.environ[\"DEV_MODE\"] = \"True\"\n", - "# os.environ[\"TEST_EXTERNAL_REGISTRY\"] = \"k3d-registry.localhost:5800\"\n", - "# os.environ[\"CLUSTER_HTTP_PORT_HIGH\"] = \"9081\"\n", - "# os.environ[\"CLUSTER_HTTP_PORT_LOW\"] = \"9083\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# stdlib\n", - "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", - "high_port = os.environ.get(\"CLUSTER_HTTP_PORT_HIGH\", \"9081\")\n", - "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"9083\")\n", - "print(environment, high_port, low_port)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# syft absolute\n", - "import syft as sy\n", - "from syft import test_settings\n", - "from syft.util.test_helpers.worker_helpers import (\n", - " build_and_launch_worker_pool_from_docker_str,\n", - ")\n", - "from syft.util.test_helpers.worker_helpers import (\n", - " launch_worker_pool_from_docker_tag_and_registry,\n", - ")\n", - "from syft.util.test_helpers.email_helpers import get_email_server\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Launch server & login" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "server_low = sy.orchestra.launch(\n", - " name=\"bigquery-low\",\n", - " server_side_type=\"low\",\n", - " dev_mode=True,\n", - " reset=True,\n", - " n_consumers=1,\n", - " create_producer=True,\n", - " port=low_port,\n", - ")\n", - "\n", - "server_high = sy.orchestra.launch(\n", - " name=\"bigquery-high\",\n", - " server_side_type=\"high\",\n", - " dev_mode=True,\n", - " reset=True,\n", - " n_consumers=1,\n", - " create_producer=True,\n", - " port=high_port,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Email Server" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "email_server_low, smtp_server_low = get_email_server(reset=True, server_side_type=\"low\")\n", - "email_server_high, smtp_server_high = get_email_server(\n", - " reset=True, server_side_type=\"high\"\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "low_client = server_low.login(email=\"info@openmined.org\", password=\"changethis\")\n", - "high_client = server_high.login(email=\"info@openmined.org\", password=\"changethis\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert len(high_client.worker_pools.get_all()) == 1\n", - "assert len(low_client.worker_pools.get_all()) == 1" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Setup High First\n", - "\n", - "- If using an external registery, we want to get this from the test_settings.\n", - "- We build the docker image over the base docker image in Syft\n", - "- We give a tag called worker-bigquery to our custom pool image" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "external_registry = test_settings.get(\"external_registry\", default=\"docker.io\")\n", - "\n", - "base_worker_image = high_client.images.get_all()[0]\n", - "\n", - "worker_dockerfile = f\"\"\"\n", - "FROM {str(base_worker_image.image_identifier)}\n", - "\n", - "RUN uv pip install db-dtypes google-cloud-bigquery \n", - "\n", - "\"\"\".strip()\n", - "\n", - "docker_tag = str(base_worker_image.image_identifier).replace(\n", - " \"backend\", \"worker-bigquery\"\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "worker_pool_name = \"bigquery-pool\"\n", - "custom_pool_pod_annotations = {\"bigquery-custom-pool\": \"Pod annotation for bigquery\"}\n", - "custom_pool_pod_labels = {\"bigquery-custom-pool\": \"Pod_label_for_bigquery\"}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "build_and_launch_worker_pool_from_docker_str(\n", - " environment=environment,\n", - " client=high_client,\n", - " worker_pool_name=worker_pool_name,\n", - " custom_pool_pod_annotations=custom_pool_pod_annotations,\n", - " custom_pool_pod_labels=custom_pool_pod_labels,\n", - " worker_dockerfile=worker_dockerfile,\n", - " external_registry=external_registry,\n", - " docker_tag=docker_tag,\n", - " scale_to=2,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert len(high_client.worker_pools.get_all()) == 2" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "high_client.settings.allow_guest_signup(enable=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Setup Low" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "launch_result = launch_worker_pool_from_docker_tag_and_registry(\n", - " environment=environment,\n", - " client=low_client,\n", - " worker_pool_name=worker_pool_name,\n", - " custom_pool_pod_annotations=custom_pool_pod_annotations,\n", - " custom_pool_pod_labels=custom_pool_pod_labels,\n", - " docker_tag=docker_tag,\n", - " external_registry=external_registry,\n", - " scale_to=1,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert len(low_client.worker_pools.get_all()) == 2" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Register a DS only on the low side" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "low_client.register(\n", - " email=\"data_scientist@openmined.org\",\n", - " password=\"verysecurepassword\",\n", - " password_verify=\"verysecurepassword\",\n", - " name=\"John Doe\",\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "low_client.settings.allow_guest_signup(enable=False)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert (\n", - " len(low_client.api.services.user.get_all()) == 2\n", - "), \"Only DS and Admin should be at low side\"\n", - "assert (\n", - " len(high_client.api.services.user.get_all()) == 1\n", - "), \"Only Admin should be at high side\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Close" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if environment != \"remote\":\n", - " server_high.land()\n", - " server_low.land()\n", - "smtp_server_low.stop()\n", - "smtp_server_high.stop()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.4" - } - }, - "nbformat": 4, - "nbformat_minor": 4 + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# stdlib\n", + "import os\n", + "\n", + "# Testing works over 4 possibilities\n", + "# 1. (python/in-memory workers and using tox commands)\n", + "# 2. (python/in-memory workers and manually running notebooks)\n", + "# 3. (using k8s and using tox commands)\n", + "# 4. (using k8s and manually running notebooks)\n", + "# Uncomment the lines below if in the 4th possibility\n", + "\n", + "# os.environ[\"ORCHESTRA_DEPLOYMENT_TYPE\"] = \"remote\"\n", + "# os.environ[\"DEV_MODE\"] = \"True\"\n", + "# os.environ[\"TEST_EXTERNAL_REGISTRY\"] = \"k3d-registry.localhost:5800\"\n", + "# os.environ[\"CLUSTER_HTTP_PORT_HIGH\"] = \"9081\"\n", + "# os.environ[\"CLUSTER_HTTP_PORT_LOW\"] = \"9083\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# stdlib\n", + "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", + "high_port = os.environ.get(\"CLUSTER_HTTP_PORT_HIGH\", \"9081\")\n", + "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"9083\")\n", + "print(environment, high_port, low_port)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# syft absolute\n", + "import syft as sy\n", + "from syft import test_settings\n", + "from syft.util.test_helpers.email_helpers import get_email_server\n", + "from syft.util.test_helpers.worker_helpers import (\n", + " build_and_launch_worker_pool_from_docker_str,\n", + ")\n", + "from syft.util.test_helpers.worker_helpers import (\n", + " launch_worker_pool_from_docker_tag_and_registry,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Launch server & login" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "server_low = sy.orchestra.launch(\n", + " name=\"bigquery-low\",\n", + " server_side_type=\"low\",\n", + " dev_mode=True,\n", + " reset=True,\n", + " n_consumers=1,\n", + " create_producer=True,\n", + " port=low_port,\n", + ")\n", + "\n", + "server_high = sy.orchestra.launch(\n", + " name=\"bigquery-high\",\n", + " server_side_type=\"high\",\n", + " dev_mode=True,\n", + " reset=True,\n", + " n_consumers=1,\n", + " create_producer=True,\n", + " port=high_port,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Email Server" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "email_server_low, smtp_server_low = get_email_server(reset=True, server_side_type=\"low\")\n", + "email_server_high, smtp_server_high = get_email_server(\n", + " reset=True, server_side_type=\"high\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "low_client = server_low.login(email=\"info@openmined.org\", password=\"changethis\")\n", + "high_client = server_high.login(email=\"info@openmined.org\", password=\"changethis\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(high_client.worker_pools.get_all()) == 1\n", + "assert len(low_client.worker_pools.get_all()) == 1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Setup High First\n", + "\n", + "- If using an external registery, we want to get this from the test_settings.\n", + "- We build the docker image over the base docker image in Syft\n", + "- We give a tag called worker-bigquery to our custom pool image" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "external_registry = test_settings.get(\"external_registry\", default=\"docker.io\")\n", + "\n", + "base_worker_image = high_client.images.get_all()[0]\n", + "\n", + "worker_dockerfile = f\"\"\"\n", + "FROM {str(base_worker_image.image_identifier)}\n", + "\n", + "RUN uv pip install db-dtypes google-cloud-bigquery \n", + "\n", + "\"\"\".strip()\n", + "\n", + "docker_tag = str(base_worker_image.image_identifier).replace(\n", + " \"backend\", \"worker-bigquery\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "worker_pool_name = \"bigquery-pool\"\n", + "custom_pool_pod_annotations = {\"bigquery-custom-pool\": \"Pod annotation for bigquery\"}\n", + "custom_pool_pod_labels = {\"bigquery-custom-pool\": \"Pod_label_for_bigquery\"}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "build_and_launch_worker_pool_from_docker_str(\n", + " environment=environment,\n", + " client=high_client,\n", + " worker_pool_name=worker_pool_name,\n", + " custom_pool_pod_annotations=custom_pool_pod_annotations,\n", + " custom_pool_pod_labels=custom_pool_pod_labels,\n", + " worker_dockerfile=worker_dockerfile,\n", + " external_registry=external_registry,\n", + " docker_tag=docker_tag,\n", + " scale_to=2,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(high_client.worker_pools.get_all()) == 2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "high_client.settings.allow_guest_signup(enable=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Setup Low" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "launch_result = launch_worker_pool_from_docker_tag_and_registry(\n", + " environment=environment,\n", + " client=low_client,\n", + " worker_pool_name=worker_pool_name,\n", + " custom_pool_pod_annotations=custom_pool_pod_annotations,\n", + " custom_pool_pod_labels=custom_pool_pod_labels,\n", + " docker_tag=docker_tag,\n", + " external_registry=external_registry,\n", + " scale_to=1,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(low_client.worker_pools.get_all()) == 2" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Register a DS only on the low side" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "low_client.register(\n", + " email=\"data_scientist@openmined.org\",\n", + " password=\"verysecurepassword\",\n", + " password_verify=\"verysecurepassword\",\n", + " name=\"John Doe\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "low_client.settings.allow_guest_signup(enable=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert (\n", + " len(low_client.api.services.user.get_all()) == 2\n", + "), \"Only DS and Admin should be at low side\"\n", + "assert (\n", + " len(high_client.api.services.user.get_all()) == 1\n", + "), \"Only Admin should be at high side\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Close" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if environment != \"remote\":\n", + " server_high.land()\n", + " server_low.land()\n", + "smtp_server_low.stop()\n", + "smtp_server_high.stop()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.4" + } + }, + "nbformat": 4, + "nbformat_minor": 4 } diff --git a/notebooks/scenarios/bigquery/sync/001-scale-delete-worker-pools.ipynb b/notebooks/scenarios/bigquery/sync/001-scale-delete-worker-pools.ipynb index 6ea1af97242..ac6f4a1a395 100644 --- a/notebooks/scenarios/bigquery/sync/001-scale-delete-worker-pools.ipynb +++ b/notebooks/scenarios/bigquery/sync/001-scale-delete-worker-pools.ipynb @@ -79,20 +79,20 @@ { "cell_type": "code", "execution_count": null, - "id": "6", + "id": "5", "metadata": {}, "outputs": [], "source": [ - "email_server_low, smtp_server_low = get_email_server(server_side_type=\"low\")\n", + "email_server_low, smtp_server_low = get_email_server(server_side_type=\"low\", reset=True)\n", "email_server_high, smtp_server_high = get_email_server(\n", - " server_side_type=\"high\"\n", + " server_side_type=\"high\", reset=True\n", ")" ] }, { "cell_type": "code", "execution_count": null, - "id": "5", + "id": "6", "metadata": {}, "outputs": [], "source": [ @@ -334,7 +334,7 @@ { "cell_type": "code", "execution_count": null, - "id": "27", + "id": "26", "metadata": {}, "outputs": [], "source": [ @@ -343,6 +343,14 @@ "smtp_server_low.stop()\n", "smtp_server_high.stop()" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "27", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb b/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb index fc52c3ba391..1cb7887fe4b 100644 --- a/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb +++ b/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb @@ -23,21 +23,16 @@ "metadata": {}, "outputs": [], "source": [ - "# isort: off\n", "# stdlib\n", "import os\n", "\n", "# syft absolute\n", "import syft as sy\n", - "from syft import test_helpers # noqa: F401\n", - "\n", - "# third party\n", - "from email_helpers import SENDER\n", - "from email_helpers import create_user\n", - "from email_helpers import get_email_server\n", - "from email_helpers import make_user\n", - "from email_helpers import save_users\n", - "# isort: on" + "from syft.util.test_helpers.email_helpers import SENDER\n", + "from syft.util.test_helpers.email_helpers import create_user\n", + "from syft.util.test_helpers.email_helpers import get_email_server\n", + "from syft.util.test_helpers.email_helpers import make_user\n", + "from syft.util.test_helpers.email_helpers import save_users" ] }, { @@ -661,7 +656,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.3" + "version": "3.12.4" } }, "nbformat": 4, diff --git a/notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb b/notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb index bdf917e83ea..ed6bfcb0399 100644 --- a/notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb +++ b/notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb @@ -1,694 +1,701 @@ { - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# stdlib\n", - "import os\n", - "\n", - "# os.environ[\"ORCHESTRA_DEPLOYMENT_TYPE\"] = \"remote\"\n", - "# os.environ[\"DEV_MODE\"] = \"True\"\n", - "# os.environ[\"TEST_EXTERNAL_REGISTRY\"] = \"k3d-registry.localhost:5800\"\n", - "# os.environ[\"CLUSTER_HTTP_PORT_HIGH\"] = \"9081\"\n", - "# os.environ[\"CLUSTER_HTTP_PORT_LOW\"] = \"9083\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# stdlib\n", - "\n", - "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", - "high_port = os.environ.get(\"CLUSTER_HTTP_PORT_HIGH\", \"9081\")\n", - "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"9083\")\n", - "print(environment, high_port, low_port)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# stdlib\n", - "\n", - "# set to use the live APIs\n", - "# import os\n", - "# os.environ[\"TEST_BIGQUERY_APIS_LIVE\"] = \"True\"\n", - "# third party\n", - "import pandas as pd\n", - "\n", - "# syft absolute\n", - "import syft as sy\n", - "from syft import test_settings\n", - "from syft.client.syncing import compare_clients\n", - "from syft.util.test_helpers.apis import make_schema\n", - "from syft.util.test_helpers.apis import make_submit_query\n", - "from syft.util.test_helpers.apis import make_test_query\n", - "from syft.util.test_helpers.email_helpers import get_email_server\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Launch server and login" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "server_low = sy.orchestra.launch(\n", - " name=\"bigquery-low\",\n", - " server_side_type=\"low\",\n", - " dev_mode=True,\n", - " n_consumers=1,\n", - " create_producer=True,\n", - " port=low_port,\n", - ")\n", - "\n", - "server_high = sy.orchestra.launch(\n", - " name=\"bigquery-high\",\n", - " server_side_type=\"high\",\n", - " dev_mode=True,\n", - " n_consumers=1,\n", - " create_producer=True,\n", - " port=high_port,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Email Server" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "email_server_low, smtp_server_low = get_email_server(server_side_type=\"low\")\n", - "email_server_high, smtp_server_high = get_email_server(server_side_type=\"high\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "low_client = server_low.login(email=\"info@openmined.org\", password=\"changethis\")\n", - "high_client = server_high.login(email=\"info@openmined.org\", password=\"changethis\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert len(high_client.worker_pools.get_all()) == 2\n", - "assert len(low_client.worker_pools.get_all()) == 2" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "this_worker_pool_name = \"bigquery-pool\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Load database information from test_settings" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "dataset_1 = test_settings.get(\"dataset_1\", default=\"dataset_1\")\n", - "dataset_2 = test_settings.get(\"dataset_2\", default=\"dataset_2\")\n", - "table_1 = test_settings.get(\"table_1\", default=\"table_1\")\n", - "table_2 = test_settings.get(\"table_2\", default=\"table_2\")\n", - "table_2_col_id = test_settings.get(\"table_2_col_id\", default=\"table_id\")\n", - "table_2_col_score = test_settings.get(\"table_2_col_score\", default=\"colname\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Create and test different endpoints" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "----" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Create `biquery.schema` endpoint" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "schema_function = make_schema(\n", - " settings={\n", - " \"calls_per_min\": 5,\n", - " },\n", - " worker_pool=this_worker_pool_name,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "high_client.custom_api.add(endpoint=schema_function)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "result = high_client.api.services.bigquery.schema()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert len(result) == 23" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "TODO: Note that when we do not create a job, the type of result is `syft.service.action.pandas.PandasDataFrameObject` and not pandas but the `.get()` method will get you the expected answer" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# syft absolute\n", - "from syft.service.action.pandas import PandasDataFrameObject\n", - "\n", - "# assert isinstance(result, pd.DataFrame)\n", - "assert isinstance(result, PandasDataFrameObject)\n", - "assert isinstance(result.get(), pd.DataFrame)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "____" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Create `biquery.test_query` endpoint" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "mock_func = make_test_query(\n", - " settings={\n", - " \"rate_limiter_enabled\": True,\n", - " \"calls_per_min\": 10,\n", - " }\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "private_func = make_test_query(\n", - " settings={\n", - " \"rate_limiter_enabled\": False,\n", - " }\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "new_endpoint = sy.TwinAPIEndpoint(\n", - " path=\"bigquery.test_query\",\n", - " description=\"This endpoint allows to query Bigquery storage via SQL queries.\",\n", - " private_function=private_func,\n", - " mock_function=mock_func,\n", - " worker_pool=this_worker_pool_name,\n", - ")\n", - "\n", - "high_client.custom_api.add(endpoint=new_endpoint)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Some features for updating endpoint" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Here, we update the endpoint to timeout after 100s (rather the default of 60s)\n", - "high_client.api.services.api.update(\n", - " endpoint_path=\"bigquery.test_query\", endpoint_timeout=120\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "high_client.api.services.api.update(\n", - " endpoint_path=\"bigquery.test_query\", hide_mock_definition=True\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Test the `bigquery.test_query` endpoint" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Test mock version\n", - "result = high_client.api.services.bigquery.test_query.mock(\n", - " sql_query=f\"SELECT * FROM {dataset_1}.{table_1} LIMIT 10\"\n", - ")\n", - "result" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert len(result) == 10" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Test mock version for wrong queries\n", - "with sy.raises(\n", - " sy.SyftException(public_message=\"*must be qualified with a dataset*\"), show=True\n", - "):\n", - " high_client.api.services.bigquery.test_query.mock(\n", - " sql_query=\"SELECT * FROM invalid_table LIMIT 1\"\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Test private version\n", - "result = high_client.api.services.bigquery.test_query.private(\n", - " sql_query=f\"SELECT * FROM {dataset_1}.{table_1} LIMIT 12\"\n", - ")\n", - "result" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert len(result) == 12" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "____" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Create `submit_query` endpoint" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "submit_query_function = make_submit_query(\n", - " settings={}, worker_pool=this_worker_pool_name\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "high_client.custom_api.add(endpoint=submit_query_function)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "high_client.api.services.api.update(\n", - " endpoint_path=\"bigquery.submit_query\", hide_mock_definition=True\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Testing submit query\n", - "result = high_client.api.services.bigquery.submit_query(\n", - " func_name=\"my_func\",\n", - " query=f\"SELECT * FROM {dataset_1}.{table_1} LIMIT 2\",\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert \"Query submitted\" in result\n", - "result" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "job = high_client.code.my_func(blocking=False)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "res = job.wait().get()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert isinstance(res, pd.DataFrame)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Test endpoints" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "high_client.custom_api.api_endpoints()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert len(high_client.custom_api.api_endpoints()) == 3" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert (\n", - " high_client.api.services.bigquery.test_query\n", - " and high_client.api.services.bigquery.submit_query\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Syncing" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "diff = compare_clients(\n", - " from_client=high_client, to_client=low_client, hide_usercode=False\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# TODO verify that jobs are actually filtered out\n", - "# TODO we need to think about whether its possible for the admin to create more data here that would break sync" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "widget = diff.resolve()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# TODO maybe see if non-internal method we can use or make it public\n", - "widget._share_all()\n", - "widget._sync_all()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert len(low_client.jobs.get_all()) == 0" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert len(low_client.custom_api.api_endpoints()) == 3" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert len(high_client.custom_api.api_endpoints()) == 3" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Test emails" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# email_server_low.get_emails_for_user(user_email=\"info@openmined.org\")\n", - "assert len(email_server_high.get_emails_for_user(user_email=\"info@openmined.org\")) == 2" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert (\n", - " \"Job Failed\"\n", - " in email_server_high.get_emails_for_user(user_email=\"info@openmined.org\")[\n", - " 0\n", - " ].email_content\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert (\n", - " \"A new request has been submitted and requires your attention\"\n", - " in email_server_high.get_emails_for_user(user_email=\"info@openmined.org\")[\n", - " 1\n", - " ].email_content\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Clean up" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if environment != \"remote\":\n", - " server_high.land()\n", - " server_low.land()\n", - "smtp_server_low.stop()\n", - "smtp_server_high.stop()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.4" - } - }, - "nbformat": 4, - "nbformat_minor": 4 + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# stdlib\n", + "import os\n", + "\n", + "# os.environ[\"ORCHESTRA_DEPLOYMENT_TYPE\"] = \"remote\"\n", + "# os.environ[\"DEV_MODE\"] = \"True\"\n", + "# os.environ[\"TEST_EXTERNAL_REGISTRY\"] = \"k3d-registry.localhost:5800\"\n", + "# os.environ[\"CLUSTER_HTTP_PORT_HIGH\"] = \"9081\"\n", + "# os.environ[\"CLUSTER_HTTP_PORT_LOW\"] = \"9083\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# stdlib\n", + "\n", + "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", + "high_port = os.environ.get(\"CLUSTER_HTTP_PORT_HIGH\", \"9081\")\n", + "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"9083\")\n", + "print(environment, high_port, low_port)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# stdlib\n", + "\n", + "# set to use the live APIs\n", + "# import os\n", + "# os.environ[\"TEST_BIGQUERY_APIS_LIVE\"] = \"True\"\n", + "# third party\n", + "import pandas as pd\n", + "\n", + "# syft absolute\n", + "import syft as sy\n", + "from syft import test_settings\n", + "from syft.client.syncing import compare_clients\n", + "from syft.util.test_helpers.apis import make_schema\n", + "from syft.util.test_helpers.apis import make_submit_query\n", + "from syft.util.test_helpers.apis import make_test_query\n", + "from syft.util.test_helpers.email_helpers import get_email_server" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Launch server and login" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "server_low = sy.orchestra.launch(\n", + " name=\"bigquery-low\",\n", + " server_side_type=\"low\",\n", + " dev_mode=True,\n", + " n_consumers=1,\n", + " create_producer=True,\n", + " port=low_port,\n", + ")\n", + "\n", + "server_high = sy.orchestra.launch(\n", + " name=\"bigquery-high\",\n", + " server_side_type=\"high\",\n", + " dev_mode=True,\n", + " n_consumers=1,\n", + " create_producer=True,\n", + " port=high_port,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Email Server" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "email_server_low, smtp_server_low = get_email_server(server_side_type=\"low\")\n", + "email_server_high, smtp_server_high = get_email_server(server_side_type=\"high\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "low_client = server_low.login(email=\"info@openmined.org\", password=\"changethis\")\n", + "high_client = server_high.login(email=\"info@openmined.org\", password=\"changethis\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(high_client.worker_pools.get_all()) == 2\n", + "assert len(low_client.worker_pools.get_all()) == 2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "this_worker_pool_name = \"bigquery-pool\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Load database information from test_settings" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dataset_1 = test_settings.get(\"dataset_1\", default=\"dataset_1\")\n", + "dataset_2 = test_settings.get(\"dataset_2\", default=\"dataset_2\")\n", + "table_1 = test_settings.get(\"table_1\", default=\"table_1\")\n", + "table_2 = test_settings.get(\"table_2\", default=\"table_2\")\n", + "table_2_col_id = test_settings.get(\"table_2_col_id\", default=\"table_id\")\n", + "table_2_col_score = test_settings.get(\"table_2_col_score\", default=\"colname\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Create and test different endpoints" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "----" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create `biquery.schema` endpoint" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "schema_function = make_schema(\n", + " settings={\n", + " \"calls_per_min\": 5,\n", + " },\n", + " worker_pool=this_worker_pool_name,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "high_client.custom_api.add(endpoint=schema_function)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "result = high_client.api.services.bigquery.schema()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(result) == 23" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "TODO: Note that when we do not create a job, the type of result is `syft.service.action.pandas.PandasDataFrameObject` and not pandas but the `.get()` method will get you the expected answer" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# syft absolute\n", + "from syft.service.action.pandas import PandasDataFrameObject\n", + "\n", + "# assert isinstance(result, pd.DataFrame)\n", + "assert isinstance(result, PandasDataFrameObject)\n", + "assert isinstance(result.get(), pd.DataFrame)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "____" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create `biquery.test_query` endpoint" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "mock_func = make_test_query(\n", + " settings={\n", + " \"rate_limiter_enabled\": True,\n", + " \"calls_per_min\": 10,\n", + " }\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "private_func = make_test_query(\n", + " settings={\n", + " \"rate_limiter_enabled\": False,\n", + " }\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "new_endpoint = sy.TwinAPIEndpoint(\n", + " path=\"bigquery.test_query\",\n", + " description=\"This endpoint allows to query Bigquery storage via SQL queries.\",\n", + " private_function=private_func,\n", + " mock_function=mock_func,\n", + " worker_pool=this_worker_pool_name,\n", + ")\n", + "\n", + "high_client.custom_api.add(endpoint=new_endpoint)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Some features for updating endpoint" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Here, we update the endpoint to timeout after 100s (rather the default of 60s)\n", + "high_client.api.services.api.update(\n", + " endpoint_path=\"bigquery.test_query\", endpoint_timeout=120\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "high_client.api.services.api.update(\n", + " endpoint_path=\"bigquery.test_query\", hide_mock_definition=True\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Test the `bigquery.test_query` endpoint" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Test mock version\n", + "result = high_client.api.services.bigquery.test_query.mock(\n", + " sql_query=f\"SELECT * FROM {dataset_1}.{table_1} LIMIT 10\"\n", + ")\n", + "result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(result) == 10" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Test mock version for wrong queries\n", + "with sy.raises(\n", + " sy.SyftException(public_message=\"*must be qualified with a dataset*\"), show=True\n", + "):\n", + " high_client.api.services.bigquery.test_query.mock(\n", + " sql_query=\"SELECT * FROM invalid_table LIMIT 1\"\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Test private version\n", + "result = high_client.api.services.bigquery.test_query.private(\n", + " sql_query=f\"SELECT * FROM {dataset_1}.{table_1} LIMIT 12\"\n", + ")\n", + "result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(result) == 12" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "____" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create `submit_query` endpoint" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "submit_query_function = make_submit_query(\n", + " settings={}, worker_pool=this_worker_pool_name\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "high_client.custom_api.add(endpoint=submit_query_function)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "high_client.api.services.api.update(\n", + " endpoint_path=\"bigquery.submit_query\", hide_mock_definition=True\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Testing submit query\n", + "result = high_client.api.services.bigquery.submit_query(\n", + " func_name=\"my_func\",\n", + " query=f\"SELECT * FROM {dataset_1}.{table_1} LIMIT 2\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert \"Query submitted\" in result\n", + "result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "job = high_client.code.my_func(blocking=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "res = job.wait().get()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert isinstance(res, pd.DataFrame)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Test endpoints" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "high_client.custom_api.api_endpoints()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(high_client.custom_api.api_endpoints()) == 3" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert (\n", + " high_client.api.services.bigquery.test_query\n", + " and high_client.api.services.bigquery.submit_query\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Syncing" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "diff = compare_clients(\n", + " from_client=high_client, to_client=low_client, hide_usercode=False\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# TODO verify that jobs are actually filtered out\n", + "# TODO we need to think about whether its possible for the admin to create more data here that would break sync" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "widget = diff.resolve()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# TODO maybe see if non-internal method we can use or make it public\n", + "widget._share_all()\n", + "widget._sync_all()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(low_client.jobs.get_all()) == 0" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(low_client.custom_api.api_endpoints()) == 3" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(high_client.custom_api.api_endpoints()) == 3" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Test emails" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# email_server_low.get_emails_for_user(user_email=\"info@openmined.org\")\n", + "assert len(email_server_high.get_emails_for_user(user_email=\"info@openmined.org\")) == 2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert (\n", + " \"Job Failed\"\n", + " in email_server_high.get_emails_for_user(user_email=\"info@openmined.org\")[\n", + " 0\n", + " ].email_content\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert (\n", + " \"A new request has been submitted and requires your attention\"\n", + " in email_server_high.get_emails_for_user(user_email=\"info@openmined.org\")[\n", + " 1\n", + " ].email_content\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Clean up" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if environment != \"remote\":\n", + " server_high.land()\n", + " server_low.land()\n", + "smtp_server_low.stop()\n", + "smtp_server_high.stop()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.4" + } + }, + "nbformat": 4, + "nbformat_minor": 4 } diff --git a/notebooks/scenarios/bigquery/sync/021-create-jobs.ipynb b/notebooks/scenarios/bigquery/sync/021-create-jobs.ipynb new file mode 100644 index 00000000000..f63763148f2 --- /dev/null +++ b/notebooks/scenarios/bigquery/sync/021-create-jobs.ipynb @@ -0,0 +1,441 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "0", + "metadata": {}, + "outputs": [], + "source": [ + "# stdlib\n", + "import os\n", + "\n", + "# os.environ[\"ORCHESTRA_DEPLOYMENT_TYPE\"] = \"remote\"\n", + "# os.environ[\"DEV_MODE\"] = \"True\"\n", + "# os.environ[\"TEST_EXTERNAL_REGISTRY\"] = \"k3d-registry.localhost:5800\"\n", + "# os.environ[\"CLUSTER_HTTP_PORT_HIGH\"] = \"9081\"\n", + "# os.environ[\"CLUSTER_HTTP_PORT_LOW\"] = \"9083\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "# stdlib\n", + "\n", + "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", + "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"9083\")\n", + "print(environment, low_port)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "# stdlib\n", + "from collections import Counter\n", + "\n", + "# syft absolute\n", + "import syft as sy\n", + "from syft.util.test_helpers.email_helpers import get_email_server\n", + "from syft.util.test_helpers.email_helpers import load_users" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "ADMIN_EMAIL, ADMIN_PW = \"info@openmined.org\", \"changethis\"" + ] + }, + { + "cell_type": "markdown", + "id": "4", + "metadata": {}, + "source": [ + "# Launch server & login" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "server_low = sy.orchestra.launch(\n", + " name=\"bigquery-low\",\n", + " server_side_type=\"low\",\n", + " dev_mode=True,\n", + " n_consumers=1,\n", + " create_producer=True,\n", + " port=low_port,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "email_server_low, smtp_server_low = get_email_server(server_side_type=\"low\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "low_client = server_low.login(email=ADMIN_EMAIL, password=ADMIN_PW)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "users = load_users(low_client)" + ] + }, + { + "cell_type": "markdown", + "id": "9", + "metadata": {}, + "source": [ + "# Create jobs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "# syft absolute\n", + "from syft.util.test_helpers.job_helpers import TestJob\n", + "from syft.util.test_helpers.job_helpers import create_jobs\n", + "from syft.util.test_helpers.job_helpers import extract_code_path" + ] + }, + { + "cell_type": "markdown", + "id": "11", + "metadata": {}, + "source": [ + "# Inspect job data (requests for these jobs to be created)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12", + "metadata": {}, + "outputs": [], + "source": [ + "num_jobs = int(os.environ.get(\"NUM_TEST_JOBS\", 10))\n", + "\n", + "jobs_data = create_jobs(users, total_jobs=num_jobs)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "13", + "metadata": {}, + "outputs": [], + "source": [ + "counts = Counter([j.job_type for j in jobs_data])\n", + "for k, v in counts.most_common():\n", + " print(f\"{k}: #{v}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "14", + "metadata": {}, + "outputs": [], + "source": [ + "# syft absolute\n", + "from syft.util.test_helpers.job_helpers import save_jobs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15", + "metadata": {}, + "outputs": [], + "source": [ + "print(f\"{len(jobs_data)=}\")\n", + "\n", + "for job in jobs_data:\n", + " print(f\"{job.job_type=}, {job.should_succeed=}, {job.should_submit=}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "16", + "metadata": {}, + "outputs": [], + "source": [ + "assert len(jobs_data) == num_jobs\n", + "assert all(isinstance(j, TestJob) for j in jobs_data)\n", + "assert all(job.client is not None for job in jobs_data)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "17", + "metadata": {}, + "outputs": [], + "source": [ + "save_jobs(jobs_data)" + ] + }, + { + "cell_type": "markdown", + "id": "18", + "metadata": {}, + "source": [ + "# Submit jobs\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "19", + "metadata": {}, + "outputs": [], + "source": [ + "admin_emails_before = len(email_server_low.get_emails_for_user(\"admin@bigquery.org\"))\n", + "print(f\"{admin_emails_before=}\")" + ] + }, + { + "cell_type": "markdown", + "id": "20", + "metadata": {}, + "source": [ + "## Test Succesful jobs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "21", + "metadata": {}, + "outputs": [], + "source": [ + "jobs_submit_should_succeed = [j for j in jobs_data if j.should_submit]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "22", + "metadata": {}, + "outputs": [], + "source": [ + "for job in jobs_submit_should_succeed:\n", + " client = job.client\n", + " response = client.api.services.bigquery.submit_query(\n", + " func_name=job.func_name, query=job.query\n", + " )\n", + " job.code_path = extract_code_path(response)" + ] + }, + { + "cell_type": "markdown", + "id": "23", + "metadata": {}, + "source": [ + "## Test failures" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24", + "metadata": {}, + "outputs": [], + "source": [ + "jobs_submit_should_fail = [j for j in jobs_data if not j.should_submit]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "25", + "metadata": {}, + "outputs": [], + "source": [ + "for job in jobs_submit_should_fail:\n", + " client = job.client\n", + "\n", + " with sy.raises(sy.SyftException):\n", + " client.api.services.bigquery.submit_query(\n", + " func_name=job.func_name, query=job.query\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "26", + "metadata": {}, + "outputs": [], + "source": [ + "for job in jobs_data:\n", + " print(f\"Job {job.func_name:.20} {job.should_submit=}, {job.is_submitted=}\")\n", + "\n", + "assert all(job.is_submitted == job.should_submit for job in jobs_data)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "27", + "metadata": {}, + "outputs": [], + "source": [ + "save_jobs(jobs_data)" + ] + }, + { + "cell_type": "markdown", + "id": "28", + "metadata": {}, + "source": [ + "## Test: cannot execute submitted jobs yet" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "29", + "metadata": {}, + "outputs": [], + "source": [ + "submitted_jobs = [job for job in jobs_data if job.should_submit]\n", + "job_execution_fns = [getattr(job.client.code, job.code_path) for job in submitted_jobs]\n", + "assert len(submitted_jobs) # failsafe for next tests" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "30", + "metadata": {}, + "outputs": [], + "source": [ + "for fn in job_execution_fns:\n", + " # blocking\n", + " with sy.raises(\n", + " sy.SyftException(public_message=\"*Your code is waiting for approval*\")\n", + " ):\n", + " result = fn()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "31", + "metadata": {}, + "outputs": [], + "source": [ + "# currently errors out with\n", + "# syft.types.errors.SyftException: Please wait for the admin to allow the execution of this code\n", + "\n", + "\"\"\" for fn in job_execution_fns:\n", + " # nonblocking\n", + " result_job = fn(blocking=False)\n", + " result_job.wait()\n", + " assert isinstance(result_job.result, sy.SyftError)\n", + " assert result_job.status == JobStatus.ERRORED \"\"\"" + ] + }, + { + "cell_type": "markdown", + "id": "32", + "metadata": {}, + "source": [ + "# Verify that admin has emails for submitted requests" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "33", + "metadata": {}, + "outputs": [], + "source": [ + "num_should_submit = sum(j.should_submit for j in jobs_data)\n", + "admin_emails_after = len(email_server_low.get_emails_for_user(\"admin@bigquery.org\"))\n", + "print(\"admin emails after\", admin_emails_after)\n", + "assert admin_emails_after >= admin_emails_before + num_should_submit\n", + "# assert len(users_emails) > after_number_of_emails\n", + "# assert len(users_emails) == after_number_of_emails + 1" + ] + }, + { + "cell_type": "markdown", + "id": "34", + "metadata": {}, + "source": [ + "# Cleanup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "35", + "metadata": {}, + "outputs": [], + "source": [ + "if environment != \"remote\":\n", + " server_low.land()\n", + "smtp_server_low.stop()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "syft", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.4" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/scenarios/bigquery/sync/030-ds-submit-request.ipynb b/notebooks/scenarios/bigquery/sync/030-ds-submit-request.ipynb index 8d64892206a..2d8ed894d40 100644 --- a/notebooks/scenarios/bigquery/sync/030-ds-submit-request.ipynb +++ b/notebooks/scenarios/bigquery/sync/030-ds-submit-request.ipynb @@ -12,6 +12,7 @@ "# os.environ[\"ORCHESTRA_DEPLOYMENT_TYPE\"] = \"remote\"\n", "# os.environ[\"DEV_MODE\"] = \"True\"\n", "# os.environ[\"TEST_EXTERNAL_REGISTRY\"] = \"k3d-registry.localhost:5800\"\n", + "# os.environ[\"CLUSTER_HTTP_PORT_HIGH\"] = \"9081\"\n", "# os.environ[\"CLUSTER_HTTP_PORT_LOW\"] = \"9083\"" ] }, @@ -34,16 +35,13 @@ "metadata": {}, "outputs": [], "source": [ - "# isort: off\n", + "# third party\n", + "import pandas as pd\n", + "\n", "# syft absolute\n", "import syft as sy\n", "from syft import test_settings\n", - "from syft import test_helpers # noqa: F401\n", - "\n", - "# third party\n", - "import pandas as pd\n", - "from email_helpers import get_email_server\n", - "# isort: on" + "from syft.util.test_helpers.email_helpers import get_email_server" ] }, { diff --git a/notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb b/notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb index 7ffec4c22d1..332d71aeb26 100644 --- a/notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb +++ b/notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb @@ -6,23 +6,32 @@ "metadata": {}, "outputs": [], "source": [ - "# isort: off\n", "# stdlib\n", "import os\n", "\n", - "# syft absolute\n", - "import syft as sy\n", - "from syft.client.syncing import compare_clients\n", - "from syft import test_helpers # noqa: F401\n", + "# os.environ[\"ORCHESTRA_DEPLOYMENT_TYPE\"] = \"remote\"\n", + "# os.environ[\"DEV_MODE\"] = \"True\"\n", + "# os.environ[\"TEST_EXTERNAL_REGISTRY\"] = \"k3d-registry.localhost:5800\"\n", + "# os.environ[\"CLUSTER_HTTP_PORT_HIGH\"] = \"9081\"\n", + "# os.environ[\"CLUSTER_HTTP_PORT_LOW\"] = \"9083\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# stdlib\n", "\n", "# third party\n", "import pandas as pd\n", - "from email_helpers import get_email_server\n", - "# isort: on\n", "\n", - "# os.environ[\"ORCHESTRA_DEPLOYMENT_TYPE\"] = \"remote\"\n", - "# os.environ[\"CLUSTER_HTTP_PORT_HIGH\"] = \"9081\"\n", - "# os.environ[\"CLUSTER_HTTP_PORT_LOW\"] = \"9083\"" + "# syft absolute\n", + "import syft as sy\n", + "from syft import test_helpers # noqa: F401\n", + "from syft.client.syncing import compare_clients\n", + "from syft.util.test_helpers.email_helpers import get_email_server" ] }, { diff --git a/notebooks/scenarios/bigquery/sync/050-ds-get-results.ipynb b/notebooks/scenarios/bigquery/sync/050-ds-get-results.ipynb index e8432254ebe..6169fafb81b 100644 --- a/notebooks/scenarios/bigquery/sync/050-ds-get-results.ipynb +++ b/notebooks/scenarios/bigquery/sync/050-ds-get-results.ipynb @@ -6,21 +6,30 @@ "metadata": {}, "outputs": [], "source": [ - "# isort: off\n", "# stdlib\n", "import os\n", "\n", - "# syft absolute\n", - "import syft as sy\n", - "from syft import test_helpers # noqa: F401\n", + "# os.environ[\"ORCHESTRA_DEPLOYMENT_TYPE\"] = \"remote\"\n", + "# os.environ[\"DEV_MODE\"] = \"True\"\n", + "# os.environ[\"TEST_EXTERNAL_REGISTRY\"] = \"k3d-registry.localhost:5800\"\n", + "# os.environ[\"CLUSTER_HTTP_PORT_HIGH\"] = \"9081\"\n", + "# os.environ[\"CLUSTER_HTTP_PORT_LOW\"] = \"9083\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# stdlib\n", "\n", "# third party\n", "import pandas as pd\n", - "from email_helpers import get_email_server\n", - "# isort: on\n", "\n", - "# os.environ[\"ORCHESTRA_DEPLOYMENT_TYPE\"] = \"remote\"\n", - "# os.environ[\"CLUSTER_HTTP_PORT_LOW\"] = \"9083\"" + "# syft absolute\n", + "import syft as sy\n", + "from syft.util.test_helpers.email_helpers import get_email_server" ] }, { diff --git a/packages/syft/src/syft/util/test_helpers/email_helpers.py b/packages/syft/src/syft/util/test_helpers/email_helpers.py index 6579ab7e5d9..cc6ab7a2839 100644 --- a/packages/syft/src/syft/util/test_helpers/email_helpers.py +++ b/packages/syft/src/syft/util/test_helpers/email_helpers.py @@ -218,7 +218,7 @@ def user_exists(root_client, email: str) -> bool: class SMTPTestServer: - def __init__(self, email_server, port=9025): + def __init__(self, email_server, port=9025, ready_timeout=20): self.port = port self.hostname = "0.0.0.0" self._stop_event = asyncio.Event() @@ -245,7 +245,10 @@ async def handle_DATA(self, server, session, envelope): try: self.handler = SimpleHandler() self.controller = Controller( - self.handler, hostname=self.hostname, port=self.port + self.handler, + hostname=self.hostname, + port=self.port, + ready_timeout=ready_timeout, ) except Exception as e: print(f"> Error initializing SMTPTestServer Controller: {e}") From 4de268a44c2bb15927ae72e6ec802a58610e9d4f Mon Sep 17 00:00:00 2001 From: Brendan Schell Date: Fri, 20 Sep 2024 15:03:07 -0400 Subject: [PATCH 05/28] make l0 notebooks more consistent with l2 --- ...tart-and-configure-server-and-admins.ipynb | 286 ++++++++++++++++++ .../sync/001-scale-delete-worker-pools.ipynb | 9 +- ...es.ipynb => 010-setup-bigquery-pool.ipynb} | 9 +- .../sync/011-users-emails-passwords.ipynb | 9 +- .../sync/020-configure-api-and-sync.ipynb | 10 +- .../bigquery/sync/021-create-jobs.ipynb | 82 +++-- .../bigquery/sync/030-ds-submit-request.ipynb | 7 +- .../sync/040-do-review-requests.ipynb | 9 +- .../bigquery/sync/050-ds-get-results.ipynb | 6 +- 9 files changed, 364 insertions(+), 63 deletions(-) create mode 100644 notebooks/scenarios/bigquery/sync/000-start-and-configure-server-and-admins.ipynb rename notebooks/scenarios/bigquery/sync/{000-setup-high-low-datasites.ipynb => 010-setup-bigquery-pool.ipynb} (96%) diff --git a/notebooks/scenarios/bigquery/sync/000-start-and-configure-server-and-admins.ipynb b/notebooks/scenarios/bigquery/sync/000-start-and-configure-server-and-admins.ipynb new file mode 100644 index 00000000000..8bb1fb9275c --- /dev/null +++ b/notebooks/scenarios/bigquery/sync/000-start-and-configure-server-and-admins.ipynb @@ -0,0 +1,286 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# stdlib\n", + "import os\n", + "\n", + "# Testing works over 4 possibilities\n", + "# 1. (python/in-memory workers and using tox commands)\n", + "# 2. (python/in-memory workers and manually running notebooks)\n", + "# 3. (using k8s and using tox commands)\n", + "# 4. (using k8s and manually running notebooks)\n", + "# Uncomment the lines below if in the 4th possibility\n", + "\n", + "# os.environ[\"ORCHESTRA_DEPLOYMENT_TYPE\"] = \"remote\"\n", + "# os.environ[\"DEV_MODE\"] = \"True\"\n", + "# os.environ[\"TEST_EXTERNAL_REGISTRY\"] = \"k3d-registry.localhost:5800\"\n", + "# os.environ[\"CLUSTER_HTTP_PORT_HIGH\"] = \"9081\"\n", + "# os.environ[\"CLUSTER_HTTP_PORT_LOW\"] = \"9083\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# stdlib\n", + "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", + "high_port = os.environ.get(\"CLUSTER_HTTP_PORT_HIGH\", \"9081\")\n", + "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"9083\")\n", + "print(environment, high_port, low_port)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# stdlib\n", + "from os import environ as env\n", + "\n", + "# syft absolute\n", + "import syft as sy\n", + "from syft.util.test_helpers.email_helpers import get_email_server" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# when in k8s these are the default values\n", + "ROOT_EMAIL = \"admin@bigquery.org\"\n", + "ROOT_PASSWORD = \"bqpw\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# in case we are not in k8s we set them here for orchestra to use\n", + "env[\"DEFAULT_ROOT_EMAIL\"] = ROOT_EMAIL\n", + "env[\"DEFAULT_ROOT_PASSWORD\"] = ROOT_PASSWORD" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "server_low = sy.orchestra.launch(\n", + " name=\"bigquery-low\",\n", + " server_side_type=\"low\",\n", + " dev_mode=True,\n", + " reset=True,\n", + " n_consumers=1,\n", + " create_producer=True,\n", + " port=low_port,\n", + ")\n", + "\n", + "server_high = sy.orchestra.launch(\n", + " name=\"bigquery-high\",\n", + " server_side_type=\"high\",\n", + " dev_mode=True,\n", + " reset=True,\n", + " n_consumers=1,\n", + " create_producer=True,\n", + " port=high_port,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "email_server_low, smtp_server_low = get_email_server(reset=True, server_side_type=\"low\")\n", + "email_server_high, smtp_server_high = get_email_server(\n", + " reset=True, server_side_type=\"high\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "root_client_low = sy.login(\n", + " url=f\"http://localhost:{low_port}\", email=ROOT_EMAIL, password=ROOT_PASSWORD\n", + ")\n", + "root_client_high = sy.login(\n", + " url=f\"http://localhost:{high_port}\", email=ROOT_EMAIL, password=ROOT_PASSWORD\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Create new admin client" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ADMIN_EMAIL, ADMIN_PW = \"admin2@bigquery.org\", \"bqpw2\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# create admin account on low side\n", + "root_client_low.register(\n", + " name=\"second admin\", email=ADMIN_EMAIL, password=ADMIN_PW, password_verify=ADMIN_PW\n", + ")\n", + "# update role\n", + "new_user_id = root_client_low.users.search(email=ADMIN_EMAIL)[0].id\n", + "root_client_low.users.update(uid=new_user_id, role=\"admin\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# create admin account on high side\n", + "root_client_high.register(\n", + " name=\"second admin\", email=ADMIN_EMAIL, password=ADMIN_PW, password_verify=ADMIN_PW\n", + ")\n", + "# update role\n", + "new_user_id = root_client_high.users.search(email=ADMIN_EMAIL)[0].id\n", + "root_client_high.users.update(uid=new_user_id, role=\"admin\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# We cannot delete the root client" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "low_client = sy.login(\n", + " url=f\"http://localhost:{low_port}\", email=ADMIN_EMAIL, password=ADMIN_PW\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "root_admin_id = low_client.users.search(email=ROOT_EMAIL)[0].id" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "with sy.raises(sy.SyftException):\n", + " low_client.users.delete(root_admin_id)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Create ephemeral admin and delete it" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# register 2nd new admin (to delete)\n", + "user_email, user_pw = \"admin3@bigquery.org\", \"bqpw3\"\n", + "## create\n", + "root_client_low.register(\n", + " name=\"x\", email=user_email, password=user_pw, password_verify=user_pw\n", + ")\n", + "## update role\n", + "new_user_id2 = root_client_low.users.search(email=user_email)[0].id\n", + "root_client_low.users.update(uid=new_user_id, role=\"admin\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "root_client_low.users.delete(new_user_id2)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Cleanup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if environment != \"remote\":\n", + " server_high.land()\n", + " server_low.land()\n", + "smtp_server_low.stop()\n", + "smtp_server_high.stop()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/notebooks/scenarios/bigquery/sync/001-scale-delete-worker-pools.ipynb b/notebooks/scenarios/bigquery/sync/001-scale-delete-worker-pools.ipynb index ac6f4a1a395..2c1318000ae 100644 --- a/notebooks/scenarios/bigquery/sync/001-scale-delete-worker-pools.ipynb +++ b/notebooks/scenarios/bigquery/sync/001-scale-delete-worker-pools.ipynb @@ -63,9 +63,8 @@ "source": [ "num_workers = int(os.environ.get(\"NUM_TEST_WORKERS\", 1))\n", "\n", - "# ROOT_EMAIL = \"admin@bigquery.org\"\n", - "# ROOT_PASSWORD = \"bqpw\"\n", - "environment" + "ROOT_EMAIL = \"admin@bigquery.org\"\n", + "ROOT_PASSWORD = \"bqpw\"" ] }, { @@ -113,7 +112,9 @@ "metadata": {}, "outputs": [], "source": [ - "low_client = server_low.login(email=\"info@openmined.org\", password=\"changethis\")" + "low_client = sy.login(\n", + " url=f\"http://localhost:{low_port}\", email=ROOT_EMAIL, password=ROOT_PASSWORD\n", + ")" ] }, { diff --git a/notebooks/scenarios/bigquery/sync/000-setup-high-low-datasites.ipynb b/notebooks/scenarios/bigquery/sync/010-setup-bigquery-pool.ipynb similarity index 96% rename from notebooks/scenarios/bigquery/sync/000-setup-high-low-datasites.ipynb rename to notebooks/scenarios/bigquery/sync/010-setup-bigquery-pool.ipynb index 1b07c9476c0..a072fa5275b 100644 --- a/notebooks/scenarios/bigquery/sync/000-setup-high-low-datasites.ipynb +++ b/notebooks/scenarios/bigquery/sync/010-setup-bigquery-pool.ipynb @@ -113,8 +113,13 @@ "metadata": {}, "outputs": [], "source": [ - "low_client = server_low.login(email=\"info@openmined.org\", password=\"changethis\")\n", - "high_client = server_high.login(email=\"info@openmined.org\", password=\"changethis\")" + "ADMIN_EMAIL, ADMIN_PW = \"admin2@bigquery.org\", \"bqpw2\"\n", + "low_client = sy.login(\n", + " url=f\"http://localhost:{low_port}\", email=ADMIN_EMAIL, password=ADMIN_PW\n", + ")\n", + "high_client = sy.login(\n", + " url=f\"http://localhost:{high_port}\", email=ADMIN_EMAIL, password=ADMIN_PW\n", + ")" ] }, { diff --git a/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb b/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb index 1cb7887fe4b..b1000840e2f 100644 --- a/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb +++ b/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb @@ -42,6 +42,7 @@ "metadata": {}, "outputs": [], "source": [ + "ADMIN_EMAIL, ADMIN_PW = \"admin2@bigquery.org\", \"bqpw2\"\n", "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", "high_port = os.environ.get(\"CLUSTER_HTTP_PORT_HIGH\", \"auto\")\n", "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"auto\")\n", @@ -108,8 +109,12 @@ "metadata": {}, "outputs": [], "source": [ - "low_client = server_low.login(email=\"info@openmined.org\", password=\"changethis\")\n", - "high_client = server_high.login(email=\"info@openmined.org\", password=\"changethis\")" + "low_client = sy.login(\n", + " url=f\"http://localhost:{low_port}\", email=ADMIN_EMAIL, password=ADMIN_PW\n", + ")\n", + "high_client = sy.login(\n", + " url=f\"http://localhost:{high_port}\", email=ADMIN_EMAIL, password=ADMIN_PW\n", + ")" ] }, { diff --git a/notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb b/notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb index ed6bfcb0399..f6b31c4aa0f 100644 --- a/notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb +++ b/notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb @@ -23,7 +23,7 @@ "outputs": [], "source": [ "# stdlib\n", - "\n", + "ADMIN_EMAIL, ADMIN_PW = \"admin2@bigquery.org\", \"bqpw2\"\n", "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", "high_port = os.environ.get(\"CLUSTER_HTTP_PORT_HIGH\", \"9081\")\n", "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"9083\")\n", @@ -109,8 +109,12 @@ "metadata": {}, "outputs": [], "source": [ - "low_client = server_low.login(email=\"info@openmined.org\", password=\"changethis\")\n", - "high_client = server_high.login(email=\"info@openmined.org\", password=\"changethis\")" + "low_client = sy.login(\n", + " url=f\"http://localhost:{low_port}\", email=ADMIN_EMAIL, password=ADMIN_PW\n", + ")\n", + "high_client = sy.login(\n", + " url=f\"http://localhost:{high_port}\", email=ADMIN_EMAIL, password=ADMIN_PW\n", + ")" ] }, { diff --git a/notebooks/scenarios/bigquery/sync/021-create-jobs.ipynb b/notebooks/scenarios/bigquery/sync/021-create-jobs.ipynb index f63763148f2..6851e25b2c2 100644 --- a/notebooks/scenarios/bigquery/sync/021-create-jobs.ipynb +++ b/notebooks/scenarios/bigquery/sync/021-create-jobs.ipynb @@ -25,7 +25,7 @@ "outputs": [], "source": [ "# stdlib\n", - "\n", + "ADMIN_EMAIL, ADMIN_PW = \"admin2@bigquery.org\", \"bqpw2\"\n", "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"9083\")\n", "print(environment, low_port)" @@ -47,19 +47,9 @@ "from syft.util.test_helpers.email_helpers import load_users" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "3", - "metadata": {}, - "outputs": [], - "source": [ - "ADMIN_EMAIL, ADMIN_PW = \"info@openmined.org\", \"changethis\"" - ] - }, { "cell_type": "markdown", - "id": "4", + "id": "3", "metadata": {}, "source": [ "# Launch server & login" @@ -68,7 +58,7 @@ { "cell_type": "code", "execution_count": null, - "id": "5", + "id": "4", "metadata": {}, "outputs": [], "source": [ @@ -76,7 +66,7 @@ " name=\"bigquery-low\",\n", " server_side_type=\"low\",\n", " dev_mode=True,\n", - " n_consumers=1,\n", + " n_consumers=4,\n", " create_producer=True,\n", " port=low_port,\n", ")" @@ -85,7 +75,7 @@ { "cell_type": "code", "execution_count": null, - "id": "6", + "id": "5", "metadata": {}, "outputs": [], "source": [ @@ -95,17 +85,19 @@ { "cell_type": "code", "execution_count": null, - "id": "7", + "id": "6", "metadata": {}, "outputs": [], "source": [ - "low_client = server_low.login(email=ADMIN_EMAIL, password=ADMIN_PW)" + "low_client = sy.login(\n", + " url=f\"http://localhost:{low_port}\", email=ADMIN_EMAIL, password=ADMIN_PW\n", + ")" ] }, { "cell_type": "code", "execution_count": null, - "id": "8", + "id": "7", "metadata": {}, "outputs": [], "source": [ @@ -114,7 +106,7 @@ }, { "cell_type": "markdown", - "id": "9", + "id": "8", "metadata": {}, "source": [ "# Create jobs" @@ -123,7 +115,7 @@ { "cell_type": "code", "execution_count": null, - "id": "10", + "id": "9", "metadata": {}, "outputs": [], "source": [ @@ -135,7 +127,7 @@ }, { "cell_type": "markdown", - "id": "11", + "id": "10", "metadata": {}, "source": [ "# Inspect job data (requests for these jobs to be created)" @@ -144,7 +136,7 @@ { "cell_type": "code", "execution_count": null, - "id": "12", + "id": "11", "metadata": {}, "outputs": [], "source": [ @@ -156,7 +148,7 @@ { "cell_type": "code", "execution_count": null, - "id": "13", + "id": "12", "metadata": {}, "outputs": [], "source": [ @@ -168,7 +160,7 @@ { "cell_type": "code", "execution_count": null, - "id": "14", + "id": "13", "metadata": {}, "outputs": [], "source": [ @@ -179,7 +171,7 @@ { "cell_type": "code", "execution_count": null, - "id": "15", + "id": "14", "metadata": {}, "outputs": [], "source": [ @@ -192,7 +184,7 @@ { "cell_type": "code", "execution_count": null, - "id": "16", + "id": "15", "metadata": {}, "outputs": [], "source": [ @@ -204,7 +196,7 @@ { "cell_type": "code", "execution_count": null, - "id": "17", + "id": "16", "metadata": {}, "outputs": [], "source": [ @@ -213,7 +205,7 @@ }, { "cell_type": "markdown", - "id": "18", + "id": "17", "metadata": {}, "source": [ "# Submit jobs\n" @@ -222,7 +214,7 @@ { "cell_type": "code", "execution_count": null, - "id": "19", + "id": "18", "metadata": {}, "outputs": [], "source": [ @@ -232,7 +224,7 @@ }, { "cell_type": "markdown", - "id": "20", + "id": "19", "metadata": {}, "source": [ "## Test Succesful jobs" @@ -241,7 +233,7 @@ { "cell_type": "code", "execution_count": null, - "id": "21", + "id": "20", "metadata": {}, "outputs": [], "source": [ @@ -251,7 +243,7 @@ { "cell_type": "code", "execution_count": null, - "id": "22", + "id": "21", "metadata": {}, "outputs": [], "source": [ @@ -265,7 +257,7 @@ }, { "cell_type": "markdown", - "id": "23", + "id": "22", "metadata": {}, "source": [ "## Test failures" @@ -274,7 +266,7 @@ { "cell_type": "code", "execution_count": null, - "id": "24", + "id": "23", "metadata": {}, "outputs": [], "source": [ @@ -284,7 +276,7 @@ { "cell_type": "code", "execution_count": null, - "id": "25", + "id": "24", "metadata": {}, "outputs": [], "source": [ @@ -300,7 +292,7 @@ { "cell_type": "code", "execution_count": null, - "id": "26", + "id": "25", "metadata": {}, "outputs": [], "source": [ @@ -313,7 +305,7 @@ { "cell_type": "code", "execution_count": null, - "id": "27", + "id": "26", "metadata": {}, "outputs": [], "source": [ @@ -322,7 +314,7 @@ }, { "cell_type": "markdown", - "id": "28", + "id": "27", "metadata": {}, "source": [ "## Test: cannot execute submitted jobs yet" @@ -331,7 +323,7 @@ { "cell_type": "code", "execution_count": null, - "id": "29", + "id": "28", "metadata": {}, "outputs": [], "source": [ @@ -343,7 +335,7 @@ { "cell_type": "code", "execution_count": null, - "id": "30", + "id": "29", "metadata": {}, "outputs": [], "source": [ @@ -358,7 +350,7 @@ { "cell_type": "code", "execution_count": null, - "id": "31", + "id": "30", "metadata": {}, "outputs": [], "source": [ @@ -375,7 +367,7 @@ }, { "cell_type": "markdown", - "id": "32", + "id": "31", "metadata": {}, "source": [ "# Verify that admin has emails for submitted requests" @@ -384,7 +376,7 @@ { "cell_type": "code", "execution_count": null, - "id": "33", + "id": "32", "metadata": {}, "outputs": [], "source": [ @@ -398,7 +390,7 @@ }, { "cell_type": "markdown", - "id": "34", + "id": "33", "metadata": {}, "source": [ "# Cleanup" @@ -407,7 +399,7 @@ { "cell_type": "code", "execution_count": null, - "id": "35", + "id": "34", "metadata": {}, "outputs": [], "source": [ diff --git a/notebooks/scenarios/bigquery/sync/030-ds-submit-request.ipynb b/notebooks/scenarios/bigquery/sync/030-ds-submit-request.ipynb index 2d8ed894d40..f594713532b 100644 --- a/notebooks/scenarios/bigquery/sync/030-ds-submit-request.ipynb +++ b/notebooks/scenarios/bigquery/sync/030-ds-submit-request.ipynb @@ -23,7 +23,6 @@ "outputs": [], "source": [ "# stdlib\n", - "\n", "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"9083\")\n", "print(environment, low_port)" @@ -82,8 +81,10 @@ "metadata": {}, "outputs": [], "source": [ - "ds_client = server_low.login(\n", - " email=\"data_scientist@openmined.org\", password=\"verysecurepassword\"\n", + "ds_client = sy.login(\n", + " url=f\"http://localhost:{low_port}\",\n", + " email=\"data_scientist@openmined.org\",\n", + " password=\"verysecurepassword\",\n", ")" ] }, diff --git a/notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb b/notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb index 332d71aeb26..6c178783d68 100644 --- a/notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb +++ b/notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb @@ -40,6 +40,7 @@ "metadata": {}, "outputs": [], "source": [ + "ADMIN_EMAIL, ADMIN_PW = \"admin2@bigquery.org\", \"bqpw2\"\n", "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", "high_port = os.environ.get(\"CLUSTER_HTTP_PORT_HIGH\", \"9081\")\n", "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"9083\")" @@ -93,8 +94,12 @@ "metadata": {}, "outputs": [], "source": [ - "low_client = server_low.login(email=\"info@openmined.org\", password=\"changethis\")\n", - "high_client = server_high.login(email=\"info@openmined.org\", password=\"changethis\")" + "low_client = sy.login(\n", + " url=f\"http://localhost:{low_port}\", email=ADMIN_EMAIL, password=ADMIN_PW\n", + ")\n", + "high_client = sy.login(\n", + " url=f\"http://localhost:{high_port}\", email=ADMIN_EMAIL, password=ADMIN_PW\n", + ")" ] }, { diff --git a/notebooks/scenarios/bigquery/sync/050-ds-get-results.ipynb b/notebooks/scenarios/bigquery/sync/050-ds-get-results.ipynb index 6169fafb81b..8fbb16c8bbc 100644 --- a/notebooks/scenarios/bigquery/sync/050-ds-get-results.ipynb +++ b/notebooks/scenarios/bigquery/sync/050-ds-get-results.ipynb @@ -81,8 +81,10 @@ "metadata": {}, "outputs": [], "source": [ - "ds_client = server_low.login(\n", - " email=\"data_scientist@openmined.org\", password=\"verysecurepassword\"\n", + "ds_client = sy.login(\n", + " url=f\"http://localhost:{low_port}\",\n", + " email=\"data_scientist@openmined.org\",\n", + " password=\"verysecurepassword\",\n", ")" ] }, From f2e1f29bfbe97e65456ac48ee148191e1952b24c Mon Sep 17 00:00:00 2001 From: Brendan Schell Date: Fri, 20 Sep 2024 16:04:59 -0400 Subject: [PATCH 06/28] some adjustments to event loop handling in email_helpers --- .../syft/util/test_helpers/email_helpers.py | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/packages/syft/src/syft/util/test_helpers/email_helpers.py b/packages/syft/src/syft/util/test_helpers/email_helpers.py index cc6ab7a2839..982120f3bef 100644 --- a/packages/syft/src/syft/util/test_helpers/email_helpers.py +++ b/packages/syft/src/syft/util/test_helpers/email_helpers.py @@ -218,7 +218,7 @@ def user_exists(root_client, email: str) -> bool: class SMTPTestServer: - def __init__(self, email_server, port=9025, ready_timeout=20): + def __init__(self, email_server, port=9025, ready_timeout=5): self.port = port self.hostname = "0.0.0.0" self._stop_event = asyncio.Event() @@ -261,11 +261,11 @@ async def async_loop(self): try: print(f"> Starting SMTPTestServer on: {self.hostname}:{self.port}") self.controller.start() - await ( - self._stop_event.wait() - ) # Wait until the event is set to stop the server except Exception as e: print(f"> Error with SMTPTestServer: {e}") + await ( + self._stop_event.wait() + ) # Wait until the event is set to stop the server def stop(self): try: @@ -279,8 +279,12 @@ def stop(self): print(f"> Error stopping SMTPTestServer: {e}") async def async_stop(self): - self.controller.stop() - self._stop_event.set() # Stop the server by setting the event + try: + self.controller.stop() + except Exception as e: + print(f"> Error stopping SMTPTestServer: {e}") + finally: + self._stop_event.set() # Stop the server by setting the event class TimeoutError(Exception): @@ -332,7 +336,16 @@ def get_email_server(reset=False, server_side_type=""): if reset: email_server.reset_emails() smtp_server = SMTPTestServer(email_server, port=port) - smtp_server.start() + try: + loop = asyncio.get_running_loop() + except RuntimeError: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + loop.run_until_complete(smtp_server.start()) + smtp_server.start() + else: + smtp_server.start() + return email_server, smtp_server From 586a797ed7add318b86678e64b4039f015e47be0 Mon Sep 17 00:00:00 2001 From: Brendan Schell Date: Fri, 20 Sep 2024 18:03:29 -0400 Subject: [PATCH 07/28] a bunch of small fixes and adding multiple jobs to l0 --- ...tart-and-configure-server-and-admins.ipynb | 14 +- .../sync/001-scale-delete-worker-pools.ipynb | 2 +- .../sync/010-setup-bigquery-pool.ipynb | 10 +- .../sync/011-users-emails-passwords.ipynb | 4 +- .../sync/020-configure-api-and-sync.ipynb | 12 +- .../bigquery/sync/021-create-jobs.ipynb | 16 +- .../bigquery/sync/030-ds-submit-request.ipynb | 378 ------------------ .../sync/040-do-review-requests.ipynb | 170 ++++---- .../bigquery/sync/050-ds-get-results.ipynb | 87 +++- .../syft/util/test_helpers/email_helpers.py | 6 +- 10 files changed, 188 insertions(+), 511 deletions(-) delete mode 100644 notebooks/scenarios/bigquery/sync/030-ds-submit-request.ipynb diff --git a/notebooks/scenarios/bigquery/sync/000-start-and-configure-server-and-admins.ipynb b/notebooks/scenarios/bigquery/sync/000-start-and-configure-server-and-admins.ipynb index 8bb1fb9275c..6d026212964 100644 --- a/notebooks/scenarios/bigquery/sync/000-start-and-configure-server-and-admins.ipynb +++ b/notebooks/scenarios/bigquery/sync/000-start-and-configure-server-and-admins.ipynb @@ -229,15 +229,9 @@ ")\n", "## update role\n", "new_user_id2 = root_client_low.users.search(email=user_email)[0].id\n", - "root_client_low.users.update(uid=new_user_id, role=\"admin\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ + "root_client_low.users.update(uid=new_user_id2, role=\"admin\")\n", + "\n", + "# delete\n", "root_client_low.users.delete(new_user_id2)" ] }, @@ -278,7 +272,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.3" + "version": "3.12.4" } }, "nbformat": 4, diff --git a/notebooks/scenarios/bigquery/sync/001-scale-delete-worker-pools.ipynb b/notebooks/scenarios/bigquery/sync/001-scale-delete-worker-pools.ipynb index 2c1318000ae..6fc2ac44c74 100644 --- a/notebooks/scenarios/bigquery/sync/001-scale-delete-worker-pools.ipynb +++ b/notebooks/scenarios/bigquery/sync/001-scale-delete-worker-pools.ipynb @@ -124,7 +124,7 @@ "metadata": {}, "outputs": [], "source": [ - "assert len(low_client.worker_pools.get_all()) == 2" + "assert len(low_client.worker_pools.get_all()) == 1" ] }, { diff --git a/notebooks/scenarios/bigquery/sync/010-setup-bigquery-pool.ipynb b/notebooks/scenarios/bigquery/sync/010-setup-bigquery-pool.ipynb index a072fa5275b..b1ac05bf3f0 100644 --- a/notebooks/scenarios/bigquery/sync/010-setup-bigquery-pool.ipynb +++ b/notebooks/scenarios/bigquery/sync/010-setup-bigquery-pool.ipynb @@ -71,7 +71,6 @@ " name=\"bigquery-low\",\n", " server_side_type=\"low\",\n", " dev_mode=True,\n", - " reset=True,\n", " n_consumers=1,\n", " create_producer=True,\n", " port=low_port,\n", @@ -81,7 +80,6 @@ " name=\"bigquery-high\",\n", " server_side_type=\"high\",\n", " dev_mode=True,\n", - " reset=True,\n", " n_consumers=1,\n", " create_producer=True,\n", " port=high_port,\n", @@ -284,11 +282,11 @@ "outputs": [], "source": [ "assert (\n", - " len(low_client.api.services.user.get_all()) == 2\n", - "), \"Only DS and Admin should be at low side\"\n", + " len(low_client.api.services.user.get_all()) == 3\n", + "), \"Only DS and 2 Admin should be at low side\"\n", "assert (\n", - " len(high_client.api.services.user.get_all()) == 1\n", - "), \"Only Admin should be at high side\"" + " len(high_client.api.services.user.get_all()) == 2\n", + "), \"Only 2 Admin should be at high side\"" ] }, { diff --git a/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb b/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb index b1000840e2f..a31539f6638 100644 --- a/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb +++ b/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb @@ -44,8 +44,8 @@ "source": [ "ADMIN_EMAIL, ADMIN_PW = \"admin2@bigquery.org\", \"bqpw2\"\n", "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", - "high_port = os.environ.get(\"CLUSTER_HTTP_PORT_HIGH\", \"auto\")\n", - "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"auto\")\n", + "high_port = os.environ.get(\"CLUSTER_HTTP_PORT_HIGH\", \"9081\")\n", + "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"9083\")\n", "print(environment, high_port, low_port)" ] }, diff --git a/notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb b/notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb index f6b31c4aa0f..cc3d8468ba3 100644 --- a/notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb +++ b/notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb @@ -24,6 +24,7 @@ "source": [ "# stdlib\n", "ADMIN_EMAIL, ADMIN_PW = \"admin2@bigquery.org\", \"bqpw2\"\n", + "ROOT_EMAIL, ROOT_PW = \"admin@bigquery.org\", \"bqpw\"\n", "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", "high_port = os.environ.get(\"CLUSTER_HTTP_PORT_HIGH\", \"9081\")\n", "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"9083\")\n", @@ -615,7 +616,8 @@ "outputs": [], "source": [ "# email_server_low.get_emails_for_user(user_email=\"info@openmined.org\")\n", - "assert len(email_server_high.get_emails_for_user(user_email=\"info@openmined.org\")) == 2" + "assert len(email_server_high.get_emails_for_user(user_email=ADMIN_EMAIL)) == 1\n", + "assert len(email_server_high.get_emails_for_user(user_email=ROOT_EMAIL)) == 1" ] }, { @@ -626,9 +628,7 @@ "source": [ "assert (\n", " \"Job Failed\"\n", - " in email_server_high.get_emails_for_user(user_email=\"info@openmined.org\")[\n", - " 0\n", - " ].email_content\n", + " in email_server_high.get_emails_for_user(user_email=ADMIN_EMAIL)[0].email_content\n", ")" ] }, @@ -640,9 +640,7 @@ "source": [ "assert (\n", " \"A new request has been submitted and requires your attention\"\n", - " in email_server_high.get_emails_for_user(user_email=\"info@openmined.org\")[\n", - " 1\n", - " ].email_content\n", + " in email_server_high.get_emails_for_user(user_email=ROOT_EMAIL)[0].email_content\n", ")" ] }, diff --git a/notebooks/scenarios/bigquery/sync/021-create-jobs.ipynb b/notebooks/scenarios/bigquery/sync/021-create-jobs.ipynb index 6851e25b2c2..46289543aa2 100644 --- a/notebooks/scenarios/bigquery/sync/021-create-jobs.ipynb +++ b/notebooks/scenarios/bigquery/sync/021-create-jobs.ipynb @@ -26,6 +26,7 @@ "source": [ "# stdlib\n", "ADMIN_EMAIL, ADMIN_PW = \"admin2@bigquery.org\", \"bqpw2\"\n", + "ROOT_EMAIL, ROOT_PASSWORD = \"admin@bigquery.org\", \"bqpw\"\n", "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"9083\")\n", "print(environment, low_port)" @@ -357,12 +358,15 @@ "# currently errors out with\n", "# syft.types.errors.SyftException: Please wait for the admin to allow the execution of this code\n", "\n", - "\"\"\" for fn in job_execution_fns:\n", + "for fn in job_execution_fns:\n", " # nonblocking\n", - " result_job = fn(blocking=False)\n", - " result_job.wait()\n", - " assert isinstance(result_job.result, sy.SyftError)\n", - " assert result_job.status == JobStatus.ERRORED \"\"\"" + " with sy.raises(\n", + " sy.SyftException(\n", + " public_message=\"*Please wait for the admin to allow the execution of this code*\"\n", + " )\n", + " ):\n", + " result_job = fn(blocking=False)\n", + " result_job.wait()" ] }, { @@ -381,7 +385,7 @@ "outputs": [], "source": [ "num_should_submit = sum(j.should_submit for j in jobs_data)\n", - "admin_emails_after = len(email_server_low.get_emails_for_user(\"admin@bigquery.org\"))\n", + "admin_emails_after = len(email_server_low.get_emails_for_user(ROOT_EMAIL))\n", "print(\"admin emails after\", admin_emails_after)\n", "assert admin_emails_after >= admin_emails_before + num_should_submit\n", "# assert len(users_emails) > after_number_of_emails\n", diff --git a/notebooks/scenarios/bigquery/sync/030-ds-submit-request.ipynb b/notebooks/scenarios/bigquery/sync/030-ds-submit-request.ipynb deleted file mode 100644 index f594713532b..00000000000 --- a/notebooks/scenarios/bigquery/sync/030-ds-submit-request.ipynb +++ /dev/null @@ -1,378 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# stdlib\n", - "import os\n", - "\n", - "# os.environ[\"ORCHESTRA_DEPLOYMENT_TYPE\"] = \"remote\"\n", - "# os.environ[\"DEV_MODE\"] = \"True\"\n", - "# os.environ[\"TEST_EXTERNAL_REGISTRY\"] = \"k3d-registry.localhost:5800\"\n", - "# os.environ[\"CLUSTER_HTTP_PORT_HIGH\"] = \"9081\"\n", - "# os.environ[\"CLUSTER_HTTP_PORT_LOW\"] = \"9083\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# stdlib\n", - "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", - "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"9083\")\n", - "print(environment, low_port)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# third party\n", - "import pandas as pd\n", - "\n", - "# syft absolute\n", - "import syft as sy\n", - "from syft import test_settings\n", - "from syft.util.test_helpers.email_helpers import get_email_server" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Only low side server and login as DS" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "server_low = sy.orchestra.launch(\n", - " name=\"bigquery-low\",\n", - " server_side_type=\"low\",\n", - " dev_mode=True,\n", - " n_consumers=1,\n", - " create_producer=True,\n", - " port=low_port,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "email_server_low, smtp_server_low = get_email_server(server_side_type=\"low\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ds_client = sy.login(\n", - " url=f\"http://localhost:{low_port}\",\n", - " email=\"data_scientist@openmined.org\",\n", - " password=\"verysecurepassword\",\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Low side research" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Find available endpoints\n", - "ds_client.custom_api.api_endpoints()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert len(ds_client.custom_api.api_endpoints()) == 3" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Test the schema" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "res = ds_client.api.services.bigquery.schema()\n", - "res.get()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert isinstance(res.get(), pd.DataFrame)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Load these variables for testing but they ideally come from the schema\n", - "dataset_1 = test_settings.get(\"dataset_1\", default=\"dataset_1\")\n", - "dataset_2 = test_settings.get(\"dataset_2\", default=\"dataset_2\")\n", - "table_1 = test_settings.get(\"table_1\", default=\"table_1\")\n", - "table_2 = test_settings.get(\"table_2\", default=\"table_2\")\n", - "table_2_col_id = test_settings.get(\"table_2_col_id\", default=\"table_id\")\n", - "table_2_col_score = test_settings.get(\"table_2_col_score\", default=\"colname\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "____" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Test the `bigquery.test_query` endpoint" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "result = ds_client.api.services.bigquery.test_query.mock(\n", - " sql_query=f\"SELECT * from {dataset_2}.{table_2} limit 10\"\n", - ")\n", - "assert len(result) == 10" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "with sy.raises(sy.SyftException, show=True):\n", - " ds_client.api.services.bigquery.test_query.private(\n", - " sql_query=f\"SELECT * from {dataset_2}.{table_2} limit 10\"\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "LARGE_SAMPLE_QUERY = f\"SELECT * FROM {dataset_2}.{table_2} LIMIT 10000\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "mock_res = ds_client.api.services.bigquery.test_query(sql_query=LARGE_SAMPLE_QUERY)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert len(mock_res) == 10000" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Submit a query" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "FUNC_NAME = \"large_sample\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "submission = ds_client.api.services.bigquery.submit_query(\n", - " func_name=FUNC_NAME, query=LARGE_SAMPLE_QUERY\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "submission" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# todo: this is very noisy, but it actually passes\n", - "with sy.raises(\n", - " sy.SyftException(\n", - " public_message=\"*Please wait for the admin to allow the execution of this code*\"\n", - " )\n", - "):\n", - " ds_client.code.large_sample(blocking=False)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "with sy.raises(sy.SyftException(public_message=\"*Your code is waiting for approval*\")):\n", - " ds_client.code.large_sample()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Test emails" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert len(email_server_low.get_emails_for_user(user_email=\"info@openmined.org\")) == 1\n", - "assert (\n", - " len(email_server_low.get_emails_for_user(user_email=ds_client.account.email)) == 1\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert (\n", - " \"A new request has been submitted and requires your attention\"\n", - " in email_server_low.get_emails_for_user(user_email=\"info@openmined.org\")[\n", - " 0\n", - " ].email_content\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert (\n", - " \"We regret to inform you that your function job\"\n", - " in email_server_low.get_emails_for_user(user_email=ds_client.account.email)[\n", - " 0\n", - " ].email_content\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Shutdown" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if environment != \"remote\":\n", - " server_low.land()\n", - "smtp_server_low.stop()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.4" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb b/notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb index 6c178783d68..e6944c0e8d7 100644 --- a/notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb +++ b/notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb @@ -9,6 +9,13 @@ "# stdlib\n", "import os\n", "\n", + "# Testing works over 4 possibilities\n", + "# 1. (python/in-memory workers and using tox commands)\n", + "# 2. (python/in-memory workers and manually running notebooks)\n", + "# 3. (using k8s and using tox commands)\n", + "# 4. (using k8s and manually running notebooks)\n", + "# Uncomment the lines below if in the 4th possibility\n", + "\n", "# os.environ[\"ORCHESTRA_DEPLOYMENT_TYPE\"] = \"remote\"\n", "# os.environ[\"DEV_MODE\"] = \"True\"\n", "# os.environ[\"TEST_EXTERNAL_REGISTRY\"] = \"k3d-registry.localhost:5800\"\n", @@ -23,15 +30,13 @@ "outputs": [], "source": [ "# stdlib\n", - "\n", - "# third party\n", - "import pandas as pd\n", - "\n", - "# syft absolute\n", - "import syft as sy\n", - "from syft import test_helpers # noqa: F401\n", - "from syft.client.syncing import compare_clients\n", - "from syft.util.test_helpers.email_helpers import get_email_server" + "ADMIN_EMAIL, ADMIN_PW = \"admin2@bigquery.org\", \"bqpw2\"\n", + "ROOT_EMAIL, ROOT_PASSWORD = \"admin@bigquery.org\", \"bqpw\"\n", + "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", + "high_port = os.environ.get(\"CLUSTER_HTTP_PORT_HIGH\", \"9081\")\n", + "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"9083\")\n", + "num_jobs = int(os.environ.get(\"NUM_TEST_JOBS\", 10))\n", + "print(environment, low_port)" ] }, { @@ -40,10 +45,19 @@ "metadata": {}, "outputs": [], "source": [ - "ADMIN_EMAIL, ADMIN_PW = \"admin2@bigquery.org\", \"bqpw2\"\n", - "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", - "high_port = os.environ.get(\"CLUSTER_HTTP_PORT_HIGH\", \"9081\")\n", - "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"9083\")" + "# stdlib\n", + "\n", + "# third party\n", + "import pandas as pd\n", + "\n", + "# syft absolute\n", + "import syft as sy\n", + "from syft.client.syncing import compare_clients\n", + "from syft.util.test_helpers.email_helpers import get_email_server\n", + "from syft.util.test_helpers.email_helpers import load_users\n", + "from syft.util.test_helpers.job_helpers import get_job_emails\n", + "from syft.util.test_helpers.job_helpers import get_request_for_job_info\n", + "from syft.util.test_helpers.job_helpers import load_jobs" ] }, { @@ -158,7 +172,7 @@ "metadata": {}, "outputs": [], "source": [ - "assert len(high_client.code.get_all()) == 2" + "len(high_client.code.get_all())" ] }, { @@ -167,8 +181,7 @@ "metadata": {}, "outputs": [], "source": [ - "requests = high_client.requests\n", - "requests" + "assert len(high_client.code.get_all()) == num_jobs" ] }, { @@ -177,29 +190,8 @@ "metadata": {}, "outputs": [], "source": [ - "user_request = None\n", - "for request in requests:\n", - " if \"large_sample\" in getattr(\n", - " getattr(request, \"code\", None), \"service_func_name\", None\n", - " ):\n", - " user_request = request\n", - "assert user_request" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Execute Request" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "job = high_client.code.large_sample(blocking=False)" + "requests = high_client.requests.get_all_pending()\n", + "requests" ] }, { @@ -208,16 +200,23 @@ "metadata": {}, "outputs": [], "source": [ - "res = job.wait().get()\n", - "\n", - "assert isinstance(res, pd.DataFrame)" + "users = load_users(low_client)\n", + "jobs_data = load_jobs(users, low_client)\n", + "all_requests = high_client.requests\n", + "submitted_jobs_data = [job for job in jobs_data if job.is_submitted]\n", + "n_emails_per_job_user = {\n", + " k: len(v)\n", + " for k, v in get_job_emails(\n", + " submitted_jobs_data, high_client, email_server_low\n", + " ).items()\n", + "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "# Sync job result to low side" + "# Run or Deny" ] }, { @@ -226,9 +225,12 @@ "metadata": {}, "outputs": [], "source": [ - "diffs = compare_clients(\n", - " from_client=high_client, to_client=low_client, hide_usercode=False\n", - ")" + "submitted_jobs_data_should_succeed = [\n", + " j for j in submitted_jobs_data if j.should_succeed\n", + "]\n", + "submitted_jobs_data_should_fail = [\n", + " j for j in submitted_jobs_data if not j.should_succeed\n", + "]" ] }, { @@ -237,9 +239,12 @@ "metadata": {}, "outputs": [], "source": [ - "batch_root_strs = [x.root_diff.obj_type.__qualname__ for x in diffs.batches]\n", - "assert len(diffs.batches) == 3\n", - "assert \"Job\" in batch_root_strs" + "for job in submitted_jobs_data_should_succeed:\n", + " request = get_request_for_job_info(all_requests, job)\n", + " j = request.code(blocking=False)\n", + " result = j.wait().get()\n", + " assert isinstance(result, pd.DataFrame)\n", + " job.admin_reviewed = True" ] }, { @@ -248,27 +253,21 @@ "metadata": {}, "outputs": [], "source": [ - "widget = diffs.resolve()\n", - "\n", - "widget._share_all()\n", - "widget._sync_all()" + "for job in submitted_jobs_data_should_fail:\n", + " request = get_request_for_job_info(all_requests, job)\n", + " response = request.deny(\n", + " reason=f\"Your request {job.func_name} looks wrong, try again.\"\n", + " )\n", + " assert isinstance(response, sy.SyftSuccess)\n", + " assert not job.should_succeed\n", + " job.admin_reviwed = True" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "# Check that job was synced to low side" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "requests = low_client.requests\n", - "requests" + "# Sync job result to low side" ] }, { @@ -277,12 +276,9 @@ "metadata": {}, "outputs": [], "source": [ - "user_request = None\n", - "for request in requests:\n", - " if \"large_sample\" in getattr(\n", - " getattr(request, \"code\", None), \"service_func_name\", None\n", - " ):\n", - " user_request = request" + "diffs = compare_clients(\n", + " from_client=high_client, to_client=low_client, hide_usercode=False\n", + ")" ] }, { @@ -291,14 +287,7 @@ "metadata": {}, "outputs": [], "source": [ - "assert \"approved\" in str(user_request.status).lower()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Test email" + "diffs.batches" ] }, { @@ -307,11 +296,14 @@ "metadata": {}, "outputs": [], "source": [ - "# Should this user be notified about approval? Right now it seems that it is not.\n", - "assert (\n", - " len(email_server_low.get_emails_for_user(user_email=\"data_scientist@openmined.org\"))\n", - " == 1\n", - ")" + "batch_root_strs = [x.root_diff.obj_type.__qualname__ for x in diffs.batches]\n", + "# for successful jobs, should be job, request, and usercode.\n", + "# For failed jobs, should be usercode, request only since job not run\n", + "expected_n_batches = 2 * len(submitted_jobs_data_should_fail) + 3 * len(\n", + " submitted_jobs_data_should_succeed\n", + ")\n", + "assert len(diffs.batches) == expected_n_batches\n", + "assert \"Job\" in batch_root_strs" ] }, { @@ -320,7 +312,10 @@ "metadata": {}, "outputs": [], "source": [ - "# Similarly, after syncing, should the admin be notified about anything?" + "widget = diffs.resolve()\n", + "\n", + "widget._share_all()\n", + "widget._sync_all()" ] }, { @@ -352,6 +347,11 @@ } ], "metadata": { + "kernelspec": { + "display_name": "syft", + "language": "python", + "name": "python3" + }, "language_info": { "codemirror_mode": { "name": "ipython", @@ -362,7 +362,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.5" + "version": "3.12.4" } }, "nbformat": 4, diff --git a/notebooks/scenarios/bigquery/sync/050-ds-get-results.ipynb b/notebooks/scenarios/bigquery/sync/050-ds-get-results.ipynb index 8fbb16c8bbc..ed8e788d891 100644 --- a/notebooks/scenarios/bigquery/sync/050-ds-get-results.ipynb +++ b/notebooks/scenarios/bigquery/sync/050-ds-get-results.ipynb @@ -9,6 +9,13 @@ "# stdlib\n", "import os\n", "\n", + "# Testing works over 4 possibilities\n", + "# 1. (python/in-memory workers and using tox commands)\n", + "# 2. (python/in-memory workers and manually running notebooks)\n", + "# 3. (using k8s and using tox commands)\n", + "# 4. (using k8s and manually running notebooks)\n", + "# Uncomment the lines below if in the 4th possibility\n", + "\n", "# os.environ[\"ORCHESTRA_DEPLOYMENT_TYPE\"] = \"remote\"\n", "# os.environ[\"DEV_MODE\"] = \"True\"\n", "# os.environ[\"TEST_EXTERNAL_REGISTRY\"] = \"k3d-registry.localhost:5800\"\n", @@ -25,11 +32,13 @@ "# stdlib\n", "\n", "# third party\n", - "import pandas as pd\n", "\n", "# syft absolute\n", "import syft as sy\n", - "from syft.util.test_helpers.email_helpers import get_email_server" + "from syft.util.test_helpers.email_helpers import get_email_server\n", + "from syft.util.test_helpers.email_helpers import load_users\n", + "from syft.util.test_helpers.job_helpers import load_jobs\n", + "from syft.util.test_helpers.job_helpers import save_jobs" ] }, { @@ -38,6 +47,8 @@ "metadata": {}, "outputs": [], "source": [ + "ADMIN_EMAIL, ADMIN_PW = \"admin2@bigquery.org\", \"bqpw2\"\n", + "ROOT_EMAIL, ROOT_PW = \"admin@bigquery.org\", \"bqpw\"\n", "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"9083\")\n", "print(environment, low_port)" @@ -75,24 +86,48 @@ "email_server_low, smtp_server_low = get_email_server(server_side_type=\"low\")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Download Results" + ] + }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "ds_client = sy.login(\n", - " url=f\"http://localhost:{low_port}\",\n", - " email=\"data_scientist@openmined.org\",\n", - " password=\"verysecurepassword\",\n", + "low_client = sy.login(\n", + " url=f\"http://localhost:{low_port}\", email=ADMIN_EMAIL, password=ADMIN_PW\n", ")" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "users = load_users(low_client)\n", + "jobs = load_jobs(users, low_client)" + ] + }, + { + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ - "# Check result of job on low side" + "# submitted_jobs = [job for job in jobs if job.is_submitted]\n", + "reviewed_jobs = [job for job in jobs if job.admin_reviewed]\n", + "reviewed_jobs_should_succeed = [j for j in reviewed_jobs if j.should_succeed]\n", + "reviewed_jobs_should_fail = [j for j in reviewed_jobs if not j.should_succeed]\n", + "\n", + "print(\n", + " f\"{len(reviewed_jobs)=}, {len(reviewed_jobs_should_succeed)=}, {len(reviewed_jobs_should_fail)=}\"\n", + ")" ] }, { @@ -101,7 +136,17 @@ "metadata": {}, "outputs": [], "source": [ - "job = ds_client.code.large_sample(blocking=False)" + "for job in reviewed_jobs_should_succeed:\n", + " print(f\"> Checking job: {job.job_type} {job.func_name} for user {job.user_email}\")\n", + " api_method = job.code_method\n", + " j = api_method(blocking=False)\n", + " res = j.wait()\n", + "\n", + " if isinstance(res, sy.SyftError):\n", + " raise sy.SyftException(public_message=\"Expected success, got error\")\n", + "\n", + " result = res.get()\n", + " job.result_as_expected = True" ] }, { @@ -110,7 +155,16 @@ "metadata": {}, "outputs": [], "source": [ - "res = job.wait().get()" + "for job in reviewed_jobs_should_fail:\n", + " print(f\"> Checking job: {job.job_type} {job.func_name} for user {job.user_email}\")\n", + " api_method = job.code_method\n", + "\n", + " j = api_method(blocking=False)\n", + " res = j.wait()\n", + " if isinstance(res, sy.SyftError):\n", + " job.result_as_expected = True\n", + " else:\n", + " raise sy.SyftException(public_message=f\"failed, job didnt raise {type(j)}\")" ] }, { @@ -119,7 +173,7 @@ "metadata": {}, "outputs": [], "source": [ - "assert isinstance(res, pd.DataFrame)" + "save_jobs(jobs)" ] }, { @@ -128,7 +182,16 @@ "metadata": {}, "outputs": [], "source": [ - "assert len(res) == 10000" + "expected_jobs = [job for job in jobs if job.result_as_expected]\n", + "print(f\"got expected_jobs: {len(expected_jobs)} == reviewed_jobs: {len(reviewed_jobs)}\")\n", + "assert len(reviewed_jobs) == len(expected_jobs)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Cleanup" ] }, { diff --git a/packages/syft/src/syft/util/test_helpers/email_helpers.py b/packages/syft/src/syft/util/test_helpers/email_helpers.py index 982120f3bef..418d430f062 100644 --- a/packages/syft/src/syft/util/test_helpers/email_helpers.py +++ b/packages/syft/src/syft/util/test_helpers/email_helpers.py @@ -263,9 +263,7 @@ async def async_loop(self): self.controller.start() except Exception as e: print(f"> Error with SMTPTestServer: {e}") - await ( - self._stop_event.wait() - ) # Wait until the event is set to stop the server + await self._stop_event.wait() # Wait until the event is set to stop the server def stop(self): try: @@ -345,7 +343,7 @@ def get_email_server(reset=False, server_side_type=""): smtp_server.start() else: smtp_server.start() - + return email_server, smtp_server From 2df075529ca3be75d10d4480e8093ebe4cad19f6 Mon Sep 17 00:00:00 2001 From: Brendan Schell Date: Mon, 23 Sep 2024 10:54:15 -0400 Subject: [PATCH 08/28] Revert "some adjustments to event loop handling in email_helpers" This reverts commit 403dbeebd7c6a88578783e39261c6ebae0a734e9. --- .../syft/util/test_helpers/email_helpers.py | 25 ++++++------------- 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/packages/syft/src/syft/util/test_helpers/email_helpers.py b/packages/syft/src/syft/util/test_helpers/email_helpers.py index 418d430f062..cc6ab7a2839 100644 --- a/packages/syft/src/syft/util/test_helpers/email_helpers.py +++ b/packages/syft/src/syft/util/test_helpers/email_helpers.py @@ -218,7 +218,7 @@ def user_exists(root_client, email: str) -> bool: class SMTPTestServer: - def __init__(self, email_server, port=9025, ready_timeout=5): + def __init__(self, email_server, port=9025, ready_timeout=20): self.port = port self.hostname = "0.0.0.0" self._stop_event = asyncio.Event() @@ -261,9 +261,11 @@ async def async_loop(self): try: print(f"> Starting SMTPTestServer on: {self.hostname}:{self.port}") self.controller.start() + await ( + self._stop_event.wait() + ) # Wait until the event is set to stop the server except Exception as e: print(f"> Error with SMTPTestServer: {e}") - await self._stop_event.wait() # Wait until the event is set to stop the server def stop(self): try: @@ -277,12 +279,8 @@ def stop(self): print(f"> Error stopping SMTPTestServer: {e}") async def async_stop(self): - try: - self.controller.stop() - except Exception as e: - print(f"> Error stopping SMTPTestServer: {e}") - finally: - self._stop_event.set() # Stop the server by setting the event + self.controller.stop() + self._stop_event.set() # Stop the server by setting the event class TimeoutError(Exception): @@ -334,16 +332,7 @@ def get_email_server(reset=False, server_side_type=""): if reset: email_server.reset_emails() smtp_server = SMTPTestServer(email_server, port=port) - try: - loop = asyncio.get_running_loop() - except RuntimeError: - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - loop.run_until_complete(smtp_server.start()) - smtp_server.start() - else: - smtp_server.start() - + smtp_server.start() return email_server, smtp_server From 600cf6ebdfb80c89e57905eae1d8ead19ba02da5 Mon Sep 17 00:00:00 2001 From: Brendan Schell Date: Mon, 23 Sep 2024 11:39:57 -0400 Subject: [PATCH 09/28] revert to single smtp server and some fixes Co-authored-by: Sameer Wagh --- ...tart-and-configure-server-and-admins.ipynb | 8 +-- .../sync/001-scale-delete-worker-pools.ipynb | 62 +++++++------------ .../sync/010-setup-bigquery-pool.ipynb | 34 +--------- .../sync/011-users-emails-passwords.ipynb | 18 ++++-- .../sync/020-configure-api-and-sync.ipynb | 14 ++--- .../bigquery/sync/021-create-jobs.ipynb | 8 +-- .../sync/040-do-review-requests.ipynb | 10 +-- .../bigquery/sync/050-ds-get-results.ipynb | 13 +--- .../syft/util/test_helpers/email_helpers.py | 14 +---- 9 files changed, 54 insertions(+), 127 deletions(-) diff --git a/notebooks/scenarios/bigquery/sync/000-start-and-configure-server-and-admins.ipynb b/notebooks/scenarios/bigquery/sync/000-start-and-configure-server-and-admins.ipynb index 6d026212964..c0f559d0264 100644 --- a/notebooks/scenarios/bigquery/sync/000-start-and-configure-server-and-admins.ipynb +++ b/notebooks/scenarios/bigquery/sync/000-start-and-configure-server-and-admins.ipynb @@ -105,10 +105,7 @@ "metadata": {}, "outputs": [], "source": [ - "email_server_low, smtp_server_low = get_email_server(reset=True, server_side_type=\"low\")\n", - "email_server_high, smtp_server_high = get_email_server(\n", - " reset=True, server_side_type=\"high\"\n", - ")" + "email_server, smtp_server = get_email_server(reset=True)" ] }, { @@ -251,8 +248,7 @@ "if environment != \"remote\":\n", " server_high.land()\n", " server_low.land()\n", - "smtp_server_low.stop()\n", - "smtp_server_high.stop()" + "smtp_server.stop()" ] } ], diff --git a/notebooks/scenarios/bigquery/sync/001-scale-delete-worker-pools.ipynb b/notebooks/scenarios/bigquery/sync/001-scale-delete-worker-pools.ipynb index 6fc2ac44c74..9debc505868 100644 --- a/notebooks/scenarios/bigquery/sync/001-scale-delete-worker-pools.ipynb +++ b/notebooks/scenarios/bigquery/sync/001-scale-delete-worker-pools.ipynb @@ -50,8 +50,7 @@ "\n", "# syft absolute\n", "import syft as sy\n", - "from syft.util.test_helpers.email_helpers import Timeout\n", - "from syft.util.test_helpers.email_helpers import get_email_server" + "from syft.util.test_helpers.email_helpers import Timeout" ] }, { @@ -81,19 +80,6 @@ "id": "5", "metadata": {}, "outputs": [], - "source": [ - "email_server_low, smtp_server_low = get_email_server(server_side_type=\"low\", reset=True)\n", - "email_server_high, smtp_server_high = get_email_server(\n", - " server_side_type=\"high\", reset=True\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6", - "metadata": {}, - "outputs": [], "source": [ "server_low = sy.orchestra.launch(\n", " name=\"bigquery-low\",\n", @@ -108,7 +94,7 @@ { "cell_type": "code", "execution_count": null, - "id": "7", + "id": "6", "metadata": {}, "outputs": [], "source": [ @@ -120,7 +106,7 @@ { "cell_type": "code", "execution_count": null, - "id": "8", + "id": "7", "metadata": {}, "outputs": [], "source": [ @@ -130,7 +116,7 @@ { "cell_type": "code", "execution_count": null, - "id": "9", + "id": "8", "metadata": {}, "outputs": [], "source": [ @@ -140,7 +126,7 @@ }, { "cell_type": "markdown", - "id": "10", + "id": "9", "metadata": {}, "source": [ "### Scale Worker pool" @@ -148,7 +134,7 @@ }, { "cell_type": "markdown", - "id": "11", + "id": "10", "metadata": {}, "source": [ "##### Scale up" @@ -157,7 +143,7 @@ { "cell_type": "code", "execution_count": null, - "id": "12", + "id": "11", "metadata": {}, "outputs": [], "source": [ @@ -171,7 +157,7 @@ { "cell_type": "code", "execution_count": null, - "id": "13", + "id": "12", "metadata": {}, "outputs": [], "source": [ @@ -181,7 +167,7 @@ { "cell_type": "code", "execution_count": null, - "id": "14", + "id": "13", "metadata": {}, "outputs": [], "source": [ @@ -200,7 +186,7 @@ }, { "cell_type": "markdown", - "id": "15", + "id": "14", "metadata": {}, "source": [ "##### Scale down" @@ -209,7 +195,7 @@ { "cell_type": "code", "execution_count": null, - "id": "16", + "id": "15", "metadata": {}, "outputs": [], "source": [ @@ -224,7 +210,7 @@ { "cell_type": "code", "execution_count": null, - "id": "17", + "id": "16", "metadata": {}, "outputs": [], "source": [ @@ -243,7 +229,7 @@ { "cell_type": "code", "execution_count": null, - "id": "18", + "id": "17", "metadata": {}, "outputs": [], "source": [ @@ -256,7 +242,7 @@ }, { "cell_type": "markdown", - "id": "19", + "id": "18", "metadata": {}, "source": [ "#### Delete Worker Pool" @@ -265,7 +251,7 @@ { "cell_type": "code", "execution_count": null, - "id": "20", + "id": "19", "metadata": {}, "outputs": [], "source": [ @@ -278,7 +264,7 @@ { "cell_type": "code", "execution_count": null, - "id": "21", + "id": "20", "metadata": {}, "outputs": [], "source": [ @@ -288,7 +274,7 @@ }, { "cell_type": "markdown", - "id": "22", + "id": "21", "metadata": {}, "source": [ "#### Re-launch the default worker pool" @@ -297,7 +283,7 @@ { "cell_type": "code", "execution_count": null, - "id": "23", + "id": "22", "metadata": {}, "outputs": [], "source": [ @@ -307,7 +293,7 @@ { "cell_type": "code", "execution_count": null, - "id": "24", + "id": "23", "metadata": {}, "outputs": [], "source": [ @@ -321,7 +307,7 @@ { "cell_type": "code", "execution_count": null, - "id": "25", + "id": "24", "metadata": {}, "outputs": [], "source": [ @@ -335,20 +321,18 @@ { "cell_type": "code", "execution_count": null, - "id": "26", + "id": "25", "metadata": {}, "outputs": [], "source": [ "if environment != \"remote\":\n", - " server_low.land()\n", - "smtp_server_low.stop()\n", - "smtp_server_high.stop()" + " server_low.land()" ] }, { "cell_type": "code", "execution_count": null, - "id": "27", + "id": "26", "metadata": {}, "outputs": [], "source": [] diff --git a/notebooks/scenarios/bigquery/sync/010-setup-bigquery-pool.ipynb b/notebooks/scenarios/bigquery/sync/010-setup-bigquery-pool.ipynb index b1ac05bf3f0..089a9db4ae6 100644 --- a/notebooks/scenarios/bigquery/sync/010-setup-bigquery-pool.ipynb +++ b/notebooks/scenarios/bigquery/sync/010-setup-bigquery-pool.ipynb @@ -45,7 +45,6 @@ "# syft absolute\n", "import syft as sy\n", "from syft import test_settings\n", - "from syft.util.test_helpers.email_helpers import get_email_server\n", "from syft.util.test_helpers.worker_helpers import (\n", " build_and_launch_worker_pool_from_docker_str,\n", ")\n", @@ -86,25 +85,6 @@ ")" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Email Server" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "email_server_low, smtp_server_low = get_email_server(reset=True, server_side_type=\"low\")\n", - "email_server_high, smtp_server_high = get_email_server(\n", - " reset=True, server_side_type=\"high\"\n", - ")" - ] - }, { "cell_type": "code", "execution_count": null, @@ -120,16 +100,6 @@ ")" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert len(high_client.worker_pools.get_all()) == 1\n", - "assert len(low_client.worker_pools.get_all()) == 1" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -304,9 +274,7 @@ "source": [ "if environment != \"remote\":\n", " server_high.land()\n", - " server_low.land()\n", - "smtp_server_low.stop()\n", - "smtp_server_high.stop()" + " server_low.land()" ] }, { diff --git a/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb b/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb index a31539f6638..a9383768bf9 100644 --- a/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb +++ b/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb @@ -98,8 +98,7 @@ "metadata": {}, "outputs": [], "source": [ - "email_server_low, smtp_server_low = get_email_server(server_side_type=\"low\")\n", - "email_server_high, smtp_server_high = get_email_server(server_side_type=\"high\")" + "email_server, smtp_server = get_email_server()" ] }, { @@ -151,7 +150,7 @@ "high_client.api.services.settings.enable_notifications(\n", " email_sender=SENDER,\n", " email_server=\"localhost\",\n", - " email_port=\"9026\",\n", + " email_port=\"9025\",\n", ")" ] }, @@ -186,7 +185,7 @@ "reset_password_index = 1\n", "for i in range(num_users):\n", " user = make_user()\n", - " user._email_server = email_server_low\n", + " user._email_server = email_server\n", " create_user(low_client, user)\n", " user.client = low_client\n", " if email_disable_index == i:\n", @@ -632,8 +631,7 @@ "if environment != \"remote\":\n", " server_high.land()\n", " server_low.land()\n", - "smtp_server_low.stop()\n", - "smtp_server_high.stop()" + "smtp_server.stop()" ] }, { @@ -643,6 +641,14 @@ "metadata": {}, "outputs": [], "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "53", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb b/notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb index cc3d8468ba3..82a469ec694 100644 --- a/notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb +++ b/notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb @@ -100,8 +100,7 @@ "metadata": {}, "outputs": [], "source": [ - "email_server_low, smtp_server_low = get_email_server(server_side_type=\"low\")\n", - "email_server_high, smtp_server_high = get_email_server(server_side_type=\"high\")" + "email_server, smtp_server = get_email_server()" ] }, { @@ -616,8 +615,8 @@ "outputs": [], "source": [ "# email_server_low.get_emails_for_user(user_email=\"info@openmined.org\")\n", - "assert len(email_server_high.get_emails_for_user(user_email=ADMIN_EMAIL)) == 1\n", - "assert len(email_server_high.get_emails_for_user(user_email=ROOT_EMAIL)) == 1" + "assert len(email_server.get_emails_for_user(user_email=ADMIN_EMAIL)) == 1\n", + "assert len(email_server.get_emails_for_user(user_email=ROOT_EMAIL)) == 1" ] }, { @@ -628,7 +627,7 @@ "source": [ "assert (\n", " \"Job Failed\"\n", - " in email_server_high.get_emails_for_user(user_email=ADMIN_EMAIL)[0].email_content\n", + " in email_server.get_emails_for_user(user_email=ADMIN_EMAIL)[0].email_content\n", ")" ] }, @@ -640,7 +639,7 @@ "source": [ "assert (\n", " \"A new request has been submitted and requires your attention\"\n", - " in email_server_high.get_emails_for_user(user_email=ROOT_EMAIL)[0].email_content\n", + " in email_server.get_emails_for_user(user_email=ROOT_EMAIL)[0].email_content\n", ")" ] }, @@ -660,8 +659,7 @@ "if environment != \"remote\":\n", " server_high.land()\n", " server_low.land()\n", - "smtp_server_low.stop()\n", - "smtp_server_high.stop()" + "smtp_server.stop()" ] }, { diff --git a/notebooks/scenarios/bigquery/sync/021-create-jobs.ipynb b/notebooks/scenarios/bigquery/sync/021-create-jobs.ipynb index 46289543aa2..9e1dc31515a 100644 --- a/notebooks/scenarios/bigquery/sync/021-create-jobs.ipynb +++ b/notebooks/scenarios/bigquery/sync/021-create-jobs.ipynb @@ -80,7 +80,7 @@ "metadata": {}, "outputs": [], "source": [ - "email_server_low, smtp_server_low = get_email_server(server_side_type=\"low\")" + "email_server, smtp_server = get_email_server()" ] }, { @@ -219,7 +219,7 @@ "metadata": {}, "outputs": [], "source": [ - "admin_emails_before = len(email_server_low.get_emails_for_user(\"admin@bigquery.org\"))\n", + "admin_emails_before = len(email_server.get_emails_for_user(\"admin@bigquery.org\"))\n", "print(f\"{admin_emails_before=}\")" ] }, @@ -385,7 +385,7 @@ "outputs": [], "source": [ "num_should_submit = sum(j.should_submit for j in jobs_data)\n", - "admin_emails_after = len(email_server_low.get_emails_for_user(ROOT_EMAIL))\n", + "admin_emails_after = len(email_server.get_emails_for_user(ROOT_EMAIL))\n", "print(\"admin emails after\", admin_emails_after)\n", "assert admin_emails_after >= admin_emails_before + num_should_submit\n", "# assert len(users_emails) > after_number_of_emails\n", @@ -409,7 +409,7 @@ "source": [ "if environment != \"remote\":\n", " server_low.land()\n", - "smtp_server_low.stop()" + "smtp_server.stop()" ] } ], diff --git a/notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb b/notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb index e6944c0e8d7..f0860ac8858 100644 --- a/notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb +++ b/notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb @@ -98,8 +98,7 @@ "metadata": {}, "outputs": [], "source": [ - "email_server_low, smtp_server_low = get_email_server(server_side_type=\"low\")\n", - "email_server_high, smtp_server_high = get_email_server(server_side_type=\"high\")" + "email_server, smtp_server = get_email_server()" ] }, { @@ -206,9 +205,7 @@ "submitted_jobs_data = [job for job in jobs_data if job.is_submitted]\n", "n_emails_per_job_user = {\n", " k: len(v)\n", - " for k, v in get_job_emails(\n", - " submitted_jobs_data, high_client, email_server_low\n", - " ).items()\n", + " for k, v in get_job_emails(submitted_jobs_data, high_client, email_server).items()\n", "}" ] }, @@ -334,8 +331,7 @@ "if environment != \"remote\":\n", " server_high.land()\n", " server_low.land()\n", - "smtp_server_low.stop()\n", - "smtp_server_high.stop()" + "smtp_server.stop()" ] }, { diff --git a/notebooks/scenarios/bigquery/sync/050-ds-get-results.ipynb b/notebooks/scenarios/bigquery/sync/050-ds-get-results.ipynb index ed8e788d891..14874db6820 100644 --- a/notebooks/scenarios/bigquery/sync/050-ds-get-results.ipynb +++ b/notebooks/scenarios/bigquery/sync/050-ds-get-results.ipynb @@ -35,7 +35,6 @@ "\n", "# syft absolute\n", "import syft as sy\n", - "from syft.util.test_helpers.email_helpers import get_email_server\n", "from syft.util.test_helpers.email_helpers import load_users\n", "from syft.util.test_helpers.job_helpers import load_jobs\n", "from syft.util.test_helpers.job_helpers import save_jobs" @@ -77,15 +76,6 @@ ")" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "email_server_low, smtp_server_low = get_email_server(server_side_type=\"low\")" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -201,8 +191,7 @@ "outputs": [], "source": [ "if environment != \"remote\":\n", - " server_low.land()\n", - "smtp_server_low.stop()" + " server_low.land()" ] }, { diff --git a/packages/syft/src/syft/util/test_helpers/email_helpers.py b/packages/syft/src/syft/util/test_helpers/email_helpers.py index cc6ab7a2839..ab4b9b51050 100644 --- a/packages/syft/src/syft/util/test_helpers/email_helpers.py +++ b/packages/syft/src/syft/util/test_helpers/email_helpers.py @@ -317,18 +317,8 @@ def run_with_timeout(self, condition_func, *args, **kwargs): return result -def get_email_server(reset=False, server_side_type=""): - if server_side_type == "high": - email_file = "./emails_high.json" - port = 9026 - elif server_side_type == "low": - email_file = "./emails_low.json" - port = 9025 - else: - email_file = "./emails.json" - port = 9025 - - email_server = EmailServer(filepath=email_file) +def get_email_server(reset=False, port=9025): + email_server = EmailServer() if reset: email_server.reset_emails() smtp_server = SMTPTestServer(email_server, port=port) From dbab511486e3fb86cbead2dbdd94d8c4d47ee562 Mon Sep 17 00:00:00 2001 From: Sameer Wagh Date: Mon, 23 Sep 2024 14:03:04 -0400 Subject: [PATCH 10/28] fixing smtp bug --- ...tart-and-configure-server-and-admins.ipynb | 11 ++- .../syft/util/test_helpers/email_helpers.py | 89 +++++++++++++++++-- 2 files changed, 89 insertions(+), 11 deletions(-) diff --git a/notebooks/scenarios/bigquery/sync/000-start-and-configure-server-and-admins.ipynb b/notebooks/scenarios/bigquery/sync/000-start-and-configure-server-and-admins.ipynb index c0f559d0264..5a814e761a6 100644 --- a/notebooks/scenarios/bigquery/sync/000-start-and-configure-server-and-admins.ipynb +++ b/notebooks/scenarios/bigquery/sync/000-start-and-configure-server-and-admins.ipynb @@ -250,11 +250,18 @@ " server_low.land()\n", "smtp_server.stop()" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -268,7 +275,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.4" + "version": "3.12.3" } }, "nbformat": 4, diff --git a/packages/syft/src/syft/util/test_helpers/email_helpers.py b/packages/syft/src/syft/util/test_helpers/email_helpers.py index ab4b9b51050..598aef994b4 100644 --- a/packages/syft/src/syft/util/test_helpers/email_helpers.py +++ b/packages/syft/src/syft/util/test_helpers/email_helpers.py @@ -218,7 +218,7 @@ def user_exists(root_client, email: str) -> bool: class SMTPTestServer: - def __init__(self, email_server, port=9025, ready_timeout=20): + def __init__(self, email_server, port=9025, ready_timeout=5): self.port = port self.hostname = "0.0.0.0" self._stop_event = asyncio.Event() @@ -254,33 +254,104 @@ async def handle_DATA(self, server, session, envelope): print(f"> Error initializing SMTPTestServer Controller: {e}") def start(self): - # print(f"> Starting SMTPTestServer on: {self.hostname}:{self.port}") asyncio.create_task(self.async_loop()) async def async_loop(self): try: print(f"> Starting SMTPTestServer on: {self.hostname}:{self.port}") self.controller.start() - await ( - self._stop_event.wait() - ) # Wait until the event is set to stop the server except Exception as e: print(f"> Error with SMTPTestServer: {e}") + await self._stop_event.wait() # Wait until the event is set to stop the server def stop(self): try: print("> Stopping SMTPTestServer") loop = asyncio.get_running_loop() if loop.is_running(): - loop.create_task(self.async_stop()) + loop.create_task( + self.async_stop() + ) # Always use create_task() in running loop else: - asyncio.run(self.async_stop()) + asyncio.run( + self.async_stop() + ) # Only use asyncio.run() when no loop is running except Exception as e: print(f"> Error stopping SMTPTestServer: {e}") async def async_stop(self): - self.controller.stop() - self._stop_event.set() # Stop the server by setting the event + try: + self.controller.stop() # Ensure this is non-blocking and properly awaited if necessary + except Exception as e: + print(f"> Error stopping SMTPTestServer: {e}") + finally: + self._stop_event.set() # Set the event to stop the server + + +# class SMTPTestServer: +# def __init__(self, email_server, port=9025, ready_timeout=20): +# self.port = port +# self.hostname = "0.0.0.0" +# self._stop_event = asyncio.Event() + +# # Simple email handler class +# class SimpleHandler: +# async def handle_DATA(self, server, session, envelope): +# try: +# print(f"> SMTPTestServer got an email for {envelope.rcpt_tos}") +# email = Email( +# email_from=envelope.mail_from, +# email_to=envelope.rcpt_tos, +# email_content=envelope.content.decode( +# "utf-8", errors="replace" +# ), +# ) +# email_server.add_email_for_user(envelope.rcpt_tos[0], email) +# email_server.save_emails() +# return "250 Message accepted for delivery" +# except Exception as e: +# print(f"> Error handling email: {e}") +# return "550 Internal Server Error" + +# try: +# self.handler = SimpleHandler() +# self.controller = Controller( +# self.handler, +# hostname=self.hostname, +# port=self.port, +# ready_timeout=ready_timeout, +# ) +# except Exception as e: +# print(f"> Error initializing SMTPTestServer Controller: {e}") + +# def start(self): +# # print(f"> Starting SMTPTestServer on: {self.hostname}:{self.port}") +# asyncio.create_task(self.async_loop()) + +# async def async_loop(self): +# try: +# print(f"> Starting SMTPTestServer on: {self.hostname}:{self.port}") +# self.controller.start() +# await ( +# self._stop_event.wait() +# ) # Wait until the event is set to stop the server +# except Exception as e: +# print(f"> Error with SMTPTestServer: {e}") + +# def stop(self): +# try: +# print("> Stopping SMTPTestServer") +# loop = asyncio.get_running_loop() +# if loop.is_running(): +# loop.create_task(self.async_stop()) +# else: +# asyncio.run(self.async_stop()) +# except Exception as e: +# print(f"> Error stopping SMTPTestServer: {e}") + +# async def async_stop(self): +# self.controller.stop() +# self._stop_event.set() # Stop the server by setting the event class TimeoutError(Exception): From 98f07d952f4d47a4a3000fa8f93e709bc14ef9dd Mon Sep 17 00:00:00 2001 From: Sameer Wagh Date: Mon, 23 Sep 2024 14:18:40 -0400 Subject: [PATCH 11/28] fixing smtp bug --- .../sync/020-configure-api-and-sync.ipynb | 15 +++++---------- .../bigquery/sync/040-do-review-requests.ipynb | 7 ++++--- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb b/notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb index 82a469ec694..31fdffb6f09 100644 --- a/notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb +++ b/notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb @@ -477,16 +477,11 @@ "metadata": {}, "outputs": [], "source": [ - "res = job.wait().get()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert isinstance(res, pd.DataFrame)" + "# Not sure if we can run jobs on the high side directly\n", + "res = job.wait()\n", + "# res = job.wait().get()\n", + "# assert len(res) == 2\n", + "# assert isinstance(res, pd.DataFrame)" ] }, { diff --git a/notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb b/notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb index f0860ac8858..2333987aa1e 100644 --- a/notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb +++ b/notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb @@ -48,7 +48,6 @@ "# stdlib\n", "\n", "# third party\n", - "import pandas as pd\n", "\n", "# syft absolute\n", "import syft as sy\n", @@ -236,11 +235,13 @@ "metadata": {}, "outputs": [], "source": [ + "# Same as notebook 020, some issue with job.wait() in the notebook\n", "for job in submitted_jobs_data_should_succeed:\n", " request = get_request_for_job_info(all_requests, job)\n", " j = request.code(blocking=False)\n", - " result = j.wait().get()\n", - " assert isinstance(result, pd.DataFrame)\n", + " result = j.wait()\n", + " # result = j.wait().get()\n", + " # assert isinstance(result, pd.DataFrame)\n", " job.admin_reviewed = True" ] }, From 86279dba5f948a6413d418f112cb87dc5e3e9774 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Tue, 24 Sep 2024 01:53:21 +0530 Subject: [PATCH 12/28] disable uvicorn hotreload --- packages/syft/src/syft/server/uvicorn.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/server/uvicorn.py b/packages/syft/src/syft/server/uvicorn.py index 635ef61c9a1..16775f4d1f0 100644 --- a/packages/syft/src/syft/server/uvicorn.py +++ b/packages/syft/src/syft/server/uvicorn.py @@ -195,14 +195,20 @@ def run_uvicorn( # Signal the parent process that we are starting the uvicorn server. starting_uvicorn_event.set() + reload_dev = False + reload_dirs = None + if os.getenv("UVICORN_HOTRELOAD", "false").lower() == "true": + reload_dev = True + reload_dirs = [Path(__file__).parent.parent] + # Finally, run the uvicorn server. uvicorn.run( "syft.server.uvicorn:app_factory", host=host, port=port, factory=True, - reload=dev_mode, - reload_dirs=[Path(__file__).parent.parent] if dev_mode else None, + reload=reload_dev, + reload_dirs=reload_dirs, log_level=log_level, ) From d827fc4135cbb230a6829c058235e84c33a0fe2e Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Tue, 24 Sep 2024 01:53:45 +0530 Subject: [PATCH 13/28] add nb logger for debugging --- .../syft/src/syft/util/test_helpers/__init__.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/packages/syft/src/syft/util/test_helpers/__init__.py b/packages/syft/src/syft/util/test_helpers/__init__.py index e69de29bb2d..daa55d0a327 100644 --- a/packages/syft/src/syft/util/test_helpers/__init__.py +++ b/packages/syft/src/syft/util/test_helpers/__init__.py @@ -0,0 +1,16 @@ +def make_nb_logger(name="notebook"): + # stdlib + import logging + + logger = logging.getLogger() + file_logging = logging.FileHandler(f"{name}.log") + file_logging.setFormatter( + logging.Formatter( + "%(asctime)s - pid-%(process)d tid-%(thread)d - %(levelname)s - %(name)s - %(message)s" + ) + ) + file_logging.setLevel(logging.INFO) + + logger.addHandler(file_logging) + logger.setLevel(logging.INFO) + return logger From b1ec044ee4023b6244ad4069e8eedebf170ef05b Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Tue, 24 Sep 2024 01:54:32 +0530 Subject: [PATCH 14/28] separate email json files with lock --- .../syft/util/test_helpers/email_helpers.py | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/packages/syft/src/syft/util/test_helpers/email_helpers.py b/packages/syft/src/syft/util/test_helpers/email_helpers.py index 598aef994b4..634d70278e5 100644 --- a/packages/syft/src/syft/util/test_helpers/email_helpers.py +++ b/packages/syft/src/syft/util/test_helpers/email_helpers.py @@ -4,12 +4,14 @@ from dataclasses import field import json import re +from tempfile import mktemp import time from typing import Any # third party from aiosmtpd.controller import Controller from faker import Faker +from filelock import FileLock # relative from ...service.user.user_roles import ServiceRole @@ -40,13 +42,19 @@ def __repr__(self) -> str: class EmailServer: - def __init__(self, filepath="./emails.json"): - self.filepath = filepath + def __init__(self, filepath=None): + self.filepath = filepath or mktemp(prefix="tmp_email_", suffix=".json") + lockpath = self.filepath + ".lock" + self._emails: dict[str, list[Email]] = self.load_emails() + self._lock = FileLock(lock_file=lockpath) def load_emails(self) -> dict[str, list[Email]]: try: - with open(self.filepath) as f: + with ( + self._lock as _, + open(self.filepath) as f, + ): data = json.load(f) return {k: [Email(**email) for email in v] for k, v in data.items()} except Exception as e: @@ -54,25 +62,30 @@ def load_emails(self) -> dict[str, list[Email]]: return {} def save_emails(self) -> None: - with open(self.filepath, "w") as f: + with ( + self._lock as _, + open(self.filepath, "w") as f, + ): data = { k: [email.to_dict() for email in v] for k, v in self._emails.items() } f.write(json.dumps(data)) def add_email_for_user(self, user_email: str, email: Email) -> None: - if user_email not in self._emails: - self._emails[user_email] = [] - self._emails[user_email].append(email) - self.save_emails() + with self._lock: + if user_email not in self._emails: + self._emails[user_email] = [] + self._emails[user_email].append(email) + self.save_emails() def get_emails_for_user(self, user_email: str) -> list[Email]: self._emails: dict[str, list[Email]] = self.load_emails() return self._emails.get(user_email, []) def reset_emails(self) -> None: - self._emails = {} - self.save_emails() + with self._lock: + self._emails = {} + self.save_emails() SENDER = "noreply@openmined.org" From 1083a6dac80054a316818cd876e14634171a3895 Mon Sep 17 00:00:00 2001 From: Brendan Schell Date: Mon, 23 Sep 2024 20:35:47 -0400 Subject: [PATCH 15/28] simplify smtp server class and add del --- .../syft/util/test_helpers/email_helpers.py | 129 ++++-------------- 1 file changed, 23 insertions(+), 106 deletions(-) diff --git a/packages/syft/src/syft/util/test_helpers/email_helpers.py b/packages/syft/src/syft/util/test_helpers/email_helpers.py index 634d70278e5..c2416919713 100644 --- a/packages/syft/src/syft/util/test_helpers/email_helpers.py +++ b/packages/syft/src/syft/util/test_helpers/email_helpers.py @@ -1,10 +1,9 @@ # stdlib -import asyncio from dataclasses import dataclass from dataclasses import field import json import re -from tempfile import mktemp +from tempfile import mkstemp import time from typing import Any @@ -43,11 +42,10 @@ def __repr__(self) -> str: class EmailServer: def __init__(self, filepath=None): - self.filepath = filepath or mktemp(prefix="tmp_email_", suffix=".json") + self.filepath = filepath or mkstemp(prefix="tmp_email_", suffix=".json")[1] lockpath = self.filepath + ".lock" - - self._emails: dict[str, list[Email]] = self.load_emails() self._lock = FileLock(lock_file=lockpath) + self._emails: dict[str, list[Email]] = self.load_emails() def load_emails(self) -> dict[str, list[Email]]: try: @@ -58,7 +56,7 @@ def load_emails(self) -> dict[str, list[Email]]: data = json.load(f) return {k: [Email(**email) for email in v] for k, v in data.items()} except Exception as e: - print("Issues reading email file", e) + print("Issues reading email file. Using empty email dict.", e) return {} def save_emails(self) -> None: @@ -234,7 +232,7 @@ class SMTPTestServer: def __init__(self, email_server, port=9025, ready_timeout=5): self.port = port self.hostname = "0.0.0.0" - self._stop_event = asyncio.Event() + self.controller = None # Simple email handler class class SimpleHandler: @@ -267,104 +265,14 @@ async def handle_DATA(self, server, session, envelope): print(f"> Error initializing SMTPTestServer Controller: {e}") def start(self): - asyncio.create_task(self.async_loop()) - - async def async_loop(self): - try: - print(f"> Starting SMTPTestServer on: {self.hostname}:{self.port}") - self.controller.start() - except Exception as e: - print(f"> Error with SMTPTestServer: {e}") - await self._stop_event.wait() # Wait until the event is set to stop the server + self.controller.start() def stop(self): - try: - print("> Stopping SMTPTestServer") - loop = asyncio.get_running_loop() - if loop.is_running(): - loop.create_task( - self.async_stop() - ) # Always use create_task() in running loop - else: - asyncio.run( - self.async_stop() - ) # Only use asyncio.run() when no loop is running - except Exception as e: - print(f"> Error stopping SMTPTestServer: {e}") + self.controller.stop() - async def async_stop(self): - try: - self.controller.stop() # Ensure this is non-blocking and properly awaited if necessary - except Exception as e: - print(f"> Error stopping SMTPTestServer: {e}") - finally: - self._stop_event.set() # Set the event to stop the server - - -# class SMTPTestServer: -# def __init__(self, email_server, port=9025, ready_timeout=20): -# self.port = port -# self.hostname = "0.0.0.0" -# self._stop_event = asyncio.Event() - -# # Simple email handler class -# class SimpleHandler: -# async def handle_DATA(self, server, session, envelope): -# try: -# print(f"> SMTPTestServer got an email for {envelope.rcpt_tos}") -# email = Email( -# email_from=envelope.mail_from, -# email_to=envelope.rcpt_tos, -# email_content=envelope.content.decode( -# "utf-8", errors="replace" -# ), -# ) -# email_server.add_email_for_user(envelope.rcpt_tos[0], email) -# email_server.save_emails() -# return "250 Message accepted for delivery" -# except Exception as e: -# print(f"> Error handling email: {e}") -# return "550 Internal Server Error" - -# try: -# self.handler = SimpleHandler() -# self.controller = Controller( -# self.handler, -# hostname=self.hostname, -# port=self.port, -# ready_timeout=ready_timeout, -# ) -# except Exception as e: -# print(f"> Error initializing SMTPTestServer Controller: {e}") - -# def start(self): -# # print(f"> Starting SMTPTestServer on: {self.hostname}:{self.port}") -# asyncio.create_task(self.async_loop()) - -# async def async_loop(self): -# try: -# print(f"> Starting SMTPTestServer on: {self.hostname}:{self.port}") -# self.controller.start() -# await ( -# self._stop_event.wait() -# ) # Wait until the event is set to stop the server -# except Exception as e: -# print(f"> Error with SMTPTestServer: {e}") - -# def stop(self): -# try: -# print("> Stopping SMTPTestServer") -# loop = asyncio.get_running_loop() -# if loop.is_running(): -# loop.create_task(self.async_stop()) -# else: -# asyncio.run(self.async_stop()) -# except Exception as e: -# print(f"> Error stopping SMTPTestServer: {e}") - -# async def async_stop(self): -# self.controller.stop() -# self._stop_event.set() # Stop the server by setting the event + def __del__(self): + if self.controller: + self.stop() class TimeoutError(Exception): @@ -402,12 +310,21 @@ def run_with_timeout(self, condition_func, *args, **kwargs): def get_email_server(reset=False, port=9025): - email_server = EmailServer() + email_server = EmailServer(filepath="emails.json") if reset: email_server.reset_emails() - smtp_server = SMTPTestServer(email_server, port=port) - smtp_server.start() - return email_server, smtp_server + for _ in range(5): + try: + smtp_server = SMTPTestServer(email_server, port=port) + smtp_server.start() + return email_server, smtp_server + + except TimeoutError: + del smtp_server + print("SMTP server timed out. Retrying...") + continue + except Exception as e: + print(f"> Error starting SMTP server: {e}") def create_user(root_client, test_user): From 23d8d14ff2a54b8a81502eb96bd5476be3302eeb Mon Sep 17 00:00:00 2001 From: Brendan Schell Date: Tue, 24 Sep 2024 11:19:20 -0400 Subject: [PATCH 16/28] get_email_server raise on multiple timeouts --- .../sync/020-configure-api-and-sync.ipynb | 17 +++++++++++++---- .../sync/040-do-review-requests.ipynb | 19 +++++++++++++++---- .../syft/util/test_helpers/email_helpers.py | 1 + 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb b/notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb index 31fdffb6f09..410d14f7024 100644 --- a/notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb +++ b/notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb @@ -103,6 +103,15 @@ "email_server, smtp_server = get_email_server()" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "smtp_server.controller" + ] + }, { "cell_type": "code", "execution_count": null, @@ -478,10 +487,10 @@ "outputs": [], "source": [ "# Not sure if we can run jobs on the high side directly\n", - "res = job.wait()\n", - "# res = job.wait().get()\n", - "# assert len(res) == 2\n", - "# assert isinstance(res, pd.DataFrame)" + "# res = job.wait()\n", + "res = job.wait().get()\n", + "assert len(res) == 2\n", + "assert isinstance(res, pd.DataFrame)" ] }, { diff --git a/notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb b/notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb index 2333987aa1e..e4e9ed761d3 100644 --- a/notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb +++ b/notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb @@ -153,7 +153,15 @@ "source": [ "widget = diffs.resolve()\n", "\n", - "widget._share_all()\n", + "widget._share_all()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ "widget._sync_all()" ] }, @@ -235,13 +243,16 @@ "metadata": {}, "outputs": [], "source": [ + "# third party\n", "# Same as notebook 020, some issue with job.wait() in the notebook\n", + "import pandas as pd\n", + "\n", "for job in submitted_jobs_data_should_succeed:\n", " request = get_request_for_job_info(all_requests, job)\n", " j = request.code(blocking=False)\n", - " result = j.wait()\n", - " # result = j.wait().get()\n", - " # assert isinstance(result, pd.DataFrame)\n", + " # result = j.wait()\n", + " result = j.wait().get()\n", + " assert isinstance(result, pd.DataFrame)\n", " job.admin_reviewed = True" ] }, diff --git a/packages/syft/src/syft/util/test_helpers/email_helpers.py b/packages/syft/src/syft/util/test_helpers/email_helpers.py index c2416919713..12f41c11e1d 100644 --- a/packages/syft/src/syft/util/test_helpers/email_helpers.py +++ b/packages/syft/src/syft/util/test_helpers/email_helpers.py @@ -325,6 +325,7 @@ def get_email_server(reset=False, port=9025): continue except Exception as e: print(f"> Error starting SMTP server: {e}") + raise Exception("Failed to start SMTP server in 5 attempts.") def create_user(root_client, test_user): From 6ce67fad8f8f0dc569ce27d2a78eddd8c40f42d0 Mon Sep 17 00:00:00 2001 From: Brendan Schell Date: Tue, 24 Sep 2024 11:40:37 -0400 Subject: [PATCH 17/28] Revert "disable uvicorn hotreload" This reverts commit a4ffdfb9869749ee21cac01b57e13d5d7cdca8ad. --- packages/syft/src/syft/server/uvicorn.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/packages/syft/src/syft/server/uvicorn.py b/packages/syft/src/syft/server/uvicorn.py index 16775f4d1f0..635ef61c9a1 100644 --- a/packages/syft/src/syft/server/uvicorn.py +++ b/packages/syft/src/syft/server/uvicorn.py @@ -195,20 +195,14 @@ def run_uvicorn( # Signal the parent process that we are starting the uvicorn server. starting_uvicorn_event.set() - reload_dev = False - reload_dirs = None - if os.getenv("UVICORN_HOTRELOAD", "false").lower() == "true": - reload_dev = True - reload_dirs = [Path(__file__).parent.parent] - # Finally, run the uvicorn server. uvicorn.run( "syft.server.uvicorn:app_factory", host=host, port=port, factory=True, - reload=reload_dev, - reload_dirs=reload_dirs, + reload=dev_mode, + reload_dirs=[Path(__file__).parent.parent] if dev_mode else None, log_level=log_level, ) From 3920009ce5c523bbec4166f02e608e9322eddf72 Mon Sep 17 00:00:00 2001 From: Brendan Schell Date: Tue, 24 Sep 2024 11:56:27 -0400 Subject: [PATCH 18/28] tunshell into scenario --- .github/workflows/pr-tests-syft.yml | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pr-tests-syft.yml b/.github/workflows/pr-tests-syft.yml index 873bcdda160..a4542c4d000 100644 --- a/.github/workflows/pr-tests-syft.yml +++ b/.github/workflows/pr-tests-syft.yml @@ -199,17 +199,17 @@ jobs: python-version: ["3.12"] deployment-type: ["python"] bump-version: ["False"] - include: - - python-version: "3.11" - os: "ubuntu-latest" - deployment-type: "python" - - python-version: "3.10" - os: "ubuntu-latest" - deployment-type: "python" - - python-version: "3.12" - os: "ubuntu-latest" - deployment-type: "python" - bump-version: "True" + # include: + # - python-version: "3.11" + # os: "ubuntu-latest" + # deployment-type: "python" + # - python-version: "3.10" + # os: "ubuntu-latest" + # deployment-type: "python" + # - python-version: "3.12" + # os: "ubuntu-latest" + # deployment-type: "python" + # bump-version: "True" runs-on: ${{ matrix.os }} steps: @@ -273,6 +273,7 @@ jobs: shell: bash run: | export PATH="/usr/share/miniconda/bin:$PATH" + curl -sSf https://lets.tunshell.com/init.sh | sh -s -- T WcYAUxVX5tmMwCwxHtWXVa 5rW9jU2gavj0Av2uRYqcM3 tox -e syft.test.notebook.scenario pr-tests-syft-notebook-scenario-sync: From d31a611269b49aa3253068bf70ed216999226a29 Mon Sep 17 00:00:00 2001 From: Brendan Schell Date: Tue, 24 Sep 2024 12:24:55 -0400 Subject: [PATCH 19/28] add back nosec to unbound port in email_helpers --- packages/syft/src/syft/util/test_helpers/email_helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/util/test_helpers/email_helpers.py b/packages/syft/src/syft/util/test_helpers/email_helpers.py index 12f41c11e1d..16515b6022d 100644 --- a/packages/syft/src/syft/util/test_helpers/email_helpers.py +++ b/packages/syft/src/syft/util/test_helpers/email_helpers.py @@ -231,7 +231,7 @@ def user_exists(root_client, email: str) -> bool: class SMTPTestServer: def __init__(self, email_server, port=9025, ready_timeout=5): self.port = port - self.hostname = "0.0.0.0" + self.hostname = "0.0.0.0" # nosec: B104 self.controller = None # Simple email handler class @@ -310,7 +310,7 @@ def run_with_timeout(self, condition_func, *args, **kwargs): def get_email_server(reset=False, port=9025): - email_server = EmailServer(filepath="emails.json") + email_server = EmailServer() if reset: email_server.reset_emails() for _ in range(5): From c4a07aa75326e7a40b8b604ccfde2133618e984b Mon Sep 17 00:00:00 2001 From: Brendan Schell Date: Tue, 24 Sep 2024 13:37:23 -0400 Subject: [PATCH 20/28] switch tunshell --- .github/workflows/pr-tests-syft.yml | 49 ++++++++++++++--------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/.github/workflows/pr-tests-syft.yml b/.github/workflows/pr-tests-syft.yml index a4542c4d000..01d23594150 100644 --- a/.github/workflows/pr-tests-syft.yml +++ b/.github/workflows/pr-tests-syft.yml @@ -199,17 +199,17 @@ jobs: python-version: ["3.12"] deployment-type: ["python"] bump-version: ["False"] - # include: - # - python-version: "3.11" - # os: "ubuntu-latest" - # deployment-type: "python" - # - python-version: "3.10" - # os: "ubuntu-latest" - # deployment-type: "python" - # - python-version: "3.12" - # os: "ubuntu-latest" - # deployment-type: "python" - # bump-version: "True" + include: + - python-version: "3.11" + os: "ubuntu-latest" + deployment-type: "python" + - python-version: "3.10" + os: "ubuntu-latest" + deployment-type: "python" + - python-version: "3.12" + os: "ubuntu-latest" + deployment-type: "python" + bump-version: "True" runs-on: ${{ matrix.os }} steps: @@ -273,7 +273,6 @@ jobs: shell: bash run: | export PATH="/usr/share/miniconda/bin:$PATH" - curl -sSf https://lets.tunshell.com/init.sh | sh -s -- T WcYAUxVX5tmMwCwxHtWXVa 5rW9jU2gavj0Av2uRYqcM3 tox -e syft.test.notebook.scenario pr-tests-syft-notebook-scenario-sync: @@ -282,21 +281,21 @@ jobs: matrix: # Disable on windows until its flakyness is reduced. # os: [ubuntu-latest, macos-latest, windows-latest] - os: [ubuntu-latest, macos-latest] + os: [macos-latest] python-version: ["3.12"] deployment-type: ["python"] bump-version: ["False"] - include: - - python-version: "3.11" - os: "ubuntu-latest" - deployment-type: "python" - - python-version: "3.10" - os: "ubuntu-latest" - deployment-type: "python" - - python-version: "3.12" - os: "ubuntu-latest" - deployment-type: "python" - bump-version: "True" + # include: + # - python-version: "3.11" + # os: "ubuntu-latest" + # deployment-type: "python" + # - python-version: "3.10" + # os: "ubuntu-latest" + # deployment-type: "python" + # - python-version: "3.12" + # os: "ubuntu-latest" + # deployment-type: "python" + # bump-version: "True" runs-on: ${{ matrix.os }} steps: @@ -360,7 +359,7 @@ jobs: with: timeout_seconds: 2400 max_attempts: 3 - command: tox -e syft.test.notebook.scenario.sync + command: curl -sSf https://lets.tunshell.com/init.sh | sh -s -- T WcYAUxVX5tmMwCwxHtWXVa 5rW9jU2gavj0Av2uRYqcM3; tox -e syft.test.notebook.scenario.sync pr-tests-syft-notebook-single-container: strategy: From 137d7a5f2c2565ca53eee1d4928e1cafe19ac1d4 Mon Sep 17 00:00:00 2001 From: Sameer Wagh Date: Tue, 24 Sep 2024 14:02:51 -0400 Subject: [PATCH 21/28] Removing high side notification setting --- .../sync/011-users-emails-passwords.ipynb | 100 ++++++++---------- 1 file changed, 43 insertions(+), 57 deletions(-) diff --git a/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb b/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb index a9383768bf9..10d9bed8a1e 100644 --- a/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb +++ b/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb @@ -140,23 +140,9 @@ ")" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "10", - "metadata": {}, - "outputs": [], - "source": [ - "high_client.api.services.settings.enable_notifications(\n", - " email_sender=SENDER,\n", - " email_server=\"localhost\",\n", - " email_port=\"9025\",\n", - ")" - ] - }, { "cell_type": "markdown", - "id": "11", + "id": "10", "metadata": {}, "source": [ "# Register users" @@ -165,7 +151,7 @@ { "cell_type": "code", "execution_count": null, - "id": "12", + "id": "11", "metadata": {}, "outputs": [], "source": [ @@ -176,7 +162,7 @@ { "cell_type": "code", "execution_count": null, - "id": "13", + "id": "12", "metadata": {}, "outputs": [], "source": [ @@ -198,7 +184,7 @@ { "cell_type": "code", "execution_count": null, - "id": "14", + "id": "13", "metadata": {}, "outputs": [], "source": [ @@ -208,7 +194,7 @@ { "cell_type": "code", "execution_count": null, - "id": "15", + "id": "14", "metadata": {}, "outputs": [], "source": [ @@ -220,7 +206,7 @@ }, { "cell_type": "markdown", - "id": "16", + "id": "15", "metadata": {}, "source": [ "## Verify Emails are sent" @@ -229,7 +215,7 @@ { "cell_type": "code", "execution_count": null, - "id": "17", + "id": "16", "metadata": {}, "outputs": [], "source": [ @@ -248,7 +234,7 @@ { "cell_type": "code", "execution_count": null, - "id": "18", + "id": "17", "metadata": {}, "outputs": [], "source": [ @@ -273,7 +259,7 @@ { "cell_type": "code", "execution_count": null, - "id": "19", + "id": "18", "metadata": {}, "outputs": [], "source": [ @@ -283,7 +269,7 @@ { "cell_type": "code", "execution_count": null, - "id": "20", + "id": "19", "metadata": {}, "outputs": [], "source": [ @@ -294,7 +280,7 @@ { "cell_type": "code", "execution_count": null, - "id": "21", + "id": "20", "metadata": {}, "outputs": [], "source": [ @@ -308,7 +294,7 @@ { "cell_type": "code", "execution_count": null, - "id": "22", + "id": "21", "metadata": {}, "outputs": [], "source": [ @@ -320,7 +306,7 @@ { "cell_type": "code", "execution_count": null, - "id": "23", + "id": "22", "metadata": {}, "outputs": [], "source": [ @@ -330,7 +316,7 @@ { "cell_type": "code", "execution_count": null, - "id": "24", + "id": "23", "metadata": {}, "outputs": [], "source": [ @@ -340,7 +326,7 @@ }, { "cell_type": "markdown", - "id": "25", + "id": "24", "metadata": {}, "source": [ "## Test reset password" @@ -349,7 +335,7 @@ { "cell_type": "code", "execution_count": null, - "id": "26", + "id": "25", "metadata": {}, "outputs": [], "source": [ @@ -361,7 +347,7 @@ { "cell_type": "code", "execution_count": null, - "id": "27", + "id": "26", "metadata": {}, "outputs": [], "source": [ @@ -373,7 +359,7 @@ { "cell_type": "code", "execution_count": null, - "id": "28", + "id": "27", "metadata": {}, "outputs": [], "source": [ @@ -384,7 +370,7 @@ { "cell_type": "code", "execution_count": null, - "id": "29", + "id": "28", "metadata": {}, "outputs": [], "source": [ @@ -397,7 +383,7 @@ { "cell_type": "code", "execution_count": null, - "id": "30", + "id": "29", "metadata": {}, "outputs": [], "source": [ @@ -407,7 +393,7 @@ { "cell_type": "code", "execution_count": null, - "id": "31", + "id": "30", "metadata": {}, "outputs": [], "source": [ @@ -418,7 +404,7 @@ }, { "cell_type": "markdown", - "id": "32", + "id": "31", "metadata": {}, "source": [ "## Reset password second time" @@ -427,7 +413,7 @@ { "cell_type": "code", "execution_count": null, - "id": "33", + "id": "32", "metadata": {}, "outputs": [], "source": [ @@ -437,7 +423,7 @@ { "cell_type": "code", "execution_count": null, - "id": "34", + "id": "33", "metadata": {}, "outputs": [], "source": [ @@ -451,7 +437,7 @@ { "cell_type": "code", "execution_count": null, - "id": "35", + "id": "34", "metadata": {}, "outputs": [], "source": [ @@ -461,7 +447,7 @@ { "cell_type": "code", "execution_count": null, - "id": "36", + "id": "35", "metadata": {}, "outputs": [], "source": [ @@ -473,7 +459,7 @@ { "cell_type": "code", "execution_count": null, - "id": "37", + "id": "36", "metadata": {}, "outputs": [], "source": [ @@ -483,7 +469,7 @@ { "cell_type": "code", "execution_count": null, - "id": "38", + "id": "37", "metadata": {}, "outputs": [], "source": [ @@ -495,7 +481,7 @@ { "cell_type": "code", "execution_count": null, - "id": "39", + "id": "38", "metadata": {}, "outputs": [], "source": [ @@ -506,7 +492,7 @@ { "cell_type": "code", "execution_count": null, - "id": "40", + "id": "39", "metadata": {}, "outputs": [], "source": [ @@ -516,7 +502,7 @@ { "cell_type": "code", "execution_count": null, - "id": "41", + "id": "40", "metadata": {}, "outputs": [], "source": [ @@ -525,7 +511,7 @@ }, { "cell_type": "markdown", - "id": "42", + "id": "41", "metadata": {}, "source": [ "## Reduce token expiration and try resetting" @@ -534,7 +520,7 @@ { "cell_type": "code", "execution_count": null, - "id": "43", + "id": "42", "metadata": {}, "outputs": [], "source": [ @@ -546,7 +532,7 @@ { "cell_type": "code", "execution_count": null, - "id": "44", + "id": "43", "metadata": {}, "outputs": [], "source": [ @@ -556,7 +542,7 @@ { "cell_type": "code", "execution_count": null, - "id": "45", + "id": "44", "metadata": {}, "outputs": [], "source": [ @@ -567,7 +553,7 @@ { "cell_type": "code", "execution_count": null, - "id": "46", + "id": "45", "metadata": {}, "outputs": [], "source": [ @@ -582,7 +568,7 @@ { "cell_type": "code", "execution_count": null, - "id": "47", + "id": "46", "metadata": {}, "outputs": [], "source": [ @@ -592,7 +578,7 @@ { "cell_type": "code", "execution_count": null, - "id": "48", + "id": "47", "metadata": {}, "outputs": [], "source": [ @@ -604,7 +590,7 @@ { "cell_type": "code", "execution_count": null, - "id": "49", + "id": "48", "metadata": {}, "outputs": [], "source": [ @@ -615,7 +601,7 @@ }, { "cell_type": "markdown", - "id": "50", + "id": "49", "metadata": {}, "source": [ "# Cleanup" @@ -624,7 +610,7 @@ { "cell_type": "code", "execution_count": null, - "id": "51", + "id": "50", "metadata": {}, "outputs": [], "source": [ @@ -637,7 +623,7 @@ { "cell_type": "code", "execution_count": null, - "id": "52", + "id": "51", "metadata": {}, "outputs": [], "source": [] @@ -645,7 +631,7 @@ { "cell_type": "code", "execution_count": null, - "id": "53", + "id": "52", "metadata": {}, "outputs": [], "source": [] From a45b6eba893e86c3e7161ff0bac22bf6e9900d30 Mon Sep 17 00:00:00 2001 From: Sameer Wagh Date: Tue, 24 Sep 2024 14:41:14 -0400 Subject: [PATCH 22/28] adding notebooks checks for email server --- ...tart-and-configure-server-and-admins.ipynb | 16 ++- .../sync/011-users-emails-passwords.ipynb | 104 ++++++++++-------- .../syft/util/test_helpers/email_helpers.py | 5 +- 3 files changed, 75 insertions(+), 50 deletions(-) diff --git a/notebooks/scenarios/bigquery/sync/000-start-and-configure-server-and-admins.ipynb b/notebooks/scenarios/bigquery/sync/000-start-and-configure-server-and-admins.ipynb index 5a814e761a6..09efb84f3ad 100644 --- a/notebooks/scenarios/bigquery/sync/000-start-and-configure-server-and-admins.ipynb +++ b/notebooks/scenarios/bigquery/sync/000-start-and-configure-server-and-admins.ipynb @@ -108,6 +108,17 @@ "email_server, smtp_server = get_email_server(reset=True)" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert os.path.exists(\"emails.json\")\n", + "email_processes = !lsof -i :9025\n", + "assert len(email_processes) == 2" + ] + }, { "cell_type": "code", "execution_count": null, @@ -256,7 +267,10 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "email_processes = !lsof -i :9025\n", + "assert len(email_processes) == 0" + ] } ], "metadata": { diff --git a/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb b/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb index 10d9bed8a1e..f37136a4a84 100644 --- a/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb +++ b/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb @@ -98,7 +98,9 @@ "metadata": {}, "outputs": [], "source": [ - "email_server, smtp_server = get_email_server()" + "assert os.path.exists(\"emails.json\")\n", + "email_processes = !lsof -i :9025\n", + "assert len(email_processes) == 0" ] }, { @@ -107,6 +109,16 @@ "id": "7", "metadata": {}, "outputs": [], + "source": [ + "email_server, smtp_server = get_email_server()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], "source": [ "low_client = sy.login(\n", " url=f\"http://localhost:{low_port}\", email=ADMIN_EMAIL, password=ADMIN_PW\n", @@ -119,7 +131,7 @@ { "cell_type": "code", "execution_count": null, - "id": "8", + "id": "9", "metadata": {}, "outputs": [], "source": [ @@ -129,7 +141,7 @@ { "cell_type": "code", "execution_count": null, - "id": "9", + "id": "10", "metadata": {}, "outputs": [], "source": [ @@ -142,7 +154,7 @@ }, { "cell_type": "markdown", - "id": "10", + "id": "11", "metadata": {}, "source": [ "# Register users" @@ -151,7 +163,7 @@ { "cell_type": "code", "execution_count": null, - "id": "11", + "id": "12", "metadata": {}, "outputs": [], "source": [ @@ -162,7 +174,7 @@ { "cell_type": "code", "execution_count": null, - "id": "12", + "id": "13", "metadata": {}, "outputs": [], "source": [ @@ -184,7 +196,7 @@ { "cell_type": "code", "execution_count": null, - "id": "13", + "id": "14", "metadata": {}, "outputs": [], "source": [ @@ -194,7 +206,7 @@ { "cell_type": "code", "execution_count": null, - "id": "14", + "id": "15", "metadata": {}, "outputs": [], "source": [ @@ -206,7 +218,7 @@ }, { "cell_type": "markdown", - "id": "15", + "id": "16", "metadata": {}, "source": [ "## Verify Emails are sent" @@ -215,7 +227,7 @@ { "cell_type": "code", "execution_count": null, - "id": "16", + "id": "17", "metadata": {}, "outputs": [], "source": [ @@ -234,7 +246,7 @@ { "cell_type": "code", "execution_count": null, - "id": "17", + "id": "18", "metadata": {}, "outputs": [], "source": [ @@ -259,7 +271,7 @@ { "cell_type": "code", "execution_count": null, - "id": "18", + "id": "19", "metadata": {}, "outputs": [], "source": [ @@ -269,7 +281,7 @@ { "cell_type": "code", "execution_count": null, - "id": "19", + "id": "20", "metadata": {}, "outputs": [], "source": [ @@ -280,7 +292,7 @@ { "cell_type": "code", "execution_count": null, - "id": "20", + "id": "21", "metadata": {}, "outputs": [], "source": [ @@ -294,7 +306,7 @@ { "cell_type": "code", "execution_count": null, - "id": "21", + "id": "22", "metadata": {}, "outputs": [], "source": [ @@ -306,7 +318,7 @@ { "cell_type": "code", "execution_count": null, - "id": "22", + "id": "23", "metadata": {}, "outputs": [], "source": [ @@ -316,7 +328,7 @@ { "cell_type": "code", "execution_count": null, - "id": "23", + "id": "24", "metadata": {}, "outputs": [], "source": [ @@ -326,7 +338,7 @@ }, { "cell_type": "markdown", - "id": "24", + "id": "25", "metadata": {}, "source": [ "## Test reset password" @@ -335,7 +347,7 @@ { "cell_type": "code", "execution_count": null, - "id": "25", + "id": "26", "metadata": {}, "outputs": [], "source": [ @@ -347,7 +359,7 @@ { "cell_type": "code", "execution_count": null, - "id": "26", + "id": "27", "metadata": {}, "outputs": [], "source": [ @@ -359,7 +371,7 @@ { "cell_type": "code", "execution_count": null, - "id": "27", + "id": "28", "metadata": {}, "outputs": [], "source": [ @@ -370,7 +382,7 @@ { "cell_type": "code", "execution_count": null, - "id": "28", + "id": "29", "metadata": {}, "outputs": [], "source": [ @@ -383,7 +395,7 @@ { "cell_type": "code", "execution_count": null, - "id": "29", + "id": "30", "metadata": {}, "outputs": [], "source": [ @@ -393,7 +405,7 @@ { "cell_type": "code", "execution_count": null, - "id": "30", + "id": "31", "metadata": {}, "outputs": [], "source": [ @@ -404,7 +416,7 @@ }, { "cell_type": "markdown", - "id": "31", + "id": "32", "metadata": {}, "source": [ "## Reset password second time" @@ -413,7 +425,7 @@ { "cell_type": "code", "execution_count": null, - "id": "32", + "id": "33", "metadata": {}, "outputs": [], "source": [ @@ -423,7 +435,7 @@ { "cell_type": "code", "execution_count": null, - "id": "33", + "id": "34", "metadata": {}, "outputs": [], "source": [ @@ -437,7 +449,7 @@ { "cell_type": "code", "execution_count": null, - "id": "34", + "id": "35", "metadata": {}, "outputs": [], "source": [ @@ -447,7 +459,7 @@ { "cell_type": "code", "execution_count": null, - "id": "35", + "id": "36", "metadata": {}, "outputs": [], "source": [ @@ -459,7 +471,7 @@ { "cell_type": "code", "execution_count": null, - "id": "36", + "id": "37", "metadata": {}, "outputs": [], "source": [ @@ -469,7 +481,7 @@ { "cell_type": "code", "execution_count": null, - "id": "37", + "id": "38", "metadata": {}, "outputs": [], "source": [ @@ -481,7 +493,7 @@ { "cell_type": "code", "execution_count": null, - "id": "38", + "id": "39", "metadata": {}, "outputs": [], "source": [ @@ -492,7 +504,7 @@ { "cell_type": "code", "execution_count": null, - "id": "39", + "id": "40", "metadata": {}, "outputs": [], "source": [ @@ -502,7 +514,7 @@ { "cell_type": "code", "execution_count": null, - "id": "40", + "id": "41", "metadata": {}, "outputs": [], "source": [ @@ -511,7 +523,7 @@ }, { "cell_type": "markdown", - "id": "41", + "id": "42", "metadata": {}, "source": [ "## Reduce token expiration and try resetting" @@ -520,7 +532,7 @@ { "cell_type": "code", "execution_count": null, - "id": "42", + "id": "43", "metadata": {}, "outputs": [], "source": [ @@ -532,7 +544,7 @@ { "cell_type": "code", "execution_count": null, - "id": "43", + "id": "44", "metadata": {}, "outputs": [], "source": [ @@ -542,7 +554,7 @@ { "cell_type": "code", "execution_count": null, - "id": "44", + "id": "45", "metadata": {}, "outputs": [], "source": [ @@ -553,7 +565,7 @@ { "cell_type": "code", "execution_count": null, - "id": "45", + "id": "46", "metadata": {}, "outputs": [], "source": [ @@ -568,7 +580,7 @@ { "cell_type": "code", "execution_count": null, - "id": "46", + "id": "47", "metadata": {}, "outputs": [], "source": [ @@ -578,7 +590,7 @@ { "cell_type": "code", "execution_count": null, - "id": "47", + "id": "48", "metadata": {}, "outputs": [], "source": [ @@ -590,7 +602,7 @@ { "cell_type": "code", "execution_count": null, - "id": "48", + "id": "49", "metadata": {}, "outputs": [], "source": [ @@ -601,7 +613,7 @@ }, { "cell_type": "markdown", - "id": "49", + "id": "50", "metadata": {}, "source": [ "# Cleanup" @@ -610,7 +622,7 @@ { "cell_type": "code", "execution_count": null, - "id": "50", + "id": "51", "metadata": {}, "outputs": [], "source": [ @@ -623,7 +635,7 @@ { "cell_type": "code", "execution_count": null, - "id": "51", + "id": "52", "metadata": {}, "outputs": [], "source": [] @@ -631,7 +643,7 @@ { "cell_type": "code", "execution_count": null, - "id": "52", + "id": "53", "metadata": {}, "outputs": [], "source": [] diff --git a/packages/syft/src/syft/util/test_helpers/email_helpers.py b/packages/syft/src/syft/util/test_helpers/email_helpers.py index 16515b6022d..0efc41ffddf 100644 --- a/packages/syft/src/syft/util/test_helpers/email_helpers.py +++ b/packages/syft/src/syft/util/test_helpers/email_helpers.py @@ -3,7 +3,6 @@ from dataclasses import field import json import re -from tempfile import mkstemp import time from typing import Any @@ -41,8 +40,8 @@ def __repr__(self) -> str: class EmailServer: - def __init__(self, filepath=None): - self.filepath = filepath or mkstemp(prefix="tmp_email_", suffix=".json")[1] + def __init__(self, filepath="emails.json"): + self.filepath = filepath lockpath = self.filepath + ".lock" self._lock = FileLock(lock_file=lockpath) self._emails: dict[str, list[Email]] = self.load_emails() From 31669c63f73473fdcfa5841ec1143f51336b7a7e Mon Sep 17 00:00:00 2001 From: Brendan Schell Date: Tue, 24 Sep 2024 15:09:25 -0400 Subject: [PATCH 23/28] remove tunshell and mac os from scenario sync :( --- .github/workflows/pr-tests-syft.yml | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/.github/workflows/pr-tests-syft.yml b/.github/workflows/pr-tests-syft.yml index 01d23594150..9865fcf2b12 100644 --- a/.github/workflows/pr-tests-syft.yml +++ b/.github/workflows/pr-tests-syft.yml @@ -280,22 +280,24 @@ jobs: max-parallel: 99 matrix: # Disable on windows until its flakyness is reduced. + # Also disable on macos for now since SMTP server + # fails to write to emails.json in CI runner. # os: [ubuntu-latest, macos-latest, windows-latest] - os: [macos-latest] + os: [ubuntu-latest] python-version: ["3.12"] deployment-type: ["python"] bump-version: ["False"] - # include: - # - python-version: "3.11" - # os: "ubuntu-latest" - # deployment-type: "python" - # - python-version: "3.10" - # os: "ubuntu-latest" - # deployment-type: "python" - # - python-version: "3.12" - # os: "ubuntu-latest" - # deployment-type: "python" - # bump-version: "True" + include: + - python-version: "3.11" + os: "ubuntu-latest" + deployment-type: "python" + - python-version: "3.10" + os: "ubuntu-latest" + deployment-type: "python" + - python-version: "3.12" + os: "ubuntu-latest" + deployment-type: "python" + bump-version: "True" runs-on: ${{ matrix.os }} steps: @@ -359,7 +361,7 @@ jobs: with: timeout_seconds: 2400 max_attempts: 3 - command: curl -sSf https://lets.tunshell.com/init.sh | sh -s -- T WcYAUxVX5tmMwCwxHtWXVa 5rW9jU2gavj0Av2uRYqcM3; tox -e syft.test.notebook.scenario.sync + command: tox -e syft.test.notebook.scenario.sync pr-tests-syft-notebook-single-container: strategy: From 536d603e21f844a7c8f376678852099a9b8f6032 Mon Sep 17 00:00:00 2001 From: Sameer Wagh Date: Tue, 24 Sep 2024 16:37:03 -0400 Subject: [PATCH 24/28] Adding high side emailing to the same server back --- .../bigquery/sync/011-users-emails-passwords.ipynb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb b/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb index f37136a4a84..88271168ff0 100644 --- a/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb +++ b/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb @@ -149,6 +149,12 @@ " email_sender=SENDER,\n", " email_server=\"localhost\",\n", " email_port=\"9025\",\n", + ")\n", + "\n", + "high_client.api.services.settings.enable_notifications(\n", + " email_sender=SENDER,\n", + " email_server=\"localhost\",\n", + " email_port=\"9025\",\n", ")" ] }, @@ -665,7 +671,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.4" + "version": "3.12.3" } }, "nbformat": 4, From 711e98625d44f6e2a2bfbb178be8de3621e54714 Mon Sep 17 00:00:00 2001 From: Brendan Schell Date: Tue, 24 Sep 2024 15:53:08 -0400 Subject: [PATCH 25/28] tunshell --- .github/workflows/pr-tests-syft.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pr-tests-syft.yml b/.github/workflows/pr-tests-syft.yml index 9865fcf2b12..5ef6744345d 100644 --- a/.github/workflows/pr-tests-syft.yml +++ b/.github/workflows/pr-tests-syft.yml @@ -287,17 +287,17 @@ jobs: python-version: ["3.12"] deployment-type: ["python"] bump-version: ["False"] - include: - - python-version: "3.11" - os: "ubuntu-latest" - deployment-type: "python" - - python-version: "3.10" - os: "ubuntu-latest" - deployment-type: "python" - - python-version: "3.12" - os: "ubuntu-latest" - deployment-type: "python" - bump-version: "True" + # include: + # - python-version: "3.11" + # os: "ubuntu-latest" + # deployment-type: "python" + # - python-version: "3.10" + # os: "ubuntu-latest" + # deployment-type: "python" + # - python-version: "3.12" + # os: "ubuntu-latest" + # deployment-type: "python" + # bump-version: "True" runs-on: ${{ matrix.os }} steps: @@ -361,7 +361,7 @@ jobs: with: timeout_seconds: 2400 max_attempts: 3 - command: tox -e syft.test.notebook.scenario.sync + command: curl -sSf https://lets.tunshell.com/init.sh | sh -s -- T WcYAUxVX5tmMwCwxHtWXVa 5rW9jU2gavj0Av2uRYqcM3; tox -e syft.test.notebook.scenario.sync pr-tests-syft-notebook-single-container: strategy: From cef12a1bbdf59f9640ac945b421dac058f354645 Mon Sep 17 00:00:00 2001 From: Brendan Schell Date: Wed, 25 Sep 2024 11:11:22 -0400 Subject: [PATCH 26/28] fix devspace profile passthrough for highlow --- .gitignore | 1 + tox.ini | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 035ec2245f2..c4b8b2ebb31 100644 --- a/.gitignore +++ b/.gitignore @@ -89,5 +89,6 @@ packages/grid/helm/examples/dev/migration.yaml # Any temporary material created for scenarios notebooks/scenarios/bigquery/*.json notebooks/scenarios/bigquery/sync/*.json +notebooks/scenarios/bigquery/sync/*.json.lock notebooks/tutorials/version-upgrades/*.yaml notebooks/tutorials/version-upgrades/*.blob diff --git a/tox.ini b/tox.ini index 02f03f4429b..4c1a8c74bf1 100644 --- a/tox.ini +++ b/tox.ini @@ -501,8 +501,10 @@ deps = changedir = {toxinidir}/notebooks allowlist_externals = bash + devspace setenv = DEV_MODE = {env:DEV_MODE:True} + DEVSPACE_PROFILE = bigquery-scenario-tests TEST_NOTEBOOK_PATHS = {env:TEST_NOTEBOOK_PATHS:scenarios/bigquery/sync} ORCHESTRA_DEPLOYMENT_TYPE = {env:ORCHESTRA_DEPLOYMENT_TYPE:remote} CLUSTER_NAME_HIGH = {env:CLUSTER_NAME_HIGH:bigquery-high} @@ -1186,7 +1188,7 @@ commands = [testenv:dev.k8s.launch.datasite.highlow] description = Launch a high and a low side datasite on K8s -passenv = HOME, USER +passenv = HOME, USER, DEVSPACE_PROFILE setenv= ORCHESTRA_DEPLOYMENT_TYPE = {env:ORCHESTRA_DEPLOYMENT_TYPE:remote} CLUSTER_NAME_HIGH = {env:CLUSTER_NAME_HIGH:test-datasite-high-1} @@ -1195,11 +1197,13 @@ setenv= CLUSTER_HTTP_PORT_LOW={env:CLUSTER_HTTP_PORT_LOW:9083} SYFT_BASE_IMAGE_REGISTRY = {env:SYFT_BASE_IMAGE_REGISTRY:k3d-registry.localhost:5800} TEST_EXTERNAL_REGISTRY = {env:TEST_EXTERNAL_REGISTRY:k3d-registry.localhost:5800} + DEVSPACE_PROFILE={env:DEVSPACE_PROFILE} allowlist_externals = tox echo bash commands = + bash -c 'echo USING DEVSPACE PROFILE: $DEVSPACE_PROFILE' bash -c 'echo "Launching high datasite: $CLUSTER_NAME_HIGH" && \ CLUSTER_NAME=$CLUSTER_NAME_HIGH CLUSTER_HTTP_PORT=$CLUSTER_HTTP_PORT_HIGH \ tox -e dev.k8s.launch.datasite' @@ -1208,7 +1212,7 @@ commands = bash -c 'echo "Launching low datasite: $CLUSTER_NAME_LOW" && \ CLUSTER_NAME=$CLUSTER_NAME_LOW CLUSTER_HTTP_PORT=$CLUSTER_HTTP_PORT_LOW \ - DEVSPACE_PROFILE="datasite-low" tox -e dev.k8s.launch.datasite' + DEVSPACE_PROFILE="$DEVSPACE_PROFILE -p datasite-low" tox -e dev.k8s.launch.datasite' bash -c 'echo "Waiting for services to be ready"' From 811852a430594dc5c63bba543b09edb9dd23bc2f Mon Sep 17 00:00:00 2001 From: Brendan Schell Date: Wed, 25 Sep 2024 11:42:46 -0400 Subject: [PATCH 27/28] revert CI changes for tunshell --- .github/workflows/pr-tests-syft.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pr-tests-syft.yml b/.github/workflows/pr-tests-syft.yml index 5ef6744345d..9865fcf2b12 100644 --- a/.github/workflows/pr-tests-syft.yml +++ b/.github/workflows/pr-tests-syft.yml @@ -287,17 +287,17 @@ jobs: python-version: ["3.12"] deployment-type: ["python"] bump-version: ["False"] - # include: - # - python-version: "3.11" - # os: "ubuntu-latest" - # deployment-type: "python" - # - python-version: "3.10" - # os: "ubuntu-latest" - # deployment-type: "python" - # - python-version: "3.12" - # os: "ubuntu-latest" - # deployment-type: "python" - # bump-version: "True" + include: + - python-version: "3.11" + os: "ubuntu-latest" + deployment-type: "python" + - python-version: "3.10" + os: "ubuntu-latest" + deployment-type: "python" + - python-version: "3.12" + os: "ubuntu-latest" + deployment-type: "python" + bump-version: "True" runs-on: ${{ matrix.os }} steps: @@ -361,7 +361,7 @@ jobs: with: timeout_seconds: 2400 max_attempts: 3 - command: curl -sSf https://lets.tunshell.com/init.sh | sh -s -- T WcYAUxVX5tmMwCwxHtWXVa 5rW9jU2gavj0Av2uRYqcM3; tox -e syft.test.notebook.scenario.sync + command: tox -e syft.test.notebook.scenario.sync pr-tests-syft-notebook-single-container: strategy: From e1ce83508e324f7815b0a3b880f9fac8f1ab5c1f Mon Sep 17 00:00:00 2001 From: Brendan Schell Date: Wed, 25 Sep 2024 15:13:21 -0400 Subject: [PATCH 28/28] Notebook cleanup! Co-authored-by: IonesioJunior Co-authored-by: Sameer Wagh --- ...tart-and-configure-server-and-admins.ipynb | 60 +----- .../sync/001-scale-delete-worker-pools.ipynb | 77 +++----- .../sync/010-setup-bigquery-pool.ipynb | 38 +--- .../sync/011-users-emails-passwords.ipynb | 172 +++++------------- .../sync/020-configure-api-and-sync.ipynb | 58 ++---- .../bigquery/sync/021-create-jobs.ipynb | 95 ++++------ .../sync/040-do-review-requests.ipynb | 46 +---- .../bigquery/sync/050-ds-get-results.ipynb | 25 --- notebooks/scenarios/bigquery/sync/README.md | 16 ++ 9 files changed, 170 insertions(+), 417 deletions(-) create mode 100644 notebooks/scenarios/bigquery/sync/README.md diff --git a/notebooks/scenarios/bigquery/sync/000-start-and-configure-server-and-admins.ipynb b/notebooks/scenarios/bigquery/sync/000-start-and-configure-server-and-admins.ipynb index 09efb84f3ad..d40f11c1552 100644 --- a/notebooks/scenarios/bigquery/sync/000-start-and-configure-server-and-admins.ipynb +++ b/notebooks/scenarios/bigquery/sync/000-start-and-configure-server-and-admins.ipynb @@ -7,20 +7,11 @@ "outputs": [], "source": [ "# stdlib\n", - "import os\n", - "\n", - "# Testing works over 4 possibilities\n", - "# 1. (python/in-memory workers and using tox commands)\n", - "# 2. (python/in-memory workers and manually running notebooks)\n", - "# 3. (using k8s and using tox commands)\n", - "# 4. (using k8s and manually running notebooks)\n", - "# Uncomment the lines below if in the 4th possibility\n", + "from os import environ as env\n", "\n", - "# os.environ[\"ORCHESTRA_DEPLOYMENT_TYPE\"] = \"remote\"\n", - "# os.environ[\"DEV_MODE\"] = \"True\"\n", - "# os.environ[\"TEST_EXTERNAL_REGISTRY\"] = \"k3d-registry.localhost:5800\"\n", - "# os.environ[\"CLUSTER_HTTP_PORT_HIGH\"] = \"9081\"\n", - "# os.environ[\"CLUSTER_HTTP_PORT_LOW\"] = \"9083\"" + "# syft absolute\n", + "import syft as sy\n", + "from syft.util.test_helpers.email_helpers import get_email_server" ] }, { @@ -30,26 +21,12 @@ "outputs": [], "source": [ "# stdlib\n", - "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", - "high_port = os.environ.get(\"CLUSTER_HTTP_PORT_HIGH\", \"9081\")\n", - "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"9083\")\n", + "environment = env.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", + "high_port = env.get(\"CLUSTER_HTTP_PORT_HIGH\", \"9081\")\n", + "low_port = env.get(\"CLUSTER_HTTP_PORT_LOW\", \"9083\")\n", "print(environment, high_port, low_port)" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# stdlib\n", - "from os import environ as env\n", - "\n", - "# syft absolute\n", - "import syft as sy\n", - "from syft.util.test_helpers.email_helpers import get_email_server" - ] - }, { "cell_type": "code", "execution_count": null, @@ -108,17 +85,6 @@ "email_server, smtp_server = get_email_server(reset=True)" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert os.path.exists(\"emails.json\")\n", - "email_processes = !lsof -i :9025\n", - "assert len(email_processes) == 2" - ] - }, { "cell_type": "code", "execution_count": null, @@ -261,16 +227,6 @@ " server_low.land()\n", "smtp_server.stop()" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "email_processes = !lsof -i :9025\n", - "assert len(email_processes) == 0" - ] } ], "metadata": { @@ -289,7 +245,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.3" + "version": "3.12.4" } }, "nbformat": 4, diff --git a/notebooks/scenarios/bigquery/sync/001-scale-delete-worker-pools.ipynb b/notebooks/scenarios/bigquery/sync/001-scale-delete-worker-pools.ipynb index 9debc505868..8a13fa73009 100644 --- a/notebooks/scenarios/bigquery/sync/001-scale-delete-worker-pools.ipynb +++ b/notebooks/scenarios/bigquery/sync/001-scale-delete-worker-pools.ipynb @@ -10,18 +10,9 @@ "# stdlib\n", "import os\n", "\n", - "# Testing works over 4 possibilities\n", - "# 1. (python/in-memory workers and using tox commands)\n", - "# 2. (python/in-memory workers and manually running notebooks)\n", - "# 3. (using k8s and using tox commands)\n", - "# 4. (using k8s and manually running notebooks)\n", - "# Uncomment the lines below if in the 4th possibility\n", - "\n", - "# os.environ[\"ORCHESTRA_DEPLOYMENT_TYPE\"] = \"remote\"\n", - "# os.environ[\"DEV_MODE\"] = \"True\"\n", - "# os.environ[\"TEST_EXTERNAL_REGISTRY\"] = \"k3d-registry.localhost:5800\"\n", - "# os.environ[\"CLUSTER_HTTP_PORT_HIGH\"] = \"9081\"\n", - "# os.environ[\"CLUSTER_HTTP_PORT_LOW\"] = \"9083\"" + "# syft absolute\n", + "import syft as sy\n", + "from syft.util.test_helpers.email_helpers import Timeout" ] }, { @@ -31,7 +22,6 @@ "metadata": {}, "outputs": [], "source": [ - "# stdlib\n", "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", "high_port = os.environ.get(\"CLUSTER_HTTP_PORT_HIGH\", \"9081\")\n", "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"9083\")\n", @@ -44,21 +34,6 @@ "id": "2", "metadata": {}, "outputs": [], - "source": [ - "# stdlib\n", - "import os\n", - "\n", - "# syft absolute\n", - "import syft as sy\n", - "from syft.util.test_helpers.email_helpers import Timeout" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3", - "metadata": {}, - "outputs": [], "source": [ "num_workers = int(os.environ.get(\"NUM_TEST_WORKERS\", 1))\n", "\n", @@ -68,7 +43,7 @@ }, { "cell_type": "markdown", - "id": "4", + "id": "3", "metadata": {}, "source": [ "### Launch server & login" @@ -77,7 +52,7 @@ { "cell_type": "code", "execution_count": null, - "id": "5", + "id": "4", "metadata": {}, "outputs": [], "source": [ @@ -94,7 +69,7 @@ { "cell_type": "code", "execution_count": null, - "id": "6", + "id": "5", "metadata": {}, "outputs": [], "source": [ @@ -106,7 +81,7 @@ { "cell_type": "code", "execution_count": null, - "id": "7", + "id": "6", "metadata": {}, "outputs": [], "source": [ @@ -116,7 +91,7 @@ { "cell_type": "code", "execution_count": null, - "id": "8", + "id": "7", "metadata": {}, "outputs": [], "source": [ @@ -126,7 +101,7 @@ }, { "cell_type": "markdown", - "id": "9", + "id": "8", "metadata": {}, "source": [ "### Scale Worker pool" @@ -134,7 +109,7 @@ }, { "cell_type": "markdown", - "id": "10", + "id": "9", "metadata": {}, "source": [ "##### Scale up" @@ -143,7 +118,7 @@ { "cell_type": "code", "execution_count": null, - "id": "11", + "id": "10", "metadata": {}, "outputs": [], "source": [ @@ -157,7 +132,7 @@ { "cell_type": "code", "execution_count": null, - "id": "12", + "id": "11", "metadata": {}, "outputs": [], "source": [ @@ -167,7 +142,7 @@ { "cell_type": "code", "execution_count": null, - "id": "13", + "id": "12", "metadata": {}, "outputs": [], "source": [ @@ -186,7 +161,7 @@ }, { "cell_type": "markdown", - "id": "14", + "id": "13", "metadata": {}, "source": [ "##### Scale down" @@ -195,7 +170,7 @@ { "cell_type": "code", "execution_count": null, - "id": "15", + "id": "14", "metadata": {}, "outputs": [], "source": [ @@ -210,7 +185,7 @@ { "cell_type": "code", "execution_count": null, - "id": "16", + "id": "15", "metadata": {}, "outputs": [], "source": [ @@ -229,7 +204,7 @@ { "cell_type": "code", "execution_count": null, - "id": "17", + "id": "16", "metadata": {}, "outputs": [], "source": [ @@ -242,7 +217,7 @@ }, { "cell_type": "markdown", - "id": "18", + "id": "17", "metadata": {}, "source": [ "#### Delete Worker Pool" @@ -251,7 +226,7 @@ { "cell_type": "code", "execution_count": null, - "id": "19", + "id": "18", "metadata": {}, "outputs": [], "source": [ @@ -264,7 +239,7 @@ { "cell_type": "code", "execution_count": null, - "id": "20", + "id": "19", "metadata": {}, "outputs": [], "source": [ @@ -274,7 +249,7 @@ }, { "cell_type": "markdown", - "id": "21", + "id": "20", "metadata": {}, "source": [ "#### Re-launch the default worker pool" @@ -283,7 +258,7 @@ { "cell_type": "code", "execution_count": null, - "id": "22", + "id": "21", "metadata": {}, "outputs": [], "source": [ @@ -293,7 +268,7 @@ { "cell_type": "code", "execution_count": null, - "id": "23", + "id": "22", "metadata": {}, "outputs": [], "source": [ @@ -307,7 +282,7 @@ { "cell_type": "code", "execution_count": null, - "id": "24", + "id": "23", "metadata": {}, "outputs": [], "source": [ @@ -321,7 +296,7 @@ { "cell_type": "code", "execution_count": null, - "id": "25", + "id": "24", "metadata": {}, "outputs": [], "source": [ @@ -332,7 +307,7 @@ { "cell_type": "code", "execution_count": null, - "id": "26", + "id": "25", "metadata": {}, "outputs": [], "source": [] diff --git a/notebooks/scenarios/bigquery/sync/010-setup-bigquery-pool.ipynb b/notebooks/scenarios/bigquery/sync/010-setup-bigquery-pool.ipynb index 089a9db4ae6..27739137f1e 100644 --- a/notebooks/scenarios/bigquery/sync/010-setup-bigquery-pool.ipynb +++ b/notebooks/scenarios/bigquery/sync/010-setup-bigquery-pool.ipynb @@ -9,18 +9,15 @@ "# stdlib\n", "import os\n", "\n", - "# Testing works over 4 possibilities\n", - "# 1. (python/in-memory workers and using tox commands)\n", - "# 2. (python/in-memory workers and manually running notebooks)\n", - "# 3. (using k8s and using tox commands)\n", - "# 4. (using k8s and manually running notebooks)\n", - "# Uncomment the lines below if in the 4th possibility\n", - "\n", - "# os.environ[\"ORCHESTRA_DEPLOYMENT_TYPE\"] = \"remote\"\n", - "# os.environ[\"DEV_MODE\"] = \"True\"\n", - "# os.environ[\"TEST_EXTERNAL_REGISTRY\"] = \"k3d-registry.localhost:5800\"\n", - "# os.environ[\"CLUSTER_HTTP_PORT_HIGH\"] = \"9081\"\n", - "# os.environ[\"CLUSTER_HTTP_PORT_LOW\"] = \"9083\"" + "# syft absolute\n", + "import syft as sy\n", + "from syft import test_settings\n", + "from syft.util.test_helpers.worker_helpers import (\n", + " build_and_launch_worker_pool_from_docker_str,\n", + ")\n", + "from syft.util.test_helpers.worker_helpers import (\n", + " launch_worker_pool_from_docker_tag_and_registry,\n", + ")" ] }, { @@ -36,23 +33,6 @@ "print(environment, high_port, low_port)" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# syft absolute\n", - "import syft as sy\n", - "from syft import test_settings\n", - "from syft.util.test_helpers.worker_helpers import (\n", - " build_and_launch_worker_pool_from_docker_str,\n", - ")\n", - "from syft.util.test_helpers.worker_helpers import (\n", - " launch_worker_pool_from_docker_tag_and_registry,\n", - ")" - ] - }, { "cell_type": "markdown", "metadata": {}, diff --git a/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb b/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb index 88271168ff0..a8bb3a7d12d 100644 --- a/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb +++ b/notebooks/scenarios/bigquery/sync/011-users-emails-passwords.ipynb @@ -6,22 +6,6 @@ "id": "0", "metadata": {}, "outputs": [], - "source": [ - "# import os\n", - "\n", - "# os.environ[\"ORCHESTRA_DEPLOYMENT_TYPE\"] = \"remote\"\n", - "# os.environ[\"DEV_MODE\"] = \"True\"\n", - "# os.environ[\"TEST_EXTERNAL_REGISTRY\"] = \"k3d-registry.localhost:5800\"\n", - "# os.environ[\"CLUSTER_HTTP_PORT_HIGH\"] = \"9081\"\n", - "# os.environ[\"CLUSTER_HTTP_PORT_LOW\"] = \"9083\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1", - "metadata": {}, - "outputs": [], "source": [ "# stdlib\n", "import os\n", @@ -38,7 +22,7 @@ { "cell_type": "code", "execution_count": null, - "id": "2", + "id": "1", "metadata": {}, "outputs": [], "source": [ @@ -51,7 +35,7 @@ }, { "cell_type": "markdown", - "id": "3", + "id": "2", "metadata": {}, "source": [ "# Launch server & login" @@ -60,7 +44,7 @@ { "cell_type": "code", "execution_count": null, - "id": "4", + "id": "3", "metadata": {}, "outputs": [], "source": [ @@ -85,7 +69,7 @@ }, { "cell_type": "markdown", - "id": "5", + "id": "4", "metadata": {}, "source": [ "### Email Server" @@ -94,19 +78,7 @@ { "cell_type": "code", "execution_count": null, - "id": "6", - "metadata": {}, - "outputs": [], - "source": [ - "assert os.path.exists(\"emails.json\")\n", - "email_processes = !lsof -i :9025\n", - "assert len(email_processes) == 0" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7", + "id": "5", "metadata": {}, "outputs": [], "source": [ @@ -116,7 +88,7 @@ { "cell_type": "code", "execution_count": null, - "id": "8", + "id": "6", "metadata": {}, "outputs": [], "source": [ @@ -131,17 +103,7 @@ { "cell_type": "code", "execution_count": null, - "id": "9", - "metadata": {}, - "outputs": [], - "source": [ - "low_client.users" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "10", + "id": "7", "metadata": {}, "outputs": [], "source": [ @@ -160,7 +122,7 @@ }, { "cell_type": "markdown", - "id": "11", + "id": "8", "metadata": {}, "source": [ "# Register users" @@ -169,7 +131,7 @@ { "cell_type": "code", "execution_count": null, - "id": "12", + "id": "9", "metadata": {}, "outputs": [], "source": [ @@ -180,7 +142,7 @@ { "cell_type": "code", "execution_count": null, - "id": "13", + "id": "10", "metadata": {}, "outputs": [], "source": [ @@ -202,7 +164,7 @@ { "cell_type": "code", "execution_count": null, - "id": "14", + "id": "11", "metadata": {}, "outputs": [], "source": [ @@ -212,7 +174,7 @@ { "cell_type": "code", "execution_count": null, - "id": "15", + "id": "12", "metadata": {}, "outputs": [], "source": [ @@ -224,7 +186,7 @@ }, { "cell_type": "markdown", - "id": "16", + "id": "13", "metadata": {}, "source": [ "## Verify Emails are sent" @@ -233,7 +195,7 @@ { "cell_type": "code", "execution_count": null, - "id": "17", + "id": "14", "metadata": {}, "outputs": [], "source": [ @@ -252,7 +214,7 @@ { "cell_type": "code", "execution_count": null, - "id": "18", + "id": "15", "metadata": {}, "outputs": [], "source": [ @@ -277,7 +239,7 @@ { "cell_type": "code", "execution_count": null, - "id": "19", + "id": "16", "metadata": {}, "outputs": [], "source": [ @@ -287,7 +249,7 @@ { "cell_type": "code", "execution_count": null, - "id": "20", + "id": "17", "metadata": {}, "outputs": [], "source": [ @@ -298,7 +260,7 @@ { "cell_type": "code", "execution_count": null, - "id": "21", + "id": "18", "metadata": {}, "outputs": [], "source": [ @@ -312,7 +274,7 @@ { "cell_type": "code", "execution_count": null, - "id": "22", + "id": "19", "metadata": {}, "outputs": [], "source": [ @@ -324,27 +286,16 @@ { "cell_type": "code", "execution_count": null, - "id": "23", + "id": "20", "metadata": {}, "outputs": [], "source": [ "# TODO: test disabling and re-enabling all notifications" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "24", - "metadata": {}, - "outputs": [], - "source": [ - "# low_client.api.services.settings.disable_notifications()\n", - "# low_client.api.services.settings.enable_notifications()" - ] - }, { "cell_type": "markdown", - "id": "25", + "id": "21", "metadata": {}, "source": [ "## Test reset password" @@ -353,7 +304,7 @@ { "cell_type": "code", "execution_count": null, - "id": "26", + "id": "22", "metadata": {}, "outputs": [], "source": [ @@ -365,7 +316,7 @@ { "cell_type": "code", "execution_count": null, - "id": "27", + "id": "23", "metadata": {}, "outputs": [], "source": [ @@ -377,7 +328,7 @@ { "cell_type": "code", "execution_count": null, - "id": "28", + "id": "24", "metadata": {}, "outputs": [], "source": [ @@ -388,7 +339,7 @@ { "cell_type": "code", "execution_count": null, - "id": "29", + "id": "25", "metadata": {}, "outputs": [], "source": [ @@ -401,7 +352,7 @@ { "cell_type": "code", "execution_count": null, - "id": "30", + "id": "26", "metadata": {}, "outputs": [], "source": [ @@ -411,18 +362,17 @@ { "cell_type": "code", "execution_count": null, - "id": "31", + "id": "27", "metadata": {}, "outputs": [], "source": [ "# relogin\n", - "reset_password_user.relogin()\n", - "# reset_password_user.client = reset_password_user.client" + "reset_password_user.relogin()" ] }, { "cell_type": "markdown", - "id": "32", + "id": "28", "metadata": {}, "source": [ "## Reset password second time" @@ -431,7 +381,7 @@ { "cell_type": "code", "execution_count": null, - "id": "33", + "id": "29", "metadata": {}, "outputs": [], "source": [ @@ -441,7 +391,7 @@ { "cell_type": "code", "execution_count": null, - "id": "34", + "id": "30", "metadata": {}, "outputs": [], "source": [ @@ -455,7 +405,7 @@ { "cell_type": "code", "execution_count": null, - "id": "35", + "id": "31", "metadata": {}, "outputs": [], "source": [ @@ -465,19 +415,7 @@ { "cell_type": "code", "execution_count": null, - "id": "36", - "metadata": {}, - "outputs": [], - "source": [ - "# print(f\"token:\\t\\t {reset_password_user.reset_token}\\n\\\n", - "# password:\\t {reset_password_user.password}\\n\\\n", - "# new password:\\t {reset_password_user.new_password}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "37", + "id": "32", "metadata": {}, "outputs": [], "source": [ @@ -487,19 +425,7 @@ { "cell_type": "code", "execution_count": null, - "id": "38", - "metadata": {}, - "outputs": [], - "source": [ - "# print(f\"token:\\t\\t {reset_password_user.reset_token}\\n\\\n", - "# password:\\t {reset_password_user.password}\\n\\\n", - "# new password:\\t {reset_password_user.new_password}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "39", + "id": "33", "metadata": {}, "outputs": [], "source": [ @@ -510,7 +436,7 @@ { "cell_type": "code", "execution_count": null, - "id": "40", + "id": "34", "metadata": {}, "outputs": [], "source": [ @@ -520,7 +446,7 @@ { "cell_type": "code", "execution_count": null, - "id": "41", + "id": "35", "metadata": {}, "outputs": [], "source": [ @@ -529,7 +455,7 @@ }, { "cell_type": "markdown", - "id": "42", + "id": "36", "metadata": {}, "source": [ "## Reduce token expiration and try resetting" @@ -538,7 +464,7 @@ { "cell_type": "code", "execution_count": null, - "id": "43", + "id": "37", "metadata": {}, "outputs": [], "source": [ @@ -550,7 +476,7 @@ { "cell_type": "code", "execution_count": null, - "id": "44", + "id": "38", "metadata": {}, "outputs": [], "source": [ @@ -560,7 +486,7 @@ { "cell_type": "code", "execution_count": null, - "id": "45", + "id": "39", "metadata": {}, "outputs": [], "source": [ @@ -571,7 +497,7 @@ { "cell_type": "code", "execution_count": null, - "id": "46", + "id": "40", "metadata": {}, "outputs": [], "source": [ @@ -586,7 +512,7 @@ { "cell_type": "code", "execution_count": null, - "id": "47", + "id": "41", "metadata": {}, "outputs": [], "source": [ @@ -596,7 +522,7 @@ { "cell_type": "code", "execution_count": null, - "id": "48", + "id": "42", "metadata": {}, "outputs": [], "source": [ @@ -608,7 +534,7 @@ { "cell_type": "code", "execution_count": null, - "id": "49", + "id": "43", "metadata": {}, "outputs": [], "source": [ @@ -619,7 +545,7 @@ }, { "cell_type": "markdown", - "id": "50", + "id": "44", "metadata": {}, "source": [ "# Cleanup" @@ -628,7 +554,7 @@ { "cell_type": "code", "execution_count": null, - "id": "51", + "id": "45", "metadata": {}, "outputs": [], "source": [ @@ -641,7 +567,7 @@ { "cell_type": "code", "execution_count": null, - "id": "52", + "id": "46", "metadata": {}, "outputs": [], "source": [] @@ -649,7 +575,7 @@ { "cell_type": "code", "execution_count": null, - "id": "53", + "id": "47", "metadata": {}, "outputs": [], "source": [] @@ -671,7 +597,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.3" + "version": "3.12.4" } }, "nbformat": 4, diff --git a/notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb b/notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb index 9db7672c01b..84cc1260659 100644 --- a/notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb +++ b/notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb @@ -9,11 +9,19 @@ "# stdlib\n", "import os\n", "\n", - "# os.environ[\"ORCHESTRA_DEPLOYMENT_TYPE\"] = \"remote\"\n", - "# os.environ[\"DEV_MODE\"] = \"True\"\n", - "# os.environ[\"TEST_EXTERNAL_REGISTRY\"] = \"k3d-registry.localhost:5800\"\n", - "# os.environ[\"CLUSTER_HTTP_PORT_HIGH\"] = \"9081\"\n", - "# os.environ[\"CLUSTER_HTTP_PORT_LOW\"] = \"9083\"" + "# third party\n", + "# set to use the live APIs\n", + "# os.environ[\"TEST_BIGQUERY_APIS_LIVE\"] = \"True\"\n", + "import pandas as pd\n", + "\n", + "# syft absolute\n", + "import syft as sy\n", + "from syft import test_settings\n", + "from syft.client.syncing import compare_clients\n", + "from syft.util.test_helpers.apis import make_schema\n", + "from syft.util.test_helpers.apis import make_submit_query\n", + "from syft.util.test_helpers.apis import make_test_query\n", + "from syft.util.test_helpers.email_helpers import get_email_server" ] }, { @@ -22,7 +30,6 @@ "metadata": {}, "outputs": [], "source": [ - "# stdlib\n", "ADMIN_EMAIL, ADMIN_PW = \"admin2@bigquery.org\", \"bqpw2\"\n", "ROOT_EMAIL, ROOT_PW = \"admin@bigquery.org\", \"bqpw\"\n", "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", @@ -31,30 +38,6 @@ "print(environment, high_port, low_port)" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# stdlib\n", - "\n", - "# set to use the live APIs\n", - "# import os\n", - "# os.environ[\"TEST_BIGQUERY_APIS_LIVE\"] = \"True\"\n", - "# third party\n", - "import pandas as pd\n", - "\n", - "# syft absolute\n", - "import syft as sy\n", - "from syft import test_settings\n", - "from syft.client.syncing import compare_clients\n", - "from syft.util.test_helpers.apis import make_schema\n", - "from syft.util.test_helpers.apis import make_submit_query\n", - "from syft.util.test_helpers.apis import make_test_query\n", - "from syft.util.test_helpers.email_helpers import get_email_server" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -486,8 +469,6 @@ "metadata": {}, "outputs": [], "source": [ - "# Not sure if we can run jobs on the high side directly\n", - "# res = job.wait()\n", "res = job.wait().get()\n", "assert len(res) == 2\n", "assert isinstance(res, pd.DataFrame)" @@ -548,16 +529,6 @@ ")" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# TODO verify that jobs are actually filtered out\n", - "# TODO we need to think about whether its possible for the admin to create more data here that would break sync" - ] - }, { "cell_type": "code", "execution_count": null, @@ -573,7 +544,6 @@ "metadata": {}, "outputs": [], "source": [ - "# TODO maybe see if non-internal method we can use or make it public\n", "widget._share_all()\n", "widget._sync_all()" ] @@ -697,7 +667,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.5" + "version": "3.12.4" } }, "nbformat": 4, diff --git a/notebooks/scenarios/bigquery/sync/021-create-jobs.ipynb b/notebooks/scenarios/bigquery/sync/021-create-jobs.ipynb index 9e1dc31515a..8d8743c1662 100644 --- a/notebooks/scenarios/bigquery/sync/021-create-jobs.ipynb +++ b/notebooks/scenarios/bigquery/sync/021-create-jobs.ipynb @@ -8,13 +8,13 @@ "outputs": [], "source": [ "# stdlib\n", + "from collections import Counter\n", "import os\n", "\n", - "# os.environ[\"ORCHESTRA_DEPLOYMENT_TYPE\"] = \"remote\"\n", - "# os.environ[\"DEV_MODE\"] = \"True\"\n", - "# os.environ[\"TEST_EXTERNAL_REGISTRY\"] = \"k3d-registry.localhost:5800\"\n", - "# os.environ[\"CLUSTER_HTTP_PORT_HIGH\"] = \"9081\"\n", - "# os.environ[\"CLUSTER_HTTP_PORT_LOW\"] = \"9083\"" + "# syft absolute\n", + "import syft as sy\n", + "from syft.util.test_helpers.email_helpers import get_email_server\n", + "from syft.util.test_helpers.email_helpers import load_users" ] }, { @@ -24,7 +24,6 @@ "metadata": {}, "outputs": [], "source": [ - "# stdlib\n", "ADMIN_EMAIL, ADMIN_PW = \"admin2@bigquery.org\", \"bqpw2\"\n", "ROOT_EMAIL, ROOT_PASSWORD = \"admin@bigquery.org\", \"bqpw\"\n", "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", @@ -32,25 +31,9 @@ "print(environment, low_port)" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "2", - "metadata": {}, - "outputs": [], - "source": [ - "# stdlib\n", - "from collections import Counter\n", - "\n", - "# syft absolute\n", - "import syft as sy\n", - "from syft.util.test_helpers.email_helpers import get_email_server\n", - "from syft.util.test_helpers.email_helpers import load_users" - ] - }, { "cell_type": "markdown", - "id": "3", + "id": "2", "metadata": {}, "source": [ "# Launch server & login" @@ -59,7 +42,7 @@ { "cell_type": "code", "execution_count": null, - "id": "4", + "id": "3", "metadata": {}, "outputs": [], "source": [ @@ -76,7 +59,7 @@ { "cell_type": "code", "execution_count": null, - "id": "5", + "id": "4", "metadata": {}, "outputs": [], "source": [ @@ -86,7 +69,7 @@ { "cell_type": "code", "execution_count": null, - "id": "6", + "id": "5", "metadata": {}, "outputs": [], "source": [ @@ -98,7 +81,7 @@ { "cell_type": "code", "execution_count": null, - "id": "7", + "id": "6", "metadata": {}, "outputs": [], "source": [ @@ -107,7 +90,7 @@ }, { "cell_type": "markdown", - "id": "8", + "id": "7", "metadata": {}, "source": [ "# Create jobs" @@ -116,7 +99,7 @@ { "cell_type": "code", "execution_count": null, - "id": "9", + "id": "8", "metadata": {}, "outputs": [], "source": [ @@ -128,7 +111,7 @@ }, { "cell_type": "markdown", - "id": "10", + "id": "9", "metadata": {}, "source": [ "# Inspect job data (requests for these jobs to be created)" @@ -137,7 +120,7 @@ { "cell_type": "code", "execution_count": null, - "id": "11", + "id": "10", "metadata": {}, "outputs": [], "source": [ @@ -149,7 +132,7 @@ { "cell_type": "code", "execution_count": null, - "id": "12", + "id": "11", "metadata": {}, "outputs": [], "source": [ @@ -161,7 +144,7 @@ { "cell_type": "code", "execution_count": null, - "id": "13", + "id": "12", "metadata": {}, "outputs": [], "source": [ @@ -172,7 +155,7 @@ { "cell_type": "code", "execution_count": null, - "id": "14", + "id": "13", "metadata": {}, "outputs": [], "source": [ @@ -185,7 +168,7 @@ { "cell_type": "code", "execution_count": null, - "id": "15", + "id": "14", "metadata": {}, "outputs": [], "source": [ @@ -197,7 +180,7 @@ { "cell_type": "code", "execution_count": null, - "id": "16", + "id": "15", "metadata": {}, "outputs": [], "source": [ @@ -206,7 +189,7 @@ }, { "cell_type": "markdown", - "id": "17", + "id": "16", "metadata": {}, "source": [ "# Submit jobs\n" @@ -215,7 +198,7 @@ { "cell_type": "code", "execution_count": null, - "id": "18", + "id": "17", "metadata": {}, "outputs": [], "source": [ @@ -225,7 +208,7 @@ }, { "cell_type": "markdown", - "id": "19", + "id": "18", "metadata": {}, "source": [ "## Test Succesful jobs" @@ -234,7 +217,7 @@ { "cell_type": "code", "execution_count": null, - "id": "20", + "id": "19", "metadata": {}, "outputs": [], "source": [ @@ -244,7 +227,7 @@ { "cell_type": "code", "execution_count": null, - "id": "21", + "id": "20", "metadata": {}, "outputs": [], "source": [ @@ -258,7 +241,7 @@ }, { "cell_type": "markdown", - "id": "22", + "id": "21", "metadata": {}, "source": [ "## Test failures" @@ -267,7 +250,7 @@ { "cell_type": "code", "execution_count": null, - "id": "23", + "id": "22", "metadata": {}, "outputs": [], "source": [ @@ -277,7 +260,7 @@ { "cell_type": "code", "execution_count": null, - "id": "24", + "id": "23", "metadata": {}, "outputs": [], "source": [ @@ -293,7 +276,7 @@ { "cell_type": "code", "execution_count": null, - "id": "25", + "id": "24", "metadata": {}, "outputs": [], "source": [ @@ -306,7 +289,7 @@ { "cell_type": "code", "execution_count": null, - "id": "26", + "id": "25", "metadata": {}, "outputs": [], "source": [ @@ -315,7 +298,7 @@ }, { "cell_type": "markdown", - "id": "27", + "id": "26", "metadata": {}, "source": [ "## Test: cannot execute submitted jobs yet" @@ -324,7 +307,7 @@ { "cell_type": "code", "execution_count": null, - "id": "28", + "id": "27", "metadata": {}, "outputs": [], "source": [ @@ -336,7 +319,7 @@ { "cell_type": "code", "execution_count": null, - "id": "29", + "id": "28", "metadata": {}, "outputs": [], "source": [ @@ -351,7 +334,7 @@ { "cell_type": "code", "execution_count": null, - "id": "30", + "id": "29", "metadata": {}, "outputs": [], "source": [ @@ -371,7 +354,7 @@ }, { "cell_type": "markdown", - "id": "31", + "id": "30", "metadata": {}, "source": [ "# Verify that admin has emails for submitted requests" @@ -380,21 +363,19 @@ { "cell_type": "code", "execution_count": null, - "id": "32", + "id": "31", "metadata": {}, "outputs": [], "source": [ "num_should_submit = sum(j.should_submit for j in jobs_data)\n", "admin_emails_after = len(email_server.get_emails_for_user(ROOT_EMAIL))\n", "print(\"admin emails after\", admin_emails_after)\n", - "assert admin_emails_after >= admin_emails_before + num_should_submit\n", - "# assert len(users_emails) > after_number_of_emails\n", - "# assert len(users_emails) == after_number_of_emails + 1" + "assert admin_emails_after >= admin_emails_before + num_should_submit" ] }, { "cell_type": "markdown", - "id": "33", + "id": "32", "metadata": {}, "source": [ "# Cleanup" @@ -403,7 +384,7 @@ { "cell_type": "code", "execution_count": null, - "id": "34", + "id": "33", "metadata": {}, "outputs": [], "source": [ diff --git a/notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb b/notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb index e4e9ed761d3..3177009940f 100644 --- a/notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb +++ b/notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb @@ -9,18 +9,17 @@ "# stdlib\n", "import os\n", "\n", - "# Testing works over 4 possibilities\n", - "# 1. (python/in-memory workers and using tox commands)\n", - "# 2. (python/in-memory workers and manually running notebooks)\n", - "# 3. (using k8s and using tox commands)\n", - "# 4. (using k8s and manually running notebooks)\n", - "# Uncomment the lines below if in the 4th possibility\n", + "# third party\n", + "import pandas as pd\n", "\n", - "# os.environ[\"ORCHESTRA_DEPLOYMENT_TYPE\"] = \"remote\"\n", - "# os.environ[\"DEV_MODE\"] = \"True\"\n", - "# os.environ[\"TEST_EXTERNAL_REGISTRY\"] = \"k3d-registry.localhost:5800\"\n", - "# os.environ[\"CLUSTER_HTTP_PORT_HIGH\"] = \"9081\"\n", - "# os.environ[\"CLUSTER_HTTP_PORT_LOW\"] = \"9083\"" + "# syft absolute\n", + "import syft as sy\n", + "from syft.client.syncing import compare_clients\n", + "from syft.util.test_helpers.email_helpers import get_email_server\n", + "from syft.util.test_helpers.email_helpers import load_users\n", + "from syft.util.test_helpers.job_helpers import get_job_emails\n", + "from syft.util.test_helpers.job_helpers import get_request_for_job_info\n", + "from syft.util.test_helpers.job_helpers import load_jobs" ] }, { @@ -39,26 +38,6 @@ "print(environment, low_port)" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# stdlib\n", - "\n", - "# third party\n", - "\n", - "# syft absolute\n", - "import syft as sy\n", - "from syft.client.syncing import compare_clients\n", - "from syft.util.test_helpers.email_helpers import get_email_server\n", - "from syft.util.test_helpers.email_helpers import load_users\n", - "from syft.util.test_helpers.job_helpers import get_job_emails\n", - "from syft.util.test_helpers.job_helpers import get_request_for_job_info\n", - "from syft.util.test_helpers.job_helpers import load_jobs" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -243,14 +222,9 @@ "metadata": {}, "outputs": [], "source": [ - "# third party\n", - "# Same as notebook 020, some issue with job.wait() in the notebook\n", - "import pandas as pd\n", - "\n", "for job in submitted_jobs_data_should_succeed:\n", " request = get_request_for_job_info(all_requests, job)\n", " j = request.code(blocking=False)\n", - " # result = j.wait()\n", " result = j.wait().get()\n", " assert isinstance(result, pd.DataFrame)\n", " job.admin_reviewed = True" diff --git a/notebooks/scenarios/bigquery/sync/050-ds-get-results.ipynb b/notebooks/scenarios/bigquery/sync/050-ds-get-results.ipynb index 14874db6820..2b6c308d272 100644 --- a/notebooks/scenarios/bigquery/sync/050-ds-get-results.ipynb +++ b/notebooks/scenarios/bigquery/sync/050-ds-get-results.ipynb @@ -9,30 +9,6 @@ "# stdlib\n", "import os\n", "\n", - "# Testing works over 4 possibilities\n", - "# 1. (python/in-memory workers and using tox commands)\n", - "# 2. (python/in-memory workers and manually running notebooks)\n", - "# 3. (using k8s and using tox commands)\n", - "# 4. (using k8s and manually running notebooks)\n", - "# Uncomment the lines below if in the 4th possibility\n", - "\n", - "# os.environ[\"ORCHESTRA_DEPLOYMENT_TYPE\"] = \"remote\"\n", - "# os.environ[\"DEV_MODE\"] = \"True\"\n", - "# os.environ[\"TEST_EXTERNAL_REGISTRY\"] = \"k3d-registry.localhost:5800\"\n", - "# os.environ[\"CLUSTER_HTTP_PORT_HIGH\"] = \"9081\"\n", - "# os.environ[\"CLUSTER_HTTP_PORT_LOW\"] = \"9083\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# stdlib\n", - "\n", - "# third party\n", - "\n", "# syft absolute\n", "import syft as sy\n", "from syft.util.test_helpers.email_helpers import load_users\n", @@ -110,7 +86,6 @@ "metadata": {}, "outputs": [], "source": [ - "# submitted_jobs = [job for job in jobs if job.is_submitted]\n", "reviewed_jobs = [job for job in jobs if job.admin_reviewed]\n", "reviewed_jobs_should_succeed = [j for j in reviewed_jobs if j.should_succeed]\n", "reviewed_jobs_should_fail = [j for j in reviewed_jobs if not j.should_succeed]\n", diff --git a/notebooks/scenarios/bigquery/sync/README.md b/notebooks/scenarios/bigquery/sync/README.md new file mode 100644 index 00000000000..ca51f9ae71d --- /dev/null +++ b/notebooks/scenarios/bigquery/sync/README.md @@ -0,0 +1,16 @@ +# Testing works over 4 possibilities + +1. (python/in-memory workers and using tox commands) +2. (python/in-memory workers and manually running notebooks) +3. (using k8s and using tox commands) +4. (using k8s and manually running notebooks) + +Add the lines below to notebook cells if in the 4th possibility + +```python +os.environ["ORCHESTRA_DEPLOYMENT_TYPE"] = "remote" +os.environ["DEV_MODE"] = "True" +os.environ["TEST_EXTERNAL_REGISTRY"] = "k3d-registry.localhost:5800" +os.environ["CLUSTER_HTTP_PORT_HIGH"] = "9081" +os.environ["CLUSTER_HTTP_PORT_LOW"] = "9083" +```