From 7fca3284bf555c03837bf8a64ed7a84b62bf7a4d Mon Sep 17 00:00:00 2001 From: Brendan Schell Date: Thu, 19 Sep 2024 11:16:34 -0400 Subject: [PATCH] rename notebooks to be closer to non-sync and add scale worker nb --- ...ynb => 000-setup-high-low-datasites.ipynb} | 6 +- .../sync/001-scale-delete-worker-pools.ipynb | 387 ++++++++++++++++++ ...ipynb => 020-configure-api-and-sync.ipynb} | 6 +- ...uest.ipynb => 030-ds-submit-request.ipynb} | 4 +- ...sts.ipynb => 040-do-review-requests.ipynb} | 6 +- ...results.ipynb => 050-ds-get-results.ipynb} | 4 +- 6 files changed, 400 insertions(+), 13 deletions(-) rename notebooks/scenarios/bigquery/sync/{01-setup-high-low-datasites.ipynb => 000-setup-high-low-datasites.ipynb} (98%) create mode 100644 notebooks/scenarios/bigquery/sync/001-scale-delete-worker-pools.ipynb rename notebooks/scenarios/bigquery/sync/{02-configure-api-and-sync.ipynb => 020-configure-api-and-sync.ipynb} (98%) rename notebooks/scenarios/bigquery/sync/{03-ds-submit-request.ipynb => 030-ds-submit-request.ipynb} (98%) rename notebooks/scenarios/bigquery/sync/{04-do-review-requests.ipynb => 040-do-review-requests.ipynb} (97%) rename notebooks/scenarios/bigquery/sync/{05-ds-get-results.ipynb => 050-ds-get-results.ipynb} (97%) diff --git a/notebooks/scenarios/bigquery/sync/01-setup-high-low-datasites.ipynb b/notebooks/scenarios/bigquery/sync/000-setup-high-low-datasites.ipynb similarity index 98% rename from notebooks/scenarios/bigquery/sync/01-setup-high-low-datasites.ipynb rename to notebooks/scenarios/bigquery/sync/000-setup-high-low-datasites.ipynb index 691c58d4b00..82e69b5b829 100644 --- a/notebooks/scenarios/bigquery/sync/01-setup-high-low-datasites.ipynb +++ b/notebooks/scenarios/bigquery/sync/000-setup-high-low-datasites.ipynb @@ -31,8 +31,8 @@ "source": [ "# stdlib\n", "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", - "high_port = os.environ.get(\"CLUSTER_HTTP_PORT_HIGH\", \"auto\")\n", - "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"auto\")\n", + "high_port = os.environ.get(\"CLUSTER_HTTP_PORT_HIGH\", \"9081\")\n", + "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"9083\")\n", "print(environment, high_port, low_port)" ] }, @@ -327,7 +327,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.3" + "version": "3.12.4" } }, "nbformat": 4, diff --git a/notebooks/scenarios/bigquery/sync/001-scale-delete-worker-pools.ipynb b/notebooks/scenarios/bigquery/sync/001-scale-delete-worker-pools.ipynb new file mode 100644 index 00000000000..fb4bd0379e6 --- /dev/null +++ b/notebooks/scenarios/bigquery/sync/001-scale-delete-worker-pools.ipynb @@ -0,0 +1,387 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "0", + "metadata": {}, + "outputs": [], + "source": [ + "# stdlib\n", + "import os\n", + "\n", + "# Testing works over 4 possibilities\n", + "# 1. (python/in-memory workers and using tox commands)\n", + "# 2. (python/in-memory workers and manually running notebooks)\n", + "# 3. (using k8s and using tox commands)\n", + "# 4. (using k8s and manually running notebooks)\n", + "# Uncomment the lines below if in the 4th possibility\n", + "\n", + "# os.environ[\"ORCHESTRA_DEPLOYMENT_TYPE\"] = \"remote\"\n", + "# os.environ[\"DEV_MODE\"] = \"True\"\n", + "# os.environ[\"TEST_EXTERNAL_REGISTRY\"] = \"k3d-registry.localhost:5800\"\n", + "# os.environ[\"CLUSTER_HTTP_PORT_HIGH\"] = \"9081\"\n", + "# os.environ[\"CLUSTER_HTTP_PORT_LOW\"] = \"9083\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "589bbac0", + "metadata": {}, + "outputs": [], + "source": [ + "# stdlib\n", + "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", + "high_port = os.environ.get(\"CLUSTER_HTTP_PORT_HIGH\", \"9081\")\n", + "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"9083\")\n", + "print(environment, high_port, low_port)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "# stdlib\n", + "import os\n", + "\n", + "# syft absolute\n", + "import syft as sy\n", + "from syft.util.test_helpers.email_helpers import Timeout\n", + "from syft.util.test_helpers.email_helpers import get_email_server" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", + "\n", + "num_workers = int(os.environ.get(\"NUM_TEST_WORKERS\", 1))\n", + "\n", + "ROOT_EMAIL = \"admin@bigquery.org\"\n", + "ROOT_PASSWORD = \"bqpw\"\n", + "environment" + ] + }, + { + "cell_type": "markdown", + "id": "3", + "metadata": {}, + "source": [ + "### Launch server & login" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "server_low = sy.orchestra.launch(\n", + " name=\"bigquery-low\",\n", + " server_side_type=\"low\",\n", + " dev_mode=True,\n", + " reset=True,\n", + " n_consumers=1,\n", + " create_producer=True,\n", + " port=low_port,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "email_server, smtp_server = get_email_server(reset=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "low_client = server_low.login(\n", + " email=ROOT_EMAIL, password=ROOT_PASSWORD\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "low_client.worker_pools" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "default_worker_pool = low_client.worker_pools.get_by_name(\"default-pool\")\n", + "default_worker_pool" + ] + }, + { + "cell_type": "markdown", + "id": "9", + "metadata": {}, + "source": [ + "### Scale Worker pool" + ] + }, + { + "cell_type": "markdown", + "id": "10", + "metadata": {}, + "source": [ + "##### Scale up" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11", + "metadata": {}, + "outputs": [], + "source": [ + "# Scale to 1\n", + "if environment == \"remote\":\n", + " low_client.api.worker_pool.scale(\n", + " number=num_workers, pool_name=default_worker_pool.name\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12", + "metadata": {}, + "outputs": [], + "source": [ + "low_client.api.services.worker_pool[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "13", + "metadata": {}, + "outputs": [], + "source": [ + "# Scale up workers\n", + "if environment == \"remote\":\n", + " scale_up_result = low_client.api.worker_pool.scale(\n", + " number=5, pool_name=default_worker_pool.name\n", + " )\n", + " if environment == \"remote\":\n", + " assert scale_up_result, scale_up_result\n", + "\n", + " assert (\n", + " low_client.api.services.worker_pool[default_worker_pool.name].max_count\n", + " == 5\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "14", + "metadata": {}, + "source": [ + "##### Scale down" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15", + "metadata": {}, + "outputs": [], + "source": [ + "# Scale down workers, this gracefully shutdowns the consumers\n", + "if environment == \"remote\":\n", + " scale_down_result = low_client.api.worker_pool.scale(\n", + " number=num_workers, pool_name=default_worker_pool.name\n", + " )\n", + " assert scale_down_result, scale_down_result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "16", + "metadata": {}, + "outputs": [], + "source": [ + "if environment == \"remote\":\n", + "\n", + " def has_worker_scaled_down():\n", + " return (\n", + " low_client.api.worker_pool[default_worker_pool.name].max_count\n", + " == num_workers\n", + " )\n", + "\n", + " worker_scale_timeout = Timeout(timeout_duration=20)\n", + " worker_scale_timeout.run_with_timeout(has_worker_scaled_down)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "17", + "metadata": {}, + "outputs": [], + "source": [ + "if environment == \"remote\":\n", + " assert (\n", + " low_client.api.services.worker_pool[default_worker_pool.name].max_count\n", + " == num_workers\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "18", + "metadata": {}, + "source": [ + "#### Delete Worker Pool" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "19", + "metadata": {}, + "outputs": [], + "source": [ + "pool_delete_result = low_client.api.services.worker_pool.delete(\n", + " pool_name=default_worker_pool.name\n", + ")\n", + "pool_delete_result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "20", + "metadata": {}, + "outputs": [], + "source": [ + "with sy.raises(KeyError):\n", + " _ = low_client.api.services.worker_pool[default_worker_pool.name]" + ] + }, + { + "cell_type": "markdown", + "id": "21", + "metadata": {}, + "source": [ + "#### Re-launch the default worker pool" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "22", + "metadata": {}, + "outputs": [], + "source": [ + "default_worker_image = default_worker_pool.image" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "23", + "metadata": {}, + "outputs": [], + "source": [ + "launch_result = low_client.api.services.worker_pool.launch(\n", + " pool_name=default_worker_pool.name,\n", + " image_uid=default_worker_image.id,\n", + " num_workers=num_workers,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24", + "metadata": {}, + "outputs": [], + "source": [ + "assert low_client.api.services.worker_pool[default_worker_pool.name]\n", + "assert (\n", + " low_client.api.services.worker_pool[default_worker_pool.name].max_count\n", + " == num_workers\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "25", + "metadata": {}, + "outputs": [], + "source": [ + "smtp_server.stop()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "26", + "metadata": {}, + "outputs": [], + "source": [ + "server_low.land()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "27", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "syft", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/scenarios/bigquery/sync/02-configure-api-and-sync.ipynb b/notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb similarity index 98% rename from notebooks/scenarios/bigquery/sync/02-configure-api-and-sync.ipynb rename to notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb index 99274aba2a8..2f9658634e3 100644 --- a/notebooks/scenarios/bigquery/sync/02-configure-api-and-sync.ipynb +++ b/notebooks/scenarios/bigquery/sync/020-configure-api-and-sync.ipynb @@ -25,8 +25,8 @@ "# stdlib\n", "\n", "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", - "high_port = os.environ.get(\"CLUSTER_HTTP_PORT_HIGH\", \"auto\")\n", - "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"auto\")\n", + "high_port = os.environ.get(\"CLUSTER_HTTP_PORT_HIGH\", \"9081\")\n", + "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"9083\")\n", "print(environment, high_port, low_port)" ] }, @@ -614,7 +614,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.3" + "version": "3.12.4" } }, "nbformat": 4, diff --git a/notebooks/scenarios/bigquery/sync/03-ds-submit-request.ipynb b/notebooks/scenarios/bigquery/sync/030-ds-submit-request.ipynb similarity index 98% rename from notebooks/scenarios/bigquery/sync/03-ds-submit-request.ipynb rename to notebooks/scenarios/bigquery/sync/030-ds-submit-request.ipynb index a2759038134..21e8437f283 100644 --- a/notebooks/scenarios/bigquery/sync/03-ds-submit-request.ipynb +++ b/notebooks/scenarios/bigquery/sync/030-ds-submit-request.ipynb @@ -24,7 +24,7 @@ "# stdlib\n", "\n", "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", - "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"auto\")\n", + "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"9083\")\n", "print(environment, low_port)" ] }, @@ -319,7 +319,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.3" + "version": "3.12.4" } }, "nbformat": 4, diff --git a/notebooks/scenarios/bigquery/sync/04-do-review-requests.ipynb b/notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb similarity index 97% rename from notebooks/scenarios/bigquery/sync/04-do-review-requests.ipynb rename to notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb index 4eec3d6e7b1..07b32abbc34 100644 --- a/notebooks/scenarios/bigquery/sync/04-do-review-requests.ipynb +++ b/notebooks/scenarios/bigquery/sync/040-do-review-requests.ipynb @@ -28,8 +28,8 @@ "outputs": [], "source": [ "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", - "high_port = os.environ.get(\"CLUSTER_HTTP_PORT_HIGH\", \"auto\")\n", - "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"auto\")" + "high_port = os.environ.get(\"CLUSTER_HTTP_PORT_HIGH\", \"9081\")\n", + "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"9083\")" ] }, { @@ -300,7 +300,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.3" + "version": "3.12.4" } }, "nbformat": 4, diff --git a/notebooks/scenarios/bigquery/sync/05-ds-get-results.ipynb b/notebooks/scenarios/bigquery/sync/050-ds-get-results.ipynb similarity index 97% rename from notebooks/scenarios/bigquery/sync/05-ds-get-results.ipynb rename to notebooks/scenarios/bigquery/sync/050-ds-get-results.ipynb index 1e61e0d8587..531f0c8f410 100644 --- a/notebooks/scenarios/bigquery/sync/05-ds-get-results.ipynb +++ b/notebooks/scenarios/bigquery/sync/050-ds-get-results.ipynb @@ -26,7 +26,7 @@ "outputs": [], "source": [ "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", - "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"auto\")\n", + "low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"9083\")\n", "print(environment, low_port)" ] }, @@ -141,7 +141,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.3" + "version": "3.12.4" } }, "nbformat": 4,