diff --git a/notebooks/scenarios/bigquery/00-start-and-configure-server-and-admins.ipynb b/notebooks/scenarios/bigquery/000-start-and-configure-server-and-admins.ipynb similarity index 94% rename from notebooks/scenarios/bigquery/00-start-and-configure-server-and-admins.ipynb rename to notebooks/scenarios/bigquery/000-start-and-configure-server-and-admins.ipynb index 8a438cbc935..444ba2fe199 100644 --- a/notebooks/scenarios/bigquery/00-start-and-configure-server-and-admins.ipynb +++ b/notebooks/scenarios/bigquery/000-start-and-configure-server-and-admins.ipynb @@ -23,10 +23,21 @@ "# stdlib\n", "from os import environ as env\n", "\n", + "# third party\n", + "# run email server\n", + "from helpers import get_email_server\n", + "\n", "# syft absolute\n", "import syft as sy" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Launch & login" + ] + }, { "cell_type": "code", "execution_count": null, @@ -49,23 +60,6 @@ "env[\"DEFAULT_ROOT_PASSWORD\"] = ROOT_PASSWORD" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# third party\n", - "# run email server\n", - "from helpers import EmailServer\n", - "from helpers import SMTPTestServer\n", - "\n", - "email_server = EmailServer()\n", - "email_server.reset_emails()\n", - "smtp_server = SMTPTestServer(email_server)\n", - "smtp_server.start()" - ] - }, { "cell_type": "code", "execution_count": null, @@ -83,6 +77,15 @@ ")" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "email_server, smtp_server = get_email_server(reset=True)" + ] + }, { "cell_type": "code", "execution_count": null, @@ -195,6 +198,13 @@ "root_client.users.delete(new_user_id2)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Cleanup" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/notebooks/scenarios/bigquery/001-scale-delete-worker-pools.ipynb b/notebooks/scenarios/bigquery/001-scale-delete-worker-pools.ipynb index 4741a74ebce..e46587d23e5 100644 --- a/notebooks/scenarios/bigquery/001-scale-delete-worker-pools.ipynb +++ b/notebooks/scenarios/bigquery/001-scale-delete-worker-pools.ipynb @@ -13,126 +13,52 @@ "# os.environ[\"TEST_EXTERNAL_REGISTRY\"] = \"k3d-registry.localhost:5800\"" ] }, - { - "cell_type": "markdown", - "id": "1", - "metadata": {}, - "source": [ - "#### Helpers" - ] - }, { "cell_type": "code", "execution_count": null, - "id": "2", + "id": "1", "metadata": {}, "outputs": [], "source": [ "# stdlib\n", - "import time\n", - "\n", - "\n", - "class TimeoutError(Exception):\n", - " pass\n", - "\n", - "\n", - "class Timeout:\n", - " def __init__(self, timeout_duration):\n", - " if timeout_duration > 60:\n", - " raise ValueError(\"Timeout duration cannot exceed 60 seconds.\")\n", - " self.timeout_duration = timeout_duration\n", - "\n", - " def run_with_timeout(self, condition_func, *args, **kwargs):\n", - " start_time = time.time()\n", - " result = None\n", - "\n", - " while True:\n", - " elapsed_time = time.time() - start_time\n", - " if elapsed_time > self.timeout_duration:\n", - " raise TimeoutError(\n", - " f\"Function execution exceeded {self.timeout_duration} seconds.\"\n", - " )\n", + "import os\n", "\n", - " # Check if the condition is met\n", - " try:\n", - " if condition_func():\n", - " print(\"Condition met, exiting early.\")\n", - " break\n", - " except Exception as e:\n", - " print(f\"Exception in target function: {e}\")\n", - " break # Exit the loop if an exception occurs in the function\n", - " time.sleep(1)\n", + "# third party\n", + "from helpers import Timeout\n", + "from helpers import get_email_server\n", "\n", - " return result" - ] - }, - { - "cell_type": "markdown", - "id": "3", - "metadata": {}, - "source": [ - "### Import lib" + "# syft absolute\n", + "import syft as sy" ] }, { "cell_type": "code", "execution_count": null, - "id": "4", + "id": "2", "metadata": {}, "outputs": [], "source": [ - "# stdlib\n", - "import os\n", - "\n", "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", - "environment" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5", - "metadata": {}, - "outputs": [], - "source": [ - "num_workers = int(os.environ.get(\"NUM_TEST_WORKERS\", 1))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6", - "metadata": {}, - "outputs": [], - "source": [ - "# stdlib\n", "\n", - "# syft absolute\n", - "import syft as sy" + "num_workers = int(os.environ.get(\"NUM_TEST_WORKERS\", 1))\n", + "\n", + "ROOT_EMAIL = \"admin@bigquery.org\"\n", + "ROOT_PASSWORD = \"bqpw\"\n", + "environment" ] }, { - "cell_type": "code", - "execution_count": null, - "id": "7", + "cell_type": "markdown", + "id": "3", "metadata": {}, - "outputs": [], "source": [ - "# third party\n", - "# run email server\n", - "from helpers import EmailServer\n", - "from helpers import SMTPTestServer\n", - "\n", - "email_server = EmailServer()\n", - "email_server.reset_emails()\n", - "smtp_server = SMTPTestServer(email_server)\n", - "smtp_server.start()" + "### Launch server & login" ] }, { "cell_type": "code", "execution_count": null, - "id": "8", + "id": "4", "metadata": {}, "outputs": [], "source": [ @@ -149,18 +75,17 @@ { "cell_type": "code", "execution_count": null, - "id": "9", + "id": "5", "metadata": {}, "outputs": [], "source": [ - "ROOT_EMAIL = \"admin@bigquery.org\"\n", - "ROOT_PASSWORD = \"bqpw\"" + "email_server, smtp_server = get_email_server(reset=True)" ] }, { "cell_type": "code", "execution_count": null, - "id": "10", + "id": "6", "metadata": {}, "outputs": [], "source": [ @@ -172,7 +97,7 @@ { "cell_type": "code", "execution_count": null, - "id": "11", + "id": "7", "metadata": {}, "outputs": [], "source": [ @@ -182,7 +107,7 @@ { "cell_type": "code", "execution_count": null, - "id": "12", + "id": "8", "metadata": {}, "outputs": [], "source": [ @@ -192,7 +117,7 @@ }, { "cell_type": "markdown", - "id": "13", + "id": "9", "metadata": {}, "source": [ "### Scale Worker pool" @@ -200,7 +125,7 @@ }, { "cell_type": "markdown", - "id": "14", + "id": "10", "metadata": {}, "source": [ "##### Scale up" @@ -209,7 +134,7 @@ { "cell_type": "code", "execution_count": null, - "id": "15", + "id": "11", "metadata": {}, "outputs": [], "source": [ @@ -223,7 +148,7 @@ { "cell_type": "code", "execution_count": null, - "id": "16", + "id": "12", "metadata": {}, "outputs": [], "source": [ @@ -233,7 +158,7 @@ { "cell_type": "code", "execution_count": null, - "id": "17", + "id": "13", "metadata": {}, "outputs": [], "source": [ @@ -253,7 +178,7 @@ }, { "cell_type": "markdown", - "id": "18", + "id": "14", "metadata": {}, "source": [ "##### Scale down" @@ -262,7 +187,7 @@ { "cell_type": "code", "execution_count": null, - "id": "19", + "id": "15", "metadata": {}, "outputs": [], "source": [ @@ -277,7 +202,7 @@ { "cell_type": "code", "execution_count": null, - "id": "20", + "id": "16", "metadata": {}, "outputs": [], "source": [ @@ -296,7 +221,7 @@ { "cell_type": "code", "execution_count": null, - "id": "21", + "id": "17", "metadata": {}, "outputs": [], "source": [ @@ -309,7 +234,7 @@ }, { "cell_type": "markdown", - "id": "22", + "id": "18", "metadata": {}, "source": [ "#### Delete Worker Pool" @@ -318,7 +243,7 @@ { "cell_type": "code", "execution_count": null, - "id": "23", + "id": "19", "metadata": {}, "outputs": [], "source": [ @@ -331,7 +256,7 @@ { "cell_type": "code", "execution_count": null, - "id": "24", + "id": "20", "metadata": {}, "outputs": [], "source": [ @@ -341,7 +266,7 @@ }, { "cell_type": "markdown", - "id": "25", + "id": "21", "metadata": {}, "source": [ "#### Re-launch the default worker pool" @@ -350,7 +275,7 @@ { "cell_type": "code", "execution_count": null, - "id": "26", + "id": "22", "metadata": {}, "outputs": [], "source": [ @@ -360,7 +285,7 @@ { "cell_type": "code", "execution_count": null, - "id": "27", + "id": "23", "metadata": {}, "outputs": [], "source": [ @@ -374,7 +299,7 @@ { "cell_type": "code", "execution_count": null, - "id": "28", + "id": "24", "metadata": {}, "outputs": [], "source": [ @@ -388,7 +313,7 @@ { "cell_type": "code", "execution_count": null, - "id": "29", + "id": "25", "metadata": {}, "outputs": [], "source": [ @@ -398,7 +323,7 @@ { "cell_type": "code", "execution_count": null, - "id": "30", + "id": "26", "metadata": {}, "outputs": [], "source": [ @@ -408,7 +333,7 @@ { "cell_type": "code", "execution_count": null, - "id": "31", + "id": "27", "metadata": {}, "outputs": [], "source": [] diff --git a/notebooks/scenarios/bigquery/01-setup-datasite.ipynb b/notebooks/scenarios/bigquery/010-setup-bigquery-pool.ipynb similarity index 92% rename from notebooks/scenarios/bigquery/01-setup-datasite.ipynb rename to notebooks/scenarios/bigquery/010-setup-bigquery-pool.ipynb index 6ce2e541480..1280a4e19d7 100644 --- a/notebooks/scenarios/bigquery/01-setup-datasite.ipynb +++ b/notebooks/scenarios/bigquery/010-setup-bigquery-pool.ipynb @@ -21,8 +21,12 @@ "# stdlib\n", "import os\n", "\n", - "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", - "environment" + "# third party\n", + "from helpers import get_email_server\n", + "\n", + "# syft absolute\n", + "import syft as sy\n", + "from syft import test_settings" ] }, { @@ -31,27 +35,15 @@ "metadata": {}, "outputs": [], "source": [ - "# stdlib\n", - "\n", - "# syft absolute\n", - "import syft as sy\n", - "from syft import test_settings" + "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", + "environment" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "# third party\n", - "# run email server\n", - "from helpers import EmailServer\n", - "from helpers import SMTPTestServer\n", - "\n", - "email_server = EmailServer()\n", - "smtp_server = SMTPTestServer(email_server)\n", - "smtp_server.start()" + "# Launch & login" ] }, { @@ -70,6 +62,15 @@ ")" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "_, smtp_server = get_email_server(reset=True)" + ] + }, { "cell_type": "code", "execution_count": null, @@ -82,6 +83,20 @@ ")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Submit images and build pools" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Add registry" + ] + }, { "cell_type": "code", "execution_count": null, @@ -140,6 +155,13 @@ "local_registry" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Upload Image" + ] + }, { "cell_type": "code", "execution_count": null, @@ -259,6 +281,13 @@ "# worker_docker_tag" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Build image" + ] + }, { "cell_type": "code", "execution_count": null, @@ -338,23 +367,6 @@ "dockerfile_list" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# worker_image = next(\n", - "# (\n", - "# image\n", - "# for image in dockerfile_list\n", - "# if image.is_prebuilt and docker_tag in str(image.image_identifier)\n", - "# ),\n", - "# None,\n", - "# )\n", - "# worker_image" - ] - }, { "cell_type": "code", "execution_count": null, @@ -407,6 +419,13 @@ "num_workers = int(os.environ.get(\"NUM_TEST_WORKERS\", 1))" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Launch pool" + ] + }, { "cell_type": "code", "execution_count": null, @@ -423,6 +442,13 @@ "result" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Scale pool" + ] + }, { "cell_type": "code", "execution_count": null, @@ -463,6 +489,13 @@ "assert len(high_client.api.services.user.get_all()) == 2" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Cleanup" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/notebooks/scenarios/bigquery/011-users-emails-passwords.ipynb b/notebooks/scenarios/bigquery/011-users-emails-passwords.ipynb index c6a4c8c865d..990f91183d8 100644 --- a/notebooks/scenarios/bigquery/011-users-emails-passwords.ipynb +++ b/notebooks/scenarios/bigquery/011-users-emails-passwords.ipynb @@ -22,23 +22,14 @@ "metadata": {}, "outputs": [], "source": [ + "# stdlib\n", "# stdlib\n", "import os\n", "\n", - "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", - "environment" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2", - "metadata": {}, - "outputs": [], - "source": [ "# third party\n", "from helpers import SENDER\n", "from helpers import create_user\n", + "from helpers import get_email_server\n", "from helpers import make_user\n", "from helpers import save_users\n", "\n", @@ -49,18 +40,22 @@ { "cell_type": "code", "execution_count": null, - "id": "3", + "id": "2", "metadata": {}, "outputs": [], "source": [ - "# third party\n", - "# run email server\n", - "from helpers import EmailServer\n", - "from helpers import SMTPTestServer\n", + "ADMIN_EMAIL, ADMIN_PW = \"admin2@bigquery.org\", \"bqpw2\"\n", "\n", - "email_server = EmailServer()\n", - "smtp_server = SMTPTestServer(email_server)\n", - "smtp_server.start()" + "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", + "environment" + ] + }, + { + "cell_type": "markdown", + "id": "3", + "metadata": {}, + "source": [ + "# Launch server & login" ] }, { @@ -87,7 +82,16 @@ "metadata": {}, "outputs": [], "source": [ - "ADMIN_EMAIL, ADMIN_PW = \"admin2@bigquery.org\", \"bqpw2\"\n", + "email_server, smtp_server = get_email_server()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ "high_client = sy.login(\n", " url=\"http://localhost:8080\", email=ADMIN_EMAIL, password=ADMIN_PW\n", ")" @@ -96,7 +100,7 @@ { "cell_type": "code", "execution_count": null, - "id": "6", + "id": "7", "metadata": {}, "outputs": [], "source": [ @@ -106,7 +110,7 @@ { "cell_type": "code", "execution_count": null, - "id": "7", + "id": "8", "metadata": {}, "outputs": [], "source": [ @@ -121,7 +125,7 @@ { "cell_type": "code", "execution_count": null, - "id": "8", + "id": "9", "metadata": {}, "outputs": [], "source": [ @@ -131,7 +135,7 @@ }, { "cell_type": "markdown", - "id": "9", + "id": "10", "metadata": {}, "source": [ "# Register users" @@ -140,7 +144,7 @@ { "cell_type": "code", "execution_count": null, - "id": "10", + "id": "11", "metadata": {}, "outputs": [], "source": [ @@ -151,7 +155,7 @@ { "cell_type": "code", "execution_count": null, - "id": "11", + "id": "12", "metadata": {}, "outputs": [], "source": [ @@ -173,7 +177,7 @@ { "cell_type": "code", "execution_count": null, - "id": "12", + "id": "13", "metadata": {}, "outputs": [], "source": [ @@ -183,7 +187,7 @@ { "cell_type": "code", "execution_count": null, - "id": "13", + "id": "14", "metadata": {}, "outputs": [], "source": [ @@ -193,10 +197,18 @@ "await asyncio.sleep(5)" ] }, + { + "cell_type": "markdown", + "id": "15", + "metadata": {}, + "source": [ + "## Verify Emails are sent" + ] + }, { "cell_type": "code", "execution_count": null, - "id": "14", + "id": "16", "metadata": {}, "outputs": [], "source": [ @@ -215,7 +227,7 @@ { "cell_type": "code", "execution_count": null, - "id": "15", + "id": "17", "metadata": {}, "outputs": [], "source": [ @@ -240,7 +252,7 @@ { "cell_type": "code", "execution_count": null, - "id": "16", + "id": "18", "metadata": {}, "outputs": [], "source": [ @@ -250,7 +262,7 @@ { "cell_type": "code", "execution_count": null, - "id": "17", + "id": "19", "metadata": {}, "outputs": [], "source": [ @@ -261,7 +273,7 @@ { "cell_type": "code", "execution_count": null, - "id": "18", + "id": "20", "metadata": {}, "outputs": [], "source": [ @@ -275,7 +287,7 @@ { "cell_type": "code", "execution_count": null, - "id": "19", + "id": "21", "metadata": {}, "outputs": [], "source": [ @@ -287,7 +299,7 @@ { "cell_type": "code", "execution_count": null, - "id": "20", + "id": "22", "metadata": {}, "outputs": [], "source": [ @@ -297,7 +309,7 @@ { "cell_type": "code", "execution_count": null, - "id": "21", + "id": "23", "metadata": {}, "outputs": [], "source": [ @@ -305,10 +317,18 @@ "# high_client.api.services.settings.enable_notifications()" ] }, + { + "cell_type": "markdown", + "id": "24", + "metadata": {}, + "source": [ + "## Test reset password" + ] + }, { "cell_type": "code", "execution_count": null, - "id": "22", + "id": "25", "metadata": {}, "outputs": [], "source": [ @@ -320,7 +340,7 @@ { "cell_type": "code", "execution_count": null, - "id": "23", + "id": "26", "metadata": {}, "outputs": [], "source": [ @@ -332,7 +352,7 @@ { "cell_type": "code", "execution_count": null, - "id": "24", + "id": "27", "metadata": {}, "outputs": [], "source": [ @@ -343,7 +363,7 @@ { "cell_type": "code", "execution_count": null, - "id": "25", + "id": "28", "metadata": {}, "outputs": [], "source": [ @@ -356,7 +376,7 @@ { "cell_type": "code", "execution_count": null, - "id": "26", + "id": "29", "metadata": {}, "outputs": [], "source": [ @@ -366,7 +386,7 @@ { "cell_type": "code", "execution_count": null, - "id": "27", + "id": "30", "metadata": {}, "outputs": [], "source": [ @@ -377,7 +397,7 @@ }, { "cell_type": "markdown", - "id": "28", + "id": "31", "metadata": {}, "source": [ "## Reset password second time" @@ -386,7 +406,7 @@ { "cell_type": "code", "execution_count": null, - "id": "29", + "id": "32", "metadata": {}, "outputs": [], "source": [ @@ -396,7 +416,7 @@ { "cell_type": "code", "execution_count": null, - "id": "30", + "id": "33", "metadata": {}, "outputs": [], "source": [ @@ -410,7 +430,7 @@ { "cell_type": "code", "execution_count": null, - "id": "31", + "id": "34", "metadata": {}, "outputs": [], "source": [ @@ -420,7 +440,7 @@ { "cell_type": "code", "execution_count": null, - "id": "32", + "id": "35", "metadata": {}, "outputs": [], "source": [ @@ -432,7 +452,7 @@ { "cell_type": "code", "execution_count": null, - "id": "33", + "id": "36", "metadata": {}, "outputs": [], "source": [ @@ -442,7 +462,7 @@ { "cell_type": "code", "execution_count": null, - "id": "34", + "id": "37", "metadata": {}, "outputs": [], "source": [ @@ -454,7 +474,7 @@ { "cell_type": "code", "execution_count": null, - "id": "35", + "id": "38", "metadata": {}, "outputs": [], "source": [ @@ -465,7 +485,7 @@ { "cell_type": "code", "execution_count": null, - "id": "36", + "id": "39", "metadata": {}, "outputs": [], "source": [ @@ -475,7 +495,7 @@ { "cell_type": "code", "execution_count": null, - "id": "37", + "id": "40", "metadata": {}, "outputs": [], "source": [ @@ -484,7 +504,7 @@ }, { "cell_type": "markdown", - "id": "38", + "id": "41", "metadata": {}, "source": [ "## Reduce token expiration and try resetting" @@ -493,7 +513,7 @@ { "cell_type": "code", "execution_count": null, - "id": "39", + "id": "42", "metadata": {}, "outputs": [], "source": [ @@ -505,7 +525,7 @@ { "cell_type": "code", "execution_count": null, - "id": "40", + "id": "43", "metadata": {}, "outputs": [], "source": [ @@ -515,7 +535,7 @@ { "cell_type": "code", "execution_count": null, - "id": "41", + "id": "44", "metadata": {}, "outputs": [], "source": [ @@ -526,7 +546,7 @@ { "cell_type": "code", "execution_count": null, - "id": "42", + "id": "45", "metadata": {}, "outputs": [], "source": [ @@ -541,7 +561,7 @@ { "cell_type": "code", "execution_count": null, - "id": "43", + "id": "46", "metadata": {}, "outputs": [], "source": [ @@ -551,7 +571,7 @@ { "cell_type": "code", "execution_count": null, - "id": "44", + "id": "47", "metadata": {}, "outputs": [], "source": [ @@ -563,7 +583,7 @@ { "cell_type": "code", "execution_count": null, - "id": "45", + "id": "48", "metadata": {}, "outputs": [], "source": [ @@ -572,10 +592,18 @@ "high_client.refresh()" ] }, + { + "cell_type": "markdown", + "id": "49", + "metadata": {}, + "source": [ + "# Cleanup" + ] + }, { "cell_type": "code", "execution_count": null, - "id": "46", + "id": "50", "metadata": {}, "outputs": [], "source": [ @@ -585,7 +613,7 @@ { "cell_type": "code", "execution_count": null, - "id": "47", + "id": "51", "metadata": {}, "outputs": [], "source": [ @@ -595,18 +623,13 @@ { "cell_type": "code", "execution_count": null, - "id": "48", + "id": "52", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, "language_info": { "codemirror_mode": { "name": "ipython", @@ -617,7 +640,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.3" + "version": "3.12.5" } }, "nbformat": 4, diff --git a/notebooks/scenarios/bigquery/02-configure-api.ipynb b/notebooks/scenarios/bigquery/020-configure-api.ipynb similarity index 87% rename from notebooks/scenarios/bigquery/02-configure-api.ipynb rename to notebooks/scenarios/bigquery/020-configure-api.ipynb index 9f0051c141c..90e35d65459 100644 --- a/notebooks/scenarios/bigquery/02-configure-api.ipynb +++ b/notebooks/scenarios/bigquery/020-configure-api.ipynb @@ -19,12 +19,7 @@ "outputs": [], "source": [ "# set to use the live APIs\n", - "# import os\n", - "# os.environ[\"TEST_BIGQUERY_APIS_LIVE\"] = \"True\"\n", - "# third party\n", - "from apis import make_schema\n", - "from apis import make_submit_query\n", - "from apis import make_test_query" + "# os.environ[\"TEST_BIGQUERY_APIS_LIVE\"] = \"True\"" ] }, { @@ -33,6 +28,14 @@ "metadata": {}, "outputs": [], "source": [ + "# third party\n", + "from apis import make_schema\n", + "from apis import make_submit_query\n", + "from apis import make_test_query\n", + "\n", + "# run email server\n", + "from helpers import get_email_server\n", + "\n", "# syft absolute\n", "import syft as sy\n", "from syft import test_settings" @@ -44,14 +47,14 @@ "metadata": {}, "outputs": [], "source": [ - "# third party\n", - "# run email server\n", - "from helpers import EmailServer\n", - "from helpers import SMTPTestServer\n", - "\n", - "email_server = EmailServer()\n", - "smtp_server = SMTPTestServer(email_server)\n", - "smtp_server.start()" + "ADMIN_EMAIL, ADMIN_PW = \"admin2@bigquery.org\", \"bqpw2\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Launch server & login" ] }, { @@ -76,7 +79,7 @@ "metadata": {}, "outputs": [], "source": [ - "this_worker_pool_name = \"bigquery-pool\"" + "email_server, smtp_server = get_email_server()" ] }, { @@ -85,12 +88,20 @@ "metadata": {}, "outputs": [], "source": [ - "ADMIN_EMAIL, ADMIN_PW = \"admin2@bigquery.org\", \"bqpw2\"\n", "high_client = sy.login(\n", " url=\"http://localhost:8080\", email=ADMIN_EMAIL, password=ADMIN_PW\n", ")" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "this_worker_pool_name = \"bigquery-pool\"" + ] + }, { "cell_type": "code", "execution_count": null, @@ -100,6 +111,13 @@ "assert len(high_client.worker_pools.get_all()) == 2" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Create `test_query` endpoint" + ] + }, { "cell_type": "code", "execution_count": null, @@ -173,6 +191,13 @@ "high_client.custom_api.add(endpoint=new_endpoint)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Update `test_query` endpoint" + ] + }, { "cell_type": "code", "execution_count": null, @@ -196,6 +221,13 @@ ")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Test `test_query` endpoint" + ] + }, { "cell_type": "code", "execution_count": null, @@ -238,12 +270,11 @@ "metadata": {}, "outputs": [], "source": [ - "schema_function = make_schema(\n", - " settings={\n", - " \"calls_per_min\": 5,\n", - " },\n", - " worker_pool=this_worker_pool_name,\n", - ")" + "# Test private version\n", + "result = high_client.api.services.bigquery.test_query.private(\n", + " sql_query=f\"SELECT * FROM {dataset_1}.{table_1} LIMIT 10\"\n", + ")\n", + "result" ] }, { @@ -252,8 +283,7 @@ "metadata": {}, "outputs": [], "source": [ - "high_client.custom_api.add(endpoint=schema_function)\n", - "high_client.refresh()" + "assert len(result) == 10" ] }, { @@ -262,9 +292,13 @@ "metadata": {}, "outputs": [], "source": [ - "submit_query_function = make_submit_query(\n", - " settings={}, worker_pool=this_worker_pool_name\n", - ")" + "# Test mock version for wrong queries\n", + "with sy.raises(\n", + " sy.SyftException(public_message=\"*must be qualified with a dataset*\"), show=True\n", + "):\n", + " _ = high_client.api.services.bigquery.test_query.mock(\n", + " sql_query=\"SELECT * FROM invalid_table LIMIT 1\"\n", + " )" ] }, { @@ -273,7 +307,11 @@ "metadata": {}, "outputs": [], "source": [ - "high_client.custom_api.add(endpoint=submit_query_function)" + "# Test private version\n", + "result = high_client.api.services.bigquery.test_query.private(\n", + " sql_query=f\"SELECT * FROM {dataset_1}.{table_1} LIMIT 1\"\n", + ")\n", + "result" ] }, { @@ -282,9 +320,14 @@ "metadata": {}, "outputs": [], "source": [ - "high_client.api.services.api.update(\n", - " endpoint_path=\"bigquery.submit_query\", hide_mock_definition=True\n", - ")" + "assert len(result) == 1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Inspect endpoint state" ] }, { @@ -293,7 +336,9 @@ "metadata": {}, "outputs": [], "source": [ - "high_client.custom_api.api_endpoints()" + "# Inspect the context state on an endpoint\n", + "state = high_client.api.services.bigquery.test_query.mock.context.state\n", + "state" ] }, { @@ -302,7 +347,7 @@ "metadata": {}, "outputs": [], "source": [ - "assert len(high_client.custom_api.api_endpoints()) == 3" + "len(state[ADMIN_EMAIL])" ] }, { @@ -311,7 +356,14 @@ "metadata": {}, "outputs": [], "source": [ - "high_client.api.services.bigquery.test_query" + "assert len(state[ADMIN_EMAIL]) >= 2" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Create `schema` endpoint" ] }, { @@ -320,7 +372,12 @@ "metadata": {}, "outputs": [], "source": [ - "high_client.api.services.bigquery.submit_query" + "schema_function = make_schema(\n", + " settings={\n", + " \"calls_per_min\": 5,\n", + " },\n", + " worker_pool=this_worker_pool_name,\n", + ")" ] }, { @@ -329,11 +386,15 @@ "metadata": {}, "outputs": [], "source": [ - "# Test mock version\n", - "result = high_client.api.services.bigquery.test_query.mock(\n", - " sql_query=f\"SELECT * FROM {dataset_1}.{table_1} LIMIT 10\"\n", - ")\n", - "result" + "high_client.custom_api.add(endpoint=schema_function)\n", + "high_client.refresh()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Test `schema` endpoint" ] }, { @@ -342,10 +403,8 @@ "metadata": {}, "outputs": [], "source": [ - "# Test private version\n", - "result = high_client.api.services.bigquery.test_query.private(\n", - " sql_query=f\"SELECT * FROM {dataset_1}.{table_1} LIMIT 10\"\n", - ")\n", + "# Testing schema\n", + "result = high_client.api.services.bigquery.schema()\n", "result" ] }, @@ -355,7 +414,14 @@ "metadata": {}, "outputs": [], "source": [ - "assert len(result) == 10" + "assert len(result) == 23" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Create `submit_query` endpoint" ] }, { @@ -364,13 +430,9 @@ "metadata": {}, "outputs": [], "source": [ - "# Test mock version for wrong queries\n", - "with sy.raises(\n", - " sy.SyftException(public_message=\"*must be qualified with a dataset*\"), show=True\n", - "):\n", - " _ = high_client.api.services.bigquery.test_query.mock(\n", - " sql_query=\"SELECT * FROM invalid_table LIMIT 1\"\n", - " )" + "submit_query_function = make_submit_query(\n", + " settings={}, worker_pool=this_worker_pool_name\n", + ")" ] }, { @@ -379,11 +441,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Test private version\n", - "result = high_client.api.services.bigquery.test_query.private(\n", - " sql_query=f\"SELECT * FROM {dataset_1}.{table_1} LIMIT 1\"\n", - ")\n", - "result" + "high_client.custom_api.add(endpoint=submit_query_function)" ] }, { @@ -392,7 +450,9 @@ "metadata": {}, "outputs": [], "source": [ - "assert len(result) == 1" + "high_client.api.services.api.update(\n", + " endpoint_path=\"bigquery.submit_query\", hide_mock_definition=True\n", + ")" ] }, { @@ -401,9 +461,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Inspect the context state on an endpoint\n", - "state = high_client.api.services.bigquery.test_query.mock.context.state\n", - "state" + "high_client.custom_api.api_endpoints()" ] }, { @@ -412,7 +470,7 @@ "metadata": {}, "outputs": [], "source": [ - "assert len(state[ADMIN_EMAIL]) >= 3" + "assert len(high_client.custom_api.api_endpoints()) == 3" ] }, { @@ -421,9 +479,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Testing schema\n", - "result = high_client.api.services.bigquery.schema()\n", - "result" + "high_client.api.services.bigquery.test_query" ] }, { @@ -432,7 +488,14 @@ "metadata": {}, "outputs": [], "source": [ - "assert len(result) == 23" + "high_client.api.services.bigquery.submit_query" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Test `submit_query` endpoint" ] }, { @@ -448,6 +511,13 @@ ")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Test emails" + ] + }, { "cell_type": "code", "execution_count": null, @@ -502,6 +572,13 @@ "assert \"Query submitted\" in result" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Cleanup" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/notebooks/scenarios/bigquery/021-create-jobs.ipynb b/notebooks/scenarios/bigquery/021-create-jobs.ipynb index ea549d3625d..3625af65c88 100644 --- a/notebooks/scenarios/bigquery/021-create-jobs.ipynb +++ b/notebooks/scenarios/bigquery/021-create-jobs.ipynb @@ -34,10 +34,15 @@ "outputs": [], "source": [ "# stdlib\n", + "from collections import Counter\n", "import os\n", "\n", - "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", - "environment" + "# third party\n", + "from helpers import get_email_server\n", + "\n", + "# syft absolute\n", + "import syft as sy\n", + "from syft.service.job.job_stash import JobStatus" ] }, { @@ -47,10 +52,8 @@ "metadata": {}, "outputs": [], "source": [ - "# third party\n", - "\n", - "# syft absolute\n", - "import syft as sy" + "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", + "environment" ] }, { @@ -60,26 +63,17 @@ "metadata": {}, "outputs": [], "source": [ - "# third party\n", - "# run email server\n", - "from helpers import EmailServer\n", - "from helpers import SMTPTestServer\n", - "\n", - "email_server = EmailServer()\n", - "email_server.reset_emails()\n", - "smtp_server = SMTPTestServer(email_server)\n", - "smtp_server.start()" + "SERVER_PORT = \"8080\"\n", + "SERVER_URL = f\"http://localhost:{SERVER_PORT}\"\n", + "ADMIN_EMAIL, ADMIN_PW = \"admin2@bigquery.org\", \"bqpw2\"" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "id": "5", "metadata": {}, - "outputs": [], "source": [ - "SERVER_PORT = \"8080\"\n", - "SERVER_URL = f\"http://localhost:{SERVER_PORT}\"" + "# Launch server & login" ] }, { @@ -106,7 +100,16 @@ "metadata": {}, "outputs": [], "source": [ - "ADMIN_EMAIL, ADMIN_PW = \"admin2@bigquery.org\", \"bqpw2\"\n", + "email_server, smtp_server = get_email_server(reset=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ "high_client = sy.login(\n", " url=\"http://localhost:8080\", email=ADMIN_EMAIL, password=ADMIN_PW\n", ")" @@ -115,7 +118,7 @@ { "cell_type": "code", "execution_count": null, - "id": "8", + "id": "9", "metadata": {}, "outputs": [], "source": [ @@ -126,7 +129,7 @@ { "cell_type": "code", "execution_count": null, - "id": "9", + "id": "10", "metadata": {}, "outputs": [], "source": [ @@ -135,7 +138,7 @@ }, { "cell_type": "markdown", - "id": "10", + "id": "11", "metadata": {}, "source": [ "# Create jobs" @@ -144,7 +147,7 @@ { "cell_type": "code", "execution_count": null, - "id": "11", + "id": "12", "metadata": {}, "outputs": [], "source": [ @@ -156,43 +159,40 @@ }, { "cell_type": "markdown", - "id": "12", + "id": "13", "metadata": {}, "source": [ - "# Test queries" + "# Inspect job data (requests for these jobs to be created)" ] }, { "cell_type": "code", "execution_count": null, - "id": "13", + "id": "14", "metadata": {}, "outputs": [], "source": [ "num_jobs = int(os.environ.get(\"NUM_TEST_JOBS\", 10))\n", "\n", - "jobs = create_jobs(users, total_jobs=num_jobs)" + "jobs_data = create_jobs(users, total_jobs=num_jobs)" ] }, { "cell_type": "code", "execution_count": null, - "id": "14", + "id": "15", "metadata": {}, "outputs": [], "source": [ - "# stdlib\n", - "from collections import Counter\n", - "\n", - "counts = Counter([j.job_type for j in jobs])\n", + "counts = Counter([j.job_type for j in jobs_data])\n", "for k, v in counts.most_common():\n", - " print(f\"number of {k}: {v}\")" + " print(f\"{k}: #{v}\")" ] }, { "cell_type": "code", "execution_count": null, - "id": "15", + "id": "16", "metadata": {}, "outputs": [], "source": [ @@ -203,45 +203,41 @@ { "cell_type": "code", "execution_count": null, - "id": "16", + "id": "17", "metadata": {}, "outputs": [], "source": [ - "print(f\"num jobs: {len(jobs)}\")\n", - "num_should_submit = 0\n", - "for job in jobs:\n", - " print(\n", - " f\"Job type: {job.job_type}, should succeed: {job.should_succeed}, should submit: {job.should_submit}\"\n", - " )\n", - " if job.should_submit:\n", - " num_should_submit += 1" + "print(f\"{len(jobs_data)=}\")\n", + "\n", + "for job in jobs_data:\n", + " print(f\"{job.job_type=}, {job.should_succeed=}, {job.should_submit=}\")" ] }, { "cell_type": "code", "execution_count": null, - "id": "17", + "id": "18", "metadata": {}, "outputs": [], "source": [ - "assert len(jobs) == num_jobs\n", - "assert all(isinstance(j, TestJob) for j in jobs)\n", - "assert all(job.client is not None for job in jobs)" + "assert len(jobs_data) == num_jobs\n", + "assert all(isinstance(j, TestJob) for j in jobs_data)\n", + "assert all(job.client is not None for job in jobs_data)" ] }, { "cell_type": "code", "execution_count": null, - "id": "18", + "id": "19", "metadata": {}, "outputs": [], "source": [ - "save_jobs(jobs)" + "save_jobs(jobs_data)" ] }, { "cell_type": "markdown", - "id": "19", + "id": "20", "metadata": {}, "source": [ "# Submit jobs\n" @@ -250,136 +246,170 @@ { "cell_type": "code", "execution_count": null, - "id": "20", + "id": "21", "metadata": {}, "outputs": [], "source": [ "admin_emails_before = len(email_server.get_emails_for_user(\"admin@bigquery.org\"))\n", - "print(\"admin emails before\", admin_emails_before)\n", - "admin_emails_before" + "print(f\"{admin_emails_before=}\")" + ] + }, + { + "cell_type": "markdown", + "id": "22", + "metadata": {}, + "source": [ + "## Test Succesful jobs" ] }, { "cell_type": "code", "execution_count": null, - "id": "21", + "id": "23", "metadata": {}, "outputs": [], "source": [ - "# stdlib\n", - "\n", - "responses = []\n", - "\n", - "for job in jobs:\n", + "jobs_submit_should_succeed = [j for j in jobs_data if j.should_submit]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24", + "metadata": {}, + "outputs": [], + "source": [ + "for job in jobs_submit_should_succeed:\n", + " client = job.client\n", + " response = client.api.services.bigquery.submit_query(\n", + " func_name=job.func_name, query=job.query\n", + " )\n", + " job.code_path = extract_code_path(response)" + ] + }, + { + "cell_type": "markdown", + "id": "25", + "metadata": {}, + "source": [ + "## Test failures" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "26", + "metadata": {}, + "outputs": [], + "source": [ + "jobs_submit_should_fail = [j for j in jobs_data if not j.should_submit]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "27", + "metadata": {}, + "outputs": [], + "source": [ + "for job in jobs_submit_should_fail:\n", " client = job.client\n", "\n", - " if not job.should_submit:\n", - " # Submitting should throw error (eg func_name invalid syntax)\n", - " with sy.raises(sy.SyftException):\n", - " response = client.api.services.bigquery.submit_query(\n", - " func_name=job.func_name, query=job.query\n", - " )\n", - " responses.append(None)\n", - " else:\n", - " response = client.api.services.bigquery.submit_query(\n", + " with sy.raises(sy.SyftException):\n", + " client.api.services.bigquery.submit_query(\n", " func_name=job.func_name, query=job.query\n", - " )\n", - " job.code_path = extract_code_path(response)\n", - " responses.append(response)\n", - "\n", - " # time.sleep(1)" + " )" ] }, { "cell_type": "code", "execution_count": null, - "id": "22", + "id": "28", "metadata": {}, "outputs": [], "source": [ - "for job in jobs:\n", - " print(\n", - " f\"Job {job.func_name:.20} is submitted, {job.should_submit}, should be submitted {job.is_submitted}\"\n", - " )\n", + "for job in jobs_data:\n", + " print(f\"Job {job.func_name:.20} {job.should_submit=}, {job.is_submitted=}\")\n", "\n", - "assert all(job.is_submitted == job.should_submit for job in jobs)" + "assert all(job.is_submitted == job.should_submit for job in jobs_data)" ] }, { "cell_type": "code", "execution_count": null, - "id": "23", + "id": "29", "metadata": {}, "outputs": [], "source": [ - "save_jobs(jobs)" + "save_jobs(jobs_data)" ] }, { "cell_type": "markdown", - "id": "24", + "id": "30", "metadata": {}, "source": [ - "## Test: cannot execute" + "## Test: cannot execute submitted jobs yet" ] }, { "cell_type": "code", "execution_count": null, - "id": "25", + "id": "31", "metadata": {}, "outputs": [], "source": [ - "submitted_jobs = [job for job in jobs if job.should_submit]\n", - "\n", + "submitted_jobs = [job for job in jobs_data if job.should_submit]\n", + "job_execution_fns = [getattr(job.client.code, job.code_path) for job in submitted_jobs]\n", "assert len(submitted_jobs) # failsafe for next tests" ] }, { "cell_type": "code", "execution_count": null, - "id": "26", + "id": "32", "metadata": {}, "outputs": [], "source": [ - "# Blocking\n", - "\n", - "for job in submitted_jobs:\n", - " execute_code_fn = getattr(job.client.code, job.code_path)\n", + "for fn in job_execution_fns:\n", + " # blocking\n", " with sy.raises(\n", " sy.SyftException(public_message=\"*Your code is waiting for approval*\")\n", " ):\n", - " result = execute_code_fn()" + " result = fn()" ] }, { "cell_type": "code", "execution_count": null, - "id": "27", + "id": "33", "metadata": {}, "outputs": [], "source": [ - "# Nonblocking\n", - "\n", - "# syft absolute\n", - "from syft.service.job.job_stash import JobStatus\n", - "\n", - "for job in submitted_jobs:\n", - " execute_code_fn = getattr(job.client.code, job.code_path)\n", - " result_job = execute_code_fn(blocking=False)\n", + "for fn in job_execution_fns:\n", + " # nonblocking\n", + " result_job = fn(blocking=False)\n", " result_job.wait()\n", " assert isinstance(result_job.result, sy.SyftError)\n", - " assert result_job.status == JobStatus.ERRORED\n", - " # time.sleep(1)" + " assert result_job.status == JobStatus.ERRORED" + ] + }, + { + "cell_type": "markdown", + "id": "34", + "metadata": {}, + "source": [ + "# Verify that admin has emails for submitted requests" ] }, { "cell_type": "code", "execution_count": null, - "id": "28", + "id": "35", "metadata": {}, "outputs": [], "source": [ + "num_should_submit = sum(j.should_submit for j in jobs_data)\n", "admin_emails_after = len(email_server.get_emails_for_user(\"admin@bigquery.org\"))\n", "print(\"admin emails after\", admin_emails_after)\n", "assert admin_emails_after >= admin_emails_before + num_should_submit\n", @@ -387,10 +417,18 @@ "# assert len(users_emails) == after_number_of_emails + 1" ] }, + { + "cell_type": "markdown", + "id": "36", + "metadata": {}, + "source": [ + "# Cleanup" + ] + }, { "cell_type": "code", "execution_count": null, - "id": "29", + "id": "37", "metadata": {}, "outputs": [], "source": [ @@ -400,7 +438,7 @@ { "cell_type": "code", "execution_count": null, - "id": "30", + "id": "38", "metadata": {}, "outputs": [], "source": [ @@ -410,7 +448,7 @@ { "cell_type": "code", "execution_count": null, - "id": "31", + "id": "39", "metadata": {}, "outputs": [], "source": [] diff --git a/notebooks/scenarios/bigquery/04-do-review-requests.ipynb b/notebooks/scenarios/bigquery/040-do-review-requests.ipynb similarity index 61% rename from notebooks/scenarios/bigquery/04-do-review-requests.ipynb rename to notebooks/scenarios/bigquery/040-do-review-requests.ipynb index 716eb756c43..9f0a301f2e3 100644 --- a/notebooks/scenarios/bigquery/04-do-review-requests.ipynb +++ b/notebooks/scenarios/bigquery/040-do-review-requests.ipynb @@ -18,14 +18,18 @@ "metadata": {}, "outputs": [], "source": [ + "# stdlib\n", + "import random\n", + "\n", "# third party\n", - "# run email server\n", - "from helpers import EmailServer\n", - "from helpers import SMTPTestServer\n", + "from helpers import get_email_server\n", + "from job_helpers import approve_by_running\n", + "from job_helpers import get_job_emails\n", + "from job_helpers import get_request_for_job_info\n", "\n", - "email_server = EmailServer()\n", - "smtp_server = SMTPTestServer(email_server)\n", - "smtp_server.start()" + "# syft absolute\n", + "import syft as sy\n", + "from syft.service.job.job_stash import Job" ] }, { @@ -34,12 +38,14 @@ "metadata": {}, "outputs": [], "source": [ - "# stdlib\n", - "import random\n", - "\n", - "# syft absolute\n", - "import syft as sy\n", - "from syft.service.job.job_stash import Job" + "ADMIN_EMAIL, ADMIN_PW = \"admin2@bigquery.org\", \"bqpw2\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Start server & login" ] }, { @@ -64,7 +70,6 @@ "metadata": {}, "outputs": [], "source": [ - "ADMIN_EMAIL, ADMIN_PW = \"admin2@bigquery.org\", \"bqpw2\"\n", "high_client = sy.login(\n", " url=\"http://localhost:8080\", email=ADMIN_EMAIL, password=ADMIN_PW\n", ")" @@ -76,7 +81,14 @@ "metadata": {}, "outputs": [], "source": [ - "high_client.requests.get_all_pending()" + "email_server, smtp_server = get_email_server()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Review requests" ] }, { @@ -97,7 +109,7 @@ "metadata": {}, "outputs": [], "source": [ - "users = load_users(high_client)" + "high_client.requests.get_all_pending()" ] }, { @@ -106,7 +118,14 @@ "metadata": {}, "outputs": [], "source": [ - "jobs = load_jobs(users, high_client)" + "users = load_users(high_client)\n", + "jobs_data = load_jobs(users, high_client)\n", + "all_requests = high_client.requests\n", + "submitted_jobs_data = [job for job in jobs_data if job.is_submitted]\n", + "n_emails_per_job_user = {\n", + " k: len(v)\n", + " for k, v in get_job_emails(submitted_jobs_data, high_client, email_server).items()\n", + "}" ] }, { @@ -115,7 +134,17 @@ "metadata": {}, "outputs": [], "source": [ - "all_requests = high_client.requests" + "# TODO we should record whether it was approved or deposited\n", + "# and test doing both in either order as there might be a bug when\n", + "# force overwriting\n", + "# also changing deny to approve and back again" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Run or deny" ] }, { @@ -124,15 +153,12 @@ "metadata": {}, "outputs": [], "source": [ - "def get_request_for_job(requests, job):\n", - " job_requests = [\n", - " r for r in all_requests if r.code.service_func_name == job.func_name\n", - " ]\n", - " if len(job_requests) == 1:\n", - " return job_requests[0]\n", - " if len(job_requests) > 1:\n", - " raise Exception(f\"Multiple of the same job: {job} in requests: {requests}\")\n", - " return None" + "submitted_jobs_data_should_succeed = [\n", + " j for j in submitted_jobs_data if j.should_succeed\n", + "]\n", + "submitted_jobs_data_should_fail = [\n", + " j for j in submitted_jobs_data if not j.should_succeed\n", + "]" ] }, { @@ -141,7 +167,18 @@ "metadata": {}, "outputs": [], "source": [ - "submitted_jobs = [job for job in jobs if job.is_submitted]" + "for job in submitted_jobs_data_should_succeed:\n", + " request = get_request_for_job_info(all_requests, job)\n", + " if random.randrange(2):\n", + " choice = \"approved with deposit_result\"\n", + " response = approve_by_running(request)\n", + " assert isinstance(response, Job)\n", + " else:\n", + " choice = \"approved\"\n", + " response = request.approve()\n", + " assert isinstance(response, sy.SyftSuccess)\n", + " print(f\"Job {job.func_name} should succeed: {job.should_succeed} and was {choice}\")\n", + " job.admin_reviewed = True" ] }, { @@ -150,17 +187,21 @@ "metadata": {}, "outputs": [], "source": [ - "def approve_by_running(request):\n", - " job = request.code(blocking=False)\n", - " result = job.wait()\n", - " print(\"got result of type\", type(result), \"bool\", bool(result))\n", - " # got result of type bool False\n", - " # assert result won't work unless we know what type is coming back\n", - " job_info = job.info(result=True)\n", - " # need force when running multiple times\n", - " # todo check and dont run if its already done\n", - " response = request.deposit_result(job_info, approve=True, force=True)\n", - " return response" + "for job in submitted_jobs_data_should_fail:\n", + " request = get_request_for_job_info(all_requests, job)\n", + " response = request.deny(\n", + " reason=f\"Your request {job.func_name} looks wrong, try again.\"\n", + " )\n", + " assert isinstance(response, sy.SyftSuccess)\n", + " assert not job.should_succeed\n", + " job.admin_reviewed = True" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Verify that users have new emails" ] }, { @@ -169,10 +210,10 @@ "metadata": {}, "outputs": [], "source": [ - "# TODO we should record whether it was approved or deposited\n", - "# and test doing both in either order as there might be a bug when\n", - "# force overwriting\n", - "# also changing deny to approve and back again" + "new_n_emails_per_job_user = {\n", + " k: len(v)\n", + " for k, v in get_job_emails(submitted_jobs_data, high_client, email_server).items()\n", + "}" ] }, { @@ -181,34 +222,16 @@ "metadata": {}, "outputs": [], "source": [ - "for job in submitted_jobs:\n", - " request = get_request_for_job(all_requests, job)\n", - " choice = None\n", - " number_of_emails = len(\n", - " email_server.get_emails_for_user(request.requesting_user_email)\n", - " )\n", - " if job.should_succeed:\n", - " if random.randrange(2):\n", - " choice = \"approved with deposit_result\"\n", - " response = approve_by_running(request)\n", - " assert isinstance(response, Job)\n", - " else:\n", - " choice = \"approved\"\n", - " response = request.approve()\n", - " assert isinstance(response, sy.SyftSuccess)\n", - " else:\n", - " choice = \"denied\"\n", - " response = request.deny(\n", - " reason=f\"Your request {job.func_name} looks wrong, try again.\"\n", - " )\n", - " assert isinstance(response, sy.SyftSuccess)\n", - "\n", - " after_users_emails = len(\n", - " email_server.get_emails_for_user(request.requesting_user_email)\n", - " )\n", - " # assert after_users_emails > number_of_emails\n", - " job.admin_reviewed = True\n", - " print(f\"Job {job.func_name} should succeed: {job.should_succeed} and was {choice}\")" + "# for user_email, new_count in new_n_emails_per_job_user.items():\n", + "# old_count = n_emails_per_job_user[user_email]\n", + "# assert new_count > old_count" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Save state" ] }, { @@ -217,7 +240,7 @@ "metadata": {}, "outputs": [], "source": [ - "save_jobs(jobs)" + "save_jobs(jobs_data)" ] }, { @@ -238,6 +261,13 @@ "high_client.requests.get_all_rejected()" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Cleanup" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/notebooks/scenarios/bigquery/05-ds-get-results.ipynb b/notebooks/scenarios/bigquery/050-ds-get-results.ipynb similarity index 68% rename from notebooks/scenarios/bigquery/05-ds-get-results.ipynb rename to notebooks/scenarios/bigquery/050-ds-get-results.ipynb index 843fdefcaf6..72ab42122c1 100644 --- a/notebooks/scenarios/bigquery/05-ds-get-results.ipynb +++ b/notebooks/scenarios/bigquery/050-ds-get-results.ipynb @@ -19,13 +19,13 @@ "outputs": [], "source": [ "# third party\n", - "# run email server\n", - "from helpers import EmailServer\n", - "from helpers import SMTPTestServer\n", + "from helpers import get_email_server\n", + "from helpers import load_users\n", + "from job_helpers import load_jobs\n", + "from job_helpers import save_jobs\n", "\n", - "email_server = EmailServer()\n", - "smtp_server = SMTPTestServer(email_server)\n", - "smtp_server.start()" + "# syft absolute\n", + "import syft as sy" ] }, { @@ -34,8 +34,14 @@ "metadata": {}, "outputs": [], "source": [ - "# syft absolute\n", - "import syft as sy" + "ADMIN_EMAIL, ADMIN_PW = \"admin2@bigquery.org\", \"bqpw2\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Launch server & login" ] }, { @@ -60,7 +66,6 @@ "metadata": {}, "outputs": [], "source": [ - "ADMIN_EMAIL, ADMIN_PW = \"admin2@bigquery.org\", \"bqpw2\"\n", "high_client = sy.login(\n", " url=\"http://localhost:8080\", email=ADMIN_EMAIL, password=ADMIN_PW\n", ")" @@ -72,19 +77,14 @@ "metadata": {}, "outputs": [], "source": [ - "# third party\n", - "from helpers import load_users\n", - "from job_helpers import load_jobs\n", - "from job_helpers import save_jobs" + "email_server, smtp_server = get_email_server()" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "users = load_users(high_client)" + "# Download results" ] }, { @@ -93,6 +93,7 @@ "metadata": {}, "outputs": [], "source": [ + "users = load_users(high_client)\n", "jobs = load_jobs(users, high_client)" ] }, @@ -104,7 +105,12 @@ "source": [ "# submitted_jobs = [job for job in jobs if job.is_submitted]\n", "reviewed_jobs = [job for job in jobs if job.admin_reviewed]\n", - "len(reviewed_jobs)" + "reviewed_jobs_should_succeed = [j for j in reviewed_jobs if j.should_succeed]\n", + "reviewed_jobs_should_fail = [j for j in reviewed_jobs if not j.should_succeed]\n", + "\n", + "print(\n", + " f\"{len(reviewed_jobs)=}, {len(reviewed_jobs_should_succeed)=}, {len(reviewed_jobs_should_fail)=}\"\n", + ")" ] }, { @@ -124,42 +130,35 @@ "metadata": {}, "outputs": [], "source": [ - "for job in reviewed_jobs:\n", - " print(\n", - " f\"> Checking job: {job.job_type} {job.func_name} for user {job.user_email} {job.should_succeed}\"\n", - " )\n", + "for job in reviewed_jobs_should_succeed:\n", + " print(f\"> Checking job: {job.job_type} {job.func_name} for user {job.user_email}\")\n", " api_method = job.code_method\n", + " j = api_method(blocking=False)\n", + " res = j.wait()\n", "\n", - " if job.should_succeed:\n", - " # print(\"Expecting job to succeed\")\n", - " j = api_method(blocking=False)\n", - " res = j.wait()\n", - " if isinstance(res, sy.SyftError):\n", - " job.result_as_expected = False\n", - " print(\"Expected success, got error\")\n", - " continue\n", + " if isinstance(res, sy.SyftError):\n", + " raise sy.SyftException(public_message=\"Expected success, got error\")\n", "\n", - " result = res.get()\n", - " if hasattr(result, \"__len__\"):\n", - " pass\n", - " # print(\"> Result length\", len(result))\n", + " result = res.get()\n", + " job.result_as_expected = True" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for job in reviewed_jobs_should_fail:\n", + " print(f\"> Checking job: {job.job_type} {job.func_name} for user {job.user_email}\")\n", + " api_method = job.code_method\n", "\n", - " # assert len(result) == job.settings[\"limit\"]\n", - " print(\"success\")\n", + " j = api_method(blocking=False)\n", + " res = j.wait()\n", + " if isinstance(res, sy.SyftError):\n", " job.result_as_expected = True\n", - " save_jobs(jobs)\n", " else:\n", - " # print(\"Expecting job to fail\")\n", - " j = api_method(blocking=False)\n", - " res = j.wait()\n", - " if isinstance(res, sy.SyftError):\n", - " job.result_as_expected = True\n", - " print(\"Expected to error, success\")\n", - " else:\n", - " print(\"failed job didnt raise\", type(j))\n", - " job.result_as_expected = False\n", - "\n", - " save_jobs(jobs)" + " raise sy.SyftException(public_message=f\"failed, job didnt raise {type(j)}\")" ] }, { @@ -168,8 +167,7 @@ "metadata": {}, "outputs": [], "source": [ - "expected_jobs = [job for job in jobs if job.result_as_expected]\n", - "len(expected_jobs)" + "save_jobs(jobs)" ] }, { @@ -179,10 +177,18 @@ "outputs": [], "source": [ "# TODO fix\n", + "expected_jobs = [job for job in jobs if job.result_as_expected]\n", "print(f\"got expected_jobs: {len(expected_jobs)} == reviewed_jobs: {len(reviewed_jobs)}\")\n", "assert len(reviewed_jobs) == len(expected_jobs)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Cleanup" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/notebooks/scenarios/bigquery/helpers.py b/notebooks/scenarios/bigquery/helpers.py index a57bdb1e9d3..f58d41a20f8 100644 --- a/notebooks/scenarios/bigquery/helpers.py +++ b/notebooks/scenarios/bigquery/helpers.py @@ -4,6 +4,7 @@ from dataclasses import field import json import re +import time from typing import Any # third party @@ -279,6 +280,49 @@ async def async_stop(self): self._stop_event.set() # Stop the server by setting the event +class TimeoutError(Exception): + pass + + +class Timeout: + def __init__(self, timeout_duration): + if timeout_duration > 60: + raise ValueError("Timeout duration cannot exceed 60 seconds.") + self.timeout_duration = timeout_duration + + def run_with_timeout(self, condition_func, *args, **kwargs): + start_time = time.time() + result = None + + while True: + elapsed_time = time.time() - start_time + if elapsed_time > self.timeout_duration: + raise TimeoutError( + f"Function execution exceeded {self.timeout_duration} seconds." + ) + + # Check if the condition is met + try: + if condition_func(): + print("Condition met, exiting early.") + break + except Exception as e: + print(f"Exception in target function: {e}") + break # Exit the loop if an exception occurs in the function + time.sleep(1) + + return result + + +def get_email_server(reset=False): + email_server = EmailServer() + if reset: + email_server.reset_emails() + smtp_server = SMTPTestServer(email_server) + smtp_server.start() + return email_server, smtp_server + + def create_user(root_client, test_user): if not user_exists(root_client, test_user.email): fake = Faker() diff --git a/notebooks/scenarios/bigquery/job_helpers.py b/notebooks/scenarios/bigquery/job_helpers.py index 777a781836d..f87cfdc7324 100644 --- a/notebooks/scenarios/bigquery/job_helpers.py +++ b/notebooks/scenarios/bigquery/job_helpers.py @@ -212,6 +212,13 @@ def create_job_funcname_xss(user: TestUser) -> TestJob: return job +def get_request_for_job_info(requests, job): + job_requests = [r for r in requests if r.code.service_func_name == job.func_name] + if len(job_requests) != 1: + raise Exception(f"Too many or too few requests: {job} in requests: {requests}") + return job_requests[0] + + def create_job_query_xss(user: TestUser) -> TestJob: job_type = "job_query_xss" func_name = f"{job_type}_{secrets.token_hex(3)}" @@ -301,6 +308,29 @@ def extract_code_path(response) -> str | None: return None +def approve_by_running(request): + job = request.code(blocking=False) + result = job.wait() + print("got result of type", type(result), "bool", bool(result)) + # got result of type bool False + # assert result won't work unless we know what type is coming back + job_info = job.info(result=True) + # need force when running multiple times + # todo check and dont run if its already done + response = request.deposit_result(job_info, approve=True, force=True) + return response + + +def get_job_emails(jobs, client, email_server): + all_requests = client.requests + res = {} + for job in jobs: + request = get_request_for_job_info(all_requests, job) + emails = email_server.get_emails_for_user(request.requesting_user_email) + res[request.requesting_user_email] = emails + return res + + def resolve_request(request): service_func_name = request.code.service_func_name if service_func_name.startswith("simple_query"):