Merge pull request #9197 from OpenMined/snwagh/bigquery-notebooks-tox…

…-and-secrets-json Improve secrets loading and tox, CI support for Bigquery Scenario
OpenMined · Aug 26, 2024 · 9379692 · 9379692
2 parents 8f7c991 + 12b6fe9
commit 9379692
Show file tree

Hide file tree

Showing 13 changed files with 513 additions and 230 deletions.
diff --git a/.gitignore b/.gitignore
@@ -29,6 +29,8 @@ build
 # docker compose volumes
 docker/data/*
 
+# env files
+.env
 
 # vagrant
 .vagrant

diff --git a/notebooks/.gitignore b/notebooks/.gitignore
@@ -1 +1,2 @@
-**/secrets.json
+**/secrets.json
+**/settings.yaml
diff --git a/notebooks/scenarios/bigquery/.gitignore b/notebooks/scenarios/bigquery/.gitignore
diff --git a/notebooks/scenarios/bigquery/01-setup-datasite.ipynb b/notebooks/scenarios/bigquery/01-setup-datasite.ipynb
@@ -6,8 +6,11 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "# third party\n",
+    "\n",
     "# syft absolute\n",
-    "import syft as sy"
+    "import syft as sy\n",
+    "from syft import test_settings"
    ]
   },
   {
@@ -18,7 +21,7 @@
    "source": [
     "server = sy.orchestra.launch(\n",
     "    name=\"bigquery-high\",\n",
-    "    dev_mode=False,\n",
+    "    dev_mode=True,\n",
     "    server_side_type=\"high\",\n",
     "    reset=True,\n",
     "    port=\"8080\",\n",
@@ -53,25 +56,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# add your values to secrets.json in this folder\n",
-    "secrets = sy.get_nb_secrets(\n",
-    "    {\n",
-    "        \"external_registry\": \"docker.io\",\n",
-    "    }\n",
-    ")\n",
-    "external_registry = secrets[\"external_registry\"]\n",
-    "external_registry = \"docker.io\"\n",
-    "external_registry"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "worker_docker_tag = f\"openmined/bigquery:{sy.__version__}\"\n",
-    "worker_docker_tag"
+    "assert len(high_client.worker_pools.get_all()) == 1"
    ]
   },
   {
@@ -80,8 +65,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "docker_config = sy.PrebuiltWorkerConfig(tag=f\"{external_registry}/{worker_docker_tag}\")\n",
-    "docker_config"
+    "external_registry = test_settings.get(\"external_registry\", default=\"docker.io\")\n",
+    "external_registry"
    ]
   },
   {
@@ -90,10 +75,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "submit_result = high_client.api.services.worker_image.submit(\n",
-    "    worker_config=docker_config\n",
-    ")\n",
-    "submit_result"
+    "worker_docker_tag = f\"openmined/bigquery:{sy.__version__}\"\n",
+    "worker_docker_tag"
    ]
   },
   {
@@ -102,8 +85,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "worker_image = high_client.images.get_all()[1]\n",
-    "worker_image"
+    "docker_config = sy.PrebuiltWorkerConfig(tag=f\"{external_registry}/{worker_docker_tag}\")\n",
+    "docker_config"
    ]
   },
   {
@@ -112,8 +95,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "registry_add_result = high_client.api.services.image_registry.add(external_registry)\n",
-    "registry_add_result"
+    "result = high_client.api.services.worker_image.submit(worker_config=docker_config)\n",
+    "result"
    ]
   },
   {
@@ -122,8 +105,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "registries = high_client.api.services.image_registry.get_all()\n",
-    "registries"
+    "worker_image = high_client.images.get_all()[1]\n",
+    "worker_image"
    ]
   },
   {
@@ -132,7 +115,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "registry_uid = next((r.id for r in registries if r.url == external_registry), None)"
+    "result = high_client.api.services.image_registry.add(external_registry)\n",
+    "result"
    ]
   },
   {
@@ -150,11 +134,12 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "high_client.api.services.worker_pool.launch(\n",
+    "result = high_client.api.services.worker_pool.launch(\n",
     "    pool_name=worker_pool_name,\n",
     "    image_uid=worker_image.id,\n",
     "    num_workers=1,\n",
-    ")"
+    ")\n",
+    "result"
    ]
   },
   {
@@ -163,9 +148,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "high_client.worker_pools.scale(number=5, pool_name=worker_pool_name)\n",
-    "\n",
-    "high_client.worker_pools"
+    "# result = high_client.worker_pools.scale(number=5, pool_name=worker_pool_name)\n",
+    "# result"
    ]
   },
   {
@@ -174,16 +158,18 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "base_worker_image = high_client.images.get_all()[0]\n",
-    "base_worker_image"
+    "assert len(high_client.worker_pools.get_all()) == 2"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
    "outputs": [],
-   "source": []
+   "source": [
+    "base_worker_image = high_client.images.get_all()[0]\n",
+    "base_worker_image"
+   ]
   },
   {
    "cell_type": "code",
@@ -208,6 +194,15 @@
     "high_client.settings.allow_guest_signup(enable=False)"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "assert len(high_client.api.services.user.get_all()) == 2"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -241,17 +236,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# # can't shutdown because in python the workers are gone in the next notebook\n",
-    "# assert False"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# server.land()"
+    "server.land()"
    ]
   },
   {