From 0f9937d8a03a1d5982cfdf189d244c5893843389 Mon Sep 17 00:00:00 2001 From: Brendan Schell Date: Mon, 9 Sep 2024 15:59:09 -0400 Subject: [PATCH 01/11] k8s scale down 2nd approach --- .../syft/src/syft/service/job/job_service.py | 6 ++- .../src/syft/service/queue/zmq_consumer.py | 1 + .../service/worker/worker_pool_service.py | 37 +++++++++++-------- .../src/syft/service/worker/worker_stash.py | 4 +- 4 files changed, 30 insertions(+), 18 deletions(-) diff --git a/packages/syft/src/syft/service/job/job_service.py b/packages/syft/src/syft/service/job/job_service.py index fbbdfd7d856..fadf90c6f18 100644 --- a/packages/syft/src/syft/service/job/job_service.py +++ b/packages/syft/src/syft/service/job/job_service.py @@ -162,10 +162,12 @@ def _kill(self, context: AuthedServiceContext, job: Job) -> SyftSuccess: results.append(res) # wait for job and subjobs to be killed by MonitorThread - wait_until(lambda: job.fetched_status == JobStatus.INTERRUPTED) + + wait_until(lambda: self.get(context, uid=job.id).status == JobStatus.INTERRUPTED) + subjob_uids = [subjob.id for subjob in self.get_subjobs(context, uid=job.id)] wait_until( lambda: all( - subjob.fetched_status == JobStatus.INTERRUPTED for subjob in job.subjobs + self.get(context, uid=subjob_id).status == JobStatus.INTERRUPTED for subjob_id in subjob_uids ) ) diff --git a/packages/syft/src/syft/service/queue/zmq_consumer.py b/packages/syft/src/syft/service/queue/zmq_consumer.py index f6993d6b032..2ec06916826 100644 --- a/packages/syft/src/syft/service/queue/zmq_consumer.py +++ b/packages/syft/src/syft/service/queue/zmq_consumer.py @@ -291,6 +291,7 @@ def _set_worker_job(self, job_id: UID | None) -> None: credentials=self.worker_stash.partition.root_verify_key, worker_uid=self.syft_worker_id, consumer_state=consumer_state, + job_id=job_id, ) if res.is_err(): logger.error( diff --git a/packages/syft/src/syft/service/worker/worker_pool_service.py b/packages/syft/src/syft/service/worker/worker_pool_service.py index 4ceced2bf26..0c8c3bf6807 100644 --- a/packages/syft/src/syft/service/worker/worker_pool_service.py +++ b/packages/syft/src/syft/service/worker/worker_pool_service.py @@ -449,28 +449,35 @@ def scale( ) else: # scale down at kubernetes control plane - runner = KubernetesRunner() - scale_kubernetes_pool( - runner, - pool_name=worker_pool.name, - replicas=number, - ).unwrap() + # scale down removes the last "n" workers # workers to delete = len(workers) - number - workers_to_delete = worker_pool.worker_list[ - -(current_worker_count - number) : + workers = [ + worker.resolve_with_context(context=context).unwrap() + for worker in worker_pool.worker_list ] - worker_stash = context.server.services.worker.stash - # delete linkedobj workers + + # get last "n" workers from pod list + runner = KubernetesRunner() + k8s_pods = [pod.name for pod in runner.get_pool_pods(pool_name=worker_pool.name)] + k8s_pods_to_kill = k8s_pods[-(current_worker_count - number) :] + workers_to_delete = [worker for worker in workers if worker.name in k8s_pods_to_kill] + worker_service = cast( + WorkerService, context.server.get_service("WorkerService") + ) + for worker in workers_to_delete: - worker_stash.delete_by_uid( - credentials=context.credentials, - uid=worker.object_uid, - ).unwrap() + worker_service.delete(context=context, uid=worker.id, force=True) + + + scale_kubernetes_pool( + runner, + pool_name=worker_pool.name, + replicas=number, + ).unwrap() - client_warning += "Scaling down workers doesn't kill the associated jobs. Please delete them manually." # update worker_pool worker_pool.max_count = number diff --git a/packages/syft/src/syft/service/worker/worker_stash.py b/packages/syft/src/syft/service/worker/worker_stash.py index b2b059ffec5..95d719f6dc4 100644 --- a/packages/syft/src/syft/service/worker/worker_stash.py +++ b/packages/syft/src/syft/service/worker/worker_stash.py @@ -67,8 +67,10 @@ def get_worker_by_name( @as_result(StashException, NotFoundException) def update_consumer_state( - self, credentials: SyftVerifyKey, worker_uid: UID, consumer_state: ConsumerState + self, credentials: SyftVerifyKey, worker_uid: UID, consumer_state: ConsumerState, job_id: UID | None = None ) -> SyftWorker: worker = self.get_by_uid(credentials=credentials, uid=worker_uid).unwrap() worker.consumer_state = consumer_state + if job_id is not None: + worker.job_id = job_id return self.update(credentials=credentials, obj=worker).unwrap() From 5b789c2e3e5922501da9b29f087ef4f515d8b285 Mon Sep 17 00:00:00 2001 From: Brendan Schell Date: Mon, 9 Sep 2024 16:00:06 -0400 Subject: [PATCH 02/11] add notebook --- .../001-scale-delete-worker-pools.ipynb | 211 ++++++++++++++++-- 1 file changed, 196 insertions(+), 15 deletions(-) diff --git a/notebooks/scenarios/bigquery/001-scale-delete-worker-pools.ipynb b/notebooks/scenarios/bigquery/001-scale-delete-worker-pools.ipynb index e46587d23e5..2c698230bda 100644 --- a/notebooks/scenarios/bigquery/001-scale-delete-worker-pools.ipynb +++ b/notebooks/scenarios/bigquery/001-scale-delete-worker-pools.ipynb @@ -10,7 +10,8 @@ "# import os\n", "# os.environ[\"ORCHESTRA_DEPLOYMENT_TYPE\"] = \"remote\"\n", "# os.environ[\"DEV_MODE\"] = \"True\"\n", - "# os.environ[\"TEST_EXTERNAL_REGISTRY\"] = \"k3d-registry.localhost:5800\"" + "# os.environ[\"TEST_EXTERNAL_REGISTRY\"] = \"k3d-registry.localhost:5800\"\n", + "# stdlib" ] }, { @@ -181,7 +182,7 @@ "id": "14", "metadata": {}, "source": [ - "##### Scale down" + "##### Give workers some long-running jobs\n" ] }, { @@ -190,6 +191,171 @@ "id": "15", "metadata": {}, "outputs": [], + "source": [ + "@sy.syft_function_single_use(worker_pool_name=default_worker_pool.name)\n", + "def wait_100_seconds_1():\n", + " # stdlib\n", + " import time\n", + "\n", + " time.sleep(100)\n", + "\n", + "\n", + "@sy.syft_function_single_use(worker_pool_name=default_worker_pool.name)\n", + "def wait_100_seconds_2():\n", + " # stdlib\n", + " import time\n", + "\n", + " time.sleep(100)\n", + "\n", + "\n", + "@sy.syft_function_single_use(worker_pool_name=default_worker_pool.name)\n", + "def wait_100_seconds_3():\n", + " # stdlib\n", + " import time\n", + "\n", + " time.sleep(100)\n", + "\n", + "\n", + "@sy.syft_function_single_use(worker_pool_name=default_worker_pool.name)\n", + "def wait_100_seconds_4():\n", + " # stdlib\n", + " import time\n", + "\n", + " time.sleep(100)\n", + "\n", + "\n", + "@sy.syft_function_single_use(worker_pool_name=default_worker_pool.name)\n", + "def wait_100_seconds_5():\n", + " # stdlib\n", + " import time\n", + "\n", + " time.sleep(100)\n", + "\n", + "\n", + "@sy.syft_function_single_use(worker_pool_name=default_worker_pool.name)\n", + "def wait_100_seconds_6():\n", + " # stdlib\n", + " import time\n", + "\n", + " time.sleep(100)\n", + "\n", + "\n", + "@sy.syft_function_single_use(worker_pool_name=default_worker_pool.name)\n", + "def wait_100_seconds_7():\n", + " # stdlib\n", + " import time\n", + "\n", + " time.sleep(100)\n", + "\n", + "\n", + "@sy.syft_function_single_use(worker_pool_name=default_worker_pool.name)\n", + "def wait_100_seconds_8():\n", + " # stdlib\n", + " import time\n", + "\n", + " time.sleep(100)\n", + "\n", + "\n", + "@sy.syft_function_single_use(worker_pool_name=default_worker_pool.name)\n", + "def wait_100_seconds_9():\n", + " # stdlib\n", + " import time\n", + "\n", + " time.sleep(100)\n", + "\n", + "\n", + "@sy.syft_function_single_use(worker_pool_name=default_worker_pool.name)\n", + "def wait_100_seconds_10():\n", + " # stdlib\n", + " import time\n", + "\n", + " time.sleep(100)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "20", + "metadata": {}, + "outputs": [], + "source": [ + "jobs = []\n", + "high_client.code.request_code_execution(wait_100_seconds_1)\n", + "high_client.code.request_code_execution(wait_100_seconds_2)\n", + "high_client.code.request_code_execution(wait_100_seconds_3)\n", + "high_client.code.request_code_execution(wait_100_seconds_4)\n", + "high_client.code.request_code_execution(wait_100_seconds_5)\n", + "high_client.code.request_code_execution(wait_100_seconds_6)\n", + "high_client.code.request_code_execution(wait_100_seconds_7)\n", + "high_client.code.request_code_execution(wait_100_seconds_8)\n", + "high_client.code.request_code_execution(wait_100_seconds_9)\n", + "high_client.code.request_code_execution(wait_100_seconds_10)\n", + "\n", + "assert len(list(high_client.requests)) == 10\n", + "for request in high_client.requests:\n", + " request.approve()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "21", + "metadata": {}, + "outputs": [], + "source": [ + "jobs = []\n", + "jobs.append(high_client.code.wait_100_seconds_1(blocking=False))\n", + "jobs.append(high_client.code.wait_100_seconds_2(blocking=False))\n", + "jobs.append(high_client.code.wait_100_seconds_3(blocking=False))\n", + "jobs.append(high_client.code.wait_100_seconds_4(blocking=False))\n", + "jobs.append(high_client.code.wait_100_seconds_5(blocking=False))\n", + "jobs.append(high_client.code.wait_100_seconds_6(blocking=False))\n", + "jobs.append(high_client.code.wait_100_seconds_7(blocking=False))\n", + "jobs.append(high_client.code.wait_100_seconds_8(blocking=False))\n", + "jobs.append(high_client.code.wait_100_seconds_9(blocking=False))\n", + "jobs.append(high_client.code.wait_100_seconds_10(blocking=False))\n", + "\n", + "\n", + "assert len(list(high_client.jobs)) == 10" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "22", + "metadata": {}, + "outputs": [], + "source": [ + "# check that at least three workers have a job (since scaling down to 2)\n", + "# try 3 times with a 1 second sleep in case it takes time for the workers to accept the jobs\n", + "for _ in range(3):\n", + " worker_to_job_map = {}\n", + " syft_workers_ids = set()\n", + " for job in high_client.jobs:\n", + " if job.status == \"processing\":\n", + " syft_workers_ids.add(job.worker.id)\n", + " worker_to_job_map[job.worker.id] = job.id\n", + " if len(syft_workers_ids) < 3:\n", + " time.sleep(1)\n", + " else:\n", + " break\n", + "assert len(syft_workers_ids) >= 3" + ] + }, + { + "cell_type": "markdown", + "id": "23", + "metadata": {}, + "source": [ + "##### Scale down" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24", + "metadata": {}, + "outputs": [], "source": [ "# Scale down workers, this gracefully shutdowns the consumers\n", "if environment == \"remote\":\n", @@ -202,17 +368,32 @@ { "cell_type": "code", "execution_count": null, - "id": "16", + "id": "25", "metadata": {}, "outputs": [], "source": [ "if environment == \"remote\":\n", "\n", " def has_worker_scaled_down():\n", - " return (\n", + " worker_count_condition = (\n", " high_client.api.worker_pool[default_worker_pool.name].max_count\n", " == num_workers\n", " )\n", + " current_worker_ids = {\n", + " worker.id\n", + " for worker in high_client.api.services.worker_pool[\n", + " default_worker_pool.name\n", + " ].workers\n", + " }\n", + " job_status_condition = [\n", + " job.status == \"interrupted\"\n", + " for job in high_client.jobs\n", + " if job.job_worker_id is not None\n", + " and job.job_worker_id not in current_worker_ids\n", + " ]\n", + "\n", + " jobs_on_old_workers_are_interrupted = all(job_status_condition)\n", + " return worker_count_condition and jobs_on_old_workers_are_interrupted\n", "\n", " worker_scale_timeout = Timeout(timeout_duration=20)\n", " worker_scale_timeout.run_with_timeout(has_worker_scaled_down)" @@ -221,7 +402,7 @@ { "cell_type": "code", "execution_count": null, - "id": "17", + "id": "26", "metadata": {}, "outputs": [], "source": [ @@ -234,7 +415,7 @@ }, { "cell_type": "markdown", - "id": "18", + "id": "27", "metadata": {}, "source": [ "#### Delete Worker Pool" @@ -243,7 +424,7 @@ { "cell_type": "code", "execution_count": null, - "id": "19", + "id": "28", "metadata": {}, "outputs": [], "source": [ @@ -256,7 +437,7 @@ { "cell_type": "code", "execution_count": null, - "id": "20", + "id": "29", "metadata": {}, "outputs": [], "source": [ @@ -266,7 +447,7 @@ }, { "cell_type": "markdown", - "id": "21", + "id": "30", "metadata": {}, "source": [ "#### Re-launch the default worker pool" @@ -275,7 +456,7 @@ { "cell_type": "code", "execution_count": null, - "id": "22", + "id": "31", "metadata": {}, "outputs": [], "source": [ @@ -285,7 +466,7 @@ { "cell_type": "code", "execution_count": null, - "id": "23", + "id": "32", "metadata": {}, "outputs": [], "source": [ @@ -299,7 +480,7 @@ { "cell_type": "code", "execution_count": null, - "id": "24", + "id": "33", "metadata": {}, "outputs": [], "source": [ @@ -313,7 +494,7 @@ { "cell_type": "code", "execution_count": null, - "id": "25", + "id": "34", "metadata": {}, "outputs": [], "source": [ @@ -323,7 +504,7 @@ { "cell_type": "code", "execution_count": null, - "id": "26", + "id": "35", "metadata": {}, "outputs": [], "source": [ @@ -333,7 +514,7 @@ { "cell_type": "code", "execution_count": null, - "id": "27", + "id": "36", "metadata": {}, "outputs": [], "source": [] From 655946359a7097dcbc324b7cd483c417717f0f9a Mon Sep 17 00:00:00 2001 From: Brendan Schell Date: Mon, 9 Sep 2024 16:01:09 -0400 Subject: [PATCH 03/11] format --- packages/syft/src/syft/service/job/job_service.py | 9 ++++++--- .../src/syft/service/worker/worker_pool_service.py | 12 ++++++------ .../syft/src/syft/service/worker/worker_stash.py | 6 +++++- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/packages/syft/src/syft/service/job/job_service.py b/packages/syft/src/syft/service/job/job_service.py index fadf90c6f18..9d8334280fd 100644 --- a/packages/syft/src/syft/service/job/job_service.py +++ b/packages/syft/src/syft/service/job/job_service.py @@ -162,12 +162,15 @@ def _kill(self, context: AuthedServiceContext, job: Job) -> SyftSuccess: results.append(res) # wait for job and subjobs to be killed by MonitorThread - - wait_until(lambda: self.get(context, uid=job.id).status == JobStatus.INTERRUPTED) + + wait_until( + lambda: self.get(context, uid=job.id).status == JobStatus.INTERRUPTED + ) subjob_uids = [subjob.id for subjob in self.get_subjobs(context, uid=job.id)] wait_until( lambda: all( - self.get(context, uid=subjob_id).status == JobStatus.INTERRUPTED for subjob_id in subjob_uids + self.get(context, uid=subjob_id).status == JobStatus.INTERRUPTED + for subjob_id in subjob_uids ) ) diff --git a/packages/syft/src/syft/service/worker/worker_pool_service.py b/packages/syft/src/syft/service/worker/worker_pool_service.py index 0c8c3bf6807..0a62b3756c7 100644 --- a/packages/syft/src/syft/service/worker/worker_pool_service.py +++ b/packages/syft/src/syft/service/worker/worker_pool_service.py @@ -449,7 +449,6 @@ def scale( ) else: # scale down at kubernetes control plane - # scale down removes the last "n" workers # workers to delete = len(workers) - number @@ -458,12 +457,15 @@ def scale( for worker in worker_pool.worker_list ] - # get last "n" workers from pod list runner = KubernetesRunner() - k8s_pods = [pod.name for pod in runner.get_pool_pods(pool_name=worker_pool.name)] + k8s_pods = [ + pod.name for pod in runner.get_pool_pods(pool_name=worker_pool.name) + ] k8s_pods_to_kill = k8s_pods[-(current_worker_count - number) :] - workers_to_delete = [worker for worker in workers if worker.name in k8s_pods_to_kill] + workers_to_delete = [ + worker for worker in workers if worker.name in k8s_pods_to_kill + ] worker_service = cast( WorkerService, context.server.get_service("WorkerService") ) @@ -471,14 +473,12 @@ def scale( for worker in workers_to_delete: worker_service.delete(context=context, uid=worker.id, force=True) - scale_kubernetes_pool( runner, pool_name=worker_pool.name, replicas=number, ).unwrap() - # update worker_pool worker_pool.max_count = number worker_pool.worker_list = worker_pool.worker_list[:number] diff --git a/packages/syft/src/syft/service/worker/worker_stash.py b/packages/syft/src/syft/service/worker/worker_stash.py index 95d719f6dc4..41511457106 100644 --- a/packages/syft/src/syft/service/worker/worker_stash.py +++ b/packages/syft/src/syft/service/worker/worker_stash.py @@ -67,7 +67,11 @@ def get_worker_by_name( @as_result(StashException, NotFoundException) def update_consumer_state( - self, credentials: SyftVerifyKey, worker_uid: UID, consumer_state: ConsumerState, job_id: UID | None = None + self, + credentials: SyftVerifyKey, + worker_uid: UID, + consumer_state: ConsumerState, + job_id: UID | None = None, ) -> SyftWorker: worker = self.get_by_uid(credentials=credentials, uid=worker_uid).unwrap() worker.consumer_state = consumer_state From 138fef4e472ffaab54f608878871f94bc1f84ae7 Mon Sep 17 00:00:00 2001 From: Brendan Schell Date: Mon, 9 Sep 2024 16:06:02 -0400 Subject: [PATCH 04/11] update worker service call --- packages/syft/src/syft/service/worker/worker_pool_service.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/syft/src/syft/service/worker/worker_pool_service.py b/packages/syft/src/syft/service/worker/worker_pool_service.py index 0a62b3756c7..e495e0ecfc4 100644 --- a/packages/syft/src/syft/service/worker/worker_pool_service.py +++ b/packages/syft/src/syft/service/worker/worker_pool_service.py @@ -466,9 +466,7 @@ def scale( workers_to_delete = [ worker for worker in workers if worker.name in k8s_pods_to_kill ] - worker_service = cast( - WorkerService, context.server.get_service("WorkerService") - ) + worker_service = context.server.services.worker for worker in workers_to_delete: worker_service.delete(context=context, uid=worker.id, force=True) From 7782ec9fb6b729593fc3cd0a08d0b75779358d65 Mon Sep 17 00:00:00 2001 From: Brendan Schell Date: Mon, 9 Sep 2024 18:12:21 -0400 Subject: [PATCH 05/11] reduce number of jobs and increase timeout --- .../001-scale-delete-worker-pools.ipynb | 123 ++++++------------ .../syft/src/syft/custom_worker/runner_k8s.py | 2 +- 2 files changed, 38 insertions(+), 87 deletions(-) diff --git a/notebooks/scenarios/bigquery/001-scale-delete-worker-pools.ipynb b/notebooks/scenarios/bigquery/001-scale-delete-worker-pools.ipynb index 2c698230bda..e01fc413b8c 100644 --- a/notebooks/scenarios/bigquery/001-scale-delete-worker-pools.ipynb +++ b/notebooks/scenarios/bigquery/001-scale-delete-worker-pools.ipynb @@ -193,83 +193,27 @@ "outputs": [], "source": [ "@sy.syft_function_single_use(worker_pool_name=default_worker_pool.name)\n", - "def wait_100_seconds_1():\n", + "def wait_1000_seconds_1():\n", " # stdlib\n", " import time\n", "\n", - " time.sleep(100)\n", + " time.sleep(1000)\n", "\n", "\n", "@sy.syft_function_single_use(worker_pool_name=default_worker_pool.name)\n", - "def wait_100_seconds_2():\n", + "def wait_1000_seconds_2():\n", " # stdlib\n", " import time\n", "\n", - " time.sleep(100)\n", + " time.sleep(1000)\n", "\n", "\n", "@sy.syft_function_single_use(worker_pool_name=default_worker_pool.name)\n", - "def wait_100_seconds_3():\n", + "def wait_1000_seconds_3():\n", " # stdlib\n", " import time\n", "\n", - " time.sleep(100)\n", - "\n", - "\n", - "@sy.syft_function_single_use(worker_pool_name=default_worker_pool.name)\n", - "def wait_100_seconds_4():\n", - " # stdlib\n", - " import time\n", - "\n", - " time.sleep(100)\n", - "\n", - "\n", - "@sy.syft_function_single_use(worker_pool_name=default_worker_pool.name)\n", - "def wait_100_seconds_5():\n", - " # stdlib\n", - " import time\n", - "\n", - " time.sleep(100)\n", - "\n", - "\n", - "@sy.syft_function_single_use(worker_pool_name=default_worker_pool.name)\n", - "def wait_100_seconds_6():\n", - " # stdlib\n", - " import time\n", - "\n", - " time.sleep(100)\n", - "\n", - "\n", - "@sy.syft_function_single_use(worker_pool_name=default_worker_pool.name)\n", - "def wait_100_seconds_7():\n", - " # stdlib\n", - " import time\n", - "\n", - " time.sleep(100)\n", - "\n", - "\n", - "@sy.syft_function_single_use(worker_pool_name=default_worker_pool.name)\n", - "def wait_100_seconds_8():\n", - " # stdlib\n", - " import time\n", - "\n", - " time.sleep(100)\n", - "\n", - "\n", - "@sy.syft_function_single_use(worker_pool_name=default_worker_pool.name)\n", - "def wait_100_seconds_9():\n", - " # stdlib\n", - " import time\n", - "\n", - " time.sleep(100)\n", - "\n", - "\n", - "@sy.syft_function_single_use(worker_pool_name=default_worker_pool.name)\n", - "def wait_100_seconds_10():\n", - " # stdlib\n", - " import time\n", - "\n", - " time.sleep(100)" + " time.sleep(1000)" ] }, { @@ -280,18 +224,11 @@ "outputs": [], "source": [ "jobs = []\n", - "high_client.code.request_code_execution(wait_100_seconds_1)\n", - "high_client.code.request_code_execution(wait_100_seconds_2)\n", - "high_client.code.request_code_execution(wait_100_seconds_3)\n", - "high_client.code.request_code_execution(wait_100_seconds_4)\n", - "high_client.code.request_code_execution(wait_100_seconds_5)\n", - "high_client.code.request_code_execution(wait_100_seconds_6)\n", - "high_client.code.request_code_execution(wait_100_seconds_7)\n", - "high_client.code.request_code_execution(wait_100_seconds_8)\n", - "high_client.code.request_code_execution(wait_100_seconds_9)\n", - "high_client.code.request_code_execution(wait_100_seconds_10)\n", + "high_client.code.request_code_execution(wait_1000_seconds_1)\n", + "high_client.code.request_code_execution(wait_1000_seconds_2)\n", + "high_client.code.request_code_execution(wait_1000_seconds_3)\n", "\n", - "assert len(list(high_client.requests)) == 10\n", + "assert len(list(high_client.requests)) == 3 \n", "for request in high_client.requests:\n", " request.approve()" ] @@ -304,19 +241,12 @@ "outputs": [], "source": [ "jobs = []\n", - "jobs.append(high_client.code.wait_100_seconds_1(blocking=False))\n", - "jobs.append(high_client.code.wait_100_seconds_2(blocking=False))\n", - "jobs.append(high_client.code.wait_100_seconds_3(blocking=False))\n", - "jobs.append(high_client.code.wait_100_seconds_4(blocking=False))\n", - "jobs.append(high_client.code.wait_100_seconds_5(blocking=False))\n", - "jobs.append(high_client.code.wait_100_seconds_6(blocking=False))\n", - "jobs.append(high_client.code.wait_100_seconds_7(blocking=False))\n", - "jobs.append(high_client.code.wait_100_seconds_8(blocking=False))\n", - "jobs.append(high_client.code.wait_100_seconds_9(blocking=False))\n", - "jobs.append(high_client.code.wait_100_seconds_10(blocking=False))\n", + "jobs.append(high_client.code.wait_1000_seconds_1(blocking=False))\n", + "jobs.append(high_client.code.wait_1000_seconds_2(blocking=False))\n", + "jobs.append(high_client.code.wait_1000_seconds_3(blocking=False))\n", "\n", "\n", - "assert len(list(high_client.jobs)) == 10" + "assert len(list(high_client.jobs)) == 3" ] }, { @@ -395,7 +325,7 @@ " jobs_on_old_workers_are_interrupted = all(job_status_condition)\n", " return worker_count_condition and jobs_on_old_workers_are_interrupted\n", "\n", - " worker_scale_timeout = Timeout(timeout_duration=20)\n", + " worker_scale_timeout = Timeout(timeout_duration=60)\n", " worker_scale_timeout.run_with_timeout(has_worker_scaled_down)" ] }, @@ -445,6 +375,22 @@ " _ = high_client.api.services.worker_pool[default_worker_pool.name]" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "52864347", + "metadata": {}, + "outputs": [], + "source": [ + "# check that all jobs are interrupted - should be the case since the entire pool was deleted and all jobs were previously assigned\n", + "assert all(\n", + " [\n", + " job.status == \"interrupted\"\n", + " for job in high_client.jobs\n", + " ]\n", + ")" + ] + }, { "cell_type": "markdown", "id": "30", @@ -521,6 +467,11 @@ } ], "metadata": { + "kernelspec": { + "display_name": "syft", + "language": "python", + "name": "python3" + }, "language_info": { "codemirror_mode": { "name": "ipython", @@ -531,7 +482,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.5" + "version": "3.12.4" } }, "nbformat": 4, diff --git a/packages/syft/src/syft/custom_worker/runner_k8s.py b/packages/syft/src/syft/custom_worker/runner_k8s.py index e320bae8e94..07e88d860d5 100644 --- a/packages/syft/src/syft/custom_worker/runner_k8s.py +++ b/packages/syft/src/syft/custom_worker/runner_k8s.py @@ -14,7 +14,7 @@ JSONPATH_AVAILABLE_REPLICAS = "{.status.availableReplicas}" CREATE_POOL_TIMEOUT_SEC = 380 -SCALE_POOL_TIMEOUT_SEC = 60 +SCALE_POOL_TIMEOUT_SEC = 120 class KubernetesRunner: From ec750d68b395b86d63ce9070d690bbc5aab05bf8 Mon Sep 17 00:00:00 2001 From: Brendan Schell Date: Mon, 9 Sep 2024 18:14:51 -0400 Subject: [PATCH 06/11] format --- .../001-scale-delete-worker-pools.ipynb | 25 ++++++++----------- .../syft/src/syft/custom_worker/runner_k8s.py | 2 +- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/notebooks/scenarios/bigquery/001-scale-delete-worker-pools.ipynb b/notebooks/scenarios/bigquery/001-scale-delete-worker-pools.ipynb index e01fc413b8c..aa99efdb3e2 100644 --- a/notebooks/scenarios/bigquery/001-scale-delete-worker-pools.ipynb +++ b/notebooks/scenarios/bigquery/001-scale-delete-worker-pools.ipynb @@ -228,7 +228,7 @@ "high_client.code.request_code_execution(wait_1000_seconds_2)\n", "high_client.code.request_code_execution(wait_1000_seconds_3)\n", "\n", - "assert len(list(high_client.requests)) == 3 \n", + "assert len(list(high_client.requests)) == 3\n", "for request in high_client.requests:\n", " request.approve()" ] @@ -378,22 +378,17 @@ { "cell_type": "code", "execution_count": null, - "id": "52864347", + "id": "30", "metadata": {}, "outputs": [], "source": [ "# check that all jobs are interrupted - should be the case since the entire pool was deleted and all jobs were previously assigned\n", - "assert all(\n", - " [\n", - " job.status == \"interrupted\"\n", - " for job in high_client.jobs\n", - " ]\n", - ")" + "assert all([job.status == \"interrupted\" for job in high_client.jobs])" ] }, { "cell_type": "markdown", - "id": "30", + "id": "31", "metadata": {}, "source": [ "#### Re-launch the default worker pool" @@ -402,7 +397,7 @@ { "cell_type": "code", "execution_count": null, - "id": "31", + "id": "32", "metadata": {}, "outputs": [], "source": [ @@ -412,7 +407,7 @@ { "cell_type": "code", "execution_count": null, - "id": "32", + "id": "33", "metadata": {}, "outputs": [], "source": [ @@ -426,7 +421,7 @@ { "cell_type": "code", "execution_count": null, - "id": "33", + "id": "34", "metadata": {}, "outputs": [], "source": [ @@ -440,7 +435,7 @@ { "cell_type": "code", "execution_count": null, - "id": "34", + "id": "35", "metadata": {}, "outputs": [], "source": [ @@ -450,7 +445,7 @@ { "cell_type": "code", "execution_count": null, - "id": "35", + "id": "36", "metadata": {}, "outputs": [], "source": [ @@ -460,7 +455,7 @@ { "cell_type": "code", "execution_count": null, - "id": "36", + "id": "37", "metadata": {}, "outputs": [], "source": [] diff --git a/packages/syft/src/syft/custom_worker/runner_k8s.py b/packages/syft/src/syft/custom_worker/runner_k8s.py index 07e88d860d5..f57f6071a88 100644 --- a/packages/syft/src/syft/custom_worker/runner_k8s.py +++ b/packages/syft/src/syft/custom_worker/runner_k8s.py @@ -14,7 +14,7 @@ JSONPATH_AVAILABLE_REPLICAS = "{.status.availableReplicas}" CREATE_POOL_TIMEOUT_SEC = 380 -SCALE_POOL_TIMEOUT_SEC = 120 +SCALE_POOL_TIMEOUT_SEC = 120 class KubernetesRunner: From 292c3562e75b7d75fd2b4d255bb9485a955d80e1 Mon Sep 17 00:00:00 2001 From: Brendan Schell Date: Mon, 9 Sep 2024 18:39:15 -0400 Subject: [PATCH 07/11] updating notebook --- .../scenarios/bigquery/001-scale-delete-worker-pools.ipynb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/notebooks/scenarios/bigquery/001-scale-delete-worker-pools.ipynb b/notebooks/scenarios/bigquery/001-scale-delete-worker-pools.ipynb index aa99efdb3e2..48212d29e11 100644 --- a/notebooks/scenarios/bigquery/001-scale-delete-worker-pools.ipynb +++ b/notebooks/scenarios/bigquery/001-scale-delete-worker-pools.ipynb @@ -382,8 +382,9 @@ "metadata": {}, "outputs": [], "source": [ - "# check that all jobs are interrupted - should be the case since the entire pool was deleted and all jobs were previously assigned\n", - "assert all([job.status == \"interrupted\" for job in high_client.jobs])" + "# check that all jobs are interrupted\n", + "# should be the case since the entire pool was deleted and all jobs were previously assigned\n", + "assert all(job.status == \"interrupted\" for job in high_client.jobs)" ] }, { From ef629da6012690d8f921fca4728f09449a0ea60b Mon Sep 17 00:00:00 2001 From: Brendan Schell Date: Mon, 9 Sep 2024 18:44:32 -0400 Subject: [PATCH 08/11] try increased sleep for CI workers to take on tasks --- .../scenarios/bigquery/001-scale-delete-worker-pools.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/notebooks/scenarios/bigquery/001-scale-delete-worker-pools.ipynb b/notebooks/scenarios/bigquery/001-scale-delete-worker-pools.ipynb index 48212d29e11..c6ae12912fd 100644 --- a/notebooks/scenarios/bigquery/001-scale-delete-worker-pools.ipynb +++ b/notebooks/scenarios/bigquery/001-scale-delete-worker-pools.ipynb @@ -257,7 +257,7 @@ "outputs": [], "source": [ "# check that at least three workers have a job (since scaling down to 2)\n", - "# try 3 times with a 1 second sleep in case it takes time for the workers to accept the jobs\n", + "# try 3 times with a 5 second sleep in case it takes time for the workers to accept the jobs\n", "for _ in range(3):\n", " worker_to_job_map = {}\n", " syft_workers_ids = set()\n", @@ -266,7 +266,7 @@ " syft_workers_ids.add(job.worker.id)\n", " worker_to_job_map[job.worker.id] = job.id\n", " if len(syft_workers_ids) < 3:\n", - " time.sleep(1)\n", + " time.sleep(5)\n", " else:\n", " break\n", "assert len(syft_workers_ids) >= 3" From d7da09ed2e8ac242ddedbf5de4eb380cc06d79a0 Mon Sep 17 00:00:00 2001 From: Brendan Schell Date: Mon, 9 Sep 2024 18:54:59 -0400 Subject: [PATCH 09/11] try increase timeout again --- .../scenarios/bigquery/001-scale-delete-worker-pools.ipynb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/notebooks/scenarios/bigquery/001-scale-delete-worker-pools.ipynb b/notebooks/scenarios/bigquery/001-scale-delete-worker-pools.ipynb index c6ae12912fd..884b77501ab 100644 --- a/notebooks/scenarios/bigquery/001-scale-delete-worker-pools.ipynb +++ b/notebooks/scenarios/bigquery/001-scale-delete-worker-pools.ipynb @@ -265,8 +265,9 @@ " if job.status == \"processing\":\n", " syft_workers_ids.add(job.worker.id)\n", " worker_to_job_map[job.worker.id] = job.id\n", + " print(worker_to_job_map)\n", " if len(syft_workers_ids) < 3:\n", - " time.sleep(5)\n", + " time.sleep(20)\n", " else:\n", " break\n", "assert len(syft_workers_ids) >= 3" From 32dc57ff1fa5fb8e56c4d0f106e2b31be4b01d3d Mon Sep 17 00:00:00 2001 From: Brendan Schell Date: Thu, 12 Sep 2024 10:41:25 -0400 Subject: [PATCH 10/11] push latest state for future debugging (deprioritized) --- ...tart-and-configure-server-and-admins.ipynb | 149 +- .../001-scale-delete-worker-pools.ipynb | 15722 +++++++++- .../bigquery/010-setup-bigquery-pool.ipynb | 25354 +++++++++++++++- .../bigquery/011-users-emails-passwords.ipynb | 7400 ++++- .../bigquery/020-configure-api.ipynb | 5360 +++- .../scenarios/bigquery/021-create-jobs.ipynb | 28 +- .../bigquery/040-do-review-requests.ipynb | 27 +- .../bigquery/050-ds-get-results.ipynb | 27 +- packages/syft/src/syft/service/queue/queue.py | 5 +- .../src/syft/service/queue/zmq_producer.py | 9 +- .../service/worker/worker_pool_service.py | 16 +- .../src/syft/service/worker/worker_service.py | 33 +- .../src/syft/service/worker/worker_stash.py | 3 + tox.ini | 4 +- 14 files changed, 53868 insertions(+), 269 deletions(-) diff --git a/notebooks/scenarios/bigquery/000-start-and-configure-server-and-admins.ipynb b/notebooks/scenarios/bigquery/000-start-and-configure-server-and-admins.ipynb index 444ba2fe199..066ba974ee2 100644 --- a/notebooks/scenarios/bigquery/000-start-and-configure-server-and-admins.ipynb +++ b/notebooks/scenarios/bigquery/000-start-and-configure-server-and-admins.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -16,7 +16,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "os.environ['ORCHESTRA_DEPLOYMENT_TYPE'] = 'remote'\n", + "os.environ['DEVSPACE_PROFILE'] = 'bigquery-scenario-tests'\n", + "os.environ['GITHUB_CI'] = 'false'\n", + "os.environ['SYFT_BASE_IMAGE_REGISTRY'] = 'k3d-registry.localhost:5800'\n", + "os.environ['DATASITE_CLUSTER_NAME'] = 'test-datasite-1'\n", + "os.environ['SERVER_URL'] = 'http://localhost'\n", + "os.environ['SERVER_PORT'] = '9081'\n", + "os.environ['TEST_EXTERNAL_REGISTRY'] = 'k3d-registry.localhost:5800'\n", + "os.environ['TEST_QUERY_LIMIT_SIZE'] = '500000'\n", + "os.environ['TRACING'] = 'False'\n", + "os.environ['NUM_TEST_USERS'] = '5'\n", + "os.environ['NUM_TEST_JOBS'] = '10'\n", + "os.environ['NUM_TEST_WORKERS'] = '2'" + ] + }, + { + "cell_type": "code", + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -40,7 +63,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -51,7 +74,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -62,7 +85,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -71,7 +94,7 @@ " dev_mode=True,\n", " server_side_type=\"high\",\n", " reset=True,\n", - " port=\"8080\",\n", + " port=\"9081\",\n", " n_consumers=1, # How many workers to be spawned\n", " create_producer=True, # Can produce more workers\n", ")" @@ -79,21 +102,88 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> Starting SMTPTestServer on: 0.0.0.0:9025\n" + ] + } + ], "source": [ "email_server, smtp_server = get_email_server(reset=True)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:urllib3.connectionpool:Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused')': /api/v2/metadata\n", + "WARNING:urllib3.connectionpool:Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused')': /api/v2/metadata\n", + "WARNING:urllib3.connectionpool:Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused')': /api/v2/metadata\n" + ] + }, + { + "ename": "ConnectionError", + "evalue": "HTTPConnectionPool(host='localhost', port=8080): Max retries exceeded with url: /api/v2/metadata (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused'))", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mConnectionRefusedError\u001b[0m Traceback (most recent call last)", + "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/site-packages/urllib3/connection.py:196\u001b[0m, in \u001b[0;36mHTTPConnection._new_conn\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 195\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 196\u001b[0m sock \u001b[38;5;241m=\u001b[39m \u001b[43mconnection\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcreate_connection\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 197\u001b[0m \u001b[43m \u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_dns_host\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mport\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 198\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 199\u001b[0m \u001b[43m \u001b[49m\u001b[43msource_address\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msource_address\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 200\u001b[0m \u001b[43m \u001b[49m\u001b[43msocket_options\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msocket_options\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 201\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 202\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m socket\u001b[38;5;241m.\u001b[39mgaierror \u001b[38;5;28;01mas\u001b[39;00m e:\n", + "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/site-packages/urllib3/util/connection.py:85\u001b[0m, in \u001b[0;36mcreate_connection\u001b[0;34m(address, timeout, source_address, socket_options)\u001b[0m\n\u001b[1;32m 84\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 85\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m err\n\u001b[1;32m 86\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m 87\u001b[0m \u001b[38;5;66;03m# Break explicitly a reference cycle\u001b[39;00m\n", + "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/site-packages/urllib3/util/connection.py:73\u001b[0m, in \u001b[0;36mcreate_connection\u001b[0;34m(address, timeout, source_address, socket_options)\u001b[0m\n\u001b[1;32m 72\u001b[0m sock\u001b[38;5;241m.\u001b[39mbind(source_address)\n\u001b[0;32m---> 73\u001b[0m \u001b[43msock\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43msa\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 74\u001b[0m \u001b[38;5;66;03m# Break explicitly a reference cycle\u001b[39;00m\n", + "\u001b[0;31mConnectionRefusedError\u001b[0m: [Errno 61] Connection refused", + "\nThe above exception was the direct cause of the following exception:\n", + "\u001b[0;31mNewConnectionError\u001b[0m Traceback (most recent call last)", + "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/site-packages/urllib3/connectionpool.py:789\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[0m\n\u001b[1;32m 788\u001b[0m \u001b[38;5;66;03m# Make the request on the HTTPConnection object\u001b[39;00m\n\u001b[0;32m--> 789\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_make_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 790\u001b[0m \u001b[43m \u001b[49m\u001b[43mconn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 791\u001b[0m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 792\u001b[0m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 793\u001b[0m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 794\u001b[0m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 795\u001b[0m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 796\u001b[0m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 797\u001b[0m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mretries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 798\u001b[0m \u001b[43m \u001b[49m\u001b[43mresponse_conn\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mresponse_conn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 799\u001b[0m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 800\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 801\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mresponse_kw\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 802\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 804\u001b[0m \u001b[38;5;66;03m# Everything went great!\u001b[39;00m\n", + "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/site-packages/urllib3/connectionpool.py:495\u001b[0m, in \u001b[0;36mHTTPConnectionPool._make_request\u001b[0;34m(self, conn, method, url, body, headers, retries, timeout, chunked, response_conn, preload_content, decode_content, enforce_content_length)\u001b[0m\n\u001b[1;32m 494\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 495\u001b[0m \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 496\u001b[0m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 497\u001b[0m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 498\u001b[0m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 499\u001b[0m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 500\u001b[0m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 501\u001b[0m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 502\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 503\u001b[0m \u001b[43m \u001b[49m\u001b[43menforce_content_length\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43menforce_content_length\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 504\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 506\u001b[0m \u001b[38;5;66;03m# We are swallowing BrokenPipeError (errno.EPIPE) since the server is\u001b[39;00m\n\u001b[1;32m 507\u001b[0m \u001b[38;5;66;03m# legitimately able to close the connection after sending a valid response.\u001b[39;00m\n\u001b[1;32m 508\u001b[0m \u001b[38;5;66;03m# With this behaviour, the received response is still readable.\u001b[39;00m\n", + "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/site-packages/urllib3/connection.py:398\u001b[0m, in \u001b[0;36mHTTPConnection.request\u001b[0;34m(self, method, url, body, headers, chunked, preload_content, decode_content, enforce_content_length)\u001b[0m\n\u001b[1;32m 397\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mputheader(header, value)\n\u001b[0;32m--> 398\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mendheaders\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 400\u001b[0m \u001b[38;5;66;03m# If we're given a body we start sending that in chunks.\u001b[39;00m\n", + "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/http/client.py:1331\u001b[0m, in \u001b[0;36mHTTPConnection.endheaders\u001b[0;34m(self, message_body, encode_chunked)\u001b[0m\n\u001b[1;32m 1330\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m CannotSendHeader()\n\u001b[0;32m-> 1331\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_send_output\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmessage_body\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencode_chunked\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/http/client.py:1091\u001b[0m, in \u001b[0;36mHTTPConnection._send_output\u001b[0;34m(self, message_body, encode_chunked)\u001b[0m\n\u001b[1;32m 1090\u001b[0m \u001b[38;5;28;01mdel\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_buffer[:]\n\u001b[0;32m-> 1091\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmsg\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1093\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m message_body \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 1094\u001b[0m \n\u001b[1;32m 1095\u001b[0m \u001b[38;5;66;03m# create a consistent interface to message_body\u001b[39;00m\n", + "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/http/client.py:1035\u001b[0m, in \u001b[0;36mHTTPConnection.send\u001b[0;34m(self, data)\u001b[0m\n\u001b[1;32m 1034\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mauto_open:\n\u001b[0;32m-> 1035\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1036\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", + "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/site-packages/urllib3/connection.py:236\u001b[0m, in \u001b[0;36mHTTPConnection.connect\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 235\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mconnect\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 236\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msock \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_new_conn\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 237\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_tunnel_host:\n\u001b[1;32m 238\u001b[0m \u001b[38;5;66;03m# If we're tunneling it means we're connected to our proxy.\u001b[39;00m\n", + "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/site-packages/urllib3/connection.py:211\u001b[0m, in \u001b[0;36mHTTPConnection._new_conn\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 210\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m--> 211\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m NewConnectionError(\n\u001b[1;32m 212\u001b[0m \u001b[38;5;28mself\u001b[39m, \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mFailed to establish a new connection: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00me\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 213\u001b[0m ) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01me\u001b[39;00m\n\u001b[1;32m 215\u001b[0m \u001b[38;5;66;03m# Audit hooks are only available in Python 3.8+\u001b[39;00m\n", + "\u001b[0;31mNewConnectionError\u001b[0m: : Failed to establish a new connection: [Errno 61] Connection refused", + "\nThe above exception was the direct cause of the following exception:\n", + "\u001b[0;31mMaxRetryError\u001b[0m Traceback (most recent call last)", + "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/site-packages/requests/adapters.py:667\u001b[0m, in \u001b[0;36mHTTPAdapter.send\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m 666\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 667\u001b[0m resp \u001b[38;5;241m=\u001b[39m \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 668\u001b[0m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 669\u001b[0m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 670\u001b[0m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 671\u001b[0m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 672\u001b[0m \u001b[43m \u001b[49m\u001b[43mredirect\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 673\u001b[0m \u001b[43m \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 674\u001b[0m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 675\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 676\u001b[0m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmax_retries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 677\u001b[0m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 678\u001b[0m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 679\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 681\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (ProtocolError, \u001b[38;5;167;01mOSError\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m err:\n", + "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/site-packages/urllib3/connectionpool.py:873\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[0m\n\u001b[1;32m 870\u001b[0m log\u001b[38;5;241m.\u001b[39mwarning(\n\u001b[1;32m 871\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mRetrying (\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[38;5;124m) after connection broken by \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m, retries, err, url\n\u001b[1;32m 872\u001b[0m )\n\u001b[0;32m--> 873\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 874\u001b[0m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 875\u001b[0m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 876\u001b[0m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 877\u001b[0m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 878\u001b[0m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 879\u001b[0m \u001b[43m \u001b[49m\u001b[43mredirect\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 880\u001b[0m \u001b[43m \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 881\u001b[0m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 882\u001b[0m \u001b[43m \u001b[49m\u001b[43mpool_timeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpool_timeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 883\u001b[0m \u001b[43m \u001b[49m\u001b[43mrelease_conn\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrelease_conn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 884\u001b[0m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 885\u001b[0m \u001b[43m \u001b[49m\u001b[43mbody_pos\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbody_pos\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 886\u001b[0m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 887\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 888\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mresponse_kw\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 889\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 891\u001b[0m \u001b[38;5;66;03m# Handle redirect?\u001b[39;00m\n", + "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/site-packages/urllib3/connectionpool.py:873\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[0m\n\u001b[1;32m 870\u001b[0m log\u001b[38;5;241m.\u001b[39mwarning(\n\u001b[1;32m 871\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mRetrying (\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[38;5;124m) after connection broken by \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m, retries, err, url\n\u001b[1;32m 872\u001b[0m )\n\u001b[0;32m--> 873\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 874\u001b[0m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 875\u001b[0m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 876\u001b[0m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 877\u001b[0m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 878\u001b[0m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 879\u001b[0m \u001b[43m \u001b[49m\u001b[43mredirect\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 880\u001b[0m \u001b[43m \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 881\u001b[0m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 882\u001b[0m \u001b[43m \u001b[49m\u001b[43mpool_timeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpool_timeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 883\u001b[0m \u001b[43m \u001b[49m\u001b[43mrelease_conn\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrelease_conn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 884\u001b[0m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 885\u001b[0m \u001b[43m \u001b[49m\u001b[43mbody_pos\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbody_pos\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 886\u001b[0m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 887\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 888\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mresponse_kw\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 889\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 891\u001b[0m \u001b[38;5;66;03m# Handle redirect?\u001b[39;00m\n", + "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/site-packages/urllib3/connectionpool.py:873\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[0m\n\u001b[1;32m 870\u001b[0m log\u001b[38;5;241m.\u001b[39mwarning(\n\u001b[1;32m 871\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mRetrying (\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[38;5;124m) after connection broken by \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m, retries, err, url\n\u001b[1;32m 872\u001b[0m )\n\u001b[0;32m--> 873\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 874\u001b[0m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 875\u001b[0m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 876\u001b[0m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 877\u001b[0m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 878\u001b[0m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 879\u001b[0m \u001b[43m \u001b[49m\u001b[43mredirect\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 880\u001b[0m \u001b[43m \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 881\u001b[0m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 882\u001b[0m \u001b[43m \u001b[49m\u001b[43mpool_timeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpool_timeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 883\u001b[0m \u001b[43m \u001b[49m\u001b[43mrelease_conn\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrelease_conn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 884\u001b[0m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 885\u001b[0m \u001b[43m \u001b[49m\u001b[43mbody_pos\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbody_pos\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 886\u001b[0m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 887\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 888\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mresponse_kw\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 889\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 891\u001b[0m \u001b[38;5;66;03m# Handle redirect?\u001b[39;00m\n", + "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/site-packages/urllib3/connectionpool.py:843\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[0m\n\u001b[1;32m 841\u001b[0m new_e \u001b[38;5;241m=\u001b[39m ProtocolError(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mConnection aborted.\u001b[39m\u001b[38;5;124m\"\u001b[39m, new_e)\n\u001b[0;32m--> 843\u001b[0m retries \u001b[38;5;241m=\u001b[39m \u001b[43mretries\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mincrement\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 844\u001b[0m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43merror\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mnew_e\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_pool\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_stacktrace\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msys\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexc_info\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m2\u001b[39;49m\u001b[43m]\u001b[49m\n\u001b[1;32m 845\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 846\u001b[0m retries\u001b[38;5;241m.\u001b[39msleep()\n", + "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/site-packages/urllib3/util/retry.py:519\u001b[0m, in \u001b[0;36mRetry.increment\u001b[0;34m(self, method, url, response, error, _pool, _stacktrace)\u001b[0m\n\u001b[1;32m 518\u001b[0m reason \u001b[38;5;241m=\u001b[39m error \u001b[38;5;129;01mor\u001b[39;00m ResponseError(cause)\n\u001b[0;32m--> 519\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m MaxRetryError(_pool, url, reason) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mreason\u001b[39;00m \u001b[38;5;66;03m# type: ignore[arg-type]\u001b[39;00m\n\u001b[1;32m 521\u001b[0m log\u001b[38;5;241m.\u001b[39mdebug(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mIncremented Retry for (url=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m): \u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[38;5;124m\"\u001b[39m, url, new_retry)\n", + "\u001b[0;31mMaxRetryError\u001b[0m: HTTPConnectionPool(host='localhost', port=8080): Max retries exceeded with url: /api/v2/metadata (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused'))", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001b[0;31mConnectionError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[8], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m root_client \u001b[38;5;241m=\u001b[39m \u001b[43msy\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlogin\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2\u001b[0m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mhttp://localhost:8080\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43memail\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mROOT_EMAIL\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpassword\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mROOT_PASSWORD\u001b[49m\n\u001b[1;32m 3\u001b[0m \u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/packages/syft/src/syft/client/client.py:1188\u001b[0m, in \u001b[0;36mlogin\u001b[0;34m(email, url, port, server, password, cache)\u001b[0m\n\u001b[1;32m 1178\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mlogin\u001b[39m(\n\u001b[1;32m 1179\u001b[0m email: \u001b[38;5;28mstr\u001b[39m,\n\u001b[1;32m 1180\u001b[0m \u001b[38;5;66;03m# HTTPConnection\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1186\u001b[0m cache: \u001b[38;5;28mbool\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 1187\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m SyftClient:\n\u001b[0;32m-> 1188\u001b[0m _client \u001b[38;5;241m=\u001b[39m \u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1189\u001b[0m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1190\u001b[0m \u001b[43m \u001b[49m\u001b[43mserver\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mserver\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1191\u001b[0m \u001b[43m \u001b[49m\u001b[43mport\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mport\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1192\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1194\u001b[0m connection \u001b[38;5;241m=\u001b[39m _client\u001b[38;5;241m.\u001b[39mconnection\n\u001b[1;32m 1196\u001b[0m login_credentials \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", + "File \u001b[0;32m~/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/packages/syft/src/syft/client/client.py:1132\u001b[0m, in \u001b[0;36mconnect\u001b[0;34m(url, server, port)\u001b[0m\n\u001b[1;32m 1129\u001b[0m url\u001b[38;5;241m.\u001b[39mset_port(\u001b[38;5;28mint\u001b[39m(port))\n\u001b[1;32m 1130\u001b[0m connection \u001b[38;5;241m=\u001b[39m HTTPConnection(url\u001b[38;5;241m=\u001b[39murl)\n\u001b[0;32m-> 1132\u001b[0m client_type \u001b[38;5;241m=\u001b[39m \u001b[43mconnection\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_client_type\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39munwrap()\n\u001b[1;32m 1133\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m client_type(connection\u001b[38;5;241m=\u001b[39mconnection)\n", + "File \u001b[0;32m~/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/packages/syft/src/syft/types/result.py:113\u001b[0m, in \u001b[0;36mas_result..decorator..wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 109\u001b[0m \u001b[38;5;129m@exclude_from_traceback\u001b[39m\n\u001b[1;32m 110\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(func)\n\u001b[1;32m 111\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwrapper\u001b[39m(\u001b[38;5;241m*\u001b[39margs: P\u001b[38;5;241m.\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs: P\u001b[38;5;241m.\u001b[39mkwargs) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Result[T, BE]:\n\u001b[1;32m 112\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 113\u001b[0m output \u001b[38;5;241m=\u001b[39m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 114\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(output, Ok) \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(output, Err):\n\u001b[1;32m 115\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m _AsResultError(\n\u001b[1;32m 116\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mFunctions decorated with `as_result` should not return Result.\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 117\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mDid you forget to unwrap() the result?\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 118\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mresult: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00moutput\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 119\u001b[0m )\n", + "File \u001b[0;32m~/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/packages/syft/src/syft/client/client.py:496\u001b[0m, in \u001b[0;36mHTTPConnection.get_client_type\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 493\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01menclave_client\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m EnclaveClient\n\u001b[1;32m 494\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mgateway_client\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m GatewayClient\n\u001b[0;32m--> 496\u001b[0m metadata \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_server_metadata\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcredentials\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mSyftSigningKey\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgenerate\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 497\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m metadata\u001b[38;5;241m.\u001b[39mserver_type \u001b[38;5;241m==\u001b[39m ServerType\u001b[38;5;241m.\u001b[39mDATASITE\u001b[38;5;241m.\u001b[39mvalue:\n\u001b[1;32m 498\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m DatasiteClient\n", + "File \u001b[0;32m~/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/packages/syft/src/syft/client/client.py:342\u001b[0m, in \u001b[0;36mHTTPConnection.get_server_metadata\u001b[0;34m(self, credentials)\u001b[0m\n\u001b[1;32m 340\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m response\n\u001b[1;32m 341\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 342\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_make_get\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mroutes\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mROUTE_METADATA\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvalue\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 343\u001b[0m metadata_json \u001b[38;5;241m=\u001b[39m json\u001b[38;5;241m.\u001b[39mloads(response)\n\u001b[1;32m 344\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ServerMetadataJSON(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mmetadata_json)\n", + "File \u001b[0;32m~/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/packages/syft/src/syft/client/client.py:204\u001b[0m, in \u001b[0;36mHTTPConnection._make_get\u001b[0;34m(self, path, params, stream)\u001b[0m\n\u001b[1;32m 200\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_make_get\u001b[39m(\n\u001b[1;32m 201\u001b[0m \u001b[38;5;28mself\u001b[39m, path: \u001b[38;5;28mstr\u001b[39m, params: \u001b[38;5;28mdict\u001b[39m \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m, stream: \u001b[38;5;28mbool\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m 202\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mbytes\u001b[39m \u001b[38;5;241m|\u001b[39m Iterable:\n\u001b[1;32m 203\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m params \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 204\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_make_get_no_params\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 206\u001b[0m url \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39murl\n\u001b[1;32m 208\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrtunnel_token:\n", + "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/site-packages/cachetools/__init__.py:737\u001b[0m, in \u001b[0;36mcached..decorator..wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 735\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m:\n\u001b[1;32m 736\u001b[0m \u001b[38;5;28;01mpass\u001b[39;00m \u001b[38;5;66;03m# key not found\u001b[39;00m\n\u001b[0;32m--> 737\u001b[0m v \u001b[38;5;241m=\u001b[39m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 738\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 739\u001b[0m cache[k] \u001b[38;5;241m=\u001b[39m v\n", + "File \u001b[0;32m~/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/packages/syft/src/syft/client/client.py:244\u001b[0m, in \u001b[0;36mHTTPConnection._make_get_no_params\u001b[0;34m(self, path, stream)\u001b[0m\n\u001b[1;32m 240\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mheaders[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mHost\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39murl\u001b[38;5;241m.\u001b[39mhost_or_ip\n\u001b[1;32m 242\u001b[0m url \u001b[38;5;241m=\u001b[39m url\u001b[38;5;241m.\u001b[39mwith_path(path)\n\u001b[0;32m--> 244\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msession\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 245\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mstr\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 246\u001b[0m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 247\u001b[0m \u001b[43m \u001b[49m\u001b[43mverify\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mverify_tls\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 248\u001b[0m \u001b[43m \u001b[49m\u001b[43mproxies\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m{\u001b[49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 249\u001b[0m \u001b[43m \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 250\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 251\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m response\u001b[38;5;241m.\u001b[39mstatus_code \u001b[38;5;241m!=\u001b[39m \u001b[38;5;241m200\u001b[39m:\n\u001b[1;32m 252\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m requests\u001b[38;5;241m.\u001b[39mConnectionError(\n\u001b[1;32m 253\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mFailed to fetch \u001b[39m\u001b[38;5;132;01m{\u001b[39;00murl\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m. Response returned with code \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mresponse\u001b[38;5;241m.\u001b[39mstatus_code\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 254\u001b[0m )\n", + "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/site-packages/requests/sessions.py:602\u001b[0m, in \u001b[0;36mSession.get\u001b[0;34m(self, url, **kwargs)\u001b[0m\n\u001b[1;32m 594\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124;03m\"\"\"Sends a GET request. Returns :class:`Response` object.\u001b[39;00m\n\u001b[1;32m 595\u001b[0m \n\u001b[1;32m 596\u001b[0m \u001b[38;5;124;03m:param url: URL for the new :class:`Request` object.\u001b[39;00m\n\u001b[1;32m 597\u001b[0m \u001b[38;5;124;03m:param \\*\\*kwargs: Optional arguments that ``request`` takes.\u001b[39;00m\n\u001b[1;32m 598\u001b[0m \u001b[38;5;124;03m:rtype: requests.Response\u001b[39;00m\n\u001b[1;32m 599\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 601\u001b[0m kwargs\u001b[38;5;241m.\u001b[39msetdefault(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mallow_redirects\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[0;32m--> 602\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mGET\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/site-packages/requests/sessions.py:589\u001b[0m, in \u001b[0;36mSession.request\u001b[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[1;32m 584\u001b[0m send_kwargs \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m 585\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtimeout\u001b[39m\u001b[38;5;124m\"\u001b[39m: timeout,\n\u001b[1;32m 586\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mallow_redirects\u001b[39m\u001b[38;5;124m\"\u001b[39m: allow_redirects,\n\u001b[1;32m 587\u001b[0m }\n\u001b[1;32m 588\u001b[0m send_kwargs\u001b[38;5;241m.\u001b[39mupdate(settings)\n\u001b[0;32m--> 589\u001b[0m resp \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprep\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43msend_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 591\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m resp\n", + "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/site-packages/requests/sessions.py:703\u001b[0m, in \u001b[0;36mSession.send\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m 700\u001b[0m start \u001b[38;5;241m=\u001b[39m preferred_clock()\n\u001b[1;32m 702\u001b[0m \u001b[38;5;66;03m# Send the request\u001b[39;00m\n\u001b[0;32m--> 703\u001b[0m r \u001b[38;5;241m=\u001b[39m \u001b[43madapter\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 705\u001b[0m \u001b[38;5;66;03m# Total elapsed time of the request (approximately)\u001b[39;00m\n\u001b[1;32m 706\u001b[0m elapsed \u001b[38;5;241m=\u001b[39m preferred_clock() \u001b[38;5;241m-\u001b[39m start\n", + "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/site-packages/requests/adapters.py:700\u001b[0m, in \u001b[0;36mHTTPAdapter.send\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m 696\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(e\u001b[38;5;241m.\u001b[39mreason, _SSLError):\n\u001b[1;32m 697\u001b[0m \u001b[38;5;66;03m# This branch is for urllib3 v1.22 and later.\u001b[39;00m\n\u001b[1;32m 698\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m SSLError(e, request\u001b[38;5;241m=\u001b[39mrequest)\n\u001b[0;32m--> 700\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m(e, request\u001b[38;5;241m=\u001b[39mrequest)\n\u001b[1;32m 702\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m ClosedPoolError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m 703\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m(e, request\u001b[38;5;241m=\u001b[39mrequest)\n", + "\u001b[0;31mConnectionError\u001b[0m: HTTPConnectionPool(host='localhost', port=8080): Max retries exceeded with url: /api/v2/metadata (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused'))" + ] + } + ], "source": [ "root_client = sy.login(\n", - " url=\"http://localhost:8080\", email=ROOT_EMAIL, password=ROOT_PASSWORD\n", + " url=\"http://localhost:9081\", email=ROOT_EMAIL, password=ROOT_PASSWORD\n", ")" ] }, @@ -142,7 +232,7 @@ "outputs": [], "source": [ "high_client = sy.login(\n", - " url=\"http://localhost:8080\", email=ADMIN_EMAIL, password=ADMIN_PW\n", + " url=\"http://localhost:9081\", email=ADMIN_EMAIL, password=ADMIN_PW\n", ")" ] }, @@ -223,6 +313,34 @@ "server.land()" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "code", "execution_count": null, @@ -232,6 +350,11 @@ } ], "metadata": { + "kernelspec": { + "display_name": "syft", + "language": "python", + "name": "python3" + }, "language_info": { "codemirror_mode": { "name": "ipython", @@ -242,7 +365,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.5" + "version": "3.12.4" } }, "nbformat": 4, diff --git a/notebooks/scenarios/bigquery/001-scale-delete-worker-pools.ipynb b/notebooks/scenarios/bigquery/001-scale-delete-worker-pools.ipynb index 884b77501ab..175f5b86e22 100644 --- a/notebooks/scenarios/bigquery/001-scale-delete-worker-pools.ipynb +++ b/notebooks/scenarios/bigquery/001-scale-delete-worker-pools.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "0", "metadata": {}, "outputs": [], @@ -16,7 +16,31 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, + "id": "c89fb0b7", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "os.environ['ORCHESTRA_DEPLOYMENT_TYPE'] = 'remote'\n", + "os.environ['DEVSPACE_PROFILE'] = 'bigquery-scenario-tests'\n", + "os.environ['GITHUB_CI'] = 'false'\n", + "os.environ['SYFT_BASE_IMAGE_REGISTRY'] = 'k3d-registry.localhost:5800'\n", + "os.environ['DATASITE_CLUSTER_NAME'] = 'test-datasite-1'\n", + "os.environ['SERVER_URL'] = 'http://localhost'\n", + "os.environ['SERVER_PORT'] = '9081'\n", + "os.environ['TEST_EXTERNAL_REGISTRY'] = 'k3d-registry.localhost:5800'\n", + "os.environ['TEST_QUERY_LIMIT_SIZE'] = '500000'\n", + "os.environ['TRACING'] = 'False'\n", + "os.environ['NUM_TEST_USERS'] = '5'\n", + "os.environ['NUM_TEST_JOBS'] = '10'\n", + "os.environ['NUM_TEST_WORKERS'] = '2'" + ] + }, + { + "cell_type": "code", + "execution_count": 3, "id": "1", "metadata": {}, "outputs": [], @@ -34,7 +58,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "2", "metadata": {}, "outputs": [], @@ -44,8 +68,7 @@ "num_workers = int(os.environ.get(\"NUM_TEST_WORKERS\", 1))\n", "\n", "ROOT_EMAIL = \"admin@bigquery.org\"\n", - "ROOT_PASSWORD = \"bqpw\"\n", - "environment" + "ROOT_PASSWORD = \"bqpw\"" ] }, { @@ -58,7 +81,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "4", "metadata": {}, "outputs": [], @@ -67,7 +90,7 @@ " name=\"bigquery-high\",\n", " dev_mode=True,\n", " server_side_type=\"high\",\n", - " port=\"8080\",\n", + " port=\"9081\",\n", " n_consumers=num_workers, # How many workers to be spawned\n", " create_producer=True, # Can produce more workers\n", ")" @@ -75,42 +98,6141 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "5", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> Starting SMTPTestServer on: 0.0.0.0:9025\n" + ] + } + ], "source": [ "email_server, smtp_server = get_email_server(reset=True)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "6", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Logged into as \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> Starting SMTPTestServer on: 0.0.0.0:9025\n", + "> Error with SMTPTestServer: [Errno 48] error while attempting to bind on address ('0.0.0.0', 9025): address already in use\n" + ] + } + ], "source": [ "high_client = sy.login(\n", - " url=\"http://localhost:8080\", email=ROOT_EMAIL, password=ROOT_PASSWORD\n", + " url=\"http://localhost:9081\", email=ROOT_EMAIL, password=ROOT_PASSWORD\n", ")" ] }, { "cell_type": "code", - "execution_count": null, - "id": "7", + "execution_count": 8, + "id": "c8e1689f", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "\n", + "
\n", + "
\n", + " \n", + "
\n", + "

WorkerPool Dicttuple

\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "

Total: 0

\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "\n", + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "high_client.worker_pools" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "8", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "

default-pool

\n", + "

\n", + " Created on: \n", + " 2024-09-11 20:07:15\n", + "

\n", + "

\n", + " Healthy Workers:\n", + " 1 / 1\n", + "

\n", + "

\n", + " Running Workers:\n", + " 1 / 1\n", + "

\n", + " \n", + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "\n", + "
\n", + "
\n", + " \n", + "
\n", + "

SyftWorker List

\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "

Total: 0

\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "\n", + "" + ], + "text/markdown": [ + "```python\n", + "class WorkerPool:\n", + " id: str = 1cc42404c3c54f9b9c6b226086ccccc6\n", + " name: str = \"default-pool\"\n", + " image: str = syft.service.worker.worker_image.SyftWorkerImage\n", + " max_count: str = 1\n", + " workers: str = [syft.service.worker.worker_pool.SyftWorker]\n", + " created_at: str = 2024-09-11 20:07:15\n", + "\n", + "```" + ], + "text/plain": [ + "syft.service.worker.worker_pool.WorkerPool" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "default_worker_pool = high_client.worker_pools.get_by_name(\"default-pool\")\n", "default_worker_pool" @@ -134,7 +6256,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "11", "metadata": {}, "outputs": [], @@ -148,17 +6270,3483 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "12", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "

default-pool

\n", + "

\n", + " Created on: \n", + " 2024-09-11 20:07:15\n", + "

\n", + "

\n", + " Healthy Workers:\n", + " 2 / 2\n", + "

\n", + "

\n", + " Running Workers:\n", + " 2 / 2\n", + "

\n", + " \n", + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "\n", + "
\n", + "
\n", + " \n", + "
\n", + "

SyftWorker List

\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "

Total: 0

\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "\n", + "" + ], + "text/markdown": [ + "```python\n", + "class WorkerPool:\n", + " id: str = 1cc42404c3c54f9b9c6b226086ccccc6\n", + " name: str = \"default-pool\"\n", + " image: str = syft.service.worker.worker_image.SyftWorkerImage\n", + " max_count: str = 2\n", + " workers: str = [syft.service.worker.worker_pool.SyftWorker, syft.service.worker.worker_pool.SyftWorker]\n", + " created_at: str = 2024-09-11 20:07:15\n", + "\n", + "```" + ], + "text/plain": [ + "syft.service.worker.worker_pool.WorkerPool" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "high_client.api.services.worker_pool[0]" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "id": "13", "metadata": {}, "outputs": [], @@ -187,10 +9775,2534 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "id": "15", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
SyftSuccess: 
Syft function 'wait_1000_seconds_1' successfully created. To add a code request, please create a project using `project = syft.Project(...)`, then use command `project.create_code_request`.

" + ], + "text/plain": [ + "SyftSuccess: Syft function 'wait_1000_seconds_1' successfully created. To add a code request, please create a project using `project = syft.Project(...)`, then use command `project.create_code_request`." + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
SyftSuccess: 
Syft function 'wait_1000_seconds_2' successfully created. To add a code request, please create a project using `project = syft.Project(...)`, then use command `project.create_code_request`.

" + ], + "text/plain": [ + "SyftSuccess: Syft function 'wait_1000_seconds_2' successfully created. To add a code request, please create a project using `project = syft.Project(...)`, then use command `project.create_code_request`." + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
SyftSuccess: 
Syft function 'wait_1000_seconds_3' successfully created. To add a code request, please create a project using `project = syft.Project(...)`, then use command `project.create_code_request`.

" + ], + "text/plain": [ + "SyftSuccess: Syft function 'wait_1000_seconds_3' successfully created. To add a code request, please create a project using `project = syft.Project(...)`, then use command `project.create_code_request`." + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "@sy.syft_function_single_use(worker_pool_name=default_worker_pool.name)\n", "def wait_1000_seconds_1():\n", @@ -218,10 +12330,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "id": "20", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Approving request on change wait_1000_seconds_3 for datasite syft-dev-server\n", + "Approving request on change wait_1000_seconds_2 for datasite syft-dev-server\n", + "Approving request on change wait_1000_seconds_1 for datasite syft-dev-server\n" + ] + } + ], "source": [ "jobs = []\n", "high_client.code.request_code_execution(wait_1000_seconds_1)\n", @@ -235,7 +12357,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "id": "21", "metadata": {}, "outputs": [], @@ -251,14 +12373,24 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "id": "22", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{}\n", + "{: , : , : }\n" + ] + } + ], "source": [ "# check that at least three workers have a job (since scaling down to 2)\n", "# try 3 times with a 5 second sleep in case it takes time for the workers to accept the jobs\n", - "for _ in range(3):\n", + "import time\n", + "for _ in range(10):\n", " worker_to_job_map = {}\n", " syft_workers_ids = set()\n", " for job in high_client.jobs:\n", @@ -267,7 +12399,7 @@ " worker_to_job_map[job.worker.id] = job.id\n", " print(worker_to_job_map)\n", " if len(syft_workers_ids) < 3:\n", - " time.sleep(20)\n", + " time.sleep(5)\n", " else:\n", " break\n", "assert len(syft_workers_ids) >= 3" @@ -283,7 +12415,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "id": "24", "metadata": {}, "outputs": [], @@ -298,10 +12430,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "id": "25", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Condition met, exiting early.\n" + ] + } + ], "source": [ "if environment == \"remote\":\n", "\n", @@ -332,7 +12472,2626 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 29, + "id": "2773f0b6", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "\n", + "
\n", + "
\n", + " \n", + "
\n", + "

Job List

\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "

Total: 0

\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "\n", + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "high_client.jobs" + ] + }, + { + "cell_type": "code", + "execution_count": 20, "id": "26", "metadata": {}, "outputs": [], @@ -354,10 +15113,853 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "id": "28", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
SyftSuccess: 
Successfully deleted worker pool with id 1cc42404c3c54f9b9c6b226086ccccc6

" + ], + "text/plain": [ + "SyftSuccess: Successfully deleted worker pool with id 1cc42404c3c54f9b9c6b226086ccccc6" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "pool_delete_result = high_client.api.services.worker_pool.delete(\n", " pool_name=default_worker_pool.name\n", @@ -367,7 +15969,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "id": "29", "metadata": {}, "outputs": [], @@ -378,10 +15980,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "id": "30", "metadata": {}, - "outputs": [], + "outputs": [ + { + "ename": "AssertionError", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mAssertionError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[25], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;66;03m# check that all jobs are interrupted\u001b[39;00m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;66;03m# should be the case since the entire pool was deleted and all jobs were previously assigned\u001b[39;00m\n\u001b[0;32m----> 3\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28mall\u001b[39m(job\u001b[38;5;241m.\u001b[39mstatus \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124minterrupted\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m job \u001b[38;5;129;01min\u001b[39;00m high_client\u001b[38;5;241m.\u001b[39mjobs)\n", + "\u001b[0;31mAssertionError\u001b[0m: " + ] + } + ], "source": [ "# check that all jobs are interrupted\n", "# should be the case since the entire pool was deleted and all jobs were previously assigned\n", @@ -398,7 +16012,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "id": "32", "metadata": {}, "outputs": [], @@ -408,7 +16022,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "id": "33", "metadata": {}, "outputs": [], @@ -422,7 +16036,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "id": "34", "metadata": {}, "outputs": [], @@ -436,20 +16050,36 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "id": "35", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> Stopping SMTPTestServer\n" + ] + } + ], "source": [ "smtp_server.stop()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 29, "id": "36", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Shutdown not implemented for the deployment type:remote\n" + ] + } + ], "source": [ "server.land()" ] diff --git a/notebooks/scenarios/bigquery/010-setup-bigquery-pool.ipynb b/notebooks/scenarios/bigquery/010-setup-bigquery-pool.ipynb index 1280a4e19d7..625355e215c 100644 --- a/notebooks/scenarios/bigquery/010-setup-bigquery-pool.ipynb +++ b/notebooks/scenarios/bigquery/010-setup-bigquery-pool.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -14,7 +14,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "os.environ['ORCHESTRA_DEPLOYMENT_TYPE'] = 'remote'\n", + "os.environ['DEVSPACE_PROFILE'] = 'bigquery-scenario-tests'\n", + "os.environ['GITHUB_CI'] = 'false'\n", + "os.environ['SYFT_BASE_IMAGE_REGISTRY'] = 'k3d-registry.localhost:5800'\n", + "os.environ['DATASITE_CLUSTER_NAME'] = 'test-datasite-1'\n", + "os.environ['SERVER_URL'] = 'http://localhost'\n", + "os.environ['SERVER_PORT'] = '9081'\n", + "os.environ['TEST_EXTERNAL_REGISTRY'] = 'k3d-registry.localhost:5800'\n", + "os.environ['TEST_QUERY_LIMIT_SIZE'] = '500000'\n", + "os.environ['TRACING'] = 'False'\n", + "os.environ['NUM_TEST_USERS'] = '5'\n", + "os.environ['NUM_TEST_JOBS'] = '10'\n", + "os.environ['NUM_TEST_WORKERS'] = '2'" + ] + }, + { + "cell_type": "code", + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -31,9 +54,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'remote'" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", "environment" @@ -48,7 +82,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -56,7 +90,7 @@ " name=\"bigquery-high\",\n", " dev_mode=True,\n", " server_side_type=\"high\",\n", - " port=\"8080\",\n", + " port=\"9081\",\n", " n_consumers=1, # How many workers to be spawned\n", " create_producer=True, # Can produce more workers\n", ")" @@ -64,22 +98,45 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> Starting SMTPTestServer on: 0.0.0.0:9025\n" + ] + } + ], "source": [ "_, smtp_server = get_email_server(reset=True)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> Starting SMTPTestServer on: 0.0.0.0:9025\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Logged into as \n" + ] + } + ], "source": [ "ADMIN_EMAIL, ADMIN_PW = \"admin2@bigquery.org\", \"bqpw2\"\n", "high_client = sy.login(\n", - " url=\"http://localhost:8080\", email=ADMIN_EMAIL, password=ADMIN_PW\n", + " url=\"http://localhost:9081\", email=ADMIN_EMAIL, password=ADMIN_PW\n", ")" ] }, @@ -99,16 +156,2625 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "\n", + "
\n", + "
\n", + " \n", + "
\n", + "

WorkerPool Dicttuple

\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "

Total: 0

\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "\n", + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "high_client.worker_pools" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -117,9 +2783,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'k3d-registry.localhost:5800'" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "external_registry = test_settings.get(\"external_registry\", default=\"docker.io\")\n", "external_registry" @@ -127,9 +2804,852 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
SyftSuccess: 
Image Registry ID: 80fdf8ec3e0d4b28a403683bf9860e3b created successfully

" + ], + "text/plain": [ + "SyftSuccess: Image Registry ID: 80fdf8ec3e0d4b28a403683bf9860e3b created successfully" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "result = high_client.api.services.image_registry.add(external_registry)\n", "result" @@ -137,9 +3657,2618 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "\n", + "
\n", + "
\n", + " \n", + "
\n", + "

SyftImageRegistry List

\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "

Total: 0

\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "\n", + "" + ], + "text/plain": [ + "[SyftImageRegistry(url=k3d-registry.localhost:5800)]" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "image_registry_list = high_client.api.services.image_registry.get_all()\n", "image_registry_list" @@ -147,9 +6276,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/markdown": [ + "```python\n", + "class SyftImageRegistry:\n", + " id: str = 80fdf8ec3e0d4b28a403683bf9860e3b\n", + " url: str = \"k3d-registry.localhost:5800\"\n", + "\n", + "```" + ], + "text/plain": [ + "SyftImageRegistry(url=k3d-registry.localhost:5800)" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "local_registry = image_registry_list[0]\n", "local_registry" @@ -164,9 +6312,3447 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "\n", + "
\n", + "
\n", + " \n", + "
\n", + "

SyftWorkerImage Dicttuple

\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "

Total: 0

\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "\n", + "" + ], + "text/plain": [ + "DictTuple(syft.service.worker.worker_image.SyftWorkerImage,)" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "dockerfile_list = high_client.images.get_all()\n", "dockerfile_list" @@ -174,9 +9760,32 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/markdown": [ + "```python\n", + "class SyftWorkerImage:\n", + " id: str = 77d3930961d341e5a154cf4022bd14d8\n", + " image_identifier: str = k3d-registry.localhost:5800/openmined/syft-backend:dev-1726083661\n", + " image_hash: str = None\n", + " created_at: str = 2024-09-11 19:42:10\n", + " built_at: str = None\n", + " config: str = prebuilt tag='k3d-registry.localhost:5800/openmined/syft-backend:dev-1726083661' description='Prebuilt default worker image'\n", + "\n", + "```" + ], + "text/plain": [ + "syft.service.worker.worker_image.SyftWorkerImage" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "base_worker_image = next(\n", " (\n", @@ -191,18 +9800,40 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "SyftWorkerImageIdentifier(repo=openmined/syft-backend, tag=dev-1726083661, registry=k3d-registry.localhost:5800)" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "base_worker_image.image_identifier" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'FROM k3d-registry.localhost:5800/openmined/syft-backend:dev-1726083661\\n\\nRUN uv pip install db-dtypes google-cloud-bigquery'" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "worker_dockerfile = f\"\"\"\n", "FROM {str(base_worker_image.image_identifier)}\n", @@ -215,7 +9846,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ @@ -225,9 +9856,852 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
SyftSuccess: 
Dockerfile ID: 2987a404b77d405a8b923521103e1772 successfully submitted.

" + ], + "text/plain": [ + "SyftSuccess: Dockerfile ID: 2987a404b77d405a8b923521103e1772 successfully submitted." + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "submit_result = high_client.api.services.worker_image.submit(\n", " worker_config=docker_config\n", @@ -237,9 +10711,3447 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "\n", + "
\n", + "
\n", + " \n", + "
\n", + "

SyftWorkerImage Dicttuple

\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "

Total: 0

\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "\n", + "" + ], + "text/plain": [ + "DictTuple(syft.service.worker.worker_image.SyftWorkerImage, syft.service.worker.worker_image.SyftWorkerImage)" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "dockerfile_list = high_client.images.get_all()\n", "dockerfile_list" @@ -247,9 +14159,34 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/markdown": [ + "```python\n", + "class SyftWorkerImage:\n", + " id: str = 2987a404b77d405a8b923521103e1772\n", + " image_identifier: str = None\n", + " image_hash: str = None\n", + " created_at: str = 2024-09-11 19:42:10\n", + " built_at: str = None\n", + " config: str = FROM k3d-registry.localhost:5800/openmined/syft-backend:dev-1726083661\n", + "\n", + "RUN uv pip install db-dtypes google-cloud-bigquery\n", + "\n", + "```" + ], + "text/plain": [ + "syft.service.worker.worker_image.SyftWorkerImage" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "worker_image = next(\n", " (\n", @@ -264,16 +14201,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "FROM k3d-registry.localhost:5800/openmined/syft-backend:dev-1726083661\n", + "\n", + "RUN uv pip install db-dtypes google-cloud-bigquery\n" + ] + } + ], "source": [ "print(worker_image.config.dockerfile)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "metadata": {}, "outputs": [], "source": [ @@ -290,9 +14237,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'k3d-registry.localhost:5800/openmined/syft-worker-bigquery:dev-1726083661'" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "docker_tag = str(base_worker_image.image_identifier).replace(\n", " \"backend\", \"worker-bigquery\"\n", @@ -302,9 +14260,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "SyftSuccess: Build for Worker ID: 2987a404b77d405a8b923521103e1772 succeeded.\n", + "----------Logs for pod=build-8397a8a888c2fd3e-np7l6----------\n", + "time=\"2024-09-11T19:46:35Z\" level=info msg=\"Retrieving image manifest k3d-registry.localhost:5800/openmined/syft-backend:dev-1726083661\"time=\"2024-09-11T19:46:35Z\" level=info msg=\"Retrieving image k3d-registry.localhost:5800/openmined/syft-backend:dev-1726083661 from registry k3d-registry.localhost:5800\"time=\"2024-09-11T19:46:35Z\" level=info msg=\"Retrieving image manifest k3d-registry.localhost:5800/openmined/syft-backend:dev-1726083661\"time=\"2024-09-11T19:46:35Z\" level=info msg=\"Returning cached image manifest\"time=\"2024-09-11T19:46:35Z\" level=info msg=\"Built cross stage deps: map[]\"time=\"2024-09-11T19:46:35Z\" level=info msg=\"Retrieving image manifest k3d-registry.localhost:5800/openmined/syft-backend:dev-1726083661\"time=\"2024-09-11T19:46:35Z\" level=info msg=\"Returning cached image manifest\"time=\"2024-09-11T19:46:35Z\" level=info msg=\"Retrieving image manifest k3d-registry.localhost:5800/openmined/syft-backend:dev-1726083661\"time=\"2024-09-11T19:46:35Z\" level=info msg=\"Returning cached image manifest\"time=\"2024-09-11T19:46:35Z\" level=info msg=\"Executing 0 build triggers\"time=\"2024-09-11T19:46:35Z\" level=info msg=\"Building stage 'k3d-registry.localhost:5800/openmined/syft-backend:dev-1726083661' [idx: '0', base-idx: '-1']\"time=\"2024-09-11T19:46:35Z\" level=info msg=\"Checking for cached layer registry.syft.svc.cluster.local/openmined/syft-worker-bigquery/cache:e4f3c9634d34d446ef66544b3ade0214b518061eab031c49d90a37354d73681d...\"time=\"2024-09-11T19:46:35Z\" level=info msg=\"No cached layer found for cmd RUN uv pip install db-dtypes google-cloud-bigquery\"time=\"2024-09-11T19:46:35Z\" level=info msg=\"Unpacking rootfs as cmd RUN uv pip install db-dtypes google-cloud-bigquery requires it.\"time=\"2024-09-11T19:46:56Z\" level=info msg=\"RUN uv pip install db-dtypes google-cloud-bigquery\"time=\"2024-09-11T19:46:56Z\" level=info msg=\"Initializing snapshotter ...\"time=\"2024-09-11T19:46:56Z\" level=info msg=\"Taking snapshot of full filesystem...\"time=\"2024-09-11T19:46:59Z\" level=info msg=\"Cmd: /bin/sh\"time=\"2024-09-11T19:46:59Z\" level=info msg=\"Args: [-c uv pip install db-dtypes google-cloud-bigquery]\"time=\"2024-09-11T19:46:59Z\" level=info msg=\"Util.Lookup returned: &{Uid:0 Gid:0 Username:root Name: HomeDir:/root}\"time=\"2024-09-11T19:46:59Z\" level=info msg=\"Performing slow lookup of group ids for root\"time=\"2024-09-11T19:46:59Z\" level=info msg=\"Running: [/bin/sh -c uv pip install db-dtypes google-cloud-bigquery]\"Resolved 29 packages in 690msDownloaded 12 packages in 102msInstalled 12 packages in 9ms + db-dtypes==1.3.0 + google-api-core==2.19.2 + google-auth==2.34.0 + google-cloud-bigquery==3.25.0 + google-cloud-core==2.4.1 + google-crc32c==1.6.0 + google-resumable-media==2.7.2 + grpcio-status==1.62.3 + proto-plus==1.24.0 + pyasn1==0.6.1 + pyasn1-modules==0.4.1 + rsa==4.9time=\"2024-09-11T19:47:00Z\" level=info msg=\"Taking snapshot of full filesystem...\"time=\"2024-09-11T19:47:01Z\" level=info msg=\"Pushing layer registry.syft.svc.cluster.local/openmined/syft-worker-bigquery/cache:e4f3c9634d34d446ef66544b3ade0214b518061eab031c49d90a37354d73681d to cache now\"time=\"2024-09-11T19:47:01Z\" level=info msg=\"Pushing image to registry.syft.svc.cluster.local/openmined/syft-worker-bigquery/cache:e4f3c9634d34d446ef66544b3ade0214b518061eab031c49d90a37354d73681d\"time=\"2024-09-11T19:47:01Z\" level=info msg=\"Pushed registry.syft.svc.cluster.local/openmined/syft-worker-bigquery/cache@sha256:119270096a158e05987329fe23c9bf6efd174e69b4b5f377955ec38691ad9601\"time=\"2024-09-11T19:47:01Z\" level=info msg=\"Pushing image to registry.syft.svc.cluster.local/openmined/syft-worker-bigquery:dev-1726083661\"time=\"2024-09-11T19:47:04Z\" level=info msg=\"Pushed registry.syft.svc.cluster.local/openmined/syft-worker-bigquery@sha256:925b1098fd91d8be3d310c355ddccbb3c0aca011a071f3cf1b28a9ff7ccc6340\"\n" + ] + } + ], "source": [ "if environment == \"remote\":\n", " docker_build_result = high_client.api.services.worker_image.build(\n", @@ -317,9 +14285,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "SyftSuccess: Pushed Image ID: 2987a404b77d405a8b923521103e1772 to \"k3d-registry.localhost:5800/openmined/syft-worker-bigquery:dev-1726083661\".\n" + ] + } + ], "source": [ "if environment == \"remote\":\n", " push_result = high_client.api.services.worker_image.push(worker_image.id)\n", @@ -328,9 +14304,3447 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "\n", + "
\n", + "
\n", + " \n", + "
\n", + "

SyftWorkerImage Dicttuple

\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "

Total: 0

\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "\n", + "" + ], + "text/plain": [ + "DictTuple(syft.service.worker.worker_image.SyftWorkerImage, syft.service.worker.worker_image.SyftWorkerImage)" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "dockerfile_list = high_client.images.get_all()\n", "dockerfile_list" @@ -338,9 +17752,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "PrebuiltWorkerConfig(tag='k3d-registry.localhost:5800/openmined/syft-worker-bigquery:dev-1726083661', description=None)" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "docker_config = sy.PrebuiltWorkerConfig(tag=docker_tag)\n", "docker_config" @@ -348,9 +17773,852 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 29, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
SyftSuccess: 
Dockerfile ID: c9b62bf8d060405b8a06891e68cf0b44 successfully submitted.

" + ], + "text/plain": [ + "SyftSuccess: Dockerfile ID: c9b62bf8d060405b8a06891e68cf0b44 successfully submitted." + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# overwrite it for now Mongo ignore\n", "result = high_client.api.services.worker_image.submit(worker_config=docker_config)\n", @@ -359,9 +18627,3447 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 30, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "\n", + "
\n", + "
\n", + " \n", + "
\n", + "

SyftWorkerImage Dicttuple

\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "

Total: 0

\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "\n", + "" + ], + "text/plain": [ + "DictTuple(syft.service.worker.worker_image.SyftWorkerImage, syft.service.worker.worker_image.SyftWorkerImage)" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "dockerfile_list = high_client.images.get_all()\n", "dockerfile_list" @@ -369,7 +22075,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 31, "metadata": {}, "outputs": [], "source": [ @@ -382,9 +22088,32 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 32, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/markdown": [ + "```python\n", + "class SyftWorkerImage:\n", + " id: str = c9b62bf8d060405b8a06891e68cf0b44\n", + " image_identifier: str = k3d-registry.localhost:5800/openmined/syft-worker-bigquery:dev-1726083661\n", + " image_hash: str = None\n", + " created_at: str = 2024-09-11 19:42:10\n", + " built_at: str = None\n", + " config: str = prebuilt tag='k3d-registry.localhost:5800/openmined/syft-worker-bigquery:dev-1726083661'\n", + "\n", + "```" + ], + "text/plain": [ + "syft.service.worker.worker_image.SyftWorkerImage" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "worker_image = next(\n", " (\n", @@ -399,7 +22128,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 33, "metadata": {}, "outputs": [], "source": [ @@ -408,7 +22137,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 34, "metadata": {}, "outputs": [], "source": [ @@ -428,9 +22157,2618 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 35, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "\n", + "
\n", + "
\n", + " \n", + "
\n", + "

ContainerSpawnStatus List

\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "

Total: 0

\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "\n", + "" + ], + "text/plain": [ + "[ContainerSpawnStatus(worker_name='bigquery-pool-0', worker=syft.service.worker.worker_pool.SyftWorker, error=None)]" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "result = high_client.api.services.worker_pool.launch(\n", " pool_name=worker_pool_name,\n", @@ -451,9 +24789,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 36, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "SyftSuccess: Worker pool scaled to 2 workers\n" + ] + } + ], "source": [ "if environment == \"remote\":\n", " result = high_client.worker_pools.scale(\n", @@ -464,7 +24810,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 37, "metadata": {}, "outputs": [], "source": [ @@ -473,16 +24819,859 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 38, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
SyftSuccess: 
Registration feature successfully disabled

" + ], + "text/plain": [ + "SyftSuccess: Registration feature successfully disabled" + ] + }, + "execution_count": 38, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "high_client.settings.allow_guest_signup(enable=False)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 39, "metadata": {}, "outputs": [], "source": [ @@ -498,18 +25687,34 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 40, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> Stopping SMTPTestServer\n" + ] + } + ], "source": [ "smtp_server.stop()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 41, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Shutdown not implemented for the deployment type:remote\n" + ] + } + ], "source": [ "server.land()" ] @@ -523,6 +25728,11 @@ } ], "metadata": { + "kernelspec": { + "display_name": "syft", + "language": "python", + "name": "python3" + }, "language_info": { "codemirror_mode": { "name": "ipython", @@ -533,7 +25743,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.5" + "version": "3.12.4" } }, "nbformat": 4, diff --git a/notebooks/scenarios/bigquery/011-users-emails-passwords.ipynb b/notebooks/scenarios/bigquery/011-users-emails-passwords.ipynb index 990f91183d8..adc84cbfe5b 100644 --- a/notebooks/scenarios/bigquery/011-users-emails-passwords.ipynb +++ b/notebooks/scenarios/bigquery/011-users-emails-passwords.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "0", "metadata": {}, "outputs": [], @@ -17,7 +17,31 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, + "id": "28c5ab76", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "os.environ['ORCHESTRA_DEPLOYMENT_TYPE'] = 'remote'\n", + "os.environ['DEVSPACE_PROFILE'] = 'bigquery-scenario-tests'\n", + "os.environ['GITHUB_CI'] = 'false'\n", + "os.environ['SYFT_BASE_IMAGE_REGISTRY'] = 'k3d-registry.localhost:5800'\n", + "os.environ['DATASITE_CLUSTER_NAME'] = 'test-datasite-1'\n", + "os.environ['SERVER_URL'] = 'http://localhost'\n", + "os.environ['SERVER_PORT'] = '9081'\n", + "os.environ['TEST_EXTERNAL_REGISTRY'] = 'k3d-registry.localhost:5800'\n", + "os.environ['TEST_QUERY_LIMIT_SIZE'] = '500000'\n", + "os.environ['TRACING'] = 'False'\n", + "os.environ['NUM_TEST_USERS'] = '5'\n", + "os.environ['NUM_TEST_JOBS'] = '10'\n", + "os.environ['NUM_TEST_WORKERS'] = '2'" + ] + }, + { + "cell_type": "code", + "execution_count": 3, "id": "1", "metadata": {}, "outputs": [], @@ -39,10 +63,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "2", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'remote'" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "ADMIN_EMAIL, ADMIN_PW = \"admin2@bigquery.org\", \"bqpw2\"\n", "\n", @@ -60,7 +95,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "4", "metadata": {}, "outputs": [], @@ -69,7 +104,7 @@ " name=\"bigquery-high\",\n", " dev_mode=True,\n", " server_side_type=\"high\",\n", - " port=\"8080\",\n", + " port=\"9081\",\n", " n_consumers=1, # How many workers to be spawned\n", " create_producer=True, # Can produce more workers\n", ")" @@ -77,39 +112,2679 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "5", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> Starting SMTPTestServer on: 0.0.0.0:9025\n" + ] + } + ], "source": [ "email_server, smtp_server = get_email_server()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "6", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> Starting SMTPTestServer on: 0.0.0.0:9025\n", + "> SMTPTestServer got an email for ['carrie-huffman-fake@openmined.org']\n", + "> SMTPTestServer got an email for ['brent-robertson-md-fake@openmined.org']\n", + "> SMTPTestServer got an email for ['lauren-velez-fake@openmined.org']\n", + "> SMTPTestServer got an email for ['daniel-moore-fake@openmined.org']\n", + "> SMTPTestServer got an email for ['shane-wheeler-fake@openmined.org']\n", + "> SMTPTestServer got an email for ['brent-robertson-md-fake@openmined.org']\n", + "> SMTPTestServer got an email for ['brent-robertson-md-fake@openmined.org']\n", + "> SMTPTestServer got an email for ['brent-robertson-md-fake@openmined.org']\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Logged into as \n" + ] + } + ], "source": [ "high_client = sy.login(\n", - " url=\"http://localhost:8080\", email=ADMIN_EMAIL, password=ADMIN_PW\n", + " url=\"http://localhost:9081\", email=ADMIN_EMAIL, password=ADMIN_PW\n", ")" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "7", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "\n", + "
\n", + "
\n", + " \n", + "
\n", + "

UserView List

\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "

Total: 0

\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "\n", + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "high_client.users" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "8", "metadata": {}, "outputs": [], @@ -124,7 +2799,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "9", "metadata": {}, "outputs": [], @@ -143,10 +2818,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "11", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "registering 5 users\n" + ] + } + ], "source": [ "num_users = int(os.environ.get(\"NUM_TEST_USERS\", 5))\n", "print(f\"registering {num_users} users\")" @@ -154,10 +2837,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "id": "12", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Logged into as \n", + "Logged into as \n", + "Logged into as \n", + "Logged into as \n", + "Logged into as \n" + ] + } + ], "source": [ "users = []\n", "email_disable_index = 0\n", @@ -176,17 +2871,25 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "id": "13", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[{'name': 'Carrie Huffman', 'email': 'carrie-huffman-fake@openmined.org', 'password': 'N!hLj0RF8v', 'role': 'ServiceRole.DATA_SCIENTIST', 'new_password': None, 'email_disabled': True, 'reset_password': False, 'reset_token': None}, {'name': 'Brent Robertson MD', 'email': 'brent-robertson-md-fake@openmined.org', 'password': '6i6Uf1vd&m', 'role': 'ServiceRole.DATA_SCIENTIST', 'new_password': None, 'email_disabled': False, 'reset_password': True, 'reset_token': None}, {'name': 'Lauren Velez', 'email': 'lauren-velez-fake@openmined.org', 'password': '_NJ9&yTne@', 'role': 'ServiceRole.DATA_SCIENTIST', 'new_password': None, 'email_disabled': False, 'reset_password': False, 'reset_token': None}, {'name': 'Daniel Moore', 'email': 'daniel-moore-fake@openmined.org', 'password': '_IfXa_*f8C', 'role': 'ServiceRole.DATA_SCIENTIST', 'new_password': None, 'email_disabled': False, 'reset_password': False, 'reset_token': None}, {'name': 'Shane Wheeler', 'email': 'shane-wheeler-fake@openmined.org', 'password': '42XIg#Ua!5', 'role': 'ServiceRole.DATA_SCIENTIST', 'new_password': None, 'email_disabled': False, 'reset_password': False, 'reset_token': None}]\n" + ] + } + ], "source": [ "save_users(users)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "id": "14", "metadata": {}, "outputs": [], @@ -207,7 +2910,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "id": "16", "metadata": {}, "outputs": [], @@ -226,10 +2929,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "id": "17", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Logged into as \n", + "Logged into as \n", + "Logged into as \n", + "Logged into as \n", + "Logged into as \n" + ] + } + ], "source": [ "# one user disables notifications\n", "# one user requests a password reset\n", @@ -251,7 +2966,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "id": "18", "metadata": {}, "outputs": [], @@ -261,10 +2976,34 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "id": "19", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/markdown": [ + "```python\n", + "class UserView:\n", + " id: str = 28a0aa1b5a7d4606804df83cd8269718\n", + " name: str = \"Carrie Huffman\"\n", + " email: str = \"carrie-huffman-fake@openmined.org\"\n", + " institution: str = \"Morton LLC\"\n", + " website: str = \"http://www.taylor.info/\"\n", + " role: str = ServiceRole.DATA_SCIENTIST\n", + " notifications_enabled: str = {: False, : False, : False, : False}\n", + "\n", + "```" + ], + "text/plain": [ + "syft.service.user.user.UserView" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "ds0_user = ds0.client.account\n", "ds0_user" @@ -272,10 +3011,78 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "id": "20", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "with sy.raises successfully caught the following exception:\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + " \n", + " \n", + " SyftException:\n", + " 
You are not permitted to perform this action.
\n", + " \n", + " \n", + "
\n", + " \n", + "
Client Trace:
\n", + " 
\n",
+       "      \n",
+       "        Traceback (most recent call last):\n",
+       "  File \"/var/folders/mt/hh7shybx3xq2888mhg60t29h0000gn/T/ipykernel_27171/4147657975.py, line 5, in \n",
+       "    ds0.client.users.update(uid=ds0_user.id, role=\"admin\")\n",
+       "  File \"/Users/brendanschell/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/packages/syft/src/syft/client/api.py, line 374, in __call__\n",
+       "    return self.function_call(self.path, *args, **kwargs)\n",
+       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+       "  File \"/Users/brendanschell/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/packages/syft/src/syft/client/api.py, line 371, in function_call\n",
+       "    return post_process_result(result, self.unwrap_on_success)\n",
+       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+       "syft.types.errors.SyftException: You are not permitted to perform this action.\n",
+       "\n",
+       "      \n",
+       "
\n", + "
\n", + "
\n", + "\n", + "" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "# with sy.raises(\n", "# sy.SyftException(public_message=\"*tried to update user*\"\n", @@ -286,10 +3093,81 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "id": "21", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "with sy.raises successfully caught the following exception:\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + " \n", + " \n", + " SyftException:\n", + " 
You are not permitted to perform this action.
\n", + " \n", + " \n", + "
\n", + " \n", + "
Client Trace:
\n", + " 
\n",
+       "      \n",
+       "        Traceback (most recent call last):\n",
+       "  File \"/var/folders/mt/hh7shybx3xq2888mhg60t29h0000gn/T/ipykernel_27171/931364833.py, line 3, in \n",
+       "    ds0_user.update(role=\"admin\")\n",
+       "  File \"/Users/brendanschell/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/packages/syft/src/syft/service/user/user.py, line 319, in update\n",
+       "    result = api.services.user.update(\n",
+       "             ^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+       "  File \"/Users/brendanschell/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/packages/syft/src/syft/client/api.py, line 374, in __call__\n",
+       "    return self.function_call(self.path, *args, **kwargs)\n",
+       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+       "  File \"/Users/brendanschell/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/packages/syft/src/syft/client/api.py, line 371, in function_call\n",
+       "    return post_process_result(result, self.unwrap_on_success)\n",
+       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+       "syft.types.errors.SyftException: You are not permitted to perform this action.\n",
+       "\n",
+       "      \n",
+       "
\n", + "
\n", + "
\n", + "\n", + "" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "# with sy.raises(sy.SyftException(public_message=\"*tried to update user*\"), show=True):\n", "with sy.raises(sy.SyftException, show=True):\n", @@ -298,7 +3176,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "id": "22", "metadata": {}, "outputs": [], @@ -308,7 +3186,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "id": "23", "metadata": {}, "outputs": [], @@ -327,10 +3205,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "id": "25", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'bwtDjLOoyURy'" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# This is necessary as it sets the new token value in user.reset_token\n", "token = reset_password_user.get_token()\n", @@ -339,10 +3228,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "id": "26", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'R&y3Yqs2$1'" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# This is necessary as it sets the new password value in user.new_password\n", "passw = reset_password_user.make_new_password()\n", @@ -351,7 +3251,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "id": "27", "metadata": {}, "outputs": [], @@ -362,10 +3262,853 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "id": "28", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
SyftSuccess: 
User Password updated successfully.

" + ], + "text/plain": [ + "SyftSuccess: User Password updated successfully." + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "output = reset_password_user.client.guest().reset_password(\n", " token=reset_password_user.reset_token, new_password=reset_password_user.new_password\n", @@ -375,7 +4118,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "id": "29", "metadata": {}, "outputs": [], @@ -385,10 +4128,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "id": "30", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Logged into as \n" + ] + } + ], "source": [ "# relogin\n", "reset_password_user.relogin()\n", @@ -405,20 +4156,1706 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 29, "id": "32", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
SyftSuccess: 
If the email is valid, we sent a password reset token to your email or a password request to the admin.

" + ], + "text/plain": [ + "SyftSuccess: If the email is valid, we sent a password reset token to your email or a password request to the admin." + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "reset_password_user.client.guest().forgot_password(email=reset_password_user.email)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 30, "id": "33", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
SyftSuccess: 
User Password updated successfully.

" + ], + "text/plain": [ + "SyftSuccess: User Password updated successfully." + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "output = reset_password_user.client.guest().reset_password(\n", " token=reset_password_user.get_token(),\n", @@ -429,7 +5866,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 31, "id": "34", "metadata": {}, "outputs": [], @@ -439,7 +5876,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 32, "id": "35", "metadata": {}, "outputs": [], @@ -451,7 +5888,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 33, "id": "36", "metadata": {}, "outputs": [], @@ -461,7 +5898,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 34, "id": "37", "metadata": {}, "outputs": [], @@ -473,10 +5910,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 35, "id": "38", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Logged into as \n" + ] + } + ], "source": [ "# relogin\n", "reset_password_user.relogin()" @@ -484,20 +5929,39 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 36, "id": "39", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[{'name': 'Carrie Huffman', 'email': 'carrie-huffman-fake@openmined.org', 'password': 'N!hLj0RF8v', 'role': 'ServiceRole.DATA_SCIENTIST', 'new_password': None, 'email_disabled': True, 'reset_password': False, 'reset_token': None}, {'name': 'Brent Robertson MD', 'email': 'brent-robertson-md-fake@openmined.org', 'password': '&cyxJ*9z94', 'role': 'ServiceRole.DATA_SCIENTIST', 'new_password': None, 'email_disabled': False, 'reset_password': True, 'reset_token': 'WOhBeOaE7TuQ'}, {'name': 'Lauren Velez', 'email': 'lauren-velez-fake@openmined.org', 'password': '_NJ9&yTne@', 'role': 'ServiceRole.DATA_SCIENTIST', 'new_password': None, 'email_disabled': False, 'reset_password': False, 'reset_token': None}, {'name': 'Daniel Moore', 'email': 'daniel-moore-fake@openmined.org', 'password': '_IfXa_*f8C', 'role': 'ServiceRole.DATA_SCIENTIST', 'new_password': None, 'email_disabled': False, 'reset_password': False, 'reset_token': None}, {'name': 'Shane Wheeler', 'email': 'shane-wheeler-fake@openmined.org', 'password': '42XIg#Ua!5', 'role': 'ServiceRole.DATA_SCIENTIST', 'new_password': None, 'email_disabled': False, 'reset_password': False, 'reset_token': None}]\n" + ] + } + ], "source": [ "save_users(users)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 37, "id": "40", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "TestUser(name='Brent Robertson MD', email='brent-robertson-md-fake@openmined.org', password='&cyxJ*9z94', role=, new_password=None, email_disabled=False, reset_password=True, reset_token='WOhBeOaE7TuQ', _email_server=)" + ] + }, + "execution_count": 37, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "reset_password_user" ] @@ -512,7 +5976,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 38, "id": "42", "metadata": {}, "outputs": [], @@ -524,17 +5988,860 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 39, "id": "43", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
SyftSuccess: 
If the email is valid, we sent a password reset token to your email or a password request to the admin.

" + ], + "text/plain": [ + "SyftSuccess: If the email is valid, we sent a password reset token to your email or a password request to the admin." + ] + }, + "execution_count": 39, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "reset_password_user.client.guest().forgot_password(email=reset_password_user.email)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 40, "id": "44", "metadata": {}, "outputs": [], @@ -545,10 +6852,853 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 41, "id": "45", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
SyftError: 
Failed to reset user password. Token is invalid or expired.

" + ], + "text/plain": [ + "SyftError: Failed to reset user password. Token is invalid or expired." + ] + }, + "execution_count": 41, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# This should throw a SyftError because we waited too long\n", "output = reset_password_user.client.guest().reset_password(\n", @@ -560,7 +7710,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 42, "id": "46", "metadata": {}, "outputs": [], @@ -570,10 +7720,93 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 43, "id": "47", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "with sy.raises successfully caught the following exception:\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + " \n", + " \n", + " SyftException:\n", + " 
Invalid credentials.
\n", + " \n", + " \n", + "
\n", + " \n", + "
Client Trace:
\n", + " 
\n",
+       "      \n",
+       "        Traceback (most recent call last):\n",
+       "  File \"/Users/brendanschell/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/packages/syft/src/syft/client/client.py, line 942, in login\n",
+       "    user_private_key = self.connection.login(email=email, password=password)\n",
+       "                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+       "  File \"/Users/brendanschell/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/packages/syft/src/syft/client/client.py, line 394, in login\n",
+       "    response = post_process_result(response, unwrap_on_success=True)\n",
+       "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+       "  File \"/Users/brendanschell/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/packages/syft/src/syft/client/api.py, line 92, in post_process_result\n",
+       "    raise SyftException(public_message=result.message, server_trace=result.tb)\n",
+       "syft.types.errors.SyftException: Invalid credentials.\n",
+       "\n",
+       "During handling of the above exception, another exception occurred:\n",
+       "\n",
+       "Traceback (most recent call last):\n",
+       "  File \"/var/folders/mt/hh7shybx3xq2888mhg60t29h0000gn/T/ipykernel_27171/3514082311.py, line 3, in \n",
+       "    reset_password_user.relogin()\n",
+       "  File \"/Users/brendanschell/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/notebooks/scenarios/bigquery/helpers.py, line 121, in relogin\n",
+       "    self.client = self.client\n",
+       "    ^^^^^^^^^^^\n",
+       "  File \"/Users/brendanschell/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/notebooks/scenarios/bigquery/helpers.py, line 125, in client\n",
+       "    client = client.login(email=self.email, password=self.latest_password)\n",
+       "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+       "  File \"/Users/brendanschell/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/packages/syft/src/syft/client/client.py, line 944, in login\n",
+       "    raise SyftException(public_message=e.public_message)\n",
+       "syft.types.errors.SyftException: Invalid credentials.\n",
+       "\n",
+       "      \n",
+       "
\n", + "
\n", + "
\n", + "\n", + "" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "# relogin\n", "with sy.raises(sy.SyftException, show=True):\n", @@ -582,7 +7815,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 44, "id": "48", "metadata": {}, "outputs": [], @@ -602,20 +7835,36 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 45, "id": "50", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> Stopping SMTPTestServer\n" + ] + } + ], "source": [ "smtp_server.stop()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 46, "id": "51", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Shutdown not implemented for the deployment type:remote\n" + ] + } + ], "source": [ "server.land()" ] @@ -630,6 +7879,11 @@ } ], "metadata": { + "kernelspec": { + "display_name": "syft", + "language": "python", + "name": "python3" + }, "language_info": { "codemirror_mode": { "name": "ipython", @@ -640,7 +7894,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.5" + "version": "3.12.4" } }, "nbformat": 4, diff --git a/notebooks/scenarios/bigquery/020-configure-api.ipynb b/notebooks/scenarios/bigquery/020-configure-api.ipynb index c3ad678a00b..a60110218ce 100644 --- a/notebooks/scenarios/bigquery/020-configure-api.ipynb +++ b/notebooks/scenarios/bigquery/020-configure-api.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -14,7 +14,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -24,9 +24,40 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], + "source": [ + "import os\n", + "\n", + "os.environ['ORCHESTRA_DEPLOYMENT_TYPE'] = 'remote'\n", + "os.environ['DEVSPACE_PROFILE'] = 'bigquery-scenario-tests'\n", + "os.environ['GITHUB_CI'] = 'false'\n", + "os.environ['SYFT_BASE_IMAGE_REGISTRY'] = 'k3d-registry.localhost:5800'\n", + "os.environ['DATASITE_CLUSTER_NAME'] = 'test-datasite-1'\n", + "os.environ['SERVER_URL'] = 'http://localhost'\n", + "os.environ['SERVER_PORT'] = '9081'\n", + "os.environ['TEST_EXTERNAL_REGISTRY'] = 'k3d-registry.localhost:5800'\n", + "os.environ['TEST_QUERY_LIMIT_SIZE'] = '500000'\n", + "os.environ['TRACING'] = 'False'\n", + "os.environ['NUM_TEST_USERS'] = '5'\n", + "os.environ['NUM_TEST_JOBS'] = '10'\n", + "os.environ['NUM_TEST_WORKERS'] = '2'" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Using Mock API Code, this will query BigQuery. $TEST_BIGQUERY_APIS_LIVE==False\n" + ] + } + ], "source": [ "# third party\n", "from apis import make_schema\n", @@ -43,7 +74,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -59,7 +90,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -67,7 +98,7 @@ " name=\"bigquery-high\",\n", " dev_mode=True,\n", " server_side_type=\"high\",\n", - " port=\"8080\",\n", + " port=\"9081\",\n", " n_consumers=1, # How many workers to be spawned\n", " create_producer=True, # Can produce more workers\n", ")" @@ -75,27 +106,50 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> Starting SMTPTestServer on: 0.0.0.0:9025\n" + ] + } + ], "source": [ "email_server, smtp_server = get_email_server()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> Starting SMTPTestServer on: 0.0.0.0:9025\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Logged into as \n" + ] + } + ], "source": [ "high_client = sy.login(\n", - " url=\"http://localhost:8080\", email=ADMIN_EMAIL, password=ADMIN_PW\n", + " url=\"http://localhost:9081\", email=ADMIN_EMAIL, password=ADMIN_PW\n", ")" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -104,7 +158,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -120,7 +174,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -129,7 +183,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -138,9 +192,2618 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "\n", + "
\n", + "
\n", + " \n", + "
\n", + "

WorkerPool Dicttuple

\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "

Total: 0

\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "\n", + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Look up the worker pools and identify the name of the one that has the required packages\n", "# After, bind the endpoint to that workerpool\n", @@ -149,7 +2812,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ @@ -163,7 +2826,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -176,9 +2839,852 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
SyftSuccess: 
Endpoint successfully created.

" + ], + "text/plain": [ + "SyftSuccess: Endpoint successfully created." + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "new_endpoint = sy.TwinAPIEndpoint(\n", " path=\"bigquery.test_query\",\n", @@ -200,9 +3706,852 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
SyftSuccess: 
Endpoint successfully updated.

" + ], + "text/plain": [ + "SyftSuccess: Endpoint successfully updated." + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Here, we update the endpoint to timeout after 100s (rather the default of 60s)\n", "high_client.api.services.api.update(\n", @@ -212,9 +4561,852 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
SyftSuccess: 
Endpoint successfully updated.

" + ], + "text/plain": [ + "SyftSuccess: Endpoint successfully updated." + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "high_client.api.services.api.update(\n", " endpoint_path=\"bigquery.test_query\", hide_mock_definition=True\n", @@ -230,7 +5422,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ @@ -244,9 +5436,119 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + " \n", + " \n", + " SyftException:\n", + " 
Function timed out in 120 seconds. Get the Job with id: 37cb41673fdf4fc1aa7761047b964ced to check results.
\n", + " \n", + " \n", + "
\n", + " \n", + "
Server Trace:
\n", + " 
\n",
+       "      \n",
+       "        Traceback (most recent call last):\n",
+       "  File \"/root/app/syft/src/syft/server/server.py, line 1271, in handle_api_call_with_unsigned_result\n",
+       "    result = method(context, *api_call.args, **api_call.kwargs)\n",
+       "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+       "  File \"/root/app/syft/src/syft/service/service.py, line 490, in _decorator\n",
+       "    result = func(self, *args, **kwargs)\n",
+       "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+       "  File \"/root/app/syft/src/syft/service/api/api_service.py, line 321, in call_public_in_jobs\n",
+       "    ).unwrap()\n",
+       "      ^^^^^^^^\n",
+       "  File \"/root/app/syft/src/syft/types/result.py, line 90, in unwrap\n",
+       "    raise self.value\n",
+       "  File \"/root/app/syft/src/syft/types/result.py, line 113, in wrapper\n",
+       "    output = func(*args, **kwargs)\n",
+       "             ^^^^^^^^^^^^^^^^^^^^^\n",
+       "  File \"/root/app/syft/src/syft/service/api/api_service.py, line 366, in _call_in_jobs\n",
+       "    raise SyftException(\n",
+       "syft.types.errors.SyftException: Function timed out in 120 seconds. Get the Job with id: 37cb41673fdf4fc1aa7761047b964ced to check results.\n",
+       "\n",
+       "      \n",
+       "
\n", + "
\n", + "
\n", + " \n", + "
Client Trace:
\n", + " 
\n",
+       "      \n",
+       "        Traceback (most recent call last):\n",
+       "  File \"/Users/brendanschell/miniconda3/envs/syft/lib/python3.12/site-packages/IPython/core/interactiveshell.py, line 3577, in run_code\n",
+       "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
+       "  File \"/var/folders/mt/hh7shybx3xq2888mhg60t29h0000gn/T/ipykernel_27303/3350366737.py, line 2, in \n",
+       "    result = high_client.api.services.bigquery.test_query.mock(\n",
+       "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+       "  File \"/Users/brendanschell/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/packages/syft/src/syft/client/api.py, line 383, in __call__\n",
+       "    return remote_func.function_call(\n",
+       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+       "  File \"/Users/brendanschell/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/packages/syft/src/syft/client/api.py, line 371, in function_call\n",
+       "    return post_process_result(result, self.unwrap_on_success)\n",
+       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+       "syft.types.errors.SyftException: Function timed out in 120 seconds. Get the Job with id: 37cb41673fdf4fc1aa7761047b964ced to check results.\n",
+       "server_trace: Traceback (most recent call last):\n",
+       "  File \"/root/app/syft/src/syft/server/server.py, line 1271, in handle_api_call_with_unsigned_result\n",
+       "    result = method(context, *api_call.args, **api_call.kwargs)\n",
+       "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+       "  File \"/root/app/syft/src/syft/service/service.py, line 490, in _decorator\n",
+       "    result = func(self, *args, **kwargs)\n",
+       "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
+       "  File \"/root/app/syft/src/syft/service/api/api_service.py, line 321, in call_public_in_jobs\n",
+       "    ).unwrap()\n",
+       "      ^^^^^^^^\n",
+       "  File \"/root/app/syft/src/syft/types/result.py, line 90, in unwrap\n",
+       "    raise self.value\n",
+       "  File \"/root/app/syft/src/syft/types/result.py, line 113, in wrapper\n",
+       "    output = func(*args, **kwargs)\n",
+       "             ^^^^^^^^^^^^^^^^^^^^^\n",
+       "  File \"/root/app/syft/src/syft/service/api/api_service.py, line 366, in _call_in_jobs\n",
+       "    raise SyftException(\n",
+       "syft.types.errors.SyftException: Function timed out in 120 seconds. Get the Job with id: 37cb41673fdf4fc1aa7761047b964ced to check results.\n",
+       "\n",
+       "\n",
+       "      \n",
+       "
\n", + "
\n", + "
\n", + "\n", + "" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "# Test mock version\n", "result = high_client.api.services.bigquery.test_query.mock(\n", @@ -624,7 +5926,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.5" + "version": "3.12.4" } }, "nbformat": 4, diff --git a/notebooks/scenarios/bigquery/021-create-jobs.ipynb b/notebooks/scenarios/bigquery/021-create-jobs.ipynb index 3625af65c88..3f960662204 100644 --- a/notebooks/scenarios/bigquery/021-create-jobs.ipynb +++ b/notebooks/scenarios/bigquery/021-create-jobs.ipynb @@ -26,6 +26,30 @@ "# os.environ[\"TEST_BIGQUERY_APIS_LIVE\"] = str(use_live_bigquery)" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "99308c3e", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "os.environ['ORCHESTRA_DEPLOYMENT_TYPE'] = 'remote'\n", + "os.environ['DEVSPACE_PROFILE'] = 'bigquery-scenario-tests'\n", + "os.environ['GITHUB_CI'] = 'false'\n", + "os.environ['SYFT_BASE_IMAGE_REGISTRY'] = 'k3d-registry.localhost:5800'\n", + "os.environ['DATASITE_CLUSTER_NAME'] = 'test-datasite-1'\n", + "os.environ['SERVER_URL'] = 'http://localhost'\n", + "os.environ['SERVER_PORT'] = '9081'\n", + "os.environ['TEST_EXTERNAL_REGISTRY'] = 'k3d-registry.localhost:5800'\n", + "os.environ['TEST_QUERY_LIMIT_SIZE'] = '500000'\n", + "os.environ['TRACING'] = 'False'\n", + "os.environ['NUM_TEST_USERS'] = '5'\n", + "os.environ['NUM_TEST_JOBS'] = '10'\n", + "os.environ['NUM_TEST_WORKERS'] = '2'" + ] + }, { "cell_type": "code", "execution_count": null, @@ -63,7 +87,7 @@ "metadata": {}, "outputs": [], "source": [ - "SERVER_PORT = \"8080\"\n", + "SERVER_PORT = \"9081\"\n", "SERVER_URL = f\"http://localhost:{SERVER_PORT}\"\n", "ADMIN_EMAIL, ADMIN_PW = \"admin2@bigquery.org\", \"bqpw2\"" ] @@ -111,7 +135,7 @@ "outputs": [], "source": [ "high_client = sy.login(\n", - " url=\"http://localhost:8080\", email=ADMIN_EMAIL, password=ADMIN_PW\n", + " url=\"http://localhost:9081\", email=ADMIN_EMAIL, password=ADMIN_PW\n", ")" ] }, diff --git a/notebooks/scenarios/bigquery/040-do-review-requests.ipynb b/notebooks/scenarios/bigquery/040-do-review-requests.ipynb index 9f0a301f2e3..e16cb5f3346 100644 --- a/notebooks/scenarios/bigquery/040-do-review-requests.ipynb +++ b/notebooks/scenarios/bigquery/040-do-review-requests.ipynb @@ -12,6 +12,29 @@ "# os.environ[\"TEST_EXTERNAL_REGISTRY\"] = \"k3d-registry.localhost:5800\"" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "os.environ['ORCHESTRA_DEPLOYMENT_TYPE'] = 'remote'\n", + "os.environ['DEVSPACE_PROFILE'] = 'bigquery-scenario-tests'\n", + "os.environ['GITHUB_CI'] = 'false'\n", + "os.environ['SYFT_BASE_IMAGE_REGISTRY'] = 'k3d-registry.localhost:5800'\n", + "os.environ['DATASITE_CLUSTER_NAME'] = 'test-datasite-1'\n", + "os.environ['SERVER_URL'] = 'http://localhost'\n", + "os.environ['SERVER_PORT'] = '9081'\n", + "os.environ['TEST_EXTERNAL_REGISTRY'] = 'k3d-registry.localhost:5800'\n", + "os.environ['TEST_QUERY_LIMIT_SIZE'] = '500000'\n", + "os.environ['TRACING'] = 'False'\n", + "os.environ['NUM_TEST_USERS'] = '5'\n", + "os.environ['NUM_TEST_JOBS'] = '10'\n", + "os.environ['NUM_TEST_WORKERS'] = '2'" + ] + }, { "cell_type": "code", "execution_count": null, @@ -58,7 +81,7 @@ " name=\"bigquery-high\",\n", " dev_mode=True,\n", " server_side_type=\"high\",\n", - " port=\"8080\",\n", + " port=\"9081\",\n", " n_consumers=1, # How many workers to be spawned\n", " create_producer=True, # Can produce more workers\n", ")" @@ -71,7 +94,7 @@ "outputs": [], "source": [ "high_client = sy.login(\n", - " url=\"http://localhost:8080\", email=ADMIN_EMAIL, password=ADMIN_PW\n", + " url=\"http://localhost:9081\", email=ADMIN_EMAIL, password=ADMIN_PW\n", ")" ] }, diff --git a/notebooks/scenarios/bigquery/050-ds-get-results.ipynb b/notebooks/scenarios/bigquery/050-ds-get-results.ipynb index 72ab42122c1..f126fc43749 100644 --- a/notebooks/scenarios/bigquery/050-ds-get-results.ipynb +++ b/notebooks/scenarios/bigquery/050-ds-get-results.ipynb @@ -12,6 +12,29 @@ "# os.environ[\"TEST_EXTERNAL_REGISTRY\"] = \"k3d-registry.localhost:5800\"" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "os.environ['ORCHESTRA_DEPLOYMENT_TYPE'] = 'remote'\n", + "os.environ['DEVSPACE_PROFILE'] = 'bigquery-scenario-tests'\n", + "os.environ['GITHUB_CI'] = 'false'\n", + "os.environ['SYFT_BASE_IMAGE_REGISTRY'] = 'k3d-registry.localhost:5800'\n", + "os.environ['DATASITE_CLUSTER_NAME'] = 'test-datasite-1'\n", + "os.environ['SERVER_URL'] = 'http://localhost'\n", + "os.environ['SERVER_PORT'] = '9081'\n", + "os.environ['TEST_EXTERNAL_REGISTRY'] = 'k3d-registry.localhost:5800'\n", + "os.environ['TEST_QUERY_LIMIT_SIZE'] = '500000'\n", + "os.environ['TRACING'] = 'False'\n", + "os.environ['NUM_TEST_USERS'] = '5'\n", + "os.environ['NUM_TEST_JOBS'] = '10'\n", + "os.environ['NUM_TEST_WORKERS'] = '2'" + ] + }, { "cell_type": "code", "execution_count": null, @@ -54,7 +77,7 @@ " name=\"bigquery-high\",\n", " dev_mode=True,\n", " server_side_type=\"high\",\n", - " port=\"8080\",\n", + " port=\"9081\",\n", " n_consumers=1, # How many workers to be spawned\n", " create_producer=True, # Can produce more workers\n", ")" @@ -67,7 +90,7 @@ "outputs": [], "source": [ "high_client = sy.login(\n", - " url=\"http://localhost:8080\", email=ADMIN_EMAIL, password=ADMIN_PW\n", + " url=\"http://localhost:9081\", email=ADMIN_EMAIL, password=ADMIN_PW\n", ")" ] }, diff --git a/packages/syft/src/syft/service/queue/queue.py b/packages/syft/src/syft/service/queue/queue.py index ae170cad95b..fb7d9b068ca 100644 --- a/packages/syft/src/syft/service/queue/queue.py +++ b/packages/syft/src/syft/service/queue/queue.py @@ -75,7 +75,10 @@ def monitor(self) -> None: ).unwrap() if job and job.status == JobStatus.TERMINATING: self.terminate(job) - for subjob in job.subjobs: + subjobs = self.worker.job_stash.get_by_parent_id( + self.credentials, job.id + ).unwrap() + for subjob in subjobs: self.terminate(subjob) self.queue_item.status = Status.INTERRUPTED diff --git a/packages/syft/src/syft/service/queue/zmq_producer.py b/packages/syft/src/syft/service/queue/zmq_producer.py index 5cb5056f8a2..38112aa2654 100644 --- a/packages/syft/src/syft/service/queue/zmq_producer.py +++ b/packages/syft/src/syft/service/queue/zmq_producer.py @@ -264,10 +264,11 @@ def purge_workers(self) -> None: self.delete_worker(worker, syft_worker.to_be_deleted) # relative - - self.auth_context.server.services.worker._delete( - self.auth_context, syft_worker - ) + # if worker has expired, then delete it. Otherwise, it should be handled by the monitor thread + # should also delete if monitor thread is not alive + """ self.auth_context.server.services.worker.delete( + self.auth_context, syft_worker, force=True + ) """ def update_consumer_state_for_worker( self, syft_worker_id: UID, consumer_state: ConsumerState diff --git a/packages/syft/src/syft/service/worker/worker_pool_service.py b/packages/syft/src/syft/service/worker/worker_pool_service.py index e495e0ecfc4..8a344506183 100644 --- a/packages/syft/src/syft/service/worker/worker_pool_service.py +++ b/packages/syft/src/syft/service/worker/worker_pool_service.py @@ -459,17 +459,17 @@ def scale( # get last "n" workers from pod list runner = KubernetesRunner() - k8s_pods = [ - pod.name for pod in runner.get_pool_pods(pool_name=worker_pool.name) - ] - k8s_pods_to_kill = k8s_pods[-(current_worker_count - number) :] - workers_to_delete = [ - worker for worker in workers if worker.name in k8s_pods_to_kill - ] + workers_to_delete = workers[ + -(current_worker_count - number) :] worker_service = context.server.services.worker + # update workers to to be deleted and wait for producer thread to call deletions for worker in workers_to_delete: - worker_service.delete(context=context, uid=worker.id, force=True) + worker.to_delete = True + + worker_stash = context.server.services.worker.stash + worker_stash.update(context.credentials) + #worker_service.delete(context=context, uid=worker.id, force=True) scale_kubernetes_pool( runner, diff --git a/packages/syft/src/syft/service/worker/worker_service.py b/packages/syft/src/syft/service/worker/worker_service.py index 625a88a46b4..0e717e8cb31 100644 --- a/packages/syft/src/syft/service/worker/worker_service.py +++ b/packages/syft/src/syft/service/worker/worker_service.py @@ -35,6 +35,7 @@ from .worker_pool import _get_worker_container from .worker_pool import _get_worker_container_status from .worker_stash import WorkerStash +from ..job.job_stash import JobStatus @serializable(canonical_name="WorkerService", version=1) @@ -144,11 +145,26 @@ def _delete( worker_pool = worker_pool_stash.get_by_name( credentials=context.credentials, pool_name=worker.worker_pool_name ).unwrap() + # remove the worker from the pool + try: + worker_linked_object = next( + obj for obj in worker_pool.worker_list if obj.object_uid == uid + ) + worker_pool.worker_list.remove(worker_linked_object) + except StopIteration: + pass + + # Delete worker from worker stash + self.stash.delete_by_uid(credentials=context.credentials, uid=uid).unwrap() + + # Update worker pool + + worker_pool_stash.update(context.credentials, obj=worker_pool).unwrap() if IN_KUBERNETES: # Kubernetes will only restart the worker NOT REMOVE IT runner = KubernetesRunner() - runner.delete_pod(pod_name=worker.name) + # runner.delete_pod(pod_name=worker.name) return SyftSuccess( # pod deletion is not supported in Kubernetes, removing and recreating the pod. message=( @@ -165,20 +181,7 @@ def _delete( # kill the in memory worker thread context.server.remove_consumer_with_id(syft_worker_id=worker.id) - # remove the worker from the pool - try: - worker_linked_object = next( - obj for obj in worker_pool.worker_list if obj.object_uid == uid - ) - worker_pool.worker_list.remove(worker_linked_object) - except StopIteration: - pass - - # Delete worker from worker stash - self.stash.delete_by_uid(credentials=context.credentials, uid=uid).unwrap() - - # Update worker pool - worker_pool_stash.update(context.credentials, obj=worker_pool).unwrap() + return SyftSuccess( message=f"Worker with id: {uid} deleted successfully from pool: {worker_pool.name}" diff --git a/packages/syft/src/syft/service/worker/worker_stash.py b/packages/syft/src/syft/service/worker/worker_stash.py index 41511457106..302395d0028 100644 --- a/packages/syft/src/syft/service/worker/worker_stash.py +++ b/packages/syft/src/syft/service/worker/worker_stash.py @@ -77,4 +77,7 @@ def update_consumer_state( worker.consumer_state = consumer_state if job_id is not None: worker.job_id = job_id + + if worker.consumer_state == ConsumerState.IDLE: + worker.job_id = None return self.update(credentials=credentials, obj=worker).unwrap() diff --git a/tox.ini b/tox.ini index 958dc2a8772..fd6cb5a8d21 100644 --- a/tox.ini +++ b/tox.ini @@ -361,7 +361,7 @@ setenv = TEST_NOTEBOOK_PATHS = {env:TEST_NOTEBOOK_PATHS:scenarios/bigquery} TEST_query_limit_size={env:test_query_limit_size:500000} SERVER_URL = {env:SERVER_URL:http://localhost} - SERVER_PORT = {env:SERVER_PORT:8080} + SERVER_PORT = {env:SERVER_PORT:9081} NUM_TEST_USERS = {env:NUM_TEST_USERS:5} NUM_TEST_JOBS = {env:NUM_TEST_JOBS:10} commands = @@ -427,7 +427,7 @@ setenv = SYFT_BASE_IMAGE_REGISTRY = {env:SYFT_BASE_IMAGE_REGISTRY:k3d-registry.localhost:5800} DATASITE_CLUSTER_NAME = {env:DATASITE_CLUSTER_NAME:test-datasite-1} SERVER_URL = {env:SERVER_URL:http://localhost} - SERVER_PORT = {env:SERVER_PORT:8080} + SERVER_PORT = {env:SERVER_PORT:9081} TEST_EXTERNAL_REGISTRY = {env:TEST_EXTERNAL_REGISTRY:k3d-registry.localhost:5800} TEST_QUERY_LIMIT_SIZE={env:TEST_QUERY_LIMIT_SIZE:500000} TRACING={env:TRACING:False} From 39afd6d2db9a62379217d059f5dd10e358237a1c Mon Sep 17 00:00:00 2001 From: Brendan Schell Date: Thu, 12 Sep 2024 10:52:21 -0400 Subject: [PATCH 11/11] revert notebook and tox changes from last commit --- ...tart-and-configure-server-and-admins.ipynb | 149 +- .../001-scale-delete-worker-pools.ipynb | 15722 +--------- .../bigquery/010-setup-bigquery-pool.ipynb | 25354 +--------------- .../bigquery/011-users-emails-passwords.ipynb | 7400 +---- .../bigquery/020-configure-api.ipynb | 5360 +--- .../scenarios/bigquery/021-create-jobs.ipynb | 28 +- .../bigquery/040-do-review-requests.ipynb | 27 +- .../bigquery/050-ds-get-results.ipynb | 27 +- tox.ini | 4 +- 9 files changed, 241 insertions(+), 53830 deletions(-) diff --git a/notebooks/scenarios/bigquery/000-start-and-configure-server-and-admins.ipynb b/notebooks/scenarios/bigquery/000-start-and-configure-server-and-admins.ipynb index 066ba974ee2..444ba2fe199 100644 --- a/notebooks/scenarios/bigquery/000-start-and-configure-server-and-admins.ipynb +++ b/notebooks/scenarios/bigquery/000-start-and-configure-server-and-admins.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -16,30 +16,7 @@ }, { "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "os.environ['ORCHESTRA_DEPLOYMENT_TYPE'] = 'remote'\n", - "os.environ['DEVSPACE_PROFILE'] = 'bigquery-scenario-tests'\n", - "os.environ['GITHUB_CI'] = 'false'\n", - "os.environ['SYFT_BASE_IMAGE_REGISTRY'] = 'k3d-registry.localhost:5800'\n", - "os.environ['DATASITE_CLUSTER_NAME'] = 'test-datasite-1'\n", - "os.environ['SERVER_URL'] = 'http://localhost'\n", - "os.environ['SERVER_PORT'] = '9081'\n", - "os.environ['TEST_EXTERNAL_REGISTRY'] = 'k3d-registry.localhost:5800'\n", - "os.environ['TEST_QUERY_LIMIT_SIZE'] = '500000'\n", - "os.environ['TRACING'] = 'False'\n", - "os.environ['NUM_TEST_USERS'] = '5'\n", - "os.environ['NUM_TEST_JOBS'] = '10'\n", - "os.environ['NUM_TEST_WORKERS'] = '2'" - ] - }, - { - "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -63,7 +40,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -74,7 +51,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -85,7 +62,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -94,7 +71,7 @@ " dev_mode=True,\n", " server_side_type=\"high\",\n", " reset=True,\n", - " port=\"9081\",\n", + " port=\"8080\",\n", " n_consumers=1, # How many workers to be spawned\n", " create_producer=True, # Can produce more workers\n", ")" @@ -102,88 +79,21 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "> Starting SMTPTestServer on: 0.0.0.0:9025\n" - ] - } - ], + "outputs": [], "source": [ "email_server, smtp_server = get_email_server(reset=True)" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:urllib3.connectionpool:Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused')': /api/v2/metadata\n", - "WARNING:urllib3.connectionpool:Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused')': /api/v2/metadata\n", - "WARNING:urllib3.connectionpool:Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused')': /api/v2/metadata\n" - ] - }, - { - "ename": "ConnectionError", - "evalue": "HTTPConnectionPool(host='localhost', port=8080): Max retries exceeded with url: /api/v2/metadata (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused'))", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mConnectionRefusedError\u001b[0m Traceback (most recent call last)", - "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/site-packages/urllib3/connection.py:196\u001b[0m, in \u001b[0;36mHTTPConnection._new_conn\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 195\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 196\u001b[0m sock \u001b[38;5;241m=\u001b[39m \u001b[43mconnection\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcreate_connection\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 197\u001b[0m \u001b[43m \u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_dns_host\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mport\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 198\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 199\u001b[0m \u001b[43m \u001b[49m\u001b[43msource_address\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msource_address\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 200\u001b[0m \u001b[43m \u001b[49m\u001b[43msocket_options\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msocket_options\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 201\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 202\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m socket\u001b[38;5;241m.\u001b[39mgaierror \u001b[38;5;28;01mas\u001b[39;00m e:\n", - "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/site-packages/urllib3/util/connection.py:85\u001b[0m, in \u001b[0;36mcreate_connection\u001b[0;34m(address, timeout, source_address, socket_options)\u001b[0m\n\u001b[1;32m 84\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 85\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m err\n\u001b[1;32m 86\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m 87\u001b[0m \u001b[38;5;66;03m# Break explicitly a reference cycle\u001b[39;00m\n", - "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/site-packages/urllib3/util/connection.py:73\u001b[0m, in \u001b[0;36mcreate_connection\u001b[0;34m(address, timeout, source_address, socket_options)\u001b[0m\n\u001b[1;32m 72\u001b[0m sock\u001b[38;5;241m.\u001b[39mbind(source_address)\n\u001b[0;32m---> 73\u001b[0m \u001b[43msock\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43msa\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 74\u001b[0m \u001b[38;5;66;03m# Break explicitly a reference cycle\u001b[39;00m\n", - "\u001b[0;31mConnectionRefusedError\u001b[0m: [Errno 61] Connection refused", - "\nThe above exception was the direct cause of the following exception:\n", - "\u001b[0;31mNewConnectionError\u001b[0m Traceback (most recent call last)", - "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/site-packages/urllib3/connectionpool.py:789\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[0m\n\u001b[1;32m 788\u001b[0m \u001b[38;5;66;03m# Make the request on the HTTPConnection object\u001b[39;00m\n\u001b[0;32m--> 789\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_make_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 790\u001b[0m \u001b[43m \u001b[49m\u001b[43mconn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 791\u001b[0m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 792\u001b[0m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 793\u001b[0m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 794\u001b[0m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 795\u001b[0m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 796\u001b[0m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 797\u001b[0m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mretries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 798\u001b[0m \u001b[43m \u001b[49m\u001b[43mresponse_conn\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mresponse_conn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 799\u001b[0m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 800\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 801\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mresponse_kw\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 802\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 804\u001b[0m \u001b[38;5;66;03m# Everything went great!\u001b[39;00m\n", - "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/site-packages/urllib3/connectionpool.py:495\u001b[0m, in \u001b[0;36mHTTPConnectionPool._make_request\u001b[0;34m(self, conn, method, url, body, headers, retries, timeout, chunked, response_conn, preload_content, decode_content, enforce_content_length)\u001b[0m\n\u001b[1;32m 494\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 495\u001b[0m \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 496\u001b[0m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 497\u001b[0m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 498\u001b[0m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 499\u001b[0m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 500\u001b[0m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 501\u001b[0m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 502\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 503\u001b[0m \u001b[43m \u001b[49m\u001b[43menforce_content_length\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43menforce_content_length\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 504\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 506\u001b[0m \u001b[38;5;66;03m# We are swallowing BrokenPipeError (errno.EPIPE) since the server is\u001b[39;00m\n\u001b[1;32m 507\u001b[0m \u001b[38;5;66;03m# legitimately able to close the connection after sending a valid response.\u001b[39;00m\n\u001b[1;32m 508\u001b[0m \u001b[38;5;66;03m# With this behaviour, the received response is still readable.\u001b[39;00m\n", - "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/site-packages/urllib3/connection.py:398\u001b[0m, in \u001b[0;36mHTTPConnection.request\u001b[0;34m(self, method, url, body, headers, chunked, preload_content, decode_content, enforce_content_length)\u001b[0m\n\u001b[1;32m 397\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mputheader(header, value)\n\u001b[0;32m--> 398\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mendheaders\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 400\u001b[0m \u001b[38;5;66;03m# If we're given a body we start sending that in chunks.\u001b[39;00m\n", - "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/http/client.py:1331\u001b[0m, in \u001b[0;36mHTTPConnection.endheaders\u001b[0;34m(self, message_body, encode_chunked)\u001b[0m\n\u001b[1;32m 1330\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m CannotSendHeader()\n\u001b[0;32m-> 1331\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_send_output\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmessage_body\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencode_chunked\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/http/client.py:1091\u001b[0m, in \u001b[0;36mHTTPConnection._send_output\u001b[0;34m(self, message_body, encode_chunked)\u001b[0m\n\u001b[1;32m 1090\u001b[0m \u001b[38;5;28;01mdel\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_buffer[:]\n\u001b[0;32m-> 1091\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmsg\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1093\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m message_body \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 1094\u001b[0m \n\u001b[1;32m 1095\u001b[0m \u001b[38;5;66;03m# create a consistent interface to message_body\u001b[39;00m\n", - "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/http/client.py:1035\u001b[0m, in \u001b[0;36mHTTPConnection.send\u001b[0;34m(self, data)\u001b[0m\n\u001b[1;32m 1034\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mauto_open:\n\u001b[0;32m-> 1035\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1036\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", - "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/site-packages/urllib3/connection.py:236\u001b[0m, in \u001b[0;36mHTTPConnection.connect\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 235\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mconnect\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 236\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msock \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_new_conn\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 237\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_tunnel_host:\n\u001b[1;32m 238\u001b[0m \u001b[38;5;66;03m# If we're tunneling it means we're connected to our proxy.\u001b[39;00m\n", - "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/site-packages/urllib3/connection.py:211\u001b[0m, in \u001b[0;36mHTTPConnection._new_conn\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 210\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m--> 211\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m NewConnectionError(\n\u001b[1;32m 212\u001b[0m \u001b[38;5;28mself\u001b[39m, \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mFailed to establish a new connection: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00me\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 213\u001b[0m ) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01me\u001b[39;00m\n\u001b[1;32m 215\u001b[0m \u001b[38;5;66;03m# Audit hooks are only available in Python 3.8+\u001b[39;00m\n", - "\u001b[0;31mNewConnectionError\u001b[0m: : Failed to establish a new connection: [Errno 61] Connection refused", - "\nThe above exception was the direct cause of the following exception:\n", - "\u001b[0;31mMaxRetryError\u001b[0m Traceback (most recent call last)", - "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/site-packages/requests/adapters.py:667\u001b[0m, in \u001b[0;36mHTTPAdapter.send\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m 666\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 667\u001b[0m resp \u001b[38;5;241m=\u001b[39m \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 668\u001b[0m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 669\u001b[0m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 670\u001b[0m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 671\u001b[0m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 672\u001b[0m \u001b[43m \u001b[49m\u001b[43mredirect\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 673\u001b[0m \u001b[43m \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 674\u001b[0m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 675\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 676\u001b[0m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmax_retries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 677\u001b[0m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 678\u001b[0m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 679\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 681\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (ProtocolError, \u001b[38;5;167;01mOSError\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m err:\n", - "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/site-packages/urllib3/connectionpool.py:873\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[0m\n\u001b[1;32m 870\u001b[0m log\u001b[38;5;241m.\u001b[39mwarning(\n\u001b[1;32m 871\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mRetrying (\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[38;5;124m) after connection broken by \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m, retries, err, url\n\u001b[1;32m 872\u001b[0m )\n\u001b[0;32m--> 873\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 874\u001b[0m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 875\u001b[0m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 876\u001b[0m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 877\u001b[0m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 878\u001b[0m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 879\u001b[0m \u001b[43m \u001b[49m\u001b[43mredirect\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 880\u001b[0m \u001b[43m \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 881\u001b[0m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 882\u001b[0m \u001b[43m \u001b[49m\u001b[43mpool_timeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpool_timeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 883\u001b[0m \u001b[43m \u001b[49m\u001b[43mrelease_conn\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrelease_conn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 884\u001b[0m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 885\u001b[0m \u001b[43m \u001b[49m\u001b[43mbody_pos\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbody_pos\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 886\u001b[0m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 887\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 888\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mresponse_kw\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 889\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 891\u001b[0m \u001b[38;5;66;03m# Handle redirect?\u001b[39;00m\n", - "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/site-packages/urllib3/connectionpool.py:873\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[0m\n\u001b[1;32m 870\u001b[0m log\u001b[38;5;241m.\u001b[39mwarning(\n\u001b[1;32m 871\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mRetrying (\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[38;5;124m) after connection broken by \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m, retries, err, url\n\u001b[1;32m 872\u001b[0m )\n\u001b[0;32m--> 873\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 874\u001b[0m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 875\u001b[0m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 876\u001b[0m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 877\u001b[0m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 878\u001b[0m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 879\u001b[0m \u001b[43m \u001b[49m\u001b[43mredirect\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 880\u001b[0m \u001b[43m \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 881\u001b[0m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 882\u001b[0m \u001b[43m \u001b[49m\u001b[43mpool_timeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpool_timeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 883\u001b[0m \u001b[43m \u001b[49m\u001b[43mrelease_conn\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrelease_conn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 884\u001b[0m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 885\u001b[0m \u001b[43m \u001b[49m\u001b[43mbody_pos\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbody_pos\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 886\u001b[0m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 887\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 888\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mresponse_kw\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 889\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 891\u001b[0m \u001b[38;5;66;03m# Handle redirect?\u001b[39;00m\n", - "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/site-packages/urllib3/connectionpool.py:873\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[0m\n\u001b[1;32m 870\u001b[0m log\u001b[38;5;241m.\u001b[39mwarning(\n\u001b[1;32m 871\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mRetrying (\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[38;5;124m) after connection broken by \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m, retries, err, url\n\u001b[1;32m 872\u001b[0m )\n\u001b[0;32m--> 873\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 874\u001b[0m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 875\u001b[0m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 876\u001b[0m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 877\u001b[0m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 878\u001b[0m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 879\u001b[0m \u001b[43m \u001b[49m\u001b[43mredirect\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 880\u001b[0m \u001b[43m \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 881\u001b[0m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 882\u001b[0m \u001b[43m \u001b[49m\u001b[43mpool_timeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpool_timeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 883\u001b[0m \u001b[43m \u001b[49m\u001b[43mrelease_conn\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrelease_conn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 884\u001b[0m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 885\u001b[0m \u001b[43m \u001b[49m\u001b[43mbody_pos\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbody_pos\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 886\u001b[0m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 887\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 888\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mresponse_kw\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 889\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 891\u001b[0m \u001b[38;5;66;03m# Handle redirect?\u001b[39;00m\n", - "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/site-packages/urllib3/connectionpool.py:843\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[0m\n\u001b[1;32m 841\u001b[0m new_e \u001b[38;5;241m=\u001b[39m ProtocolError(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mConnection aborted.\u001b[39m\u001b[38;5;124m\"\u001b[39m, new_e)\n\u001b[0;32m--> 843\u001b[0m retries \u001b[38;5;241m=\u001b[39m \u001b[43mretries\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mincrement\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 844\u001b[0m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43merror\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mnew_e\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_pool\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_stacktrace\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msys\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexc_info\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m2\u001b[39;49m\u001b[43m]\u001b[49m\n\u001b[1;32m 845\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 846\u001b[0m retries\u001b[38;5;241m.\u001b[39msleep()\n", - "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/site-packages/urllib3/util/retry.py:519\u001b[0m, in \u001b[0;36mRetry.increment\u001b[0;34m(self, method, url, response, error, _pool, _stacktrace)\u001b[0m\n\u001b[1;32m 518\u001b[0m reason \u001b[38;5;241m=\u001b[39m error \u001b[38;5;129;01mor\u001b[39;00m ResponseError(cause)\n\u001b[0;32m--> 519\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m MaxRetryError(_pool, url, reason) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mreason\u001b[39;00m \u001b[38;5;66;03m# type: ignore[arg-type]\u001b[39;00m\n\u001b[1;32m 521\u001b[0m log\u001b[38;5;241m.\u001b[39mdebug(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mIncremented Retry for (url=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m): \u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[38;5;124m\"\u001b[39m, url, new_retry)\n", - "\u001b[0;31mMaxRetryError\u001b[0m: HTTPConnectionPool(host='localhost', port=8080): Max retries exceeded with url: /api/v2/metadata (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused'))", - "\nDuring handling of the above exception, another exception occurred:\n", - "\u001b[0;31mConnectionError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[8], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m root_client \u001b[38;5;241m=\u001b[39m \u001b[43msy\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlogin\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2\u001b[0m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mhttp://localhost:8080\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43memail\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mROOT_EMAIL\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpassword\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mROOT_PASSWORD\u001b[49m\n\u001b[1;32m 3\u001b[0m \u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/packages/syft/src/syft/client/client.py:1188\u001b[0m, in \u001b[0;36mlogin\u001b[0;34m(email, url, port, server, password, cache)\u001b[0m\n\u001b[1;32m 1178\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mlogin\u001b[39m(\n\u001b[1;32m 1179\u001b[0m email: \u001b[38;5;28mstr\u001b[39m,\n\u001b[1;32m 1180\u001b[0m \u001b[38;5;66;03m# HTTPConnection\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1186\u001b[0m cache: \u001b[38;5;28mbool\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 1187\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m SyftClient:\n\u001b[0;32m-> 1188\u001b[0m _client \u001b[38;5;241m=\u001b[39m \u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1189\u001b[0m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1190\u001b[0m \u001b[43m \u001b[49m\u001b[43mserver\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mserver\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1191\u001b[0m \u001b[43m \u001b[49m\u001b[43mport\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mport\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1192\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1194\u001b[0m connection \u001b[38;5;241m=\u001b[39m _client\u001b[38;5;241m.\u001b[39mconnection\n\u001b[1;32m 1196\u001b[0m login_credentials \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", - "File \u001b[0;32m~/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/packages/syft/src/syft/client/client.py:1132\u001b[0m, in \u001b[0;36mconnect\u001b[0;34m(url, server, port)\u001b[0m\n\u001b[1;32m 1129\u001b[0m url\u001b[38;5;241m.\u001b[39mset_port(\u001b[38;5;28mint\u001b[39m(port))\n\u001b[1;32m 1130\u001b[0m connection \u001b[38;5;241m=\u001b[39m HTTPConnection(url\u001b[38;5;241m=\u001b[39murl)\n\u001b[0;32m-> 1132\u001b[0m client_type \u001b[38;5;241m=\u001b[39m \u001b[43mconnection\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_client_type\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39munwrap()\n\u001b[1;32m 1133\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m client_type(connection\u001b[38;5;241m=\u001b[39mconnection)\n", - "File \u001b[0;32m~/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/packages/syft/src/syft/types/result.py:113\u001b[0m, in \u001b[0;36mas_result..decorator..wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 109\u001b[0m \u001b[38;5;129m@exclude_from_traceback\u001b[39m\n\u001b[1;32m 110\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(func)\n\u001b[1;32m 111\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwrapper\u001b[39m(\u001b[38;5;241m*\u001b[39margs: P\u001b[38;5;241m.\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs: P\u001b[38;5;241m.\u001b[39mkwargs) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Result[T, BE]:\n\u001b[1;32m 112\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 113\u001b[0m output \u001b[38;5;241m=\u001b[39m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 114\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(output, Ok) \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(output, Err):\n\u001b[1;32m 115\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m _AsResultError(\n\u001b[1;32m 116\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mFunctions decorated with `as_result` should not return Result.\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 117\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mDid you forget to unwrap() the result?\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 118\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mresult: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00moutput\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 119\u001b[0m )\n", - "File \u001b[0;32m~/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/packages/syft/src/syft/client/client.py:496\u001b[0m, in \u001b[0;36mHTTPConnection.get_client_type\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 493\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01menclave_client\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m EnclaveClient\n\u001b[1;32m 494\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mgateway_client\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m GatewayClient\n\u001b[0;32m--> 496\u001b[0m metadata \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_server_metadata\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcredentials\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mSyftSigningKey\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgenerate\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 497\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m metadata\u001b[38;5;241m.\u001b[39mserver_type \u001b[38;5;241m==\u001b[39m ServerType\u001b[38;5;241m.\u001b[39mDATASITE\u001b[38;5;241m.\u001b[39mvalue:\n\u001b[1;32m 498\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m DatasiteClient\n", - "File \u001b[0;32m~/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/packages/syft/src/syft/client/client.py:342\u001b[0m, in \u001b[0;36mHTTPConnection.get_server_metadata\u001b[0;34m(self, credentials)\u001b[0m\n\u001b[1;32m 340\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m response\n\u001b[1;32m 341\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 342\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_make_get\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mroutes\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mROUTE_METADATA\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvalue\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 343\u001b[0m metadata_json \u001b[38;5;241m=\u001b[39m json\u001b[38;5;241m.\u001b[39mloads(response)\n\u001b[1;32m 344\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m ServerMetadataJSON(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mmetadata_json)\n", - "File \u001b[0;32m~/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/packages/syft/src/syft/client/client.py:204\u001b[0m, in \u001b[0;36mHTTPConnection._make_get\u001b[0;34m(self, path, params, stream)\u001b[0m\n\u001b[1;32m 200\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_make_get\u001b[39m(\n\u001b[1;32m 201\u001b[0m \u001b[38;5;28mself\u001b[39m, path: \u001b[38;5;28mstr\u001b[39m, params: \u001b[38;5;28mdict\u001b[39m \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m, stream: \u001b[38;5;28mbool\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m 202\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mbytes\u001b[39m \u001b[38;5;241m|\u001b[39m Iterable:\n\u001b[1;32m 203\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m params \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 204\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_make_get_no_params\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 206\u001b[0m url \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39murl\n\u001b[1;32m 208\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrtunnel_token:\n", - "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/site-packages/cachetools/__init__.py:737\u001b[0m, in \u001b[0;36mcached..decorator..wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 735\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m:\n\u001b[1;32m 736\u001b[0m \u001b[38;5;28;01mpass\u001b[39;00m \u001b[38;5;66;03m# key not found\u001b[39;00m\n\u001b[0;32m--> 737\u001b[0m v \u001b[38;5;241m=\u001b[39m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 738\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 739\u001b[0m cache[k] \u001b[38;5;241m=\u001b[39m v\n", - "File \u001b[0;32m~/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/packages/syft/src/syft/client/client.py:244\u001b[0m, in \u001b[0;36mHTTPConnection._make_get_no_params\u001b[0;34m(self, path, stream)\u001b[0m\n\u001b[1;32m 240\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mheaders[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mHost\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39murl\u001b[38;5;241m.\u001b[39mhost_or_ip\n\u001b[1;32m 242\u001b[0m url \u001b[38;5;241m=\u001b[39m url\u001b[38;5;241m.\u001b[39mwith_path(path)\n\u001b[0;32m--> 244\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msession\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 245\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mstr\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 246\u001b[0m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 247\u001b[0m \u001b[43m \u001b[49m\u001b[43mverify\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mverify_tls\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 248\u001b[0m \u001b[43m \u001b[49m\u001b[43mproxies\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m{\u001b[49m\u001b[43m}\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 249\u001b[0m \u001b[43m \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 250\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 251\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m response\u001b[38;5;241m.\u001b[39mstatus_code \u001b[38;5;241m!=\u001b[39m \u001b[38;5;241m200\u001b[39m:\n\u001b[1;32m 252\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m requests\u001b[38;5;241m.\u001b[39mConnectionError(\n\u001b[1;32m 253\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mFailed to fetch \u001b[39m\u001b[38;5;132;01m{\u001b[39;00murl\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m. Response returned with code \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mresponse\u001b[38;5;241m.\u001b[39mstatus_code\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 254\u001b[0m )\n", - "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/site-packages/requests/sessions.py:602\u001b[0m, in \u001b[0;36mSession.get\u001b[0;34m(self, url, **kwargs)\u001b[0m\n\u001b[1;32m 594\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124;03m\"\"\"Sends a GET request. Returns :class:`Response` object.\u001b[39;00m\n\u001b[1;32m 595\u001b[0m \n\u001b[1;32m 596\u001b[0m \u001b[38;5;124;03m:param url: URL for the new :class:`Request` object.\u001b[39;00m\n\u001b[1;32m 597\u001b[0m \u001b[38;5;124;03m:param \\*\\*kwargs: Optional arguments that ``request`` takes.\u001b[39;00m\n\u001b[1;32m 598\u001b[0m \u001b[38;5;124;03m:rtype: requests.Response\u001b[39;00m\n\u001b[1;32m 599\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 601\u001b[0m kwargs\u001b[38;5;241m.\u001b[39msetdefault(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mallow_redirects\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[0;32m--> 602\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mGET\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/site-packages/requests/sessions.py:589\u001b[0m, in \u001b[0;36mSession.request\u001b[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[1;32m 584\u001b[0m send_kwargs \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m 585\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtimeout\u001b[39m\u001b[38;5;124m\"\u001b[39m: timeout,\n\u001b[1;32m 586\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mallow_redirects\u001b[39m\u001b[38;5;124m\"\u001b[39m: allow_redirects,\n\u001b[1;32m 587\u001b[0m }\n\u001b[1;32m 588\u001b[0m send_kwargs\u001b[38;5;241m.\u001b[39mupdate(settings)\n\u001b[0;32m--> 589\u001b[0m resp \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprep\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43msend_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 591\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m resp\n", - "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/site-packages/requests/sessions.py:703\u001b[0m, in \u001b[0;36mSession.send\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m 700\u001b[0m start \u001b[38;5;241m=\u001b[39m preferred_clock()\n\u001b[1;32m 702\u001b[0m \u001b[38;5;66;03m# Send the request\u001b[39;00m\n\u001b[0;32m--> 703\u001b[0m r \u001b[38;5;241m=\u001b[39m \u001b[43madapter\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 705\u001b[0m \u001b[38;5;66;03m# Total elapsed time of the request (approximately)\u001b[39;00m\n\u001b[1;32m 706\u001b[0m elapsed \u001b[38;5;241m=\u001b[39m preferred_clock() \u001b[38;5;241m-\u001b[39m start\n", - "File \u001b[0;32m~/miniconda3/envs/syft/lib/python3.12/site-packages/requests/adapters.py:700\u001b[0m, in \u001b[0;36mHTTPAdapter.send\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m 696\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(e\u001b[38;5;241m.\u001b[39mreason, _SSLError):\n\u001b[1;32m 697\u001b[0m \u001b[38;5;66;03m# This branch is for urllib3 v1.22 and later.\u001b[39;00m\n\u001b[1;32m 698\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m SSLError(e, request\u001b[38;5;241m=\u001b[39mrequest)\n\u001b[0;32m--> 700\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m(e, request\u001b[38;5;241m=\u001b[39mrequest)\n\u001b[1;32m 702\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m ClosedPoolError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m 703\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m(e, request\u001b[38;5;241m=\u001b[39mrequest)\n", - "\u001b[0;31mConnectionError\u001b[0m: HTTPConnectionPool(host='localhost', port=8080): Max retries exceeded with url: /api/v2/metadata (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused'))" - ] - } - ], + "outputs": [], "source": [ "root_client = sy.login(\n", - " url=\"http://localhost:9081\", email=ROOT_EMAIL, password=ROOT_PASSWORD\n", + " url=\"http://localhost:8080\", email=ROOT_EMAIL, password=ROOT_PASSWORD\n", ")" ] }, @@ -232,7 +142,7 @@ "outputs": [], "source": [ "high_client = sy.login(\n", - " url=\"http://localhost:9081\", email=ADMIN_EMAIL, password=ADMIN_PW\n", + " url=\"http://localhost:8080\", email=ADMIN_EMAIL, password=ADMIN_PW\n", ")" ] }, @@ -313,34 +223,6 @@ "server.land()" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, { "cell_type": "code", "execution_count": null, @@ -350,11 +232,6 @@ } ], "metadata": { - "kernelspec": { - "display_name": "syft", - "language": "python", - "name": "python3" - }, "language_info": { "codemirror_mode": { "name": "ipython", @@ -365,7 +242,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.4" + "version": "3.12.5" } }, "nbformat": 4, diff --git a/notebooks/scenarios/bigquery/001-scale-delete-worker-pools.ipynb b/notebooks/scenarios/bigquery/001-scale-delete-worker-pools.ipynb index 175f5b86e22..884b77501ab 100644 --- a/notebooks/scenarios/bigquery/001-scale-delete-worker-pools.ipynb +++ b/notebooks/scenarios/bigquery/001-scale-delete-worker-pools.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "0", "metadata": {}, "outputs": [], @@ -16,31 +16,7 @@ }, { "cell_type": "code", - "execution_count": 2, - "id": "c89fb0b7", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "os.environ['ORCHESTRA_DEPLOYMENT_TYPE'] = 'remote'\n", - "os.environ['DEVSPACE_PROFILE'] = 'bigquery-scenario-tests'\n", - "os.environ['GITHUB_CI'] = 'false'\n", - "os.environ['SYFT_BASE_IMAGE_REGISTRY'] = 'k3d-registry.localhost:5800'\n", - "os.environ['DATASITE_CLUSTER_NAME'] = 'test-datasite-1'\n", - "os.environ['SERVER_URL'] = 'http://localhost'\n", - "os.environ['SERVER_PORT'] = '9081'\n", - "os.environ['TEST_EXTERNAL_REGISTRY'] = 'k3d-registry.localhost:5800'\n", - "os.environ['TEST_QUERY_LIMIT_SIZE'] = '500000'\n", - "os.environ['TRACING'] = 'False'\n", - "os.environ['NUM_TEST_USERS'] = '5'\n", - "os.environ['NUM_TEST_JOBS'] = '10'\n", - "os.environ['NUM_TEST_WORKERS'] = '2'" - ] - }, - { - "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "1", "metadata": {}, "outputs": [], @@ -58,7 +34,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "2", "metadata": {}, "outputs": [], @@ -68,7 +44,8 @@ "num_workers = int(os.environ.get(\"NUM_TEST_WORKERS\", 1))\n", "\n", "ROOT_EMAIL = \"admin@bigquery.org\"\n", - "ROOT_PASSWORD = \"bqpw\"" + "ROOT_PASSWORD = \"bqpw\"\n", + "environment" ] }, { @@ -81,7 +58,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "4", "metadata": {}, "outputs": [], @@ -90,7 +67,7 @@ " name=\"bigquery-high\",\n", " dev_mode=True,\n", " server_side_type=\"high\",\n", - " port=\"9081\",\n", + " port=\"8080\",\n", " n_consumers=num_workers, # How many workers to be spawned\n", " create_producer=True, # Can produce more workers\n", ")" @@ -98,6141 +75,42 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "5", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "> Starting SMTPTestServer on: 0.0.0.0:9025\n" - ] - } - ], + "outputs": [], "source": [ "email_server, smtp_server = get_email_server(reset=True)" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "6", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Logged into as \n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "> Starting SMTPTestServer on: 0.0.0.0:9025\n", - "> Error with SMTPTestServer: [Errno 48] error while attempting to bind on address ('0.0.0.0', 9025): address already in use\n" - ] - } - ], + "outputs": [], "source": [ "high_client = sy.login(\n", - " url=\"http://localhost:9081\", email=ROOT_EMAIL, password=ROOT_PASSWORD\n", + " url=\"http://localhost:8080\", email=ROOT_EMAIL, password=ROOT_PASSWORD\n", ")" ] }, { "cell_type": "code", - "execution_count": 8, - "id": "c8e1689f", + "execution_count": null, + "id": "7", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
\n", - "\n", - "
\n", - "
\n", - " \n", - "
\n", - "

WorkerPool Dicttuple

\n", - "
\n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n", - "

Total: 0

\n", - "
\n", - "
\n", - "
\n", - "\n", - "\n", - "\n", - "" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "high_client.worker_pools" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "8", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - " \n", - "
\n", - "

default-pool

\n", - "

\n", - " Created on: \n", - " 2024-09-11 20:07:15\n", - "

\n", - "

\n", - " Healthy Workers:\n", - " 1 / 1\n", - "

\n", - "

\n", - " Running Workers:\n", - " 1 / 1\n", - "

\n", - " \n", - "
\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
\n", - "\n", - "
\n", - "
\n", - " \n", - "
\n", - "

SyftWorker List

\n", - "
\n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n", - "

Total: 0

\n", - "
\n", - "
\n", - "
\n", - "\n", - "\n", - "\n", - "" - ], - "text/markdown": [ - "```python\n", - "class WorkerPool:\n", - " id: str = 1cc42404c3c54f9b9c6b226086ccccc6\n", - " name: str = \"default-pool\"\n", - " image: str = syft.service.worker.worker_image.SyftWorkerImage\n", - " max_count: str = 1\n", - " workers: str = [syft.service.worker.worker_pool.SyftWorker]\n", - " created_at: str = 2024-09-11 20:07:15\n", - "\n", - "```" - ], - "text/plain": [ - "syft.service.worker.worker_pool.WorkerPool" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "default_worker_pool = high_client.worker_pools.get_by_name(\"default-pool\")\n", "default_worker_pool" @@ -6256,7 +134,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "11", "metadata": {}, "outputs": [], @@ -6270,3483 +148,17 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "12", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - " \n", - "
\n", - "

default-pool

\n", - "

\n", - " Created on: \n", - " 2024-09-11 20:07:15\n", - "

\n", - "

\n", - " Healthy Workers:\n", - " 2 / 2\n", - "

\n", - "

\n", - " Running Workers:\n", - " 2 / 2\n", - "

\n", - " \n", - "
\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
\n", - "\n", - "
\n", - "
\n", - " \n", - "
\n", - "

SyftWorker List

\n", - "
\n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n", - "

Total: 0

\n", - "
\n", - "
\n", - "
\n", - "\n", - "\n", - "\n", - "" - ], - "text/markdown": [ - "```python\n", - "class WorkerPool:\n", - " id: str = 1cc42404c3c54f9b9c6b226086ccccc6\n", - " name: str = \"default-pool\"\n", - " image: str = syft.service.worker.worker_image.SyftWorkerImage\n", - " max_count: str = 2\n", - " workers: str = [syft.service.worker.worker_pool.SyftWorker, syft.service.worker.worker_pool.SyftWorker]\n", - " created_at: str = 2024-09-11 20:07:15\n", - "\n", - "```" - ], - "text/plain": [ - "syft.service.worker.worker_pool.WorkerPool" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "high_client.api.services.worker_pool[0]" ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "13", "metadata": {}, "outputs": [], @@ -9775,2534 +187,10 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "15", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
SyftSuccess: 
Syft function 'wait_1000_seconds_1' successfully created. To add a code request, please create a project using `project = syft.Project(...)`, then use command `project.create_code_request`.

" - ], - "text/plain": [ - "SyftSuccess: Syft function 'wait_1000_seconds_1' successfully created. To add a code request, please create a project using `project = syft.Project(...)`, then use command `project.create_code_request`." - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
SyftSuccess: 
Syft function 'wait_1000_seconds_2' successfully created. To add a code request, please create a project using `project = syft.Project(...)`, then use command `project.create_code_request`.

" - ], - "text/plain": [ - "SyftSuccess: Syft function 'wait_1000_seconds_2' successfully created. To add a code request, please create a project using `project = syft.Project(...)`, then use command `project.create_code_request`." - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
SyftSuccess: 
Syft function 'wait_1000_seconds_3' successfully created. To add a code request, please create a project using `project = syft.Project(...)`, then use command `project.create_code_request`.

" - ], - "text/plain": [ - "SyftSuccess: Syft function 'wait_1000_seconds_3' successfully created. To add a code request, please create a project using `project = syft.Project(...)`, then use command `project.create_code_request`." - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "@sy.syft_function_single_use(worker_pool_name=default_worker_pool.name)\n", "def wait_1000_seconds_1():\n", @@ -12330,20 +218,10 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "20", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Approving request on change wait_1000_seconds_3 for datasite syft-dev-server\n", - "Approving request on change wait_1000_seconds_2 for datasite syft-dev-server\n", - "Approving request on change wait_1000_seconds_1 for datasite syft-dev-server\n" - ] - } - ], + "outputs": [], "source": [ "jobs = []\n", "high_client.code.request_code_execution(wait_1000_seconds_1)\n", @@ -12357,7 +235,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "id": "21", "metadata": {}, "outputs": [], @@ -12373,24 +251,14 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "id": "22", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{}\n", - "{: , : , : }\n" - ] - } - ], + "outputs": [], "source": [ "# check that at least three workers have a job (since scaling down to 2)\n", "# try 3 times with a 5 second sleep in case it takes time for the workers to accept the jobs\n", - "import time\n", - "for _ in range(10):\n", + "for _ in range(3):\n", " worker_to_job_map = {}\n", " syft_workers_ids = set()\n", " for job in high_client.jobs:\n", @@ -12399,7 +267,7 @@ " worker_to_job_map[job.worker.id] = job.id\n", " print(worker_to_job_map)\n", " if len(syft_workers_ids) < 3:\n", - " time.sleep(5)\n", + " time.sleep(20)\n", " else:\n", " break\n", "assert len(syft_workers_ids) >= 3" @@ -12415,7 +283,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "id": "24", "metadata": {}, "outputs": [], @@ -12430,18 +298,10 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "id": "25", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Condition met, exiting early.\n" - ] - } - ], + "outputs": [], "source": [ "if environment == \"remote\":\n", "\n", @@ -12472,2626 +332,7 @@ }, { "cell_type": "code", - "execution_count": 29, - "id": "2773f0b6", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
\n", - "\n", - "
\n", - "
\n", - " \n", - "
\n", - "

Job List

\n", - "
\n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n", - "

Total: 0

\n", - "
\n", - "
\n", - "
\n", - "\n", - "\n", - "\n", - "" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 29, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "high_client.jobs" - ] - }, - { - "cell_type": "code", - "execution_count": 20, + "execution_count": null, "id": "26", "metadata": {}, "outputs": [], @@ -15113,853 +354,10 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "id": "28", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
SyftSuccess: 
Successfully deleted worker pool with id 1cc42404c3c54f9b9c6b226086ccccc6

" - ], - "text/plain": [ - "SyftSuccess: Successfully deleted worker pool with id 1cc42404c3c54f9b9c6b226086ccccc6" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "pool_delete_result = high_client.api.services.worker_pool.delete(\n", " pool_name=default_worker_pool.name\n", @@ -15969,7 +367,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "id": "29", "metadata": {}, "outputs": [], @@ -15980,22 +378,10 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "id": "30", "metadata": {}, - "outputs": [ - { - "ename": "AssertionError", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mAssertionError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[25], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;66;03m# check that all jobs are interrupted\u001b[39;00m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;66;03m# should be the case since the entire pool was deleted and all jobs were previously assigned\u001b[39;00m\n\u001b[0;32m----> 3\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28mall\u001b[39m(job\u001b[38;5;241m.\u001b[39mstatus \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124minterrupted\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m job \u001b[38;5;129;01min\u001b[39;00m high_client\u001b[38;5;241m.\u001b[39mjobs)\n", - "\u001b[0;31mAssertionError\u001b[0m: " - ] - } - ], + "outputs": [], "source": [ "# check that all jobs are interrupted\n", "# should be the case since the entire pool was deleted and all jobs were previously assigned\n", @@ -16012,7 +398,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "id": "32", "metadata": {}, "outputs": [], @@ -16022,7 +408,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": null, "id": "33", "metadata": {}, "outputs": [], @@ -16036,7 +422,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "id": "34", "metadata": {}, "outputs": [], @@ -16050,36 +436,20 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": null, "id": "35", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "> Stopping SMTPTestServer\n" - ] - } - ], + "outputs": [], "source": [ "smtp_server.stop()" ] }, { "cell_type": "code", - "execution_count": 29, + "execution_count": null, "id": "36", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Shutdown not implemented for the deployment type:remote\n" - ] - } - ], + "outputs": [], "source": [ "server.land()" ] diff --git a/notebooks/scenarios/bigquery/010-setup-bigquery-pool.ipynb b/notebooks/scenarios/bigquery/010-setup-bigquery-pool.ipynb index 625355e215c..1280a4e19d7 100644 --- a/notebooks/scenarios/bigquery/010-setup-bigquery-pool.ipynb +++ b/notebooks/scenarios/bigquery/010-setup-bigquery-pool.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -14,30 +14,7 @@ }, { "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "os.environ['ORCHESTRA_DEPLOYMENT_TYPE'] = 'remote'\n", - "os.environ['DEVSPACE_PROFILE'] = 'bigquery-scenario-tests'\n", - "os.environ['GITHUB_CI'] = 'false'\n", - "os.environ['SYFT_BASE_IMAGE_REGISTRY'] = 'k3d-registry.localhost:5800'\n", - "os.environ['DATASITE_CLUSTER_NAME'] = 'test-datasite-1'\n", - "os.environ['SERVER_URL'] = 'http://localhost'\n", - "os.environ['SERVER_PORT'] = '9081'\n", - "os.environ['TEST_EXTERNAL_REGISTRY'] = 'k3d-registry.localhost:5800'\n", - "os.environ['TEST_QUERY_LIMIT_SIZE'] = '500000'\n", - "os.environ['TRACING'] = 'False'\n", - "os.environ['NUM_TEST_USERS'] = '5'\n", - "os.environ['NUM_TEST_JOBS'] = '10'\n", - "os.environ['NUM_TEST_WORKERS'] = '2'" - ] - }, - { - "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -54,20 +31,9 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'remote'" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n", "environment" @@ -82,7 +48,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -90,7 +56,7 @@ " name=\"bigquery-high\",\n", " dev_mode=True,\n", " server_side_type=\"high\",\n", - " port=\"9081\",\n", + " port=\"8080\",\n", " n_consumers=1, # How many workers to be spawned\n", " create_producer=True, # Can produce more workers\n", ")" @@ -98,45 +64,22 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "> Starting SMTPTestServer on: 0.0.0.0:9025\n" - ] - } - ], + "outputs": [], "source": [ "_, smtp_server = get_email_server(reset=True)" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "> Starting SMTPTestServer on: 0.0.0.0:9025\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Logged into as \n" - ] - } - ], + "outputs": [], "source": [ "ADMIN_EMAIL, ADMIN_PW = \"admin2@bigquery.org\", \"bqpw2\"\n", "high_client = sy.login(\n", - " url=\"http://localhost:9081\", email=ADMIN_EMAIL, password=ADMIN_PW\n", + " url=\"http://localhost:8080\", email=ADMIN_EMAIL, password=ADMIN_PW\n", ")" ] }, @@ -156,2625 +99,16 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
\n", - "\n", - "
\n", - "
\n", - " \n", - "
\n", - "

WorkerPool Dicttuple

\n", - "
\n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n", - "

Total: 0

\n", - "
\n", - "
\n", - "
\n", - "\n", - "\n", - "\n", - "" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "high_client.worker_pools" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -2783,20 +117,9 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'k3d-registry.localhost:5800'" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "external_registry = test_settings.get(\"external_registry\", default=\"docker.io\")\n", "external_registry" @@ -2804,852 +127,9 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
SyftSuccess: 
Image Registry ID: 80fdf8ec3e0d4b28a403683bf9860e3b created successfully

" - ], - "text/plain": [ - "SyftSuccess: Image Registry ID: 80fdf8ec3e0d4b28a403683bf9860e3b created successfully" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "result = high_client.api.services.image_registry.add(external_registry)\n", "result" @@ -3657,2618 +137,9 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
\n", - "\n", - "
\n", - "
\n", - " \n", - "
\n", - "

SyftImageRegistry List

\n", - "
\n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n", - "

Total: 0

\n", - "
\n", - "
\n", - "
\n", - "\n", - "\n", - "\n", - "" - ], - "text/plain": [ - "[SyftImageRegistry(url=k3d-registry.localhost:5800)]" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "image_registry_list = high_client.api.services.image_registry.get_all()\n", "image_registry_list" @@ -6276,28 +147,9 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "```python\n", - "class SyftImageRegistry:\n", - " id: str = 80fdf8ec3e0d4b28a403683bf9860e3b\n", - " url: str = \"k3d-registry.localhost:5800\"\n", - "\n", - "```" - ], - "text/plain": [ - "SyftImageRegistry(url=k3d-registry.localhost:5800)" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "local_registry = image_registry_list[0]\n", "local_registry" @@ -6312,3447 +164,9 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - " \n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
\n", - "\n", - "
\n", - "
\n", - " \n", - "
\n", - "

SyftWorkerImage Dicttuple

\n", - "
\n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n", - "

Total: 0

\n", - "
\n", - "
\n", - "
\n", - "\n", - "\n", - "\n", - "" - ], - "text/plain": [ - "DictTuple(syft.service.worker.worker_image.SyftWorkerImage,)" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "dockerfile_list = high_client.images.get_all()\n", "dockerfile_list" @@ -9760,32 +174,9 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "```python\n", - "class SyftWorkerImage:\n", - " id: str = 77d3930961d341e5a154cf4022bd14d8\n", - " image_identifier: str = k3d-registry.localhost:5800/openmined/syft-backend:dev-1726083661\n", - " image_hash: str = None\n", - " created_at: str = 2024-09-11 19:42:10\n", - " built_at: str = None\n", - " config: str = prebuilt tag='k3d-registry.localhost:5800/openmined/syft-backend:dev-1726083661' description='Prebuilt default worker image'\n", - "\n", - "```" - ], - "text/plain": [ - "syft.service.worker.worker_image.SyftWorkerImage" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "base_worker_image = next(\n", " (\n", @@ -9800,40 +191,18 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "SyftWorkerImageIdentifier(repo=openmined/syft-backend, tag=dev-1726083661, registry=k3d-registry.localhost:5800)" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "base_worker_image.image_identifier" ] }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'FROM k3d-registry.localhost:5800/openmined/syft-backend:dev-1726083661\\n\\nRUN uv pip install db-dtypes google-cloud-bigquery'" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "worker_dockerfile = f\"\"\"\n", "FROM {str(base_worker_image.image_identifier)}\n", @@ -9846,7 +215,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -9856,852 +225,9 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
SyftSuccess: 
Dockerfile ID: 2987a404b77d405a8b923521103e1772 successfully submitted.

" - ], - "text/plain": [ - "SyftSuccess: Dockerfile ID: 2987a404b77d405a8b923521103e1772 successfully submitted." - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "submit_result = high_client.api.services.worker_image.submit(\n", " worker_config=docker_config\n", @@ -10711,3447 +237,9 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - " \n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
\n", - "\n", - "
\n", - "
\n", - " \n", - "
\n", - "

SyftWorkerImage Dicttuple

\n", - "
\n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n", - "

Total: 0

\n", - "
\n", - "
\n", - "
\n", - "\n", - "\n", - "\n", - "" - ], - "text/plain": [ - "DictTuple(syft.service.worker.worker_image.SyftWorkerImage, syft.service.worker.worker_image.SyftWorkerImage)" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "dockerfile_list = high_client.images.get_all()\n", "dockerfile_list" @@ -14159,34 +247,9 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "```python\n", - "class SyftWorkerImage:\n", - " id: str = 2987a404b77d405a8b923521103e1772\n", - " image_identifier: str = None\n", - " image_hash: str = None\n", - " created_at: str = 2024-09-11 19:42:10\n", - " built_at: str = None\n", - " config: str = FROM k3d-registry.localhost:5800/openmined/syft-backend:dev-1726083661\n", - "\n", - "RUN uv pip install db-dtypes google-cloud-bigquery\n", - "\n", - "```" - ], - "text/plain": [ - "syft.service.worker.worker_image.SyftWorkerImage" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "worker_image = next(\n", " (\n", @@ -14201,26 +264,16 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "FROM k3d-registry.localhost:5800/openmined/syft-backend:dev-1726083661\n", - "\n", - "RUN uv pip install db-dtypes google-cloud-bigquery\n" - ] - } - ], + "outputs": [], "source": [ "print(worker_image.config.dockerfile)" ] }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -14237,20 +290,9 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'k3d-registry.localhost:5800/openmined/syft-worker-bigquery:dev-1726083661'" - ] - }, - "execution_count": 24, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "docker_tag = str(base_worker_image.image_identifier).replace(\n", " \"backend\", \"worker-bigquery\"\n", @@ -14260,19 +302,9 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "SyftSuccess: Build for Worker ID: 2987a404b77d405a8b923521103e1772 succeeded.\n", - "----------Logs for pod=build-8397a8a888c2fd3e-np7l6----------\n", - "time=\"2024-09-11T19:46:35Z\" level=info msg=\"Retrieving image manifest k3d-registry.localhost:5800/openmined/syft-backend:dev-1726083661\"time=\"2024-09-11T19:46:35Z\" level=info msg=\"Retrieving image k3d-registry.localhost:5800/openmined/syft-backend:dev-1726083661 from registry k3d-registry.localhost:5800\"time=\"2024-09-11T19:46:35Z\" level=info msg=\"Retrieving image manifest k3d-registry.localhost:5800/openmined/syft-backend:dev-1726083661\"time=\"2024-09-11T19:46:35Z\" level=info msg=\"Returning cached image manifest\"time=\"2024-09-11T19:46:35Z\" level=info msg=\"Built cross stage deps: map[]\"time=\"2024-09-11T19:46:35Z\" level=info msg=\"Retrieving image manifest k3d-registry.localhost:5800/openmined/syft-backend:dev-1726083661\"time=\"2024-09-11T19:46:35Z\" level=info msg=\"Returning cached image manifest\"time=\"2024-09-11T19:46:35Z\" level=info msg=\"Retrieving image manifest k3d-registry.localhost:5800/openmined/syft-backend:dev-1726083661\"time=\"2024-09-11T19:46:35Z\" level=info msg=\"Returning cached image manifest\"time=\"2024-09-11T19:46:35Z\" level=info msg=\"Executing 0 build triggers\"time=\"2024-09-11T19:46:35Z\" level=info msg=\"Building stage 'k3d-registry.localhost:5800/openmined/syft-backend:dev-1726083661' [idx: '0', base-idx: '-1']\"time=\"2024-09-11T19:46:35Z\" level=info msg=\"Checking for cached layer registry.syft.svc.cluster.local/openmined/syft-worker-bigquery/cache:e4f3c9634d34d446ef66544b3ade0214b518061eab031c49d90a37354d73681d...\"time=\"2024-09-11T19:46:35Z\" level=info msg=\"No cached layer found for cmd RUN uv pip install db-dtypes google-cloud-bigquery\"time=\"2024-09-11T19:46:35Z\" level=info msg=\"Unpacking rootfs as cmd RUN uv pip install db-dtypes google-cloud-bigquery requires it.\"time=\"2024-09-11T19:46:56Z\" level=info msg=\"RUN uv pip install db-dtypes google-cloud-bigquery\"time=\"2024-09-11T19:46:56Z\" level=info msg=\"Initializing snapshotter ...\"time=\"2024-09-11T19:46:56Z\" level=info msg=\"Taking snapshot of full filesystem...\"time=\"2024-09-11T19:46:59Z\" level=info msg=\"Cmd: /bin/sh\"time=\"2024-09-11T19:46:59Z\" level=info msg=\"Args: [-c uv pip install db-dtypes google-cloud-bigquery]\"time=\"2024-09-11T19:46:59Z\" level=info msg=\"Util.Lookup returned: &{Uid:0 Gid:0 Username:root Name: HomeDir:/root}\"time=\"2024-09-11T19:46:59Z\" level=info msg=\"Performing slow lookup of group ids for root\"time=\"2024-09-11T19:46:59Z\" level=info msg=\"Running: [/bin/sh -c uv pip install db-dtypes google-cloud-bigquery]\"Resolved 29 packages in 690msDownloaded 12 packages in 102msInstalled 12 packages in 9ms + db-dtypes==1.3.0 + google-api-core==2.19.2 + google-auth==2.34.0 + google-cloud-bigquery==3.25.0 + google-cloud-core==2.4.1 + google-crc32c==1.6.0 + google-resumable-media==2.7.2 + grpcio-status==1.62.3 + proto-plus==1.24.0 + pyasn1==0.6.1 + pyasn1-modules==0.4.1 + rsa==4.9time=\"2024-09-11T19:47:00Z\" level=info msg=\"Taking snapshot of full filesystem...\"time=\"2024-09-11T19:47:01Z\" level=info msg=\"Pushing layer registry.syft.svc.cluster.local/openmined/syft-worker-bigquery/cache:e4f3c9634d34d446ef66544b3ade0214b518061eab031c49d90a37354d73681d to cache now\"time=\"2024-09-11T19:47:01Z\" level=info msg=\"Pushing image to registry.syft.svc.cluster.local/openmined/syft-worker-bigquery/cache:e4f3c9634d34d446ef66544b3ade0214b518061eab031c49d90a37354d73681d\"time=\"2024-09-11T19:47:01Z\" level=info msg=\"Pushed registry.syft.svc.cluster.local/openmined/syft-worker-bigquery/cache@sha256:119270096a158e05987329fe23c9bf6efd174e69b4b5f377955ec38691ad9601\"time=\"2024-09-11T19:47:01Z\" level=info msg=\"Pushing image to registry.syft.svc.cluster.local/openmined/syft-worker-bigquery:dev-1726083661\"time=\"2024-09-11T19:47:04Z\" level=info msg=\"Pushed registry.syft.svc.cluster.local/openmined/syft-worker-bigquery@sha256:925b1098fd91d8be3d310c355ddccbb3c0aca011a071f3cf1b28a9ff7ccc6340\"\n" - ] - } - ], + "outputs": [], "source": [ "if environment == \"remote\":\n", " docker_build_result = high_client.api.services.worker_image.build(\n", @@ -14285,17 +317,9 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "SyftSuccess: Pushed Image ID: 2987a404b77d405a8b923521103e1772 to \"k3d-registry.localhost:5800/openmined/syft-worker-bigquery:dev-1726083661\".\n" - ] - } - ], + "outputs": [], "source": [ "if environment == \"remote\":\n", " push_result = high_client.api.services.worker_image.push(worker_image.id)\n", @@ -14304,3447 +328,9 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - " \n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
\n", - "\n", - "
\n", - "
\n", - " \n", - "
\n", - "

SyftWorkerImage Dicttuple

\n", - "
\n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n", - "

Total: 0

\n", - "
\n", - "
\n", - "
\n", - "\n", - "\n", - "\n", - "" - ], - "text/plain": [ - "DictTuple(syft.service.worker.worker_image.SyftWorkerImage, syft.service.worker.worker_image.SyftWorkerImage)" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "dockerfile_list = high_client.images.get_all()\n", "dockerfile_list" @@ -17752,20 +338,9 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "PrebuiltWorkerConfig(tag='k3d-registry.localhost:5800/openmined/syft-worker-bigquery:dev-1726083661', description=None)" - ] - }, - "execution_count": 28, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "docker_config = sy.PrebuiltWorkerConfig(tag=docker_tag)\n", "docker_config" @@ -17773,852 +348,9 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
SyftSuccess: 
Dockerfile ID: c9b62bf8d060405b8a06891e68cf0b44 successfully submitted.

" - ], - "text/plain": [ - "SyftSuccess: Dockerfile ID: c9b62bf8d060405b8a06891e68cf0b44 successfully submitted." - ] - }, - "execution_count": 29, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# overwrite it for now Mongo ignore\n", "result = high_client.api.services.worker_image.submit(worker_config=docker_config)\n", @@ -18627,3447 +359,9 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - " \n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
\n", - "\n", - "
\n", - "
\n", - " \n", - "
\n", - "

SyftWorkerImage Dicttuple

\n", - "
\n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n", - "

Total: 0

\n", - "
\n", - "
\n", - "
\n", - "\n", - "\n", - "\n", - "" - ], - "text/plain": [ - "DictTuple(syft.service.worker.worker_image.SyftWorkerImage, syft.service.worker.worker_image.SyftWorkerImage)" - ] - }, - "execution_count": 30, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "dockerfile_list = high_client.images.get_all()\n", "dockerfile_list" @@ -22075,7 +369,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -22088,32 +382,9 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "```python\n", - "class SyftWorkerImage:\n", - " id: str = c9b62bf8d060405b8a06891e68cf0b44\n", - " image_identifier: str = k3d-registry.localhost:5800/openmined/syft-worker-bigquery:dev-1726083661\n", - " image_hash: str = None\n", - " created_at: str = 2024-09-11 19:42:10\n", - " built_at: str = None\n", - " config: str = prebuilt tag='k3d-registry.localhost:5800/openmined/syft-worker-bigquery:dev-1726083661'\n", - "\n", - "```" - ], - "text/plain": [ - "syft.service.worker.worker_image.SyftWorkerImage" - ] - }, - "execution_count": 32, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "worker_image = next(\n", " (\n", @@ -22128,7 +399,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -22137,7 +408,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -22157,2618 +428,9 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
\n", - "\n", - "
\n", - "
\n", - " \n", - "
\n", - "

ContainerSpawnStatus List

\n", - "
\n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n", - "

Total: 0

\n", - "
\n", - "
\n", - "
\n", - "\n", - "\n", - "\n", - "" - ], - "text/plain": [ - "[ContainerSpawnStatus(worker_name='bigquery-pool-0', worker=syft.service.worker.worker_pool.SyftWorker, error=None)]" - ] - }, - "execution_count": 35, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "result = high_client.api.services.worker_pool.launch(\n", " pool_name=worker_pool_name,\n", @@ -24789,17 +451,9 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "SyftSuccess: Worker pool scaled to 2 workers\n" - ] - } - ], + "outputs": [], "source": [ "if environment == \"remote\":\n", " result = high_client.worker_pools.scale(\n", @@ -24810,7 +464,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -24819,859 +473,16 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
SyftSuccess: 
Registration feature successfully disabled

" - ], - "text/plain": [ - "SyftSuccess: Registration feature successfully disabled" - ] - }, - "execution_count": 38, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "high_client.settings.allow_guest_signup(enable=False)" ] }, { "cell_type": "code", - "execution_count": 39, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -25687,34 +498,18 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "> Stopping SMTPTestServer\n" - ] - } - ], + "outputs": [], "source": [ "smtp_server.stop()" ] }, { "cell_type": "code", - "execution_count": 41, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Shutdown not implemented for the deployment type:remote\n" - ] - } - ], + "outputs": [], "source": [ "server.land()" ] @@ -25728,11 +523,6 @@ } ], "metadata": { - "kernelspec": { - "display_name": "syft", - "language": "python", - "name": "python3" - }, "language_info": { "codemirror_mode": { "name": "ipython", @@ -25743,7 +533,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.4" + "version": "3.12.5" } }, "nbformat": 4, diff --git a/notebooks/scenarios/bigquery/011-users-emails-passwords.ipynb b/notebooks/scenarios/bigquery/011-users-emails-passwords.ipynb index adc84cbfe5b..990f91183d8 100644 --- a/notebooks/scenarios/bigquery/011-users-emails-passwords.ipynb +++ b/notebooks/scenarios/bigquery/011-users-emails-passwords.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "0", "metadata": {}, "outputs": [], @@ -17,31 +17,7 @@ }, { "cell_type": "code", - "execution_count": 2, - "id": "28c5ab76", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "os.environ['ORCHESTRA_DEPLOYMENT_TYPE'] = 'remote'\n", - "os.environ['DEVSPACE_PROFILE'] = 'bigquery-scenario-tests'\n", - "os.environ['GITHUB_CI'] = 'false'\n", - "os.environ['SYFT_BASE_IMAGE_REGISTRY'] = 'k3d-registry.localhost:5800'\n", - "os.environ['DATASITE_CLUSTER_NAME'] = 'test-datasite-1'\n", - "os.environ['SERVER_URL'] = 'http://localhost'\n", - "os.environ['SERVER_PORT'] = '9081'\n", - "os.environ['TEST_EXTERNAL_REGISTRY'] = 'k3d-registry.localhost:5800'\n", - "os.environ['TEST_QUERY_LIMIT_SIZE'] = '500000'\n", - "os.environ['TRACING'] = 'False'\n", - "os.environ['NUM_TEST_USERS'] = '5'\n", - "os.environ['NUM_TEST_JOBS'] = '10'\n", - "os.environ['NUM_TEST_WORKERS'] = '2'" - ] - }, - { - "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "1", "metadata": {}, "outputs": [], @@ -63,21 +39,10 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "2", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'remote'" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "ADMIN_EMAIL, ADMIN_PW = \"admin2@bigquery.org\", \"bqpw2\"\n", "\n", @@ -95,7 +60,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "4", "metadata": {}, "outputs": [], @@ -104,7 +69,7 @@ " name=\"bigquery-high\",\n", " dev_mode=True,\n", " server_side_type=\"high\",\n", - " port=\"9081\",\n", + " port=\"8080\",\n", " n_consumers=1, # How many workers to be spawned\n", " create_producer=True, # Can produce more workers\n", ")" @@ -112,2679 +77,39 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "5", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "> Starting SMTPTestServer on: 0.0.0.0:9025\n" - ] - } - ], + "outputs": [], "source": [ "email_server, smtp_server = get_email_server()" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "6", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "> Starting SMTPTestServer on: 0.0.0.0:9025\n", - "> SMTPTestServer got an email for ['carrie-huffman-fake@openmined.org']\n", - "> SMTPTestServer got an email for ['brent-robertson-md-fake@openmined.org']\n", - "> SMTPTestServer got an email for ['lauren-velez-fake@openmined.org']\n", - "> SMTPTestServer got an email for ['daniel-moore-fake@openmined.org']\n", - "> SMTPTestServer got an email for ['shane-wheeler-fake@openmined.org']\n", - "> SMTPTestServer got an email for ['brent-robertson-md-fake@openmined.org']\n", - "> SMTPTestServer got an email for ['brent-robertson-md-fake@openmined.org']\n", - "> SMTPTestServer got an email for ['brent-robertson-md-fake@openmined.org']\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Logged into as \n" - ] - } - ], + "outputs": [], "source": [ "high_client = sy.login(\n", - " url=\"http://localhost:9081\", email=ADMIN_EMAIL, password=ADMIN_PW\n", + " url=\"http://localhost:8080\", email=ADMIN_EMAIL, password=ADMIN_PW\n", ")" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "7", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
\n", - "\n", - "
\n", - "
\n", - " \n", - "
\n", - "

UserView List

\n", - "
\n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n", - "

Total: 0

\n", - "
\n", - "
\n", - "
\n", - "\n", - "\n", - "\n", - "" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "high_client.users" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "8", "metadata": {}, "outputs": [], @@ -2799,7 +124,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "9", "metadata": {}, "outputs": [], @@ -2818,18 +143,10 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "11", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "registering 5 users\n" - ] - } - ], + "outputs": [], "source": [ "num_users = int(os.environ.get(\"NUM_TEST_USERS\", 5))\n", "print(f\"registering {num_users} users\")" @@ -2837,22 +154,10 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "12", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Logged into as \n", - "Logged into as \n", - "Logged into as \n", - "Logged into as \n", - "Logged into as \n" - ] - } - ], + "outputs": [], "source": [ "users = []\n", "email_disable_index = 0\n", @@ -2871,25 +176,17 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "13", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[{'name': 'Carrie Huffman', 'email': 'carrie-huffman-fake@openmined.org', 'password': 'N!hLj0RF8v', 'role': 'ServiceRole.DATA_SCIENTIST', 'new_password': None, 'email_disabled': True, 'reset_password': False, 'reset_token': None}, {'name': 'Brent Robertson MD', 'email': 'brent-robertson-md-fake@openmined.org', 'password': '6i6Uf1vd&m', 'role': 'ServiceRole.DATA_SCIENTIST', 'new_password': None, 'email_disabled': False, 'reset_password': True, 'reset_token': None}, {'name': 'Lauren Velez', 'email': 'lauren-velez-fake@openmined.org', 'password': '_NJ9&yTne@', 'role': 'ServiceRole.DATA_SCIENTIST', 'new_password': None, 'email_disabled': False, 'reset_password': False, 'reset_token': None}, {'name': 'Daniel Moore', 'email': 'daniel-moore-fake@openmined.org', 'password': '_IfXa_*f8C', 'role': 'ServiceRole.DATA_SCIENTIST', 'new_password': None, 'email_disabled': False, 'reset_password': False, 'reset_token': None}, {'name': 'Shane Wheeler', 'email': 'shane-wheeler-fake@openmined.org', 'password': '42XIg#Ua!5', 'role': 'ServiceRole.DATA_SCIENTIST', 'new_password': None, 'email_disabled': False, 'reset_password': False, 'reset_token': None}]\n" - ] - } - ], + "outputs": [], "source": [ "save_users(users)" ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "14", "metadata": {}, "outputs": [], @@ -2910,7 +207,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "id": "16", "metadata": {}, "outputs": [], @@ -2929,22 +226,10 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "id": "17", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Logged into as \n", - "Logged into as \n", - "Logged into as \n", - "Logged into as \n", - "Logged into as \n" - ] - } - ], + "outputs": [], "source": [ "# one user disables notifications\n", "# one user requests a password reset\n", @@ -2966,7 +251,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "id": "18", "metadata": {}, "outputs": [], @@ -2976,34 +261,10 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "id": "19", "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "```python\n", - "class UserView:\n", - " id: str = 28a0aa1b5a7d4606804df83cd8269718\n", - " name: str = \"Carrie Huffman\"\n", - " email: str = \"carrie-huffman-fake@openmined.org\"\n", - " institution: str = \"Morton LLC\"\n", - " website: str = \"http://www.taylor.info/\"\n", - " role: str = ServiceRole.DATA_SCIENTIST\n", - " notifications_enabled: str = {: False, : False, : False, : False}\n", - "\n", - "```" - ], - "text/plain": [ - "syft.service.user.user.UserView" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "ds0_user = ds0.client.account\n", "ds0_user" @@ -3011,78 +272,10 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "id": "20", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "with sy.raises successfully caught the following exception:\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - " \n", - " \n", - " SyftException:\n", - " 
You are not permitted to perform this action.
\n", - " \n", - " \n", - "
\n", - " \n", - "
Client Trace:
\n", - " 
\n",
-       "      \n",
-       "        Traceback (most recent call last):\n",
-       "  File \"/var/folders/mt/hh7shybx3xq2888mhg60t29h0000gn/T/ipykernel_27171/4147657975.py, line 5, in \n",
-       "    ds0.client.users.update(uid=ds0_user.id, role=\"admin\")\n",
-       "  File \"/Users/brendanschell/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/packages/syft/src/syft/client/api.py, line 374, in __call__\n",
-       "    return self.function_call(self.path, *args, **kwargs)\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
-       "  File \"/Users/brendanschell/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/packages/syft/src/syft/client/api.py, line 371, in function_call\n",
-       "    return post_process_result(result, self.unwrap_on_success)\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
-       "syft.types.errors.SyftException: You are not permitted to perform this action.\n",
-       "\n",
-       "      \n",
-       "
\n", - "
\n", - "
\n", - "\n", - "" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "# with sy.raises(\n", "# sy.SyftException(public_message=\"*tried to update user*\"\n", @@ -3093,81 +286,10 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "id": "21", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "with sy.raises successfully caught the following exception:\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - " \n", - " \n", - " SyftException:\n", - " 
You are not permitted to perform this action.
\n", - " \n", - " \n", - "
\n", - " \n", - "
Client Trace:
\n", - " 
\n",
-       "      \n",
-       "        Traceback (most recent call last):\n",
-       "  File \"/var/folders/mt/hh7shybx3xq2888mhg60t29h0000gn/T/ipykernel_27171/931364833.py, line 3, in \n",
-       "    ds0_user.update(role=\"admin\")\n",
-       "  File \"/Users/brendanschell/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/packages/syft/src/syft/service/user/user.py, line 319, in update\n",
-       "    result = api.services.user.update(\n",
-       "             ^^^^^^^^^^^^^^^^^^^^^^^^^\n",
-       "  File \"/Users/brendanschell/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/packages/syft/src/syft/client/api.py, line 374, in __call__\n",
-       "    return self.function_call(self.path, *args, **kwargs)\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
-       "  File \"/Users/brendanschell/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/packages/syft/src/syft/client/api.py, line 371, in function_call\n",
-       "    return post_process_result(result, self.unwrap_on_success)\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
-       "syft.types.errors.SyftException: You are not permitted to perform this action.\n",
-       "\n",
-       "      \n",
-       "
\n", - "
\n", - "
\n", - "\n", - "" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "# with sy.raises(sy.SyftException(public_message=\"*tried to update user*\"), show=True):\n", "with sy.raises(sy.SyftException, show=True):\n", @@ -3176,7 +298,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "id": "22", "metadata": {}, "outputs": [], @@ -3186,7 +308,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "id": "23", "metadata": {}, "outputs": [], @@ -3205,21 +327,10 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "id": "25", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'bwtDjLOoyURy'" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# This is necessary as it sets the new token value in user.reset_token\n", "token = reset_password_user.get_token()\n", @@ -3228,21 +339,10 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "id": "26", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'R&y3Yqs2$1'" - ] - }, - "execution_count": 24, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# This is necessary as it sets the new password value in user.new_password\n", "passw = reset_password_user.make_new_password()\n", @@ -3251,7 +351,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "id": "27", "metadata": {}, "outputs": [], @@ -3262,853 +362,10 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": null, "id": "28", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
SyftSuccess: 
User Password updated successfully.

" - ], - "text/plain": [ - "SyftSuccess: User Password updated successfully." - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "output = reset_password_user.client.guest().reset_password(\n", " token=reset_password_user.reset_token, new_password=reset_password_user.new_password\n", @@ -4118,7 +375,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "id": "29", "metadata": {}, "outputs": [], @@ -4128,18 +385,10 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": null, "id": "30", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Logged into as \n" - ] - } - ], + "outputs": [], "source": [ "# relogin\n", "reset_password_user.relogin()\n", @@ -4156,1706 +405,20 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": null, "id": "32", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
SyftSuccess: 
If the email is valid, we sent a password reset token to your email or a password request to the admin.

" - ], - "text/plain": [ - "SyftSuccess: If the email is valid, we sent a password reset token to your email or a password request to the admin." - ] - }, - "execution_count": 29, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "reset_password_user.client.guest().forgot_password(email=reset_password_user.email)" ] }, { "cell_type": "code", - "execution_count": 30, + "execution_count": null, "id": "33", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
SyftSuccess: 
User Password updated successfully.

" - ], - "text/plain": [ - "SyftSuccess: User Password updated successfully." - ] - }, - "execution_count": 30, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "output = reset_password_user.client.guest().reset_password(\n", " token=reset_password_user.get_token(),\n", @@ -5866,7 +429,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": null, "id": "34", "metadata": {}, "outputs": [], @@ -5876,7 +439,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": null, "id": "35", "metadata": {}, "outputs": [], @@ -5888,7 +451,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": null, "id": "36", "metadata": {}, "outputs": [], @@ -5898,7 +461,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": null, "id": "37", "metadata": {}, "outputs": [], @@ -5910,18 +473,10 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": null, "id": "38", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Logged into as \n" - ] - } - ], + "outputs": [], "source": [ "# relogin\n", "reset_password_user.relogin()" @@ -5929,39 +484,20 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": null, "id": "39", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[{'name': 'Carrie Huffman', 'email': 'carrie-huffman-fake@openmined.org', 'password': 'N!hLj0RF8v', 'role': 'ServiceRole.DATA_SCIENTIST', 'new_password': None, 'email_disabled': True, 'reset_password': False, 'reset_token': None}, {'name': 'Brent Robertson MD', 'email': 'brent-robertson-md-fake@openmined.org', 'password': '&cyxJ*9z94', 'role': 'ServiceRole.DATA_SCIENTIST', 'new_password': None, 'email_disabled': False, 'reset_password': True, 'reset_token': 'WOhBeOaE7TuQ'}, {'name': 'Lauren Velez', 'email': 'lauren-velez-fake@openmined.org', 'password': '_NJ9&yTne@', 'role': 'ServiceRole.DATA_SCIENTIST', 'new_password': None, 'email_disabled': False, 'reset_password': False, 'reset_token': None}, {'name': 'Daniel Moore', 'email': 'daniel-moore-fake@openmined.org', 'password': '_IfXa_*f8C', 'role': 'ServiceRole.DATA_SCIENTIST', 'new_password': None, 'email_disabled': False, 'reset_password': False, 'reset_token': None}, {'name': 'Shane Wheeler', 'email': 'shane-wheeler-fake@openmined.org', 'password': '42XIg#Ua!5', 'role': 'ServiceRole.DATA_SCIENTIST', 'new_password': None, 'email_disabled': False, 'reset_password': False, 'reset_token': None}]\n" - ] - } - ], + "outputs": [], "source": [ "save_users(users)" ] }, { "cell_type": "code", - "execution_count": 37, + "execution_count": null, "id": "40", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "TestUser(name='Brent Robertson MD', email='brent-robertson-md-fake@openmined.org', password='&cyxJ*9z94', role=, new_password=None, email_disabled=False, reset_password=True, reset_token='WOhBeOaE7TuQ', _email_server=)" - ] - }, - "execution_count": 37, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "reset_password_user" ] @@ -5976,7 +512,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": null, "id": "42", "metadata": {}, "outputs": [], @@ -5988,860 +524,17 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": null, "id": "43", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
SyftSuccess: 
If the email is valid, we sent a password reset token to your email or a password request to the admin.

" - ], - "text/plain": [ - "SyftSuccess: If the email is valid, we sent a password reset token to your email or a password request to the admin." - ] - }, - "execution_count": 39, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "reset_password_user.client.guest().forgot_password(email=reset_password_user.email)" ] }, { "cell_type": "code", - "execution_count": 40, + "execution_count": null, "id": "44", "metadata": {}, "outputs": [], @@ -6852,853 +545,10 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": null, "id": "45", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
SyftError: 
Failed to reset user password. Token is invalid or expired.

" - ], - "text/plain": [ - "SyftError: Failed to reset user password. Token is invalid or expired." - ] - }, - "execution_count": 41, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# This should throw a SyftError because we waited too long\n", "output = reset_password_user.client.guest().reset_password(\n", @@ -7710,7 +560,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": null, "id": "46", "metadata": {}, "outputs": [], @@ -7720,93 +570,10 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": null, "id": "47", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "with sy.raises successfully caught the following exception:\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - " \n", - " \n", - " SyftException:\n", - " 
Invalid credentials.
\n", - " \n", - " \n", - "
\n", - " \n", - "
Client Trace:
\n", - " 
\n",
-       "      \n",
-       "        Traceback (most recent call last):\n",
-       "  File \"/Users/brendanschell/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/packages/syft/src/syft/client/client.py, line 942, in login\n",
-       "    user_private_key = self.connection.login(email=email, password=password)\n",
-       "                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
-       "  File \"/Users/brendanschell/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/packages/syft/src/syft/client/client.py, line 394, in login\n",
-       "    response = post_process_result(response, unwrap_on_success=True)\n",
-       "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
-       "  File \"/Users/brendanschell/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/packages/syft/src/syft/client/api.py, line 92, in post_process_result\n",
-       "    raise SyftException(public_message=result.message, server_trace=result.tb)\n",
-       "syft.types.errors.SyftException: Invalid credentials.\n",
-       "\n",
-       "During handling of the above exception, another exception occurred:\n",
-       "\n",
-       "Traceback (most recent call last):\n",
-       "  File \"/var/folders/mt/hh7shybx3xq2888mhg60t29h0000gn/T/ipykernel_27171/3514082311.py, line 3, in \n",
-       "    reset_password_user.relogin()\n",
-       "  File \"/Users/brendanschell/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/notebooks/scenarios/bigquery/helpers.py, line 121, in relogin\n",
-       "    self.client = self.client\n",
-       "    ^^^^^^^^^^^\n",
-       "  File \"/Users/brendanschell/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/notebooks/scenarios/bigquery/helpers.py, line 125, in client\n",
-       "    client = client.login(email=self.email, password=self.latest_password)\n",
-       "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
-       "  File \"/Users/brendanschell/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/packages/syft/src/syft/client/client.py, line 944, in login\n",
-       "    raise SyftException(public_message=e.public_message)\n",
-       "syft.types.errors.SyftException: Invalid credentials.\n",
-       "\n",
-       "      \n",
-       "
\n", - "
\n", - "
\n", - "\n", - "" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "# relogin\n", "with sy.raises(sy.SyftException, show=True):\n", @@ -7815,7 +582,7 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": null, "id": "48", "metadata": {}, "outputs": [], @@ -7835,36 +602,20 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": null, "id": "50", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "> Stopping SMTPTestServer\n" - ] - } - ], + "outputs": [], "source": [ "smtp_server.stop()" ] }, { "cell_type": "code", - "execution_count": 46, + "execution_count": null, "id": "51", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Shutdown not implemented for the deployment type:remote\n" - ] - } - ], + "outputs": [], "source": [ "server.land()" ] @@ -7879,11 +630,6 @@ } ], "metadata": { - "kernelspec": { - "display_name": "syft", - "language": "python", - "name": "python3" - }, "language_info": { "codemirror_mode": { "name": "ipython", @@ -7894,7 +640,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.4" + "version": "3.12.5" } }, "nbformat": 4, diff --git a/notebooks/scenarios/bigquery/020-configure-api.ipynb b/notebooks/scenarios/bigquery/020-configure-api.ipynb index a60110218ce..c3ad678a00b 100644 --- a/notebooks/scenarios/bigquery/020-configure-api.ipynb +++ b/notebooks/scenarios/bigquery/020-configure-api.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -14,7 +14,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -24,40 +24,9 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], - "source": [ - "import os\n", - "\n", - "os.environ['ORCHESTRA_DEPLOYMENT_TYPE'] = 'remote'\n", - "os.environ['DEVSPACE_PROFILE'] = 'bigquery-scenario-tests'\n", - "os.environ['GITHUB_CI'] = 'false'\n", - "os.environ['SYFT_BASE_IMAGE_REGISTRY'] = 'k3d-registry.localhost:5800'\n", - "os.environ['DATASITE_CLUSTER_NAME'] = 'test-datasite-1'\n", - "os.environ['SERVER_URL'] = 'http://localhost'\n", - "os.environ['SERVER_PORT'] = '9081'\n", - "os.environ['TEST_EXTERNAL_REGISTRY'] = 'k3d-registry.localhost:5800'\n", - "os.environ['TEST_QUERY_LIMIT_SIZE'] = '500000'\n", - "os.environ['TRACING'] = 'False'\n", - "os.environ['NUM_TEST_USERS'] = '5'\n", - "os.environ['NUM_TEST_JOBS'] = '10'\n", - "os.environ['NUM_TEST_WORKERS'] = '2'" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Using Mock API Code, this will query BigQuery. $TEST_BIGQUERY_APIS_LIVE==False\n" - ] - } - ], "source": [ "# third party\n", "from apis import make_schema\n", @@ -74,7 +43,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -90,7 +59,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -98,7 +67,7 @@ " name=\"bigquery-high\",\n", " dev_mode=True,\n", " server_side_type=\"high\",\n", - " port=\"9081\",\n", + " port=\"8080\",\n", " n_consumers=1, # How many workers to be spawned\n", " create_producer=True, # Can produce more workers\n", ")" @@ -106,50 +75,27 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "> Starting SMTPTestServer on: 0.0.0.0:9025\n" - ] - } - ], + "outputs": [], "source": [ "email_server, smtp_server = get_email_server()" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "> Starting SMTPTestServer on: 0.0.0.0:9025\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Logged into as \n" - ] - } - ], + "outputs": [], "source": [ "high_client = sy.login(\n", - " url=\"http://localhost:9081\", email=ADMIN_EMAIL, password=ADMIN_PW\n", + " url=\"http://localhost:8080\", email=ADMIN_EMAIL, password=ADMIN_PW\n", ")" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -158,7 +104,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -174,7 +120,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -183,7 +129,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -192,2618 +138,9 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
\n", - "\n", - "
\n", - "
\n", - " \n", - "
\n", - "

WorkerPool Dicttuple

\n", - "
\n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n", - "

Total: 0

\n", - "
\n", - "
\n", - "
\n", - "\n", - "\n", - "\n", - "" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Look up the worker pools and identify the name of the one that has the required packages\n", "# After, bind the endpoint to that workerpool\n", @@ -2812,7 +149,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -2826,7 +163,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -2839,852 +176,9 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
SyftSuccess: 
Endpoint successfully created.

" - ], - "text/plain": [ - "SyftSuccess: Endpoint successfully created." - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "new_endpoint = sy.TwinAPIEndpoint(\n", " path=\"bigquery.test_query\",\n", @@ -3706,852 +200,9 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
SyftSuccess: 
Endpoint successfully updated.

" - ], - "text/plain": [ - "SyftSuccess: Endpoint successfully updated." - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Here, we update the endpoint to timeout after 100s (rather the default of 60s)\n", "high_client.api.services.api.update(\n", @@ -4561,852 +212,9 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
SyftSuccess: 
Endpoint successfully updated.

" - ], - "text/plain": [ - "SyftSuccess: Endpoint successfully updated." - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "high_client.api.services.api.update(\n", " endpoint_path=\"bigquery.test_query\", hide_mock_definition=True\n", @@ -5422,7 +230,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -5436,119 +244,9 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - " \n", - " \n", - " SyftException:\n", - " 
Function timed out in 120 seconds. Get the Job with id: 37cb41673fdf4fc1aa7761047b964ced to check results.
\n", - " \n", - " \n", - "
\n", - " \n", - "
Server Trace:
\n", - " 
\n",
-       "      \n",
-       "        Traceback (most recent call last):\n",
-       "  File \"/root/app/syft/src/syft/server/server.py, line 1271, in handle_api_call_with_unsigned_result\n",
-       "    result = method(context, *api_call.args, **api_call.kwargs)\n",
-       "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
-       "  File \"/root/app/syft/src/syft/service/service.py, line 490, in _decorator\n",
-       "    result = func(self, *args, **kwargs)\n",
-       "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
-       "  File \"/root/app/syft/src/syft/service/api/api_service.py, line 321, in call_public_in_jobs\n",
-       "    ).unwrap()\n",
-       "      ^^^^^^^^\n",
-       "  File \"/root/app/syft/src/syft/types/result.py, line 90, in unwrap\n",
-       "    raise self.value\n",
-       "  File \"/root/app/syft/src/syft/types/result.py, line 113, in wrapper\n",
-       "    output = func(*args, **kwargs)\n",
-       "             ^^^^^^^^^^^^^^^^^^^^^\n",
-       "  File \"/root/app/syft/src/syft/service/api/api_service.py, line 366, in _call_in_jobs\n",
-       "    raise SyftException(\n",
-       "syft.types.errors.SyftException: Function timed out in 120 seconds. Get the Job with id: 37cb41673fdf4fc1aa7761047b964ced to check results.\n",
-       "\n",
-       "      \n",
-       "
\n", - "
\n", - "
\n", - " \n", - "
Client Trace:
\n", - " 
\n",
-       "      \n",
-       "        Traceback (most recent call last):\n",
-       "  File \"/Users/brendanschell/miniconda3/envs/syft/lib/python3.12/site-packages/IPython/core/interactiveshell.py, line 3577, in run_code\n",
-       "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
-       "  File \"/var/folders/mt/hh7shybx3xq2888mhg60t29h0000gn/T/ipykernel_27303/3350366737.py, line 2, in \n",
-       "    result = high_client.api.services.bigquery.test_query.mock(\n",
-       "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
-       "  File \"/Users/brendanschell/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/packages/syft/src/syft/client/api.py, line 383, in __call__\n",
-       "    return remote_func.function_call(\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
-       "  File \"/Users/brendanschell/repos/om/PySyft.worktrees/bschell/scale-down-k8s-kills-jobs/packages/syft/src/syft/client/api.py, line 371, in function_call\n",
-       "    return post_process_result(result, self.unwrap_on_success)\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
-       "syft.types.errors.SyftException: Function timed out in 120 seconds. Get the Job with id: 37cb41673fdf4fc1aa7761047b964ced to check results.\n",
-       "server_trace: Traceback (most recent call last):\n",
-       "  File \"/root/app/syft/src/syft/server/server.py, line 1271, in handle_api_call_with_unsigned_result\n",
-       "    result = method(context, *api_call.args, **api_call.kwargs)\n",
-       "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
-       "  File \"/root/app/syft/src/syft/service/service.py, line 490, in _decorator\n",
-       "    result = func(self, *args, **kwargs)\n",
-       "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
-       "  File \"/root/app/syft/src/syft/service/api/api_service.py, line 321, in call_public_in_jobs\n",
-       "    ).unwrap()\n",
-       "      ^^^^^^^^\n",
-       "  File \"/root/app/syft/src/syft/types/result.py, line 90, in unwrap\n",
-       "    raise self.value\n",
-       "  File \"/root/app/syft/src/syft/types/result.py, line 113, in wrapper\n",
-       "    output = func(*args, **kwargs)\n",
-       "             ^^^^^^^^^^^^^^^^^^^^^\n",
-       "  File \"/root/app/syft/src/syft/service/api/api_service.py, line 366, in _call_in_jobs\n",
-       "    raise SyftException(\n",
-       "syft.types.errors.SyftException: Function timed out in 120 seconds. Get the Job with id: 37cb41673fdf4fc1aa7761047b964ced to check results.\n",
-       "\n",
-       "\n",
-       "      \n",
-       "
\n", - "
\n", - "
\n", - "\n", - "" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "# Test mock version\n", "result = high_client.api.services.bigquery.test_query.mock(\n", @@ -5926,7 +624,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.4" + "version": "3.12.5" } }, "nbformat": 4, diff --git a/notebooks/scenarios/bigquery/021-create-jobs.ipynb b/notebooks/scenarios/bigquery/021-create-jobs.ipynb index 3f960662204..3625af65c88 100644 --- a/notebooks/scenarios/bigquery/021-create-jobs.ipynb +++ b/notebooks/scenarios/bigquery/021-create-jobs.ipynb @@ -26,30 +26,6 @@ "# os.environ[\"TEST_BIGQUERY_APIS_LIVE\"] = str(use_live_bigquery)" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "99308c3e", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "os.environ['ORCHESTRA_DEPLOYMENT_TYPE'] = 'remote'\n", - "os.environ['DEVSPACE_PROFILE'] = 'bigquery-scenario-tests'\n", - "os.environ['GITHUB_CI'] = 'false'\n", - "os.environ['SYFT_BASE_IMAGE_REGISTRY'] = 'k3d-registry.localhost:5800'\n", - "os.environ['DATASITE_CLUSTER_NAME'] = 'test-datasite-1'\n", - "os.environ['SERVER_URL'] = 'http://localhost'\n", - "os.environ['SERVER_PORT'] = '9081'\n", - "os.environ['TEST_EXTERNAL_REGISTRY'] = 'k3d-registry.localhost:5800'\n", - "os.environ['TEST_QUERY_LIMIT_SIZE'] = '500000'\n", - "os.environ['TRACING'] = 'False'\n", - "os.environ['NUM_TEST_USERS'] = '5'\n", - "os.environ['NUM_TEST_JOBS'] = '10'\n", - "os.environ['NUM_TEST_WORKERS'] = '2'" - ] - }, { "cell_type": "code", "execution_count": null, @@ -87,7 +63,7 @@ "metadata": {}, "outputs": [], "source": [ - "SERVER_PORT = \"9081\"\n", + "SERVER_PORT = \"8080\"\n", "SERVER_URL = f\"http://localhost:{SERVER_PORT}\"\n", "ADMIN_EMAIL, ADMIN_PW = \"admin2@bigquery.org\", \"bqpw2\"" ] @@ -135,7 +111,7 @@ "outputs": [], "source": [ "high_client = sy.login(\n", - " url=\"http://localhost:9081\", email=ADMIN_EMAIL, password=ADMIN_PW\n", + " url=\"http://localhost:8080\", email=ADMIN_EMAIL, password=ADMIN_PW\n", ")" ] }, diff --git a/notebooks/scenarios/bigquery/040-do-review-requests.ipynb b/notebooks/scenarios/bigquery/040-do-review-requests.ipynb index e16cb5f3346..9f0a301f2e3 100644 --- a/notebooks/scenarios/bigquery/040-do-review-requests.ipynb +++ b/notebooks/scenarios/bigquery/040-do-review-requests.ipynb @@ -12,29 +12,6 @@ "# os.environ[\"TEST_EXTERNAL_REGISTRY\"] = \"k3d-registry.localhost:5800\"" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "os.environ['ORCHESTRA_DEPLOYMENT_TYPE'] = 'remote'\n", - "os.environ['DEVSPACE_PROFILE'] = 'bigquery-scenario-tests'\n", - "os.environ['GITHUB_CI'] = 'false'\n", - "os.environ['SYFT_BASE_IMAGE_REGISTRY'] = 'k3d-registry.localhost:5800'\n", - "os.environ['DATASITE_CLUSTER_NAME'] = 'test-datasite-1'\n", - "os.environ['SERVER_URL'] = 'http://localhost'\n", - "os.environ['SERVER_PORT'] = '9081'\n", - "os.environ['TEST_EXTERNAL_REGISTRY'] = 'k3d-registry.localhost:5800'\n", - "os.environ['TEST_QUERY_LIMIT_SIZE'] = '500000'\n", - "os.environ['TRACING'] = 'False'\n", - "os.environ['NUM_TEST_USERS'] = '5'\n", - "os.environ['NUM_TEST_JOBS'] = '10'\n", - "os.environ['NUM_TEST_WORKERS'] = '2'" - ] - }, { "cell_type": "code", "execution_count": null, @@ -81,7 +58,7 @@ " name=\"bigquery-high\",\n", " dev_mode=True,\n", " server_side_type=\"high\",\n", - " port=\"9081\",\n", + " port=\"8080\",\n", " n_consumers=1, # How many workers to be spawned\n", " create_producer=True, # Can produce more workers\n", ")" @@ -94,7 +71,7 @@ "outputs": [], "source": [ "high_client = sy.login(\n", - " url=\"http://localhost:9081\", email=ADMIN_EMAIL, password=ADMIN_PW\n", + " url=\"http://localhost:8080\", email=ADMIN_EMAIL, password=ADMIN_PW\n", ")" ] }, diff --git a/notebooks/scenarios/bigquery/050-ds-get-results.ipynb b/notebooks/scenarios/bigquery/050-ds-get-results.ipynb index f126fc43749..72ab42122c1 100644 --- a/notebooks/scenarios/bigquery/050-ds-get-results.ipynb +++ b/notebooks/scenarios/bigquery/050-ds-get-results.ipynb @@ -12,29 +12,6 @@ "# os.environ[\"TEST_EXTERNAL_REGISTRY\"] = \"k3d-registry.localhost:5800\"" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "os.environ['ORCHESTRA_DEPLOYMENT_TYPE'] = 'remote'\n", - "os.environ['DEVSPACE_PROFILE'] = 'bigquery-scenario-tests'\n", - "os.environ['GITHUB_CI'] = 'false'\n", - "os.environ['SYFT_BASE_IMAGE_REGISTRY'] = 'k3d-registry.localhost:5800'\n", - "os.environ['DATASITE_CLUSTER_NAME'] = 'test-datasite-1'\n", - "os.environ['SERVER_URL'] = 'http://localhost'\n", - "os.environ['SERVER_PORT'] = '9081'\n", - "os.environ['TEST_EXTERNAL_REGISTRY'] = 'k3d-registry.localhost:5800'\n", - "os.environ['TEST_QUERY_LIMIT_SIZE'] = '500000'\n", - "os.environ['TRACING'] = 'False'\n", - "os.environ['NUM_TEST_USERS'] = '5'\n", - "os.environ['NUM_TEST_JOBS'] = '10'\n", - "os.environ['NUM_TEST_WORKERS'] = '2'" - ] - }, { "cell_type": "code", "execution_count": null, @@ -77,7 +54,7 @@ " name=\"bigquery-high\",\n", " dev_mode=True,\n", " server_side_type=\"high\",\n", - " port=\"9081\",\n", + " port=\"8080\",\n", " n_consumers=1, # How many workers to be spawned\n", " create_producer=True, # Can produce more workers\n", ")" @@ -90,7 +67,7 @@ "outputs": [], "source": [ "high_client = sy.login(\n", - " url=\"http://localhost:9081\", email=ADMIN_EMAIL, password=ADMIN_PW\n", + " url=\"http://localhost:8080\", email=ADMIN_EMAIL, password=ADMIN_PW\n", ")" ] }, diff --git a/tox.ini b/tox.ini index fd6cb5a8d21..958dc2a8772 100644 --- a/tox.ini +++ b/tox.ini @@ -361,7 +361,7 @@ setenv = TEST_NOTEBOOK_PATHS = {env:TEST_NOTEBOOK_PATHS:scenarios/bigquery} TEST_query_limit_size={env:test_query_limit_size:500000} SERVER_URL = {env:SERVER_URL:http://localhost} - SERVER_PORT = {env:SERVER_PORT:9081} + SERVER_PORT = {env:SERVER_PORT:8080} NUM_TEST_USERS = {env:NUM_TEST_USERS:5} NUM_TEST_JOBS = {env:NUM_TEST_JOBS:10} commands = @@ -427,7 +427,7 @@ setenv = SYFT_BASE_IMAGE_REGISTRY = {env:SYFT_BASE_IMAGE_REGISTRY:k3d-registry.localhost:5800} DATASITE_CLUSTER_NAME = {env:DATASITE_CLUSTER_NAME:test-datasite-1} SERVER_URL = {env:SERVER_URL:http://localhost} - SERVER_PORT = {env:SERVER_PORT:9081} + SERVER_PORT = {env:SERVER_PORT:8080} TEST_EXTERNAL_REGISTRY = {env:TEST_EXTERNAL_REGISTRY:k3d-registry.localhost:5800} TEST_QUERY_LIMIT_SIZE={env:TEST_QUERY_LIMIT_SIZE:500000} TRACING={env:TRACING:False}