diff --git a/.gitignore b/.gitignore index 52b24d684cb..e016a1d9b9b 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,8 @@ build # docker compose volumes docker/data/* +# env files +.env # vagrant .vagrant diff --git a/notebooks/.gitignore b/notebooks/.gitignore index e93bc87c119..c19599ba18f 100644 --- a/notebooks/.gitignore +++ b/notebooks/.gitignore @@ -1 +1,2 @@ -**/secrets.json \ No newline at end of file +**/secrets.json +**/settings.yaml \ No newline at end of file diff --git a/notebooks/scenarios/bigquery/.gitignore b/notebooks/scenarios/bigquery/.gitignore deleted file mode 100644 index 63b990492d4..00000000000 --- a/notebooks/scenarios/bigquery/.gitignore +++ /dev/null @@ -1 +0,0 @@ -service_account.json \ No newline at end of file diff --git a/notebooks/scenarios/bigquery/01-setup-datasite.ipynb b/notebooks/scenarios/bigquery/01-setup-datasite.ipynb index 9b925b70f3a..89bdf1cddd7 100644 --- a/notebooks/scenarios/bigquery/01-setup-datasite.ipynb +++ b/notebooks/scenarios/bigquery/01-setup-datasite.ipynb @@ -6,8 +6,11 @@ "metadata": {}, "outputs": [], "source": [ + "# third party\n", + "\n", "# syft absolute\n", - "import syft as sy" + "import syft as sy\n", + "from syft import test_settings" ] }, { @@ -18,7 +21,7 @@ "source": [ "server = sy.orchestra.launch(\n", " name=\"bigquery-high\",\n", - " dev_mode=False,\n", + " dev_mode=True,\n", " server_side_type=\"high\",\n", " reset=True,\n", " port=\"8080\",\n", @@ -53,25 +56,7 @@ "metadata": {}, "outputs": [], "source": [ - "# add your values to secrets.json in this folder\n", - "secrets = sy.get_nb_secrets(\n", - " {\n", - " \"external_registry\": \"docker.io\",\n", - " }\n", - ")\n", - "external_registry = secrets[\"external_registry\"]\n", - "external_registry = \"docker.io\"\n", - "external_registry" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "worker_docker_tag = f\"openmined/bigquery:{sy.__version__}\"\n", - "worker_docker_tag" + "assert len(high_client.worker_pools.get_all()) == 1" ] }, { @@ -80,8 +65,8 @@ "metadata": {}, "outputs": [], "source": [ - "docker_config = sy.PrebuiltWorkerConfig(tag=f\"{external_registry}/{worker_docker_tag}\")\n", - "docker_config" + "external_registry = test_settings.get(\"external_registry\", default=\"docker.io\")\n", + "external_registry" ] }, { @@ -90,10 +75,8 @@ "metadata": {}, "outputs": [], "source": [ - "submit_result = high_client.api.services.worker_image.submit(\n", - " worker_config=docker_config\n", - ")\n", - "submit_result" + "worker_docker_tag = f\"openmined/bigquery:{sy.__version__}\"\n", + "worker_docker_tag" ] }, { @@ -102,8 +85,8 @@ "metadata": {}, "outputs": [], "source": [ - "worker_image = high_client.images.get_all()[1]\n", - "worker_image" + "docker_config = sy.PrebuiltWorkerConfig(tag=f\"{external_registry}/{worker_docker_tag}\")\n", + "docker_config" ] }, { @@ -112,8 +95,8 @@ "metadata": {}, "outputs": [], "source": [ - "registry_add_result = high_client.api.services.image_registry.add(external_registry)\n", - "registry_add_result" + "result = high_client.api.services.worker_image.submit(worker_config=docker_config)\n", + "result" ] }, { @@ -122,8 +105,8 @@ "metadata": {}, "outputs": [], "source": [ - "registries = high_client.api.services.image_registry.get_all()\n", - "registries" + "worker_image = high_client.images.get_all()[1]\n", + "worker_image" ] }, { @@ -132,7 +115,8 @@ "metadata": {}, "outputs": [], "source": [ - "registry_uid = next((r.id for r in registries if r.url == external_registry), None)" + "result = high_client.api.services.image_registry.add(external_registry)\n", + "result" ] }, { @@ -150,11 +134,12 @@ "metadata": {}, "outputs": [], "source": [ - "high_client.api.services.worker_pool.launch(\n", + "result = high_client.api.services.worker_pool.launch(\n", " pool_name=worker_pool_name,\n", " image_uid=worker_image.id,\n", " num_workers=1,\n", - ")" + ")\n", + "result" ] }, { @@ -163,9 +148,8 @@ "metadata": {}, "outputs": [], "source": [ - "high_client.worker_pools.scale(number=5, pool_name=worker_pool_name)\n", - "\n", - "high_client.worker_pools" + "# result = high_client.worker_pools.scale(number=5, pool_name=worker_pool_name)\n", + "# result" ] }, { @@ -174,8 +158,7 @@ "metadata": {}, "outputs": [], "source": [ - "base_worker_image = high_client.images.get_all()[0]\n", - "base_worker_image" + "assert len(high_client.worker_pools.get_all()) == 2" ] }, { @@ -183,7 +166,10 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "base_worker_image = high_client.images.get_all()[0]\n", + "base_worker_image" + ] }, { "cell_type": "code", @@ -208,6 +194,15 @@ "high_client.settings.allow_guest_signup(enable=False)" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(high_client.api.services.user.get_all()) == 2" + ] + }, { "cell_type": "code", "execution_count": null, @@ -241,17 +236,7 @@ "metadata": {}, "outputs": [], "source": [ - "# # can't shutdown because in python the workers are gone in the next notebook\n", - "# assert False" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# server.land()" + "server.land()" ] }, { diff --git a/notebooks/scenarios/bigquery/02-configure-api.ipynb b/notebooks/scenarios/bigquery/02-configure-api.ipynb index c1400401d43..e9b647b16ad 100644 --- a/notebooks/scenarios/bigquery/02-configure-api.ipynb +++ b/notebooks/scenarios/bigquery/02-configure-api.ipynb @@ -7,7 +7,8 @@ "outputs": [], "source": [ "# syft absolute\n", - "import syft as sy" + "import syft as sy\n", + "from syft import test_settings" ] }, { @@ -16,14 +17,14 @@ "metadata": {}, "outputs": [], "source": [ - "# server = sy.orchestra.launch(\n", - "# name=\"bigquery-high\",\n", - "# dev_mode=False,\n", - "# server_side_type=\"high\",\n", - "# port=\"8080\",\n", - "# n_consumers=1, # How many workers to be spawned\n", - "# create_producer=True # Can produce more workers\n", - "# )" + "server = sy.orchestra.launch(\n", + " name=\"bigquery-high\",\n", + " dev_mode=True,\n", + " server_side_type=\"high\",\n", + " port=\"8080\",\n", + " n_consumers=1, # How many workers to be spawned\n", + " create_producer=True, # Can produce more workers\n", + ")" ] }, { @@ -32,9 +33,7 @@ "metadata": {}, "outputs": [], "source": [ - "high_client = sy.login(\n", - " url=\"http://localhost:8080\", email=\"info@openmined.org\", password=\"changethis\"\n", - ")" + "this_worker_pool_name = \"bigquery-pool\"" ] }, { @@ -43,17 +42,8 @@ "metadata": {}, "outputs": [], "source": [ - "# add your values to secrets.json in this folder\n", - "secrets = sy.get_nb_secrets(\n", - " {\n", - " \"service_account_bigquery_private\": {},\n", - " \"service_account_bigquery_mock\": {},\n", - " \"region_bigquery\": \"\",\n", - " \"project_id\": \"\",\n", - " \"dataset_1\": \"dataset1\",\n", - " \"table_1\": \"table1\",\n", - " \"table_2\": \"table2\",\n", - " }\n", + "high_client = sy.login(\n", + " url=\"http://localhost:8080\", email=\"info@openmined.org\", password=\"changethis\"\n", ")" ] }, @@ -63,7 +53,7 @@ "metadata": {}, "outputs": [], "source": [ - "high_client.worker_pools" + "assert len(high_client.worker_pools.get_all()) == 2" ] }, { @@ -84,16 +74,6 @@ "# !pip install db-dtypes google-cloud-bigquery" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# third party\n", - "from google.cloud import bigquery" - ] - }, { "cell_type": "code", "execution_count": null, @@ -102,9 +82,9 @@ "source": [ "@sy.api_endpoint_method(\n", " settings={\n", - " \"credentials\": secrets[\"service_account_bigquery_private\"],\n", - " \"region\": secrets[\"region_bigquery\"],\n", - " \"project_id\": secrets[\"project_id\"],\n", + " \"credentials\": test_settings.gce_service_account.to_dict(),\n", + " \"region\": test_settings.gce_region,\n", + " \"project_id\": test_settings.gce_project_id,\n", " }\n", ")\n", "def private_query_function(\n", @@ -112,7 +92,7 @@ " sql_query: str,\n", ") -> str:\n", " # third party\n", - " from google.cloud import bigquery\n", + " from google.cloud import bigquery # noqa: F811\n", " from google.oauth2 import service_account\n", "\n", " # syft absolute\n", @@ -211,9 +191,9 @@ "\n", "@sy.api_endpoint_method(\n", " settings={\n", - " \"credentials\": secrets[\"service_account_bigquery_private\"],\n", - " \"region\": secrets[\"region_bigquery\"],\n", - " \"project_id\": secrets[\"project_id\"],\n", + " \"credentials\": test_settings.gce_service_account.to_dict(),\n", + " \"region\": test_settings.gce_region,\n", + " \"project_id\": test_settings.gce_project_id,\n", " \"CALLS_PER_MIN\": 10,\n", " },\n", " helper_functions=[is_within_rate_limit],\n", @@ -226,7 +206,7 @@ " import datetime\n", "\n", " # third party\n", - " from google.cloud import bigquery\n", + " from google.cloud import bigquery # noqa: F811\n", " from google.oauth2 import service_account\n", "\n", " # syft absolute\n", @@ -312,8 +292,6 @@ "source": [ "# Look up the worker pools and identify the name of the one that has the required packages\n", "# After, bind the endpoint to that workerpool\n", - "\n", - "\n", "high_client.worker_pools" ] }, @@ -328,7 +306,7 @@ " description=\"This endpoint allows to query Bigquery storage via SQL queries.\",\n", " private_function=private_query_function,\n", " mock_function=mock_query_function,\n", - " worker_pool=\"bigquery-pool\",\n", + " worker_pool=this_worker_pool_name,\n", ")\n", "\n", "high_client.custom_api.add(endpoint=new_endpoint)" @@ -357,6 +335,15 @@ ")" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# it currently hangs here because the reloaded server in this notebook doesnt recreate the worker consumers" + ] + }, { "cell_type": "code", "execution_count": null, @@ -365,11 +352,20 @@ "source": [ "# Test mock version\n", "result = high_client.api.services.bigquery.test_query.mock(\n", - " sql_query=f\"SELECT * FROM {secrets['dataset_1']}.{secrets['table_1']} LIMIT 10\"\n", + " sql_query=f\"SELECT * FROM {test_settings.dataset_1}.{test_settings.table_1} LIMIT 10\"\n", ")\n", "result" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(result) == 10" + ] + }, { "cell_type": "code", "execution_count": null, @@ -380,15 +376,18 @@ " path=\"bigquery.schema\",\n", " description=\"This endpoint allows for visualising the metadata of tables available in BigQuery.\",\n", " settings={\n", - " \"credentials\": secrets[\"service_account_bigquery_mock\"],\n", - " \"region\": secrets[\"region_bigquery\"],\n", - " \"project_id\": secrets[\"project_id\"],\n", + " \"credentials\": test_settings.gce_service_account.to_dict(),\n", + " \"region\": test_settings.gce_region,\n", + " \"project_id\": test_settings.gce_project_id,\n", + " \"dataset_1\": test_settings.dataset_1,\n", + " \"table_1\": test_settings.table_1,\n", + " \"table_2\": test_settings.table_2,\n", " \"CALLS_PER_MIN\": 5,\n", " },\n", " helper_functions=[\n", " is_within_rate_limit\n", " ], # Adds ratelimit as this is also a method available to data scientists\n", - " worker_pool=\"bigquery-pool\",\n", + " worker_pool=this_worker_pool_name,\n", ")\n", "def schema_function(\n", " context,\n", @@ -397,6 +396,7 @@ " import datetime\n", "\n", " # third party\n", + " from google.cloud import bigquery # noqa: F811\n", " from google.oauth2 import service_account\n", " import pandas as pd\n", "\n", @@ -430,8 +430,8 @@ "\n", " data_schema = []\n", " for table_id in [\n", - " f\"{secrets['dataset_1']}.{secrets['table_1']}\",\n", - " f\"{secrets['dataset_1']}.{secrets['table_2']}\",\n", + " f\"{context.settings[\"dataset_1\"]}.{context.settings[\"table_1\"]}\",\n", + " f\"{context.settings[\"dataset_1\"]}.{context.settings[\"table_2\"]}\",\n", " ]:\n", " table = client.get_table(table_id)\n", " for schema in table.schema:\n", @@ -476,7 +476,8 @@ "@sy.api_endpoint(\n", " path=\"bigquery.submit_query\",\n", " description=\"API endpoint that allows you to submit SQL queries to run on the private data.\",\n", - " worker_pool=\"bigquery-pool\",\n", + " worker_pool=this_worker_pool_name,\n", + " settings={\"worker\": this_worker_pool_name},\n", ")\n", "def submit_query(\n", " context,\n", @@ -503,7 +504,7 @@ " query=sy.Constant(val=query),\n", " client=context.admin_client,\n", " ),\n", - " worker_pool_name=\"bigquery-pool\",\n", + " worker_pool_name=context.settings[\"worker\"],\n", " )\n", " def execute_query(query: str, endpoint):\n", " res = endpoint(sql_query=query)\n", @@ -548,6 +549,15 @@ "high_client.custom_api.api_endpoints()" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(high_client.custom_api.api_endpoints()) == 3" + ] + }, { "cell_type": "code", "execution_count": null, @@ -574,11 +584,20 @@ "source": [ "# Test mock version\n", "result = high_client.api.services.bigquery.test_query.mock(\n", - " sql_query=f\"SELECT * FROM {secrets['dataset_1']}.{secrets['table_1']} LIMIT 10\"\n", + " sql_query=f\"SELECT * FROM {test_settings.dataset_1}.{test_settings.table_1} LIMIT 10\"\n", ")\n", "result" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(result) == 10" + ] + }, { "cell_type": "code", "execution_count": null, @@ -587,11 +606,20 @@ "source": [ "# Test private version\n", "result = high_client.api.services.bigquery.test_query.private(\n", - " sql_query=f\"SELECT * FROM {secrets['dataset_1']}.{secrets['table_1']} LIMIT 10\"\n", + " sql_query=f\"SELECT * FROM {test_settings.dataset_1}.{test_settings.table_1} LIMIT 10\"\n", ")\n", "result" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(result) == 10" + ] + }, { "cell_type": "code", "execution_count": null, @@ -605,6 +633,15 @@ "result" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert isinstance(result, sy.SyftError)" + ] + }, { "cell_type": "code", "execution_count": null, @@ -613,11 +650,20 @@ "source": [ "# Test private version\n", "result = high_client.api.services.bigquery.test_query.private(\n", - " sql_query=f\"SELECT * FROM {secrets['dataset_1']}.{secrets['table_1']} LIMIT 1\"\n", + " sql_query=f\"SELECT * FROM {test_settings.dataset_1}.{test_settings.table_1} LIMIT 1\"\n", ")\n", "result" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(result) == 1" + ] + }, { "cell_type": "code", "execution_count": null, @@ -625,7 +671,17 @@ "outputs": [], "source": [ "# Inspect the context state on an endpoint\n", - "high_client.api.services.bigquery.test_query.mock.context.state" + "state = high_client.api.services.bigquery.test_query.mock.context.state\n", + "state" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(state[\"info@openmined.org\"]) == 3" ] }, { @@ -634,9 +690,30 @@ "metadata": {}, "outputs": [], "source": [ + "# Testing schema\n", + "result = high_client.api.services.bigquery.schema()\n", + "result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(result) == 23" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Testing submit query\n", "result = high_client.api.services.bigquery.submit_query(\n", " func_name=\"my_func\",\n", - " query=f\"SELECT * FROM {secrets['dataset_1']}.{secrets['table_1']} LIMIT 1\",\n", + " query=f\"SELECT * FROM {test_settings.dataset_1}.{test_settings.table_1} LIMIT 1\",\n", ")" ] }, @@ -649,6 +726,24 @@ "result" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert \"Query submitted\" in result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "server.land()" + ] + }, { "cell_type": "code", "execution_count": null, @@ -673,7 +768,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.2" + "version": "3.12.3" } }, "nbformat": 4, diff --git a/notebooks/scenarios/bigquery/03-ds-submit-request.ipynb b/notebooks/scenarios/bigquery/03-ds-submit-request.ipynb index 44250404b71..829cdd7dda3 100644 --- a/notebooks/scenarios/bigquery/03-ds-submit-request.ipynb +++ b/notebooks/scenarios/bigquery/03-ds-submit-request.ipynb @@ -7,7 +7,8 @@ "outputs": [], "source": [ "# syft absolute\n", - "import syft as sy" + "import syft as sy\n", + "from syft import test_settings" ] }, { @@ -16,10 +17,13 @@ "metadata": {}, "outputs": [], "source": [ - "high_client = sy.login(\n", - " url=\"http://localhost:8080\",\n", - " email=\"data_scientist@openmined.org\",\n", - " password=\"verysecurepassword\",\n", + "server = sy.orchestra.launch(\n", + " name=\"bigquery-high\",\n", + " dev_mode=True,\n", + " server_side_type=\"high\",\n", + " port=\"8080\",\n", + " n_consumers=1, # How many workers to be spawned\n", + " create_producer=True, # Can produce more workers\n", ")" ] }, @@ -29,15 +33,10 @@ "metadata": {}, "outputs": [], "source": [ - "# add your values to secrets.json in this folder\n", - "secrets = sy.get_nb_secrets(\n", - " {\n", - " \"dataset_2\": \"dataset2\",\n", - " \"table_1\": \"table1\",\n", - " \"table_2\": \"table2\",\n", - " \"table_2_col_id\": \"id\",\n", - " \"table_2_col_score\": \"score\",\n", - " }\n", + "high_client = sy.login(\n", + " url=\"http://localhost:8080\",\n", + " email=\"data_scientist@openmined.org\",\n", + " password=\"verysecurepassword\",\n", ")" ] }, @@ -66,12 +65,21 @@ "outputs": [], "source": [ "FUNC_NAME = \"popular\"\n", - "QUERY = f\"SELECT {secrets['table_2_col_id']}, AVG({secrets['table_2_col_score']}) AS average_score \\\n", - " FROM {secrets['dataset_2']}.{secrets['table_2']} \\\n", - " GROUP BY {secrets['table_2_col_id']} \\\n", + "QUERY = f\"SELECT {test_settings.table_2_col_id}, AVG({test_settings.table_2_col_score}) AS average_score \\\n", + " FROM {test_settings.dataset_2}.{test_settings.table_2} \\\n", + " GROUP BY {test_settings.table_2_col_id} \\\n", " LIMIT 10000\"\n", "\n", - "high_client.api.services.bigquery.test_query(sql_query=QUERY)" + "result = high_client.api.services.bigquery.test_query(sql_query=QUERY)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(result) == 10000" ] }, { @@ -109,7 +117,24 @@ "outputs": [], "source": [ "func_name = extract_code_path(res1)\n", - "print(func_name)\n", + "print(func_name)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert \"popular\" in func_name" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ "api_method = getattr(high_client.code, func_name, None)\n", "api_method" ] @@ -120,7 +145,18 @@ "metadata": {}, "outputs": [], "source": [ - "api_method()" + "result = api_method()\n", + "result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert isinstance(result, sy.SyftError)\n", + "assert \"waiting for approval\" in str(result)" ] }, { @@ -131,7 +167,7 @@ "source": [ "FUNC_NAME = \"large_sample\"\n", "LARGE_SAMPLE_QUERY = (\n", - " f\"SELECT * FROM {secrets['dataset_2']}.{secrets['table_2']} LIMIT 1000000\"\n", + " f\"SELECT * FROM {test_settings.dataset_2}.{test_settings.table_2} LIMIT 1000000\"\n", ")" ] }, @@ -153,7 +189,24 @@ "outputs": [], "source": [ "func_name = extract_code_path(res2)\n", - "print(func_name)\n", + "print(func_name)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert \"large_sample\" in func_name" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ "api_method_2 = getattr(high_client.code, func_name, None)\n", "api_method_2" ] @@ -164,7 +217,17 @@ "metadata": {}, "outputs": [], "source": [ - "api_method_2()" + "result = api_method_2()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert isinstance(result, sy.SyftError)\n", + "assert \"waiting for approval\" in str(result)" ] }, { @@ -172,7 +235,9 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "server.land()" + ] } ], "metadata": { diff --git a/notebooks/scenarios/bigquery/04-do-review-requests.ipynb b/notebooks/scenarios/bigquery/04-do-review-requests.ipynb index 95fd063c6a1..8ae6e4cd674 100644 --- a/notebooks/scenarios/bigquery/04-do-review-requests.ipynb +++ b/notebooks/scenarios/bigquery/04-do-review-requests.ipynb @@ -7,7 +7,26 @@ "outputs": [], "source": [ "# syft absolute\n", - "import syft as sy" + "import syft as sy\n", + "from syft.service.code.user_code import UserCode\n", + "from syft.service.job.job_stash import Job\n", + "from syft.service.request.request import Request" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "server = sy.orchestra.launch(\n", + " name=\"bigquery-high\",\n", + " dev_mode=True,\n", + " server_side_type=\"high\",\n", + " port=\"8080\",\n", + " n_consumers=1, # How many workers to be spawned\n", + " create_producer=True, # Can produce more workers\n", + ")" ] }, { @@ -54,11 +73,6 @@ "metadata": {}, "outputs": [], "source": [ - "# syft absolute\n", - "from syft.service.code.user_code import UserCode\n", - "from syft.service.request.request import Request\n", - "\n", - "\n", "def execute_request(client_high, request) -> dict:\n", " if not isinstance(request, Request):\n", " return \"This is not a request\"\n", @@ -100,7 +114,26 @@ "metadata": {}, "outputs": [], "source": [ - "job.wait()" + "isinstance(job, Job)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "result = job.wait()\n", + "result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(result) == 1" ] }, { @@ -110,7 +143,8 @@ "outputs": [], "source": [ "job_info = job.info(result=True)\n", - "request.deposit_result(job_info, approve=True)" + "response = request.deposit_result(job_info, approve=True)\n", + "response" ] }, { @@ -119,11 +153,40 @@ "metadata": {}, "outputs": [], "source": [ - "# syft absolute\n", - "\n", - "\n", - "def approve_and_deposit(do_client, request_id):\n", - " request = do_client.requests.get_by_uid(uid=sy.UID(request_id))\n", + "isinstance(response, Job)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def list_pending(do_client):\n", + " results = []\n", + " for request in do_client.requests:\n", + " if str(request.status) == \"RequestStatus.PENDING\":\n", + " results.append(request)\n", + " print(request.id)\n", + " print(request.code.constants[\"query\"].val)\n", + " print()\n", + " return results" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def approve_and_deposit(do_client, request_id_or_idx):\n", + " if isinstance(request_id_or_idx, int):\n", + " pending = list_pending(do_client)\n", + " request = pending[request_id_or_idx]\n", + " elif isinstance(request_id_or_idx, str):\n", + " request = do_client.requests.get_by_uid(uid=sy.UID(request_id_or_idx))\n", + " else:\n", + " raise Exception(f\"Wrong type for request_id_or_idx: {type(request_id_or_idx)}\")\n", "\n", " if not isinstance(request, Request):\n", " return \"This is not a request\"\n", @@ -150,12 +213,8 @@ "metadata": {}, "outputs": [], "source": [ - "def list_pending(do_client):\n", - " for request in do_client.requests:\n", - " if str(request.status) == \"RequestStatus.PENDING\":\n", - " print(request.id)\n", - " print(request.code.constants[\"query\"].val)\n", - " print()" + "pending = list_pending(high_client)\n", + "pending" ] }, { @@ -164,7 +223,7 @@ "metadata": {}, "outputs": [], "source": [ - "list_pending(high_client)" + "result = approve_and_deposit(high_client, 0)" ] }, { @@ -173,7 +232,7 @@ "metadata": {}, "outputs": [], "source": [ - "approve_and_deposit(high_client, \"5c0e75dc293b43c3a16b138a58232408\")" + "assert isinstance(result, Job)" ] }, { @@ -182,15 +241,8 @@ "metadata": {}, "outputs": [], "source": [ - "approve_and_deposit(high_client, \"936b2a5e8d2f4106980650c6214aa23d\")" + "server.land()" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/notebooks/scenarios/bigquery/05-ds-get-results.ipynb b/notebooks/scenarios/bigquery/05-ds-get-results.ipynb index edf5e6927b1..c6f86c5fdd0 100644 --- a/notebooks/scenarios/bigquery/05-ds-get-results.ipynb +++ b/notebooks/scenarios/bigquery/05-ds-get-results.ipynb @@ -10,6 +10,22 @@ "import syft as sy" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "server = sy.orchestra.launch(\n", + " name=\"bigquery-high\",\n", + " dev_mode=True,\n", + " server_side_type=\"high\",\n", + " port=\"8080\",\n", + " n_consumers=1, # How many workers to be spawned\n", + " create_producer=True, # Can produce more workers\n", + ")" + ] + }, { "cell_type": "code", "execution_count": null, @@ -54,7 +70,26 @@ "metadata": {}, "outputs": [], "source": [ - "api_methods[\"popular_bed17f\"]()" + "result = api_methods[\"popular_bed17f\"]()\n", + "result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(result) == 10000" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "server.land()" ] }, { diff --git a/notebooks/scenarios/getting-started/03-configuring-datasite.ipynb b/notebooks/scenarios/getting-started/03-configuring-datasite.ipynb index fd3973b6720..58294a9f86f 100644 --- a/notebooks/scenarios/getting-started/03-configuring-datasite.ipynb +++ b/notebooks/scenarios/getting-started/03-configuring-datasite.ipynb @@ -6,32 +6,13 @@ "id": "0", "metadata": {}, "outputs": [], - "source": [ - "# -- all the configuration and server settings options\n", - "# -- optional: adding email settings (smtp with sendgrid free account)\n", - "# -- changing passwords (importance of root account)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1", - "metadata": {}, - "outputs": [], "source": [ "SYFT_VERSION = \">=0.9,<0.9.2\"\n", - "package_string = f'\"syft{SYFT_VERSION}\"'" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2", - "metadata": {}, - "outputs": [], - "source": [ + "package_string = f'\"syft{SYFT_VERSION}\"'\n", + "\n", "# syft absolute\n", - "import syft as sy\n", + "import syft as sy # noqa: E402\n", + "from syft import test_settings # noqa: E402\n", "\n", "sy.requires(SYFT_VERSION)" ] @@ -39,7 +20,7 @@ { "cell_type": "code", "execution_count": null, - "id": "3", + "id": "1", "metadata": {}, "outputs": [], "source": [ @@ -50,7 +31,7 @@ { "cell_type": "code", "execution_count": null, - "id": "4", + "id": "2", "metadata": {}, "outputs": [], "source": [ @@ -59,7 +40,7 @@ }, { "cell_type": "markdown", - "id": "5", + "id": "3", "metadata": {}, "source": [ "# email" @@ -68,23 +49,23 @@ { "cell_type": "code", "execution_count": null, - "id": "6", + "id": "4", "metadata": {}, "outputs": [], "source": [ - "secrets = sy.get_nb_secrets(defaults={\"smtp_token\": \"ADD_POSTMARK_TOKEN\"})" + "smtp_token = test_settings.get(\"smtp_token\", default=\"ADD_POSTMARK_TOKEN\")" ] }, { "cell_type": "code", "execution_count": null, - "id": "7", + "id": "5", "metadata": {}, "outputs": [], "source": [ "smtp_username, smtp_password, smtp_server, smtp_port = (\n", - " secrets[\"smtp_token\"],\n", - " secrets[\"smtp_token\"],\n", + " smtp_token,\n", + " smtp_token,\n", " \"smtp.postmarkapp.com\",\n", " \"25\",\n", ")\n", @@ -101,7 +82,7 @@ { "cell_type": "code", "execution_count": null, - "id": "8", + "id": "6", "metadata": {}, "outputs": [], "source": [ @@ -111,7 +92,7 @@ { "cell_type": "code", "execution_count": null, - "id": "9", + "id": "7", "metadata": {}, "outputs": [], "source": [ @@ -121,7 +102,7 @@ { "cell_type": "code", "execution_count": null, - "id": "10", + "id": "8", "metadata": {}, "outputs": [], "source": [ @@ -131,7 +112,7 @@ { "cell_type": "code", "execution_count": null, - "id": "11", + "id": "9", "metadata": {}, "outputs": [], "source": [ @@ -141,7 +122,7 @@ { "cell_type": "code", "execution_count": null, - "id": "12", + "id": "10", "metadata": {}, "outputs": [], "source": [ @@ -151,7 +132,7 @@ { "cell_type": "code", "execution_count": null, - "id": "13", + "id": "11", "metadata": {}, "outputs": [], "source": [ @@ -161,7 +142,7 @@ { "cell_type": "code", "execution_count": null, - "id": "14", + "id": "12", "metadata": {}, "outputs": [], "source": [ @@ -179,7 +160,7 @@ { "cell_type": "code", "execution_count": null, - "id": "15", + "id": "13", "metadata": {}, "outputs": [], "source": [ @@ -214,7 +195,7 @@ { "cell_type": "code", "execution_count": null, - "id": "16", + "id": "14", "metadata": {}, "outputs": [], "source": [ @@ -224,7 +205,7 @@ { "cell_type": "code", "execution_count": null, - "id": "17", + "id": "15", "metadata": {}, "outputs": [], "source": [ @@ -234,7 +215,7 @@ { "cell_type": "code", "execution_count": null, - "id": "18", + "id": "16", "metadata": {}, "outputs": [], "source": [ @@ -244,7 +225,7 @@ { "cell_type": "code", "execution_count": null, - "id": "19", + "id": "17", "metadata": {}, "outputs": [], "source": [ @@ -255,7 +236,7 @@ { "cell_type": "code", "execution_count": null, - "id": "20", + "id": "18", "metadata": {}, "outputs": [], "source": [ @@ -265,7 +246,7 @@ { "cell_type": "code", "execution_count": null, - "id": "21", + "id": "19", "metadata": {}, "outputs": [], "source": [ @@ -275,7 +256,7 @@ { "cell_type": "code", "execution_count": null, - "id": "22", + "id": "20", "metadata": {}, "outputs": [], "source": [ @@ -285,7 +266,7 @@ { "cell_type": "code", "execution_count": null, - "id": "23", + "id": "21", "metadata": {}, "outputs": [], "source": [ @@ -295,7 +276,7 @@ { "cell_type": "code", "execution_count": null, - "id": "24", + "id": "22", "metadata": {}, "outputs": [], "source": [ @@ -305,7 +286,7 @@ { "cell_type": "code", "execution_count": null, - "id": "25", + "id": "23", "metadata": {}, "outputs": [], "source": [] diff --git a/packages/syft/setup.cfg b/packages/syft/setup.cfg index 9d97a98589e..7c6f3f852a3 100644 --- a/packages/syft/setup.cfg +++ b/packages/syft/setup.cfg @@ -125,6 +125,7 @@ test_plugins = coverage faker distro + dynaconf [options.entry_points] console_scripts = diff --git a/packages/syft/src/syft/__init__.py b/packages/syft/src/syft/__init__.py index 09c91e99254..39e019682dd 100644 --- a/packages/syft/src/syft/__init__.py +++ b/packages/syft/src/syft/__init__.py @@ -85,7 +85,6 @@ from .util.patch_ipython import patch_ipython from .util.telemetry import instrument from .util.util import autocache -from .util.util import get_nb_secrets from .util.util import get_root_data_path from .util.version_compare import make_requires @@ -141,6 +140,14 @@ def _settings() -> UserSettings: return settings +@module_property +def _test_settings() -> Any: + # relative + from .util.util import test_settings + + return test_settings() + + @module_property def hello_baby() -> None: print("Hello baby!") diff --git a/packages/syft/src/syft/util/util.py b/packages/syft/src/syft/util/util.py index c9011c82ffc..10a99212655 100644 --- a/packages/syft/src/syft/util/util.py +++ b/packages/syft/src/syft/util/util.py @@ -1070,19 +1070,53 @@ def get_latest_tag(registry: str, repo: str) -> str | None: return None -def get_nb_secrets(defaults: dict | None = None) -> dict: - if defaults is None: - defaults = {} +def find_base_dir_with_tox_ini(start_path: str = ".") -> str | None: + base_path = os.path.abspath(start_path) + while True: + if os.path.exists(os.path.join(base_path, "tox.ini")): + return base_path + parent_path = os.path.abspath(os.path.join(base_path, os.pardir)) + if parent_path == base_path: # Reached the root directory + break + base_path = parent_path + return None - try: - filename = "./secrets.json" - with open(filename) as f: - loaded = json.loads(f.read()) - defaults.update(loaded) - except Exception: - print(f"Unable to load {filename}") - return defaults +def get_all_config_files(base_path: str, current_path: str) -> list[str]: + config_files = [] + current_path = os.path.abspath(current_path) + + while current_path.startswith(base_path): + config_file = os.path.join(current_path, "settings.yaml") + if os.path.exists(config_file): + config_files.append(config_file) + if current_path == base_path: # Stop if we reach the base directory + break + current_path = os.path.abspath(os.path.join(current_path, os.pardir)) + + return config_files + + +def test_settings() -> Any: + # third party + from dynaconf import Dynaconf + + base_dir = find_base_dir_with_tox_ini() + config_files = get_all_config_files(base_dir, ".") if base_dir else [] + # create + # can override with + # import os + # os.environ["TEST_KEY"] = "var" + # third party + + # Dynaconf settings + test_settings = Dynaconf( + settings_files=list(reversed(config_files)), + environments=True, + envvar_prefix="TEST", + ) + + return test_settings class CustomRepr(reprlib.Repr): diff --git a/tox.ini b/tox.ini index 110aafd69db..a700d5c2ae0 100644 --- a/tox.ini +++ b/tox.ini @@ -18,6 +18,7 @@ envlist = syft.test.security syft.test.unit syft.test.notebook + syft.test.notebook.scenario single_container.launch single_container.destroy stack.test.notebook @@ -301,6 +302,31 @@ commands = ; pytest -x --nbmake tutorials/pandas-cookbook -p no:randomly -vvvv + +[testenv:syft.test.notebook.scenario] +description = Syft Notebook Scenario Tests +deps = + {[testenv:syft]deps} + nbmake + db-dtypes + google-cloud-bigquery +changedir = {toxinidir}/notebooks +allowlist_externals = + bash +setenv = + ORCHESTRA_DEPLOYMENT_TYPE = {env:ORCHESTRA_DEPLOYMENT_TYPE:python} + DEV_MODE = {env:DEV_MODE:True} + TEST_NOTEBOOK_PATHS = {env:TEST_NOTEBOOK_PATHS:scenarios/bigquery} + SERVER_URL = {env:SERVER_URL:http://localhost} + SERVER_PORT = {env:SERVER_PORT:8080} +commands = + + bash -c "echo Running with ORCHESTRA_DEPLOYMENT_TYPE=$ORCHESTRA_DEPLOYMENT_TYPE DEV_MODE=$DEV_MODE TEST_NOTEBOOK_PATHS=$TEST_NOTEBOOK_PATHS; date" + bash -c "for subfolder in $(echo ${TEST_NOTEBOOK_PATHS} | tr ',' ' ');\ + do \ + pytest -x --nbmake --nbmake-timeout=1000 "$subfolder" -p no:randomly -vvvv;\ + done" + [testenv:single_container.launch] description = Launch a single backend container using the dockerfile changedir = {toxinidir}/packages