Skip to content

Commit

Permalink
old sync notebooks, dump db for high-low
Browse files Browse the repository at this point in the history
  • Loading branch information
@eelcovdw
eelcovdw committed Sep 19, 2024
1 parent a664364 commit 4906ac2
Show file tree
Hide file tree
Showing 24 changed files with 3,241 additions and 1 deletion.
2 changes: 1 addition & 1 deletion .pre-commit-config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ repos:
exclude: ^(packages/grid/frontend/|.vscode)
- id: check-added-large-files
always_run: true
exclude: ^(packages/grid/backend/wheels/.*|docs/img/header.png|docs/img/terminalizer.gif)
exclude: ^(packages/grid/backend/wheels/.*|docs/img/header.png|docs/img/terminalizer.gif|^notebooks/scenarios/bigquery/upgradability/sync/migration_.*\.blob)
- id: check-yaml
always_run: true
exclude: ^(packages/grid/k8s/rendered/|packages/grid/helm/)
Expand Down
241 changes: 241 additions & 0 deletions ...s/scenarios/bigquery/upgradability/sync/0.9.1_notebooks/01-setup-high-low-datasites.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,241 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# syft absolute\n",
"import syft as sy\n",
"from syft import test_settings\n",
"\n",
"print(f\"syft version: {sy.__version__}\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"server_low = sy.orchestra.launch(\n",
" name=\"bigquery-low\",\n",
" server_side_type=\"low\",\n",
" dev_mode=True,\n",
" reset=True,\n",
" n_consumers=1,\n",
" create_producer=True,\n",
" port=\"auto\",\n",
")\n",
"\n",
"server_high = sy.orchestra.launch(\n",
" name=\"bigquery-high\",\n",
" server_side_type=\"high\",\n",
" dev_mode=True,\n",
" reset=True,\n",
" n_consumers=1,\n",
" create_producer=True,\n",
" port=\"auto\",\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"low_client = server_low.login(email=\"[email protected]\", password=\"changethis\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"high_client = server_high.login(email=\"[email protected]\", password=\"changethis\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"low_client.worker_pools.get_all()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"assert len(high_client.worker_pools.get_all()) == 1\n",
"assert len(low_client.worker_pools.get_all()) == 1"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def launch_worker_pool(client, pool_name):\n",
" if pool_name not in [x.name for x in client.worker_pools]:\n",
" external_registry = test_settings.get(\"external_registry\", default=\"docker.io\")\n",
" worker_docker_tag = f\"openmined/bigquery:{sy.__version__}\"\n",
" result = client.api.services.worker_image.submit(\n",
" worker_config=sy.PrebuiltWorkerConfig(\n",
" tag=f\"{external_registry}/{worker_docker_tag}\"\n",
" )\n",
" )\n",
" worker_image = client.images.get_all()[1]\n",
" result = client.api.services.image_registry.add(external_registry)\n",
" result = client.api.services.worker_pool.launch(\n",
" pool_name=pool_name,\n",
" image_uid=worker_image.id,\n",
" num_workers=1,\n",
" )\n",
" return result\n",
" else:\n",
" print(\"Pool already exists\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"pool_name = \"bigquery-pool\""
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"launch_worker_pool(high_client, pool_name)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"launch_worker_pool(low_client, pool_name)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# result = high_client.worker_pools.scale(number=5, pool_name=pool_name)\n",
"# result"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"assert len(high_client.worker_pools.get_all()) == 2\n",
"assert len(low_client.worker_pools.get_all()) == 2"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"base_worker_image = high_client.images.get_all()[0]\n",
"base_worker_image"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"low_client.register(\n",
" email=\"[email protected]\",\n",
" password=\"verysecurepassword\",\n",
" password_verify=\"verysecurepassword\",\n",
" name=\"John Doe\",\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"high_client.settings.allow_guest_signup(enable=False)\n",
"low_client.settings.allow_guest_signup(enable=False)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"assert (\n",
" len(low_client.api.services.user.get_all()) == 2\n",
"), \"Only DS and Admin should be at low side\"\n",
"assert (\n",
" len(high_client.api.services.user.get_all()) == 1\n",
"), \"Only Admin should be at high side\""
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"server_high.land()\n",
"server_low.land()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.13"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
Loading

0 comments on commit 4906ac2

Please sign in to comment.