Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Rename args in some API endpoints to be more consistent #8853

Merged
merged 5 commits into from
Jun 3, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion notebooks/admin/Custom API + Custom Worker.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -235,7 +235,7 @@
"source": [
"worker_pool_name = \"bigquery-pool\"\n",
"domain_client.api.services.worker_pool.launch(\n",
" name=worker_pool_name,\n",
" pool_name=worker_pool_name,\n",
" image_uid=workerimage.id,\n",
" num_workers=1,\n",
")"
Expand Down
6 changes: 3 additions & 3 deletions notebooks/api/0.8/10-container-images.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -393,7 +393,7 @@
" image_uid=workerimage.id,\n",
" tag=docker_tag,\n",
" registry_uid=registry_uid,\n",
" pull=pull,\n",
" pull_image=pull,\n",
")\n",
"docker_build_result"
]
Expand Down Expand Up @@ -600,7 +600,7 @@
"source": [
"worker_pool_name = \"opendp-pool\"\n",
"worker_pool_res = domain_client.api.services.worker_pool.launch(\n",
" name=worker_pool_name,\n",
" pool_name=worker_pool_name,\n",
" image_uid=workerimage.id,\n",
" num_workers=2,\n",
")"
Expand Down Expand Up @@ -1483,7 +1483,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.7"
"version": "3.12.2"
}
},
"nbformat": 4,
Expand Down
12 changes: 7 additions & 5 deletions notebooks/api/0.8/11-container-images-k8s.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -567,11 +567,11 @@
"source": [
"worker_pool_name = \"custom-pool\"\n",
"worker_pool_res = domain_client.api.services.worker_pool.launch(\n",
" name=worker_pool_name,\n",
" pool_name=worker_pool_name,\n",
" image_uid=workerimage.id,\n",
" num_workers=3,\n",
" reg_username=external_registry_username,\n",
" reg_password=external_registry_password,\n",
" registry_username=external_registry_username,\n",
" registry_password=external_registry_password,\n",
")"
]
},
Expand Down Expand Up @@ -1097,7 +1097,8 @@
"source": [
"# get the pending request and approve it\n",
"req_result = pool_create_request.approve(\n",
" reg_username=external_registry_username, reg_password=external_registry_password\n",
" registry_username=external_registry_username,\n",
" registry_password=external_registry_password,\n",
")\n",
"req_result"
]
Expand Down Expand Up @@ -1243,7 +1244,8 @@
"outputs": [],
"source": [
"req_result = pool_image_create_request.approve(\n",
" reg_username=external_registry_username, reg_password=external_registry_password\n",
" registry_username=external_registry_username,\n",
" registry_password=external_registry_password,\n",
")\n",
"req_result"
]
Expand Down
16 changes: 8 additions & 8 deletions packages/syft/src/syft/custom_worker/runner_k8s.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,19 +28,19 @@ def create_pool(
replicas: int = 1,
env_vars: list[dict] | None = None,
mount_secrets: dict | None = None,
reg_username: str | None = None,
reg_password: str | None = None,
registry_username: str | None = None,
registry_password: str | None = None,
reg_url: str | None = None,
**kwargs: Any,
) -> StatefulSet:
try:
# create pull secret if registry credentials are passed
pull_secret = None
if reg_username and reg_password and reg_url:
if registry_username and registry_password and reg_url:
pull_secret = self._create_image_pull_secret(
pool_name,
reg_username,
reg_password,
registry_username,
registry_password,
reg_url,
)

Expand Down Expand Up @@ -126,16 +126,16 @@ def get_pod_env_vars(self, pod: str | Pod) -> list[dict] | None:
def _create_image_pull_secret(
self,
pool_name: str,
reg_username: str,
reg_password: str,
registry_username: str,
registry_password: str,
reg_url: str,
**kwargs: Any,
) -> Secret:
return KubeUtils.create_dockerconfig_secret(
secret_name=f"pull-secret-{pool_name}",
component=pool_name,
registries=[
(reg_url, reg_username, reg_password),
(reg_url, registry_username, registry_password),
],
)

Expand Down
4 changes: 2 additions & 2 deletions packages/syft/src/syft/node/node.py
Original file line number Diff line number Diff line change
Expand Up @@ -1752,7 +1752,7 @@ def create_default_worker_pool(node: Node) -> SyftError | None:
context,
image_uid=default_image.id,
tag=DEFAULT_WORKER_IMAGE_TAG,
pull=pull_image,
pull_image=pull_image,
)

if isinstance(result, SyftError):
Expand All @@ -1772,7 +1772,7 @@ def create_default_worker_pool(node: Node) -> SyftError | None:
create_pool_method = node.get_service_method(SyftWorkerPoolService.launch)
result = create_pool_method(
context,
name=default_pool_name,
pool_name=default_pool_name,
image_uid=default_image.id,
num_workers=worker_count,
)
Expand Down
14 changes: 7 additions & 7 deletions packages/syft/src/syft/service/request/request.py
Original file line number Diff line number Diff line change
Expand Up @@ -251,7 +251,7 @@ def _run(
image_uid=worker_image.id,
tag=self.tag,
registry_uid=self.registry_uid,
pull=self.pull_image,
pull_image=self.pull_image,
)

if isinstance(build_result, SyftError):
Expand All @@ -266,9 +266,9 @@ def _run(
if IN_KUBERNETES and not worker_image.is_prebuilt:
push_result = worker_image_service.push(
service_context,
image=worker_image.id,
username=context.extra_kwargs.get("reg_username", None),
password=context.extra_kwargs.get("reg_password", None),
image_uid=worker_image.id,
username=context.extra_kwargs.get("registry_username", None),
password=context.extra_kwargs.get("registry_password", None),
)

if isinstance(push_result, SyftError):
Expand Down Expand Up @@ -332,11 +332,11 @@ def _run(

result = worker_pool_service.launch(
context=service_context,
name=self.pool_name,
pool_name=self.pool_name,
image_uid=self.image_uid,
num_workers=self.num_workers,
reg_username=context.extra_kwargs.get("reg_username", None),
reg_password=context.extra_kwargs.get("reg_password", None),
registry_username=context.extra_kwargs.get("registry_username", None),
registry_password=context.extra_kwargs.get("registry_password", None),
)
if isinstance(result, SyftError):
return Err(result)
Expand Down
28 changes: 14 additions & 14 deletions packages/syft/src/syft/service/worker/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -325,8 +325,8 @@ def create_kubernetes_pool(
replicas: int,
queue_port: int,
debug: bool,
reg_username: str | None = None,
reg_password: str | None = None,
registry_username: str | None = None,
registry_password: str | None = None,
reg_url: str | None = None,
**kwargs: Any,
) -> list[Pod] | SyftError:
Expand Down Expand Up @@ -360,8 +360,8 @@ def create_kubernetes_pool(
replicas=replicas,
env_vars=env_vars,
mount_secrets=mount_secrets,
reg_username=reg_username,
reg_password=reg_password,
registry_username=registry_username,
registry_password=registry_password,
reg_url=reg_url,
)
except Exception as e:
Expand Down Expand Up @@ -402,8 +402,8 @@ def run_workers_in_kubernetes(
queue_port: int,
start_idx: int = 0,
debug: bool = False,
reg_username: str | None = None,
reg_password: str | None = None,
registry_username: str | None = None,
registry_password: str | None = None,
reg_url: str | None = None,
**kwargs: Any,
) -> list[ContainerSpawnStatus] | SyftError:
Expand All @@ -419,8 +419,8 @@ def run_workers_in_kubernetes(
replicas=worker_count,
queue_port=queue_port,
debug=debug,
reg_username=reg_username,
reg_password=reg_password,
registry_username=registry_username,
registry_password=registry_password,
reg_url=reg_url,
)
else:
Expand Down Expand Up @@ -501,8 +501,8 @@ def run_containers(
queue_port: int,
dev_mode: bool = False,
start_idx: int = 0,
reg_username: str | None = None,
reg_password: str | None = None,
registry_username: str | None = None,
registry_password: str | None = None,
reg_url: str | None = None,
) -> list[ContainerSpawnStatus] | SyftError:
results = []
Expand All @@ -524,8 +524,8 @@ def run_containers(
pool_name=pool_name,
queue_port=queue_port,
debug=dev_mode,
username=reg_username,
password=reg_password,
username=registry_username,
password=registry_password,
registry_url=reg_url,
)
results.append(spawn_result)
Expand All @@ -537,8 +537,8 @@ def run_containers(
queue_port=queue_port,
debug=dev_mode,
start_idx=start_idx,
reg_username=reg_username,
reg_password=reg_password,
registry_username=registry_username,
registry_password=registry_password,
reg_url=reg_url,
)

Expand Down
10 changes: 5 additions & 5 deletions packages/syft/src/syft/service/worker/worker_image_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ def build(
image_uid: UID,
tag: str,
registry_uid: UID | None = None,
pull: bool = True,
pull_image: bool = True,
) -> SyftSuccess | SyftError:
registry: SyftImageRegistry | None = None

Expand Down Expand Up @@ -130,7 +130,7 @@ def build(
result = None

if not context.node.in_memory_workers:
build_result = image_build(worker_image, pull=pull)
build_result = image_build(worker_image, pull=pull_image)
if isinstance(build_result, SyftError):
return build_result

Expand Down Expand Up @@ -162,14 +162,14 @@ def build(
def push(
self,
context: AuthedServiceContext,
image: UID,
image_uid: UID,
username: str | None = None,
password: str | None = None,
) -> SyftSuccess | SyftError:
result = self.stash.get_by_uid(credentials=context.credentials, uid=image)
result = self.stash.get_by_uid(credentials=context.credentials, uid=image_uid)
if result.is_err():
return SyftError(
message=f"Failed to get Image ID: {image}. Error: {result.err()}"
message=f"Failed to get Image ID: {image_uid}. Error: {result.err()}"
)
worker_image: SyftWorkerImage = result.ok()

Expand Down
40 changes: 21 additions & 19 deletions packages/syft/src/syft/service/worker/worker_pool_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,11 +64,11 @@ def __init__(self, store: DocumentStore) -> None:
def launch(
self,
context: AuthedServiceContext,
name: str,
pool_name: str,
image_uid: UID | None,
num_workers: int,
reg_username: str | None = None,
reg_password: str | None = None,
registry_username: str | None = None,
registry_password: str | None = None,
) -> list[ContainerSpawnStatus] | SyftError:
"""Creates a pool of workers from the given SyftWorkerImage.

Expand All @@ -84,13 +84,15 @@ def launch(
num_workers (int): the number of SyftWorker that needs to be created in the pool
"""

result = self.stash.get_by_name(context.credentials, pool_name=name)
result = self.stash.get_by_name(context.credentials, pool_name=pool_name)

if result.is_err():
return SyftError(message=f"{result.err()}")

if result.ok() is not None:
return SyftError(message=f"Worker Pool with name: {name} already exists !!")
return SyftError(
message=f"Worker Pool with name: {pool_name} already exists !!"
)

# If image uid is not passed, then use the default worker image
# to create the worker pool
Expand Down Expand Up @@ -119,13 +121,13 @@ def launch(
# and with the desired number of workers
result = _create_workers_in_pool(
context=context,
pool_name=name,
pool_name=pool_name,
existing_worker_cnt=0,
worker_cnt=num_workers,
worker_image=worker_image,
worker_stash=worker_stash,
reg_username=reg_username,
reg_password=reg_password,
registry_username=registry_username,
registry_password=registry_password,
)

if isinstance(result, SyftError):
Expand All @@ -135,7 +137,7 @@ def launch(

# Update the Database with the pool information
worker_pool = WorkerPool(
name=name,
name=pool_name,
max_count=num_workers,
image_id=worker_image.id,
worker_list=worker_list,
Expand Down Expand Up @@ -364,8 +366,8 @@ def add_workers(
number: int,
pool_id: UID | None = None,
pool_name: str | None = None,
reg_username: str | None = None,
reg_password: str | None = None,
registry_username: str | None = None,
registry_password: str | None = None,
) -> list[ContainerSpawnStatus] | SyftError:
"""Add workers to existing worker pool.

Expand Down Expand Up @@ -423,8 +425,8 @@ def add_workers(
worker_cnt=number,
worker_image=worker_image,
worker_stash=worker_stash,
reg_username=reg_username,
reg_password=reg_password,
registry_username=registry_username,
registry_password=registry_password,
)

if isinstance(result, SyftError):
Expand Down Expand Up @@ -485,8 +487,8 @@ def scale(
pool_id=pool_id,
pool_name=pool_name,
# kube scaling doesn't require password as it replicates an existing deployment
reg_username=None,
reg_password=None,
registry_username=None,
registry_password=None,
)
if isinstance(result, SyftError):
return result
Expand Down Expand Up @@ -654,8 +656,8 @@ def _create_workers_in_pool(
worker_cnt: int,
worker_image: SyftWorkerImage,
worker_stash: WorkerStash,
reg_username: str | None = None,
reg_password: str | None = None,
registry_username: str | None = None,
registry_password: str | None = None,
) -> tuple[list[LinkedObject], list[ContainerSpawnStatus]] | SyftError:
queue_port = context.node.queue_config.client_config.queue_port

Expand Down Expand Up @@ -684,8 +686,8 @@ def _create_workers_in_pool(
orchestration=get_orchestration_type(),
queue_port=queue_port,
dev_mode=context.node.dev_mode,
reg_username=reg_username,
reg_password=reg_password,
registry_username=registry_username,
registry_password=registry_password,
reg_url=registry_host,
)
if isinstance(result, SyftError):
Expand Down
Loading
Loading