Skip to content

Commit

Permalink
Merge pull request #8437 from OpenMined/issue-955
Browse files Browse the repository at this point in the history
Add a test image build method to DockerWorkerConfig
  • Loading branch information
shubham3121 authored Feb 7, 2024
2 parents d10873f + 1a7714e commit c110d7e
Show file tree
Hide file tree
Showing 5 changed files with 65 additions and 12 deletions.
24 changes: 23 additions & 1 deletion notebooks/api/0.8/10-container-images.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,28 @@
"docker_config = sy.DockerWorkerConfig(dockerfile=custom_dockerfile_str)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "62762ceb-38da-46f1-acac-cdf5bbf29513",
"metadata": {},
"outputs": [],
"source": [
"# test image build locally\n",
"test_build_res = docker_config.test_image_build(tag=\"openmined/custom-worker:0.7.8\")\n",
"test_build_res"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0235e567-c65c-48fe-825d-79ea3e219166",
"metadata": {},
"outputs": [],
"source": [
"assert isinstance(test_build_res, sy.SyftSuccess), str(test_build_res)"
]
},
{
"cell_type": "code",
"execution_count": null,
Expand Down Expand Up @@ -1406,7 +1428,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.7"
"version": "3.9.7"
}
},
"nbformat": 4,
Expand Down
13 changes: 2 additions & 11 deletions packages/syft/src/syft/custom_worker/builder_docker.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
# stdlib
import contextlib
import io
import json
from pathlib import Path
from typing import Iterable
from typing import Optional
Expand All @@ -13,6 +12,7 @@
from .builder_types import BuilderBase
from .builder_types import ImageBuildResult
from .builder_types import ImagePushResult
from .utils import iterator_to_string

__all__ = ["DockerBuilder"]

Expand Down Expand Up @@ -70,13 +70,4 @@ def push_image(
return ImagePushResult(logs=result, exit_code=0)

def _parse_output(self, log_iterator: Iterable) -> str:
log = ""
for line in log_iterator:
for item in line.values():
if isinstance(item, str):
log += item
elif isinstance(item, dict):
log += json.dumps(item) + "\n"
else:
log += str(item)
return log
return iterator_to_string(iterator=log_iterator)
23 changes: 23 additions & 0 deletions packages/syft/src/syft/custom_worker/config.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
# stdlib
import contextlib
from hashlib import sha256
import io
from pathlib import Path
from typing import Any
from typing import Dict
Expand All @@ -8,14 +10,18 @@
from typing import Union

# third party
import docker
from packaging import version
from pydantic import validator
from typing_extensions import Self
import yaml

# relative
from ..serde.serializable import serializable
from ..service.response import SyftError
from ..service.response import SyftSuccess
from ..types.base import SyftBaseModel
from .utils import iterator_to_string

PYTHON_DEFAULT_VER = "3.11"
PYTHON_MIN_VER = version.parse("3.10")
Expand Down Expand Up @@ -159,3 +165,20 @@ def __str__(self) -> str:

def set_description(self, description_text: str) -> None:
self.description = description_text

def test_image_build(self, tag: str, **kwargs) -> Union[SyftSuccess, SyftError]:
try:
with contextlib.closing(docker.from_env()) as client:
if not client.ping():
return SyftError(
"Cannot reach docker server. Please check if docker is running."
)

kwargs["fileobj"] = io.BytesIO(self.dockerfile.encode("utf-8"))
_, logs = client.images.build(
tag=tag,
**kwargs,
)
return SyftSuccess(message=iterator_to_string(iterator=logs))
except Exception as e:
return SyftError(message=f"Failed to build: {e}")
15 changes: 15 additions & 0 deletions packages/syft/src/syft/custom_worker/utils.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,23 @@
# stdlib
import json
from typing import Iterable
from typing import Optional
from typing import Tuple


def iterator_to_string(iterator: Iterable) -> str:
log = ""
for line in iterator:
for item in line.values():
if isinstance(item, str):
log += item
elif isinstance(item, dict):
log += json.dumps(item) + "\n"
else:
log += str(item)
return log


class ImageUtils:
@staticmethod
def parse_tag(tag: str) -> Tuple[Optional[str], str, str]:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ def dataset_2(client_do_2):
return client_do_2.datasets[0].assets[0]


@pytest.mark.flaky(reruns=2, reruns_delay=1)
def test_transfer_request_blocking(
client_ds_1, client_do_1, client_do_2, dataset_1, dataset_2
):
Expand Down Expand Up @@ -147,6 +148,7 @@ def compute_sum(data) -> float:
assert result_ds_blocking == result_ds_nonblocking == dataset_2.data.mean()


@pytest.mark.flaky(reruns=2, reruns_delay=1)
def test_transfer_request_nonblocking(
client_ds_1, client_do_1, client_do_2, dataset_1, dataset_2
):
Expand Down

0 comments on commit c110d7e

Please sign in to comment.