diff --git a/notebooks/api/0.8/10-container-images.ipynb b/notebooks/api/0.8/10-container-images.ipynb index 81ba6a05d13..35f26e791ff 100644 --- a/notebooks/api/0.8/10-container-images.ipynb +++ b/notebooks/api/0.8/10-container-images.ipynb @@ -133,6 +133,28 @@ "docker_config = sy.DockerWorkerConfig(dockerfile=custom_dockerfile_str)" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "62762ceb-38da-46f1-acac-cdf5bbf29513", + "metadata": {}, + "outputs": [], + "source": [ + "# test image build locally\n", + "test_build_res = docker_config.test_image_build(tag=\"openmined/custom-worker:0.7.8\")\n", + "test_build_res" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0235e567-c65c-48fe-825d-79ea3e219166", + "metadata": {}, + "outputs": [], + "source": [ + "assert isinstance(test_build_res, sy.SyftSuccess), str(test_build_res)" + ] + }, { "cell_type": "code", "execution_count": null, @@ -1406,7 +1428,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.7" + "version": "3.9.7" } }, "nbformat": 4, diff --git a/packages/syft/src/syft/custom_worker/builder_docker.py b/packages/syft/src/syft/custom_worker/builder_docker.py index 9eda180e08b..3f7f16cf185 100644 --- a/packages/syft/src/syft/custom_worker/builder_docker.py +++ b/packages/syft/src/syft/custom_worker/builder_docker.py @@ -1,7 +1,6 @@ # stdlib import contextlib import io -import json from pathlib import Path from typing import Iterable from typing import Optional @@ -14,6 +13,7 @@ from .builder_types import BuilderBase from .builder_types import ImageBuildResult from .builder_types import ImagePushResult +from .utils import iterator_to_string __all__ = ["DockerBuilder"] @@ -69,13 +69,4 @@ def push_image( return ImagePushResult(logs=result, exit_code=0) def _parse_output(self, log_iterator: Iterable) -> str: - log = "" - for line in log_iterator: - for item in line.values(): - if isinstance(item, str): - log += item - elif isinstance(item, dict): - log += json.dumps(item) + "\n" - else: - log += str(item) - return log + return iterator_to_string(iterator=log_iterator) diff --git a/packages/syft/src/syft/custom_worker/config.py b/packages/syft/src/syft/custom_worker/config.py index 7c3867a5392..c54d4f77c40 100644 --- a/packages/syft/src/syft/custom_worker/config.py +++ b/packages/syft/src/syft/custom_worker/config.py @@ -1,5 +1,7 @@ # stdlib +import contextlib from hashlib import sha256 +import io from pathlib import Path from typing import Any from typing import Dict @@ -8,6 +10,7 @@ from typing import Union # third party +import docker from packaging import version from pydantic import validator from typing_extensions import Self @@ -15,7 +18,10 @@ # relative from ..serde.serializable import serializable +from ..service.response import SyftError +from ..service.response import SyftSuccess from ..types.base import SyftBaseModel +from .utils import iterator_to_string PYTHON_DEFAULT_VER = "3.11" PYTHON_MIN_VER = version.parse("3.10") @@ -159,3 +165,20 @@ def __str__(self) -> str: def set_description(self, description_text: str) -> None: self.description = description_text + + def test_image_build(self, tag: str, **kwargs) -> Union[SyftSuccess, SyftError]: + try: + with contextlib.closing(docker.from_env()) as client: + if not client.ping(): + return SyftError( + "Cannot reach docker server. Please check if docker is running." + ) + + kwargs["fileobj"] = io.BytesIO(self.dockerfile.encode("utf-8")) + _, logs = client.images.build( + tag=tag, + **kwargs, + ) + return SyftSuccess(message=iterator_to_string(iterator=logs)) + except Exception as e: + return SyftError(message=f"Failed to build: {e}") diff --git a/packages/syft/src/syft/custom_worker/utils.py b/packages/syft/src/syft/custom_worker/utils.py index 4ec45cc1358..597e4bb6aff 100644 --- a/packages/syft/src/syft/custom_worker/utils.py +++ b/packages/syft/src/syft/custom_worker/utils.py @@ -1,8 +1,23 @@ # stdlib +import json +from typing import Iterable from typing import Optional from typing import Tuple +def iterator_to_string(iterator: Iterable) -> str: + log = "" + for line in iterator: + for item in line.values(): + if isinstance(item, str): + log += item + elif isinstance(item, dict): + log += json.dumps(item) + "\n" + else: + log += str(item) + return log + + class ImageUtils: @staticmethod def parse_tag(tag: str) -> Tuple[Optional[str], str, str]: diff --git a/packages/syft/src/syft/service/code/user_code.py b/packages/syft/src/syft/service/code/user_code.py index 020830d2508..a449845115d 100644 --- a/packages/syft/src/syft/service/code/user_code.py +++ b/packages/syft/src/syft/service/code/user_code.py @@ -462,6 +462,10 @@ def input_policy(self) -> Optional[InputPolicy]: print(f"Failed to deserialize custom input policy state. {e}") return None + @property + def output_policy_approved(self): + return self.status.approved + @property def output_policy(self) -> Optional[OutputPolicy]: if not self.status.approved: diff --git a/packages/syft/src/syft/service/code/user_code_service.py b/packages/syft/src/syft/service/code/user_code_service.py index 20c8630c231..da409b1dac0 100644 --- a/packages/syft/src/syft/service/code/user_code_service.py +++ b/packages/syft/src/syft/service/code/user_code_service.py @@ -311,7 +311,7 @@ def is_execution_allowed(self, code, context, output_policy): # Check if the user has permission to execute the code. elif not (has_code_permission := self.has_code_permission(code, context)): return has_code_permission - elif code.output_policy is None: + elif not code.output_policy_approved: return SyftError("Output policy not approved", code) elif not output_policy.valid: return output_policy.valid @@ -399,9 +399,9 @@ def _call( code=code, context=context, output_policy=output_policy ) if not can_execute: - if output_policy is None: + if not code.output_policy_approved: return Err( - "UserCodeStatus.DENIED: Function has no output policy" + "Execution denied: Your code is waiting for approval" ) if not (is_valid := output_policy.valid): if ( diff --git a/packages/syft/tests/syft/request/request_code_accept_deny_test.py b/packages/syft/tests/syft/request/request_code_accept_deny_test.py index 242de50ff85..7451dd26d91 100644 --- a/packages/syft/tests/syft/request/request_code_accept_deny_test.py +++ b/packages/syft/tests/syft/request/request_code_accept_deny_test.py @@ -205,4 +205,4 @@ def simple_function(data): result = ds_client.code.simple_function(data=action_obj) assert isinstance(result, SyftError) - assert "UserCodeStatus.DENIED" in result.message + assert "Execution denied" in result.message diff --git a/packages/syft/tests/syft/request/request_multiple_nodes_test.py b/packages/syft/tests/syft/request/request_multiple_nodes_test.py index 4beb780cc31..9ec214ea7fe 100644 --- a/packages/syft/tests/syft/request/request_multiple_nodes_test.py +++ b/packages/syft/tests/syft/request/request_multiple_nodes_test.py @@ -110,6 +110,7 @@ def dataset_2(client_do_2): return client_do_2.datasets[0].assets[0] +@pytest.mark.flaky(reruns=2, reruns_delay=1) def test_transfer_request_blocking( client_ds_1, client_do_1, client_do_2, dataset_1, dataset_2 ): @@ -147,6 +148,7 @@ def compute_sum(data) -> float: assert result_ds_blocking == result_ds_nonblocking == dataset_2.data.mean() +@pytest.mark.flaky(reruns=2, reruns_delay=1) def test_transfer_request_nonblocking( client_ds_1, client_do_1, client_do_2, dataset_1, dataset_2 ):