Merge branch 'dev' into yash/scale

notebooks/api/0.8/10-container-images.ipynb

@@ -133,6 +133,28 @@
     "docker_config = sy.DockerWorkerConfig(dockerfile=custom_dockerfile_str)"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "62762ceb-38da-46f1-acac-cdf5bbf29513",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# test image build locally\n",
+    "test_build_res = docker_config.test_image_build(tag=\"openmined/custom-worker:0.7.8\")\n",
+    "test_build_res"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0235e567-c65c-48fe-825d-79ea3e219166",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "assert isinstance(test_build_res, sy.SyftSuccess), str(test_build_res)"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -1406,7 +1428,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.7"
+   "version": "3.9.7"
   }
  },
  "nbformat": 4,

packages/syft/src/syft/custom_worker/builder_docker.py

@@ -1,7 +1,6 @@
 # stdlib
 import contextlib
 import io
-import json
 from pathlib import Path
 from typing import Iterable
 from typing import Optional
@@ -14,6 +13,7 @@
 from .builder_types import BuilderBase
 from .builder_types import ImageBuildResult
 from .builder_types import ImagePushResult
+from .utils import iterator_to_string
 
 __all__ = ["DockerBuilder"]
 
@@ -69,13 +69,4 @@ def push_image(
             return ImagePushResult(logs=result, exit_code=0)
 
     def _parse_output(self, log_iterator: Iterable) -> str:
-        log = ""
-        for line in log_iterator:
-            for item in line.values():
-                if isinstance(item, str):
-                    log += item
-                elif isinstance(item, dict):
-                    log += json.dumps(item) + "\n"
-                else:
-                    log += str(item)
-        return log
+        return iterator_to_string(iterator=log_iterator)

packages/syft/src/syft/custom_worker/config.py

@@ -1,5 +1,7 @@
 # stdlib
+import contextlib
 from hashlib import sha256
+import io
 from pathlib import Path
 from typing import Any
 from typing import Dict
@@ -8,14 +10,18 @@
 from typing import Union
 
 # third party
+import docker
 from packaging import version
 from pydantic import validator
 from typing_extensions import Self
 import yaml
 
 # relative
 from ..serde.serializable import serializable
+from ..service.response import SyftError
+from ..service.response import SyftSuccess
 from ..types.base import SyftBaseModel
+from .utils import iterator_to_string
 
 PYTHON_DEFAULT_VER = "3.11"
 PYTHON_MIN_VER = version.parse("3.10")
@@ -159,3 +165,20 @@ def __str__(self) -> str:
 
     def set_description(self, description_text: str) -> None:
         self.description = description_text
+
+    def test_image_build(self, tag: str, **kwargs) -> Union[SyftSuccess, SyftError]:
+        try:
+            with contextlib.closing(docker.from_env()) as client:
+                if not client.ping():
+                    return SyftError(
+                        "Cannot reach docker server. Please check if docker is running."
+                    )
+
+                kwargs["fileobj"] = io.BytesIO(self.dockerfile.encode("utf-8"))
+                _, logs = client.images.build(
+                    tag=tag,
+                    **kwargs,
+                )
+                return SyftSuccess(message=iterator_to_string(iterator=logs))
+        except Exception as e:
+            return SyftError(message=f"Failed to build: {e}")

packages/syft/src/syft/custom_worker/utils.py

@@ -1,8 +1,23 @@
 # stdlib
+import json
+from typing import Iterable
 from typing import Optional
 from typing import Tuple
 
 
+def iterator_to_string(iterator: Iterable) -> str:
+    log = ""
+    for line in iterator:
+        for item in line.values():
+            if isinstance(item, str):
+                log += item
+            elif isinstance(item, dict):
+                log += json.dumps(item) + "\n"
+            else:
+                log += str(item)
+    return log
+
+
 class ImageUtils:
     @staticmethod
     def parse_tag(tag: str) -> Tuple[Optional[str], str, str]:

packages/syft/src/syft/service/code/user_code.py

@@ -462,6 +462,10 @@ def input_policy(self) -> Optional[InputPolicy]:
             print(f"Failed to deserialize custom input policy state. {e}")
             return None
 
+    @property
+    def output_policy_approved(self):
+        return self.status.approved
+
     @property
     def output_policy(self) -> Optional[OutputPolicy]:
         if not self.status.approved:

packages/syft/src/syft/service/code/user_code_service.py

@@ -311,7 +311,7 @@ def is_execution_allowed(self, code, context, output_policy):
         # Check if the user has permission to execute the code.
         elif not (has_code_permission := self.has_code_permission(code, context)):
             return has_code_permission
-        elif code.output_policy is None:
+        elif not code.output_policy_approved:
             return SyftError("Output policy not approved", code)
         elif not output_policy.valid:
             return output_policy.valid
@@ -399,9 +399,9 @@ def _call(
                     code=code, context=context, output_policy=output_policy
                 )
                 if not can_execute:
-                    if output_policy is None:
+                    if not code.output_policy_approved:
                         return Err(
-                            "UserCodeStatus.DENIED: Function has no output policy"
+                            "Execution denied: Your code is waiting for approval"
                         )
                     if not (is_valid := output_policy.valid):
                         if (

packages/syft/tests/syft/request/request_code_accept_deny_test.py

@@ -205,4 +205,4 @@ def simple_function(data):
 
     result = ds_client.code.simple_function(data=action_obj)
     assert isinstance(result, SyftError)
-    assert "UserCodeStatus.DENIED" in result.message
+    assert "Execution denied" in result.message

packages/syft/tests/syft/request/request_multiple_nodes_test.py

@@ -110,6 +110,7 @@ def dataset_2(client_do_2):
     return client_do_2.datasets[0].assets[0]
 
 
+@pytest.mark.flaky(reruns=2, reruns_delay=1)
 def test_transfer_request_blocking(
     client_ds_1, client_do_1, client_do_2, dataset_1, dataset_2
 ):
@@ -147,6 +148,7 @@ def compute_sum(data) -> float:
     assert result_ds_blocking == result_ds_nonblocking == dataset_2.data.mean()
 
 
+@pytest.mark.flaky(reruns=2, reruns_delay=1)
 def test_transfer_request_nonblocking(
     client_ds_1, client_do_1, client_do_2, dataset_1, dataset_2
 ):