Merge pull request #8311 from OpenMined/container-workload-1

Container Workload I

notebooks/api/0.8/10-container-images.ipynb

@@ -0,0 +1,364 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ab8aca22-8bd7-4764-8f2d-27dd5f33d8c6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "SYFT_VERSION = \">=0.8.2.b0,<0.9\"\n",
+    "package_string = f'\"syft{SYFT_VERSION}\"'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2cb8c995-c806-4b8e-a892-9bc461c61935",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import syft as sy\n",
+    "sy.requires(SYFT_VERSION)\n",
+    "from syft.service.worker.worker_image import SyftWorkerImage, SyftWorkerImageTag\n",
+    "from syft.custom_worker.config import DockerWorkerConfig\n",
+    "from syft.service.worker.worker_image import build_using_docker\n",
+    "from syft.service.worker.utils import run_container_using_docker\n",
+    "\n",
+    "\n",
+    "#third party\n",
+    "import docker"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0bc7b5dc-1565-4261-ac98-db2602c5877b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "domain  = sy.orchestra.launch(name=\"test-domain-1\", reset=True, dev_mode=True, port=\"auto\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "91f1988a-daa3-42f0-9bfe-f9fdd9597fdc",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "domain_client = domain.login(email=\"[email protected]\", password=\"changethis\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "75193f9f-3622-4071-9aba-d42a5dc5b301",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "nginx_dockerfile_str = \"\"\"\n",
+    "# Use the official Nginx image as the base\n",
+    "FROM nginx:latest\n",
+    "\n",
+    "# Expose port 80 for the Nginx server\n",
+    "EXPOSE 80\n",
+    "\n",
+    "# Start Nginx when the container has provisioned\n",
+    "CMD [\"nginx\", \"-g\", \"daemon off;\"]\n",
+    "\"\"\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b6bfe92a-e873-4dc3-b3a0-6715f8843785",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "docker_config = DockerWorkerConfig(dockerfile=nginx_dockerfile_str)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "941cf5e2-4ba8-488f-880b-de908d23a4c3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "assert docker_config.dockerfile == nginx_dockerfile_str"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d4a60bf8-22d3-4052-b9cc-f6dcf68b2dd8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res = domain_client.api.services.worker_image.submit_dockerfile(docker_config=docker_config)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "730df31b-7c23-4068-a275-419526c3ee6f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ebb3b7e9-c7a4-4c99-866b-13c6a75d04e8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "assert isinstance( res, sy.SyftSuccess)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d9cc2eb9-9f28-454f-96bc-fbb722f78bb5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dockerfile_list = domain_client.api.services.worker_image.get_all()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8e56f9e8-5cf3-418b-9774-75a47c8ef276",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "assert len(dockerfile_list) == 1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "21836707-891c-4e2b-87b5-51de8050da9f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "workerimage = dockerfile_list[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "133dacbe-4d2e-458e-830b-2c18bce018e4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "assert isinstance(workerimage, SyftWorkerImage)\n",
+    "assert workerimage.config.dockerfile == nginx_dockerfile_str"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "aa6573e1-ea18-4049-b6bf-1615521d8ced",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "docker_tag = \"openmined/test-nginx:0.7.8\"\n",
+    "docker_build_res = domain_client.api.services.worker_image.build(uid=workerimage.id, tag=docker_tag)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "573869c9-2bf9-4110-892b-edc75f0cd48d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "docker_build_res"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "21e3679d-ef71-44af-a2ab-91bed47472c1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "assert isinstance(docker_build_res, sy.SyftSuccess)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c7ccfd18-2bae-4879-b6c2-27b6bcccf8f5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import subprocess\n",
+    "\n",
+    "def check_image_exists(tag) -> bool:\n",
+    "    result = subprocess.run(['docker', 'images', '-q', tag], stdout=subprocess.PIPE)\n",
+    "    return result.stdout.strip() != b''\n",
+    "assert check_image_exists(docker_tag)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c540043d-4485-4213-b93c-358e4c507f5a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "image_list = domain_client.api.services.worker_image.get_all()\n",
+    "image_list"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2dd5d306-1726-4d68-a1fd-2513bbaf568c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "assert len(image_list) == 1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "990d2cf3-2148-4a67-b17f-486efc5ccb02",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import docker\n",
+    "\n",
+    "def get_image_hash(tag) -> str:\n",
+    "    client = docker.from_env()\n",
+    "    try:\n",
+    "        image = client.images.get(tag)\n",
+    "        return image.id\n",
+    "    except docker.errors.ImageNotFound:\n",
+    "        return None"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b2829070-f156-4dbd-b1ee-1e3f654f5b7b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "assert image_list[0].image_hash == get_image_hash(docker_tag)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5692f3a9-f493-467a-a394-7d8e62b970e5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_container_id(container_name: str) -> str:\n",
+    "    client = docker.from_env()\n",
+    "    try:\n",
+    "        container = client.containers.get(container_name)\n",
+    "        return container.id\n",
+    "    except docker.errors.NotFound:\n",
+    "        return None"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f57b5443-8519-4464-89a2-37deb25f6923",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "worker_pool_name = \"my_first_worker_pool\"\n",
+    "worker_pool_res = domain_client.api.services.worker_pool.create(name=worker_pool_name, image_uid = image_list[0].id, number=3)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f418fb83-4111-412c-ab11-8d4587239dc6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "assert len(worker_pool_res) == 3"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "64b5d651-3dd6-45e6-b189-c7e278a7ddd1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for status in worker_pool_res:\n",
+    "    assert status.error == None\n",
+    "    assert status.worker.image_hash == get_image_hash(docker_tag)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "98ad4890-0424-4d84-af00-15bfa445c64d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# It would be move to api in worker_pool_service, currently it cleans up the resources of the created\n",
+    "# workerpool\n",
+    "client = docker.from_env()\n",
+    "for status in worker_pool_res:\n",
+    "    container = client.containers.get(status.worker.container_id)\n",
+    "    container.stop()\n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b2ee8588-f435-4431-b456-dd5b1ff7eec6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "delete_res = domain_client.api.services.worker_image.delete(workerimage.id)\n",
+    "assert isinstance(delete_res, sy.SyftSuccess)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1b5256d6-a9bc-4de2-a771-d3c0147ba1b4",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

packages/grid/docker-compose.yml

@@ -110,6 +110,7 @@ services:
       - "${HTTP_PORT}:${HTTP_PORT}"
     volumes:
       - credentials-data:/root/data/creds/
+      - /var/run/docker.sock:/var/run/docker.sock
     extra_hosts:
       - "host.docker.internal:host-gateway"
     labels: