From 64831318ddcd68e6a940aa713573a8b31a9dae2e Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Mon, 20 Nov 2023 21:32:16 +0100 Subject: [PATCH 01/12] nb --- notebooks/helm/local_jobs_execution.ipynb | 2665 +++++++++++++++++++++ 1 file changed, 2665 insertions(+) create mode 100644 notebooks/helm/local_jobs_execution.ipynb diff --git a/notebooks/helm/local_jobs_execution.ipynb b/notebooks/helm/local_jobs_execution.ipynb new file mode 100644 index 00000000000..9eef50bedcc --- /dev/null +++ b/notebooks/helm/local_jobs_execution.ipynb @@ -0,0 +1,2665 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "a196017f", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "kj/filesystem-disk-unix.c++:1703: warning: PWD environment variable doesn't match current directory; pwd = /Users/koen/workspace/pysyft\n" + ] + } + ], + "source": [ + "import syft as sy\n", + "from syft import ActionObject\n", + "from syft import syft_function, syft_function_single_use\n", + "from time import sleep\n", + "import os\n", + "import psutil\n", + "import inspect" + ] + }, + { + "cell_type": "markdown", + "id": "cb2d07de", + "metadata": {}, + "source": [ + "with server" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "9b31c627", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Staging Protocol Changes...\n", + "Logged into as \n" + ] + }, + { + "data": { + "text/html": [ + "
SyftWarning: You are using a default password. Please change the password using `[your_client].me.set_password([new_password])`.

" + ], + "text/plain": [ + "SyftWarning: You are using a default password. Please change the password using `[your_client].me.set_password([new_password])`." + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "node = sy.orchestra.launch(name=\"test-domain-helm2\",\n", + " dev_mode=True,\n", + " reset=True, \n", + " n_consumers=3,\n", + " create_producer=True,\n", + " queue_port=3322)\n", + " \n", + "client = node.login(email=\"info@openmined.org\", password=\"changethis\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "507740d2", + "metadata": {}, + "outputs": [], + "source": [ + "res = client.register(name=\"a\", email=\"aa@b.org\", password=\"c\", password_verify=\"c\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "0c33d096", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Logged into as \n" + ] + } + ], + "source": [ + "ds_client = node.login(email=\"aa@b.org\", password=\"c\")" + ] + }, + { + "cell_type": "markdown", + "id": "176addfb", + "metadata": {}, + "source": [ + "setup: compute train-test overlap between a very large train set and a smaller test set. Small test is still to big for memory, so we split it into 54 parts. We keep 1 of those parts in memory. We dont keep the train set in memory, but read and compare with 1/54 parts line by line. Each part takes ~30 hours, but we can run 54 processes in parallel." + ] + }, + { + "cell_type": "markdown", + "id": "a0cea81b", + "metadata": {}, + "source": [ + "# Setup syft functions" + ] + }, + { + "cell_type": "markdown", + "id": "da2b114a", + "metadata": {}, + "source": [ + "## Dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "fbe87adc", + "metadata": {}, + "outputs": [], + "source": [ + "dataset = sy.Dataset(\n", + " name=\"My dataset\",\n", + " asset_list=[\n", + " sy.Asset(\n", + " name=\"input asset\",\n", + " data=ActionObject.from_obj([1, 2]),\n", + " mock=ActionObject.from_obj([1, 2]),\n", + " ),\n", + " ]\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "a76e2d15", + "metadata": {}, + "outputs": [], + "source": [ + "# dataset.assets[0].mock.syft_action_data\n", + "\n", + "# dataset.assets[0].data.syft_action_data\n", + "\n", + "# dataset.assets[0].mock.__len__()\n", + "\n", + "# dataset.assets[0].data.__len__()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "7661e9a7", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "[1, 2]" + ], + "text/plain": [ + "[1, 2]" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataset.assets[0].data.syft_action_data" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "b0714fba", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 10.25it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Uploading: input asset\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "text/html": [ + "
SyftSuccess: Dataset uploaded to 'test-domain-helm2'. To see the datasets uploaded by a client on this node, use command `[your_client].datasets`

" + ], + "text/plain": [ + "SyftSuccess: Dataset uploaded to 'test-domain-helm2'. To see the datasets uploaded by a client on this node, use command `[your_client].datasets`" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "client.upload_dataset(dataset)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "18ccb235", + "metadata": {}, + "outputs": [], + "source": [ + "asset = ds_client.datasets[0].assets[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "83307a2f", + "metadata": {}, + "outputs": [], + "source": [ + "# x = ActionObject.from_obj([1, 2])\n", + "# x_ptr = x.send(ds_client)" + ] + }, + { + "cell_type": "markdown", + "id": "31bbb3ff", + "metadata": {}, + "source": [ + "## Batch function" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "5d2fd248", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
SyftSuccess: Syft function 'process_batch' successfully created. To add a code request, please create a project using `project = syft.Project(...)`, then use command `project.create_code_request`.

" + ], + "text/plain": [ + "SyftSuccess: Syft function 'process_batch' successfully created. To add a code request, please create a project using `project = syft.Project(...)`, then use command `project.create_code_request`." + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "@syft_function()\n", + "def process_batch(batch):\n", + " # takes 30 hours normally\n", + " print(f\"starting batch {batch}\")\n", + " from time import sleep\n", + " sleep(1)\n", + " print(\"done\")\n", + " return batch+1" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "9ba22655", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
SyftSuccess: User Code Submitted

" + ], + "text/plain": [ + "SyftSuccess: User Code Submitted" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds_client.code.submit(process_batch)" + ] + }, + { + "cell_type": "markdown", + "id": "01319f1f", + "metadata": {}, + "source": [ + "## Main function" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "ca1b95ee", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
SyftSuccess: Syft function 'process_all' successfully created. To add a code request, please create a project using `project = syft.Project(...)`, then use command `project.create_code_request`.

" + ], + "text/plain": [ + "SyftSuccess: Syft function 'process_all' successfully created. To add a code request, please create a project using `project = syft.Project(...)`, then use command `project.create_code_request`." + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "@syft_function_single_use(x=asset)\n", + "def process_all(domain, x):\n", + " \n", + " jobs = []\n", + " print(\"Launching jobs\")\n", + " for elem in x:\n", + " # We inject a domain object in the scope\n", + " batch_job = domain.launch_job(process_batch, batch=elem)\n", + " jobs += [batch_job]\n", + " print(\"starting aggregation\")\n", + " print(\"Done\")\n", + " return 3" + ] + }, + { + "cell_type": "markdown", + "id": "1e77c5db", + "metadata": {}, + "source": [ + "# Approve & run" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0ab572f9", + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Request approved for domain test-domain-helm2\n" + ] + }, + { + "data": { + "text/html": [ + "
SyftSuccess: Request 5147062631e1422f8a113120f153afa4 changes applied

" + ], + "text/plain": [ + "SyftSuccess: Request 5147062631e1422f8a113120f153afa4 changes applied" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds_client.code.request_code_execution(process_all)\n", + "client.requests[-1].approve()" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "70a97140", + "metadata": {}, + "outputs": [], + "source": [ + "# ds_client.code.process_all(x=asset, blocking=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "9eb4f262", + "metadata": {}, + "outputs": [], + "source": [ + "mock =asset.mock" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "4f899e3c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "[1, 2]" + ], + "text/plain": [ + "[1, 2]" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "mock" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0c2567a7", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "10a606cc", + "metadata": {}, + "outputs": [], + "source": [ + "# \"domain\" in sig.parameters" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "e8d689e1", + "metadata": {}, + "outputs": [], + "source": [ + "from syft.node.worker import Worker" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "1fe858a0", + "metadata": {}, + "outputs": [], + "source": [ + "from syft.node.node import Node" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "d0b018e4", + "metadata": {}, + "outputs": [], + "source": [ + "# process_all\n", + "# process_all(x=[3,4])" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "45799cce", + "metadata": {}, + "outputs": [], + "source": [ + "# domain = Node.named(name=\"plan_building\", reset=True, processes=0)" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "cfb1f932", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Staging Protocol Changes...\n", + "Logged into as \n" + ] + }, + { + "data": { + "text/html": [ + "
SyftWarning: You are using a default password. Please change the password using `[your_client].me.set_password([new_password])`.

" + ], + "text/plain": [ + "SyftWarning: You are using a default password. Please change the password using `[your_client].me.set_password([new_password])`." + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "domain = sy.orchestra.launch(name=\"test\",\n", + " dev_mode=True,\n", + " reset=True, \n", + " n_consumers=3,\n", + " create_producer=True,\n", + " queue_port=22221)\n", + " \n", + "temp_client = node.login(email=\"info@openmined.org\", password=\"changethis\")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "3033771b", + "metadata": {}, + "outputs": [], + "source": [ + "# handle = NodeHandle(\n", + "# node_type=node_type_enum,\n", + "# deployment_type=deployment_type_enum,\n", + "# name=name,\n", + "# python_node=worker,\n", + "# node_side_type=node_side_type,\n", + "# )" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "abf60134", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + ".wrapper(*args, **kwargs)>" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds_client.code.process_all" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "f672b6da", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + ">" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "client.launch_job" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "a23c34bd", + "metadata": {}, + "outputs": [], + "source": [ + "# from hagrid.hagrid.orchestra" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "6697fa53", + "metadata": {}, + "outputs": [], + "source": [ + "# handle.login(email=\"info@openmined.org\", password=\"changethis\")" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "fcadf9c1", + "metadata": {}, + "outputs": [], + "source": [ + "filtered_kwargs = {\"x\": [3,4]}" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "7e42eb6d", + "metadata": {}, + "outputs": [], + "source": [ + "filtered_kwargs[\"domain\"] = temp_client" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "57de2a1e", + "metadata": {}, + "outputs": [], + "source": [ + "action_kwargs = {}\n", + "for k, v in filtered_kwargs.items():\n", + " val = ActionObject.from_obj(v)\n", + " action_kwargs[k] = val.send(temp_client)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "id": "23e9686a", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Launching jobs\n" + ] + }, + { + "ename": "AttributeError", + "evalue": "'SQLiteStorePartition' object has no attribute 'lock'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[38], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mprocess_all\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlocal_function\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43maction_kwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "Cell \u001b[0;32mIn[13], line 8\u001b[0m, in \u001b[0;36mprocess_all\u001b[0;34m(domain, x)\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mLaunching jobs\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 6\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m elem \u001b[38;5;129;01min\u001b[39;00m x:\n\u001b[1;32m 7\u001b[0m \u001b[38;5;66;03m# We inject a domain object in the scope\u001b[39;00m\n\u001b[0;32m----> 8\u001b[0m batch_job \u001b[38;5;241m=\u001b[39m \u001b[43mdomain\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlaunch_job\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprocess_batch\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbatch\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43melem\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 9\u001b[0m jobs \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m [batch_job]\n\u001b[1;32m 10\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mstarting aggregation\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", + "File \u001b[0;32m~/workspace/PySyft/packages/syft/src/syft/service/action/action_object.py:1400\u001b[0m, in \u001b[0;36mActionObject._syft_wrap_attribute_for_methods..wrapper\u001b[0;34m(_self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1399\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwrapper\u001b[39m(_self: Any, \u001b[38;5;241m*\u001b[39margs: Any, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs: Any):\n\u001b[0;32m-> 1400\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_base_wrapper\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/workspace/PySyft/packages/syft/src/syft/service/action/action_object.py:1389\u001b[0m, in \u001b[0;36mActionObject._syft_wrap_attribute_for_methods.._base_wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 1385\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 1386\u001b[0m original_args, original_kwargs \u001b[38;5;241m=\u001b[39m debox_args_and_kwargs(\n\u001b[1;32m 1387\u001b[0m pre_hook_args, pre_hook_kwargs\n\u001b[1;32m 1388\u001b[0m )\n\u001b[0;32m-> 1389\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[43moriginal_func\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43moriginal_args\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43moriginal_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1391\u001b[0m post_result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_syft_run_post_hooks__(context, name, result)\n\u001b[1;32m 1392\u001b[0m post_result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_syft_attr_propagate_ids(context, name, post_result)\n", + "File \u001b[0;32m~/workspace/PySyft/packages/syft/src/syft/client/domain_client.py:151\u001b[0m, in \u001b[0;36mDomainClient.launch_job\u001b[0;34m(self, func, *args, **kwargs)\u001b[0m\n\u001b[1;32m 149\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mlaunch_job\u001b[39m(\u001b[38;5;28mself\u001b[39m, func, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 150\u001b[0m kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mblocking\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[0;32m--> 151\u001b[0m func \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcode\u001b[49m, func\u001b[38;5;241m.\u001b[39mfunc_name)\n\u001b[1;32m 152\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m func(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n", + "File \u001b[0;32m~/workspace/PySyft/packages/syft/src/syft/client/domain_client.py:158\u001b[0m, in \u001b[0;36mDomainClient.code\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 154\u001b[0m \u001b[38;5;129m@property\u001b[39m\n\u001b[1;32m 155\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcode\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Optional[APIModule]:\n\u001b[1;32m 156\u001b[0m \u001b[38;5;66;03m# if self.api.refresh_api_callback is not None:\u001b[39;00m\n\u001b[1;32m 157\u001b[0m \u001b[38;5;66;03m# self.api.refresh_api_callback()\u001b[39;00m\n\u001b[0;32m--> 158\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mapi\u001b[49m\u001b[38;5;241m.\u001b[39mhas_service(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m):\n\u001b[1;32m 159\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mapi\u001b[38;5;241m.\u001b[39mservices\u001b[38;5;241m.\u001b[39mcode\n\u001b[1;32m 160\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n", + "File \u001b[0;32m~/workspace/PySyft/packages/syft/src/syft/client/client.py:586\u001b[0m, in \u001b[0;36mSyftClient.api\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 582\u001b[0m \u001b[38;5;129m@property\u001b[39m\n\u001b[1;32m 583\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mapi\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m SyftAPI:\n\u001b[1;32m 584\u001b[0m \u001b[38;5;66;03m# invalidate API\u001b[39;00m\n\u001b[1;32m 585\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_api \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mor\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_api\u001b[38;5;241m.\u001b[39msigning_key \u001b[38;5;241m!=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcredentials):\n\u001b[0;32m--> 586\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_fetch_api\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcredentials\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 588\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_api\n", + "File \u001b[0;32m~/workspace/PySyft/packages/syft/src/syft/client/client.py:824\u001b[0m, in \u001b[0;36mSyftClient._fetch_api\u001b[0;34m(self, credentials)\u001b[0m\n\u001b[1;32m 823\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_fetch_api\u001b[39m(\u001b[38;5;28mself\u001b[39m, credentials: SyftSigningKey):\n\u001b[0;32m--> 824\u001b[0m _api: SyftAPI \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconnection\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_api\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 825\u001b[0m \u001b[43m \u001b[49m\u001b[43mcredentials\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcredentials\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 826\u001b[0m \u001b[43m \u001b[49m\u001b[43mcommunication_protocol\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcommunication_protocol\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 827\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 829\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mrefresh_callback\u001b[39m():\n\u001b[1;32m 830\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_fetch_api(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcredentials)\n", + "File \u001b[0;32m~/workspace/PySyft/packages/syft/src/syft/client/client.py:371\u001b[0m, in \u001b[0;36mPythonConnection.get_api\u001b[0;34m(self, credentials, communication_protocol)\u001b[0m\n\u001b[1;32m 360\u001b[0m obj \u001b[38;5;241m=\u001b[39m forward_message_to_proxy(\n\u001b[1;32m 361\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmake_call,\n\u001b[1;32m 362\u001b[0m proxy_target_uid\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mproxy_target_uid,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 368\u001b[0m credentials\u001b[38;5;241m=\u001b[39mcredentials,\n\u001b[1;32m 369\u001b[0m )\n\u001b[1;32m 370\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 371\u001b[0m obj \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mnode\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_api\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 372\u001b[0m \u001b[43m \u001b[49m\u001b[43mfor_user\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcredentials\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mverify_key\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 373\u001b[0m \u001b[43m \u001b[49m\u001b[43mcommunication_protocol\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcommunication_protocol\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 374\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 375\u001b[0m obj\u001b[38;5;241m.\u001b[39mconnection \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\n\u001b[1;32m 376\u001b[0m obj\u001b[38;5;241m.\u001b[39msigning_key \u001b[38;5;241m=\u001b[39m credentials\n", + "File \u001b[0;32m~/workspace/PySyft/packages/syft/src/syft/node/node.py:1090\u001b[0m, in \u001b[0;36mNode.get_api\u001b[0;34m(self, for_user, communication_protocol)\u001b[0m\n\u001b[1;32m 1085\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget_api\u001b[39m(\n\u001b[1;32m 1086\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 1087\u001b[0m for_user: Optional[SyftVerifyKey] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m 1088\u001b[0m communication_protocol: Optional[PROTOCOL_TYPE] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m 1089\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m SyftAPI:\n\u001b[0;32m-> 1090\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mSyftAPI\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfor_user\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1091\u001b[0m \u001b[43m \u001b[49m\u001b[43mnode\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1092\u001b[0m \u001b[43m \u001b[49m\u001b[43muser_verify_key\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfor_user\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1093\u001b[0m \u001b[43m \u001b[49m\u001b[43mcommunication_protocol\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcommunication_protocol\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1094\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/workspace/PySyft/packages/syft/src/syft/client/api.py:507\u001b[0m, in \u001b[0;36mSyftAPI.for_user\u001b[0;34m(node, communication_protocol, user_verify_key)\u001b[0m\n\u001b[1;32m 503\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mservice\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcode\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01muser_code_service\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m UserCodeService\n\u001b[1;32m 505\u001b[0m \u001b[38;5;66;03m# find user role by verify_key\u001b[39;00m\n\u001b[1;32m 506\u001b[0m \u001b[38;5;66;03m# TODO: we should probably not allow empty verify keys but instead make user always register\u001b[39;00m\n\u001b[0;32m--> 507\u001b[0m role \u001b[38;5;241m=\u001b[39m \u001b[43mnode\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_role_for_credentials\u001b[49m\u001b[43m(\u001b[49m\u001b[43muser_verify_key\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 508\u001b[0m _user_service_config_registry \u001b[38;5;241m=\u001b[39m UserServiceConfigRegistry\u001b[38;5;241m.\u001b[39mfrom_role(role)\n\u001b[1;32m 509\u001b[0m _user_lib_config_registry \u001b[38;5;241m=\u001b[39m UserLibConfigRegistry\u001b[38;5;241m.\u001b[39mfrom_user(user_verify_key)\n", + "File \u001b[0;32m~/workspace/PySyft/packages/syft/src/syft/node/node.py:921\u001b[0m, in \u001b[0;36mNode.get_role_for_credentials\u001b[0;34m(self, credentials)\u001b[0m\n\u001b[1;32m 920\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget_role_for_credentials\u001b[39m(\u001b[38;5;28mself\u001b[39m, credentials: SyftVerifyKey) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m ServiceRole:\n\u001b[0;32m--> 921\u001b[0m role \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_service\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43muserservice\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_role_for_credentials\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 922\u001b[0m \u001b[43m \u001b[49m\u001b[43mcredentials\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcredentials\u001b[49m\n\u001b[1;32m 923\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 924\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m role\n", + "File \u001b[0;32m~/workspace/PySyft/packages/syft/src/syft/service/user/user_service.py:138\u001b[0m, in \u001b[0;36mUserService.get_role_for_credentials\u001b[0;34m(self, credentials)\u001b[0m\n\u001b[1;32m 132\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget_role_for_credentials\u001b[39m(\n\u001b[1;32m 133\u001b[0m \u001b[38;5;28mself\u001b[39m, credentials: Union[SyftVerifyKey, SyftSigningKey]\n\u001b[1;32m 134\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Union[Optional[ServiceRole], SyftError]:\n\u001b[1;32m 135\u001b[0m \u001b[38;5;66;03m# they could be different\u001b[39;00m\n\u001b[1;32m 137\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(credentials, SyftVerifyKey):\n\u001b[0;32m--> 138\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstash\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_by_verify_key\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 139\u001b[0m \u001b[43m \u001b[49m\u001b[43mcredentials\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcredentials\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mverify_key\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcredentials\u001b[49m\n\u001b[1;32m 140\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 141\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 142\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstash\u001b[38;5;241m.\u001b[39mget_by_signing_key(\n\u001b[1;32m 143\u001b[0m credentials\u001b[38;5;241m=\u001b[39mcredentials, signing_key\u001b[38;5;241m=\u001b[39mcredentials\n\u001b[1;32m 144\u001b[0m )\n", + "File \u001b[0;32m~/workspace/PySyft/packages/syft/src/syft/service/user/user_stash.py:106\u001b[0m, in \u001b[0;36mUserStash.get_by_verify_key\u001b[0;34m(self, credentials, verify_key)\u001b[0m\n\u001b[1;32m 104\u001b[0m verify_key \u001b[38;5;241m=\u001b[39m SyftVerifyKey\u001b[38;5;241m.\u001b[39mfrom_string(verify_key)\n\u001b[1;32m 105\u001b[0m qks \u001b[38;5;241m=\u001b[39m QueryKeys(qks\u001b[38;5;241m=\u001b[39m[VerifyKeyPartitionKey\u001b[38;5;241m.\u001b[39mwith_obj(verify_key)])\n\u001b[0;32m--> 106\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mquery_one\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcredentials\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcredentials\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mqks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mqks\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/workspace/PySyft/packages/syft/src/syft/store/document_store.py:636\u001b[0m, in \u001b[0;36mBaseStash.query_one\u001b[0;34m(self, credentials, qks, order_by)\u001b[0m\n\u001b[1;32m 630\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mquery_one\u001b[39m(\n\u001b[1;32m 631\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 632\u001b[0m credentials: SyftVerifyKey,\n\u001b[1;32m 633\u001b[0m qks: Union[QueryKey, QueryKeys],\n\u001b[1;32m 634\u001b[0m order_by: Optional[PartitionKey] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m 635\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Result[Optional[BaseStash\u001b[38;5;241m.\u001b[39mobject_type], \u001b[38;5;28mstr\u001b[39m]:\n\u001b[0;32m--> 636\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mquery_all\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 637\u001b[0m \u001b[43m \u001b[49m\u001b[43mcredentials\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcredentials\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mqks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mqks\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43morder_by\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43morder_by\u001b[49m\n\u001b[1;32m 638\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mand_then(first_or_none)\n", + "File \u001b[0;32m~/workspace/PySyft/packages/syft/src/syft/store/document_store.py:614\u001b[0m, in \u001b[0;36mBaseStash.query_all\u001b[0;34m(self, credentials, qks, order_by)\u001b[0m\n\u001b[1;32m 611\u001b[0m index_qks \u001b[38;5;241m=\u001b[39m QueryKeys(qks\u001b[38;5;241m=\u001b[39munique_keys)\n\u001b[1;32m 612\u001b[0m search_qks \u001b[38;5;241m=\u001b[39m QueryKeys(qks\u001b[38;5;241m=\u001b[39msearchable_keys)\n\u001b[0;32m--> 614\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpartition\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfind_index_or_search_keys\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 615\u001b[0m \u001b[43m \u001b[49m\u001b[43mcredentials\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcredentials\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 616\u001b[0m \u001b[43m \u001b[49m\u001b[43mindex_qks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mindex_qks\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 617\u001b[0m \u001b[43m \u001b[49m\u001b[43msearch_qks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msearch_qks\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 618\u001b[0m \u001b[43m \u001b[49m\u001b[43morder_by\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43morder_by\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 619\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/workspace/PySyft/packages/syft/src/syft/store/document_store.py:401\u001b[0m, in \u001b[0;36mStorePartition.find_index_or_search_keys\u001b[0;34m(self, credentials, index_qks, search_qks, order_by)\u001b[0m\n\u001b[1;32m 394\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mfind_index_or_search_keys\u001b[39m(\n\u001b[1;32m 395\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 396\u001b[0m credentials: SyftVerifyKey,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 399\u001b[0m order_by: Optional[PartitionKey] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m 400\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Result[List[SyftObject], \u001b[38;5;28mstr\u001b[39m]:\n\u001b[0;32m--> 401\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_thread_safe_cbk\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 402\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_find_index_or_search_keys\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 403\u001b[0m \u001b[43m \u001b[49m\u001b[43mcredentials\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 404\u001b[0m \u001b[43m \u001b[49m\u001b[43mindex_qks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mindex_qks\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 405\u001b[0m \u001b[43m \u001b[49m\u001b[43msearch_qks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msearch_qks\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 406\u001b[0m \u001b[43m \u001b[49m\u001b[43morder_by\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43morder_by\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 407\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/workspace/PySyft/packages/syft/src/syft/store/document_store.py:355\u001b[0m, in \u001b[0;36mStorePartition._thread_safe_cbk\u001b[0;34m(self, cbk, *args, **kwargs)\u001b[0m\n\u001b[1;32m 354\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_thread_safe_cbk\u001b[39m(\u001b[38;5;28mself\u001b[39m, cbk: Callable, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m--> 355\u001b[0m locked \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlock\u001b[49m\u001b[38;5;241m.\u001b[39macquire(blocking\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m 356\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m locked:\n\u001b[1;32m 357\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mFAILED TO LOCK\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", + "\u001b[0;31mAttributeError\u001b[0m: 'SQLiteStorePartition' object has no attribute 'lock'" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Idle worker expired: b'5C5B-F4BE'\n", + "Idle worker expired: b'D140-D0EF'\n", + "Idle worker expired: b'4A71-B0A6'\n", + "Idle worker expired: b'4999-97E5'\n", + "Idle worker expired: b'897B-20D9'\n" + ] + } + ], + "source": [ + "process_all.local_function(**action_kwargs)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f0c2f8c0", + "metadata": {}, + "outputs": [], + "source": [ + "# args, _, _, values = inspect.getargvalues(process_all.local_function)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "33cad669", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "8c5ef2e8", + "metadata": {}, + "outputs": [ + { + "ename": "TypeError", + "evalue": "process_all() missing 1 required positional argument: 'domain'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[11], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mprocess_all\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m3\u001b[39;49m\u001b[43m,\u001b[49m\u001b[38;5;241;43m4\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/workspace/PySyft/packages/syft/src/syft/service/code/user_code.py:593\u001b[0m, in \u001b[0;36mSubmitUserCode.__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 591\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m on_mock_data:\n\u001b[1;32m 592\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mWarning: The result you see is computed on MOCK data.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 593\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlocal_function\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mfiltered_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 594\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 595\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mNotImplementedError\u001b[39;00m\n", + "\u001b[0;31mTypeError\u001b[0m: process_all() missing 1 required positional argument: 'domain'" + ] + } + ], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a3a91fde", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dc22e8cf", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5283e423", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "345e5472", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "434fcdfe", + "metadata": {}, + "source": [ + "Usecase 1: What if you have a syft function thats nested with jobs?\n", + "\n", + "Subproblem 1.1: We need to code before submitting\n", + "\n", + "- solution 1: remove the job aspect from local testing\n", + "- solution 2: you mimick the domain architecture locally\n", + "- solution 3: you run this on the low side on mock data\n", + "\n", + "\n", + "Subproblem 1.2: We need to be able to run this on the high side\n", + "\n", + "Solution\n", + "- move requests / code / inputs from low side to high side\n", + "- run job on the high side\n", + "- get results from job and load them into the low side\n", + "\n", + "Usecase 2: What if you have a syft function that takes days to run on the high side?\n", + "\n", + "See solution for 1.2\n", + "\n", + "Solutions based on example\n", + "\n", + "- we have an optional mock sample in asset creation\n", + "- we split submission and code_request on the DS side\n", + "- DS can call code that is submitted\n", + "\n", + "- what is the UX for running on the low side as a test vs a code request\n", + "\n", + "ux for running as a test locally\n", + "my_function(asset.mock_sample)\n", + "\n", + "ux for running as a test on the low side\n", + "\n", + "ds_client.code.my_function(asset.mock)\n", + "-> Should print info: runs on mock data\n", + "\n", + "ux for running the code on real data\n", + "\n", + "ds_client.code.my_function(asset)\n", + "-> Should print info: runs on real data\n", + "\n", + "the ux for a code request\n", + "\n", + "ds_client.make_code_request(\n", + "\n", + "ds_client.submit(code2)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0025e611", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b7e3720c", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8d24ddef", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ceef5cee", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "f39b522e", + "metadata": {}, + "outputs": [ + { + "ename": "AttributeError", + "evalue": "'DomainClient' object has no attribute 'submit'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[17], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mds_client\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msubmit\u001b[49m(code1)\n", + "\u001b[0;31mAttributeError\u001b[0m: 'DomainClient' object has no attribute 'submit'" + ] + } + ], + "source": [ + "ds_client.submit(code1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ecbc1bd5", + "metadata": {}, + "outputs": [], + "source": [ + "ds_client.request(code1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bfabc90f", + "metadata": {}, + "outputs": [], + "source": [ + "do_client.code" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7491560f", + "metadata": {}, + "outputs": [], + "source": [ + "request = do_client.requests[0]" + ] + }, + { + "cell_type": "markdown", + "id": "13ae252b", + "metadata": {}, + "source": [ + "code1:\n", + " blbalblab\n", + " code()\n", + "\n", + "code2:\n", + " blballbbla" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2ee32ea9", + "metadata": {}, + "outputs": [], + "source": [ + "request.code2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d0de9b26", + "metadata": {}, + "outputs": [], + "source": [ + "do_client.all_code.code2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "76530cbf", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0200af88", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "115be9c9", + "metadata": {}, + "source": [ + "When doing local execution **without** a domain" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9e67763c", + "metadata": {}, + "outputs": [], + "source": [ + "my_syft_function(x, y)" + ] + }, + { + "cell_type": "markdown", + "id": "199e03c6", + "metadata": {}, + "source": [ + "When doing local execution **with** a domain" + ] + }, + { + "cell_type": "markdown", + "id": "67811546", + "metadata": {}, + "source": [ + "- is it relevant to think about whether we need to configure the domain (e.g. consumsers)\n", + "- do we create the domain explicitly or does it happen on the fly?\n", + "- how do we indicate in the function call that we want to create a domain?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b13821c6", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "113fbe74", + "metadata": {}, + "source": [ + "When doing **high side** execution **without** a domain" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "da91ae61", + "metadata": {}, + "outputs": [], + "source": [ + "users_function = func.unsafe_function\n", + "real_result = users_function(trade_data=pvt_data)" + ] + }, + { + "cell_type": "markdown", + "id": "f540c39e", + "metadata": {}, + "source": [ + "When doing **high side** exeuction **with** a domain" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "375ed965", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "16/11/23 19:29:10 FUNCTION LOG (6470a06b282445ab8b912302c8c33a52): Launching jobs\n", + "16/11/23 19:29:10 FUNCTION LOG (6470a06b282445ab8b912302c8c33a52): starting aggregation\n", + "16/11/23 19:29:10 FUNCTION LOG (6470a06b282445ab8b912302c8c33a52): Done\n", + "16/11/23 19:29:11 FUNCTION LOG (511d17d79d0b450ab0f4d410bbbbe688): starting batch 1\n", + "16/11/23 19:29:11 EXCEPTION LOG (511d17d79d0b450ab0f4d410bbbbe688):\n", + "\n", + "Encountered while executing process_batch:\n", + "Traceback (most recent call last):\n", + " File \"/Users/koen/workspace/PySyft/packages/syft/src/syft/service/code/user_code.py\", line 1087, in execute_byte_code\n", + " result = eval(evil_string, _globals, _locals) # nosec\n", + " File \"\", line 1, in \n", + " File \"\", line 10, in user_func_process_batch_563f3f1899d092d0501158fbb086f3fd2c79a010c4e85bfe4fe12e0e1c01abde_3fa7f836b5a65455ef117b87f10f6a3f711d956132679e32522b6ae4c1440ec5\n", + " File \"\", line 6, in process_batch\n", + "IndexError: list index out of range\n", + "\n", + " 3 print(f'starting batch {batch}')\n", + " 4 from time import sleep\n", + "--> 5 [1, 2, 3, 4][5]\n", + " 6 sleep(1)\n", + "16/11/23 19:29:11 FUNCTION LOG (1f4245154b5a4e8ab20f7b3907203542): starting batch 2\n", + "16/11/23 19:29:11 EXCEPTION LOG (1f4245154b5a4e8ab20f7b3907203542):\n", + "\n", + "Encountered while executing process_batch:\n", + "Traceback (most recent call last):\n", + " File \"/Users/koen/workspace/PySyft/packages/syft/src/syft/service/code/user_code.py\", line 1087, in execute_byte_code\n", + " result = eval(evil_string, _globals, _locals) # nosec\n", + " File \"\", line 1, in \n", + " File \"\", line 10, in user_func_process_batch_563f3f1899d092d0501158fbb086f3fd2c79a010c4e85bfe4fe12e0e1c01abde_3fa7f836b5a65455ef117b87f10f6a3f711d956132679e32522b6ae4c1440ec5\n", + " File \"\", line 6, in process_batch\n", + "IndexError: list index out of range\n", + "\n", + " 3 print(f'starting batch {batch}')\n", + " 4 from time import sleep\n", + "--> 5 [1, 2, 3, 4][5]\n", + " 6 sleep(1)\n" + ] + } + ], + "source": [ + "job = ds_client.code.process_all(x=x_ptr, blocking=False)\n", + "sleep(5)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "0f8e12c0", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "starting batch 1\n", + "\n", + "Exception encountered while running process_batch, please contact the Node Admin for more info.\n" + ] + } + ], + "source": [ + "job.subjobs[0].logs()" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6d97d2a4", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "
\n", + "
\n", + "
\n", + "

Job List

\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + " \n", + "
\n", + " \n", + "
\n", + "\n", + "

0

\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + " \n", + " \n" + ], + "text/plain": [ + "[syft.service.job.job_stash.Job, syft.service.job.job_stash.Job]" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "job.subjobs" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "afbe026b", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "starting batch 1\n", + "\n", + "\n", + "Encountered while executing process_batch:\n", + "Traceback (most recent call last):\n", + " File \"/Users/koen/workspace/PySyft/packages/syft/src/syft/service/code/user_code.py\", line 1087, in execute_byte_code\n", + " result = eval(evil_string, _globals, _locals) # nosec\n", + " File \"\", line 1, in \n", + " File \"\", line 10, in user_func_process_batch_563f3f1899d092d0501158fbb086f3fd2c79a010c4e85bfe4fe12e0e1c01abde_3fa7f836b5a65455ef117b87f10f6a3f711d956132679e32522b6ae4c1440ec5\n", + " File \"\", line 6, in process_batch\n", + "IndexError: list index out of range\n", + "\n", + " 3 print(f'starting batch {batch}')\n", + " 4 from time import sleep\n", + "--> 5 [1, 2, 3, 4][5]\n", + " 6 sleep(1)\n" + ] + } + ], + "source": [ + "client.jobs[0].subjobs[0].logs()" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "81c23727", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "
\n", + "
\n", + "
\n", + "

Job List

\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + " \n", + "
\n", + " \n", + "
\n", + "\n", + "

0

\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + " \n", + " \n" + ], + "text/plain": [ + "[syft.service.job.job_stash.Job, syft.service.job.job_stash.Job]" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "job.subjobs" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "a9fde08d", + "metadata": {}, + "outputs": [ + { + "ename": "IndexError", + "evalue": "list index out of range", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mIndexError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[12], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mjob\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msubjobs\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[38;5;241m.\u001b[39mlogs(stderr\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n", + "\u001b[0;31mIndexError\u001b[0m: list index out of range" + ] + } + ], + "source": [ + "job.subjobs[0].logs(stderr=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "78bd08dc", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "starting batch 2\n", + "\n", + "\n", + "Encountered while executing process_batch:\n", + "Traceback (most recent call last):\n", + " File \"/Users/koen/workspace/PySyft/packages/syft/src/syft/service/code/user_code.py\", line 1072, in execute_byte_code\n", + " result = eval(evil_string, _globals, _locals) # nosec\n", + " File \"\", line 1, in \n", + " File \"\", line 10, in user_func_process_batch_47f8c8f3db3a30695a28e4a51e44916669ac3d111924cb614181c64b2c3b8323_3fa7f836b5a65455ef117b87f10f6a3f711d956132679e32522b6ae4c1440ec5\n", + " File \"\", line 6, in process_batch\n", + "IndexError: list index out of range\n", + "\n", + " 3 print(f'starting batch {batch}')\n", + " 4 from time import sleep\n", + "--> 5 [1, 2, 3, 4][5]\n", + " 6 sleep(1)\n" + ] + } + ], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "cc0db669", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Pointer:\n", + "None" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "job.wait()" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "5bf0974f", + "metadata": {}, + "outputs": [ + { + "ename": "TypeError", + "evalue": "unsupported operand type(s) for +: 'int' and 'Err'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[11], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;43msum\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m[\u001b[49m\u001b[43mj\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwait\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mj\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mjob\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msubjobs\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[0;31mTypeError\u001b[0m: unsupported operand type(s) for +: 'int' and 'Err'" + ] + } + ], + "source": [ + "sum([j.wait().get() for j in job.subjobs])" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "55892031", + "metadata": {}, + "outputs": [], + "source": [ + "# import sys" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "b0800f1d", + "metadata": {}, + "outputs": [], + "source": [ + "# print(\"D\")\n", + "# print(\"A\")\n", + "\n", + "# try:\n", + "# raise ValueError()\n", + "# except Exception as e:\n", + "# tb = e.__traceback__\n", + "# # print(len([tb.tb_next while tb is not None])\n", + "# # trace = []\n", + "# # while tb is not None:\n", + "# # trace.append({\n", + "# # \"filename\": tb.tb_frame.f_code.co_filename,\n", + "# # \"name\": tb.tb_frame.f_code.co_name,\n", + "# # \"lineno\": tb.tb_lineno\n", + "# # })\n", + "# # tb = tb.tb_next\n", + "\n", + "\n", + "# # print(sys.exc_info()[-1].tb_lineno)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "6c8fd04c", + "metadata": {}, + "outputs": [], + "source": [ + "# n = 0\n", + "# while tb is not None:\n", + "# tb = tb.tb_next\n", + "# n+=1" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "0c9368c9", + "metadata": {}, + "outputs": [], + "source": [ + "# n" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "1d73ef1e", + "metadata": {}, + "outputs": [], + "source": [ + "# trace" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "e61760f5", + "metadata": {}, + "outputs": [], + "source": [ + "import traceback\n", + "import sys" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "f5f2c781", + "metadata": {}, + "outputs": [], + "source": [ + "# try:\n", + "# raise ValueError()\n", + "# except Exception as e:\n", + "# print(sys.exception())\n", + "# # print()\n", + "\n", + "\n", + "# # # traceback.print_stack()\n", + "\n", + "\n", + "# # # tb = e.__traceback__\n", + "# # # traceback.print_tb(type(e), e, tb)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ead1e368", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "e956f350", + "metadata": {}, + "outputs": [], + "source": [ + "# print(\"\\n\".join(['def user_func_process_batch_47f8c8f3db3a30695a28e4a51e44916669ac3d111924cb614181c64b2c3b8323_bd42b6607712b09a0b187e3c67030f772c5dcd3c4152e4abebcb3a9f0a8259e4(batch):',\n", + "# '', ' def process_batch(batch):', \" print(f'starting batch {batch}')\",\n", + "# ' from time import sleep', ' sleep(1)', ' [1, 2, 3][5]',\n", + "# \" print('done')\", ' return batch + 1', ' result = process_batch(batch=batch)',\n", + "# ' return result']))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.16" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": true + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 6c9351ebdb2396954e4b358b175ee31686389b8d Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Mon, 20 Nov 2023 21:33:09 +0100 Subject: [PATCH 02/12] nb --- notebooks/helm/local_jobs_execution.ipynb | 764 ++++++++++++++++++++-- 1 file changed, 710 insertions(+), 54 deletions(-) diff --git a/notebooks/helm/local_jobs_execution.ipynb b/notebooks/helm/local_jobs_execution.ipynb index 9eef50bedcc..e43a9b9b86b 100644 --- a/notebooks/helm/local_jobs_execution.ipynb +++ b/notebooks/helm/local_jobs_execution.ipynb @@ -125,7 +125,7 @@ { "cell_type": "code", "execution_count": 5, - "id": "fbe87adc", + "id": "354455ee", "metadata": {}, "outputs": [], "source": [ @@ -144,7 +144,7 @@ { "cell_type": "code", "execution_count": 6, - "id": "a76e2d15", + "id": "9445c528", "metadata": {}, "outputs": [], "source": [ @@ -160,7 +160,7 @@ { "cell_type": "code", "execution_count": 7, - "id": "7661e9a7", + "id": "b16f6d96", "metadata": {}, "outputs": [ { @@ -184,7 +184,7 @@ { "cell_type": "code", "execution_count": 8, - "id": "b0714fba", + "id": "14b73a4e", "metadata": {}, "outputs": [ { @@ -229,7 +229,7 @@ { "cell_type": "code", "execution_count": 9, - "id": "18ccb235", + "id": "f68842d5", "metadata": {}, "outputs": [], "source": [ @@ -396,7 +396,7 @@ { "cell_type": "code", "execution_count": 15, - "id": "70a97140", + "id": "1a56aa14", "metadata": {}, "outputs": [], "source": [ @@ -406,7 +406,7 @@ { "cell_type": "code", "execution_count": 17, - "id": "9eb4f262", + "id": "9d23d22b", "metadata": {}, "outputs": [], "source": [ @@ -416,7 +416,7 @@ { "cell_type": "code", "execution_count": 18, - "id": "4f899e3c", + "id": "f3009e13", "metadata": {}, "outputs": [ { @@ -440,7 +440,7 @@ { "cell_type": "code", "execution_count": null, - "id": "0c2567a7", + "id": "7dfb2268", "metadata": {}, "outputs": [], "source": [] @@ -448,7 +448,7 @@ { "cell_type": "code", "execution_count": 19, - "id": "10a606cc", + "id": "2d71b2e5", "metadata": {}, "outputs": [], "source": [ @@ -458,7 +458,7 @@ { "cell_type": "code", "execution_count": 20, - "id": "e8d689e1", + "id": "f0892ff1", "metadata": {}, "outputs": [], "source": [ @@ -468,7 +468,7 @@ { "cell_type": "code", "execution_count": 21, - "id": "1fe858a0", + "id": "0f4324fd", "metadata": {}, "outputs": [], "source": [ @@ -478,7 +478,7 @@ { "cell_type": "code", "execution_count": 22, - "id": "d0b018e4", + "id": "98bcfdd6", "metadata": {}, "outputs": [], "source": [ @@ -489,7 +489,7 @@ { "cell_type": "code", "execution_count": 23, - "id": "45799cce", + "id": "408310a7", "metadata": {}, "outputs": [], "source": [ @@ -499,7 +499,7 @@ { "cell_type": "code", "execution_count": 24, - "id": "cfb1f932", + "id": "10813d11", "metadata": {}, "outputs": [ { @@ -537,7 +537,7 @@ { "cell_type": "code", "execution_count": 25, - "id": "3033771b", + "id": "1fb9b507", "metadata": {}, "outputs": [], "source": [ @@ -553,7 +553,7 @@ { "cell_type": "code", "execution_count": 26, - "id": "abf60134", + "id": "4a413b93", "metadata": {}, "outputs": [ { @@ -574,7 +574,7 @@ { "cell_type": "code", "execution_count": 29, - "id": "f672b6da", + "id": "123412b6", "metadata": {}, "outputs": [ { @@ -595,7 +595,7 @@ { "cell_type": "code", "execution_count": 30, - "id": "a23c34bd", + "id": "69b0ae40", "metadata": {}, "outputs": [], "source": [ @@ -605,7 +605,7 @@ { "cell_type": "code", "execution_count": 31, - "id": "6697fa53", + "id": "e0ce30b7", "metadata": {}, "outputs": [], "source": [ @@ -615,7 +615,7 @@ { "cell_type": "code", "execution_count": 32, - "id": "fcadf9c1", + "id": "0e76802f", "metadata": {}, "outputs": [], "source": [ @@ -625,7 +625,7 @@ { "cell_type": "code", "execution_count": 33, - "id": "7e42eb6d", + "id": "bd38811e", "metadata": {}, "outputs": [], "source": [ @@ -635,7 +635,7 @@ { "cell_type": "code", "execution_count": 37, - "id": "57de2a1e", + "id": "a34a4f43", "metadata": {}, "outputs": [], "source": [ @@ -648,7 +648,7 @@ { "cell_type": "code", "execution_count": 38, - "id": "23e9686a", + "id": "b2d81f84", "metadata": {}, "outputs": [ { @@ -694,7 +694,663 @@ "Idle worker expired: b'D140-D0EF'\n", "Idle worker expired: b'4A71-B0A6'\n", "Idle worker expired: b'4999-97E5'\n", - "Idle worker expired: b'897B-20D9'\n" + "Idle worker expired: b'897B-20D9'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Exception in thread Thread-6:\n", + "Traceback (most recent call last):\n", + " File \"/opt/anaconda3/envs/syft/lib/python3.9/threading.py\", line 980, in _bootstrap_inner\n", + " self.run()\n", + " File \"/opt/anaconda3/envs/syft/lib/python3.9/threading.py\", line 917, in run\n", + " self._target(*self._args, **self._kwargs)\n", + " File \"/Users/koen/workspace/PySyft/packages/syft/src/syft/service/queue/zmq_queue.py\", line 113, in read_items\n", + " for item in items:\n", + "TypeError: 'NoneType' object is not iterable\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "\n", + "Timeout elapsed after 30 seconds while trying to acquiring lock.\n", + "FAILED TO LOCK\n", + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Exception in thread Thread-12:\n", + "Traceback (most recent call last):\n", + " File \"/opt/anaconda3/envs/syft/lib/python3.9/threading.py\", line 980, in _bootstrap_inner\n", + " self.run()\n", + " File \"/opt/anaconda3/envs/syft/lib/python3.9/threading.py\", line 917, in run\n", + " self._target(*self._args, **self._kwargs)\n", + " File \"/Users/koen/workspace/PySyft/packages/syft/src/syft/service/queue/zmq_queue.py\", line 113, in read_items\n", + " for item in items:\n", + "TypeError: 'NoneType' object is not iterable\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", + "Timeout elapsed after 30 seconds while trying to acquiring lock.\n", + "FAILED TO LOCK\n" ] } ], @@ -705,7 +1361,7 @@ { "cell_type": "code", "execution_count": null, - "id": "f0c2f8c0", + "id": "00af3a02", "metadata": {}, "outputs": [], "source": [ @@ -715,7 +1371,7 @@ { "cell_type": "code", "execution_count": 12, - "id": "33cad669", + "id": "defa0fd2", "metadata": {}, "outputs": [ { @@ -734,7 +1390,7 @@ { "cell_type": "code", "execution_count": 11, - "id": "8c5ef2e8", + "id": "a0eda41c", "metadata": {}, "outputs": [ { @@ -755,7 +1411,7 @@ { "cell_type": "code", "execution_count": null, - "id": "a3a91fde", + "id": "624b3d73", "metadata": {}, "outputs": [], "source": [] @@ -763,7 +1419,7 @@ { "cell_type": "code", "execution_count": null, - "id": "dc22e8cf", + "id": "7240fa0c", "metadata": {}, "outputs": [], "source": [] @@ -771,7 +1427,7 @@ { "cell_type": "code", "execution_count": null, - "id": "5283e423", + "id": "294c1e75", "metadata": {}, "outputs": [], "source": [] @@ -779,14 +1435,14 @@ { "cell_type": "code", "execution_count": null, - "id": "345e5472", + "id": "e4efec44", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", - "id": "434fcdfe", + "id": "16877085", "metadata": {}, "source": [ "Usecase 1: What if you have a syft function thats nested with jobs?\n", @@ -840,7 +1496,7 @@ { "cell_type": "code", "execution_count": null, - "id": "0025e611", + "id": "91202992", "metadata": {}, "outputs": [], "source": [] @@ -848,7 +1504,7 @@ { "cell_type": "code", "execution_count": null, - "id": "b7e3720c", + "id": "acc27591", "metadata": {}, "outputs": [], "source": [] @@ -856,7 +1512,7 @@ { "cell_type": "code", "execution_count": null, - "id": "8d24ddef", + "id": "a92a1606", "metadata": {}, "outputs": [], "source": [] @@ -864,7 +1520,7 @@ { "cell_type": "code", "execution_count": null, - "id": "ceef5cee", + "id": "6957dee7", "metadata": {}, "outputs": [], "source": [] @@ -872,7 +1528,7 @@ { "cell_type": "code", "execution_count": 17, - "id": "f39b522e", + "id": "c30585d8", "metadata": {}, "outputs": [ { @@ -894,7 +1550,7 @@ { "cell_type": "code", "execution_count": null, - "id": "ecbc1bd5", + "id": "4104c432", "metadata": {}, "outputs": [], "source": [ @@ -904,7 +1560,7 @@ { "cell_type": "code", "execution_count": null, - "id": "bfabc90f", + "id": "c5a62e47", "metadata": {}, "outputs": [], "source": [ @@ -914,7 +1570,7 @@ { "cell_type": "code", "execution_count": null, - "id": "7491560f", + "id": "57773a01", "metadata": {}, "outputs": [], "source": [ @@ -923,7 +1579,7 @@ }, { "cell_type": "markdown", - "id": "13ae252b", + "id": "3d8a0a1f", "metadata": {}, "source": [ "code1:\n", @@ -937,7 +1593,7 @@ { "cell_type": "code", "execution_count": null, - "id": "2ee32ea9", + "id": "e227c845", "metadata": {}, "outputs": [], "source": [ @@ -947,7 +1603,7 @@ { "cell_type": "code", "execution_count": null, - "id": "d0de9b26", + "id": "30afd225", "metadata": {}, "outputs": [], "source": [ @@ -957,7 +1613,7 @@ { "cell_type": "code", "execution_count": null, - "id": "76530cbf", + "id": "fb7adb6b", "metadata": {}, "outputs": [], "source": [] @@ -965,14 +1621,14 @@ { "cell_type": "code", "execution_count": null, - "id": "0200af88", + "id": "1b437a65", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", - "id": "115be9c9", + "id": "b15d4db8", "metadata": {}, "source": [ "When doing local execution **without** a domain" @@ -981,7 +1637,7 @@ { "cell_type": "code", "execution_count": null, - "id": "9e67763c", + "id": "1e7ca2d8", "metadata": {}, "outputs": [], "source": [ @@ -990,7 +1646,7 @@ }, { "cell_type": "markdown", - "id": "199e03c6", + "id": "edc738c5", "metadata": {}, "source": [ "When doing local execution **with** a domain" @@ -998,7 +1654,7 @@ }, { "cell_type": "markdown", - "id": "67811546", + "id": "6604dd30", "metadata": {}, "source": [ "- is it relevant to think about whether we need to configure the domain (e.g. consumsers)\n", @@ -1009,14 +1665,14 @@ { "cell_type": "code", "execution_count": null, - "id": "b13821c6", + "id": "ae8fec19", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", - "id": "113fbe74", + "id": "a0e6837f", "metadata": {}, "source": [ "When doing **high side** execution **without** a domain" @@ -1025,7 +1681,7 @@ { "cell_type": "code", "execution_count": null, - "id": "da91ae61", + "id": "df9bcd99", "metadata": {}, "outputs": [], "source": [ @@ -1035,7 +1691,7 @@ }, { "cell_type": "markdown", - "id": "f540c39e", + "id": "b3959b10", "metadata": {}, "source": [ "When doing **high side** exeuction **with** a domain" From 786e8bbd2540c653a3d4250b15426c940d99281f Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Mon, 27 Nov 2023 13:52:40 +0000 Subject: [PATCH 03/12] 0 --- packages/syft/src/syft/service/queue/queue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/queue/queue.py b/packages/syft/src/syft/service/queue/queue.py index 6049d7645c1..14c19709a0f 100644 --- a/packages/syft/src/syft/service/queue/queue.py +++ b/packages/syft/src/syft/service/queue/queue.py @@ -142,7 +142,7 @@ def handle_message(message: bytes): except Exception as e: # nosec status = Status.ERRORED job_status = JobStatus.ERRORED - # stdlib + # stdlib raise e # result = SyftError( From 1ee310cc01e3c14a7648927ed53cf0590165585d Mon Sep 17 00:00:00 2001 From: teo Date: Mon, 4 Dec 2023 17:44:49 +0200 Subject: [PATCH 04/12] added client verify key to the dataset at upload --- packages/syft/src/syft/client/domain_client.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/syft/src/syft/client/domain_client.py b/packages/syft/src/syft/client/domain_client.py index 05cbb2294d6..b1f9d21e348 100644 --- a/packages/syft/src/syft/client/domain_client.py +++ b/packages/syft/src/syft/client/domain_client.py @@ -71,6 +71,8 @@ def upload_dataset(self, dataset: CreateDataset) -> Union[SyftSuccess, SyftError asset = dataset.asset_list[i] dataset.asset_list[i] = add_default_uploader(user, asset) + dataset.syft_client_verify_key = self.verify_key + dataset._check_asset_must_contain_mock() dataset_size = 0 From 8054bd5566865a1b24e29c0b78b0dbf31f182c4c Mon Sep 17 00:00:00 2001 From: teo Date: Mon, 4 Dec 2023 17:45:20 +0200 Subject: [PATCH 05/12] added default node_uid for asset and removed mock and data length check --- .../syft/src/syft/service/dataset/dataset.py | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/packages/syft/src/syft/service/dataset/dataset.py b/packages/syft/src/syft/service/dataset/dataset.py index 6d656c206dc..cebd66a6ede 100644 --- a/packages/syft/src/syft/service/dataset/dataset.py +++ b/packages/syft/src/syft/service/dataset/dataset.py @@ -413,13 +413,13 @@ def check(self) -> Union[SyftSuccess, SyftError]: return SyftError( message=f"set_obj type {type(self.data)} must match set_mock type {type(self.mock)}" ) - if not _is_action_data_empty(self.mock): - data_shape = get_shape_or_len(self.data) - mock_shape = get_shape_or_len(self.mock) - if data_shape != mock_shape: - return SyftError( - message=f"set_obj shape {data_shape} must match set_mock shape {mock_shape}" - ) + # if not _is_action_data_empty(self.mock): + # data_shape = get_shape_or_len(self.data) + # mock_shape = get_shape_or_len(self.mock) + # if data_shape != mock_shape: + # return SyftError( + # message=f"set_obj shape {data_shape} must match set_mock shape {mock_shape}" + # ) total_size_mb = get_mb_size(self.data) + get_mb_size(self.mock) if total_size_mb > DATA_SIZE_WARNING_LIMIT: print( @@ -438,7 +438,10 @@ def get_shape_or_len(obj: Any) -> Optional[Union[Tuple[int, ...], int]]: return shape len_attr = getattr(obj, "__len__", None) if len_attr is not None: - return len_attr() + len_value = len_attr() + if isinstance(len_value, int): + return (len_value,) + return len_value return None @@ -752,6 +755,7 @@ def infer_shape(context: TransformContext) -> TransformContext: if context.output["shape"] is None: if not _is_action_data_empty(context.obj.mock): context.output["shape"] = get_shape_or_len(context.obj.mock) + print(context.output["shape"]) return context @@ -773,6 +777,10 @@ def add_msg_creation_time(context: TransformContext) -> TransformContext: context.output["created_at"] = DateTime.now() return context +def add_default_node_uid(context: TransformContext) -> TransformContext: + if context.output["node_uid"] is None: + context.output["node_uid"] = context.node.id + return context @transform(CreateAsset, Asset) def createasset_to_asset() -> List[Callable]: @@ -782,6 +790,7 @@ def createasset_to_asset() -> List[Callable]: infer_shape, create_and_store_twin, set_data_subjects, + add_default_node_uid, ] From 00b605c6d7ba5993c7b4b55c7821e35ab3d16b2f Mon Sep 17 00:00:00 2001 From: teo Date: Mon, 4 Dec 2023 17:58:59 +0200 Subject: [PATCH 06/12] reverse bad conflict solve --- packages/hagrid/tests/hagrid/cli_test.py | 10 +-- .../syft/src/syft/service/dataset/dataset.py | 2 + packages/syft/src/syft/service/queue/queue.py | 62 ++++--------------- .../src/syft/store/sqlite_document_store.py | 8 +-- 4 files changed, 23 insertions(+), 59 deletions(-) diff --git a/packages/hagrid/tests/hagrid/cli_test.py b/packages/hagrid/tests/hagrid/cli_test.py index c6d9f794e1f..0ca6e577de8 100644 --- a/packages/hagrid/tests/hagrid/cli_test.py +++ b/packages/hagrid/tests/hagrid/cli_test.py @@ -23,7 +23,7 @@ def test_hagrid_launch() -> None: verb=verb, kwargs=defaultdict(lambda: None), ignore_docker_version_check=True ) - cmd = cmd["Launching"][0] # type: ignore + cmd = cmd["Launching"][0] # check that it's a domain by default assert "NODE_TYPE=domain" in cmd or "NODE_TYPE='domain'" in cmd @@ -63,7 +63,7 @@ def test_hagrid_launch_without_name_with_preposition() -> None: verb=verb, kwargs=defaultdict(lambda: None), ignore_docker_version_check=True ) - cmd = cmd["Launching"][0] # type: ignore + cmd = cmd["Launching"][0] # check that it's a domain by default assert "NODE_TYPE=domain" in cmd or "NODE_TYPE='domain'" in cmd @@ -102,7 +102,7 @@ def test_launch_with_multiword_domain_name() -> None: verb=verb, kwargs=defaultdict(lambda: None), ignore_docker_version_check=True ) - cmd = cmd["Launching"][0] # type: ignore + cmd = cmd["Launching"][0] # check that it's a domain by default assert "NODE_TYPE=domain" in cmd or "NODE_TYPE='domain'" in cmd @@ -128,7 +128,7 @@ def test_launch_with_longer_multiword_domain_name() -> None: verb=verb, kwargs=defaultdict(lambda: None), ignore_docker_version_check=True ) - cmd = cmd["Launching"][0] # type: ignore + cmd = cmd["Launching"][0] # check that it's a domain by default assert "NODE_TYPE=domain" in cmd or "NODE_TYPE='domain'" in cmd @@ -157,7 +157,7 @@ def test_launch_with_longer_multiword_domain_name_with_preposition() -> None: verb=verb, kwargs=defaultdict(lambda: None), ignore_docker_version_check=True ) - cmd = cmd["Launching"][0] # type: ignore + cmd = cmd["Launching"][0] # check that it's a domain by default assert "NODE_TYPE=domain" in cmd or "NODE_TYPE='domain'" in cmd diff --git a/packages/syft/src/syft/service/dataset/dataset.py b/packages/syft/src/syft/service/dataset/dataset.py index cebd66a6ede..381c44aac2d 100644 --- a/packages/syft/src/syft/service/dataset/dataset.py +++ b/packages/syft/src/syft/service/dataset/dataset.py @@ -777,11 +777,13 @@ def add_msg_creation_time(context: TransformContext) -> TransformContext: context.output["created_at"] = DateTime.now() return context + def add_default_node_uid(context: TransformContext) -> TransformContext: if context.output["node_uid"] is None: context.output["node_uid"] = context.node.id return context + @transform(CreateAsset, Asset) def createasset_to_asset() -> List[Callable]: return [ diff --git a/packages/syft/src/syft/service/queue/queue.py b/packages/syft/src/syft/service/queue/queue.py index cbf8948dac1..c56a16e711e 100644 --- a/packages/syft/src/syft/service/queue/queue.py +++ b/packages/syft/src/syft/service/queue/queue.py @@ -243,56 +243,18 @@ def handle_message(message: bytes): job_item.status = JobStatus.PROCESSING job_item.node_uid = worker.id - worker.queue_stash.set_result(credentials, queue_item) - worker.job_stash.set_result(credentials, job_item) - - status = Status.COMPLETED - job_status = JobStatus.COMPLETED - - try: - call_method = getattr( - worker.get_service(queue_item.service), queue_item.method - ) - - role = worker.get_role_for_credentials(credentials=credentials) - context = AuthedServiceContext( - node=worker, - credentials=credentials, - role=role, - job_id=queue_item.job_id, - ) - - result: Any = call_method(context, *queue_item.args, **queue_item.kwargs) - - if isinstance(result, Ok): - result = result.ok() - elif isinstance(result, SyftError) or isinstance(result, Err): - status = Status.ERRORED - job_status = JobStatus.ERRORED - except Exception as e: # nosec - status = Status.ERRORED - job_status = JobStatus.ERRORED - # stdlib - - raise e - # result = SyftError( - # message=f"Failed with exception: {e}, {traceback.format_exc()}" - # ) - # print("HAD AN ERROR WHILE HANDLING MESSAGE", result.message) - - queue_item.result = result - queue_item.resolved = True - queue_item.status = status - - # get new job item to get latest iter status - job_item = worker.job_stash.get_by_uid(credentials, job_item.id).ok() - - # if result.is_ok(): - - job_item.node_uid = worker.id - job_item.result = result - job_item.resolved = True - job_item.status = job_status + queue_result = worker.queue_stash.set_result(credentials, queue_item) + if isinstance(queue_result, SyftError): + raise Exception(message=f"{queue_result.err()}") + worker_result = worker.job_stash.set_result(credentials, job_item) + if isinstance(worker_result, SyftError): + raise Exception(message=f"{worker_result.err()}") + + p = multiprocessing.Process( + target=handle_message_multiprocessing, + args=(worker_settings, queue_item, credentials), + ) + p.start() job_item.job_pid = p.pid worker.job_stash.set_result(credentials, job_item) diff --git a/packages/syft/src/syft/store/sqlite_document_store.py b/packages/syft/src/syft/store/sqlite_document_store.py index 2ec313a1a8a..43cd5e2952b 100644 --- a/packages/syft/src/syft/store/sqlite_document_store.py +++ b/packages/syft/src/syft/store/sqlite_document_store.py @@ -206,8 +206,8 @@ def _set(self, key: UID, value: Any) -> None: self._update(key, value) else: insert_sql = ( - f"insert into {self.table_name} (uid, repr, value) VALUES (?, ?, ?)" # nosec - ) + f"insert into {self.table_name} (uid, repr, value) VALUES (?, ?, ?)" + ) # nosec data = _serialize(value, to_bytes=True) res = self._execute(insert_sql, [str(key), _repr_debug_(value), data]) if res.is_err(): @@ -215,8 +215,8 @@ def _set(self, key: UID, value: Any) -> None: def _update(self, key: UID, value: Any) -> None: insert_sql = ( - f"update {self.table_name} set uid = ?, repr = ?, value = ? where uid = ?" # nosec - ) + f"update {self.table_name} set uid = ?, repr = ?, value = ? where uid = ?" + ) # nosec data = _serialize(value, to_bytes=True) res = self._execute(insert_sql, [str(key), _repr_debug_(value), data, str(key)]) if res.is_err(): From cd839a927646da7af1c800f260bda3270a32a313 Mon Sep 17 00:00:00 2001 From: teo Date: Mon, 4 Dec 2023 18:05:36 +0200 Subject: [PATCH 07/12] fix lint --- packages/hagrid/tests/hagrid/cli_test.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/hagrid/tests/hagrid/cli_test.py b/packages/hagrid/tests/hagrid/cli_test.py index 0ca6e577de8..c6d9f794e1f 100644 --- a/packages/hagrid/tests/hagrid/cli_test.py +++ b/packages/hagrid/tests/hagrid/cli_test.py @@ -23,7 +23,7 @@ def test_hagrid_launch() -> None: verb=verb, kwargs=defaultdict(lambda: None), ignore_docker_version_check=True ) - cmd = cmd["Launching"][0] + cmd = cmd["Launching"][0] # type: ignore # check that it's a domain by default assert "NODE_TYPE=domain" in cmd or "NODE_TYPE='domain'" in cmd @@ -63,7 +63,7 @@ def test_hagrid_launch_without_name_with_preposition() -> None: verb=verb, kwargs=defaultdict(lambda: None), ignore_docker_version_check=True ) - cmd = cmd["Launching"][0] + cmd = cmd["Launching"][0] # type: ignore # check that it's a domain by default assert "NODE_TYPE=domain" in cmd or "NODE_TYPE='domain'" in cmd @@ -102,7 +102,7 @@ def test_launch_with_multiword_domain_name() -> None: verb=verb, kwargs=defaultdict(lambda: None), ignore_docker_version_check=True ) - cmd = cmd["Launching"][0] + cmd = cmd["Launching"][0] # type: ignore # check that it's a domain by default assert "NODE_TYPE=domain" in cmd or "NODE_TYPE='domain'" in cmd @@ -128,7 +128,7 @@ def test_launch_with_longer_multiword_domain_name() -> None: verb=verb, kwargs=defaultdict(lambda: None), ignore_docker_version_check=True ) - cmd = cmd["Launching"][0] + cmd = cmd["Launching"][0] # type: ignore # check that it's a domain by default assert "NODE_TYPE=domain" in cmd or "NODE_TYPE='domain'" in cmd @@ -157,7 +157,7 @@ def test_launch_with_longer_multiword_domain_name_with_preposition() -> None: verb=verb, kwargs=defaultdict(lambda: None), ignore_docker_version_check=True ) - cmd = cmd["Launching"][0] + cmd = cmd["Launching"][0] # type: ignore # check that it's a domain by default assert "NODE_TYPE=domain" in cmd or "NODE_TYPE='domain'" in cmd From 02d4acc670af89f58bac0a04a9f2bd3ff55827de Mon Sep 17 00:00:00 2001 From: teo Date: Mon, 8 Jan 2024 15:04:08 +0200 Subject: [PATCH 08/12] removed manually setting the data verify key --- packages/syft/src/syft/client/domain_client.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/syft/src/syft/client/domain_client.py b/packages/syft/src/syft/client/domain_client.py index b1f9d21e348..05cbb2294d6 100644 --- a/packages/syft/src/syft/client/domain_client.py +++ b/packages/syft/src/syft/client/domain_client.py @@ -71,8 +71,6 @@ def upload_dataset(self, dataset: CreateDataset) -> Union[SyftSuccess, SyftError asset = dataset.asset_list[i] dataset.asset_list[i] = add_default_uploader(user, asset) - dataset.syft_client_verify_key = self.verify_key - dataset._check_asset_must_contain_mock() dataset_size = 0 From 180a47f41f27c17c4662d97baf4a4164231fd2d3 Mon Sep 17 00:00:00 2001 From: teo Date: Mon, 8 Jan 2024 15:05:14 +0200 Subject: [PATCH 09/12] removed stale notebook --- notebooks/helm/local_jobs_execution.ipynb | 3321 --------------------- 1 file changed, 3321 deletions(-) delete mode 100644 notebooks/helm/local_jobs_execution.ipynb diff --git a/notebooks/helm/local_jobs_execution.ipynb b/notebooks/helm/local_jobs_execution.ipynb deleted file mode 100644 index e43a9b9b86b..00000000000 --- a/notebooks/helm/local_jobs_execution.ipynb +++ /dev/null @@ -1,3321 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "id": "a196017f", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "kj/filesystem-disk-unix.c++:1703: warning: PWD environment variable doesn't match current directory; pwd = /Users/koen/workspace/pysyft\n" - ] - } - ], - "source": [ - "import syft as sy\n", - "from syft import ActionObject\n", - "from syft import syft_function, syft_function_single_use\n", - "from time import sleep\n", - "import os\n", - "import psutil\n", - "import inspect" - ] - }, - { - "cell_type": "markdown", - "id": "cb2d07de", - "metadata": {}, - "source": [ - "with server" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "9b31c627", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Staging Protocol Changes...\n", - "Logged into as \n" - ] - }, - { - "data": { - "text/html": [ - "
SyftWarning: You are using a default password. Please change the password using `[your_client].me.set_password([new_password])`.

" - ], - "text/plain": [ - "SyftWarning: You are using a default password. Please change the password using `[your_client].me.set_password([new_password])`." - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "node = sy.orchestra.launch(name=\"test-domain-helm2\",\n", - " dev_mode=True,\n", - " reset=True, \n", - " n_consumers=3,\n", - " create_producer=True,\n", - " queue_port=3322)\n", - " \n", - "client = node.login(email=\"info@openmined.org\", password=\"changethis\")" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "507740d2", - "metadata": {}, - "outputs": [], - "source": [ - "res = client.register(name=\"a\", email=\"aa@b.org\", password=\"c\", password_verify=\"c\")" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "0c33d096", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Logged into as \n" - ] - } - ], - "source": [ - "ds_client = node.login(email=\"aa@b.org\", password=\"c\")" - ] - }, - { - "cell_type": "markdown", - "id": "176addfb", - "metadata": {}, - "source": [ - "setup: compute train-test overlap between a very large train set and a smaller test set. Small test is still to big for memory, so we split it into 54 parts. We keep 1 of those parts in memory. We dont keep the train set in memory, but read and compare with 1/54 parts line by line. Each part takes ~30 hours, but we can run 54 processes in parallel." - ] - }, - { - "cell_type": "markdown", - "id": "a0cea81b", - "metadata": {}, - "source": [ - "# Setup syft functions" - ] - }, - { - "cell_type": "markdown", - "id": "da2b114a", - "metadata": {}, - "source": [ - "## Dataset" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "354455ee", - "metadata": {}, - "outputs": [], - "source": [ - "dataset = sy.Dataset(\n", - " name=\"My dataset\",\n", - " asset_list=[\n", - " sy.Asset(\n", - " name=\"input asset\",\n", - " data=ActionObject.from_obj([1, 2]),\n", - " mock=ActionObject.from_obj([1, 2]),\n", - " ),\n", - " ]\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "9445c528", - "metadata": {}, - "outputs": [], - "source": [ - "# dataset.assets[0].mock.syft_action_data\n", - "\n", - "# dataset.assets[0].data.syft_action_data\n", - "\n", - "# dataset.assets[0].mock.__len__()\n", - "\n", - "# dataset.assets[0].data.__len__()" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "b16f6d96", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "[1, 2]" - ], - "text/plain": [ - "[1, 2]" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "dataset.assets[0].data.syft_action_data" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "14b73a4e", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 10.25it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Uploading: input asset\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "data": { - "text/html": [ - "
SyftSuccess: Dataset uploaded to 'test-domain-helm2'. To see the datasets uploaded by a client on this node, use command `[your_client].datasets`

" - ], - "text/plain": [ - "SyftSuccess: Dataset uploaded to 'test-domain-helm2'. To see the datasets uploaded by a client on this node, use command `[your_client].datasets`" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "client.upload_dataset(dataset)" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "f68842d5", - "metadata": {}, - "outputs": [], - "source": [ - "asset = ds_client.datasets[0].assets[0]" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "83307a2f", - "metadata": {}, - "outputs": [], - "source": [ - "# x = ActionObject.from_obj([1, 2])\n", - "# x_ptr = x.send(ds_client)" - ] - }, - { - "cell_type": "markdown", - "id": "31bbb3ff", - "metadata": {}, - "source": [ - "## Batch function" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "5d2fd248", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
SyftSuccess: Syft function 'process_batch' successfully created. To add a code request, please create a project using `project = syft.Project(...)`, then use command `project.create_code_request`.

" - ], - "text/plain": [ - "SyftSuccess: Syft function 'process_batch' successfully created. To add a code request, please create a project using `project = syft.Project(...)`, then use command `project.create_code_request`." - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "@syft_function()\n", - "def process_batch(batch):\n", - " # takes 30 hours normally\n", - " print(f\"starting batch {batch}\")\n", - " from time import sleep\n", - " sleep(1)\n", - " print(\"done\")\n", - " return batch+1" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "9ba22655", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
SyftSuccess: User Code Submitted

" - ], - "text/plain": [ - "SyftSuccess: User Code Submitted" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds_client.code.submit(process_batch)" - ] - }, - { - "cell_type": "markdown", - "id": "01319f1f", - "metadata": {}, - "source": [ - "## Main function" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "ca1b95ee", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
SyftSuccess: Syft function 'process_all' successfully created. To add a code request, please create a project using `project = syft.Project(...)`, then use command `project.create_code_request`.

" - ], - "text/plain": [ - "SyftSuccess: Syft function 'process_all' successfully created. To add a code request, please create a project using `project = syft.Project(...)`, then use command `project.create_code_request`." - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "@syft_function_single_use(x=asset)\n", - "def process_all(domain, x):\n", - " \n", - " jobs = []\n", - " print(\"Launching jobs\")\n", - " for elem in x:\n", - " # We inject a domain object in the scope\n", - " batch_job = domain.launch_job(process_batch, batch=elem)\n", - " jobs += [batch_job]\n", - " print(\"starting aggregation\")\n", - " print(\"Done\")\n", - " return 3" - ] - }, - { - "cell_type": "markdown", - "id": "1e77c5db", - "metadata": {}, - "source": [ - "# Approve & run" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "0ab572f9", - "metadata": { - "scrolled": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Request approved for domain test-domain-helm2\n" - ] - }, - { - "data": { - "text/html": [ - "
SyftSuccess: Request 5147062631e1422f8a113120f153afa4 changes applied

" - ], - "text/plain": [ - "SyftSuccess: Request 5147062631e1422f8a113120f153afa4 changes applied" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds_client.code.request_code_execution(process_all)\n", - "client.requests[-1].approve()" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "1a56aa14", - "metadata": {}, - "outputs": [], - "source": [ - "# ds_client.code.process_all(x=asset, blocking=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "id": "9d23d22b", - "metadata": {}, - "outputs": [], - "source": [ - "mock =asset.mock" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "id": "f3009e13", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "[1, 2]" - ], - "text/plain": [ - "[1, 2]" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "mock" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7dfb2268", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 19, - "id": "2d71b2e5", - "metadata": {}, - "outputs": [], - "source": [ - "# \"domain\" in sig.parameters" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "id": "f0892ff1", - "metadata": {}, - "outputs": [], - "source": [ - "from syft.node.worker import Worker" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "id": "0f4324fd", - "metadata": {}, - "outputs": [], - "source": [ - "from syft.node.node import Node" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "id": "98bcfdd6", - "metadata": {}, - "outputs": [], - "source": [ - "# process_all\n", - "# process_all(x=[3,4])" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "id": "408310a7", - "metadata": {}, - "outputs": [], - "source": [ - "# domain = Node.named(name=\"plan_building\", reset=True, processes=0)" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "id": "10813d11", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Staging Protocol Changes...\n", - "Logged into as \n" - ] - }, - { - "data": { - "text/html": [ - "
SyftWarning: You are using a default password. Please change the password using `[your_client].me.set_password([new_password])`.

" - ], - "text/plain": [ - "SyftWarning: You are using a default password. Please change the password using `[your_client].me.set_password([new_password])`." - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "domain = sy.orchestra.launch(name=\"test\",\n", - " dev_mode=True,\n", - " reset=True, \n", - " n_consumers=3,\n", - " create_producer=True,\n", - " queue_port=22221)\n", - " \n", - "temp_client = node.login(email=\"info@openmined.org\", password=\"changethis\")" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "id": "1fb9b507", - "metadata": {}, - "outputs": [], - "source": [ - "# handle = NodeHandle(\n", - "# node_type=node_type_enum,\n", - "# deployment_type=deployment_type_enum,\n", - "# name=name,\n", - "# python_node=worker,\n", - "# node_side_type=node_side_type,\n", - "# )" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "id": "4a413b93", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - ".wrapper(*args, **kwargs)>" - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds_client.code.process_all" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "id": "123412b6", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - ">" - ] - }, - "execution_count": 29, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "client.launch_job" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "id": "69b0ae40", - "metadata": {}, - "outputs": [], - "source": [ - "# from hagrid.hagrid.orchestra" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "id": "e0ce30b7", - "metadata": {}, - "outputs": [], - "source": [ - "# handle.login(email=\"info@openmined.org\", password=\"changethis\")" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "id": "0e76802f", - "metadata": {}, - "outputs": [], - "source": [ - "filtered_kwargs = {\"x\": [3,4]}" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "id": "bd38811e", - "metadata": {}, - "outputs": [], - "source": [ - "filtered_kwargs[\"domain\"] = temp_client" - ] - }, - { - "cell_type": "code", - "execution_count": 37, - "id": "a34a4f43", - "metadata": {}, - "outputs": [], - "source": [ - "action_kwargs = {}\n", - "for k, v in filtered_kwargs.items():\n", - " val = ActionObject.from_obj(v)\n", - " action_kwargs[k] = val.send(temp_client)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "id": "b2d81f84", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Launching jobs\n" - ] - }, - { - "ename": "AttributeError", - "evalue": "'SQLiteStorePartition' object has no attribute 'lock'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[38], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mprocess_all\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlocal_function\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43maction_kwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "Cell \u001b[0;32mIn[13], line 8\u001b[0m, in \u001b[0;36mprocess_all\u001b[0;34m(domain, x)\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mLaunching jobs\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 6\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m elem \u001b[38;5;129;01min\u001b[39;00m x:\n\u001b[1;32m 7\u001b[0m \u001b[38;5;66;03m# We inject a domain object in the scope\u001b[39;00m\n\u001b[0;32m----> 8\u001b[0m batch_job \u001b[38;5;241m=\u001b[39m \u001b[43mdomain\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlaunch_job\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprocess_batch\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbatch\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43melem\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 9\u001b[0m jobs \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m [batch_job]\n\u001b[1;32m 10\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mstarting aggregation\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", - "File \u001b[0;32m~/workspace/PySyft/packages/syft/src/syft/service/action/action_object.py:1400\u001b[0m, in \u001b[0;36mActionObject._syft_wrap_attribute_for_methods..wrapper\u001b[0;34m(_self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1399\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwrapper\u001b[39m(_self: Any, \u001b[38;5;241m*\u001b[39margs: Any, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs: Any):\n\u001b[0;32m-> 1400\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_base_wrapper\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/workspace/PySyft/packages/syft/src/syft/service/action/action_object.py:1389\u001b[0m, in \u001b[0;36mActionObject._syft_wrap_attribute_for_methods.._base_wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 1385\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 1386\u001b[0m original_args, original_kwargs \u001b[38;5;241m=\u001b[39m debox_args_and_kwargs(\n\u001b[1;32m 1387\u001b[0m pre_hook_args, pre_hook_kwargs\n\u001b[1;32m 1388\u001b[0m )\n\u001b[0;32m-> 1389\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[43moriginal_func\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43moriginal_args\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43moriginal_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1391\u001b[0m post_result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_syft_run_post_hooks__(context, name, result)\n\u001b[1;32m 1392\u001b[0m post_result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_syft_attr_propagate_ids(context, name, post_result)\n", - "File \u001b[0;32m~/workspace/PySyft/packages/syft/src/syft/client/domain_client.py:151\u001b[0m, in \u001b[0;36mDomainClient.launch_job\u001b[0;34m(self, func, *args, **kwargs)\u001b[0m\n\u001b[1;32m 149\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mlaunch_job\u001b[39m(\u001b[38;5;28mself\u001b[39m, func, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 150\u001b[0m kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mblocking\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[0;32m--> 151\u001b[0m func \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcode\u001b[49m, func\u001b[38;5;241m.\u001b[39mfunc_name)\n\u001b[1;32m 152\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m func(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n", - "File \u001b[0;32m~/workspace/PySyft/packages/syft/src/syft/client/domain_client.py:158\u001b[0m, in \u001b[0;36mDomainClient.code\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 154\u001b[0m \u001b[38;5;129m@property\u001b[39m\n\u001b[1;32m 155\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcode\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Optional[APIModule]:\n\u001b[1;32m 156\u001b[0m \u001b[38;5;66;03m# if self.api.refresh_api_callback is not None:\u001b[39;00m\n\u001b[1;32m 157\u001b[0m \u001b[38;5;66;03m# self.api.refresh_api_callback()\u001b[39;00m\n\u001b[0;32m--> 158\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mapi\u001b[49m\u001b[38;5;241m.\u001b[39mhas_service(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcode\u001b[39m\u001b[38;5;124m\"\u001b[39m):\n\u001b[1;32m 159\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mapi\u001b[38;5;241m.\u001b[39mservices\u001b[38;5;241m.\u001b[39mcode\n\u001b[1;32m 160\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n", - "File \u001b[0;32m~/workspace/PySyft/packages/syft/src/syft/client/client.py:586\u001b[0m, in \u001b[0;36mSyftClient.api\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 582\u001b[0m \u001b[38;5;129m@property\u001b[39m\n\u001b[1;32m 583\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mapi\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m SyftAPI:\n\u001b[1;32m 584\u001b[0m \u001b[38;5;66;03m# invalidate API\u001b[39;00m\n\u001b[1;32m 585\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_api \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mor\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_api\u001b[38;5;241m.\u001b[39msigning_key \u001b[38;5;241m!=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcredentials):\n\u001b[0;32m--> 586\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_fetch_api\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcredentials\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 588\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_api\n", - "File \u001b[0;32m~/workspace/PySyft/packages/syft/src/syft/client/client.py:824\u001b[0m, in \u001b[0;36mSyftClient._fetch_api\u001b[0;34m(self, credentials)\u001b[0m\n\u001b[1;32m 823\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_fetch_api\u001b[39m(\u001b[38;5;28mself\u001b[39m, credentials: SyftSigningKey):\n\u001b[0;32m--> 824\u001b[0m _api: SyftAPI \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconnection\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_api\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 825\u001b[0m \u001b[43m \u001b[49m\u001b[43mcredentials\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcredentials\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 826\u001b[0m \u001b[43m \u001b[49m\u001b[43mcommunication_protocol\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcommunication_protocol\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 827\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 829\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mrefresh_callback\u001b[39m():\n\u001b[1;32m 830\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_fetch_api(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcredentials)\n", - "File \u001b[0;32m~/workspace/PySyft/packages/syft/src/syft/client/client.py:371\u001b[0m, in \u001b[0;36mPythonConnection.get_api\u001b[0;34m(self, credentials, communication_protocol)\u001b[0m\n\u001b[1;32m 360\u001b[0m obj \u001b[38;5;241m=\u001b[39m forward_message_to_proxy(\n\u001b[1;32m 361\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmake_call,\n\u001b[1;32m 362\u001b[0m proxy_target_uid\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mproxy_target_uid,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 368\u001b[0m credentials\u001b[38;5;241m=\u001b[39mcredentials,\n\u001b[1;32m 369\u001b[0m )\n\u001b[1;32m 370\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 371\u001b[0m obj \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mnode\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_api\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 372\u001b[0m \u001b[43m \u001b[49m\u001b[43mfor_user\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcredentials\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mverify_key\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 373\u001b[0m \u001b[43m \u001b[49m\u001b[43mcommunication_protocol\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcommunication_protocol\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 374\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 375\u001b[0m obj\u001b[38;5;241m.\u001b[39mconnection \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\n\u001b[1;32m 376\u001b[0m obj\u001b[38;5;241m.\u001b[39msigning_key \u001b[38;5;241m=\u001b[39m credentials\n", - "File \u001b[0;32m~/workspace/PySyft/packages/syft/src/syft/node/node.py:1090\u001b[0m, in \u001b[0;36mNode.get_api\u001b[0;34m(self, for_user, communication_protocol)\u001b[0m\n\u001b[1;32m 1085\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget_api\u001b[39m(\n\u001b[1;32m 1086\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 1087\u001b[0m for_user: Optional[SyftVerifyKey] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m 1088\u001b[0m communication_protocol: Optional[PROTOCOL_TYPE] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m 1089\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m SyftAPI:\n\u001b[0;32m-> 1090\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mSyftAPI\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfor_user\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1091\u001b[0m \u001b[43m \u001b[49m\u001b[43mnode\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1092\u001b[0m \u001b[43m \u001b[49m\u001b[43muser_verify_key\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfor_user\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1093\u001b[0m \u001b[43m \u001b[49m\u001b[43mcommunication_protocol\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcommunication_protocol\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1094\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/workspace/PySyft/packages/syft/src/syft/client/api.py:507\u001b[0m, in \u001b[0;36mSyftAPI.for_user\u001b[0;34m(node, communication_protocol, user_verify_key)\u001b[0m\n\u001b[1;32m 503\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mservice\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcode\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01muser_code_service\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m UserCodeService\n\u001b[1;32m 505\u001b[0m \u001b[38;5;66;03m# find user role by verify_key\u001b[39;00m\n\u001b[1;32m 506\u001b[0m \u001b[38;5;66;03m# TODO: we should probably not allow empty verify keys but instead make user always register\u001b[39;00m\n\u001b[0;32m--> 507\u001b[0m role \u001b[38;5;241m=\u001b[39m \u001b[43mnode\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_role_for_credentials\u001b[49m\u001b[43m(\u001b[49m\u001b[43muser_verify_key\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 508\u001b[0m _user_service_config_registry \u001b[38;5;241m=\u001b[39m UserServiceConfigRegistry\u001b[38;5;241m.\u001b[39mfrom_role(role)\n\u001b[1;32m 509\u001b[0m _user_lib_config_registry \u001b[38;5;241m=\u001b[39m UserLibConfigRegistry\u001b[38;5;241m.\u001b[39mfrom_user(user_verify_key)\n", - "File \u001b[0;32m~/workspace/PySyft/packages/syft/src/syft/node/node.py:921\u001b[0m, in \u001b[0;36mNode.get_role_for_credentials\u001b[0;34m(self, credentials)\u001b[0m\n\u001b[1;32m 920\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget_role_for_credentials\u001b[39m(\u001b[38;5;28mself\u001b[39m, credentials: SyftVerifyKey) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m ServiceRole:\n\u001b[0;32m--> 921\u001b[0m role \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_service\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43muserservice\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_role_for_credentials\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 922\u001b[0m \u001b[43m \u001b[49m\u001b[43mcredentials\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcredentials\u001b[49m\n\u001b[1;32m 923\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 924\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m role\n", - "File \u001b[0;32m~/workspace/PySyft/packages/syft/src/syft/service/user/user_service.py:138\u001b[0m, in \u001b[0;36mUserService.get_role_for_credentials\u001b[0;34m(self, credentials)\u001b[0m\n\u001b[1;32m 132\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget_role_for_credentials\u001b[39m(\n\u001b[1;32m 133\u001b[0m \u001b[38;5;28mself\u001b[39m, credentials: Union[SyftVerifyKey, SyftSigningKey]\n\u001b[1;32m 134\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Union[Optional[ServiceRole], SyftError]:\n\u001b[1;32m 135\u001b[0m \u001b[38;5;66;03m# they could be different\u001b[39;00m\n\u001b[1;32m 137\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(credentials, SyftVerifyKey):\n\u001b[0;32m--> 138\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstash\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_by_verify_key\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 139\u001b[0m \u001b[43m \u001b[49m\u001b[43mcredentials\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcredentials\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mverify_key\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcredentials\u001b[49m\n\u001b[1;32m 140\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 141\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 142\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstash\u001b[38;5;241m.\u001b[39mget_by_signing_key(\n\u001b[1;32m 143\u001b[0m credentials\u001b[38;5;241m=\u001b[39mcredentials, signing_key\u001b[38;5;241m=\u001b[39mcredentials\n\u001b[1;32m 144\u001b[0m )\n", - "File \u001b[0;32m~/workspace/PySyft/packages/syft/src/syft/service/user/user_stash.py:106\u001b[0m, in \u001b[0;36mUserStash.get_by_verify_key\u001b[0;34m(self, credentials, verify_key)\u001b[0m\n\u001b[1;32m 104\u001b[0m verify_key \u001b[38;5;241m=\u001b[39m SyftVerifyKey\u001b[38;5;241m.\u001b[39mfrom_string(verify_key)\n\u001b[1;32m 105\u001b[0m qks \u001b[38;5;241m=\u001b[39m QueryKeys(qks\u001b[38;5;241m=\u001b[39m[VerifyKeyPartitionKey\u001b[38;5;241m.\u001b[39mwith_obj(verify_key)])\n\u001b[0;32m--> 106\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mquery_one\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcredentials\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcredentials\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mqks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mqks\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/workspace/PySyft/packages/syft/src/syft/store/document_store.py:636\u001b[0m, in \u001b[0;36mBaseStash.query_one\u001b[0;34m(self, credentials, qks, order_by)\u001b[0m\n\u001b[1;32m 630\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mquery_one\u001b[39m(\n\u001b[1;32m 631\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 632\u001b[0m credentials: SyftVerifyKey,\n\u001b[1;32m 633\u001b[0m qks: Union[QueryKey, QueryKeys],\n\u001b[1;32m 634\u001b[0m order_by: Optional[PartitionKey] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m 635\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Result[Optional[BaseStash\u001b[38;5;241m.\u001b[39mobject_type], \u001b[38;5;28mstr\u001b[39m]:\n\u001b[0;32m--> 636\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mquery_all\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 637\u001b[0m \u001b[43m \u001b[49m\u001b[43mcredentials\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcredentials\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mqks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mqks\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43morder_by\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43morder_by\u001b[49m\n\u001b[1;32m 638\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mand_then(first_or_none)\n", - "File \u001b[0;32m~/workspace/PySyft/packages/syft/src/syft/store/document_store.py:614\u001b[0m, in \u001b[0;36mBaseStash.query_all\u001b[0;34m(self, credentials, qks, order_by)\u001b[0m\n\u001b[1;32m 611\u001b[0m index_qks \u001b[38;5;241m=\u001b[39m QueryKeys(qks\u001b[38;5;241m=\u001b[39munique_keys)\n\u001b[1;32m 612\u001b[0m search_qks \u001b[38;5;241m=\u001b[39m QueryKeys(qks\u001b[38;5;241m=\u001b[39msearchable_keys)\n\u001b[0;32m--> 614\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpartition\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfind_index_or_search_keys\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 615\u001b[0m \u001b[43m \u001b[49m\u001b[43mcredentials\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcredentials\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 616\u001b[0m \u001b[43m \u001b[49m\u001b[43mindex_qks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mindex_qks\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 617\u001b[0m \u001b[43m \u001b[49m\u001b[43msearch_qks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msearch_qks\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 618\u001b[0m \u001b[43m \u001b[49m\u001b[43morder_by\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43morder_by\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 619\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/workspace/PySyft/packages/syft/src/syft/store/document_store.py:401\u001b[0m, in \u001b[0;36mStorePartition.find_index_or_search_keys\u001b[0;34m(self, credentials, index_qks, search_qks, order_by)\u001b[0m\n\u001b[1;32m 394\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mfind_index_or_search_keys\u001b[39m(\n\u001b[1;32m 395\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 396\u001b[0m credentials: SyftVerifyKey,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 399\u001b[0m order_by: Optional[PartitionKey] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m 400\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Result[List[SyftObject], \u001b[38;5;28mstr\u001b[39m]:\n\u001b[0;32m--> 401\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_thread_safe_cbk\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 402\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_find_index_or_search_keys\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 403\u001b[0m \u001b[43m \u001b[49m\u001b[43mcredentials\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 404\u001b[0m \u001b[43m \u001b[49m\u001b[43mindex_qks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mindex_qks\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 405\u001b[0m \u001b[43m \u001b[49m\u001b[43msearch_qks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msearch_qks\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 406\u001b[0m \u001b[43m \u001b[49m\u001b[43morder_by\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43morder_by\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 407\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/workspace/PySyft/packages/syft/src/syft/store/document_store.py:355\u001b[0m, in \u001b[0;36mStorePartition._thread_safe_cbk\u001b[0;34m(self, cbk, *args, **kwargs)\u001b[0m\n\u001b[1;32m 354\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_thread_safe_cbk\u001b[39m(\u001b[38;5;28mself\u001b[39m, cbk: Callable, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m--> 355\u001b[0m locked \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlock\u001b[49m\u001b[38;5;241m.\u001b[39macquire(blocking\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m 356\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m locked:\n\u001b[1;32m 357\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mFAILED TO LOCK\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", - "\u001b[0;31mAttributeError\u001b[0m: 'SQLiteStorePartition' object has no attribute 'lock'" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Idle worker expired: b'5C5B-F4BE'\n", - "Idle worker expired: b'D140-D0EF'\n", - "Idle worker expired: b'4A71-B0A6'\n", - "Idle worker expired: b'4999-97E5'\n", - "Idle worker expired: b'897B-20D9'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Exception in thread Thread-6:\n", - "Traceback (most recent call last):\n", - " File \"/opt/anaconda3/envs/syft/lib/python3.9/threading.py\", line 980, in _bootstrap_inner\n", - " self.run()\n", - " File \"/opt/anaconda3/envs/syft/lib/python3.9/threading.py\", line 917, in run\n", - " self._target(*self._args, **self._kwargs)\n", - " File \"/Users/koen/workspace/PySyft/packages/syft/src/syft/service/queue/zmq_queue.py\", line 113, in read_items\n", - " for item in items:\n", - "TypeError: 'NoneType' object is not iterable\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "\n", - "Timeout elapsed after 30 seconds while trying to acquiring lock.\n", - "FAILED TO LOCK\n", - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Exception in thread Thread-12:\n", - "Traceback (most recent call last):\n", - " File \"/opt/anaconda3/envs/syft/lib/python3.9/threading.py\", line 980, in _bootstrap_inner\n", - " self.run()\n", - " File \"/opt/anaconda3/envs/syft/lib/python3.9/threading.py\", line 917, in run\n", - " self._target(*self._args, **self._kwargs)\n", - " File \"/Users/koen/workspace/PySyft/packages/syft/src/syft/service/queue/zmq_queue.py\", line 113, in read_items\n", - " for item in items:\n", - "TypeError: 'NoneType' object is not iterable\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[Errno 2] No such file or directory: '/tmp/sherlock/QueueItem.lock'\n", - "Timeout elapsed after 30 seconds while trying to acquiring lock.\n", - "FAILED TO LOCK\n" - ] - } - ], - "source": [ - "process_all.local_function(**action_kwargs)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "00af3a02", - "metadata": {}, - "outputs": [], - "source": [ - "# args, _, _, values = inspect.getargvalues(process_all.local_function)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "defa0fd2", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "a0eda41c", - "metadata": {}, - "outputs": [ - { - "ename": "TypeError", - "evalue": "process_all() missing 1 required positional argument: 'domain'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[11], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mprocess_all\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m3\u001b[39;49m\u001b[43m,\u001b[49m\u001b[38;5;241;43m4\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/workspace/PySyft/packages/syft/src/syft/service/code/user_code.py:593\u001b[0m, in \u001b[0;36mSubmitUserCode.__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 591\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m on_mock_data:\n\u001b[1;32m 592\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mWarning: The result you see is computed on MOCK data.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 593\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlocal_function\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mfiltered_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 594\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 595\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mNotImplementedError\u001b[39;00m\n", - "\u001b[0;31mTypeError\u001b[0m: process_all() missing 1 required positional argument: 'domain'" - ] - } - ], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "624b3d73", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7240fa0c", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "294c1e75", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e4efec44", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "id": "16877085", - "metadata": {}, - "source": [ - "Usecase 1: What if you have a syft function thats nested with jobs?\n", - "\n", - "Subproblem 1.1: We need to code before submitting\n", - "\n", - "- solution 1: remove the job aspect from local testing\n", - "- solution 2: you mimick the domain architecture locally\n", - "- solution 3: you run this on the low side on mock data\n", - "\n", - "\n", - "Subproblem 1.2: We need to be able to run this on the high side\n", - "\n", - "Solution\n", - "- move requests / code / inputs from low side to high side\n", - "- run job on the high side\n", - "- get results from job and load them into the low side\n", - "\n", - "Usecase 2: What if you have a syft function that takes days to run on the high side?\n", - "\n", - "See solution for 1.2\n", - "\n", - "Solutions based on example\n", - "\n", - "- we have an optional mock sample in asset creation\n", - "- we split submission and code_request on the DS side\n", - "- DS can call code that is submitted\n", - "\n", - "- what is the UX for running on the low side as a test vs a code request\n", - "\n", - "ux for running as a test locally\n", - "my_function(asset.mock_sample)\n", - "\n", - "ux for running as a test on the low side\n", - "\n", - "ds_client.code.my_function(asset.mock)\n", - "-> Should print info: runs on mock data\n", - "\n", - "ux for running the code on real data\n", - "\n", - "ds_client.code.my_function(asset)\n", - "-> Should print info: runs on real data\n", - "\n", - "the ux for a code request\n", - "\n", - "ds_client.make_code_request(\n", - "\n", - "ds_client.submit(code2)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "91202992", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "acc27591", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a92a1606", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6957dee7", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 17, - "id": "c30585d8", - "metadata": {}, - "outputs": [ - { - "ename": "AttributeError", - "evalue": "'DomainClient' object has no attribute 'submit'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[17], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mds_client\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msubmit\u001b[49m(code1)\n", - "\u001b[0;31mAttributeError\u001b[0m: 'DomainClient' object has no attribute 'submit'" - ] - } - ], - "source": [ - "ds_client.submit(code1)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4104c432", - "metadata": {}, - "outputs": [], - "source": [ - "ds_client.request(code1)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c5a62e47", - "metadata": {}, - "outputs": [], - "source": [ - "do_client.code" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "57773a01", - "metadata": {}, - "outputs": [], - "source": [ - "request = do_client.requests[0]" - ] - }, - { - "cell_type": "markdown", - "id": "3d8a0a1f", - "metadata": {}, - "source": [ - "code1:\n", - " blbalblab\n", - " code()\n", - "\n", - "code2:\n", - " blballbbla" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e227c845", - "metadata": {}, - "outputs": [], - "source": [ - "request.code2" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "30afd225", - "metadata": {}, - "outputs": [], - "source": [ - "do_client.all_code.code2" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "fb7adb6b", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1b437a65", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "id": "b15d4db8", - "metadata": {}, - "source": [ - "When doing local execution **without** a domain" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1e7ca2d8", - "metadata": {}, - "outputs": [], - "source": [ - "my_syft_function(x, y)" - ] - }, - { - "cell_type": "markdown", - "id": "edc738c5", - "metadata": {}, - "source": [ - "When doing local execution **with** a domain" - ] - }, - { - "cell_type": "markdown", - "id": "6604dd30", - "metadata": {}, - "source": [ - "- is it relevant to think about whether we need to configure the domain (e.g. consumsers)\n", - "- do we create the domain explicitly or does it happen on the fly?\n", - "- how do we indicate in the function call that we want to create a domain?" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ae8fec19", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "id": "a0e6837f", - "metadata": {}, - "source": [ - "When doing **high side** execution **without** a domain" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "df9bcd99", - "metadata": {}, - "outputs": [], - "source": [ - "users_function = func.unsafe_function\n", - "real_result = users_function(trade_data=pvt_data)" - ] - }, - { - "cell_type": "markdown", - "id": "b3959b10", - "metadata": {}, - "source": [ - "When doing **high side** exeuction **with** a domain" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "375ed965", - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "16/11/23 19:29:10 FUNCTION LOG (6470a06b282445ab8b912302c8c33a52): Launching jobs\n", - "16/11/23 19:29:10 FUNCTION LOG (6470a06b282445ab8b912302c8c33a52): starting aggregation\n", - "16/11/23 19:29:10 FUNCTION LOG (6470a06b282445ab8b912302c8c33a52): Done\n", - "16/11/23 19:29:11 FUNCTION LOG (511d17d79d0b450ab0f4d410bbbbe688): starting batch 1\n", - "16/11/23 19:29:11 EXCEPTION LOG (511d17d79d0b450ab0f4d410bbbbe688):\n", - "\n", - "Encountered while executing process_batch:\n", - "Traceback (most recent call last):\n", - " File \"/Users/koen/workspace/PySyft/packages/syft/src/syft/service/code/user_code.py\", line 1087, in execute_byte_code\n", - " result = eval(evil_string, _globals, _locals) # nosec\n", - " File \"\", line 1, in \n", - " File \"\", line 10, in user_func_process_batch_563f3f1899d092d0501158fbb086f3fd2c79a010c4e85bfe4fe12e0e1c01abde_3fa7f836b5a65455ef117b87f10f6a3f711d956132679e32522b6ae4c1440ec5\n", - " File \"\", line 6, in process_batch\n", - "IndexError: list index out of range\n", - "\n", - " 3 print(f'starting batch {batch}')\n", - " 4 from time import sleep\n", - "--> 5 [1, 2, 3, 4][5]\n", - " 6 sleep(1)\n", - "16/11/23 19:29:11 FUNCTION LOG (1f4245154b5a4e8ab20f7b3907203542): starting batch 2\n", - "16/11/23 19:29:11 EXCEPTION LOG (1f4245154b5a4e8ab20f7b3907203542):\n", - "\n", - "Encountered while executing process_batch:\n", - "Traceback (most recent call last):\n", - " File \"/Users/koen/workspace/PySyft/packages/syft/src/syft/service/code/user_code.py\", line 1087, in execute_byte_code\n", - " result = eval(evil_string, _globals, _locals) # nosec\n", - " File \"\", line 1, in \n", - " File \"\", line 10, in user_func_process_batch_563f3f1899d092d0501158fbb086f3fd2c79a010c4e85bfe4fe12e0e1c01abde_3fa7f836b5a65455ef117b87f10f6a3f711d956132679e32522b6ae4c1440ec5\n", - " File \"\", line 6, in process_batch\n", - "IndexError: list index out of range\n", - "\n", - " 3 print(f'starting batch {batch}')\n", - " 4 from time import sleep\n", - "--> 5 [1, 2, 3, 4][5]\n", - " 6 sleep(1)\n" - ] - } - ], - "source": [ - "job = ds_client.code.process_all(x=x_ptr, blocking=False)\n", - "sleep(5)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "0f8e12c0", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "starting batch 1\n", - "\n", - "Exception encountered while running process_batch, please contact the Node Admin for more info.\n" - ] - } - ], - "source": [ - "job.subjobs[0].logs()" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "6d97d2a4", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "

Job List

\n", - "
\n", - "\n", - "
\n", - "
\n", - "
\n", - "
\n", - "
\n", - " \n", - "
\n", - " \n", - "
\n", - " \n", - "
\n", - "\n", - "

0

\n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n", - " \n", - " \n" - ], - "text/plain": [ - "[syft.service.job.job_stash.Job, syft.service.job.job_stash.Job]" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "job.subjobs" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "afbe026b", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "starting batch 1\n", - "\n", - "\n", - "Encountered while executing process_batch:\n", - "Traceback (most recent call last):\n", - " File \"/Users/koen/workspace/PySyft/packages/syft/src/syft/service/code/user_code.py\", line 1087, in execute_byte_code\n", - " result = eval(evil_string, _globals, _locals) # nosec\n", - " File \"\", line 1, in \n", - " File \"\", line 10, in user_func_process_batch_563f3f1899d092d0501158fbb086f3fd2c79a010c4e85bfe4fe12e0e1c01abde_3fa7f836b5a65455ef117b87f10f6a3f711d956132679e32522b6ae4c1440ec5\n", - " File \"\", line 6, in process_batch\n", - "IndexError: list index out of range\n", - "\n", - " 3 print(f'starting batch {batch}')\n", - " 4 from time import sleep\n", - "--> 5 [1, 2, 3, 4][5]\n", - " 6 sleep(1)\n" - ] - } - ], - "source": [ - "client.jobs[0].subjobs[0].logs()" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "id": "81c23727", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "

Job List

\n", - "
\n", - "\n", - "
\n", - "
\n", - "
\n", - "
\n", - "
\n", - " \n", - "
\n", - " \n", - "
\n", - " \n", - "
\n", - "\n", - "

0

\n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n", - " \n", - " \n" - ], - "text/plain": [ - "[syft.service.job.job_stash.Job, syft.service.job.job_stash.Job]" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "job.subjobs" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "a9fde08d", - "metadata": {}, - "outputs": [ - { - "ename": "IndexError", - "evalue": "list index out of range", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mIndexError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[12], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mjob\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msubjobs\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[38;5;241m.\u001b[39mlogs(stderr\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n", - "\u001b[0;31mIndexError\u001b[0m: list index out of range" - ] - } - ], - "source": [ - "job.subjobs[0].logs(stderr=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "78bd08dc", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "starting batch 2\n", - "\n", - "\n", - "Encountered while executing process_batch:\n", - "Traceback (most recent call last):\n", - " File \"/Users/koen/workspace/PySyft/packages/syft/src/syft/service/code/user_code.py\", line 1072, in execute_byte_code\n", - " result = eval(evil_string, _globals, _locals) # nosec\n", - " File \"\", line 1, in \n", - " File \"\", line 10, in user_func_process_batch_47f8c8f3db3a30695a28e4a51e44916669ac3d111924cb614181c64b2c3b8323_3fa7f836b5a65455ef117b87f10f6a3f711d956132679e32522b6ae4c1440ec5\n", - " File \"\", line 6, in process_batch\n", - "IndexError: list index out of range\n", - "\n", - " 3 print(f'starting batch {batch}')\n", - " 4 from time import sleep\n", - "--> 5 [1, 2, 3, 4][5]\n", - " 6 sleep(1)\n" - ] - } - ], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "cc0db669", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Pointer:\n", - "None" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "job.wait()" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "5bf0974f", - "metadata": {}, - "outputs": [ - { - "ename": "TypeError", - "evalue": "unsupported operand type(s) for +: 'int' and 'Err'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[11], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;43msum\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m[\u001b[49m\u001b[43mj\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwait\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mj\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mjob\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msubjobs\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n", - "\u001b[0;31mTypeError\u001b[0m: unsupported operand type(s) for +: 'int' and 'Err'" - ] - } - ], - "source": [ - "sum([j.wait().get() for j in job.subjobs])" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "55892031", - "metadata": {}, - "outputs": [], - "source": [ - "# import sys" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "b0800f1d", - "metadata": {}, - "outputs": [], - "source": [ - "# print(\"D\")\n", - "# print(\"A\")\n", - "\n", - "# try:\n", - "# raise ValueError()\n", - "# except Exception as e:\n", - "# tb = e.__traceback__\n", - "# # print(len([tb.tb_next while tb is not None])\n", - "# # trace = []\n", - "# # while tb is not None:\n", - "# # trace.append({\n", - "# # \"filename\": tb.tb_frame.f_code.co_filename,\n", - "# # \"name\": tb.tb_frame.f_code.co_name,\n", - "# # \"lineno\": tb.tb_lineno\n", - "# # })\n", - "# # tb = tb.tb_next\n", - "\n", - "\n", - "# # print(sys.exc_info()[-1].tb_lineno)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "6c8fd04c", - "metadata": {}, - "outputs": [], - "source": [ - "# n = 0\n", - "# while tb is not None:\n", - "# tb = tb.tb_next\n", - "# n+=1" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "0c9368c9", - "metadata": {}, - "outputs": [], - "source": [ - "# n" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "1d73ef1e", - "metadata": {}, - "outputs": [], - "source": [ - "# trace" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "e61760f5", - "metadata": {}, - "outputs": [], - "source": [ - "import traceback\n", - "import sys" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "f5f2c781", - "metadata": {}, - "outputs": [], - "source": [ - "# try:\n", - "# raise ValueError()\n", - "# except Exception as e:\n", - "# print(sys.exception())\n", - "# # print()\n", - "\n", - "\n", - "# # # traceback.print_stack()\n", - "\n", - "\n", - "# # # tb = e.__traceback__\n", - "# # # traceback.print_tb(type(e), e, tb)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ead1e368", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "e956f350", - "metadata": {}, - "outputs": [], - "source": [ - "# print(\"\\n\".join(['def user_func_process_batch_47f8c8f3db3a30695a28e4a51e44916669ac3d111924cb614181c64b2c3b8323_bd42b6607712b09a0b187e3c67030f772c5dcd3c4152e4abebcb3a9f0a8259e4(batch):',\n", - "# '', ' def process_batch(batch):', \" print(f'starting batch {batch}')\",\n", - "# ' from time import sleep', ' sleep(1)', ' [1, 2, 3][5]',\n", - "# \" print('done')\", ' return batch + 1', ' result = process_batch(batch=batch)',\n", - "# ' return result']))" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.16" - }, - "toc": { - "base_numbering": 1, - "nav_menu": {}, - "number_sections": true, - "sideBar": true, - "skip_h1_title": false, - "title_cell": "Table of Contents", - "title_sidebar": "Contents", - "toc_cell": false, - "toc_position": {}, - "toc_section_display": true, - "toc_window_display": true - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} From 0b8abc4842f2ed789d24b3a93edb152e5e22ce64 Mon Sep 17 00:00:00 2001 From: teo Date: Mon, 8 Jan 2024 15:51:29 +0200 Subject: [PATCH 10/12] remove print debugging --- packages/syft/src/syft/service/dataset/dataset.py | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/syft/src/syft/service/dataset/dataset.py b/packages/syft/src/syft/service/dataset/dataset.py index 381c44aac2d..87fed2a65b2 100644 --- a/packages/syft/src/syft/service/dataset/dataset.py +++ b/packages/syft/src/syft/service/dataset/dataset.py @@ -755,7 +755,6 @@ def infer_shape(context: TransformContext) -> TransformContext: if context.output["shape"] is None: if not _is_action_data_empty(context.obj.mock): context.output["shape"] = get_shape_or_len(context.obj.mock) - print(context.output["shape"]) return context From cced40bbca88a3ca63fd6c4751b8894f5bbc8b8b Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Mon, 8 Jan 2024 17:37:56 +0100 Subject: [PATCH 11/12] lint --- .../src/syft/protocol/protocol_version.json | 2 +- .../src/syft/service/blob_storage/service.py | 26 +++++++++++++------ packages/syft/src/syft/types/blob_storage.py | 2 +- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/packages/syft/src/syft/protocol/protocol_version.json b/packages/syft/src/syft/protocol/protocol_version.json index fa31a1419b6..f0100bd0182 100644 --- a/packages/syft/src/syft/protocol/protocol_version.json +++ b/packages/syft/src/syft/protocol/protocol_version.json @@ -985,7 +985,7 @@ "SeaweedSecureFilePathLocation": { "2": { "version": 2, - "hash": "3ca49db7536a33d5712485164e95406000df9af2aed78e9f9fa2bb2bbbb34fe6", + "hash": "5fd63fed2a4efba8c2b6c7a7b5e9b5939181781c331230896aa130b6fd558739", "action": "add" } }, diff --git a/packages/syft/src/syft/service/blob_storage/service.py b/packages/syft/src/syft/service/blob_storage/service.py index f485c2e67bf..d1ef73aab9f 100644 --- a/packages/syft/src/syft/service/blob_storage/service.py +++ b/packages/syft/src/syft/service/blob_storage/service.py @@ -20,6 +20,7 @@ from ...types.blob_storage import BlobStorageEntry from ...types.blob_storage import BlobStorageMetadata from ...types.blob_storage import CreateBlobStorageEntry +from ...types.blob_storage import SeaweedSecureFilePathLocation from ...types.uid import UID from ..context import AuthedServiceContext from ..response import SyftError @@ -63,6 +64,7 @@ def mount_azure( account_key: str, container_name: str, bucket_name: str, + use_direct_connections=True, ): # stdlib @@ -105,14 +107,22 @@ def mount_azure( objects = res["Contents"] file_sizes = [object["Size"] for object in objects] file_paths = [object["Key"] for object in objects] - secure_file_paths = [ - AzureSecureFilePathLocation( - path=file_path, - azure_profile_name=remote_name, - bucket_name=bucket_name, - ) - for file_path in file_paths - ] + if use_direct_connections: + secure_file_paths = [ + AzureSecureFilePathLocation( + path=file_path, + azure_profile_name=remote_name, + bucket_name=bucket_name, + ) + for file_path in file_paths + ] + else: + secure_file_paths = [ + SeaweedSecureFilePathLocation( + path=file_path, + ) + for file_path in file_paths + ] for sfp, file_size in zip(secure_file_paths, file_sizes): blob_storage_entry = BlobStorageEntry( diff --git a/packages/syft/src/syft/types/blob_storage.py b/packages/syft/src/syft/types/blob_storage.py index 7f0438acf2e..899cb04fcbd 100644 --- a/packages/syft/src/syft/types/blob_storage.py +++ b/packages/syft/src/syft/types/blob_storage.py @@ -201,7 +201,7 @@ class SeaweedSecureFilePathLocation(SecureFilePathLocation): __canonical_name__ = "SeaweedSecureFilePathLocation" __version__ = SYFT_OBJECT_VERSION_2 - upload_id: str + upload_id: Optional[str] = None def generate_url(self, connection, type_, bucket_name): try: From f36d291b225832280c08241e821ebb54b8a0830a Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Mon, 8 Jan 2024 21:51:50 +0100 Subject: [PATCH 12/12] fix bugs --- packages/syft/src/syft/service/worker/worker_pool_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/service/worker/worker_pool_service.py b/packages/syft/src/syft/service/worker/worker_pool_service.py index 53577f5d4b1..6bd8143833d 100644 --- a/packages/syft/src/syft/service/worker/worker_pool_service.py +++ b/packages/syft/src/syft/service/worker/worker_pool_service.py @@ -185,7 +185,7 @@ def add_workers( result = self.image_stash.get_by_uid( credentials=context.credentials, - uid=worker_pool.syft_worker_image_id, + uid=worker_pool.image.id, ) if result.is_err(): @@ -204,7 +204,7 @@ def add_workers( worker_stash=self.worker_stash, ) - worker_pool.worker_list.append(worker_list) + worker_pool.worker_list += worker_list worker_pool.max_count = existing_worker_cnt + number update_result = self.stash.update(