From c7153c87cdc10eaca31d5a16ecb1ffed05b41525 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Mon, 8 Apr 2024 17:34:33 +0530 Subject: [PATCH 1/4] delete references of veilid from code --- .github/workflows/cd-syft-dev.yml | 11 - .github/workflows/cd-syft.yml | 27 +- .../Testing/Veilid/Alice-Python-Server.ipynb | 281 --- .../Testing/Veilid/Alice-Veilid-Peer.ipynb | 302 --- .../Testing/Veilid/Bob-Python-Server.ipynb | 202 -- .../Testing/Veilid/Bob-Veilid-Peer.ipynb | 211 --- .../Veilid/Large-Message-Testing.ipynb | 397 ---- .../Veilid Route-Connection-Testing.ipynb | 116 -- .../Veilid/Veilid-Connection-Test.ipynb | 554 ------ .../Veilid/Veilid-Gateway-Testing.ipynb | 226 --- notebooks/Testing/Veilid/utils.py | 12 - packages/grid/default.env | 3 - packages/grid/veilid/.dockerignore | 1 - packages/grid/veilid/development.md | 67 - packages/grid/veilid/requirements.txt | 4 - packages/grid/veilid/server/constants.py | 14 - packages/grid/veilid/server/main.py | 161 -- packages/grid/veilid/server/models.py | 16 - packages/grid/veilid/server/utils.py | 58 - .../grid/veilid/server/veilid_callback.py | 66 - .../grid/veilid/server/veilid_connection.py | 36 - .../server/veilid_connection_singleton.py | 35 - packages/grid/veilid/server/veilid_core.py | 182 -- packages/grid/veilid/server/veilid_db.py | 54 - .../veilid/server/veilid_streamer.excalidraw | 1676 ----------------- .../grid/veilid/server/veilid_streamer.py | 513 ----- packages/grid/veilid/start.sh | 21 - packages/grid/veilid/veilid-server.conf | 9 - packages/grid/veilid/veilid.dockerfile | 43 - packages/syft/src/syft/client/client.py | 248 --- packages/syft/src/syft/client/protocol.py | 1 - packages/syft/src/syft/node/node.py | 4 +- .../syft/service/network/network_service.py | 99 - .../src/syft/service/network/node_peer.py | 10 - .../syft/src/syft/service/network/routes.py | 26 +- packages/syft/src/syft/service/service.py | 19 - .../syft/src/syft/service/veilid/__init__.py | 18 - .../syft/service/veilid/veilid_endpoints.py | 8 - .../src/syft/service/veilid/veilid_service.py | 92 - .../integration/veilid/gateway_veilid_test.py | 96 - tox.ini | 4 - 41 files changed, 3 insertions(+), 5920 deletions(-) delete mode 100644 notebooks/Testing/Veilid/Alice-Python-Server.ipynb delete mode 100644 notebooks/Testing/Veilid/Alice-Veilid-Peer.ipynb delete mode 100644 notebooks/Testing/Veilid/Bob-Python-Server.ipynb delete mode 100644 notebooks/Testing/Veilid/Bob-Veilid-Peer.ipynb delete mode 100644 notebooks/Testing/Veilid/Large-Message-Testing.ipynb delete mode 100644 notebooks/Testing/Veilid/Veilid Route-Connection-Testing.ipynb delete mode 100644 notebooks/Testing/Veilid/Veilid-Connection-Test.ipynb delete mode 100644 notebooks/Testing/Veilid/Veilid-Gateway-Testing.ipynb delete mode 100644 notebooks/Testing/Veilid/utils.py delete mode 100644 packages/grid/veilid/.dockerignore delete mode 100644 packages/grid/veilid/development.md delete mode 100644 packages/grid/veilid/requirements.txt delete mode 100644 packages/grid/veilid/server/constants.py delete mode 100644 packages/grid/veilid/server/main.py delete mode 100644 packages/grid/veilid/server/models.py delete mode 100644 packages/grid/veilid/server/utils.py delete mode 100644 packages/grid/veilid/server/veilid_callback.py delete mode 100644 packages/grid/veilid/server/veilid_connection.py delete mode 100644 packages/grid/veilid/server/veilid_connection_singleton.py delete mode 100644 packages/grid/veilid/server/veilid_core.py delete mode 100644 packages/grid/veilid/server/veilid_db.py delete mode 100644 packages/grid/veilid/server/veilid_streamer.excalidraw delete mode 100644 packages/grid/veilid/server/veilid_streamer.py delete mode 100644 packages/grid/veilid/start.sh delete mode 100644 packages/grid/veilid/veilid-server.conf delete mode 100644 packages/grid/veilid/veilid.dockerfile delete mode 100644 packages/syft/src/syft/service/veilid/__init__.py delete mode 100644 packages/syft/src/syft/service/veilid/veilid_endpoints.py delete mode 100644 packages/syft/src/syft/service/veilid/veilid_service.py delete mode 100644 tests/integration/veilid/gateway_veilid_test.py diff --git a/.github/workflows/cd-syft-dev.yml b/.github/workflows/cd-syft-dev.yml index cca81c6e2b6..7accad71745 100644 --- a/.github/workflows/cd-syft-dev.yml +++ b/.github/workflows/cd-syft-dev.yml @@ -118,17 +118,6 @@ jobs: ${{ secrets.ACR_SERVER }}/openmined/grid-seaweedfs:dev-${{ github.sha }} ${{ secrets.ACR_SERVER }}/openmined/grid-seaweedfs:${{ steps.grid.outputs.GRID_VERSION }} - - name: Build and push `grid-veilid` image to registry - uses: docker/build-push-action@v5 - with: - context: ./packages/grid/veilid - file: ./packages/grid/veilid/veilid.dockerfile - push: true - tags: | - ${{ secrets.ACR_SERVER }}/openmined/grid-veilid:dev - ${{ secrets.ACR_SERVER }}/openmined/grid-veilid:dev-${{ github.sha }} - ${{ secrets.ACR_SERVER }}/openmined/grid-veilid:${{ steps.grid.outputs.GRID_VERSION }} - - name: Build Helm Chart & Copy to infra if: github.ref == 'refs/heads/dev' || github.event.inputs.deploy-helm == 'true' shell: bash diff --git a/.github/workflows/cd-syft.yml b/.github/workflows/cd-syft.yml index 27f756ea348..baba3343fac 100644 --- a/.github/workflows/cd-syft.yml +++ b/.github/workflows/cd-syft.yml @@ -248,24 +248,7 @@ jobs: digest="${{ steps.grid-seaweedfs-build.outputs.digest }}" touch "/tmp/digests/grid-seaweedfs/${digest#sha256:}" - - name: Build and push `grid-veilid` image to DockerHub - id: grid-veilid-build - uses: docker/build-push-action@v5 - with: - context: ./packages/grid/veilid - file: ./packages/grid/veilid/veilid.dockerfile - platforms: ${{ steps.release_metadata.outputs.release_platform }} - outputs: type=image,name=openmined/grid-veilid,push-by-digest=true,name-canonical=true,push=true - cache-from: type=registry,ref=openmined/grid-veilid:cache-${{ steps.release_metadata.outputs.short_release_platform }} - cache-to: type=registry,ref=openmined/grid-veilid:cache-${{ steps.release_metadata.outputs.short_release_platform}},mode=max - - - name: Export digest for grid-veilid - run: | - mkdir -p /tmp/digests/grid-veilid - digest="${{ steps.grid-veilid-build.outputs.digest }}" - touch "/tmp/digests/grid-veilid/${digest#sha256:}" - - - name: Upload digest for grid-backend, grid-frontend and grid-seaweedfs, grid-veilid + - name: Upload digest for grid-backend, grid-frontend and grid-seaweedfs uses: actions/upload-artifact@v4 with: name: digests-${{ steps.release_metadata.outputs.grid_version }}-${{ steps.release_metadata.outputs.short_release_platform }} @@ -324,14 +307,6 @@ jobs: -t openmined/grid-seaweedfs:${{ needs.build-and-push-docker-images.outputs.release_tag }} \ $(printf 'openmined/grid-seaweedfs@sha256:%s ' *) - - name: Create manifest list and push for grid-veilid - working-directory: /tmp/digests/grid-veilid - run: | - docker buildx imagetools create \ - -t openmined/grid-veilid:${{ needs.build-and-push-docker-images.outputs.grid_version }} \ - -t openmined/grid-veilid:${{ needs.build-and-push-docker-images.outputs.release_tag }} \ - $(printf 'openmined/grid-veilid@sha256:%s ' *) - deploy-syft: needs: [merge-docker-images] if: always() && needs.merge-docker-images.result == 'success' diff --git a/notebooks/Testing/Veilid/Alice-Python-Server.ipynb b/notebooks/Testing/Veilid/Alice-Python-Server.ipynb deleted file mode 100644 index 3e1b7065c2c..00000000000 --- a/notebooks/Testing/Veilid/Alice-Python-Server.ipynb +++ /dev/null @@ -1,281 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "c74990eb-d769-4117-8c88-e9210136606e", - "metadata": {}, - "source": [ - "## Alice Python Server" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "20df98d8-de6c-496c-b30e-6421ac99401c", - "metadata": {}, - "outputs": [], - "source": [ - "# third party\n", - "import requests" - ] - }, - { - "cell_type": "markdown", - "id": "54885cd0-f803-4911-8423-e595dc4cd7c3", - "metadata": {}, - "source": [ - "### 1. Create DHT Key and Private Route" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "41d82ff3-ceda-4569-8178-8758ef635cb0", - "metadata": {}, - "outputs": [], - "source": [ - "host = \"localhost\"\n", - "port = 4000" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0d9f3cca-66a7-4e6c-a332-b38a8f5c02db", - "metadata": {}, - "outputs": [], - "source": [ - "res = requests.post(f\"http://{host}:{port}/generate_vld_key\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "81c6aa9d-26b4-4672-a059-643edfeeed95", - "metadata": {}, - "outputs": [], - "source": [ - "res.content" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4a9487e3-f5c8-468e-acd0-261e21bc3e14", - "metadata": {}, - "outputs": [], - "source": [ - "res = requests.get(f\"http://{host}:{port}/retrieve_vld_key\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5b87e9e6-244f-47f7-a31a-fa7cbce65b88", - "metadata": {}, - "outputs": [], - "source": [ - "self_vld_key = res.json()[\"message\"]\n", - "print(\"=\" * 30)\n", - "print(self_vld_key)\n", - "print(\"=\" * 30)" - ] - }, - { - "cell_type": "markdown", - "id": "a8c70d99-6814-453d-80bf-d141c40ba24e", - "metadata": {}, - "source": [ - "### Send AppMessage using VLD Key to Self" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a7495805-817d-44d9-ad62-32407b42316c", - "metadata": {}, - "outputs": [], - "source": [ - "# Cannot send messages to self, due to local routing feature not\n", - "# available in direct routing" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "aca01ec6-1bbe-44b5-ad4a-053ba1edcfe6", - "metadata": {}, - "outputs": [], - "source": [ - "# json_data = {\"dht_key\": self_dht_key, \"message\": \"Hello to me again\"}\n", - "# app_message = requests.post(f\"http://{host}:{port}/app_message\", json=json_data)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ff09ab92-3423-483a-abf3-51e8c2448cf9", - "metadata": {}, - "outputs": [], - "source": [ - "# app_message.content" - ] - }, - { - "cell_type": "markdown", - "id": "4d0d9e39-bf05-4ef3-b00a-2bb605f041ee", - "metadata": {}, - "source": [ - "### Send AppCall using VLD Key to Self" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b8bc9f54-b2f0-4f88-8897-f640866ba2ed", - "metadata": {}, - "outputs": [], - "source": [ - "# json_data = {\"dht_key\": self_dht_key, \"message\": \"Hello to app call\"}\n", - "# app_call = requests.post(f\"http://{host}:{port}/app_call\", json=json_data)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2c1c4148-461a-459e-846a-fad332a7ce3a", - "metadata": {}, - "outputs": [], - "source": [ - "# app_call.json()" - ] - }, - { - "cell_type": "markdown", - "id": "ddba6e22-96ee-46d7-8251-fcaa4140253b", - "metadata": {}, - "source": [ - "### Ping Peer " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3de4b843-f3a2-4d96-bd48-121ae2b6f197", - "metadata": {}, - "outputs": [], - "source": [ - "peer_vld_key = str(input(\"Enter Peer VLD Key\"))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "575c3441-cd11-4a42-ab4e-0bde3e5d5c72", - "metadata": {}, - "outputs": [], - "source": [ - "peer_vld_key" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "64d0b338-a439-4982-b739-24c056833be1", - "metadata": {}, - "outputs": [], - "source": [ - "res = requests.post(f\"http://{host}:{port}/ping/{peer_vld_key}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3ce13553-dae5-442e-bd56-2dddb526c0f2", - "metadata": {}, - "outputs": [], - "source": [ - "res.json()" - ] - }, - { - "cell_type": "markdown", - "id": "fd824cca-2a7f-4ea9-9e67-1c06d1f8bec2", - "metadata": {}, - "source": [ - "### Send AppMessage using VLD Key to Peer" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2e2c1341-d840-4429-b3e5-093d8e90365e", - "metadata": {}, - "outputs": [], - "source": [ - "json_data = {\"vld_key\": peer_vld_key, \"message\": \"How are you doing , Bob\"}\n", - "app_message = requests.post(f\"http://{host}:{port}/app_message\", json=json_data)" - ] - }, - { - "cell_type": "markdown", - "id": "153377f6-698e-4013-9be3-0833b71ee0c4", - "metadata": {}, - "source": [ - "### Send Proxy Message " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "271d7316-eaab-438c-9192-55a4e44b9dea", - "metadata": {}, - "outputs": [], - "source": [ - "res = requests.get(\n", - " f\"http://{host}:{port}/proxy\",\n", - " json={\"url\": \"https://www.google.com\", \"method\": \"GET\", \"vld_key\": self_vld_key},\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "77e1ad1d-379a-4899-8805-c703ad437c0d", - "metadata": {}, - "outputs": [], - "source": [ - "res.content" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "73c1f0b0-d240-4964-a88b-365ea89b1bdd", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.8" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks/Testing/Veilid/Alice-Veilid-Peer.ipynb b/notebooks/Testing/Veilid/Alice-Veilid-Peer.ipynb deleted file mode 100644 index 0541d90a921..00000000000 --- a/notebooks/Testing/Veilid/Alice-Veilid-Peer.ipynb +++ /dev/null @@ -1,302 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "3ab7de7e-d23e-4cfc-895d-0bd02d9bc17f", - "metadata": {}, - "outputs": [], - "source": [ - "# stdlib\n", - "\n", - "# stdlib\n", - "import asyncio\n", - "\n", - "# third party\n", - "import veilid\n", - "from veilid import KeyPair" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "461ef888-76c1-4256-ad69-cbf405d830e0", - "metadata": {}, - "outputs": [], - "source": [ - "app_message_queue: asyncio.Queue = asyncio.Queue()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ad857491-4cad-4a21-bf10-8035f05ef52a", - "metadata": {}, - "outputs": [], - "source": [ - "host = \"localhost\"\n", - "port = 5959" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8d5984fd-30eb-4c9f-8bb7-e0a827b6de2c", - "metadata": {}, - "outputs": [], - "source": [ - "async def noop_callback(update: veilid.VeilidUpdate):\n", - " if update.kind == veilid.VeilidUpdateKind.APP_MESSAGE:\n", - " print(\"Received App Message\")\n", - " await app_message_queue.put(update)\n", - "\n", - "\n", - "async def connect(host: str, port: int):\n", - " conn = await veilid.json_api_connect(host, port, noop_callback)\n", - " return conn" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1296f362-60ff-471a-807b-6d96dbf36403", - "metadata": {}, - "outputs": [], - "source": [ - "conn = await connect(host, port)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "418b05e9-eaac-4bb5-b61a-e300221b10ba", - "metadata": {}, - "outputs": [], - "source": [ - "# route_id, blob = await conn.new_private_route()\n", - "# Stable and reliable route\n", - "# Creating a new one\n", - "route_id, blob = await conn.new_custom_private_route(\n", - " [veilid.CryptoKind.CRYPTO_KIND_VLD0],\n", - " veilid.Stability.RELIABLE,\n", - " veilid.Sequencing.ENSURE_ORDERED,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "dade6f0b-27f7-4317-84e0-83f5db2f93ea", - "metadata": {}, - "outputs": [], - "source": [ - "route_id, blob" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "de7eb3e2", - "metadata": {}, - "outputs": [], - "source": [ - "# Creating a new routing context\n", - "# Since it is safe by default , we could remove default safety\n", - "router = await (await conn.new_routing_context()).with_default_safety()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f76a7403", - "metadata": {}, - "outputs": [], - "source": [ - "router" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ad3b3dbc", - "metadata": {}, - "outputs": [], - "source": [ - "alice_record = await router.create_dht_record(veilid.DHTSchema.dflt(1))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3922a41b", - "metadata": {}, - "outputs": [], - "source": [ - "# Creation of a Record in DHT DFLT schema , creates a new public and private key pair for the owner\n", - "# that is different from the NodeID public key\n", - "alice_private_key = alice_record.owner_secret\n", - "alice_public_key = alice_record.owner" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "77dd5ed2", - "metadata": {}, - "outputs": [], - "source": [ - "alice_private_key, alice_public_key, alice_record.key" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "fb3c60bb", - "metadata": {}, - "outputs": [], - "source": [ - "# Close the record\n", - "await router.close_dht_record(alice_record.key)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "73249f4a", - "metadata": {}, - "outputs": [], - "source": [ - "key_pair = KeyPair.from_parts(key=alice_public_key, secret=alice_private_key)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b63cb28c", - "metadata": {}, - "outputs": [], - "source": [ - "key_pair" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3afe0d6a", - "metadata": {}, - "outputs": [], - "source": [ - "record_open = await router.open_dht_record(alice_record.key, key_pair)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b12b2281", - "metadata": {}, - "outputs": [], - "source": [ - "record_open.key" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "11d2d05d", - "metadata": {}, - "outputs": [], - "source": [ - "await router.set_dht_value(record_open.key, 0, blob)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "833bf7e0-c3f9-4280-aa8d-5b1302b00f0f", - "metadata": {}, - "outputs": [], - "source": [ - "record_open.key[5::]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "df056893-c98d-45b4-9eff-c3e6301ce7e4", - "metadata": {}, - "outputs": [], - "source": [ - "self_prr = await conn.import_remote_private_route(blob)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "75b7efff-3b20-4446-ad8d-94ea0f5f4f26", - "metadata": {}, - "outputs": [], - "source": [ - "message_send = await router.app_message(self_prr, b\"Hello to me\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b6764bf2-30c7-4baa-81e8-c097be06dbea", - "metadata": {}, - "outputs": [], - "source": [ - "value = await app_message_queue.get()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2d52874c-ad2e-46ef-a9c9-7447661bc7fa", - "metadata": {}, - "outputs": [], - "source": [ - "assert value.kind == veilid.VeilidUpdateKind.APP_MESSAGE" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b690487b-20fb-4b36-9076-33915a184354", - "metadata": {}, - "outputs": [], - "source": [ - "value.detail.message" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9e0453a0-4c6d-4d83-aa01-232cab545653", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks/Testing/Veilid/Bob-Python-Server.ipynb b/notebooks/Testing/Veilid/Bob-Python-Server.ipynb deleted file mode 100644 index 35deb460032..00000000000 --- a/notebooks/Testing/Veilid/Bob-Python-Server.ipynb +++ /dev/null @@ -1,202 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "a003292f-d8f6-4888-b47d-9e0e9b1309ec", - "metadata": {}, - "source": [ - "## Bob Python Server" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "338b22f9-938e-4628-9636-14c192e42e49", - "metadata": {}, - "outputs": [], - "source": [ - "# third party\n", - "import requests" - ] - }, - { - "cell_type": "markdown", - "id": "f1279a42-f391-4ec8-b711-e9a05d601ce2", - "metadata": {}, - "source": [ - "### 1. Create DHT Key and Private Route" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "755d48fe-9471-4474-b47f-d344d31604aa", - "metadata": {}, - "outputs": [], - "source": [ - "host = \"localhost\"\n", - "port = 4001" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f14915f1-2535-424b-bdd9-23efab16bb43", - "metadata": {}, - "outputs": [], - "source": [ - "res = requests.post(f\"http://{host}:{port}/generate_vld_key\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "29aa597d-660e-4524-82ac-62c119e10fdf", - "metadata": {}, - "outputs": [], - "source": [ - "res.content" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "632ccceb-f742-4c8a-b00f-c55e6333fdc1", - "metadata": {}, - "outputs": [], - "source": [ - "res = requests.get(f\"http://{host}:{port}/retrieve_vld_key\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a7b8581a-a73d-4d15-97ec-2869aff00e90", - "metadata": {}, - "outputs": [], - "source": [ - "self_vld_key = res.json()[\"message\"]\n", - "print(\"=\" * 30)\n", - "print(self_vld_key)\n", - "print(\"=\" * 30)" - ] - }, - { - "cell_type": "markdown", - "id": "616f208c-fead-40cc-9391-416b59d7dc15", - "metadata": {}, - "source": [ - "### Send AppMessage using DHT Key to Self" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3e810776-491d-4170-a9c5-bf7eaf2995bd", - "metadata": {}, - "outputs": [], - "source": [ - "# Cannot send messages to self, due to local routing feature not\n", - "# available in direct routing" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "538913ae-29be-41a5-9608-4c694ccb392b", - "metadata": {}, - "outputs": [], - "source": [ - "# json_data = {\"dht_key\": self_dht_key, \"message\": \"Hello to me\"}\n", - "# app_message = requests.post(f\"http://{host}:{port}/app_message\", json=json_data)" - ] - }, - { - "cell_type": "markdown", - "id": "3ed2c114-eab7-4be7-bd89-d5ec3a7ec4c2", - "metadata": {}, - "source": [ - "### Send AppCall using DHT Key to Self" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "db49c78d-9767-4358-aa00-e740ce04e000", - "metadata": {}, - "outputs": [], - "source": [ - "# json_data = {\"dht_key\": self_dht_key, \"message\": \"Hello to app call\"}\n", - "# app_call = requests.post(f\"http://{host}:{port}/app_call\", json=json_data)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9bc0a69e-7cff-42fc-8859-e5de6edacdeb", - "metadata": {}, - "outputs": [], - "source": [ - "# app_call.json()" - ] - }, - { - "cell_type": "markdown", - "id": "73eee970-bb61-4014-9380-1944587b929a", - "metadata": {}, - "source": [ - "### Send AppMessage using DHT Key to Peer" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9e5671f6-1ffd-410c-b72a-6fb39f68fe93", - "metadata": {}, - "outputs": [], - "source": [ - "peer_vld_key = input(\"Enter Peer VLD Key\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8a753450-19e3-4603-ae93-a48bfbc7f829", - "metadata": {}, - "outputs": [], - "source": [ - "json_data = {\"vld_key\": peer_vld_key, \"message\": \"Hello Alice\"}\n", - "app_message = requests.post(f\"http://{host}:{port}/app_message\", json=json_data)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0cf79332-1a88-4d02-87b7-53c19d4fd1ad", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks/Testing/Veilid/Bob-Veilid-Peer.ipynb b/notebooks/Testing/Veilid/Bob-Veilid-Peer.ipynb deleted file mode 100644 index 4c974d0bea4..00000000000 --- a/notebooks/Testing/Veilid/Bob-Veilid-Peer.ipynb +++ /dev/null @@ -1,211 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "3ab7de7e-d23e-4cfc-895d-0bd02d9bc17f", - "metadata": {}, - "outputs": [], - "source": [ - "# stdlib\n", - "\n", - "# third party\n", - "from utils import get_typed_key\n", - "import veilid" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ad857491-4cad-4a21-bf10-8035f05ef52a", - "metadata": {}, - "outputs": [], - "source": [ - "host = \"localhost\"\n", - "port = 5960" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8d5984fd-30eb-4c9f-8bb7-e0a827b6de2c", - "metadata": {}, - "outputs": [], - "source": [ - "async def noop_callback(*args, **kwargs):\n", - " return\n", - "\n", - "\n", - "async def connect(host: str, port: int):\n", - " conn = await veilid.json_api_connect(host, port, noop_callback)\n", - " return conn" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1296f362-60ff-471a-807b-6d96dbf36403", - "metadata": {}, - "outputs": [], - "source": [ - "conn = await connect(host, port)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "de7eb3e2", - "metadata": {}, - "outputs": [], - "source": [ - "# Creating a new routing context\n", - "router = await (await conn.new_routing_context()).with_default_safety()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cbb26e88-6935-471a-a248-a46fdcba8e18", - "metadata": {}, - "outputs": [], - "source": [ - "conn.new_routing_context?" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f76a7403", - "metadata": {}, - "outputs": [], - "source": [ - "router" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "40370b01", - "metadata": {}, - "outputs": [], - "source": [ - "# Get this DHT Key from the Previous Notebook\n", - "# paste only the string party without VLD0: prefix\n", - "alice_dht_key_str = input(\"Enter Alice's DHT Key: \")\n", - "dht_key = get_typed_key(alice_dht_key_str)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ad3b3dbc", - "metadata": {}, - "outputs": [], - "source": [ - "alice_record = await router.open_dht_record(key=dht_key, writer=None)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "12fdf804", - "metadata": {}, - "outputs": [], - "source": [ - "alice_record_value = await router.get_dht_value(\n", - " key=dht_key, subkey=0, force_refresh=True\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e4e44fc7", - "metadata": {}, - "outputs": [], - "source": [ - "alice_record_value.data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c4db5bf7", - "metadata": {}, - "outputs": [], - "source": [ - "# Import the private route sent by Alice:\n", - "prr_alice = await conn.import_remote_private_route(alice_record_value.data)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e249e970-30f1-4121-98e0-e1df0db37e4b", - "metadata": {}, - "outputs": [], - "source": [ - "prr_alice" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6a268e77-84d0-43ad-a9b9-21582992ea64", - "metadata": {}, - "outputs": [], - "source": [ - "message = b\"Hello Alice\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e76b3680-7e84-4e7f-8c8e-450a6de6786d", - "metadata": {}, - "outputs": [], - "source": [ - "message_send = await router.app_message(prr_alice, message)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "62bd12ec-1aef-4459-90a9-a4402a8b3a68", - "metadata": {}, - "outputs": [], - "source": [ - "message_send" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "13b87c97-0c10-4112-82c7-6b283a31cc28", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks/Testing/Veilid/Large-Message-Testing.ipynb b/notebooks/Testing/Veilid/Large-Message-Testing.ipynb deleted file mode 100644 index 46d1980a5c4..00000000000 --- a/notebooks/Testing/Veilid/Large-Message-Testing.ipynb +++ /dev/null @@ -1,397 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Instructions\n", - "\n", - "1. Follow these instructions from `packages/grid/veilid/development.md` to build veilid docker containers:\n", - " ```bash\n", - " cd packages/grid/veilid && docker build -f veilid.dockerfile -t veilid:0.1 .\n", - " ```\n", - "2. From within the `packages/grid/veilid` directory run the receiver docker container on port 4000:\n", - " ```bash\n", - " docker run -it -e DEV_MODE=True -p 4000:4000 -v $(pwd)/server:/app/server veilid:0.1\n", - " ```\n", - "3. On a separate terminal tab/window, cd into `packages/grid/veilid` directory again and run the sender docker container on port 4001:\n", - " ```bash\n", - " docker run -it -e DEV_MODE=True -p 4001:4000 -v $(pwd)/server:/app/server veilid:0.1\n", - " ```\n", - "4. Follow and run the below cells to test out sending large messages through Veilid. You may also use the **`Run All`** notebook function once the above two docker containers are up and running." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 1. Set up imports" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# stdlib\n", - "import json\n", - "import logging\n", - "from pprint import pprint\n", - "import random\n", - "import time\n", - "\n", - "# third party\n", - "import requests\n", - "\n", - "logging.basicConfig(level=logging.INFO, format=\"%(message)s\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2. Set up receiver" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "RECEIVER_HOST = \"localhost\"\n", - "RECEIVER_PORT = 4000\n", - "RECEIVER_BASE_ADDRESS = f\"http://{RECEIVER_HOST}:{RECEIVER_PORT}\"\n", - "\n", - "requests.post(f\"{RECEIVER_BASE_ADDRESS}/generate_vld_key\")\n", - "res = requests.get(f\"{RECEIVER_BASE_ADDRESS}/retrieve_vld_key\")\n", - "receiver_vld_key = res.json()[\"message\"]\n", - "logging.info(f\"{'=' * 30}\\n{receiver_vld_key}\\n{'=' * 30}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 3. Set up sender" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "SENDER_HOST = \"localhost\"\n", - "SENDER_PORT = 4001\n", - "SENDER_BASE_ADDRESS = f\"http://{SENDER_HOST}:{SENDER_PORT}\"\n", - "\n", - "requests.post(f\"{SENDER_BASE_ADDRESS}/generate_vld_key\")\n", - "res = requests.get(f\"{SENDER_BASE_ADDRESS}/retrieve_vld_key\")\n", - "sender_vld_key = res.json()[\"message\"]\n", - "logging.info(f\"{'=' * 30}\\n{sender_vld_key}\\n{'=' * 30}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 4. Declare utility functions" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def send_test_request(request_size_bytes, response_size_bytes):\n", - " \"\"\"\n", - " Send a test request of the specified size and receive a response of.\n", - "\n", - " Args:\n", - " request_size_bytes (int): Size of the request body in bytes.\n", - " response_size_bytes (int): Expected size of the response body in bytes.\n", - "\n", - " Returns:\n", - " tuple: A tuple containing the total transfer size, total time taken and success status.\n", - " \"\"\"\n", - " message = build_vld_message(request_size_bytes, response_size_bytes)\n", - " json_data = {\n", - " \"vld_key\": receiver_vld_key,\n", - " \"message\": message,\n", - " }\n", - "\n", - " logging.info(f\"Sending message of size {len(message) // 1024} KB...\")\n", - "\n", - " start = time.time()\n", - " app_call = requests.post(f\"{SENDER_BASE_ADDRESS}/app_call\", json=json_data)\n", - " end = time.time()\n", - "\n", - " response = app_call.content\n", - " response_len = len(response)\n", - " response = response.decode()\n", - " response_pretty = (\n", - " response if len(response) <= 100 else f\"{response[:50]}...{response[-50:]}\"\n", - " )\n", - "\n", - " total_xfer = request_size_bytes + response_size_bytes\n", - " total_time = round(end - start, 2)\n", - "\n", - " success = \"received_request_body_length\" in response\n", - " logging.info(f\"[{total_time}s] Response({response_len} B): {response_pretty}\")\n", - " return total_xfer, total_time, success\n", - "\n", - "\n", - "def build_vld_message(request_size_bytes, response_size_bytes):\n", - " \"\"\"\n", - " Build a message of length `request_size_bytes`. Padded with random characters.\n", - "\n", - " Args:\n", - " request_size_bytes (int): Size of the request body in bytes.\n", - " response_size_bytes (int): Expected size of the response body in bytes.\n", - "\n", - " Returns:\n", - " dict: The constructed request body.\n", - " \"\"\"\n", - " endpoint = f\"{RECEIVER_BASE_ADDRESS}/test_veilid_streamer\"\n", - " message = {\n", - " \"method\": \"POST\",\n", - " \"url\": endpoint,\n", - " \"json\": {\n", - " \"expected_response_length\": response_size_bytes,\n", - " \"random_padding\": \"\",\n", - " },\n", - " }\n", - " padding_length = request_size_bytes - len(json.dumps(message))\n", - " random_padding = generate_random_alphabets(padding_length)\n", - " message[\"json\"][\"random_padding\"] = random_padding\n", - " return json.dumps(message)\n", - "\n", - "\n", - "def generate_random_alphabets(length):\n", - " return \"\".join([random.choice(\"abcdefghijklmnopqrstuvwxyz\") for _ in range(length)])\n", - "\n", - "\n", - "def bytes_to_human_readable(size_in_bytes):\n", - " if size_in_bytes >= (2**20):\n", - " size_in_mb = size_in_bytes / (2**20)\n", - " return f\"{size_in_mb:.2f} MB\"\n", - " else:\n", - " size_in_kb = size_in_bytes / (2**10)\n", - " return f\"{size_in_kb:.2f} KB\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 5. Run manual tests" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "MIN_MESSAGE_SIZE = 1024\n", - "MAX_CHUNK_SIZE = 32744 # minus 24 bytes for single chunk header\n", - "\n", - "\n", - "def get_random_single_chunk_size():\n", - " return random.randint(MIN_MESSAGE_SIZE, MAX_CHUNK_SIZE)\n", - "\n", - "\n", - "def get_random_multi_chunk_size():\n", - " return random.randint(2 * MAX_CHUNK_SIZE, 3 * MAX_CHUNK_SIZE)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def test_for_single_chunk_request_and_single_chunk_response():\n", - " request_size = get_random_single_chunk_size()\n", - " response_size = get_random_single_chunk_size()\n", - " total_xfer, total_time, success = send_test_request(request_size, response_size)\n", - " result = \"Success\" if success else \"Failure\"\n", - " logging.info(\n", - " f\"[{request_size} B ⇅ {response_size} B] \"\n", - " f\"Transferred {bytes_to_human_readable(total_xfer)} \"\n", - " f\"in {total_time}s; \"\n", - " f\"Result: {result}\"\n", - " )\n", - "\n", - "\n", - "test_for_single_chunk_request_and_single_chunk_response()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def test_for_multi_chunk_request_and_single_chunk_response():\n", - " request_size = get_random_multi_chunk_size()\n", - " response_size = get_random_single_chunk_size()\n", - " total_xfer, total_time, success = send_test_request(request_size, response_size)\n", - " result = \"Success\" if success else \"Failure\"\n", - " logging.info(\n", - " f\"[{request_size} B ⇅ {response_size} B] \"\n", - " f\"Transferred {bytes_to_human_readable(total_xfer)} \"\n", - " f\"in {total_time}s; \"\n", - " f\"Result: {result}\"\n", - " )\n", - "\n", - "\n", - "test_for_multi_chunk_request_and_single_chunk_response()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def test_for_single_chunk_request_and_multi_chunk_response():\n", - " request_size = get_random_single_chunk_size()\n", - " response_size = get_random_multi_chunk_size()\n", - " total_xfer, total_time, success = send_test_request(request_size, response_size)\n", - " result = \"Success\" if success else \"Failure\"\n", - " logging.info(\n", - " f\"[{request_size} B ⇅ {response_size} B] \"\n", - " f\"Transferred {bytes_to_human_readable(total_xfer)} \"\n", - " f\"in {total_time}s; \"\n", - " f\"Result: {result}\"\n", - " )\n", - "\n", - "\n", - "test_for_single_chunk_request_and_multi_chunk_response()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def test_for_multi_chunk_request_and_multi_chunk_response():\n", - " request_size = get_random_multi_chunk_size()\n", - " response_size = get_random_multi_chunk_size()\n", - " total_xfer, total_time, success = send_test_request(request_size, response_size)\n", - " result = \"Success\" if success else \"Failure\"\n", - " logging.info(\n", - " f\"[{request_size} B ⇅ {response_size} B] \"\n", - " f\"Transferred {bytes_to_human_readable(total_xfer)} \"\n", - " f\"in {total_time}s; \"\n", - " f\"Result: {result}\"\n", - " )\n", - "\n", - "\n", - "test_for_multi_chunk_request_and_multi_chunk_response()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 6. Run benchmarks on requests-responses of sizes from 1 KB to 512 MB" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "benchmarks = {}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Baseline tests (Tests with single chunk messages i.e. 1 KB to 32 KB)\n", - "for powers_of_two in range(0, 6): # Test from 1 KB to 32 KB\n", - " message_size = 2**powers_of_two * 1024\n", - " total_xfer, total_time, success = send_test_request(message_size, message_size)\n", - " if success:\n", - " benchmarks[bytes_to_human_readable(total_xfer)] = total_time\n", - "pprint(benchmarks, sort_dicts=False)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Tests with smaller messages\n", - "for powers_of_two in range(6, 13): # Test from 64 KB to 4 MB\n", - " message_size = 2**powers_of_two * 1024\n", - " total_xfer, total_time, success = send_test_request(message_size, message_size)\n", - " if success:\n", - " benchmarks[bytes_to_human_readable(total_xfer)] = total_time\n", - "pprint(benchmarks, sort_dicts=False)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Tests with larger messages\n", - "for powers_of_two in range(13, 16): # Test from 8 MB to 32 MB\n", - " message_size = 2**powers_of_two * 1024\n", - " total_xfer, total_time, success = send_test_request(message_size, message_size)\n", - " if success:\n", - " benchmarks[bytes_to_human_readable(total_xfer)] = total_time\n", - "pprint(benchmarks, sort_dicts=False)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Tests with super large messages\n", - "for powers_of_two in range(16, 19): # Test from 64 MB to 256 MB\n", - " message_size = 2**powers_of_two * 1024\n", - " total_xfer, total_time, success = send_test_request(message_size, message_size)\n", - " if success:\n", - " benchmarks[bytes_to_human_readable(total_xfer)] = total_time\n", - "pprint(benchmarks, sort_dicts=False)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "PySyft", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.7" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/notebooks/Testing/Veilid/Veilid Route-Connection-Testing.ipynb b/notebooks/Testing/Veilid/Veilid Route-Connection-Testing.ipynb deleted file mode 100644 index bd2ea78c1c6..00000000000 --- a/notebooks/Testing/Veilid/Veilid Route-Connection-Testing.ipynb +++ /dev/null @@ -1,116 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "a8d2d5a4-5512-4a24-aafd-7133d64c22fc", - "metadata": {}, - "outputs": [], - "source": [ - "# syft absolute" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5a5a1b05-336d-4523-ae85-4022783acf85", - "metadata": {}, - "outputs": [], - "source": [ - "# syft absolute\n", - "from syft.client.client import VeilidConnection" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "963f96e5-8d62-44b2-a975-faa23624bbd4", - "metadata": {}, - "outputs": [], - "source": [ - "veilid_conn = VeilidConnection(dht_key=\"test\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f2d6083b-527f-46be-a582-15f4404950b5", - "metadata": {}, - "outputs": [], - "source": [ - "# syft absolute\n", - "from syft.service.network.routes import connection_to_route\n", - "from syft.service.network.routes import route_to_connection" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9e8e508f-c527-43f4-98d1-7e7c6ef0dfb3", - "metadata": {}, - "outputs": [], - "source": [ - "veilid_route = connection_to_route(veilid_conn)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7aba2e02-46c7-46a2-ab11-9253e05fd2fe", - "metadata": {}, - "outputs": [], - "source": [ - "veilid_route.dht_key" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0d50eec2-a7ed-49f6-b90c-082cd8c40e0a", - "metadata": {}, - "outputs": [], - "source": [ - "re_veilid_conn = route_to_connection(veilid_route)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ea5d2d73-1cbc-496a-a6b6-4136e9423394", - "metadata": {}, - "outputs": [], - "source": [ - "re_veilid_conn" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a535caf0-d1e6-40b9-842b-066ce2b6b897", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks/Testing/Veilid/Veilid-Connection-Test.ipynb b/notebooks/Testing/Veilid/Veilid-Connection-Test.ipynb deleted file mode 100644 index c38143c7c35..00000000000 --- a/notebooks/Testing/Veilid/Veilid-Connection-Test.ipynb +++ /dev/null @@ -1,554 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "df3d4dbb-e179-4995-9507-1f82cb417fc5", - "metadata": {}, - "outputs": [], - "source": [ - "# syft absolute\n", - "import syft as sy\n", - "from syft.client.client import connect" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cc7f02fb-b4f8-4615-a39f-dca2752b58b2", - "metadata": {}, - "outputs": [], - "source": [ - "domain_client = sy.login(email=\"info@openmined.org\", password=\"changethis\", port=8080)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4d9ce704-36e6-455b-a633-fe943848420c", - "metadata": {}, - "outputs": [], - "source": [ - "domain_client.api.services.veilid.generate_dht_key()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ef053ef6-e31a-4634-8d5e-2e8ff2e002de", - "metadata": {}, - "outputs": [], - "source": [ - "domain_client.api.services.veilid.retrieve_dht_key()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "670f2e09-3409-4545-be3a-17e1b2a97cd2", - "metadata": {}, - "outputs": [], - "source": [ - "domain_client = sy.login_as_guest(\n", - " dht_key=\"VLD0:OBeFkuuQz6LIofeIIzC5Y-zwR96NoKqbojqGCcNKu8c\",\n", - " vld_forward_proxy=\"http://localhost:4000\",\n", - " vld_reverse_proxy=\"http://proxy\",\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "63a9a5f3-a004-4523-bf70-e3ebee06408e", - "metadata": {}, - "outputs": [], - "source": [ - "domain_client.api" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b2195bbd-5ef1-4a53-8886-1b2ea6854bc3", - "metadata": {}, - "outputs": [], - "source": [ - "connect_client = connect(\n", - " dht_key=\"VLD0:OBeFkuuQz6LIofeIIzC5Y-zwR96NoKqbojqGCcNKu8c\",\n", - " vld_forward_proxy=\"http://localhost:4000\",\n", - " vld_reverse_proxy=\"http://proxy\",\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "71261091-1cfc-428f-9087-7f24395a2750", - "metadata": {}, - "outputs": [], - "source": [ - "domain_client = sy.login(\n", - " dht_key=\"VLD0:OBeFkuuQz6LIofeIIzC5Y-zwR96NoKqbojqGCcNKu8c\",\n", - " vld_forward_proxy=\"http://localhost:4000\",\n", - " vld_reverse_proxy=\"http://proxy\",\n", - " email=\"info@openmined.org\",\n", - " password=\"changethis\",\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ec181b37-71cc-411b-8b6c-0f149e45c79c", - "metadata": {}, - "outputs": [], - "source": [ - "domain_client.api" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "293b55c9-9f9b-4702-b74f-6dfe9b5eee8d", - "metadata": {}, - "outputs": [], - "source": [ - "# syft absolute\n", - "import syft as sy" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2f469470-6280-466f-85e3-ed655484178e", - "metadata": {}, - "outputs": [], - "source": [ - "domain_client = sy.login_as_guest(port=8080)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "141a0871-d322-4508-b0b1-68ad1654dcda", - "metadata": {}, - "outputs": [], - "source": [ - "res = sy.serialize(domain_client.api, to_bytes=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "61192beb-a4f7-495f-adf5-f2294ec5a199", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "af269af3-f55b-4f3d-8cc1-cbe8ee10d327", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "95facdab-92ab-42cf-b976-a9b646ae2901", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8de7d433-c26b-43e9-9a45-d960cfb18645", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7c7a97e3-9585-485f-ad41-2982bf935564", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "eb073a52-1c7a-4c02-bce3-0782c6f89064", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "98f58488-e927-4e44-a885-04740f8c8b31", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5adb6185-9f49-444c-ae26-702e17bcfabf", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6ed88528-1e23-4585-89ca-0e3cfa098d37", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "985f6211-efa8-4850-b2fa-280b064032ff", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3f3abeb1-228c-45ff-acc9-fbc2314c6e31", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "83aee788-4a14-4e41-b924-53dcbebe8e14", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0d2d9fa5-9098-4d79-a35e-2da46f615ef7", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "id": "18613355-f3bd-45c3-8ac3-97165dd6e28d", - "metadata": {}, - "source": [ - "## Debugging" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f2d4a8ea-f9e5-4411-bf68-0d4ed25f3fa6", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "77f7d4b4-7ea2-4a61-8a67-a2dacbfd054f", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9bf0aa58-b6a1-463a-8d14-76f74dcc6d7c", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "raw", - "id": "1142383d-82df-49f5-ad5f-ede5fde39b20", - "metadata": {}, - "source": [ - "import lzma" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c8026971-b496-4a24-b84f-b57d898f15d9", - "metadata": {}, - "outputs": [], - "source": [ - "# stdlib\n", - "import lzma" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "69542e59-2ba3-4721-8c39-192258180114", - "metadata": {}, - "outputs": [], - "source": [ - "len(res)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "61f3fb0e-50e1-4cca-94cf-490e5bde974b", - "metadata": {}, - "outputs": [], - "source": [ - "comp = lzma.compress(res)\n", - "print(len(comp))\n", - "decom = lzma.decompress(comp)\n", - "print(len(decom))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ee53df6e-e979-4011-8fe7-24141f7df001", - "metadata": {}, - "outputs": [], - "source": [ - "# third party\n", - "from pympler import asizeof" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8db1d8a9-ee4b-4efa-a69b-1d735ceaf129", - "metadata": {}, - "outputs": [], - "source": [ - "asizeof.asizeof(domain_client.api)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f3bdfb82-687e-49a7-a268-2bb0e74364cc", - "metadata": {}, - "outputs": [], - "source": [ - "# stdlib\n", - "import sys\n", - "\n", - "# third party\n", - "from pympler import asizeof" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "41944d4d-7613-461e-a6e7-905514bb08da", - "metadata": {}, - "outputs": [], - "source": [ - "for attr_name, attr_value in domain_client.api.__dict__.items():\n", - " if attr_name != \"refresh_api_callback\":\n", - " res = sy.serialize(attr_value, to_bytes=True)\n", - " immediate_size = sys.getsizeof(res)\n", - " total_size = asizeof.asizeof(res)\n", - " print(\n", - " f\"{attr_name}: immediate size = {immediate_size} bytes, total size = {total_size} bytes\"\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a1d4ad18-7fb0-4ec7-966d-cf86a6b280f1", - "metadata": {}, - "outputs": [], - "source": [ - "count = 0\n", - "for i in domain_client.api.lib_endpoints.values():\n", - " count += 1\n", - " print(count, \" \", i.module_path)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "59ad85c9-6acb-4fbd-b9e7-25a0e34d8f6c", - "metadata": {}, - "outputs": [], - "source": [ - "len(res)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8ee3d56b-298e-4706-9e93-055960f41654", - "metadata": {}, - "outputs": [], - "source": [ - "# stdlib\n", - "import zlib" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d4509185-ba56-42d4-aaf3-84341cdeaa52", - "metadata": {}, - "outputs": [], - "source": [ - "%%time\n", - "c = zlib.compress(res)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b9b7539e-06ce-4a92-bf8e-6a65331f3ee1", - "metadata": {}, - "outputs": [], - "source": [ - "len(c)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1eb8fc1d-1d8a-4301-bd36-618393e6ff8a", - "metadata": {}, - "outputs": [], - "source": [ - "# stdlib\n", - "import lzma" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d8b9cabe-382d-4085-861d-ca55d99a938e", - "metadata": {}, - "outputs": [], - "source": [ - "%%time\n", - "lc = lzma.compress(res)\n", - "print(len(lc))\n", - "ld = lzma.decompress(lc)\n", - "print(len(ld))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4a8462ce-6de8-472b-8685-72665f36f940", - "metadata": {}, - "outputs": [], - "source": [ - "# stdlib\n", - "import gzip" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9f357c7d-059d-46b5-bf03-c8acb5a3e7df", - "metadata": {}, - "outputs": [], - "source": [ - "%%time\n", - "c2 = gzip.compress(res)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9b4647a5-ec95-4f22-9ac2-104f30600cf5", - "metadata": {}, - "outputs": [], - "source": [ - "len(sy.serialize(domain_client.api.endpoints, to_bytes=True))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7d7b89fe-b270-40c0-bc18-066f9be62569", - "metadata": {}, - "outputs": [], - "source": [ - "# res = veilid_conn.get_node_metadata(credentials=None)\n", - "res = b\"\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4f6a98cf-ad5b-4ad0-87c7-b8cdc7d0678d", - "metadata": {}, - "outputs": [], - "source": [ - "res" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3ab82cd6-c080-46dd-b15d-da0c904e967e", - "metadata": {}, - "outputs": [], - "source": [ - "# stdlib\n", - "import json" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cba2d15b-826d-4f6b-82d1-bb70ba0e439d", - "metadata": {}, - "outputs": [], - "source": [ - "type(json.loads(res))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cbfda25f-5b2e-4c55-a906-1ca78497623f", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks/Testing/Veilid/Veilid-Gateway-Testing.ipynb b/notebooks/Testing/Veilid/Veilid-Gateway-Testing.ipynb deleted file mode 100644 index 0e3754724cd..00000000000 --- a/notebooks/Testing/Veilid/Veilid-Gateway-Testing.ipynb +++ /dev/null @@ -1,226 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "a300f01b-8357-43ca-9c64-c489839603e8", - "metadata": {}, - "outputs": [], - "source": [ - "# syft absolute\n", - "import syft as sy" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9aeed160-94d3-49c1-98c5-7795c6df7280", - "metadata": {}, - "outputs": [], - "source": [ - "domain_client = sy.login(email=\"info@openmined.org\", password=\"changethis\", port=9082)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e7a79ee9-68bf-4a93-935e-32f42e332f97", - "metadata": {}, - "outputs": [], - "source": [ - "gateway_client = sy.login(email=\"info@openmined.org\", password=\"changethis\", port=9081)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b2d66293-b573-4cdf-8721-9d91a620dd9d", - "metadata": {}, - "outputs": [], - "source": [ - "domain_client.api.services.veilid.generate_vld_key()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e3b10d70-1c30-42e2-98bd-86af6a228455", - "metadata": {}, - "outputs": [], - "source": [ - "gateway_client.api.services.veilid.generate_vld_key()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7500007e-e5f6-4c4a-bbc3-46f2357d2433", - "metadata": {}, - "outputs": [], - "source": [ - "domain_route = domain_client.api.services.veilid.get_veilid_route()\n", - "gateway_route = gateway_client.api.services.veilid.get_veilid_route()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "82bee827-ea59-4255-9c32-5b9e10e5676f", - "metadata": {}, - "outputs": [], - "source": [ - "gateway_route.vld_key" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "960f6b4c-3073-45ec-93cf-54c384262d0b", - "metadata": {}, - "outputs": [], - "source": [ - "domain_route.vld_key" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e3e916e7-2897-4d63-b8b8-a913a2baed8a", - "metadata": {}, - "outputs": [], - "source": [ - "domain_client.connect_to_gateway(gateway_client, protocol=\"veilid\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ee4b39c1-01d5-4cae-9115-a0d83667c31a", - "metadata": {}, - "outputs": [], - "source": [ - "domain_client.peers[0].node_routes[0].vld_key" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6c56a7d4-88dc-43e0-b092-4c443734e3c3", - "metadata": {}, - "outputs": [], - "source": [ - "gateway_client.api.services.network.get_all_peers()[0].node_routes[0].vld_key" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8febe455-4b82-478f-85b5-d1e2e104fb1a", - "metadata": {}, - "outputs": [], - "source": [ - "gateway_client.peers" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9f6871cb-37bf-4570-94cd-b993906c11f8", - "metadata": {}, - "outputs": [], - "source": [ - "domain_peer = gateway_client.api.services.network.get_all_peers()[0]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "91c303f5-36af-4a65-a81a-7cb24f5c3494", - "metadata": {}, - "outputs": [], - "source": [ - "connection = gateway_client.connection.with_proxy(domain_peer.id)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5b744210-dddb-4a20-a32e-146b0a92678c", - "metadata": {}, - "outputs": [], - "source": [ - "# syft absolute\n", - "from syft.node.credentials import SyftSigningKey" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "66b4f4c5-780d-4259-8360-2692ade1358f", - "metadata": {}, - "outputs": [], - "source": [ - "metadata = connection.get_node_metadata(credentials=SyftSigningKey.generate())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "508e9374-37ca-412b-af34-631994f80ff7", - "metadata": {}, - "outputs": [], - "source": [ - "proxy_client = gateway_client.domains[0]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2081964a-12da-428d-b543-7ba1a4c82600", - "metadata": {}, - "outputs": [], - "source": [ - "admin_client = proxy_client.login(email=\"info@openmined.org\", password=\"changethis\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "68a6e4bb-d6f6-4173-a8bb-dc70ea52c0b5", - "metadata": {}, - "outputs": [], - "source": [ - "admin_client" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "760f17f0-b44c-4e71-ae93-ba9f4c291fd9", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks/Testing/Veilid/utils.py b/notebooks/Testing/Veilid/utils.py deleted file mode 100644 index 814ff62758a..00000000000 --- a/notebooks/Testing/Veilid/utils.py +++ /dev/null @@ -1,12 +0,0 @@ -# third party -import veilid - - -def get_typed_key(key: str) -> veilid.types.TypedKey: - return veilid.types.TypedKey.from_value( - kind=veilid.CryptoKind.CRYPTO_KIND_VLD0, value=key - ) - - -# state = await conn.get_state() -# state.config.config diff --git a/packages/grid/default.env b/packages/grid/default.env index 0be556eb714..94463eacd9b 100644 --- a/packages/grid/default.env +++ b/packages/grid/default.env @@ -108,6 +108,3 @@ USE_BLOB_STORAGE=False # Registation ENABLE_SIGNUP=False - -# Veilid -DOCKER_IMAGE_VEILID=openmined/grid-veilid diff --git a/packages/grid/veilid/.dockerignore b/packages/grid/veilid/.dockerignore deleted file mode 100644 index cc8cc888f79..00000000000 --- a/packages/grid/veilid/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -veilid \ No newline at end of file diff --git a/packages/grid/veilid/development.md b/packages/grid/veilid/development.md deleted file mode 100644 index f6baa053ac7..00000000000 --- a/packages/grid/veilid/development.md +++ /dev/null @@ -1,67 +0,0 @@ -## Veilid - Development Instructions - -### 1. Building Veilid Container - -```sh -cd packages/grid/veilid && docker build -f veilid.dockerfile -t veilid:0.1 . -``` - -### Running veilid Container - -#### 1. Development Mode - -```sh -cd packages/grid/veilid && \ -docker run --rm -e DEV_MODE=True -p 4000:4000 -p 5959:5959 -p 5959:5959/udp -v $(pwd)/server:/app/server veilid:0.1 -``` - -##### 2. Additional Flags for Development - -``` -a. VEILID_FLAGS="--debug" (For Veilid Debug logs) -b. APP_LOG_LEVEL="debug" (For changing logging method inside the application could be info, debug, warning, critical) -c. UVICORN_LOG_LEVEL="debug" (For setting logging method for uvicorn) -``` - -#### 3. Production Mode - -```sh -cd packages/grid/veilid && \ -docker run --rm -p 4000:4000 -p 5959:5959 -p 5959:5959/udp veilid:0.1 -``` - -### Kubernetes Development - -#### 1. Gateway Node - -##### Creation - -```sh -bash -c '\ - export CLUSTER_NAME=testgateway1 CLUSTER_HTTP_PORT=9081 DEVSPACE_PROFILE=gateway && \ - tox -e dev.k8s.start && \ - tox -e dev.k8s.hotreload' -``` - -##### Deletion - -```sh -bash -c "CLUSTER_NAME=testgateway1 tox -e dev.k8s.destroy || true" -``` - -#### 2. Domain Node - -##### Creation - -```sh -bash -c '\ - export CLUSTER_NAME=testdomain1 CLUSTER_HTTP_PORT=9082 && \ - tox -e dev.k8s.start && \ - tox -e dev.k8s.hotreload' -``` - -##### Deletion - -```sh -bash -c "CLUSTER_NAME=testdomain1 tox -e dev.k8s.destroy || true" -``` diff --git a/packages/grid/veilid/requirements.txt b/packages/grid/veilid/requirements.txt deleted file mode 100644 index 6517014dc1c..00000000000 --- a/packages/grid/veilid/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -fastapi==0.109.1 -httpx==0.27.0 -loguru==0.7.2 -uvicorn[standard]==0.24.0.post1 diff --git a/packages/grid/veilid/server/constants.py b/packages/grid/veilid/server/constants.py deleted file mode 100644 index 0a4efdad8cc..00000000000 --- a/packages/grid/veilid/server/constants.py +++ /dev/null @@ -1,14 +0,0 @@ -HOST = "localhost" -PORT = 5959 -# name of the Table Database -TABLE_DB_KEY = "syft-table-db" -# name of the DHT Key in the table Database -DHT_KEY = "syft-dht-key" -# name of the DHT Key Credentials in the table Database -# Credentials refer to the Public and Private Key created for the DHT Key -DHT_KEY_CREDS = "syft-dht-key-creds" - -USE_DIRECT_CONNECTION = True -MAX_SINGLE_VEILID_MESSAGE_SIZE = 32768 # 32KB -MAX_STREAMER_CONCURRENCY = 200 -TIMEOUT = 10 # in seconds diff --git a/packages/grid/veilid/server/main.py b/packages/grid/veilid/server/main.py deleted file mode 100644 index 6df7db827a0..00000000000 --- a/packages/grid/veilid/server/main.py +++ /dev/null @@ -1,161 +0,0 @@ -# stdlib -import json -import os -import sys -from typing import Annotated - -# third party -from fastapi import Body -from fastapi import FastAPI -from fastapi import HTTPException -from fastapi import Request -from fastapi import Response -from loguru import logger - -# relative -from .models import ResponseModel -from .models import TestVeilidStreamerRequest -from .models import TestVeilidStreamerResponse -from .utils import generate_random_alphabets -from .veilid_connection_singleton import VeilidConnectionSingleton -from .veilid_core import app_call -from .veilid_core import app_message -from .veilid_core import generate_vld_key -from .veilid_core import healthcheck -from .veilid_core import ping -from .veilid_core import retrieve_vld_key - -# Logging Configuration -log_level = os.getenv("APP_LOG_LEVEL", "INFO").upper() -logger.remove() -logger.add(sys.stderr, colorize=True, level=log_level) - -app = FastAPI(title="Veilid") -veilid_conn = VeilidConnectionSingleton() - - -@app.get("/", response_model=ResponseModel) -async def read_root() -> ResponseModel: - return ResponseModel(message="Veilid has started") - - -@app.get("/healthcheck", response_model=ResponseModel) -async def healthcheck_endpoint() -> ResponseModel: - res = await healthcheck() - if res: - return ResponseModel(message="OK") - else: - return ResponseModel(message="FAIL") - - -@app.post("/generate_vld_key", response_model=ResponseModel) -async def generate_vld_key_endpoint() -> ResponseModel: - try: - res = await generate_vld_key() - return ResponseModel(message=res) - except Exception as e: - raise HTTPException(status_code=500, detail=f"Failed to generate VLD key: {e}") - - -@app.get("/retrieve_vld_key", response_model=ResponseModel) -async def retrieve_vld_key_endpoint() -> ResponseModel: - try: - res = await retrieve_vld_key() - return ResponseModel(message=res) - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -@app.post("/ping/{vld_key}", response_model=ResponseModel) -async def ping_endpoint(request: Request, vld_key: str) -> ResponseModel: - try: - logger.info(f"Received ping request:{vld_key}") - res = await ping(vld_key) - return ResponseModel(message=res) - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -@app.post("/app_message", response_model=ResponseModel) -async def app_message_endpoint( - request: Request, vld_key: Annotated[str, Body()], message: Annotated[bytes, Body()] -) -> ResponseModel: - try: - logger.info("Received app_message request") - res = await app_message(vld_key=vld_key, message=message) - return ResponseModel(message=res) - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -@app.post("/app_call") -async def app_call_endpoint( - request: Request, vld_key: Annotated[str, Body()], message: Annotated[bytes, Body()] -) -> Response: - try: - res = await app_call(vld_key=vld_key, message=message) - return Response(res, media_type="application/octet-stream") - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - -@app.api_route("/proxy", methods=["GET", "POST", "PUT"]) -async def proxy(request: Request) -> Response: - logger.info("Proxying request") - - request_data = await request.json() - logger.info(f"Request URL: {request_data}") - - vld_key = request_data.get("vld_key") - request_data.pop("vld_key") - message = json.dumps(request_data).encode() - - res = await app_call(vld_key=vld_key, message=message) - - return Response(res, media_type="application/octet-stream") - - -@app.on_event("startup") -async def startup_event() -> None: - try: - await veilid_conn.initialize_connection() - except Exception as e: - logger.exception(f"Failed to connect to Veilid: {e}") - raise e - - -@app.on_event("shutdown") -async def shutdown_event() -> None: - await veilid_conn.release_connection() - - -@app.post("/test_veilid_streamer") -async def test_veilid_streamer( - request_data: TestVeilidStreamerRequest, -) -> TestVeilidStreamerResponse: - """Test endpoint for notebooks/Testing/Veilid/Large-Message-Testing.ipynb. - - This endpoint is used to test the Veilid streamer by receiving a request body of any - arbitrary size and sending back a response of a size specified in the request body. - The length of the response body is determined by the `expected_response_length` field - in the request body. After adding the necessary fields, both the request and response - bodies are padded with random alphabets to reach the expected length using a - `random_padding` field. - """ - expected_response_length = request_data.expected_response_length - if expected_response_length <= 0: - raise HTTPException(status_code=400, detail="Length must be greater than zero") - - try: - request_body_length = len(request_data.json()) - response = TestVeilidStreamerResponse( - received_request_body_length=request_body_length, - random_padding="", - ) - response_length_so_far = len(response.json()) - padding_length = expected_response_length - response_length_so_far - random_message = generate_random_alphabets(padding_length) - response.random_padding = random_message - return response - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) diff --git a/packages/grid/veilid/server/models.py b/packages/grid/veilid/server/models.py deleted file mode 100644 index 941a8b75c59..00000000000 --- a/packages/grid/veilid/server/models.py +++ /dev/null @@ -1,16 +0,0 @@ -# third party -from pydantic import BaseModel - - -class ResponseModel(BaseModel): - message: str - - -class TestVeilidStreamerRequest(BaseModel): - expected_response_length: int - random_padding: str - - -class TestVeilidStreamerResponse(BaseModel): - received_request_body_length: int - random_padding: str diff --git a/packages/grid/veilid/server/utils.py b/packages/grid/veilid/server/utils.py deleted file mode 100644 index e3779ea3f7b..00000000000 --- a/packages/grid/veilid/server/utils.py +++ /dev/null @@ -1,58 +0,0 @@ -# stdlib -import asyncio -from collections.abc import Callable -from enum import ReprEnum -from functools import wraps -import random -from typing import Any - - -def retry( - exceptions: tuple[type[BaseException], ...] | type[BaseException], - tries: int = 3, - delay: int = 1, - backoff: int = 2, -) -> Callable: - """Retry calling the decorated function using exponential backoff. - - Args: - exceptions (Tuple or Exception): The exception(s) to catch. Can be a tuple of exceptions or a single exception. - tries (int): The maximum number of times to try the function (default: 3). - delay (int): The initial delay between retries in seconds (default: 1). - backoff (int): The exponential backoff factor (default: 2). - - Returns: - The result of the decorated function. - """ - - def decorator(func: Callable) -> Callable: - @wraps(func) - async def wrapper(*args: Any, **kwargs: Any) -> Any: - current_delay: int = delay - for _ in range(tries): - try: - return await func(*args, **kwargs) - except exceptions as e: - print( - f"Caught exception: {e}. Retrying in {current_delay} seconds..." - ) - await asyncio.sleep(current_delay) - current_delay *= backoff - # Retry one last time before raising the exception - return await func(*args, **kwargs) - - return wrapper - - return decorator - - -def generate_random_alphabets(length: int) -> str: - return "".join([random.choice("abcdefghijklmnopqrstuvwxyz") for _ in range(length)]) - - -class BytesEnum(bytes, ReprEnum): - """ - Enum where members are also (and must be) bytes - """ - - pass diff --git a/packages/grid/veilid/server/veilid_callback.py b/packages/grid/veilid/server/veilid_callback.py deleted file mode 100644 index 9d1b8fed7eb..00000000000 --- a/packages/grid/veilid/server/veilid_callback.py +++ /dev/null @@ -1,66 +0,0 @@ -# stdlib -import base64 -import json - -# third party -import httpx -from loguru import logger -import veilid -from veilid import VeilidUpdate - -# relative -from .veilid_connection import get_routing_context -from .veilid_connection import get_veilid_conn -from .veilid_streamer import VeilidStreamer - - -async def handle_app_message(update: VeilidUpdate) -> None: - logger.info(f"Received App Message: {update.detail.message}") - - -async def handle_app_call(message: bytes) -> bytes: - logger.info(f"Received App Call of {len(message)} bytes.") - message_dict: dict = json.loads(message) - - async with httpx.AsyncClient() as client: - data = message_dict.get("data", None) - # TODO: can we optimize this? - # We encode the data to base64, as while sending - # json expects valid utf-8 strings - if data: - message_dict["data"] = base64.b64decode(data) - response = await client.request( - method=message_dict.get("method"), - url=message_dict.get("url"), - data=message_dict.get("data", None), - params=message_dict.get("params", None), - json=message_dict.get("json", None), - ) - - # TODO: Currently in `dev` branch, compression is handled by the veilid internals, - # but we are decompressing it on the client side. Should both the compression and - # decompression be done either on the client side (for more client control) or by - # the veilid internals (for abstraction)? - - # compressed_response = lzma.compress(response.content) - # logger.info(f"Compression response size: {len(compressed_response)}") - # return compressed_response - return response.content - - -# TODO: Handle other types of network events like -# when our private route goes -async def main_callback(update: VeilidUpdate) -> None: - if VeilidStreamer.is_stream_update(update): - async with await get_veilid_conn() as conn: - async with await get_routing_context(conn) as router: - await VeilidStreamer().receive_stream( - conn, router, update, handle_app_call - ) - elif update.kind == veilid.VeilidUpdateKind.APP_MESSAGE: - await handle_app_message(update) - - elif update.kind == veilid.VeilidUpdateKind.APP_CALL: - response = await handle_app_call(update.detail.message) - async with await get_veilid_conn() as conn: - await conn.app_call_reply(update.detail.call_id, response) diff --git a/packages/grid/veilid/server/veilid_connection.py b/packages/grid/veilid/server/veilid_connection.py deleted file mode 100644 index 0e208893aea..00000000000 --- a/packages/grid/veilid/server/veilid_connection.py +++ /dev/null @@ -1,36 +0,0 @@ -# stdlib -from collections.abc import Callable - -# third party -import veilid -from veilid import VeilidUpdate -from veilid.json_api import _JsonRoutingContext -from veilid.json_api import _JsonVeilidAPI - -# relative -from .constants import HOST -from .constants import PORT -from .constants import USE_DIRECT_CONNECTION - - -async def noop_callback(update: VeilidUpdate) -> None: - pass - - -async def get_veilid_conn( - host: str = HOST, port: int = PORT, update_callback: Callable = noop_callback -) -> _JsonVeilidAPI: - return await veilid.json_api_connect( - host=host, port=port, update_callback=update_callback - ) - - -async def get_routing_context(conn: _JsonVeilidAPI) -> _JsonRoutingContext: - if USE_DIRECT_CONNECTION: - return await (await conn.new_routing_context()).with_safety( - veilid.SafetySelection.unsafe(veilid.Sequencing.ENSURE_ORDERED) - ) - else: - return await (await conn.new_routing_context()).with_sequencing( - veilid.Sequencing.ENSURE_ORDERED - ) diff --git a/packages/grid/veilid/server/veilid_connection_singleton.py b/packages/grid/veilid/server/veilid_connection_singleton.py deleted file mode 100644 index 2fe78676be0..00000000000 --- a/packages/grid/veilid/server/veilid_connection_singleton.py +++ /dev/null @@ -1,35 +0,0 @@ -# third party -from loguru import logger -from veilid.json_api import _JsonVeilidAPI - -# relative -from .veilid_callback import main_callback -from .veilid_connection import get_veilid_conn - - -class VeilidConnectionSingleton: - _instance = None - - def __new__(cls) -> "VeilidConnectionSingleton": - if cls._instance is None: - cls._instance = super().__new__(cls) - cls._instance._connection = None - return cls._instance - - def __init__(self) -> None: - self._connection: _JsonVeilidAPI | None = None - - @property - def connection(self) -> _JsonVeilidAPI | None: - return self._connection - - async def initialize_connection(self) -> None: - if self._connection is None: - self._connection = await get_veilid_conn(update_callback=main_callback) - logger.info("Connected to Veilid") - - async def release_connection(self) -> None: - if self._connection is not None: - await self._connection.release() - logger.info("Disconnected from Veilid") - self._connection = None diff --git a/packages/grid/veilid/server/veilid_core.py b/packages/grid/veilid/server/veilid_core.py deleted file mode 100644 index 0c95cc228c2..00000000000 --- a/packages/grid/veilid/server/veilid_core.py +++ /dev/null @@ -1,182 +0,0 @@ -# stdlib -import asyncio -from enum import Enum - -# third party -from loguru import logger -import veilid -from veilid import KeyPair -from veilid import Sequencing -from veilid import Stability -from veilid import TypedKey -from veilid import ValueData -from veilid.json_api import _JsonRoutingContext -from veilid.json_api import _JsonVeilidAPI -from veilid.types import RouteId - -# relative -from .constants import TIMEOUT -from .constants import USE_DIRECT_CONNECTION -from .veilid_connection import get_routing_context -from .veilid_connection import get_veilid_conn -from .veilid_db import load_dht_key -from .veilid_db import store_dht_key -from .veilid_db import store_dht_key_creds -from .veilid_streamer import VeilidStreamer - - -class PingResponse(Enum): - SUCCESS = "SUCCESS" - FAIL = "FAIL" - - -async def create_private_route( - conn: _JsonVeilidAPI, - stability: Stability = veilid.Stability.RELIABLE, - sequencing: Sequencing = veilid.Sequencing.ENSURE_ORDERED, -) -> tuple[RouteId, bytes]: - route_id, route_blob = await conn.new_custom_private_route( - [veilid.CryptoKind.CRYPTO_KIND_VLD0], - stability=stability, - sequencing=sequencing, - ) - logger.info(f"Private Route created with Route ID: {route_id}") - return (route_id, route_blob) - - -async def get_node_id() -> str: - logger.info("Getting Node ID") - # TODO: Cache NODE ID Retrieval - async with await get_veilid_conn() as conn: - state = await conn.get_state() - config = state.config.config - node_id = config.network.routing_table.node_id[0] - if not node_id: - raise Exception("Node ID not found.Veilid might not be ready") - return node_id - - -async def generate_dht_key() -> str: - logger.info("Generating DHT Key") - - async with await get_veilid_conn() as conn: - if await load_dht_key(conn): - return "DHT Key already exists" - - async with await get_routing_context(conn) as router: - dht_record = await router.create_dht_record(veilid.DHTSchema.dflt(1)) - - _, route_blob = await create_private_route(conn) - await router.set_dht_value(dht_record.key, 0, route_blob) - - await router.close_dht_record(dht_record.key) - - keypair = KeyPair.from_parts( - key=dht_record.owner, secret=dht_record.owner_secret - ) - - await store_dht_key(conn, dht_record.key) - await store_dht_key_creds(conn, keypair) - - return "DHT Key generated successfully" - - -async def retrieve_dht_key() -> str: - async with await get_veilid_conn() as conn: - dht_key = await load_dht_key(conn) - - if dht_key is None: - raise Exception("DHT Key does not exist. Please generate one.") - return str(dht_key) - - -async def generate_vld_key() -> str: - if USE_DIRECT_CONNECTION: - await get_node_id() - else: - await generate_dht_key() - - return "Veilid Key generated successfully" - - -async def retrieve_vld_key() -> str: - if USE_DIRECT_CONNECTION: - return await get_node_id() - else: - return await retrieve_dht_key() - - -async def get_dht_value( - router: _JsonRoutingContext, - dht_key: TypedKey, - subkey: int, - force_refresh: bool = True, -) -> ValueData: - try: - await router.open_dht_record(key=dht_key, writer=None) - except Exception as e: - raise Exception(f"Unable to open DHT Record:{dht_key} . Exception: {e}") - - try: - dht_value = await router.get_dht_value( - key=dht_key, subkey=subkey, force_refresh=force_refresh - ) - # NOTE: Always close the DHT record after reading the value - await router.close_dht_record(dht_key) - return dht_value - except Exception as e: - raise Exception( - f"Unable to get subkey value:{subkey} from DHT Record:{dht_key}. Exception: {e}" - ) - - -# TODO: change verbosity of logs to debug at appropriate places -async def get_route_from_vld_key( - vld_key: str, conn: _JsonVeilidAPI, router: _JsonRoutingContext -) -> str | RouteId: - if USE_DIRECT_CONNECTION: - route = vld_key - logger.info(f"Peer Node ID: {route}") - else: - dht_key = veilid.TypedKey(vld_key) - dht_value = await get_dht_value(router, dht_key, 0) - logger.info(f"DHT Value:{dht_value}") - route = await conn.import_remote_private_route(dht_value.data) - logger.info(f"Private Route of Peer: {route} ") - - return route - - -async def app_message(vld_key: str, message: bytes) -> str: - async with await get_veilid_conn() as conn: - async with await get_routing_context(conn) as router: - route = await get_route_from_vld_key(vld_key, conn, router) - - await router.app_message(route, message) - - return "Message sent successfully" - - -async def app_call(vld_key: str, message: bytes) -> bytes: - async with await get_veilid_conn() as conn: - async with await get_routing_context(conn) as router: - route = await get_route_from_vld_key(vld_key, conn, router) - result = await VeilidStreamer().stream(router, route, message) - return result - - -async def ping(vld_key: str) -> str: - async with await get_veilid_conn() as conn: - try: - _ = await asyncio.wait_for(conn.debug(f"ping {vld_key}"), timeout=TIMEOUT) - return PingResponse.SUCCESS.value - except Exception as e: - logger.error(f"Failed to ping {vld_key} : {e}") - return PingResponse.FAIL.value - - -# TODO: Modify healthcheck endpoint to check public internet ready -async def healthcheck() -> bool: - async with await get_veilid_conn() as conn: - state = await conn.get_state() - return state.network.started diff --git a/packages/grid/veilid/server/veilid_db.py b/packages/grid/veilid/server/veilid_db.py deleted file mode 100644 index bb295910fd2..00000000000 --- a/packages/grid/veilid/server/veilid_db.py +++ /dev/null @@ -1,54 +0,0 @@ -# Contains all the database related functions for the Veilid server -# stdlib - -# third party -from veilid import KeyPair -from veilid import TypedKey -from veilid.json_api import _JsonVeilidAPI - -# relative -from .constants import DHT_KEY -from .constants import DHT_KEY_CREDS -from .constants import TABLE_DB_KEY - - -async def load_key(conn: _JsonVeilidAPI, key: str) -> str | None: - tdb = await conn.open_table_db(TABLE_DB_KEY, 1) - - async with tdb: - key_bytes = key.encode() - value = await tdb.load(key_bytes) - if value is None: - return None - return value.decode() - - -async def store_key(conn: _JsonVeilidAPI, key: str, value: str) -> None: - tdb = await conn.open_table_db(TABLE_DB_KEY, 1) - - async with tdb: - key_bytes = key.encode() - value_bytes = value.encode() - await tdb.store(key_bytes, value_bytes) - - -async def load_dht_key(conn: _JsonVeilidAPI) -> TypedKey | None: - value = await load_key(conn, DHT_KEY) - if value is None: - return None - return TypedKey(value) - - -async def load_dht_key_creds(conn: _JsonVeilidAPI) -> KeyPair | None: - value = await load_key(conn, DHT_KEY_CREDS) - if value is None: - return None - return KeyPair(value) - - -async def store_dht_key(conn: _JsonVeilidAPI, keypair: TypedKey) -> None: - await store_key(conn, DHT_KEY, str(keypair)) - - -async def store_dht_key_creds(conn: _JsonVeilidAPI, keypair: KeyPair) -> None: - await store_key(conn, DHT_KEY_CREDS, str(keypair)) diff --git a/packages/grid/veilid/server/veilid_streamer.excalidraw b/packages/grid/veilid/server/veilid_streamer.excalidraw deleted file mode 100644 index 02805934941..00000000000 --- a/packages/grid/veilid/server/veilid_streamer.excalidraw +++ /dev/null @@ -1,1676 +0,0 @@ -{ - "type": "excalidraw", - "version": 2, - "source": "https://marketplace.visualstudio.com/items?itemName=pomdtr.excalidraw-editor", - "elements": [ - { - "type": "rectangle", - "version": 562, - "versionNonce": 1165000035, - "isDeleted": false, - "id": "YrludtoGjOLLgH4SItxmn", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 255.36837768554688, - "y": 92.89969700604388, - "strokeColor": "#2f9e44", - "backgroundColor": "#b2f2bb", - "width": 146.90756225585938, - "height": 1581.3630405473252, - "seed": 1550674715, - "groupIds": [], - "frameId": null, - "roundness": { - "type": 3 - }, - "boundElements": [ - { - "type": "text", - "id": "fk5RGSylDzKogjpQ5p26_" - }, - { - "id": "HB9fmukPCatJXRU3ojPMX", - "type": "arrow" - }, - { - "id": "j8c9qxEfS_z8URX0uxlmB", - "type": "arrow" - }, - { - "id": "x8z1zPLhc-R6MXXxAo5SG", - "type": "arrow" - }, - { - "id": "QB4PiSqJ3kiEuLV0qrt23", - "type": "arrow" - }, - { - "id": "kXuxeUFilq2oUXb_PLcWy", - "type": "arrow" - } - ], - "updated": 1709916685819, - "link": null, - "locked": false - }, - { - "type": "text", - "version": 495, - "versionNonce": 1709882957, - "isDeleted": false, - "id": "fk5RGSylDzKogjpQ5p26_", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 297.1421890258789, - "y": 871.0812172797065, - "strokeColor": "#2f9e44", - "backgroundColor": "transparent", - "width": 63.35993957519531, - "height": 25, - "seed": 1890327163, - "groupIds": [], - "frameId": null, - "roundness": null, - "boundElements": [], - "updated": 1709916685820, - "link": null, - "locked": false, - "fontSize": 20, - "fontFamily": 1, - "text": "Sender", - "textAlign": "center", - "verticalAlign": "middle", - "containerId": "YrludtoGjOLLgH4SItxmn", - "originalText": "Sender", - "lineHeight": 1.25, - "baseline": 19 - }, - { - "type": "rectangle", - "version": 1104, - "versionNonce": 1373690115, - "isDeleted": false, - "id": "2dbipfTEQI_OdDpD1O2P6", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 989.8649922904917, - "y": 68.79518058494284, - "strokeColor": "#1971c2", - "backgroundColor": "#a5d8ff", - "width": 146.90756225585938, - "height": 1563.6983458368256, - "seed": 1616292725, - "groupIds": [], - "frameId": null, - "roundness": { - "type": 3 - }, - "boundElements": [ - { - "type": "text", - "id": "oSFfPmmrWFxxMU4hrqx-v" - }, - { - "id": "LMTwR0iOlc1S-qrzvLNUS", - "type": "arrow" - }, - { - "id": "HB9fmukPCatJXRU3ojPMX", - "type": "arrow" - }, - { - "id": "j8c9qxEfS_z8URX0uxlmB", - "type": "arrow" - }, - { - "id": "TpbVwIfJxGxktZJFNOMLe", - "type": "arrow" - }, - { - "id": "QB4PiSqJ3kiEuLV0qrt23", - "type": "arrow" - }, - { - "id": "Zi22fOm6LakXEYPkXfsP4", - "type": "arrow" - }, - { - "id": "dkSURhSXe6EnyxNiyADTY", - "type": "arrow" - }, - { - "id": "F-c6erTmMrf54lH2h57Sj", - "type": "arrow" - } - ], - "updated": 1709916685820, - "link": null, - "locked": false - }, - { - "type": "text", - "version": 1042, - "versionNonce": 1542941869, - "isDeleted": false, - "id": "oSFfPmmrWFxxMU4hrqx-v", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 1023.3888112602183, - "y": 838.1443535033557, - "strokeColor": "#1971c2", - "backgroundColor": "transparent", - "width": 79.85992431640625, - "height": 25, - "seed": 263943381, - "groupIds": [], - "frameId": null, - "roundness": null, - "boundElements": [], - "updated": 1709916685820, - "link": null, - "locked": false, - "fontSize": 20, - "fontFamily": 1, - "text": "Receiver", - "textAlign": "center", - "verticalAlign": "middle", - "containerId": "2dbipfTEQI_OdDpD1O2P6", - "originalText": "Receiver", - "lineHeight": 1.25, - "baseline": 19 - }, - { - "type": "arrow", - "version": 912, - "versionNonce": 319078563, - "isDeleted": false, - "id": "LMTwR0iOlc1S-qrzvLNUS", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 401.9585266113281, - "y": 145.81704711914062, - "strokeColor": "#2f9e44", - "backgroundColor": "#a5d8ff", - "width": 583.0643216317774, - "height": 3.1745915792200776, - "seed": 2067248347, - "groupIds": [], - "frameId": null, - "roundness": { - "type": 2 - }, - "boundElements": [], - "updated": 1709916685820, - "link": null, - "locked": false, - "startBinding": null, - "endBinding": { - "elementId": "2dbipfTEQI_OdDpD1O2P6", - "focus": 0.9056299193219357, - "gap": 4.842144047386228 - }, - "lastCommittedPoint": null, - "startArrowhead": null, - "endArrowhead": "arrow", - "points": [ - [ - 0, - 0 - ], - [ - 583.0643216317774, - -3.1745915792200776 - ] - ] - }, - { - "type": "text", - "version": 187, - "versionNonce": 488591117, - "isDeleted": false, - "id": "EKld6RlsfUGUDwGQ9FLIq", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 618.4837433593403, - "y": 120.77956947683776, - "strokeColor": "#2f9e44", - "backgroundColor": "#a5d8ff", - "width": 136.99192810058594, - "height": 20, - "seed": 1997842075, - "groupIds": [], - "frameId": null, - "roundness": null, - "boundElements": [], - "updated": 1709916685820, - "link": null, - "locked": false, - "fontSize": 16, - "fontFamily": 1, - "text": "STREAM_START", - "textAlign": "left", - "verticalAlign": "top", - "containerId": null, - "originalText": "STREAM_START", - "lineHeight": 1.25, - "baseline": 14 - }, - { - "type": "rectangle", - "version": 349, - "versionNonce": 1116386371, - "isDeleted": false, - "id": "Tw3Xomcf_7Km7fBnrtIAE", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 571.3246109735403, - "y": 31.04316040239246, - "strokeColor": "#2f9e44", - "backgroundColor": "#b2f2bb", - "width": 233.94567924641032, - "height": 70, - "seed": 2001962805, - "groupIds": [], - "frameId": null, - "roundness": { - "type": 3 - }, - "boundElements": [ - { - "type": "text", - "id": "cwTQWVcrPSgf5H6oCsJzH" - } - ], - "updated": 1709916685820, - "link": null, - "locked": false - }, - { - "type": "text", - "version": 364, - "versionNonce": 131851629, - "isDeleted": false, - "id": "cwTQWVcrPSgf5H6oCsJzH", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 578.9134936875657, - "y": 36.04316040239246, - "strokeColor": "#2f9e44", - "backgroundColor": "#b2f2bb", - "width": 218.76791381835938, - "height": 60, - "seed": 345470869, - "groupIds": [], - "frameId": null, - "roundness": null, - "boundElements": [], - "updated": 1709916685820, - "link": null, - "locked": false, - "fontSize": 16, - "fontFamily": 1, - "text": "@VS@SS + + ", - "textAlign": "center", - "verticalAlign": "middle", - "containerId": "Tw3Xomcf_7Km7fBnrtIAE", - "originalText": "@VS@SS + + ", - "lineHeight": 1.25, - "baseline": 55 - }, - { - "type": "arrow", - "version": 1978, - "versionNonce": 1615934435, - "isDeleted": false, - "id": "HB9fmukPCatJXRU3ojPMX", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 983.2340653553713, - "y": 150.8663711629565, - "strokeColor": "#1971c2", - "backgroundColor": "#b2f2bb", - "width": 572.205494510023, - "height": 4.575787969564033, - "seed": 1128544635, - "groupIds": [], - "frameId": null, - "roundness": { - "type": 2 - }, - "boundElements": [], - "updated": 1709916685820, - "link": null, - "locked": false, - "startBinding": { - "elementId": "2dbipfTEQI_OdDpD1O2P6", - "focus": 0.8951759563035356, - "gap": 6.630926935120442 - }, - "endBinding": { - "elementId": "YrludtoGjOLLgH4SItxmn", - "focus": -0.9193861432081903, - "gap": 8.752630903942077 - }, - "lastCommittedPoint": null, - "startArrowhead": null, - "endArrowhead": "arrow", - "points": [ - [ - 0, - 0 - ], - [ - -572.205494510023, - 4.575787969564033 - ] - ] - }, - { - "type": "rectangle", - "version": 310, - "versionNonce": 1870049229, - "isDeleted": false, - "id": "nz0b8C3mFxPnTKj_Qr-cY", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 582.1773073936117, - "y": 168.86703178674856, - "strokeColor": "#1971c2", - "backgroundColor": "#a5d8ff", - "width": 242.73996757157292, - "height": 130, - "seed": 36183675, - "groupIds": [], - "frameId": null, - "roundness": { - "type": 3 - }, - "boundElements": [ - { - "type": "text", - "id": "xe0rQssVfeqoXL-5oGr_m" - }, - { - "id": "HB9fmukPCatJXRU3ojPMX", - "type": "arrow" - } - ], - "updated": 1709916685820, - "link": null, - "locked": false - }, - { - "type": "text", - "version": 324, - "versionNonce": 1818673027, - "isDeleted": false, - "id": "xe0rQssVfeqoXL-5oGr_m", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 590.4513515431677, - "y": 173.86703178674856, - "strokeColor": "#1971c2", - "backgroundColor": "#a5d8ff", - "width": 226.19187927246094, - "height": 120, - "seed": 1081516693, - "groupIds": [], - "frameId": null, - "roundness": null, - "boundElements": [], - "updated": 1709916685820, - "link": null, - "locked": false, - "fontSize": 16, - "fontFamily": 1, - "text": "1. Set up a buffer\nreceive_buffer = {\n\"abc123\": [None, None, None, \nNone]\n}\n2. Sends an b\"OK\" response", - "textAlign": "center", - "verticalAlign": "middle", - "containerId": "nz0b8C3mFxPnTKj_Qr-cY", - "originalText": "1. Set up a buffer\nreceive_buffer = {\n\"abc123\": [None, None, None, None]\n}\n2. Sends an b\"OK\" response", - "lineHeight": 1.25, - "baseline": 115 - }, - { - "type": "arrow", - "version": 505, - "versionNonce": 34156077, - "isDeleted": false, - "id": "F-c6erTmMrf54lH2h57Sj", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 396.6860230120161, - "y": 486.72157866798426, - "strokeColor": "#2f9e44", - "backgroundColor": "#a5d8ff", - "width": 582.9946958863416, - "height": 0.6490779749104263, - "seed": 1376785941, - "groupIds": [], - "frameId": null, - "roundness": { - "type": 2 - }, - "boundElements": [], - "updated": 1709916685820, - "link": null, - "locked": false, - "startBinding": null, - "endBinding": { - "elementId": "2dbipfTEQI_OdDpD1O2P6", - "focus": 0.46636467076296556, - "gap": 10.18427339213406 - }, - "lastCommittedPoint": null, - "startArrowhead": null, - "endArrowhead": "arrow", - "points": [ - [ - 0, - 0 - ], - [ - 582.9946958863416, - -0.6490779749104263 - ] - ] - }, - { - "type": "text", - "version": 167, - "versionNonce": 1628034851, - "isDeleted": false, - "id": "SQM0s_YEj7PWKcQoWnqkv", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 623.9229635778465, - "y": 465.32951918007956, - "strokeColor": "#2f9e44", - "backgroundColor": "#a5d8ff", - "width": 130.84791564941406, - "height": 20, - "seed": 1740189307, - "groupIds": [], - "frameId": null, - "roundness": null, - "boundElements": [], - "updated": 1709916685820, - "link": null, - "locked": false, - "fontSize": 16, - "fontFamily": 1, - "text": "STREAM_CHUNK", - "textAlign": "left", - "verticalAlign": "top", - "containerId": null, - "originalText": "STREAM_CHUNK", - "lineHeight": 1.25, - "baseline": 14 - }, - { - "type": "arrow", - "version": 524, - "versionNonce": 1853677709, - "isDeleted": false, - "id": "dkSURhSXe6EnyxNiyADTY", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 977.314340077239, - "y": 498.7643456218376, - "strokeColor": "#1971c2", - "backgroundColor": "#a5d8ff", - "width": 575.7471666090936, - "height": 1.1691808232932885, - "seed": 1151071061, - "groupIds": [], - "frameId": null, - "roundness": { - "type": 2 - }, - "boundElements": [], - "updated": 1709916685820, - "link": null, - "locked": false, - "startBinding": { - "elementId": "2dbipfTEQI_OdDpD1O2P6", - "focus": 0.45019873075569805, - "gap": 12.55065221325276 - }, - "endBinding": null, - "lastCommittedPoint": null, - "startArrowhead": null, - "endArrowhead": "arrow", - "points": [ - [ - 0, - 0 - ], - [ - -575.7471666090936, - 1.1691808232932885 - ] - ] - }, - { - "type": "rectangle", - "version": 335, - "versionNonce": 1823013571, - "isDeleted": false, - "id": "lLHzEdaDgzQtpqFl9phEy", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 568.5911704050735, - "y": 376.79110651889084, - "strokeColor": "#2f9e44", - "backgroundColor": "#b2f2bb", - "width": 261.18339774560013, - "height": 70, - "seed": 817732469, - "groupIds": [], - "frameId": null, - "roundness": { - "type": 3 - }, - "boundElements": [ - { - "type": "text", - "id": "xLH7lch7TtdU9tfjjukeH" - } - ], - "updated": 1709916685820, - "link": null, - "locked": false - }, - { - "type": "text", - "version": 294, - "versionNonce": 1250641645, - "isDeleted": false, - "id": "xLH7lch7TtdU9tfjjukeH", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 576.5109408721119, - "y": 381.79110651889084, - "strokeColor": "#2f9e44", - "backgroundColor": "#b2f2bb", - "width": 245.34385681152344, - "height": 60, - "seed": 626489403, - "groupIds": [], - "frameId": null, - "roundness": null, - "boundElements": [], - "updated": 1709916685820, - "link": null, - "locked": false, - "fontSize": 16, - "fontFamily": 1, - "text": "@VS@SC + \n+ + ", - "textAlign": "center", - "verticalAlign": "middle", - "containerId": "lLHzEdaDgzQtpqFl9phEy", - "originalText": "@VS@SC + + + ", - "lineHeight": 1.25, - "baseline": 55 - }, - { - "type": "rectangle", - "version": 453, - "versionNonce": 1810016867, - "isDeleted": false, - "id": "0dam5Yr7jdey1fBvZGSTH", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 566.5049463747594, - "y": 517.9393188557151, - "strokeColor": "#1971c2", - "backgroundColor": "#a5d8ff", - "width": 252.24534438386124, - "height": 170, - "seed": 1074622389, - "groupIds": [], - "frameId": null, - "roundness": { - "type": 3 - }, - "boundElements": [ - { - "type": "text", - "id": "fgICSUX1KMNwggRoQJUsn" - }, - { - "id": "dkSURhSXe6EnyxNiyADTY", - "type": "arrow" - } - ], - "updated": 1709916685821, - "link": null, - "locked": false - }, - { - "type": "text", - "version": 367, - "versionNonce": 951968077, - "isDeleted": false, - "id": "fgICSUX1KMNwggRoQJUsn", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 571.5049463747594, - "y": 522.9393188557151, - "strokeColor": "#1971c2", - "backgroundColor": "#a5d8ff", - "width": 207.00787353515625, - "height": 160, - "seed": 692955317, - "groupIds": [], - "frameId": null, - "roundness": null, - "boundElements": [], - "updated": 1709916685821, - "link": null, - "locked": false, - "fontSize": 16, - "fontFamily": 1, - "text": "1. Fill buffer for chunk 1\n{\"abc123\": [\n \"loremipsumdolor\",\n None,\n None,\n None\n]}\n2. Send an b\"OK\" response", - "textAlign": "left", - "verticalAlign": "middle", - "containerId": "0dam5Yr7jdey1fBvZGSTH", - "originalText": "1. Fill buffer for chunk 1\n{\"abc123\": [\n \"loremipsumdolor\",\n None,\n None,\n None\n]}\n2. Send an b\"OK\" response", - "lineHeight": 1.25, - "baseline": 155 - }, - { - "type": "arrow", - "version": 767, - "versionNonce": 595944963, - "isDeleted": false, - "id": "Zi22fOm6LakXEYPkXfsP4", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 399.04013203273087, - "y": 1025.4612437223598, - "strokeColor": "#2f9e44", - "backgroundColor": "#a5d8ff", - "width": 582.5879986164294, - "height": 6.108838529077616, - "seed": 604705717, - "groupIds": [], - "frameId": null, - "roundness": { - "type": 2 - }, - "boundElements": [], - "updated": 1709916685821, - "link": null, - "locked": false, - "startBinding": null, - "endBinding": { - "elementId": "2dbipfTEQI_OdDpD1O2P6", - "focus": -0.21447394685028323, - "gap": 8.236861641331416 - }, - "lastCommittedPoint": null, - "startArrowhead": null, - "endArrowhead": "arrow", - "points": [ - [ - 0, - 0 - ], - [ - 582.5879986164294, - -6.108838529077616 - ] - ] - }, - { - "type": "text", - "version": 302, - "versionNonce": 1315844013, - "isDeleted": false, - "id": "0PcxOgR_7Krwqf76l7_um", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 631.8916545601762, - "y": 1000.0741221033294, - "strokeColor": "#2f9e44", - "backgroundColor": "#a5d8ff", - "width": 130.84791564941406, - "height": 20, - "seed": 1765332245, - "groupIds": [], - "frameId": null, - "roundness": null, - "boundElements": [], - "updated": 1709916685821, - "link": null, - "locked": false, - "fontSize": 16, - "fontFamily": 1, - "text": "STREAM_CHUNK", - "textAlign": "left", - "verticalAlign": "top", - "containerId": null, - "originalText": "STREAM_CHUNK", - "lineHeight": 1.25, - "baseline": 14 - }, - { - "type": "arrow", - "version": 1085, - "versionNonce": 982494627, - "isDeleted": false, - "id": "TpbVwIfJxGxktZJFNOMLe", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 988.6031450614196, - "y": 1032.6353487558683, - "strokeColor": "#1971c2", - "backgroundColor": "#a5d8ff", - "width": 588.5987724496572, - "height": 1.6151065044678603, - "seed": 971467381, - "groupIds": [], - "frameId": null, - "roundness": { - "type": 2 - }, - "boundElements": [], - "updated": 1709916685821, - "link": null, - "locked": false, - "startBinding": { - "elementId": "2dbipfTEQI_OdDpD1O2P6", - "focus": -0.2324478066483819, - "gap": 1.2618472290721456 - }, - "endBinding": null, - "lastCommittedPoint": null, - "startArrowhead": null, - "endArrowhead": "arrow", - "points": [ - [ - 0, - 0 - ], - [ - -588.5987724496572, - 1.6151065044678603 - ] - ] - }, - { - "type": "rectangle", - "version": 418, - "versionNonce": 969257485, - "isDeleted": false, - "id": "E9Wcd4UKiBrdKst4CYvU6", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 556.7071756602812, - "y": 920.2619825315129, - "strokeColor": "#2f9e44", - "backgroundColor": "#b2f2bb", - "width": 261.18339774560013, - "height": 70, - "seed": 383538133, - "groupIds": [], - "frameId": null, - "roundness": { - "type": 3 - }, - "boundElements": [ - { - "type": "text", - "id": "4H1ojE-pd3F9M9IMXcHbo" - } - ], - "updated": 1709916685821, - "link": null, - "locked": false - }, - { - "type": "text", - "version": 389, - "versionNonce": 1949303107, - "isDeleted": false, - "id": "4H1ojE-pd3F9M9IMXcHbo", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 564.6269461273196, - "y": 925.2619825315129, - "strokeColor": "#2f9e44", - "backgroundColor": "#b2f2bb", - "width": 245.34385681152344, - "height": 60, - "seed": 1278021941, - "groupIds": [], - "frameId": null, - "roundness": null, - "boundElements": [], - "updated": 1709916685821, - "link": null, - "locked": false, - "fontSize": 16, - "fontFamily": 1, - "text": "@VS@SC + \n+ + ", - "textAlign": "center", - "verticalAlign": "middle", - "containerId": "E9Wcd4UKiBrdKst4CYvU6", - "originalText": "@VS@SC + + + ", - "lineHeight": 1.25, - "baseline": 55 - }, - { - "type": "rectangle", - "version": 574, - "versionNonce": 666757229, - "isDeleted": false, - "id": "5kid2LNnJzyU6XbjulzSm", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 575.2972793441436, - "y": 1051.9466233772062, - "strokeColor": "#1971c2", - "backgroundColor": "#a5d8ff", - "width": 252.24534438386124, - "height": 170, - "seed": 121578133, - "groupIds": [], - "frameId": null, - "roundness": { - "type": 3 - }, - "boundElements": [ - { - "type": "text", - "id": "djhv6FK1JWvGpJhuFGIV3" - } - ], - "updated": 1709916685821, - "link": null, - "locked": false - }, - { - "type": "text", - "version": 500, - "versionNonce": 700768483, - "isDeleted": false, - "id": "djhv6FK1JWvGpJhuFGIV3", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 580.2972793441436, - "y": 1056.9466233772062, - "strokeColor": "#1971c2", - "backgroundColor": "#a5d8ff", - "width": 207.00787353515625, - "height": 160, - "seed": 751306741, - "groupIds": [], - "frameId": null, - "roundness": null, - "boundElements": [], - "updated": 1709916685821, - "link": null, - "locked": false, - "fontSize": 16, - "fontFamily": 1, - "text": "1. Fill buffer for chunk 4\n{\"abc123\": [\n \"loremipsumdolor\",\n None,\n None,\n \"theend\",\n]}\n2. Send an b\"OK\" response", - "textAlign": "left", - "verticalAlign": "middle", - "containerId": "5kid2LNnJzyU6XbjulzSm", - "originalText": "1. Fill buffer for chunk 4\n{\"abc123\": [\n \"loremipsumdolor\",\n None,\n None,\n \"theend\",\n]}\n2. Send an b\"OK\" response", - "lineHeight": 1.25, - "baseline": 155 - }, - { - "type": "arrow", - "version": 836, - "versionNonce": 1780503245, - "isDeleted": false, - "id": "QB4PiSqJ3kiEuLV0qrt23", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 403.39163225198564, - "y": 1417.1755622288765, - "strokeColor": "#2f9e44", - "backgroundColor": "#a5d8ff", - "width": 569.5153961186734, - "height": 7.537834164851347, - "seed": 1712698427, - "groupIds": [], - "frameId": null, - "roundness": { - "type": 2 - }, - "boundElements": [], - "updated": 1709916685821, - "link": null, - "locked": false, - "startBinding": { - "elementId": "YrludtoGjOLLgH4SItxmn", - "focus": 0.6752535515151398, - "gap": 1.115692310579334 - }, - "endBinding": { - "elementId": "2dbipfTEQI_OdDpD1O2P6", - "focus": -0.7125466264457613, - "gap": 16.957963919832764 - }, - "lastCommittedPoint": null, - "startArrowhead": null, - "endArrowhead": "arrow", - "points": [ - [ - 0, - 0 - ], - [ - 569.5153961186734, - -7.537834164851347 - ] - ] - }, - { - "type": "text", - "version": 179, - "versionNonce": 81579139, - "isDeleted": false, - "id": "UtLxfjaN0S3RwsChXW5t-", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 644.444223761323, - "y": 1381.430298185337, - "strokeColor": "#2f9e44", - "backgroundColor": "#a5d8ff", - "width": 113.75993347167969, - "height": 20, - "seed": 1804174651, - "groupIds": [], - "frameId": null, - "roundness": null, - "boundElements": [], - "updated": 1709916685821, - "link": null, - "locked": false, - "fontSize": 16, - "fontFamily": 1, - "text": "STREAM_END", - "textAlign": "left", - "verticalAlign": "top", - "containerId": null, - "originalText": "STREAM_END", - "lineHeight": 1.25, - "baseline": 14 - }, - { - "type": "rectangle", - "version": 336, - "versionNonce": 1569091885, - "isDeleted": false, - "id": "Ibn8PlcF3za5608SkKdgx", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 562.962283638331, - "y": 1325.1122098575497, - "strokeColor": "#2f9e44", - "backgroundColor": "#b2f2bb", - "width": 264.62319212953884, - "height": 73.22376855407398, - "seed": 1919595317, - "groupIds": [], - "frameId": null, - "roundness": { - "type": 3 - }, - "boundElements": [ - { - "type": "text", - "id": "bJ8t86dmqgwwJmN1WqFXW" - } - ], - "updated": 1709916685821, - "link": null, - "locked": false - }, - { - "type": "text", - "version": 174, - "versionNonce": 2033599523, - "isDeleted": false, - "id": "bJ8t86dmqgwwJmN1WqFXW", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 572.3619458041746, - "y": 1351.7240941345867, - "strokeColor": "#2f9e44", - "backgroundColor": "#b2f2bb", - "width": 245.82386779785156, - "height": 20, - "seed": 1468886811, - "groupIds": [], - "frameId": null, - "roundness": null, - "boundElements": [], - "updated": 1709916685821, - "link": null, - "locked": false, - "fontSize": 16, - "fontFamily": 1, - "text": "@VS@SE + ", - "textAlign": "center", - "verticalAlign": "middle", - "containerId": "Ibn8PlcF3za5608SkKdgx", - "originalText": "@VS@SE + ", - "lineHeight": 1.25, - "baseline": 15 - }, - { - "type": "arrow", - "version": 1539, - "versionNonce": 257541005, - "isDeleted": false, - "id": "j8c9qxEfS_z8URX0uxlmB", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 964.3333645753538, - "y": 1429.2401333122662, - "strokeColor": "#1971c2", - "backgroundColor": "#b2f2bb", - "width": 557.7337714638178, - "height": 6.659368252753438, - "seed": 1466596795, - "groupIds": [], - "frameId": null, - "roundness": { - "type": 2 - }, - "boundElements": [], - "updated": 1709916685821, - "link": null, - "locked": false, - "startBinding": { - "elementId": "2dbipfTEQI_OdDpD1O2P6", - "focus": -0.7376907624049549, - "gap": 25.531627715137915 - }, - "endBinding": { - "elementId": "YrludtoGjOLLgH4SItxmn", - "focus": 0.6989336951088101, - "gap": 4.323653170129774 - }, - "lastCommittedPoint": null, - "startArrowhead": null, - "endArrowhead": "arrow", - "points": [ - [ - 0, - 0 - ], - [ - -557.7337714638178, - 6.659368252753438 - ] - ] - }, - { - "type": "rectangle", - "version": 453, - "versionNonce": 2126497731, - "isDeleted": false, - "id": "972R6Jn3pcTJI6IUiGJjh", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 585.21649789408, - "y": 1449.7538534136283, - "strokeColor": "#1971c2", - "backgroundColor": "#a5d8ff", - "width": 244.51669177702865, - "height": 170, - "seed": 488867061, - "groupIds": [], - "frameId": null, - "roundness": { - "type": 3 - }, - "boundElements": [ - { - "id": "j8c9qxEfS_z8URX0uxlmB", - "type": "arrow" - }, - { - "type": "text", - "id": "Y8bhVzeiwde11g2P-ost6" - } - ], - "updated": 1709916685821, - "link": null, - "locked": false - }, - { - "type": "text", - "version": 575, - "versionNonce": 601082349, - "isDeleted": false, - "id": "Y8bhVzeiwde11g2P-ost6", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 591.5469110433365, - "y": 1454.7538534136283, - "strokeColor": "#1971c2", - "backgroundColor": "#a5d8ff", - "width": 231.85586547851562, - "height": 160, - "seed": 1426121179, - "groupIds": [], - "frameId": null, - "roundness": null, - "boundElements": [], - "updated": 1709916685821, - "link": null, - "locked": false, - "fontSize": 16, - "fontFamily": 1, - "text": "1. Join the buffer\n2. Verify that hash matches\n3. Run callback function on \nthe whole message and store\nthe response in a var\n4. Send an b\"OK\" + response \nto the sender\n5. Clear the buffer", - "textAlign": "center", - "verticalAlign": "middle", - "containerId": "972R6Jn3pcTJI6IUiGJjh", - "originalText": "1. Join the buffer\n2. Verify that hash matches\n3. Run callback function on the whole message and store the response in a var\n4. Send an b\"OK\" + response to the sender\n5. Clear the buffer", - "lineHeight": 1.25, - "baseline": 155 - }, - { - "type": "rectangle", - "version": 242, - "versionNonce": 194500451, - "isDeleted": false, - "id": "kzV8273srMyrkf05Rl0Uz", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": -780.3797967754724, - "y": 480.5291996672489, - "strokeColor": "#e03131", - "backgroundColor": "#ffc9c9", - "width": 353.0668189889283, - "height": 608.9079235586044, - "seed": 535434363, - "groupIds": [], - "frameId": null, - "roundness": { - "type": 3 - }, - "boundElements": [ - { - "type": "text", - "id": "lm_xWApjgaW1lBCMAtpya" - }, - { - "id": "kXuxeUFilq2oUXb_PLcWy", - "type": "arrow" - }, - { - "id": "x8z1zPLhc-R6MXXxAo5SG", - "type": "arrow" - } - ], - "updated": 1709916685822, - "link": null, - "locked": false - }, - { - "type": "text", - "version": 160, - "versionNonce": 2080244813, - "isDeleted": false, - "id": "lm_xWApjgaW1lBCMAtpya", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": -641.7163671394067, - "y": 767.4831614465511, - "strokeColor": "#e03131", - "backgroundColor": "#a5d8ff", - "width": 75.73995971679688, - "height": 35, - "seed": 1153394171, - "groupIds": [], - "frameId": null, - "roundness": null, - "boundElements": [], - "updated": 1709916685822, - "link": null, - "locked": false, - "fontSize": 28, - "fontFamily": 1, - "text": "Client", - "textAlign": "center", - "verticalAlign": "middle", - "containerId": "kzV8273srMyrkf05Rl0Uz", - "originalText": "Client", - "lineHeight": 1.25, - "baseline": 26 - }, - { - "type": "arrow", - "version": 237, - "versionNonce": 1309471491, - "isDeleted": false, - "id": "x8z1zPLhc-R6MXXxAo5SG", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": -416.0163458335219, - "y": 626.2086720363646, - "strokeColor": "#e03131", - "backgroundColor": "#ffc9c9", - "width": 664.9364918145627, - "height": 1.3904905109947094, - "seed": 1840762965, - "groupIds": [], - "frameId": null, - "roundness": { - "type": 2 - }, - "boundElements": [], - "updated": 1709916685822, - "link": null, - "locked": false, - "startBinding": { - "elementId": "kzV8273srMyrkf05Rl0Uz", - "focus": -0.5221627271342325, - "gap": 11.2966319530222 - }, - "endBinding": { - "elementId": "YrludtoGjOLLgH4SItxmn", - "focus": 0.32347446239419303, - "gap": 6.448231704506043 - }, - "lastCommittedPoint": null, - "startArrowhead": null, - "endArrowhead": "arrow", - "points": [ - [ - 0, - 0 - ], - [ - 664.9364918145627, - 1.3904905109947094 - ] - ] - }, - { - "type": "text", - "version": 122, - "versionNonce": 464646829, - "isDeleted": false, - "id": "9TQFO9PtcrhI_95QlDWpn", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": -278.1043845291606, - "y": 559.1678880411183, - "strokeColor": "#e03131", - "backgroundColor": "#ffc9c9", - "width": 396.84381103515625, - "height": 35, - "seed": 670552539, - "groupIds": [], - "frameId": null, - "roundness": null, - "boundElements": [], - "updated": 1709916685822, - "link": null, - "locked": false, - "fontSize": 28, - "fontFamily": 1, - "text": "vs.stream(dht_key, message)", - "textAlign": "left", - "verticalAlign": "top", - "containerId": null, - "originalText": "vs.stream(dht_key, message)", - "lineHeight": 1.25, - "baseline": 24 - }, - { - "type": "arrow", - "version": 384, - "versionNonce": 647862947, - "isDeleted": false, - "id": "kXuxeUFilq2oUXb_PLcWy", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 226.1422528254894, - "y": 912.9365134644322, - "strokeColor": "#2f9e44", - "backgroundColor": "#ffc9c9", - "width": 642.7651108451009, - "height": 1.5734924856465113, - "seed": 1469167547, - "groupIds": [], - "frameId": null, - "roundness": { - "type": 2 - }, - "boundElements": [], - "updated": 1709916685822, - "link": null, - "locked": false, - "startBinding": { - "elementId": "YrludtoGjOLLgH4SItxmn", - "focus": -0.03680030102519655, - "gap": 29.226124860057496 - }, - "endBinding": { - "elementId": "kzV8273srMyrkf05Rl0Uz", - "focus": 0.42634007150707853, - "gap": 10.690119766932582 - }, - "lastCommittedPoint": null, - "startArrowhead": null, - "endArrowhead": "arrow", - "points": [ - [ - 0, - 0 - ], - [ - -642.7651108451009, - 1.5734924856465113 - ] - ] - }, - { - "type": "text", - "version": 69, - "versionNonce": 1335432461, - "isDeleted": false, - "id": "BQgl3ip0-sE7MAmeuPNmX", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": -134.2239612010835, - "y": 862.0448648508075, - "strokeColor": "#2f9e44", - "backgroundColor": "#ffc9c9", - "width": 115.47193908691406, - "height": 35, - "seed": 471129755, - "groupIds": [], - "frameId": null, - "roundness": null, - "boundElements": [], - "updated": 1709916685822, - "link": null, - "locked": false, - "fontSize": 28, - "fontFamily": 1, - "text": "response", - "textAlign": "left", - "verticalAlign": "top", - "containerId": null, - "originalText": "response", - "lineHeight": 1.25, - "baseline": 24 - }, - { - "type": "rectangle", - "version": 295, - "versionNonce": 1643662915, - "isDeleted": false, - "id": "yACF3JP_mOTeFaKCh7Tcw", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": -362.50801379981795, - "y": 945.4942460253155, - "strokeColor": "#2f9e44", - "backgroundColor": "#b2f2bb", - "width": 552.5887113658009, - "height": 327.7297392151165, - "seed": 1429220891, - "groupIds": [], - "frameId": null, - "roundness": { - "type": 3 - }, - "boundElements": [ - { - "type": "text", - "id": "KJKnN8IDr_IRhkIsuyDw1" - } - ], - "updated": 1709916685822, - "link": null, - "locked": false - }, - { - "type": "text", - "version": 520, - "versionNonce": 321710957, - "isDeleted": false, - "id": "KJKnN8IDr_IRhkIsuyDw1", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": -350.6875350700425, - "y": 1004.3591156328738, - "strokeColor": "#2f9e44", - "backgroundColor": "#b2f2bb", - "width": 528.94775390625, - "height": 210, - "seed": 1251718165, - "groupIds": [], - "frameId": null, - "roundness": null, - "boundElements": [], - "updated": 1709916685822, - "link": null, - "locked": false, - "fontSize": 28, - "fontFamily": 1, - "text": "1. Get the response from receiver on \nSTREAM_END request\n2. Remove any VeilidStreamer specific \nprefix or stuff from the message\n3. Send the actual response back to \nthe client", - "textAlign": "center", - "verticalAlign": "middle", - "containerId": "yACF3JP_mOTeFaKCh7Tcw", - "originalText": "1. Get the response from receiver on STREAM_END request\n2. Remove any VeilidStreamer specific prefix or stuff from the message\n3. Send the actual response back to the client", - "lineHeight": 1.25, - "baseline": 201 - }, - { - "id": "HcwQQIBWGohQgaaG-3EAG", - "type": "text", - "x": 453.06147103069236, - "y": 749.0078780548102, - "width": 483.9996643066406, - "height": 125, - "angle": 0, - "strokeColor": "#2f9e44", - "backgroundColor": "transparent", - "fillStyle": "solid", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "groupIds": [], - "frameId": null, - "roundness": null, - "seed": 2128260269, - "version": 327, - "versionNonce": 917960141, - "isDeleted": false, - "boundElements": null, - "updated": 1709916685822, - "link": null, - "locked": false, - "text": ".\n.\nSend all chunks in parallel using STREAM_CHUNK\n.\n.", - "fontSize": 20, - "fontFamily": 1, - "textAlign": "center", - "verticalAlign": "top", - "baseline": 117, - "containerId": null, - "originalText": ".\n.\nSend all chunks in parallel using STREAM_CHUNK\n.\n.", - "lineHeight": 1.25 - } - ], - "appState": { - "gridSize": null, - "viewBackgroundColor": "#ffffff" - }, - "files": {} -} \ No newline at end of file diff --git a/packages/grid/veilid/server/veilid_streamer.py b/packages/grid/veilid/server/veilid_streamer.py deleted file mode 100644 index b3ce9f39a53..00000000000 --- a/packages/grid/veilid/server/veilid_streamer.py +++ /dev/null @@ -1,513 +0,0 @@ -# stdlib -import asyncio -from collections.abc import Callable -from collections.abc import Coroutine -from enum import nonmember -import hashlib -import math -from struct import Struct -from typing import Any -import uuid - -# third party -from loguru import logger -import veilid - -# relative -from .constants import MAX_SINGLE_VEILID_MESSAGE_SIZE -from .constants import MAX_STREAMER_CONCURRENCY -from .utils import BytesEnum -from .utils import retry - -# An asynchronous callable type hint that takes bytes as input and returns bytes -AsyncReceiveStreamCallback = Callable[[bytes], Coroutine[Any, Any, bytes]] -StreamId = bytes - - -class RequestType(BytesEnum): - SIZE = nonmember(8) - - STREAM_START = b"@VS@SS" - STREAM_CHUNK = b"@VS@SC" - STREAM_END = b"@VS@SE" - STREAM_SINGLE = b"@VS@S1" # Special case for handling single chunk messages - - def __init__(self, value: bytes) -> None: - # Members must be a bytes object of length == SIZE. If length is less than - # SIZE, it'll be padded with null bytes to make it SIZE bytes long. If it is - # greater, a ValueError will be raise. - size = int(self.SIZE) # type: ignore - if len(value) > size: - raise ValueError("Value must not be greater than 8 in length") - if len(value) < size: - value = value.ljust(size, b"\x00") - self._value_ = value - - def __eq__(self, __other: object) -> bool: - return self._value_ == __other - - -class ResponseType(BytesEnum): - OK = b"@VS@OK" - ERROR = b"@VS@ER" - - -class Buffer: - def __init__(self, holds_reply: bool = False) -> None: - self.msg_hash: bytes - self.chunks: list[bytes | None] - self.message: asyncio.Future[bytes] = asyncio.Future() - self.holds_reply: bool = holds_reply - # TODO add mechanism to delete/timeout old buffers - # self.last_updated: float = asyncio.get_event_loop().time() - - def set_metadata(self, message_hash: bytes, chunks_count: int) -> None: - self.message_hash = message_hash - self.chunks = [None] * chunks_count - - def add_chunk(self, chunk_number: int, chunk: bytes) -> None: - self.chunks[chunk_number] = chunk - # self.last_updated = asyncio.get_event_loop().time() - - -class VeilidStreamer: - """Pluggable class to make veild server capable of streaming large messages. - - This class is a singleton and should be used as such. It is designed to be used - with the Veilid server to stream large messages over the network. It is capable of - sending and receiving messages of any size by dividing them into chunks and - reassembling them at the receiver's end. - - Data flow: - Sender side: - 1. Send STREAM_START request -> Get OK - 3. Send all chunks using STREAM_CHUNK requests - 4. Send STREAM_END request -> Get OK - ------ Operation for sending the message finished here ------ - 5. Await reply from the receiver (the reply could also be >32kb in size) - This will finish after step 5 of receiver side (See Below section) - 6. Return the reply once received - Receiver side: - 1. Get STREAM_START request -> Set up buffers and send OK - 2. Receive all the chunks (STREAM_CHUNK request) and fill the buffers - 3. Get STREAM_END request -> Reassemble message -> Send OK - ------ Operation for receiving the message finished here ------ - 4. Pass the reassembled message to the callback function and get the reply - 5. Stream the reply back to the sender - - Structs: - We are using 3 different structs to serialize and deserialize the metadata: - - 1. stream_start_struct = Struct("!8s16s32sQ") # 64 bytes - [RequestType.STREAM_START (8 bytes string)] + - [Stream ID (16 bytes random UUID string)] + - [Message hash (32 bytes string)] + - [Total chunks count (8 bytes unsigned long long)] - - 2. stream_chunk_header_struct = Struct("!8s16sQ") # 32 bytes - [RequestType.STREAM_CHUNK (8 bytes string)] + - [Stream ID (16 bytes random UUID string)] + - [Current Chunk Number (8 bytes unsigned long long)] - - 3. stream_end_struct = Struct("!8s16s") # 24 bytes - [RequestType.STREAM_END (8 bytes string)] + - [Stream ID (16 bytes random UUID string)] - - The message is divided into chunks of 32736 bytes each, and each chunk is sent - as a separate STREAM_CHUNK request. This helps in keeping the size of each - request within the 32KB limit of the Veilid API. - [stream_chunk_header_struct (32 bytes)] + - [Actual Message Chunk (32736 bytes)] - = 32768 bytes - - Usage: - 1. Add this singleton class anwhere in your code, preferably above the update - callback function for your connection. - ``` - vs = VeilidStreamer() - ``` - - 2. Add a callback function to handle the received message stream: - ``` - async def handle_receive_stream(message: bytes) -> bytes: - # Do something with the message once the entire stream is received. - return b'some response to the sender of the stream.' - ``` - - 3. Add the following to your connection's update_callback function to relay - updates to the VeilidStreamer properly: - ``` - def update_callback(update: veilid.VeilidUpdate) -> None: - if vs.is_stream_update(update): - vs.receive_stream(connection, update, handle_receive_stream) - ...other callback code... - ``` - - 4. Use the `stream` method to send an app_call with a message of any size. - ``` - response = await vs.stream(router, vld_key, message) - ``` - - Special case: - If the message is small enough to fit in a single chunk, we can send it as a - single STREAM_SINGLE request. This avoids the additional overhead while still - allowing large replies containing multiple chunks. - - stream_single_struct = Struct("!8s16s") # 24 bytes - [RequestType.STREAM_SINGLE (8 bytes string)] + - [Stream ID (16 bytes random UUID string)] + - - Therefore, the maximum size of the message that can be sent in a STREAM_SINGLE - request is 32768 - 24 = 32744 bytes. - [stream_single_struct (24 bytes)] + - [Actual Message (32744 bytes)] - = 32768 bytes - - Data flow for single chunk message: - Sender side: - 1. Send STREAM_SINGLE request -> Get OK - 2. Await reply from the receiver - 3. Return the reply once received - Receiver side: - 1. Get STREAM_SINGLE request -> Send OK - 2. Pass the message to the callback function and get the reply - 3. Stream the reply back to the sender - - Usage: - This is automatically handled by the VeilidStreamer class. You don't need to - do anything special for this. Just use the `stream` method as usual. If the - message is small enough to fit in a single chunk, it will be sent as a - STREAM_SINGLE request automatically. - """ - - _instance = None - buffers: dict[StreamId, Buffer] - - def __new__(cls) -> "VeilidStreamer": - # Nothing fancy here, just a simple singleton pattern - if cls._instance is None: - cls._instance = super().__new__(cls) - cls._instance.buffers = {} - return cls._instance - - def __init__(self) -> None: - self._init_structs() - self._init_message_sizes() - self._init_semaphores() - - @staticmethod - def is_stream_update(update: veilid.VeilidUpdate) -> bool: - """Checks if the update is a stream request.""" - if update.kind != veilid.VeilidUpdateKind.APP_CALL: - return False - prefix = update.detail.message[:8] - return prefix in {r.value for r in RequestType} - - async def stream( - self, - router: veilid.RoutingContext, - vld_key: str, - message: bytes, - stream_id: bytes | None = None, - ) -> bytes: - """Streams a message to the given DHT key.""" - # If stream_id is not present, this is a fresh request stream. - is_request_stream = stream_id is None - - if is_request_stream: - # Since this is a new request stream, so we need to generate a new stream_id - stream_id = uuid.uuid4().bytes - # Set up a buffer for holding the reply after the end of this request stream - self.buffers[stream_id] = Buffer(holds_reply=True) - - if len(message) <= self.max_stream_single_msg_size: - await self._stream_single_chunk_request(router, vld_key, message, stream_id) - else: - await self._stream_multi_chunk_request(router, vld_key, message, stream_id) - - if is_request_stream: - response = await self._wait_for_reply(stream_id) - self._cleanup_buffer(stream_id) - return response - - return ResponseType.OK - - async def receive_stream( - self, - connection: veilid.VeilidAPI, - router: veilid.RoutingContext, - update: veilid.VeilidUpdate, - callback: AsyncReceiveStreamCallback, - ) -> None: - """Receives a streamed message.""" - message = update.detail.message - prefix = message[:8] - - if prefix == RequestType.STREAM_SINGLE: - await self._handle_receive_stream_single( - connection, router, update, callback - ) - elif prefix == RequestType.STREAM_START: - await self._handle_receive_stream_start(connection, update) - elif prefix == RequestType.STREAM_CHUNK: - await self._handle_receive_stream_chunk(connection, update) - elif prefix == RequestType.STREAM_END: - await self._handle_receive_stream_end(connection, router, update, callback) - else: - logger.error(f"[Bad Message] Message with unknown prefix: {prefix}") - - def _init_structs(self) -> None: - # Structs for serializing and deserializing metadata as bytes of fixed length - # https://docs.python.org/3/library/struct.html#format-characters - BYTE_ORDER = "!" # big-endian is recommended for networks as per IETF RFC 1700 - REQUEST_TYPE_PREFIX = f"{RequestType.SIZE}s" - STREAM_ID = "16s" - MESSAGE_HASH = "32s" - TOTAL_CHUNKS_COUNT = "Q" - CURRENT_CHUNK_NUMBER = "Q" - - self.stream_start_struct = Struct( - BYTE_ORDER - + REQUEST_TYPE_PREFIX - + STREAM_ID - + MESSAGE_HASH - + TOTAL_CHUNKS_COUNT - ) - self.stream_chunk_header_struct = Struct( - BYTE_ORDER + REQUEST_TYPE_PREFIX + STREAM_ID + CURRENT_CHUNK_NUMBER - ) - self.stream_end_struct = Struct(BYTE_ORDER + REQUEST_TYPE_PREFIX + STREAM_ID) - self.stream_single_struct = Struct(BYTE_ORDER + REQUEST_TYPE_PREFIX + STREAM_ID) - - def _init_message_sizes(self) -> None: - self.max_stream_chunk_msg_size = ( - MAX_SINGLE_VEILID_MESSAGE_SIZE - self.stream_chunk_header_struct.size - ) - self.max_stream_single_msg_size = ( - MAX_SINGLE_VEILID_MESSAGE_SIZE - self.stream_single_struct.size - ) - - def _init_semaphores(self) -> None: - self._send_request_semaphore = asyncio.Semaphore(MAX_STREAMER_CONCURRENCY) - self._send_response_semaphore = asyncio.Semaphore(MAX_STREAMER_CONCURRENCY) - - @retry(veilid.VeilidAPIError, tries=4, delay=1, backoff=2) - async def _send_request( - self, router: veilid.RoutingContext, vld_key: str, request_data: bytes - ) -> None: - """Send an app call to the Veilid server and return the response.""" - async with self._send_request_semaphore: - response = await router.app_call(vld_key, request_data) - if response != ResponseType.OK: - raise Exception("Unexpected response from server") - - async def _send_response( - self, - connection: veilid.VeilidAPI, - update: veilid.VeilidUpdate, - response: bytes, - ) -> None: - """Send a response to an app call.""" - async with self._send_response_semaphore: - await connection.app_call_reply(update.detail.call_id, response) - - async def _send_ok_response( - self, connection: veilid.VeilidAPI, update: veilid.VeilidUpdate - ) -> None: - await self._send_response(connection, update, ResponseType.OK) - - async def _send_error_response( - self, connection: veilid.VeilidAPI, update: veilid.VeilidUpdate - ) -> None: - await self._send_response(connection, update, ResponseType.ERROR) - - def _cleanup_buffer(self, stream_id: bytes) -> None: - del self.buffers[stream_id] - - def _calculate_chunks_count(self, message_size: int) -> int: - total_no_of_chunks = math.ceil(message_size / self.max_stream_chunk_msg_size) - return total_no_of_chunks - - def _get_chunk( - self, - stream_id: bytes, - chunk_number: int, - message: bytes, - ) -> bytes: - chunk_header = self.stream_chunk_header_struct.pack( - RequestType.STREAM_CHUNK, - stream_id, - chunk_number, - ) - cursor_start = chunk_number * self.max_stream_chunk_msg_size - cursor_end = cursor_start + self.max_stream_chunk_msg_size - chunk = message[cursor_start:cursor_end] - return chunk_header + chunk - - async def _stream_single_chunk_request( - self, - router: veilid.RoutingContext, - vld_key: str, - message: bytes, - stream_id: bytes, - ) -> None: - stream_single_request_header = self.stream_single_struct.pack( - RequestType.STREAM_SINGLE, stream_id - ) - stream_single_request = stream_single_request_header + message - await self._send_request(router, vld_key, stream_single_request) - - async def _stream_multi_chunk_request( - self, - router: veilid.RoutingContext, - vld_key: str, - message: bytes, - stream_id: bytes, - ) -> None: - message_size = len(message) - message_hash = hashlib.sha256(message).digest() - total_chunks_count = self._calculate_chunks_count(message_size) - - # Send STREAM_START request - stream_start_request = self.stream_start_struct.pack( - RequestType.STREAM_START, - stream_id, - message_hash, - total_chunks_count, - ) - await self._send_request(router, vld_key, stream_start_request) - - # Send chunks - tasks = [] - for chunk_number in range(total_chunks_count): - chunk = self._get_chunk(stream_id, chunk_number, message) - tasks.append(self._send_request(router, vld_key, chunk)) - await asyncio.gather(*tasks) - - # Send STREAM_END request - stream_end_message = self.stream_end_struct.pack( - RequestType.STREAM_END, stream_id - ) - await self._send_request(router, vld_key, stream_end_message) - - async def _wait_for_reply(self, stream_id: bytes) -> bytes: - buffer = self.buffers[stream_id] - logger.debug("Waiting for reply...") - response = await buffer.message - logger.debug("Reply received") - return response - - async def _handle_receive_stream_single( - self, - connection: veilid.VeilidAPI, - router: veilid.RoutingContext, - update: veilid.VeilidUpdate, - callback: AsyncReceiveStreamCallback, - ) -> None: - """Handles receiving STREAM_SINGLE request.""" - message = update.detail.message - header_len = self.stream_single_struct.size - header, message = message[:header_len], message[header_len:] - _, stream_id = self.stream_single_struct.unpack(header) - await self._send_ok_response(connection, update) - - buffer = self.buffers.get(stream_id) - if buffer and buffer.holds_reply: - # This message is being received by the sender and the stream() method must - # be waiting for the reply. So we need to set the result in the buffer. - logger.debug(f"Received single chunk reply of {len(message)} bytes...") - buffer.message.set_result(message) - else: - # This message is being received by the receiver and we need to send back - # the reply to the sender. So we need to call the callback function and - # stream the reply back to the sender. - logger.debug(f"Received single chunk request of {len(message)} bytes...") - reply = await callback(message) - logger.debug( - f"Replying to {update.detail.sender} with {len(reply)} bytes of msg..." - ) - await self.stream(router, update.detail.sender, reply, stream_id) - - async def _handle_receive_stream_start( - self, connection: veilid.VeilidAPI, update: veilid.VeilidUpdate - ) -> None: - """Handles receiving STREAM_START request.""" - _, stream_id, message_hash, chunks_count = self.stream_start_struct.unpack( - update.detail.message - ) - buffer = self.buffers.get(stream_id) - - if buffer is None: - # If the buffer is not present, this is a new request stream. So we need to - # set up a new buffer to hold the chunks. - buffer = Buffer(holds_reply=False) - self.buffers[stream_id] = buffer - buffer.set_metadata(message_hash, chunks_count) - stream_type = "reply" if buffer.holds_reply else "request" - logger.debug(f"Receiving {stream_type} stream of {chunks_count} chunks...") - await self._send_ok_response(connection, update) - - async def _handle_receive_stream_chunk( - self, - connection: veilid.VeilidAPI, - update: veilid.VeilidUpdate, - ) -> None: - """Handles receiving STREAM_CHUNK request.""" - message = update.detail.message - chunk_header_len = self.stream_chunk_header_struct.size - chunk_header, chunk = message[:chunk_header_len], message[chunk_header_len:] - _, stream_id, chunk_number = self.stream_chunk_header_struct.unpack( - chunk_header - ) - buffer = self.buffers[stream_id] - buffer.add_chunk(chunk_number, chunk) - stream_type = "reply" if buffer.holds_reply else "request" - logger.debug( - f"Received {stream_type} chunk {chunk_number + 1}/{len(buffer.chunks)}" - ) - await self._send_ok_response(connection, update) - - async def _handle_receive_stream_end( - self, - connection: veilid.VeilidAPI, - router: veilid.RoutingContext, - update: veilid.VeilidUpdate, - callback: AsyncReceiveStreamCallback, - ) -> None: - """Handles receiving STREAM_END request.""" - _, stream_id = self.stream_end_struct.unpack(update.detail.message) - buffer = self.buffers[stream_id] - - if None in buffer.chunks: - # TODO add retry mechanism to request the missing chunks - raise Exception("Did not receive all the chunks") - - reassembled_message = b"".join(buffer.chunks) - hash_matches = ( - hashlib.sha256(reassembled_message).digest() == buffer.message_hash - ) - stream_type = "Reply" if buffer.holds_reply else "Request" - logger.debug( - f"{stream_type} message of {len(reassembled_message) // 1024} KB reassembled, hash matches: {hash_matches}" - ) - - if hash_matches: - buffer.message.set_result(reassembled_message) - await self._send_ok_response(connection, update) - else: - buffer.message.set_exception(Exception("Hash mismatch")) - await self._send_error_response(connection, update) - - is_request_stream = not buffer.holds_reply - if is_request_stream: - # This message is being received on the receiver's end and we need to send - # back the reply to the sender. So we need to call the callback function - # and stream the reply back to the sender. - reply = await callback(reassembled_message) - logger.debug( - f"Replying to {update.detail.sender} with {len(reply)} bytes of msg..." - ) - # Stream as the reply itself could be greater than the max chunk size - await self.stream(router, update.detail.sender, reply, stream_id) - # Finally delete the buffer - self._cleanup_buffer(stream_id) diff --git a/packages/grid/veilid/start.sh b/packages/grid/veilid/start.sh deleted file mode 100644 index 0675243d3d4..00000000000 --- a/packages/grid/veilid/start.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash -set -e -export PATH="/root/.local/bin:${PATH}" - -APP_MODULE=server.main:app -APP_LOG_LEVEL=${APP_LOG_LEVEL:-info} -UVICORN_LOG_LEVEL=${UVICORN_LOG_LEVEL:-info} -HOST=${HOST:-0.0.0.0} -PORT=${PORT:-4000} -RELOAD="" -VEILID_FLAGS=${VEILID_FLAGS:-""} - -if [[ ${DEV_MODE} == "True" ]]; -then - echo "DEV_MODE Enabled" - RELOAD="--reload" -fi - -/veilid/veilid-server -c /veilid/veilid-server.conf $VEILID_FLAGS & - -exec uvicorn $RELOAD --host $HOST --port $PORT --log-level $UVICORN_LOG_LEVEL "$APP_MODULE" \ No newline at end of file diff --git a/packages/grid/veilid/veilid-server.conf b/packages/grid/veilid/veilid-server.conf deleted file mode 100644 index 3644a1ef643..00000000000 --- a/packages/grid/veilid/veilid-server.conf +++ /dev/null @@ -1,9 +0,0 @@ -daemon: - enabled: false -client_api: - enabled: true - listen_address: '0.0.0.0:5959' -core: - network: - rpc: - timeout_ms: 60000 diff --git a/packages/grid/veilid/veilid.dockerfile b/packages/grid/veilid/veilid.dockerfile deleted file mode 100644 index baa168b9b12..00000000000 --- a/packages/grid/veilid/veilid.dockerfile +++ /dev/null @@ -1,43 +0,0 @@ -ARG VEILID_VERSION="0.2.5" -ARG PYTHON_VERSION="3.12" - -# ======== [Stage 1] Build Veilid Server ========== # -# TODO: Switch from building the packages to using the pre-built packages -# from debian or rpm. This will reduce the build time and the size of the -# final image. -FROM rust as build -ARG VEILID_VERSION -RUN apt update && apt install -y git -RUN git clone -b v${VEILID_VERSION} https://gitlab.com/veilid/veilid -WORKDIR /veilid -RUN bash -c "source scripts/earthly/install_capnproto.sh" -RUN bash -c "source scripts/earthly/install_protoc.sh" -RUN cd veilid-server && cargo build --release -p veilid-server - -# ========== [Stage 2] Dependency Install ========== # - -FROM python:${PYTHON_VERSION}-bookworm -ARG VEILID_VERSION -COPY --from=build /veilid/target/release/veilid-server /veilid/veilid-server -WORKDIR /app -COPY ./requirements.txt /app/requirements.txt -RUN --mount=type=cache,target=/root/.cache \ - pip install --user -r requirements.txt && \ - pip install veilid==${VEILID_VERSION} - -COPY ./start.sh /app/start.sh -RUN chmod +x /app/start.sh -COPY ./server /app/server -COPY ./veilid-server.conf /veilid - -# ========== [Final] Start Veilid Server and Python Web Server ========== # - -CMD ["sh", "-c", "/app/start.sh"] -EXPOSE 5959/udp -EXPOSE 5959 -EXPOSE 4000 -RUN apt update && apt install netcat-openbsd - -# docker build -f veilid.dockerfile . -t veilid -# docker run -it -p 4000:4000 -p 5959:5959 -p 5959:5959/udp veilid -# /root/.local/share/veilid \ No newline at end of file diff --git a/packages/syft/src/syft/client/client.py b/packages/syft/src/syft/client/client.py index c719943eed2..0de91ffb8cf 100644 --- a/packages/syft/src/syft/client/client.py +++ b/packages/syft/src/syft/client/client.py @@ -2,7 +2,6 @@ from __future__ import annotations # stdlib -import base64 from collections.abc import Callable from copy import deepcopy from enum import Enum @@ -15,7 +14,6 @@ # third party from argon2 import PasswordHasher -from pydantic import Field from pydantic import field_validator import requests from requests import Response @@ -48,11 +46,7 @@ from ..service.user.user import UserView from ..service.user.user_roles import ServiceRole from ..service.user.user_service import UserService -from ..service.veilid.veilid_endpoints import VEILID_PROXY_PATH -from ..service.veilid.veilid_endpoints import VEILID_SERVICE_URL -from ..service.veilid.veilid_endpoints import VEILID_SYFT_PROXY_URL from ..types.grid_url import GridURL -from ..types.syft_object import SYFT_OBJECT_VERSION_1 from ..types.syft_object import SYFT_OBJECT_VERSION_2 from ..types.uid import UID from ..util.logger import debug @@ -330,218 +324,6 @@ def get_client_type(self) -> type[SyftClient]: return SyftError(message=f"Unknown node type {metadata.node_type}") -@serializable( - attrs=["proxy_target_uid", "vld_key", "vld_forward_proxy", "vld_reverse_proxy"] -) -class VeilidConnection(NodeConnection): - __canonical_name__ = "VeilidConnection" - __version__ = SYFT_OBJECT_VERSION_1 - - vld_forward_proxy: GridURL = Field(default=GridURL.from_url(VEILID_SERVICE_URL)) - vld_reverse_proxy: GridURL = Field(default=GridURL.from_url(VEILID_SYFT_PROXY_URL)) - vld_key: str - proxy_target_uid: UID | None = None - routes: type[Routes] = Field(default=Routes) - session_cache: Session | None = None - - @field_validator("vld_forward_proxy", mode="before") - def make_forward_proxy_url(cls, v: GridURL | str) -> GridURL: - if isinstance(v, str): - return GridURL.from_url(v) - else: - return v - - # TODO: Remove this once when we remove reverse proxy in Veilid Connection - @field_validator("vld_reverse_proxy", mode="before") - def make_reverse_proxy_url(cls, v: GridURL | str) -> GridURL: - if isinstance(v, str): - return GridURL.from_url(v) - else: - return v - - def with_proxy(self, proxy_target_uid: UID) -> Self: - raise NotImplementedError("VeilidConnection does not support with_proxy") - - def get_cache_key(self) -> str: - return str(self.vld_key) - - # def to_blob_route(self, path: str, **kwargs) -> GridURL: - # _path = self.routes.ROUTE_BLOB_STORE.value + path - # return self.url.with_path(_path) - - @property - def session(self) -> Session: - if self.session_cache is None: - session = requests.Session() - retry = Retry(total=3, backoff_factor=0.5) - adapter = HTTPAdapter(max_retries=retry) - session.mount("http://", adapter) - session.mount("https://", adapter) - self.session_cache = session - return self.session_cache - - def _make_get(self, path: str, params: dict | None = None) -> bytes: - rev_proxy_url = self.vld_reverse_proxy.with_path(path) - forward_proxy_url = self.vld_forward_proxy.with_path(VEILID_PROXY_PATH) - - json_data = { - "url": str(rev_proxy_url), - "method": "GET", - "vld_key": self.vld_key, - "params": params, - } - response = self.session.get(str(forward_proxy_url), json=json_data) - if response.status_code != 200: - raise requests.ConnectionError( - f"Failed to fetch {forward_proxy_url}. Response returned with code {response.status_code}" - ) - - return response.content - - def _make_post( - self, - path: str, - json: dict[str, Any] | None = None, - data: bytes | None = None, - ) -> bytes: - rev_proxy_url = self.vld_reverse_proxy.with_path(path) - forward_proxy_url = self.vld_forward_proxy.with_path(VEILID_PROXY_PATH) - - # Since JSON expects strings, we need to encode the bytes to base64 - # as some bytes may not be valid utf-8 - # TODO: Can we optimize this? - data_base64 = base64.b64encode(data).decode() if data else None - - json_data = { - "url": str(rev_proxy_url), - "method": "POST", - "vld_key": self.vld_key, - "json": json, - "data": data_base64, - } - - response = self.session.post(str(forward_proxy_url), json=json_data) - if response.status_code != 200: - raise requests.ConnectionError( - f"Failed to fetch {forward_proxy_url}. Response returned with code {response.status_code}" - ) - - return response.content - - def get_node_metadata(self, credentials: SyftSigningKey) -> NodeMetadataJSON: - # TODO: Implement message proxy forwarding for gateway - - response = self._make_get(self.routes.ROUTE_METADATA.value) - metadata_json = json.loads(response) - return NodeMetadataJSON(**metadata_json) - - def get_api( - self, credentials: SyftSigningKey, communication_protocol: int - ) -> SyftAPI: - # TODO: Implement message proxy forwarding for gateway - - params = { - "verify_key": str(credentials.verify_key), - "communication_protocol": communication_protocol, - } - content = self._make_get(self.routes.ROUTE_API.value, params=params) - obj = _deserialize(content, from_bytes=True) - obj.connection = self - obj.signing_key = credentials - obj.communication_protocol = communication_protocol - if self.proxy_target_uid: - obj.node_uid = self.proxy_target_uid - return cast(SyftAPI, obj) - - def login( - self, - email: str, - password: str, - ) -> SyftSigningKey | None: - # TODO: Implement message proxy forwarding for gateway - - credentials = {"email": email, "password": password} - response = self._make_post(self.routes.ROUTE_LOGIN.value, credentials) - obj = _deserialize(response, from_bytes=True) - - return obj - - def register(self, new_user: UserCreate) -> Any: - # TODO: Implement message proxy forwarding for gateway - - data = _serialize(new_user, to_bytes=True) - response = self._make_post(self.routes.ROUTE_REGISTER.value, data=data) - response = _deserialize(response, from_bytes=True) - return response - - def make_call(self, signed_call: SignedSyftAPICall) -> Any: - msg_bytes: bytes = _serialize(obj=signed_call, to_bytes=True) - # Since JSON expects strings, we need to encode the bytes to base64 - # as some bytes may not be valid utf-8 - # TODO: Can we optimize this? - msg_base64 = base64.b64encode(msg_bytes).decode() - - rev_proxy_url = self.vld_reverse_proxy.with_path( - self.routes.ROUTE_API_CALL.value - ) - forward_proxy_url = self.vld_forward_proxy.with_path(VEILID_PROXY_PATH) - json_data = { - "url": str(rev_proxy_url), - "method": "POST", - "vld_key": self.vld_key, - "data": msg_base64, - } - response = requests.post( # nosec - url=str(forward_proxy_url), - json=json_data, - ) - - if response.status_code != 200: - raise requests.ConnectionError( - f"Failed to fetch metadata. Response returned with code {response.status_code}" - ) - - result = _deserialize(response.content, from_bytes=True) - return result - - def __repr__(self) -> str: - return self.__str__() - - def __str__(self) -> str: - res = f"{type(self).__name__}:" - res += f"\n DHT Key: {self.vld_key}" - res += f"\n Forward Proxy: {self.vld_forward_proxy}" - res += f"\n Reverse Proxy: {self.vld_reverse_proxy}" - return res - - def __hash__(self) -> int: - return ( - hash(self.proxy_target_uid) - + hash(self.vld_key) - + hash(self.vld_forward_proxy) - + hash(self.vld_reverse_proxy) - ) - - def get_client_type(self) -> type[SyftClient]: - # TODO: Rasswanth, should remove passing in credentials - # when metadata are proxy forwarded in the grid routes - # in the gateway fixes PR - # relative - from .domain_client import DomainClient - from .enclave_client import EnclaveClient - from .gateway_client import GatewayClient - - metadata = self.get_node_metadata(credentials=SyftSigningKey.generate()) - if metadata.node_type == NodeType.DOMAIN.value: - return DomainClient - elif metadata.node_type == NodeType.GATEWAY.value: - return GatewayClient - elif metadata.node_type == NodeType.ENCLAVE.value: - return EnclaveClient - else: - return SyftError(message=f"Unknown node type {metadata.node_type}") - - @serializable() class PythonConnection(NodeConnection): __canonical_name__ = "PythonConnection" @@ -882,13 +664,6 @@ def exchange_route( remote_node_route=remote_node_route, remote_node_verify_key=client.metadata.to(NodeMetadataV3).verify_key, ) - - elif protocol == SyftProtocol.VEILID: - remote_node_route = connection_to_route(client.connection) - - result = self.api.services.network.exchange_veilid_route( - remote_node_route=remote_node_route, - ) else: raise ValueError( f"Invalid Route Exchange SyftProtocol: {protocol}.Supported protocols are {SyftProtocol.all()}" @@ -1166,18 +941,9 @@ def connect( url: str | GridURL = DEFAULT_PYGRID_ADDRESS, node: AbstractNode | None = None, port: int | None = None, - vld_forward_proxy: str | GridURL | None = None, - vld_reverse_proxy: str | GridURL | None = None, - vld_key: str | None = None, ) -> SyftClient: if node: connection = PythonConnection(node=node) - elif vld_key and vld_forward_proxy and vld_reverse_proxy: - connection = VeilidConnection( - vld_forward_proxy=vld_forward_proxy, - vld_reverse_proxy=vld_reverse_proxy, - vld_key=vld_key, - ) else: url = GridURL.from_url(url) if isinstance(port, int | str): @@ -1219,19 +985,12 @@ def login_as_guest( port: int | None = None, # PythonConnection node: AbstractNode | None = None, - # Veilid Connection - vld_forward_proxy: str | GridURL | None = None, - vld_reverse_proxy: str | GridURL | None = None, - vld_key: str | None = None, verbose: bool = True, ) -> SyftClient: _client = connect( url=url, node=node, port=port, - vld_forward_proxy=vld_forward_proxy, - vld_reverse_proxy=vld_reverse_proxy, - vld_key=vld_key, ) if isinstance(_client, SyftError): @@ -1254,10 +1013,6 @@ def login( port: int | None = None, # PythonConnection node: AbstractNode | None = None, - # Veilid Connection - vld_forward_proxy: str | GridURL | None = None, - vld_reverse_proxy: str | GridURL | None = None, - vld_key: str | None = None, password: str | None = None, cache: bool = True, ) -> SyftClient: @@ -1265,9 +1020,6 @@ def login( url=url, node=node, port=port, - vld_forward_proxy=vld_forward_proxy, - vld_reverse_proxy=vld_reverse_proxy, - vld_key=vld_key, ) if isinstance(_client, SyftError): diff --git a/packages/syft/src/syft/client/protocol.py b/packages/syft/src/syft/client/protocol.py index e969d59ca5d..ffb23c5880e 100644 --- a/packages/syft/src/syft/client/protocol.py +++ b/packages/syft/src/syft/client/protocol.py @@ -6,7 +6,6 @@ class SyftProtocol(Enum): """Enum class to represent the different Syft protocols.""" HTTP = "http" - VEILID = "veilid" def all(self) -> list: return [p.value for p in SyftProtocol] diff --git a/packages/syft/src/syft/node/node.py b/packages/syft/src/syft/node/node.py index 07025fa0bcf..8f24b1ab3cd 100644 --- a/packages/syft/src/syft/node/node.py +++ b/packages/syft/src/syft/node/node.py @@ -96,7 +96,6 @@ from ..service.user.user_roles import ServiceRole from ..service.user.user_service import UserService from ..service.user.user_stash import UserStash -from ..service.veilid import VeilidServiceProvider from ..service.worker.image_registry_service import SyftImageRegistryService from ..service.worker.utils import DEFAULT_WORKER_IMAGE_TAG from ..service.worker.utils import DEFAULT_WORKER_POOL_NAME @@ -874,8 +873,7 @@ def _construct_services(self) -> None: {"svc": SyftImageRegistryService}, {"svc": SyncService}, {"svc": OutputService}, - {"svc": UserCodeStatusService}, - {"svc": VeilidServiceProvider}, # this is lazy + {"svc": UserCodeStatusService}, # this is lazy ] for svc_kwargs in default_services: diff --git a/packages/syft/src/syft/service/network/network_service.py b/packages/syft/src/syft/service/network/network_service.py index 27b3fcd18dd..6ae9e681943 100644 --- a/packages/syft/src/syft/service/network/network_service.py +++ b/packages/syft/src/syft/service/network/network_service.py @@ -11,7 +11,6 @@ from ...client.client import HTTPConnection from ...client.client import PythonConnection from ...client.client import SyftClient -from ...client.client import VeilidConnection from ...node.credentials import SyftVerifyKey from ...node.worker_settings import WorkerSettings from ...serde.serializable import serializable @@ -44,7 +43,6 @@ from .routes import HTTPNodeRoute from .routes import NodeRoute from .routes import PythonNodeRoute -from .routes import VeilidNodeRoute VerifyKeyPartitionKey = PartitionKey(key="verify_key", type_=SyftVerifyKey) NodeTypePartitionKey = PartitionKey(key="node_type", type_=NodeType) @@ -381,89 +379,6 @@ def delete_peer_by_id( return SyftError(message=str(result.err())) return SyftSuccess(message="Node Peer Deleted") - @service_method( - path="network.exchange_veilid_route", - name="exchange_veilid_route", - roles=DATA_OWNER_ROLE_LEVEL, - ) - def exchange_veilid_route( - self, - context: AuthedServiceContext, - remote_node_route: NodeRoute, - ) -> SyftSuccess | SyftError: - """Exchange Route With Another Node""" - - # Step 1: Get our own Veilid Node Peer to send to the remote node - self_node_peer: NodePeer = context.node.settings.to(NodePeer) - - veilid_service = context.node.get_service("veilidservice") - veilid_route = veilid_service.get_veilid_route(context=context) - - if isinstance(veilid_route, SyftError): - return veilid_route - - self_node_peer.node_routes = [veilid_route] - - # Step 2: Create a Remote Client - remote_client: SyftClient = remote_node_route.client_with_context( - context=context - ) - - # Step 3: Send the Node Peer to the remote node - remote_node_peer: NodePeer | SyftError = ( - remote_client.api.services.network.add_veilid_peer( - peer=self_node_peer, - ) - ) - - if not isinstance(remote_node_peer, NodePeer): - return remote_node_peer - - # Step 4: Add the remote Node Peer to our stash - result = self.stash.update_peer(context.node.verify_key, remote_node_peer) - if result.is_err(): - return SyftError(message=str(result.err())) - - return SyftSuccess(message="Routes Exchanged") - - @service_method( - path="network.add_veilid_peer", name="add_veilid_peer", roles=GUEST_ROLE_LEVEL - ) - def add_veilid_peer( - self, - context: AuthedServiceContext, - peer: NodePeer, - ) -> NodePeer | SyftError: - """Add a Veilid Node Peer""" - - # Step 1: Using the verify_key of the peer to verify the signature - # It is also our single source of truth for the peer - if peer.verify_key != context.credentials: - return SyftError( - message=( - f"The {type(peer)}.verify_key: " - f"{peer.verify_key} does not match the signature of the message" - ) - ) - - # Step 2: Save the remote peer to our stash - result = self.stash.update_peer(context.node.verify_key, peer) - if result.is_err(): - return SyftError(message=str(result.err())) - - # Step 3: Get our own Veilid Node Peer to send to the remote node - self_node_peer: NodePeer = context.node.settings.to(NodePeer) - - veilid_service = context.node.get_service("veilidservice") - veilid_route = veilid_service.get_veilid_route(context=context) - - if isinstance(veilid_route, SyftError): - return veilid_route - - self_node_peer.node_routes = [veilid_route] - - return self_node_peer - TYPE_TO_SERVICE[NodePeer] = NetworkService SERVICE_TO_TYPES[NetworkService].update({NodePeer}) @@ -517,20 +432,6 @@ def node_route_to_http_connection( return HTTPConnection(url=url, proxy_target_uid=obj.proxy_target_uid) -@transform_method(VeilidNodeRoute, VeilidConnection) -def node_route_to_veilid_connection( - obj: VeilidNodeRoute, context: TransformContext | None = None -) -> list[Callable]: - return VeilidConnection(vld_key=obj.vld_key, proxy_target_uid=obj.proxy_target_uid) - - -@transform_method(VeilidConnection, VeilidNodeRoute) -def veilid_connection_to_node_route( - obj: VeilidConnection, context: TransformContext | None = None -) -> list[Callable]: - return VeilidNodeRoute(vld_key=obj.vld_key, proxy_target_uid=obj.proxy_target_uid) - - @transform(NodeMetadataV3, NodePeer) def metadata_to_peer() -> list[Callable]: return [ diff --git a/packages/syft/src/syft/service/network/node_peer.py b/packages/syft/src/syft/service/network/node_peer.py index 5ecac3535dd..b6aa18ca9bf 100644 --- a/packages/syft/src/syft/service/network/node_peer.py +++ b/packages/syft/src/syft/service/network/node_peer.py @@ -16,7 +16,6 @@ from .routes import NodeRoute from .routes import NodeRouteType from .routes import PythonNodeRoute -from .routes import VeilidNodeRoute from .routes import connection_to_route from .routes import route_to_connection @@ -100,15 +99,6 @@ def existed_route(self, route: NodeRoute) -> tuple[bool, int | None]: ): return (True, i) return (False, None) - elif isinstance(route, VeilidNodeRoute): - for i, r in enumerate(self.node_routes): - if ( - route.vld_key == r.vld_key - and route.proxy_target_uid == r.proxy_target_uid - ): - return (True, i) - - return (False, None) else: raise ValueError(f"Unsupported route type: {type(route)}") diff --git a/packages/syft/src/syft/service/network/routes.py b/packages/syft/src/syft/service/network/routes.py index 747bcceca83..c357bd6d26b 100644 --- a/packages/syft/src/syft/service/network/routes.py +++ b/packages/syft/src/syft/service/network/routes.py @@ -15,10 +15,8 @@ from ...client.client import NodeConnection from ...client.client import PythonConnection from ...client.client import SyftClient -from ...client.client import VeilidConnection from ...node.worker_settings import WorkerSettings from ...serde.serializable import serializable -from ...types.syft_object import SYFT_OBJECT_VERSION_1 from ...types.syft_object import SYFT_OBJECT_VERSION_2 from ...types.syft_object import SyftObject from ...types.transforms import TransformContext @@ -91,24 +89,6 @@ def __hash__(self) -> int: return hash(self.host_or_ip) + hash(self.port) + hash(self.protocol) -@serializable() -class VeilidNodeRoute(SyftObject, NodeRoute): - __canonical_name__ = "VeilidNodeRoute" - __version__ = SYFT_OBJECT_VERSION_1 - - vld_key: str - proxy_target_uid: UID | None = None - priority: int = 1 - - def __eq__(self, other: Any) -> bool: - if isinstance(other, VeilidNodeRoute): - return hash(self) == hash(other) - return self == other - - def __hash__(self) -> int: - return hash(self.vld_key) - - @serializable() class PythonNodeRoute(SyftObject, NodeRoute): __canonical_name__ = "PythonNodeRoute" @@ -149,7 +129,7 @@ def __hash__(self) -> int: return hash(self.worker_settings.id) -NodeRouteType = HTTPNodeRoute | PythonNodeRoute | VeilidNodeRoute +NodeRouteType = HTTPNodeRoute | PythonNodeRoute def route_to_connection( @@ -159,8 +139,6 @@ def route_to_connection( return route.to(HTTPConnection, context=context) elif isinstance(route, PythonNodeRoute): return route.to(PythonConnection, context=context) - elif isinstance(route, VeilidNodeRoute): - return route.to(VeilidConnection, context=context) else: raise ValueError(f"Route {route} is not supported.") @@ -170,7 +148,5 @@ def connection_to_route(connection: NodeConnection) -> NodeRoute: return connection.to(HTTPNodeRoute) elif isinstance(connection, PythonConnection): # type: ignore[unreachable] return connection.to(PythonNodeRoute) - elif isinstance(connection, VeilidConnection): - return connection.to(VeilidNodeRoute) else: raise ValueError(f"Connection {connection} is not supported.") diff --git a/packages/syft/src/syft/service/service.py b/packages/syft/src/syft/service/service.py index 8a98eca633b..4e3c41ed014 100644 --- a/packages/syft/src/syft/service/service.py +++ b/packages/syft/src/syft/service/service.py @@ -21,9 +21,6 @@ from ..serde.lib_permissions import CMPCRUDPermission from ..serde.lib_permissions import CMPPermission from ..serde.lib_service_registry import CMPBase -from ..serde.lib_service_registry import CMPClass -from ..serde.lib_service_registry import CMPFunction -from ..serde.lib_service_registry import action_execute_registry_libs from ..serde.serializable import serializable from ..serde.signature import Signature from ..serde.signature import signature_remove_context @@ -39,7 +36,6 @@ from .response import SyftError from .user.user_roles import DATA_OWNER_ROLE_LEVEL from .user.user_roles import ServiceRole -from .veilid import VEILID_ENABLED from .warnings import APIEndpointWarning if TYPE_CHECKING: @@ -228,21 +224,6 @@ def register_lib_obj(lib_obj: CMPBase) -> None: LibConfigRegistry.register(lib_config) -# NOTE: Currently we disable adding library enpoints like numpy, torch when veilid is enabled -# This is because the /api endpoint which return SyftAPI along with the lib enpoints exceeds -# 2 MB . But veilid has a limit of 32 KB for sending and receiving message. -# This would be fixed, when chunking is implemented at veilid core. -if not VEILID_ENABLED: - # hacky, prevent circular imports - for lib_obj in action_execute_registry_libs.flatten(): - # # for functions - # func_name = func.__name__ - # # for classes - # func_name = path.split(".")[-1] - if isinstance(lib_obj, CMPFunction) or isinstance(lib_obj, CMPClass): - register_lib_obj(lib_obj) - - def deconstruct_param(param: inspect.Parameter) -> dict[str, Any]: # Gets the init signature form pydantic object param_type = param.annotation diff --git a/packages/syft/src/syft/service/veilid/__init__.py b/packages/syft/src/syft/service/veilid/__init__.py deleted file mode 100644 index 93f60cd6213..00000000000 --- a/packages/syft/src/syft/service/veilid/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# stdlib -import os -from typing import Any - -# relative -from ...util.util import str_to_bool - -VEILID_ENABLED: bool = str_to_bool(os.environ.get("VEILID_ENABLED", "False")) - - -# Any because circular import -def VeilidServiceProvider(*args: Any, **kwargs: Any) -> Any | None: - if VEILID_ENABLED: - # relative - from .veilid_service import VeilidService - - return VeilidService(*args, **kwargs) - return None diff --git a/packages/syft/src/syft/service/veilid/veilid_endpoints.py b/packages/syft/src/syft/service/veilid/veilid_endpoints.py deleted file mode 100644 index 0e37226dd27..00000000000 --- a/packages/syft/src/syft/service/veilid/veilid_endpoints.py +++ /dev/null @@ -1,8 +0,0 @@ -VEILID_SERVICE_URL = "http://veilid:80" -# Service name of our traefik service -# TODO: Remove this once when we remove reverse proxy in Veilid Connection -VEILID_SYFT_PROXY_URL = "http://proxy:80" -HEALTHCHECK_ENDPOINT = "/healthcheck" -GEN_VLD_KEY_ENDPOINT = "/generate_vld_key" -RET_VLD_KEY_ENDPOINT = "/retrieve_vld_key" -VEILID_PROXY_PATH = "/proxy" diff --git a/packages/syft/src/syft/service/veilid/veilid_service.py b/packages/syft/src/syft/service/veilid/veilid_service.py deleted file mode 100644 index 3fbcd064291..00000000000 --- a/packages/syft/src/syft/service/veilid/veilid_service.py +++ /dev/null @@ -1,92 +0,0 @@ -# stdlib -from collections.abc import Callable - -# third party -import requests - -# relative -from ...serde.serializable import serializable -from ...store.document_store import DocumentStore -from ...util.telemetry import instrument -from ..context import AuthedServiceContext -from ..network.routes import VeilidNodeRoute -from ..response import SyftError -from ..response import SyftSuccess -from ..service import AbstractService -from ..service import service_method -from ..user.user_roles import DATA_OWNER_ROLE_LEVEL -from .veilid_endpoints import GEN_VLD_KEY_ENDPOINT -from .veilid_endpoints import HEALTHCHECK_ENDPOINT -from .veilid_endpoints import RET_VLD_KEY_ENDPOINT -from .veilid_endpoints import VEILID_SERVICE_URL - - -@instrument -@serializable() -class VeilidService(AbstractService): - store: DocumentStore - - def __init__(self, store: DocumentStore) -> None: - self.store = store - - def perform_request( - self, method: Callable, endpoint: str, raw: bool = False - ) -> SyftSuccess | SyftError | str: - try: - response = method(f"{VEILID_SERVICE_URL}{endpoint}") - response.raise_for_status() - message = response.json().get("message") - return message if raw else SyftSuccess(message=message) - except requests.HTTPError: - return SyftError(message=f"{response.json()['detail']}") - except requests.RequestException as e: - return SyftError(message=f"Failed to perform request. {e}") - - def is_veilid_service_healthy(self) -> bool: - res = self.perform_request( - method=requests.get, endpoint=HEALTHCHECK_ENDPOINT, raw=True - ) - return res == "OK" - - @service_method( - path="veilid.generate_vld_key", - name="generate_vld_key", - roles=DATA_OWNER_ROLE_LEVEL, - ) - def generate_vld_key(self, context: AuthedServiceContext) -> str | SyftError: - if not self.is_veilid_service_healthy(): - return SyftError( - message="Veilid service is not healthy. Please try again later." - ) - return self.perform_request( - method=requests.post, - endpoint=GEN_VLD_KEY_ENDPOINT, - ) - - @service_method( - path="veilid.retrieve_vld_key", - name="retrieve_vld_key", - roles=DATA_OWNER_ROLE_LEVEL, - ) - def retrieve_vld_key(self, context: AuthedServiceContext) -> str | SyftError: - if not self.is_veilid_service_healthy(): - return SyftError( - message="Veilid service is not healthy. Please try again later." - ) - return self.perform_request( - method=requests.get, - endpoint=RET_VLD_KEY_ENDPOINT, - raw=True, - ) - - @service_method( - path="veilid.get_veilid_route", - name="get_veilid_route", - ) - def get_veilid_route( - self, context: AuthedServiceContext - ) -> VeilidNodeRoute | SyftError: - vld_key = self.retrieve_vld_key(context) - if isinstance(vld_key, SyftError): - return vld_key - return VeilidNodeRoute(vld_key=vld_key) diff --git a/tests/integration/veilid/gateway_veilid_test.py b/tests/integration/veilid/gateway_veilid_test.py deleted file mode 100644 index fa4e092aefa..00000000000 --- a/tests/integration/veilid/gateway_veilid_test.py +++ /dev/null @@ -1,96 +0,0 @@ -# third party -import pytest - -# syft absolute -import syft as sy -from syft.abstract_node import NodeType -from syft.client.domain_client import DomainClient -from syft.client.gateway_client import GatewayClient -from syft.client.protocol import SyftProtocol -from syft.service.network.node_peer import NodePeer -from syft.service.network.routes import VeilidNodeRoute -from syft.service.response import SyftSuccess -from syft.service.user.user_roles import ServiceRole - - -def remove_existing_peers(client): - for peer in client.api.services.network.get_all_peers(): - res = client.api.services.network.delete_peer_by_id(peer.id) - assert isinstance(res, SyftSuccess) - - -@pytest.mark.skip( - reason="The tests are highly flaky currently.Will be re-enabled soon!" -) -@pytest.mark.veilid -def test_domain_connect_to_gateway_veilid(domain_1_port, gateway_port): - # Revert to the guest login, when we automatically generate the dht key - # gateway_client: GatewayClient = sy.login_as_guest(port=gateway_port) - gateway_client: GatewayClient = sy.login( - port=gateway_port, email="info@openmined.org", password="changethis" - ) - domain_client: DomainClient = sy.login( - port=domain_1_port, email="info@openmined.org", password="changethis" - ) - - # Remove existing peers due to the previous gateway test - remove_existing_peers(domain_client) - remove_existing_peers(gateway_client) - - # Generate DHT Record - gateway_dht_res = gateway_client.api.services.veilid.generate_vld_key() - assert isinstance(gateway_dht_res, SyftSuccess), gateway_dht_res - domain_dht_res = domain_client.api.services.veilid.generate_vld_key() - assert isinstance(domain_dht_res, SyftSuccess), domain_dht_res - - # Retrieve DHT Record - domain_veilid_route = domain_client.api.services.veilid.get_veilid_route() - assert isinstance(domain_veilid_route, VeilidNodeRoute), domain_veilid_route - gateway_veilid_route = gateway_client.api.services.veilid.get_veilid_route() - assert isinstance(gateway_veilid_route, VeilidNodeRoute), gateway_veilid_route - - # Connect Domain to Gateway via Veilid - result = domain_client.connect_to_gateway( - gateway_client, protocol=SyftProtocol.VEILID - ) - assert isinstance(result, SyftSuccess) - - proxy_domain_client = gateway_client.peers[0] - domain_peer = domain_client.peers[0] - gateway_peer = gateway_client.api.services.network.get_all_peers()[0] - - # Domain Asserts - assert len(domain_client.peers) == 1 - assert isinstance(proxy_domain_client, DomainClient) - assert domain_peer.node_type == NodeType.GATEWAY - assert isinstance(domain_peer, NodePeer) - assert isinstance(domain_peer.node_routes[0], VeilidNodeRoute) - assert domain_peer.node_routes[0].vld_key == gateway_veilid_route.vld_key - assert domain_client.name == proxy_domain_client.name - - # Gateway Asserts - assert len(gateway_client.peers) == 1 - assert gateway_peer.node_type == NodeType.DOMAIN - assert isinstance(gateway_peer.node_routes[0], VeilidNodeRoute) - assert gateway_peer.node_routes[0].vld_key == domain_veilid_route.vld_key - assert gateway_client.name == domain_peer.name - assert len(gateway_client.domains) == 1 - assert len(gateway_client.enclaves) == 0 - - # Proxy Domain Asserts - assert proxy_domain_client.metadata == domain_client.metadata - assert proxy_domain_client.user_role == ServiceRole.NONE - - domain_client = domain_client.login( - email="info@openmined.org", password="changethis" - ) - proxy_domain_client = proxy_domain_client.login( - email="info@openmined.org", password="changethis" - ) - - assert proxy_domain_client.logged_in_user == "info@openmined.org" - assert proxy_domain_client.user_role == ServiceRole.ADMIN - assert proxy_domain_client.credentials == domain_client.credentials - assert ( - proxy_domain_client.api.endpoints.keys() == domain_client.api.endpoints.keys() - ) diff --git a/tox.ini b/tox.ini index 19f61268b92..afb901ed177 100644 --- a/tox.ini +++ b/tox.ini @@ -696,10 +696,6 @@ commands = bash -c "source ./scripts/get_k8s_secret_ci.sh; \ pytest tests/integration/network -k 'not test_domain_gateway_user_code' -p no:randomly -vvvv" - # Veilid Integration tests - bash -c "source ./scripts/get_k8s_secret_ci.sh; \ - pytest tests/integration/veilid -p no:randomly -vvvv" - # Shutting down the gateway cluster to free up space, as the # below code does not require gateway cluster bash -c "CLUSTER_NAME=testgateway1 tox -e dev.k8s.destroy || true" From 7d3e5e86076130a89a2b33d0db43dd1a126fe73e Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Mon, 8 Apr 2024 18:01:38 +0530 Subject: [PATCH 2/4] update node peer and enclave metadata classes - write migrations for node peer for dropping veilid route --- .../syft/src/syft/client/enclave_client.py | 4 +- .../src/syft/protocol/protocol_version.json | 26 +++++++++ .../src/syft/service/network/node_peer.py | 53 ++++++++++++++++++- .../syft/src/syft/service/network/routes.py | 20 +++++++ 4 files changed, 100 insertions(+), 3 deletions(-) diff --git a/packages/syft/src/syft/client/enclave_client.py b/packages/syft/src/syft/client/enclave_client.py index afd01336e77..7a29b37fd7b 100644 --- a/packages/syft/src/syft/client/enclave_client.py +++ b/packages/syft/src/syft/client/enclave_client.py @@ -17,7 +17,7 @@ from ..service.network.routes import NodeRouteType from ..service.response import SyftError from ..service.response import SyftSuccess -from ..types.syft_object import SYFT_OBJECT_VERSION_2 +from ..types.syft_object import SYFT_OBJECT_VERSION_3 from ..types.syft_object import SyftObject from ..types.uid import UID from ..util.fonts import FONT_CSS @@ -35,7 +35,7 @@ @serializable() class EnclaveMetadata(SyftObject): __canonical_name__ = "EnclaveMetadata" - __version__ = SYFT_OBJECT_VERSION_2 + __version__ = SYFT_OBJECT_VERSION_3 route: NodeRouteType diff --git a/packages/syft/src/syft/protocol/protocol_version.json b/packages/syft/src/syft/protocol/protocol_version.json index e708e4a178d..4e587772a00 100644 --- a/packages/syft/src/syft/protocol/protocol_version.json +++ b/packages/syft/src/syft/protocol/protocol_version.json @@ -111,6 +111,32 @@ "hash": "4dbfa0813f5a3f7be0b36249ff2d67e395ad7c9e138c5a122fc7342b8dcc4b92", "action": "remove" } + }, + "EnclaveMetadata": { + "2": { + "version": 2, + "hash": "6dcc26695abc6a9ecd9d7d1e6507a9f1a92cc5ccd10987e92419bf984245f9a1", + "action": "remove" + }, + "3": { + "version": 3, + "hash": "d2f23411927c68e2307a84d180ad053b3e4ba12d74aba64d34dac224c90e815d", + "action": "add" + } + }, + "NodePeer": { + "3": { + "version": 3, + "hash": "dababb03d2463b6218ae22d55293a60580f5a14bebd0c664d71da104e2f0b835", + "action": "add" + } + }, + "VeilidConnection": { + "1": { + "version": 1, + "hash": "c1796e7b01c9eae0dbf59cfd5c2c2f0e7eba593e0cea615717246572b27aae4b", + "action": "remove" + } } } } diff --git a/packages/syft/src/syft/service/network/node_peer.py b/packages/syft/src/syft/service/network/node_peer.py index b6aa18ca9bf..ecf4c08193c 100644 --- a/packages/syft/src/syft/service/network/node_peer.py +++ b/packages/syft/src/syft/service/network/node_peer.py @@ -1,5 +1,8 @@ # stdlib +# stdlib +from collections.abc import Callable + # relative from ...abstract_node import NodeType from ...client.client import SyftClient @@ -7,21 +10,26 @@ from ...node.credentials import SyftVerifyKey from ...serde.serializable import serializable from ...service.response import SyftError +from ...types.syft_migration import migrate from ...types.syft_object import SYFT_OBJECT_VERSION_2 +from ...types.syft_object import SYFT_OBJECT_VERSION_3 from ...types.syft_object import SyftObject +from ...types.transforms import TransformContext from ...types.uid import UID from ..context import NodeServiceContext from ..metadata.node_metadata import NodeMetadataV3 from .routes import HTTPNodeRoute from .routes import NodeRoute from .routes import NodeRouteType +from .routes import NodeRouteTypeV1 from .routes import PythonNodeRoute +from .routes import VeilidNodeRoute from .routes import connection_to_route from .routes import route_to_connection @serializable() -class NodePeer(SyftObject): +class NodePeerV2(SyftObject): # version __canonical_name__ = "NodePeer" __version__ = SYFT_OBJECT_VERSION_2 @@ -30,6 +38,24 @@ class NodePeer(SyftObject): __attr_unique__ = ["verify_key"] __repr_attrs__ = ["name", "node_type", "admin_email"] + id: UID | None = None # type: ignore[assignment] + name: str + verify_key: SyftVerifyKey + node_routes: list[NodeRouteTypeV1] = [] + node_type: NodeType + admin_email: str + + +@serializable() +class NodePeer(SyftObject): + # version + __canonical_name__ = "NodePeer" + __version__ = SYFT_OBJECT_VERSION_3 + + __attr_searchable__ = ["name", "node_type"] + __attr_unique__ = ["verify_key"] + __repr_attrs__ = ["name", "node_type", "admin_email"] + id: UID | None = None # type: ignore[assignment] name: str verify_key: SyftVerifyKey @@ -150,3 +176,28 @@ def pick_highest_priority_route(self) -> NodeRoute: if route.priority > final_route.priority: final_route = route return final_route + + +def drop_veilid_route() -> Callable: + def _drop_veilid_route(context: TransformContext) -> TransformContext: + if context.output: + node_routes = context.output["node_routes"] + new_routes = [ + node_route + for node_route in node_routes + if not isinstance(node_route, VeilidNodeRoute) + ] + context.output["node_routes"] = new_routes + return context + + return _drop_veilid_route + + +@migrate(NodePeerV2, NodePeer) +def upgrade_node_peer() -> list[Callable]: + return [drop_veilid_route()] + + +@migrate(NodePeerV2, NodePeer) +def downgrade_node_peer() -> list[Callable]: + return [] diff --git a/packages/syft/src/syft/service/network/routes.py b/packages/syft/src/syft/service/network/routes.py index c357bd6d26b..eea17b6b835 100644 --- a/packages/syft/src/syft/service/network/routes.py +++ b/packages/syft/src/syft/service/network/routes.py @@ -17,6 +17,7 @@ from ...client.client import SyftClient from ...node.worker_settings import WorkerSettings from ...serde.serializable import serializable +from ...types.syft_object import SYFT_OBJECT_VERSION_1 from ...types.syft_object import SYFT_OBJECT_VERSION_2 from ...types.syft_object import SyftObject from ...types.transforms import TransformContext @@ -129,6 +130,25 @@ def __hash__(self) -> int: return hash(self.worker_settings.id) +@serializable() +class VeilidNodeRoute(SyftObject, NodeRoute): + __canonical_name__ = "VeilidNodeRoute" + __version__ = SYFT_OBJECT_VERSION_1 + + vld_key: str + proxy_target_uid: UID | None = None + priority: int = 1 + + def __eq__(self, other: Any) -> bool: + if isinstance(other, VeilidNodeRoute): + return hash(self) == hash(other) + return self == other + + def __hash__(self) -> int: + return hash(self.vld_key) + + +NodeRouteTypeV1 = HTTPNodeRoute | PythonNodeRoute | VeilidNodeRoute NodeRouteType = HTTPNodeRoute | PythonNodeRoute From d755aa4d173011a52faad0bda4ff7c731509893f Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Mon, 8 Apr 2024 18:18:56 +0530 Subject: [PATCH 3/4] services:revert removing numpy library endpoint registration in SyftAPI --- packages/syft/src/syft/service/service.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/syft/src/syft/service/service.py b/packages/syft/src/syft/service/service.py index 4e3c41ed014..d9157a4be5f 100644 --- a/packages/syft/src/syft/service/service.py +++ b/packages/syft/src/syft/service/service.py @@ -21,6 +21,9 @@ from ..serde.lib_permissions import CMPCRUDPermission from ..serde.lib_permissions import CMPPermission from ..serde.lib_service_registry import CMPBase +from ..serde.lib_service_registry import CMPClass +from ..serde.lib_service_registry import CMPFunction +from ..serde.lib_service_registry import action_execute_registry_libs from ..serde.serializable import serializable from ..serde.signature import Signature from ..serde.signature import signature_remove_context @@ -223,6 +226,15 @@ def register_lib_obj(lib_obj: CMPBase) -> None: LibConfigRegistry.register(lib_config) + # hacky, prevent circular imports + for lib_obj in action_execute_registry_libs.flatten(): + # # for functions + # func_name = func.__name__ + # # for classes + # func_name = path.split(".")[-1] + if isinstance(lib_obj, CMPFunction) or isinstance(lib_obj, CMPClass): + register_lib_obj(lib_obj) + def deconstruct_param(param: inspect.Parameter) -> dict[str, Any]: # Gets the init signature form pydantic object From 049e1f064af76cad82182b1746c8d9a4cbfde7c7 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Tue, 9 Apr 2024 14:52:09 +0530 Subject: [PATCH 4/4] fix scope of loading the registering the action based lib objects --- packages/syft/src/syft/service/service.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/packages/syft/src/syft/service/service.py b/packages/syft/src/syft/service/service.py index d9157a4be5f..fee806c440c 100644 --- a/packages/syft/src/syft/service/service.py +++ b/packages/syft/src/syft/service/service.py @@ -226,14 +226,15 @@ def register_lib_obj(lib_obj: CMPBase) -> None: LibConfigRegistry.register(lib_config) - # hacky, prevent circular imports - for lib_obj in action_execute_registry_libs.flatten(): - # # for functions - # func_name = func.__name__ - # # for classes - # func_name = path.split(".")[-1] - if isinstance(lib_obj, CMPFunction) or isinstance(lib_obj, CMPClass): - register_lib_obj(lib_obj) + +# hacky, prevent circular imports +for lib_obj in action_execute_registry_libs.flatten(): + # # for functions + # func_name = func.__name__ + # # for classes + # func_name = path.split(".")[-1] + if isinstance(lib_obj, CMPFunction) or isinstance(lib_obj, CMPClass): + register_lib_obj(lib_obj) def deconstruct_param(param: inspect.Parameter) -> dict[str, Any]: