diff --git a/.bumpversion.cfg b/.bumpversion.cfg index e6051cb595f..474ab191d46 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.8.2-beta.27 +current_version = 0.8.2-beta.28 tag = False tag_name = {new_version} commit = True diff --git a/.github/workflows/pr-tests-enclave.yml b/.github/workflows/pr-tests-enclave.yml index 321bbea0726..7c6ee71cb6d 100644 --- a/.github/workflows/pr-tests-enclave.yml +++ b/.github/workflows/pr-tests-enclave.yml @@ -70,7 +70,6 @@ jobs: if: steps.changes.outputs.syft == 'true' run: | pip install --upgrade tox packaging wheel --default-timeout=60 - python -c "import platform; import os; os.system('pip install jaxlib==0.4.10 -f https://whls.blob.core.windows.net/unstable/index.html') if platform.system().lower() == 'windows' else ''" # Temporarily disabled oblv tests # - name: Run Enclave tests diff --git a/.github/workflows/pr-tests-stack-public.yml b/.github/workflows/pr-tests-stack-public.yml index 5d8d4b71811..6658e470b45 100644 --- a/.github/workflows/pr-tests-stack-public.yml +++ b/.github/workflows/pr-tests-stack-public.yml @@ -106,11 +106,6 @@ jobs: with: args: install visualstudio2019-workload-vctools -y - - name: Install jaxlib - if: steps.changes.outputs.stack == 'true' && matrix.os == 'windows' - run: | - python -c "import platform; import os; os.system('pip install jaxlib==0.4.10 -f https://whls.blob.core.windows.net/unstable/index.html') if platform.system().lower() == 'windows' else ''" - - name: Install Docker Compose if: steps.changes.outputs.stack == 'true' && runner.os == 'Linux' shell: bash diff --git a/.github/workflows/pr-tests-stack.yml b/.github/workflows/pr-tests-stack.yml index 9e9b4a913d7..2dcc1ec2f0e 100644 --- a/.github/workflows/pr-tests-stack.yml +++ b/.github/workflows/pr-tests-stack.yml @@ -118,11 +118,6 @@ jobs: with: args: install visualstudio2019-workload-vctools -y - - name: Install jaxlib - if: steps.changes.outputs.stack == 'true' && matrix.os == 'windows' - run: | - python -c "import platform; import os; os.system('pip install jaxlib==0.4.10 -f https://whls.blob.core.windows.net/unstable/index.html') if platform.system().lower() == 'windows' else ''" - - name: Install Docker Compose if: steps.changes.outputs.stack == 'true' && runner.os == 'Linux' shell: bash @@ -334,11 +329,6 @@ jobs: with: args: install visualstudio2019-workload-vctools -y - - name: Install jaxlib - if: steps.changes.outputs.stack == 'true' && matrix.os == 'windows' - run: | - python -c "import platform; import os; os.system('pip install jaxlib==0.4.10 -f https://whls.blob.core.windows.net/unstable/index.html') if platform.system().lower() == 'windows' else ''" - - name: Install Docker Compose if: steps.changes.outputs.stack == 'true' && runner.os == 'Linux' shell: bash diff --git a/.github/workflows/pr-tests-syft.yml b/.github/workflows/pr-tests-syft.yml index 1e4287ae400..520344fe39f 100644 --- a/.github/workflows/pr-tests-syft.yml +++ b/.github/workflows/pr-tests-syft.yml @@ -87,7 +87,6 @@ jobs: if: steps.changes.outputs.syft == 'true' run: | pip install --upgrade tox packaging wheel --default-timeout=60 - python -c "import platform; import os; os.system('pip install jaxlib==0.4.10 -f https://whls.blob.core.windows.net/unstable/index.html') if platform.system().lower() == 'windows' else ''" - name: Fix Colima issue on MacOS Runners if: steps.changes.outputs.syft == 'true' && matrix.os == 'macos-13' @@ -179,7 +178,6 @@ jobs: if: steps.changes.outputs.syft == 'true' || steps.changes.outputs.notebooks == 'true' run: | pip install --upgrade tox packaging wheel --default-timeout=60 - python -c "import platform; import os; os.system('pip install jaxlib==0.4.10 -f https://whls.blob.core.windows.net/unstable/index.html') if platform.system().lower() == 'windows' else ''" - name: Run notebook tests uses: nick-fields/retry@v2 @@ -249,7 +247,6 @@ jobs: if: steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true' run: | pip install --upgrade tox packaging wheel --default-timeout=60 - python -c "import platform; import os; os.system('pip install jaxlib==0.4.10 -f https://whls.blob.core.windows.net/unstable/index.html') if platform.system().lower() == 'windows' else ''" - name: Docker Compose on Linux if: (steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true') && matrix.os == 'ubuntu-latest' @@ -356,7 +353,6 @@ jobs: if: steps.changes.outputs.syft == 'true' run: | pip install --upgrade tox packaging wheel --default-timeout=60 - python -c "import platform; import os; os.system('pip install jaxlib==0.4.10 -f https://whls.blob.core.windows.net/unstable/index.html') if platform.system().lower() == 'windows' else ''" - name: Scan for security issues if: steps.changes.outputs.syft == 'true' diff --git a/README.md b/README.md index cce9d3cb22a..a8dd419645f 100644 --- a/README.md +++ b/README.md @@ -10,12 +10,12 @@ Perform data science on `data` that remains in `someone else's` server # Quickstart -✅ `Linux` ✅ `macOS` ✅ `Windows`\* ✅ `Docker` ✅ `Kubernetes` +✅ `Linux` ✅ `macOS` ✅ `Windows` ✅ `Docker` ✅ `Podman` ✅ `Kubernetes` ## Install Client ```bash -$ pip install -U syft -f https://whls.blob.core.windows.net/unstable/index.html +$ pip install -U syft ``` ## Launch Server @@ -93,9 +93,8 @@ $ helm install ... --set ingress.ingressClass="gce" - HAGrid 0.3 Requires: 🐍 `python` 🐙 `git` - Run: `pip install -U hagrid` - Interactive Install 🧙🏽‍♂️ WizardBETA Requires 🛵 `hagrid`: - Run: `hagrid quickstart` -- PySyft 0.8.1 Requires: 🐍 `python 3.9 - 3.11` - Run: `pip install -U syft` - \*`Windows` users must run this first: `pip install jaxlib==0.4.10 -f https://whls.blob.core.windows.net/unstable/index.html` -- PyGrid Requires: 🐳 `docker`, ☸️ `kubernetes` or 🐧 `ubuntu` VM - Run: `hagrid launch ...` +- PySyft 0.8.1 Requires: 🐍 `python 3.9 - 3.11` - Run: `pip install -U syft` +- PyGrid Requires: 🐳 `docker`, 🦦 `podman` or ☸️ `kubernetes` - Run: `hagrid launch ...` # Versions diff --git a/VERSION b/VERSION index 6d7567fd24a..7f3f0571209 100644 --- a/VERSION +++ b/VERSION @@ -1,5 +1,5 @@ # Mono Repo Global Version -__version__ = "0.8.2-beta.27" +__version__ = "0.8.2-beta.28" # elsewhere we can call this file: `python VERSION` and simply take the stdout # stdlib diff --git a/notebooks/api/0.8/00-load-data.ipynb b/notebooks/api/0.8/00-load-data.ipynb index ec1586da9bb..74f0eb9d10f 100644 --- a/notebooks/api/0.8/00-load-data.ipynb +++ b/notebooks/api/0.8/00-load-data.ipynb @@ -32,7 +32,7 @@ "source": [ "SYFT_VERSION = \">=0.8.2.b0,<0.9\"\n", "package_string = f'\"syft{SYFT_VERSION}\"'\n", - "# %pip install {package_string} -f https://whls.blob.core.windows.net/unstable/index.html -q" + "# %pip install {package_string} -q" ] }, { diff --git a/notebooks/api/0.8/01-submit-code.ipynb b/notebooks/api/0.8/01-submit-code.ipynb index 3850320f18e..30f4aa54864 100644 --- a/notebooks/api/0.8/01-submit-code.ipynb +++ b/notebooks/api/0.8/01-submit-code.ipynb @@ -24,7 +24,7 @@ "source": [ "SYFT_VERSION = \">=0.8.2.b0,<0.9\"\n", "package_string = f'\"syft{SYFT_VERSION}\"'\n", - "# %pip install {package_string} -f https://whls.blob.core.windows.net/unstable/index.html -q" + "# %pip install {package_string} -q" ] }, { @@ -295,11 +295,15 @@ "@sy.syft_function_single_use(trade_data=asset)\n", "def sum_trade_value_mil(trade_data):\n", " import pandas as pd\n", - " from opendp.mod import enable_features\n", - " enable_features('contrib')\n", - " from opendp.measurements import make_base_laplace\n", + " import opendp.prelude as dp\n", + " dp.enable_features(\"contrib\")\n", + " from opendp.measurements import make_laplace\n", " aggregate = 0.\n", - " base_lap = make_base_laplace(scale=5.)\n", + " base_lap = dp.m.make_base_laplace(\n", + " dp.atom_domain(T=float),\n", + " dp.absolute_distance(T=float),\n", + " scale=5.\n", + " )\n", " noise = base_lap(aggregate)\n", "\n", " df = trade_data\n", diff --git a/notebooks/api/0.8/02-review-code-and-approve.ipynb b/notebooks/api/0.8/02-review-code-and-approve.ipynb index d00d15ad166..9becd45b484 100644 --- a/notebooks/api/0.8/02-review-code-and-approve.ipynb +++ b/notebooks/api/0.8/02-review-code-and-approve.ipynb @@ -24,7 +24,7 @@ "source": [ "SYFT_VERSION = \">=0.8.2.b0,<0.9\"\n", "package_string = f'\"syft{SYFT_VERSION}\"'\n", - "# %pip install {package_string} -f https://whls.blob.core.windows.net/unstable/index.html -q" + "# %pip install {package_string} -q" ] }, { diff --git a/notebooks/api/0.8/03-data-scientist-download-result.ipynb b/notebooks/api/0.8/03-data-scientist-download-result.ipynb index 576badbafd8..1f05196dcc9 100644 --- a/notebooks/api/0.8/03-data-scientist-download-result.ipynb +++ b/notebooks/api/0.8/03-data-scientist-download-result.ipynb @@ -24,7 +24,7 @@ "source": [ "SYFT_VERSION = \">=0.8.2.b0,<0.9\"\n", "package_string = f'\"syft{SYFT_VERSION}\"'\n", - "# %pip install {package_string} -f https://whls.blob.core.windows.net/unstable/index.html -q" + "# %pip install {package_string} -q" ] }, { diff --git a/notebooks/api/0.8/04-jax-example.ipynb b/notebooks/api/0.8/04-jax-example.ipynb index e12bfd16897..54ff7a41eaf 100644 --- a/notebooks/api/0.8/04-jax-example.ipynb +++ b/notebooks/api/0.8/04-jax-example.ipynb @@ -11,7 +11,7 @@ "source": [ "SYFT_VERSION = \">=0.8.2.b0,<0.9\"\n", "package_string = f'\"syft{SYFT_VERSION}\"'\n", - "# %pip install {package_string} -f https://whls.blob.core.windows.net/unstable/index.html -q" + "# %pip install {package_string} -q" ] }, { diff --git a/notebooks/api/0.8/05-custom-policy.ipynb b/notebooks/api/0.8/05-custom-policy.ipynb index e837464fa34..b223311365b 100644 --- a/notebooks/api/0.8/05-custom-policy.ipynb +++ b/notebooks/api/0.8/05-custom-policy.ipynb @@ -11,7 +11,7 @@ "source": [ "SYFT_VERSION = \">=0.8.2.b0,<0.9\"\n", "package_string = f'\"syft{SYFT_VERSION}\"'\n", - "# %pip install {package_string} -f https://whls.blob.core.windows.net/unstable/index.html -q" + "# %pip install {package_string} -q" ] }, { diff --git a/notebooks/api/0.8/06-multiple-code-requests.ipynb b/notebooks/api/0.8/06-multiple-code-requests.ipynb index 89a9bb4a80b..c6a83ba21a5 100644 --- a/notebooks/api/0.8/06-multiple-code-requests.ipynb +++ b/notebooks/api/0.8/06-multiple-code-requests.ipynb @@ -11,7 +11,7 @@ "source": [ "SYFT_VERSION = \">=0.8.2.b0,<0.9\"\n", "package_string = f'\"syft{SYFT_VERSION}\"'\n", - "# %pip install {package_string} -f https://whls.blob.core.windows.net/unstable/index.html -q" + "# %pip install {package_string} -q" ] }, { @@ -170,16 +170,17 @@ "@sy.syft_function(input_policy=sy.ExactMatch(data=dataset_ptr.assets[0]),\n", " output_policy=sy.SingleExecutionExactOutput())\n", "def calculate_sum(data):\n", - "\n", - " \n", - " from opendp.mod import enable_features\n", - " enable_features('contrib')\n", - " from opendp.measurements import make_base_laplace\n", - " \n", + " import opendp.prelude as dp\n", + " dp.enable_features(\"contrib\")\n", + " from opendp.measurements import make_laplace\n", " \n", " #compute sum\n", " res = data.sum()\n", - " base_lap = make_base_laplace(scale=10.)\n", + " base_lap = dp.m.make_base_laplace(\n", + " dp.atom_domain(T=float),\n", + " dp.absolute_distance(T=float),\n", + " scale=10.\n", + " )\n", " noise_result = base_lap(res)\n", " return noise_result" ] @@ -265,14 +266,17 @@ "@sy.syft_function(input_policy=sy.ExactMatch(data=dataset_ptr.assets[0]),\n", " output_policy=sy.SingleExecutionExactOutput())\n", "def calculate_mean(data):\n", - " from opendp.mod import enable_features\n", - " enable_features('contrib')\n", - " from opendp.measurements import make_base_laplace\n", - " \n", + " import opendp.prelude as dp\n", + " dp.enable_features(\"contrib\")\n", + " from opendp.measurements import make_laplace\n", " \n", " #compute mean\n", " mean = data.mean()\n", - " base_lap = make_base_laplace(scale=10.)\n", + " base_lap = dp.m.make_base_laplace(\n", + " dp.atom_domain(T=float),\n", + " dp.absolute_distance(T=float),\n", + " scale=10.\n", + " )\n", " noise_result = base_lap(mean)\n", " return noise_result" ] @@ -311,7 +315,7 @@ }, "outputs": [], "source": [ - "assert len(project.events) ==2\n", + "assert len(project.events) == 2\n", "assert isinstance(project.events[1], sy.service.project.project.ProjectRequest)" ] }, @@ -335,7 +339,7 @@ "outputs": [], "source": [ "root_client_project.requests\n", - "assert len(root_client_project.requests) ==2" + "assert len(root_client_project.requests) == 2" ] }, { diff --git a/notebooks/api/0.8/07-domain-register-control-flow.ipynb b/notebooks/api/0.8/07-domain-register-control-flow.ipynb index a61d1181a3c..ff319fa8ea3 100644 --- a/notebooks/api/0.8/07-domain-register-control-flow.ipynb +++ b/notebooks/api/0.8/07-domain-register-control-flow.ipynb @@ -27,7 +27,7 @@ "source": [ "SYFT_VERSION = \">=0.8.2.b0,<0.9\"\n", "package_string = f'\"syft{SYFT_VERSION}\"'\n", - "# %pip install {package_string} -f https://whls.blob.core.windows.net/unstable/index.html -q" + "# %pip install {package_string} -q" ] }, { diff --git a/notebooks/api/0.8/08-code-version.ipynb b/notebooks/api/0.8/08-code-version.ipynb index 4fff29aaf0e..a71500ef06c 100644 --- a/notebooks/api/0.8/08-code-version.ipynb +++ b/notebooks/api/0.8/08-code-version.ipynb @@ -22,7 +22,7 @@ "source": [ "SYFT_VERSION = \">=0.8.2.b0,<0.9\"\n", "package_string = f'\"syft{SYFT_VERSION}\"'\n", - "%pip install {package_string} -f https://whls.blob.core.windows.net/unstable/index.html -q" + "%pip install {package_string} -q" ] }, { diff --git a/notebooks/api/0.8/09-blob-storage.ipynb b/notebooks/api/0.8/09-blob-storage.ipynb index 2a3cfbc78be..6f67340b608 100644 --- a/notebooks/api/0.8/09-blob-storage.ipynb +++ b/notebooks/api/0.8/09-blob-storage.ipynb @@ -8,7 +8,7 @@ "source": [ "SYFT_VERSION = \">=0.8.2.b0,<0.9\"\n", "package_string = f'\"syft{SYFT_VERSION}\"'\n", - "# %pip install {package_string} -f https://whls.blob.core.windows.net/unstable/index.html -q" + "# %pip install {package_string} -q" ] }, { diff --git a/notebooks/tutorials/data-owner/01-uploading-private-data.ipynb b/notebooks/tutorials/data-owner/01-uploading-private-data.ipynb index be631ef48f3..f63a3c81d31 100644 --- a/notebooks/tutorials/data-owner/01-uploading-private-data.ipynb +++ b/notebooks/tutorials/data-owner/01-uploading-private-data.ipynb @@ -27,7 +27,7 @@ "source": [ "SYFT_VERSION = \">=0.8.2.b0,<0.9\"\n", "package_string = f'\"syft{SYFT_VERSION}\"'\n", - "# %pip install {package_string} -f https://whls.blob.core.windows.net/unstable/index.html" + "# %pip install {package_string} -q" ] }, { diff --git a/notebooks/tutorials/data-owner/02-account-management.ipynb b/notebooks/tutorials/data-owner/02-account-management.ipynb index 0e5708baea5..7b697d1bacb 100644 --- a/notebooks/tutorials/data-owner/02-account-management.ipynb +++ b/notebooks/tutorials/data-owner/02-account-management.ipynb @@ -25,7 +25,7 @@ "source": [ "SYFT_VERSION = \">=0.8.2.b0,<0.9\"\n", "package_string = f'\"syft{SYFT_VERSION}\"'\n", - "# %pip install {package_string} -f https://whls.blob.core.windows.net/unstable/index.html" + "# %pip install {package_string} -q" ] }, { diff --git a/notebooks/tutorials/data-owner/03-messages-and-requests.ipynb b/notebooks/tutorials/data-owner/03-messages-and-requests.ipynb index 24839611c0d..9a2f97475b7 100644 --- a/notebooks/tutorials/data-owner/03-messages-and-requests.ipynb +++ b/notebooks/tutorials/data-owner/03-messages-and-requests.ipynb @@ -25,7 +25,7 @@ "source": [ "SYFT_VERSION = \">=0.8.2.b0,<0.9\"\n", "package_string = f'\"syft{SYFT_VERSION}\"'\n", - "# %pip install {package_string} -f https://whls.blob.core.windows.net/unstable/index.html" + "# %pip install {package_string} -q" ] }, { diff --git a/notebooks/tutorials/data-owner/05-syft-services-api.ipynb b/notebooks/tutorials/data-owner/05-syft-services-api.ipynb index cd6bd9970e0..522171fc455 100644 --- a/notebooks/tutorials/data-owner/05-syft-services-api.ipynb +++ b/notebooks/tutorials/data-owner/05-syft-services-api.ipynb @@ -25,7 +25,7 @@ "source": [ "SYFT_VERSION = \">=0.8.2.b0,<0.9\"\n", "package_string = f'\"syft{SYFT_VERSION}\"'\n", - "# %pip install {package_string} -f https://whls.blob.core.windows.net/unstable/index.html" + "# %pip install {package_string} -q" ] }, { diff --git a/notebooks/tutorials/data-scientist/01-installing-syft-client.ipynb b/notebooks/tutorials/data-scientist/01-installing-syft-client.ipynb index 626772a034d..04684668f9c 100644 --- a/notebooks/tutorials/data-scientist/01-installing-syft-client.ipynb +++ b/notebooks/tutorials/data-scientist/01-installing-syft-client.ipynb @@ -75,14 +75,12 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "77e1b18d", "metadata": {}, "source": [ - "`Linux`, `macOS` and `Windows` are supported.\n", - "\n", - "* `macOS` Apple Silicon users might need cmake: `brew install cmake` \n", - "* `Windows` users must run this first: `pip install jaxlib==0.4.10 -f https://whls.blob.core.windows.net/unstable/index.html`" + "`Linux`, `macOS` and `Windows` are supported." ] }, { diff --git a/notebooks/tutorials/data-scientist/03-working-with-private-datasets.ipynb b/notebooks/tutorials/data-scientist/03-working-with-private-datasets.ipynb index 341d628b167..8f046a79aaa 100644 --- a/notebooks/tutorials/data-scientist/03-working-with-private-datasets.ipynb +++ b/notebooks/tutorials/data-scientist/03-working-with-private-datasets.ipynb @@ -25,7 +25,7 @@ "source": [ "SYFT_VERSION = \">=0.8.2.b0,<0.9\"\n", "package_string = f'\"syft{SYFT_VERSION}\"'\n", - "# %pip install {package_string} -f https://whls.blob.core.windows.net/unstable/index.html" + "# %pip install {package_string} -q" ] }, { diff --git a/notebooks/tutorials/data-scientist/05-syft-functions.ipynb b/notebooks/tutorials/data-scientist/05-syft-functions.ipynb index 59605ec2e90..b88c685616a 100644 --- a/notebooks/tutorials/data-scientist/05-syft-functions.ipynb +++ b/notebooks/tutorials/data-scientist/05-syft-functions.ipynb @@ -25,7 +25,7 @@ "source": [ "SYFT_VERSION = \">=0.8.2.b0,<0.9\"\n", "package_string = f'\"syft{SYFT_VERSION}\"'\n", - "# %pip install {package_string} -f https://whls.blob.core.windows.net/unstable/index.html" + "# %pip install {package_string} -q" ] }, { @@ -198,7 +198,7 @@ "source": [ "SYFT_VERSION = \">=0.8.2.b0,<0.9\"\n", "package_string = f'\"syft{SYFT_VERSION}\"'\n", - "# %pip install {package_string} -f https://whls.blob.core.windows.net/unstable/index.html" + "# %pip install {package_string} -q" ] }, { diff --git a/notebooks/tutorials/data-scientist/06-messaging-and-requests.ipynb b/notebooks/tutorials/data-scientist/06-messaging-and-requests.ipynb index ad79bcf5681..b65fe75d404 100644 --- a/notebooks/tutorials/data-scientist/06-messaging-and-requests.ipynb +++ b/notebooks/tutorials/data-scientist/06-messaging-and-requests.ipynb @@ -25,7 +25,7 @@ "source": [ "SYFT_VERSION = \">=0.8.2.b0,<0.9\"\n", "package_string = f'\"syft{SYFT_VERSION}\"'\n", - "# %pip install {package_string} -f https://whls.blob.core.windows.net/unstable/index.html" + "# %pip install {package_string} -q" ] }, { diff --git a/notebooks/tutorials/hello-syft/01-hello-syft.ipynb b/notebooks/tutorials/hello-syft/01-hello-syft.ipynb index 430c7d2de6c..01f89edc93f 100644 --- a/notebooks/tutorials/hello-syft/01-hello-syft.ipynb +++ b/notebooks/tutorials/hello-syft/01-hello-syft.ipynb @@ -58,7 +58,7 @@ "source": [ "SYFT_VERSION = \">=0.8.2.b0,<0.9\"\n", "package_string = f'\"syft{SYFT_VERSION}\"'\n", - "# %pip install {package_string} -f https://whls.blob.core.windows.net/unstable/index.html" + "# %pip install {package_string} -q" ] }, { diff --git a/notebooks/tutorials/model-auditing/colab/01-user-log.ipynb b/notebooks/tutorials/model-auditing/colab/01-user-log.ipynb index 01fc291670c..4620c220e86 100644 --- a/notebooks/tutorials/model-auditing/colab/01-user-log.ipynb +++ b/notebooks/tutorials/model-auditing/colab/01-user-log.ipynb @@ -46,7 +46,7 @@ "# install syft\n", "SYFT_VERSION = \">=0.8.2.b0,<0.9\"\n", "package_string = f'\"syft{SYFT_VERSION}\"'\n", - "%pip install {package_string} -f https://whls.blob.core.windows.net/unstable/index.html -q" + "%pip install {package_string} -q" ] }, { diff --git a/notebooks/tutorials/pandas-cookbook/01-reading-from-a-csv.ipynb b/notebooks/tutorials/pandas-cookbook/01-reading-from-a-csv.ipynb index 7f0c1695184..ce844ee7f47 100644 --- a/notebooks/tutorials/pandas-cookbook/01-reading-from-a-csv.ipynb +++ b/notebooks/tutorials/pandas-cookbook/01-reading-from-a-csv.ipynb @@ -27,7 +27,7 @@ "source": [ "SYFT_VERSION = \">=0.8.2.b0,<0.9\"\n", "package_string = f'\"syft{SYFT_VERSION}\"'\n", - "# %pip install {package_string} -f https://whls.blob.core.windows.net/unstable/index.html" + "# %pip install {package_string} -q" ] }, { diff --git a/notebooks/tutorials/pandas-cookbook/02-selecting-data-finding-common-complain.ipynb b/notebooks/tutorials/pandas-cookbook/02-selecting-data-finding-common-complain.ipynb index 51b948a6de2..2b75c070ea3 100644 --- a/notebooks/tutorials/pandas-cookbook/02-selecting-data-finding-common-complain.ipynb +++ b/notebooks/tutorials/pandas-cookbook/02-selecting-data-finding-common-complain.ipynb @@ -27,7 +27,7 @@ "source": [ "SYFT_VERSION = \">=0.8.2.b0,<0.9\"\n", "package_string = f'\"syft{SYFT_VERSION}\"'\n", - "# %pip install {package_string} -f https://whls.blob.core.windows.net/unstable/index.html -q" + "# %pip install {package_string} -q" ] }, { diff --git a/notebooks/tutorials/pandas-cookbook/03-which-borough-has-the-most-noise-complaints.ipynb b/notebooks/tutorials/pandas-cookbook/03-which-borough-has-the-most-noise-complaints.ipynb index bb80d83dcbf..2f3e20955be 100644 --- a/notebooks/tutorials/pandas-cookbook/03-which-borough-has-the-most-noise-complaints.ipynb +++ b/notebooks/tutorials/pandas-cookbook/03-which-borough-has-the-most-noise-complaints.ipynb @@ -27,7 +27,7 @@ "source": [ "SYFT_VERSION = \">=0.8.2.b0,<0.9\"\n", "package_string = f'\"syft{SYFT_VERSION}\"'\n", - "# %pip install {package_string} -f https://whls.blob.core.windows.net/unstable/index.html -q" + "# %pip install {package_string} -q" ] }, { diff --git a/notebooks/tutorials/pandas-cookbook/04-weekday-bike-most-groupby-aggregate.ipynb b/notebooks/tutorials/pandas-cookbook/04-weekday-bike-most-groupby-aggregate.ipynb index 2e3683cd564..ed59b274c64 100644 --- a/notebooks/tutorials/pandas-cookbook/04-weekday-bike-most-groupby-aggregate.ipynb +++ b/notebooks/tutorials/pandas-cookbook/04-weekday-bike-most-groupby-aggregate.ipynb @@ -19,7 +19,7 @@ "source": [ "SYFT_VERSION = \">=0.8.2.b0,<0.9\"\n", "package_string = f'\"syft{SYFT_VERSION}\"'\n", - "# %pip install {package_string} -f https://whls.blob.core.windows.net/unstable/index.html -q" + "# %pip install {package_string} -q" ] }, { diff --git a/notebooks/tutorials/pandas-cookbook/05-combining-dataframes-scraping-weather-data.ipynb b/notebooks/tutorials/pandas-cookbook/05-combining-dataframes-scraping-weather-data.ipynb index 54c8b8a2240..1fc3212aaa2 100644 --- a/notebooks/tutorials/pandas-cookbook/05-combining-dataframes-scraping-weather-data.ipynb +++ b/notebooks/tutorials/pandas-cookbook/05-combining-dataframes-scraping-weather-data.ipynb @@ -27,7 +27,7 @@ "source": [ "SYFT_VERSION = \">=0.8.2.b0,<0.9\"\n", "package_string = f'\"syft{SYFT_VERSION}\"'\n", - "# %pip install {package_string} -f https://whls.blob.core.windows.net/unstable/index.html -q" + "# %pip install {package_string} -q" ] }, { diff --git a/notebooks/tutorials/pandas-cookbook/06-string-operations-which-month-was-the-snowiest.ipynb b/notebooks/tutorials/pandas-cookbook/06-string-operations-which-month-was-the-snowiest.ipynb index 3794877ab03..0a639bcb2b2 100644 --- a/notebooks/tutorials/pandas-cookbook/06-string-operations-which-month-was-the-snowiest.ipynb +++ b/notebooks/tutorials/pandas-cookbook/06-string-operations-which-month-was-the-snowiest.ipynb @@ -27,7 +27,7 @@ "source": [ "SYFT_VERSION = \">=0.8.2.b0,<0.9\"\n", "package_string = f'\"syft{SYFT_VERSION}\"'\n", - "# %pip install {package_string} -f https://whls.blob.core.windows.net/unstable/index.html -q" + "# %pip install {package_string} -q" ] }, { diff --git a/notebooks/tutorials/pandas-cookbook/07-cleaning-up-messy-data.ipynb b/notebooks/tutorials/pandas-cookbook/07-cleaning-up-messy-data.ipynb index 0a4b03f0378..92490b2ba40 100644 --- a/notebooks/tutorials/pandas-cookbook/07-cleaning-up-messy-data.ipynb +++ b/notebooks/tutorials/pandas-cookbook/07-cleaning-up-messy-data.ipynb @@ -27,7 +27,7 @@ "source": [ "SYFT_VERSION = \">=0.8.2.b0,<0.9\"\n", "package_string = f'\"syft{SYFT_VERSION}\"'\n", - "# %pip install {package_string} -f https://whls.blob.core.windows.net/unstable/index.html -q" + "# %pip install {package_string} -q" ] }, { diff --git a/notebooks/tutorials/pandas-cookbook/08-how-to-deal-with-timestamps.ipynb b/notebooks/tutorials/pandas-cookbook/08-how-to-deal-with-timestamps.ipynb index f6afc36cb88..60387e53d39 100644 --- a/notebooks/tutorials/pandas-cookbook/08-how-to-deal-with-timestamps.ipynb +++ b/notebooks/tutorials/pandas-cookbook/08-how-to-deal-with-timestamps.ipynb @@ -29,7 +29,7 @@ "source": [ "SYFT_VERSION = \">=0.8.2.b0,<0.9\"\n", "package_string = f'\"syft{SYFT_VERSION}\"'\n", - "# %pip install {package_string} -f https://whls.blob.core.windows.net/unstable/index.html -q" + "# %pip install {package_string} -q" ] }, { diff --git a/packages/grid/VERSION b/packages/grid/VERSION index 6d7567fd24a..7f3f0571209 100644 --- a/packages/grid/VERSION +++ b/packages/grid/VERSION @@ -1,5 +1,5 @@ # Mono Repo Global Version -__version__ = "0.8.2-beta.27" +__version__ = "0.8.2-beta.28" # elsewhere we can call this file: `python VERSION` and simply take the stdout # stdlib diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index 10f8f5a992d..316ba1e08c2 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -15,13 +15,13 @@ pipelines: run: |- run_dependencies --all ensure_pull_secrets --all - build_images --all -t $(git rev-parse --short=6 HEAD) -t 0.8.2-beta.27 -t dev-latest + build_images --all -t $(git rev-parse --short=6 HEAD) -t 0.8.2-beta.28 -t dev-latest create_deployments --all vars: DEVSPACE_ENV_FILE: "default.env" CONTAINER_REGISTRY: "docker.io/" - VERSION: "0.8.2-beta.27" + VERSION: "0.8.2-beta.28" # This is a list of `images` that DevSpace can build for this project # We recommend to skip image building during development (devspace dev) as much as possible diff --git a/packages/grid/docker-compose.yml b/packages/grid/docker-compose.yml index 2309749dfe8..ad88ad7474d 100644 --- a/packages/grid/docker-compose.yml +++ b/packages/grid/docker-compose.yml @@ -129,6 +129,8 @@ services: network_mode: service:proxy volumes: - ${BACKEND_STORAGE_PATH}:/storage + stdin_open: true + tty: true # backend_stream: # restart: always diff --git a/packages/grid/frontend/package.json b/packages/grid/frontend/package.json index edd811bd419..05598895966 100644 --- a/packages/grid/frontend/package.json +++ b/packages/grid/frontend/package.json @@ -1,6 +1,6 @@ { "name": "pygrid-ui", - "version": "0.8.2-beta.27", + "version": "0.8.2-beta.28", "private": true, "scripts": { "dev": "pnpm i && vite dev --host --port 80", diff --git a/packages/grid/helm/repo/index.yaml b/packages/grid/helm/repo/index.yaml index f5e59383d29..693eed425c3 100644 --- a/packages/grid/helm/repo/index.yaml +++ b/packages/grid/helm/repo/index.yaml @@ -1,9 +1,25 @@ apiVersion: v1 entries: syft: + - apiVersion: v2 + appVersion: 0.8.2-beta.28 + created: "2023-09-03T12:23:17.826816314Z" + dependencies: + - name: component-chart + repository: https://charts.devspace.sh + version: 0.8.6 + description: Perform numpy-like analysis on data that remains in someone elses + server + digest: 50a697c502634565c0f5bbe219173e5d3ac7dbe02adccfd62a32732be7824528 + icon: https://raw.githubusercontent.com/OpenMined/PySyft/dev/docs/img/title_syft_light.png + name: syft + type: application + urls: + - https://openmined.github.io/PySyft/helm/syft-0.8.2-beta.28.tgz + version: 0.8.2-beta.28 - apiVersion: v2 appVersion: 0.8.2-beta.27 - created: "2023-08-31T07:12:34.862061012Z" + created: "2023-09-03T12:23:17.826302845Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -19,7 +35,7 @@ entries: version: 0.8.2-beta.27 - apiVersion: v2 appVersion: 0.8.2-beta.26 - created: "2023-08-31T07:12:34.861575787Z" + created: "2023-09-03T12:23:17.825676526Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -35,7 +51,7 @@ entries: version: 0.8.2-beta.26 - apiVersion: v2 appVersion: 0.8.2-beta.25 - created: "2023-08-31T07:12:34.861095721Z" + created: "2023-09-03T12:23:17.824574854Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -51,7 +67,7 @@ entries: version: 0.8.2-beta.25 - apiVersion: v2 appVersion: 0.8.2-beta.24 - created: "2023-08-31T07:12:34.86059213Z" + created: "2023-09-03T12:23:17.824065754Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -67,7 +83,7 @@ entries: version: 0.8.2-beta.24 - apiVersion: v2 appVersion: 0.8.2-beta.23 - created: "2023-08-31T07:12:34.860110001Z" + created: "2023-09-03T12:23:17.823578334Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -83,7 +99,7 @@ entries: version: 0.8.2-beta.23 - apiVersion: v2 appVersion: 0.8.2-beta.22 - created: "2023-08-31T07:12:34.859620788Z" + created: "2023-09-03T12:23:17.823030542Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -99,7 +115,7 @@ entries: version: 0.8.2-beta.22 - apiVersion: v2 appVersion: 0.8.2-beta.21 - created: "2023-08-31T07:12:34.859134099Z" + created: "2023-09-03T12:23:17.822536359Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -115,7 +131,7 @@ entries: version: 0.8.2-beta.21 - apiVersion: v2 appVersion: 0.8.2-beta.20 - created: "2023-08-31T07:12:34.858640658Z" + created: "2023-09-03T12:23:17.822051103Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -131,7 +147,7 @@ entries: version: 0.8.2-beta.20 - apiVersion: v2 appVersion: 0.8.2-beta.19 - created: "2023-08-31T07:12:34.857651111Z" + created: "2023-09-03T12:23:17.821044815Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -147,7 +163,7 @@ entries: version: 0.8.2-beta.19 - apiVersion: v2 appVersion: 0.8.2-beta.18 - created: "2023-08-31T07:12:34.857062963Z" + created: "2023-09-03T12:23:17.82052784Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -163,7 +179,7 @@ entries: version: 0.8.2-beta.18 - apiVersion: v2 appVersion: 0.8.2-beta.17 - created: "2023-08-31T07:12:34.856033093Z" + created: "2023-09-03T12:23:17.820034419Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -179,7 +195,7 @@ entries: version: 0.8.2-beta.17 - apiVersion: v2 appVersion: 0.8.2-beta.16 - created: "2023-08-31T07:12:34.855436259Z" + created: "2023-09-03T12:23:17.819535759Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -195,7 +211,7 @@ entries: version: 0.8.2-beta.16 - apiVersion: v2 appVersion: 0.8.2-beta.15 - created: "2023-08-31T07:12:34.854858521Z" + created: "2023-09-03T12:23:17.819006741Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -211,7 +227,7 @@ entries: version: 0.8.2-beta.15 - apiVersion: v2 appVersion: 0.8.2-beta.14 - created: "2023-08-31T07:12:34.854365771Z" + created: "2023-09-03T12:23:17.817810804Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -227,7 +243,7 @@ entries: version: 0.8.2-beta.14 - apiVersion: v2 appVersion: 0.8.2-beta.13 - created: "2023-08-31T07:12:34.85387788Z" + created: "2023-09-03T12:23:17.817290573Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -243,7 +259,7 @@ entries: version: 0.8.2-beta.13 - apiVersion: v2 appVersion: 0.8.2-beta.12 - created: "2023-08-31T07:12:34.853369431Z" + created: "2023-09-03T12:23:17.816784779Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -259,7 +275,7 @@ entries: version: 0.8.2-beta.12 - apiVersion: v2 appVersion: 0.8.2-beta.11 - created: "2023-08-31T07:12:34.852852647Z" + created: "2023-09-03T12:23:17.816264829Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -275,7 +291,7 @@ entries: version: 0.8.2-beta.11 - apiVersion: v2 appVersion: 0.8.2-beta.10 - created: "2023-08-31T07:12:34.852290848Z" + created: "2023-09-03T12:23:17.815782589Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -291,7 +307,7 @@ entries: version: 0.8.2-beta.10 - apiVersion: v2 appVersion: 0.8.2-beta.9 - created: "2023-08-31T07:12:34.866122593Z" + created: "2023-09-03T12:23:17.830241697Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -307,7 +323,7 @@ entries: version: 0.8.2-beta.9 - apiVersion: v2 appVersion: 0.8.2-beta.8 - created: "2023-08-31T07:12:34.865639572Z" + created: "2023-09-03T12:23:17.829757033Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -323,7 +339,7 @@ entries: version: 0.8.2-beta.8 - apiVersion: v2 appVersion: 0.8.2-beta.7 - created: "2023-08-31T07:12:34.865154998Z" + created: "2023-09-03T12:23:17.829269984Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -339,7 +355,7 @@ entries: version: 0.8.2-beta.7 - apiVersion: v2 appVersion: 0.8.2-beta.6 - created: "2023-08-31T07:12:34.864632712Z" + created: "2023-09-03T12:23:17.828780129Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -355,7 +371,7 @@ entries: version: 0.8.2-beta.6 - apiVersion: v2 appVersion: 0.8.2-beta.5 - created: "2023-08-31T07:12:34.86414901Z" + created: "2023-09-03T12:23:17.828275287Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -371,7 +387,7 @@ entries: version: 0.8.2-beta.5 - apiVersion: v2 appVersion: 0.8.2-beta.4 - created: "2023-08-31T07:12:34.863648596Z" + created: "2023-09-03T12:23:17.827795101Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -387,7 +403,7 @@ entries: version: 0.8.2-beta.4 - apiVersion: v2 appVersion: 0.8.2-beta.3 - created: "2023-08-31T07:12:34.863068824Z" + created: "2023-09-03T12:23:17.827308643Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -403,7 +419,7 @@ entries: version: 0.8.2-beta.3 - apiVersion: v2 appVersion: 0.8.2-beta.2 - created: "2023-08-31T07:12:34.858153108Z" + created: "2023-09-03T12:23:17.821559626Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -419,7 +435,7 @@ entries: version: 0.8.2-beta.2 - apiVersion: v2 appVersion: 0.8.2-beta.1 - created: "2023-08-31T07:12:34.851802747Z" + created: "2023-09-03T12:23:17.815288356Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -435,7 +451,7 @@ entries: version: 0.8.2-beta.1 - apiVersion: v2 appVersion: 0.8.1 - created: "2023-08-31T07:12:34.851269482Z" + created: "2023-09-03T12:23:17.814780388Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -451,7 +467,7 @@ entries: version: 0.8.1 - apiVersion: v2 appVersion: 0.8.1-beta.21 - created: "2023-08-31T07:12:34.850723693Z" + created: "2023-09-03T12:23:17.814274774Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -467,7 +483,7 @@ entries: version: 0.8.1-beta.21 - apiVersion: v2 appVersion: 0.8.1-beta.20 - created: "2023-08-31T07:12:34.849615675Z" + created: "2023-09-03T12:23:17.813750626Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -483,7 +499,7 @@ entries: version: 0.8.1-beta.20 - apiVersion: v2 appVersion: 0.8.1-beta.19 - created: "2023-08-31T07:12:34.849107276Z" + created: "2023-09-03T12:23:17.813243639Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -499,7 +515,7 @@ entries: version: 0.8.1-beta.19 - apiVersion: v2 appVersion: 0.8.1-beta.15 - created: "2023-08-31T07:12:34.848542292Z" + created: "2023-09-03T12:23:17.81256905Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -515,7 +531,7 @@ entries: version: 0.8.1-beta.15 - apiVersion: v2 appVersion: 0.8.1-beta.12 - created: "2023-08-31T07:12:34.848012673Z" + created: "2023-09-03T12:23:17.811439955Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -529,4 +545,4 @@ entries: urls: - https://openmined.github.io/PySyft/helm/syft-0.8.1-beta.12.tgz version: 0.8.1-beta.12 -generated: "2023-08-31T07:12:34.847327845Z" +generated: "2023-09-03T12:23:17.81059586Z" diff --git a/packages/grid/helm/repo/syft-0.8.2-beta.28.tgz b/packages/grid/helm/repo/syft-0.8.2-beta.28.tgz new file mode 100644 index 00000000000..3ae3b6f4571 Binary files /dev/null and b/packages/grid/helm/repo/syft-0.8.2-beta.28.tgz differ diff --git a/packages/grid/helm/syft/Chart.yaml b/packages/grid/helm/syft/Chart.yaml index 1b525f84584..a577377db2a 100644 --- a/packages/grid/helm/syft/Chart.yaml +++ b/packages/grid/helm/syft/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: syft description: Perform numpy-like analysis on data that remains in someone elses server type: application -version: "0.8.2-beta.27" -appVersion: "0.8.2-beta.27" +version: "0.8.2-beta.28" +appVersion: "0.8.2-beta.28" icon: https://raw.githubusercontent.com/OpenMined/PySyft/dev/docs/img/title_syft_light.png dependencies: diff --git a/packages/grid/podman/podman-kube/podman-syft-kube.yaml b/packages/grid/podman/podman-kube/podman-syft-kube.yaml index 930b773fcf6..36fd43113d7 100644 --- a/packages/grid/podman/podman-kube/podman-syft-kube.yaml +++ b/packages/grid/podman/podman-kube/podman-syft-kube.yaml @@ -41,7 +41,7 @@ spec: - configMapRef: name: podman-syft-config - image: docker.io/openmined/grid-backend:0.8.2-beta.27 + image: docker.io/openmined/grid-backend:0.8.2-beta.28 imagePullPolicy: IfNotPresent resources: {} tty: true @@ -57,7 +57,7 @@ spec: envFrom: - configMapRef: name: podman-syft-config - image: docker.io/openmined/grid-frontend:0.8.2-beta.27 + image: docker.io/openmined/grid-frontend:0.8.2-beta.28 imagePullPolicy: IfNotPresent resources: {} tty: true diff --git a/packages/hagrid/.bumpversion.cfg b/packages/hagrid/.bumpversion.cfg index 451adb61971..9d38890f4f1 100644 --- a/packages/hagrid/.bumpversion.cfg +++ b/packages/hagrid/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.3.70 +current_version = 0.3.71 tag = False tag_name = {new_version} commit = True diff --git a/packages/hagrid/hagrid/deps.py b/packages/hagrid/hagrid/deps.py index 39113c2b6e5..a341e064d81 100644 --- a/packages/hagrid/hagrid/deps.py +++ b/packages/hagrid/hagrid/deps.py @@ -42,7 +42,7 @@ from .version import __version__ LATEST_STABLE_SYFT = "0.8.1" -LATEST_BETA_SYFT = "0.8.2-beta.27" +LATEST_BETA_SYFT = "0.8.2-beta.28" DOCKER_ERROR = """ You are running an old version of docker, possibly on Linux. You need to install v2. @@ -114,8 +114,7 @@ class DependencySyftOS(Dependency): def check(self) -> None: self.display = "✅ " + ENVIRONMENT["os"] if is_windows(): - if not get_pip_package("jaxlib"): - self.issues.append(windows_jaxlib()) + pass elif is_apple_silicon(): pass @@ -918,15 +917,3 @@ def python_version_unsupported() -> SetupIssue: command="", solution="You must install a compatible version of Python", ) - - -WINDOWS_JAXLIB_REPO = "https://whls.blob.core.windows.net/unstable/index.html" - - -def windows_jaxlib() -> SetupIssue: - return SetupIssue( - issue_name="windows_jaxlib", - description="Windows Python Wheels for Jax are not available on PyPI yet", - command=f"pip install jaxlib==0.4.10 -f {WINDOWS_JAXLIB_REPO}", - solution="Windows users must install jaxlib before syft", - ) diff --git a/packages/hagrid/hagrid/manifest_template.yml b/packages/hagrid/hagrid/manifest_template.yml index b6ad753169a..fa6cc2b4469 100644 --- a/packages/hagrid/hagrid/manifest_template.yml +++ b/packages/hagrid/hagrid/manifest_template.yml @@ -1,9 +1,9 @@ manifestVersion: 0.1 -hagrid_version: 0.3.70 -syft_version: 0.8.2-beta.27 -dockerTag: 0.8.2-beta.27 +hagrid_version: 0.3.71 +syft_version: 0.8.2-beta.28 +dockerTag: 0.8.2-beta.28 baseUrl: https://raw.githubusercontent.com/OpenMined/PySyft/ -hash: 576bd08c2a99e17ac2907433fd6dc3a3030136f5 +hash: ef6e9adb58b7a2d68ef5f41b61ae0d237f36e5be target_dir: ~/.hagrid/PySyft/ files: grid: diff --git a/packages/hagrid/hagrid/version.py b/packages/hagrid/hagrid/version.py index a02282ca438..5c26289f637 100644 --- a/packages/hagrid/hagrid/version.py +++ b/packages/hagrid/hagrid/version.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # HAGrid Version -__version__ = "0.3.70" +__version__ = "0.3.71" if __name__ == "__main__": print(__version__) diff --git a/packages/hagrid/setup.py b/packages/hagrid/setup.py index 7a50a69d26d..22eb365fdf6 100644 --- a/packages/hagrid/setup.py +++ b/packages/hagrid/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages from setuptools import setup -__version__ = "0.3.70" +__version__ = "0.3.71" DATA_FILES = {"img": ["hagrid/img/*.png"], "hagrid": ["*.yml"]} diff --git a/packages/syft/setup.cfg b/packages/syft/setup.cfg index 87de753c800..749f705fa05 100644 --- a/packages/syft/setup.cfg +++ b/packages/syft/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = syft -version = attr: "0.8.2-beta.27" +version = attr: "0.8.2-beta.28" description = Perform numpy-like analysis on data that remains in someone elses server author = OpenMined author_email = info@openmined.org @@ -30,12 +30,12 @@ syft = forbiddenfruit==0.1.4 gevent==22.10.2 gipc==1.5.0 - jax==0.4.10 - jaxlib==0.4.10 + jax==0.4.14 + jaxlib==0.4.14 loguru==0.7.0 networkx==2.8 numpy>=1.22.4,<=1.24.3 - opendp==0.7.0 + opendp==0.8.0 packaging>=21.0 pandas==1.5.3 pyarrow==11.0.0 @@ -56,7 +56,7 @@ syft = fastapi==0.97.0 hagrid>=0.3 matplotlib==3.7.1 - dm-haiku==0.0.9 + dm-haiku==0.0.10 itables==1.5.3 safetensors==0.3.2 # Pinning as arm linux wheels, not available for safetensors in 0.3.3 transformers==4.30.2 diff --git a/packages/syft/src/syft/VERSION b/packages/syft/src/syft/VERSION index 6d7567fd24a..7f3f0571209 100644 --- a/packages/syft/src/syft/VERSION +++ b/packages/syft/src/syft/VERSION @@ -1,5 +1,5 @@ # Mono Repo Global Version -__version__ = "0.8.2-beta.27" +__version__ = "0.8.2-beta.28" # elsewhere we can call this file: `python VERSION` and simply take the stdout # stdlib diff --git a/packages/syft/src/syft/__init__.py b/packages/syft/src/syft/__init__.py index 2e9b4aa4aa0..deaedacb5ee 100644 --- a/packages/syft/src/syft/__init__.py +++ b/packages/syft/src/syft/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.8.2-beta.27" +__version__ = "0.8.2-beta.28" # stdlib import pathlib diff --git a/packages/syft/src/syft/store/kv_document_store.py b/packages/syft/src/syft/store/kv_document_store.py index 8ba859c9a48..2c280c443d5 100644 --- a/packages/syft/src/syft/store/kv_document_store.py +++ b/packages/syft/src/syft/store/kv_document_store.py @@ -176,11 +176,11 @@ def _set( try: if obj.id is None: obj.id = UID() - store_query_key = self.settings.store_key.with_obj(obj) + store_query_key: QueryKey = self.settings.store_key.with_obj(obj) uid = store_query_key.value write_permission = ActionObjectWRITE(uid=uid, credentials=credentials) can_write = self.has_permission(write_permission) - unique_query_keys = self.settings.unique_keys.with_obj(obj) + unique_query_keys: QueryKeys = self.settings.unique_keys.with_obj(obj) store_key_exists = store_query_key.value in self.data searchable_query_keys = self.settings.searchable_keys.with_obj(obj) diff --git a/packages/syft/src/syft/store/mongo_client.py b/packages/syft/src/syft/store/mongo_client.py index b63a94bcb7a..f5c8ea47b05 100644 --- a/packages/syft/src/syft/store/mongo_client.py +++ b/packages/syft/src/syft/store/mongo_client.py @@ -203,3 +203,25 @@ def with_collection( return Err(str(e)) return Ok(collection) + + def with_collection_permissions( + self, collection_settings: PartitionSettings, store_config: StoreConfig + ) -> Result[MongoCollection, Err]: + """ + For each collection, create a corresponding collection + that store the permissions to the data in that collection + """ + res = self.with_db(db_name=store_config.db_name) + if res.is_err(): + return res + db = res.ok() + + try: + collection_permissions_name: str = collection_settings.name + "_permissions" + collection_permissions = db.get_collection( + name=collection_permissions_name, codec_options=SYFT_CODEC_OPTIONS + ) + except BaseException as e: + return Err(str(e)) + + return Ok(collection_permissions) diff --git a/packages/syft/src/syft/store/mongo_document_store.py b/packages/syft/src/syft/store/mongo_document_store.py index cd1f0e0aa42..46664512647 100644 --- a/packages/syft/src/syft/store/mongo_document_store.py +++ b/packages/syft/src/syft/store/mongo_document_store.py @@ -7,9 +7,7 @@ # third party from pymongo import ASCENDING -from pymongo import WriteConcern from pymongo.collection import Collection as MongoCollection -from pymongo.errors import DuplicateKeyError from result import Err from result import Ok from result import Result @@ -20,9 +18,12 @@ from ..serde.deserialize import _deserialize from ..serde.serializable import serializable from ..serde.serialize import _serialize +from ..service.action.action_permissions import ActionObjectEXECUTE +from ..service.action.action_permissions import ActionObjectOWNER from ..service.action.action_permissions import ActionObjectPermission from ..service.action.action_permissions import ActionObjectREAD from ..service.action.action_permissions import ActionObjectWRITE +from ..service.action.action_permissions import ActionPermission from ..service.response import SyftSuccess from ..types.syft_object import StorableObjectType from ..types.syft_object import SyftBaseObject @@ -30,6 +31,7 @@ from ..types.transforms import TransformContext from ..types.transforms import transform from ..types.transforms import transform_method +from ..types.uid import UID from .document_store import DocumentStore from .document_store import PartitionKey from .document_store import QueryKey @@ -124,17 +126,26 @@ def init_store(self) -> Result[Ok, Err]: return store_status client = MongoClient(config=self.store_config.client_config) + collection_status = client.with_collection( collection_settings=self.settings, store_config=self.store_config ) if collection_status.is_err(): return collection_status + collection_permissions_status = client.with_collection_permissions( + collection_settings=self.settings, store_config=self.store_config + ) + if collection_permissions_status.is_err(): + return collection_permissions_status + self._collection = collection_status.ok() + self._permissions = collection_permissions_status.ok() return self._create_update_index() # Potentially thread-unsafe methods. + # # CAUTION: # * Don't use self.lock here. # * Do not call the public thread-safe methods here(with locking). @@ -145,7 +156,7 @@ def _create_update_index(self) -> Result[Ok, Err]: collection_status = self.collection if collection_status.is_err(): return collection_status - collection = collection_status.ok() + collection: MongoCollection = collection_status.ok() def check_index_keys(current_keys, new_index_keys): current_keys.sort() @@ -202,6 +213,15 @@ def collection(self) -> Result[MongoCollection, Err]: return Ok(self._collection) + @property + def permissions(self) -> Result[MongoCollection, Err]: + if not hasattr(self, "_permissions"): + res = self.init_store() + if res.is_err(): + return res + + return Ok(self._permissions) + def _set( self, credentials: SyftVerifyKey, @@ -211,27 +231,58 @@ def _set( ) -> Result[SyftObject, str]: write_permission = ActionObjectWRITE(uid=obj.id, credentials=credentials) can_write = self.has_permission(write_permission) + + store_query_key: QueryKey = self.settings.store_key.with_obj(obj) + collection_status = self.collection + if collection_status.is_err(): + return collection_status + collection: MongoCollection = collection_status.ok() + + store_key_exists = ( + collection.find_one(store_query_key.as_dict_mongo) is not None + ) + if (not store_key_exists) and (not self.item_keys_exist(obj, collection)): + # attempt to claim ownership for writing + ownership_result = self.take_ownership(uid=obj.id, credentials=credentials) + can_write: bool = ownership_result.is_ok() + elif not ignore_duplicates: + unique_query_keys: QueryKeys = self.settings.unique_keys.with_obj(obj) + keys = ", ".join(f"`{key.key}`" for key in unique_query_keys.all) + return Err( + f"Duplication Key Error for {obj}.\n" + f"The fields that should be unique are {keys}." + ) + else: + # we are not throwing an error, because we are ignoring duplicates + # we are also not writing though + return Ok(obj) + if can_write: storage_obj = obj.to(self.storage_type) - collection_status = self.collection - if collection_status.is_err(): - return collection_status - collection = collection_status.ok() + collection.insert_one(storage_obj) + + # adding permissions + read_permission = ActionObjectPermission( + uid=obj.id, + credentials=credentials, + permission=ActionPermission.READ, + ) + self.add_permission(read_permission) - if ignore_duplicates: - collection = collection.with_options(write_concern=WriteConcern(w=0)) - try: - collection.insert_one(storage_obj) - except DuplicateKeyError as e: - return Err(f"Duplicate Key Error for {obj}: {e}") if add_permissions is not None: - pass - # TODO: update permissions + self.add_permissions(add_permissions) + return Ok(obj) else: return Err(f"No permission to write object with id {obj.id}") + def item_keys_exist(self, obj, collection): + qks: QueryKeys = self.settings.unique_keys.with_obj(obj) + query = {"$or": [{k: v} for k, v in qks.as_dict_mongo.items()]} + res = collection.find_one(query) + return res is not None + def _update( self, credentials: SyftVerifyKey, @@ -242,7 +293,7 @@ def _update( collection_status = self.collection if collection_status.is_err(): return collection_status - collection = collection_status.ok() + collection: MongoCollection = collection_status.ok() # TODO: optimize the update. The ID should not be overwritten, # but the qk doesn't necessarily have to include the `id` field either. @@ -305,7 +356,7 @@ def _get_all_from_store( collection_status = self.collection if collection_status.is_err(): return collection_status - collection = collection_status.ok() + collection: MongoCollection = collection_status.ok() if order_by is not None: storage_objs = collection.find(filter=qks.as_dict_mongo).sort(order_by.key) @@ -331,7 +382,7 @@ def _delete( collection_status = self.collection if collection_status.is_err(): return collection_status - collection = collection_status.ok() + collection: MongoCollection = collection_status.ok() if has_permission or self.has_permission( ActionObjectWRITE(uid=qk.value, credentials=credentials) @@ -345,12 +396,117 @@ def _delete( return Err(f"Failed to delete object with qk: {qk}") def has_permission(self, permission: ActionObjectPermission) -> bool: - # TODO: implement - return True + """Check if the permission is inside the permission collection""" + collection_permissions_status = self.permissions + if collection_permissions_status.is_err(): + return False + collection_permissions: MongoCollection = collection_permissions_status.ok() + + # TODO: fix for other admins + if self.root_verify_key.verify == permission.credentials.verify: + return True + + permissions: Optional[Dict] = collection_permissions.find_one( + {"_id": permission.uid} + ) + + if permissions is None: + return False + + if permission.permission_string in permissions["permissions"]: + return True + + # check ALL_READ permission + if ( + permission.permission == ActionPermission.READ + and ActionObjectPermission( + permission.uid, ActionPermission.ALL_READ + ).permission_string + in permissions["permissions"] + ): + return True + + return False + + def add_permission(self, permission: ActionObjectPermission) -> Result[None, Err]: + collection_permissions_status = self.permissions + if collection_permissions_status.is_err(): + return collection_permissions_status + collection_permissions: MongoCollection = collection_permissions_status.ok() + + # find the permissions for the given permission.uid + # e.g. permissions = {"_id": "7b88fdef6bff42a8991d294c3d66f757", + # "permissions": set(["permission_str_1", "permission_str_2"]}} + permissions: Optional[Dict] = collection_permissions.find_one( + {"_id": permission.uid} + ) + if permissions is None: + # Permission doesn't exits, add a new one + collection_permissions.insert_one( + { + "_id": permission.uid, + "permissions": {permission.permission_string}, + } + ) + else: + # update the permissions with the new permission string + permission_strings: set = permissions["permissions"] + permission_strings.add(permission.permission_string) + collection_permissions.update_one( + {"_id": permission.uid}, {"$set": {"permissions": permission_strings}} + ) def add_permissions(self, permissions: List[ActionObjectPermission]) -> None: - # TODO: implemenet - pass + for permission in permissions: + self.add_permission(permission) + + def remove_permission( + self, permission: ActionObjectPermission + ) -> Result[None, Err]: + collection_permissions_status = self.permissions + if collection_permissions_status.is_err(): + return collection_permissions_status + collection_permissions: MongoCollection = collection_permissions_status.ok() + permissions: Optional[Dict] = collection_permissions.find_one( + {"_id": permission.uid} + ) + if permissions is None: + return Err(f"permission with UID {permission.uid} not found!") + permissions_strings: set = permissions["permissions"] + permissions_strings.remove(permission.permission_string) + collection_permissions.update_one( + {"_id": permission.uid}, {"$set": {"permissions": permissions_strings}} + ) + + def take_ownership( + self, uid: UID, credentials: SyftVerifyKey + ) -> Result[SyftSuccess, str]: + collection_permissions_status = self.permissions + if collection_permissions_status.is_err(): + return collection_permissions_status + collection_permissions: MongoCollection = collection_permissions_status.ok() + + collection_status = self.collection + if collection_status.is_err(): + return collection_status + collection: MongoCollection = collection_status.ok() + + data: List[UID] = collection.find_one({"_id": uid}) + permissions: List[UID] = collection_permissions.find_one({"_id": uid}) + + # first person using this UID can claim ownership + if permissions is None and data is None: + self.add_permissions( + [ + ActionObjectOWNER(uid=uid, credentials=credentials), + ActionObjectWRITE(uid=uid, credentials=credentials), + ActionObjectREAD(uid=uid, credentials=credentials), + ActionObjectEXECUTE(uid=uid, credentials=credentials), + ] + ) + return Ok(SyftSuccess(message=f"Ownership of ID: {uid} taken.")) + + return Err(f"UID: {uid} already owned.") def _all( self, @@ -370,7 +526,7 @@ def __len__(self): collection_status = self.collection if collection_status.is_err(): return 0 - collection = collection_status.ok() + collection: MongoCollection = collection_status.ok() return collection.count_documents(filter={}) diff --git a/packages/syft/src/syft/store/sqlite_document_store.py b/packages/syft/src/syft/store/sqlite_document_store.py index 01b297ec4a2..27195ae7ac5 100644 --- a/packages/syft/src/syft/store/sqlite_document_store.py +++ b/packages/syft/src/syft/store/sqlite_document_store.py @@ -94,7 +94,7 @@ def _connect(self) -> None: # Set journal mode to WAL. # self._db[thread_ident()].execute("pragma journal_mode=wal") - def create_table(self): + def create_table(self) -> None: try: self.cur.execute( f"create table {self.table_name} (uid VARCHAR(32) NOT NULL PRIMARY KEY, " # nosec diff --git a/packages/syftcli/manifest.yml b/packages/syftcli/manifest.yml index 73e90d9c649..58cfdcb9f4f 100644 --- a/packages/syftcli/manifest.yml +++ b/packages/syftcli/manifest.yml @@ -1,11 +1,11 @@ manifestVersion: 1.0 -syftVersion: 0.8.2-beta.27 -dockerTag: 0.8.2-beta.27 +syftVersion: 0.8.2-beta.28 +dockerTag: 0.8.2-beta.28 images: - - docker.io/openmined/grid-frontend:0.8.2-beta.27 - - docker.io/openmined/grid-backend:0.8.2-beta.27 + - docker.io/openmined/grid-frontend:0.8.2-beta.28 + - docker.io/openmined/grid-backend:0.8.2-beta.28 - docker.io/library/mongo:latest - docker.io/traefik:v2.10 diff --git a/scripts/hagrid_hash b/scripts/hagrid_hash index 20384883944..ba276c38159 100644 --- a/scripts/hagrid_hash +++ b/scripts/hagrid_hash @@ -1 +1 @@ -3604d501ca849b320a73cbc6fc66c282 +e32f2c4364e17407a56a8cf9fb828d5e diff --git a/tox.ini b/tox.ini index 9b2eaee3479..46a5ae90ce4 100644 --- a/tox.ini +++ b/tox.ini @@ -27,11 +27,9 @@ skipsdist = True [testenv] basepython = python3 -install_command = pip install --find-links https://whls.blob.core.windows.net/unstable/index.html {opts} {packages} +install_command = pip install {opts} {packages} commands = python --version -setenv = - PIP_FIND_LINKS=https://whls.blob.core.windows.net/unstable/index.html # Syft [testenv:syft] @@ -39,8 +37,6 @@ deps = -e{toxinidir}/packages/syft[dev] changedir = {toxinidir}/packages/syft description = Syft -setenv = - PIP_FIND_LINKS=https://whls.blob.core.windows.net/unstable/index.html commands = pip list @@ -49,8 +45,6 @@ deps = -e{toxinidir}/packages/hagrid[dev] changedir = {toxinidir}/packages/hagrid description = Syft -setenv = - PIP_FIND_LINKS=https://whls.blob.core.windows.net/unstable/index.html commands = pip list @@ -233,7 +227,6 @@ allowlist_externals = chcp passenv=HOME, USER setenv = - PIP_FIND_LINKS=https://whls.blob.core.windows.net/unstable/index.html HAGRID_FLAGS = {env:HAGRID_FLAGS:--tag=local --test} EMULATION = {env:EMULATION:false} HAGRID_ART = false @@ -339,8 +332,6 @@ commands = [testenv:syft.jupyter] description = Jupyter Notebook with Editable Syft -setenv = - PIP_FIND_LINKS=https://whls.blob.core.windows.net/unstable/index.html deps = {[testenv:syft]deps} {[testenv:hagrid]deps} @@ -401,7 +392,6 @@ setenv = ENABLE_SIGNUP=True commands = pip install oblv-ctl==0.3.1 - python -c "import platform; import os; os.system('pip install jaxlib==0.4.10 -f https://whls.blob.core.windows.net/unstable/index.html') if platform.system().lower() == 'windows' else ''" # run at start to kill any process started beforehand bash -c 'chmod +x scripts/kill_process_in_port.sh && ./scripts/kill_process_in_port.sh $LOCAL_ENCLAVE_PORT' @@ -607,7 +597,6 @@ allowlist_externals = k3d echo setenv = - PIP_FIND_LINKS=https://whls.blob.core.windows.net/unstable/index.html ORCHESTRA_DEPLOYMENT_TYPE = {env:ORCHESTRA_DEPLOYMENT_TYPE:k8s} NODE_PORT = {env:NODE_PORT:9082} commands = @@ -760,8 +749,6 @@ allowlist_externals = echo rm helm -setenv = - PIP_FIND_LINKS=https://whls.blob.core.windows.net/unstable/index.html commands = k3d version @@ -798,8 +785,6 @@ allowlist_externals = echo rm helm -setenv = - PIP_FIND_LINKS=https://whls.blob.core.windows.net/unstable/index.html commands = bash -c 'cd packages/grid/helm && \ helm lint syft' @@ -833,7 +818,6 @@ allowlist_externals = rm helm setenv = - PIP_FIND_LINKS=https://whls.blob.core.windows.net/unstable/index.html ORCHESTRA_DEPLOYMENT_TYPE = {env:ORCHESTRA_DEPLOYMENT_TYPE:k8s} NODE_PORT = {env:NODE_PORT:8080} commands =