From 845ceebb4e3b0041c1307ac94f57154598f329b6 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Thu, 9 Nov 2023 12:57:05 +0530 Subject: [PATCH 1/3] removed / in href for notebooks links --- README.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 42fd4fcaf42..a262e6e9c16 100644 --- a/README.md +++ b/README.md @@ -44,16 +44,16 @@ domain_client = sy.login(port=8080, email="info@openmined.org", password="change ## PySyft in 10 minutes -πŸ“ API Example Notebooks +πŸ“ API Example Notebooks -- 00-load-data.ipynb -- 01-submit-code.ipynb -- 02-review-code-and-approve.ipynb -- 03-data-scientist-download-result.ipynb -- 04-jax-example.ipynb -- 05-custom-policy.ipynb -- 06-multiple-code-requests.ipynb -- 07-domain-register-control-flow.ipynb +- 00-load-data.ipynb +- 01-submit-code.ipynb +- 02-review-code-and-approve.ipynb +- 03-data-scientist-download-result.ipynb +- 04-jax-example.ipynb +- 05-custom-policy.ipynb +- 06-multiple-code-requests.ipynb +- 07-domain-register-control-flow.ipynb ## Deploy Kubernetes Helm Chart @@ -183,11 +183,11 @@ No more cold calls to get `access` to a dataset. No more weeks of `wait times` t -- Deploy a Domain Server -- Upload Private Data -- Create Accounts +- Deploy a Domain Server +- Upload Private Data +- Create Accounts - Manage Privacy Budget -- Join a Network +- Join a Network - Learn how PETs streamline Data Policies From c71eee952e88d18db21a33ff701915bc30c3648d Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Thu, 9 Nov 2023 13:15:54 +0530 Subject: [PATCH 2/3] added a new script to convert github readme to pypi readme --- packages/syft/PYPI.md | 403 ++++++++++++++++++++++++++++++ packages/syft/setup.cfg | 2 +- scripts/convert_to_pypi_readme.py | 90 +++++++ 3 files changed, 494 insertions(+), 1 deletion(-) create mode 100644 packages/syft/PYPI.md create mode 100644 scripts/convert_to_pypi_readme.py diff --git a/packages/syft/PYPI.md b/packages/syft/PYPI.md new file mode 100644 index 00000000000..434cfd354f5 --- /dev/null +++ b/packages/syft/PYPI.md @@ -0,0 +1,403 @@ +
+

+ +Syft Logo + +Perform data science on `data` that remains in `someone else's` server + +# Quickstart + +βœ… `Linux` βœ… `macOS` βœ… `Windows` βœ… `Docker` βœ… `Podman` βœ… `Kubernetes` + +## Install Client + +```bash +$ pip install -U syft +``` + +## Launch Server + +```python +# from Jupyter / Python +import syft as sy +sy.requires(">=0.8.1,<0.8.2") +node = sy.orchestra.launch(name="my-domain", port=8080, dev_mode=True, reset=True) +``` + +```bash +# or from the command line +$ syft launch --name=my-domain --port=8080 --reset=True + +Starting syft-node server on 0.0.0.0:8080 +``` + +## Launch Client + +```python +import syft as sy +sy.requires(">=0.8.1,<0.8.2") +domain_client = sy.login(port=8080, email="info@openmined.org", password="changethis") +``` + +## PySyft in 10 minutes + +πŸ“ API Example Notebooks + +- 00-load-data.ipynb +- 01-submit-code.ipynb +- 02-review-code-and-approve.ipynb +- 03-data-scientist-download-result.ipynb +- 04-jax-example.ipynb +- 05-custom-policy.ipynb +- 06-multiple-code-requests.ipynb +- 07-domain-register-control-flow.ipynb + +## Deploy Kubernetes Helm Chart + +**Note**: Assuming we have a Kubernetes cluster already setup. + +#### 1. Add and update Helm repo for Syft + +```sh +$ helm repo add openmined https://openmined.github.io/PySyft/helm +$ helm repo update openmined +``` + +#### 2. Search for available Syft versions + +```sh +$ helm search repo openmined/syft --versions --devel +``` + +#### 3. Set your preferred Syft Chart version + +```sh +SYFT_VERSION="" +``` + +#### 4. Provisioning Helm Charts + +```sh +$ helm install my-domain openmined/syft --version $SYFT_VERSION --namespace syft --create-namespace --set ingress.ingressClass=traefik +``` + +### Azure or GCP Ingress + +``` +$ helm install ... --set ingress.ingressClass="azure/application-gateway" +$ helm install ... --set ingress.ingressClass="gce" +``` + +## Deploy to a Container Engine or Cloud + +1. Install our handy πŸ›΅ cli tool which makes deploying a Domain or Gateway server to Docker or VM a one-liner: + `pip install -U hagrid` + +2. Then run our interactive jupyter Install πŸ§™πŸ½β€β™‚οΈ WizardBETA: + `hagrid quickstart` + +3. In the tutorial you will learn how to install and deploy: + `PySyft` = our `numpy`-like 🐍 Python library for computing on `private data` in someone else's `Domain` + + `PyGrid` = our 🐳 `docker` / 🐧 `vm` `Domain` & `Gateway` Servers where `private data` lives + +## Docs and Support + +- πŸ“š Docs +- `#support` on Slack + +# Install Notes + +- HAGrid 0.3 Requires: 🐍 `python` πŸ™ `git` - Run: `pip install -U hagrid` +- Interactive Install πŸ§™πŸ½β€β™‚οΈ WizardBETA Requires πŸ›΅ `hagrid`: - Run: `hagrid quickstart` +- PySyft 0.8.1 Requires: 🐍 `python 3.9 - 3.11` - Run: `pip install -U syft` +- PyGrid Requires: 🐳 `docker`, 🦦 `podman` or ☸️ `kubernetes` - Run: `hagrid launch ...` + +# Versions + +`0.9.0` - Coming soon... +`0.8.2` (Beta) - `dev` branch πŸ‘ˆπŸ½ API - Coming soon... +`0.8.1` (Stable) - API + +Deprecated: + +- `0.8.0` - API +- `0.7.0` - Course 3 Updated +- `0.6.0` - Course 3 +- `0.5.1` - Course 2 + M1 Hotfix +- `0.2.0` - `0.5.0` + +PySyft and PyGrid use the same `version` and its best to match them up where possible. We release weekly betas which can be used in each context: + +PySyft (Stable): `pip install -U syft` +PyGrid (Stable) `hagrid launch ... tag=latest` + +PySyft (Beta): `pip install -U syft --pre` +PyGrid (Beta): `hagrid launch ... tag=beta` + +HAGrid is a cli / deployment tool so the latest version of `hagrid` is usually the best. + +# What is Syft? + +Syft + +`Syft` is OpenMined's `open source` stack that provides `secure` and `private` Data Science in Python. Syft decouples `private data` from model training, using techniques like [Federated Learning](https://ai.googleblog.com/2017/04/federated-learning-collaborative.html), [Differential Privacy](https://en.wikipedia.org/wiki/Differential_privacy), and [Encrypted Computation](https://en.wikipedia.org/wiki/Homomorphic_encryption). This is done with a `numpy`-like interface and integration with `Deep Learning` frameworks, so that you as a `Data Scientist` can maintain your current workflow while using these new `privacy-enhancing techniques`. + +### Why should I use Syft? + +`Syft` allows a `Data Scientist` to ask `questions` about a `dataset` and, within `privacy limits` set by the `data owner`, get `answers` to those `questions`, all without obtaining a `copy` of the data itself. We call this process `Remote Data Science`. It means in a wide variety of `domains` across society, the current `risks` of sharing information (`copying` data) with someone such as, privacy invasion, IP theft and blackmail will no longer prevent the vast `benefits` such as innovation, insights and scientific discovery which secure access will provide. + +No more cold calls to get `access` to a dataset. No more weeks of `wait times` to get a `result` on your `query`. It also means `1000x more data` in every domain. PySyft opens the doors to a streamlined Data Scientist `workflow`, all with the individual's `privacy` at its heart. + + + +# Terminology + + + + + + + + + + + + + + + + + + + + +
+ +

πŸ‘¨πŸ»β€πŸ’Ό Data Owners

+
+ +

πŸ‘©πŸ½β€πŸ”¬ Data Scientists

+
+ + +Provide `datasets` which they would like to make available for `study` by an `outside party` they may or may not `fully trust` has good intentions. + + + + +Are end `users` who desire to perform `computations` or `answer` a specific `question` using one or more data owners' `datasets`. + +
+ +

🏰 Domain Server

+
+ +

πŸ”— Gateway Server

+
+ + +Manages the `remote study` of the data by a `Data Scientist` and allows the `Data Owner` to manage the `data` and control the `privacy guarantees` of the subjects under study. It also acts as a `gatekeeper` for the `Data Scientist's` access to the data to compute and experiment with the results. + + + + +Provides services to a group of `Data Owners` and `Data Scientists`, such as dataset `search` and bulk `project approval` (legal / technical) to participate in a project. A gateway server acts as a bridge between it's members (`Domains`) and their subscribers (`Data Scientists`) and can provide access to a collection of `domains` at once.
+ +# Community + + + + + + + +
+ +
+ + + + + +
+
+ + + + +
+ + + + + +
+
+ +# Courses + + + + + + + +
+ +
+ +
+ +
+ +
+
+ +
+ +
+
+ +# Contributors + +OpenMined and Syft appreciates all contributors, if you would like to fix a bug or suggest a new feature, please see our [guidelines](https://openmined.github.io/PySyft/developer_guide/index.html).
+ +Contributors + +# Supporters + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +# Open Collective + +`OpenMined` is a fiscally sponsored `501(c)(3)` in the USA. We are funded by our generous supporters on Open Collective.

+ +Contributors + +# Disclaimer + +Syft is under active development and is not yet ready for pilots on private data without our assistance. As early access participants, please contact us via [Slack](https://slack.openmined.org/) or email if you would like to ask a question or have a use case that you would like to discuss. + +# License + +[Apache License 2.0](LICENSE)
+Person icons created by Freepik - Flaticon + + diff --git a/packages/syft/setup.cfg b/packages/syft/setup.cfg index 4b0f8fe459f..c267a7e302e 100644 --- a/packages/syft/setup.cfg +++ b/packages/syft/setup.cfg @@ -5,7 +5,7 @@ description = Perform numpy-like analysis on data that remains in someone elses author = OpenMined author_email = info@openmined.org license = Apache-2.0 -long_description = file: README.md +long_description = file: PYPI.md long_description_content_type = text/markdown; charset=UTF-8; variant=GFM url = https://openmined.github.io/PySyft/ project_urls = diff --git a/scripts/convert_to_pypi_readme.py b/scripts/convert_to_pypi_readme.py new file mode 100644 index 00000000000..70d53ad3ae5 --- /dev/null +++ b/scripts/convert_to_pypi_readme.py @@ -0,0 +1,90 @@ +# stdlib +import argparse +from pathlib import Path +import re + + +def convert_github_markdown_to_pypi(input_file, output_file, repo, version): + with open(input_file, encoding="utf-8") as file: + content = file.read() + + # Regular expression pattern to match and tags + pattern = r"\s*]*>\s*(]*>)\s*" + + # Replace matched patterns with the contents of the tag + content = re.sub(pattern, r"\1", content) + + # Regular expression patterns to match and tags with relative links + img_pattern = r'(]* src=")(?!https://)([^"]*)"' + a_pattern = r'(> Args", args.__dict__) + print(">> Input File:", args.input_file) + print(">> Output File:", args.output_file) + print(">> Repo URL:", args.repo) + print(">> Version:", args.version) + + # Converting the readme file to pypi format + convert_github_markdown_to_pypi( + args.input_file, args.output_file, args.repo, args.version + ) + + print("\n\n") + print("-" * 50) + print(">> Done") + print("-" * 50) + + +if __name__ == "__main__": + main() From 50b5d7dffca3832fb2a6227af062c039891c2a53 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Thu, 9 Nov 2023 13:26:32 +0530 Subject: [PATCH 3/3] updated pypi script in github release --- .github/workflows/cd-syft.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cd-syft.yml b/.github/workflows/cd-syft.yml index df5491f8344..cea82117543 100644 --- a/.github/workflows/cd-syft.yml +++ b/.github/workflows/cd-syft.yml @@ -70,6 +70,7 @@ jobs: pip install --upgrade tox setuptools wheel twine bump2version PyYAML - name: Bump the Version + id: bump-version run: | ls **/VERSION | xargs -I {} python {} cat packages/grid/devspace.yaml | grep '0\.' @@ -78,6 +79,7 @@ jobs: ls **/VERSION | xargs -I {} python {} cat packages/grid/devspace.yaml | grep '0\.' python packages/hagrid/scripts/update_manifest.py $(python packages/grid/VERSION) + echo "current_version=$(python packages/grid/VERSION | sed 's/-beta.*//')" >> $GITHUB_OUTPUT - name: Check Protocol Version run: | @@ -112,13 +114,17 @@ jobs: # export BUMP=True tox -e syft.protocol.check + - name: Update PyPI Readme + run: | + python scripts/convert_to_pypi_readme.py --input-file packages/syft/README.MD --output-file packages/syft/PYPI.md --version {{ steps.bump-version.outputs.current_version }} + - name: Commit changes to Syft uses: EndBug/add-and-commit@v9 with: author_name: ${{ secrets.OM_BOT_NAME }} author_email: ${{ secrets.OM_BOT_EMAIL }} message: "[syft]bump version" - add: "['.bumpversion.cfg', 'VERSION', 'packages/grid/VERSION', 'packages/grid/devspace.yaml', 'packages/syft/src/syft/VERSION', 'packages/syft/setup.cfg', 'packages/grid/frontend/package.json', 'packages/syft/src/syft/__init__.py', 'packages/hagrid/hagrid/manifest_template.yml', 'packages/grid/helm/syft/Chart.yaml','packages/grid/helm/repo', 'packages/hagrid/hagrid/deps.py', 'packages/grid/podman/podman-kube/podman-syft-kube.yaml' ,'packages/grid/podman/podman-kube/podman-syft-kube-config.yaml' 'packages/syftcli/manifest.yml', 'packages/syft/src/syft/protocol/protocol_version.json']" + add: "['.bumpversion.cfg', 'VERSION', 'packages/grid/VERSION','packages/syft/PYPI.MD', 'packages/grid/devspace.yaml', 'packages/syft/src/syft/VERSION', 'packages/syft/setup.cfg', 'packages/grid/frontend/package.json', 'packages/syft/src/syft/__init__.py', 'packages/hagrid/hagrid/manifest_template.yml', 'packages/grid/helm/syft/Chart.yaml','packages/grid/helm/repo', 'packages/hagrid/hagrid/deps.py', 'packages/grid/podman/podman-kube/podman-syft-kube.yaml' ,'packages/grid/podman/podman-kube/podman-syft-kube-config.yaml' 'packages/syftcli/manifest.yml', 'packages/syft/src/syft/protocol/protocol_version.json']" - name: Scheduled Build and Publish if: github.event_name == 'schedule'