From f932590e5ef6ebda4e07cf23ae8d8ba589d6ddb0 Mon Sep 17 00:00:00 2001 From: Aziz Berkay Yesilyurt Date: Wed, 15 May 2024 10:50:15 +0200 Subject: [PATCH 1/7] enable parallel runs in integration tests --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 375ea2a39b8..a47a6241d33 100644 --- a/tox.ini +++ b/tox.ini @@ -649,7 +649,7 @@ allowlist_externals = setenv = PYTEST_MODULES = {env:PYTEST_MODULES:local_node} ASSOCIATION_REQUEST_AUTO_APPROVAL = {env:ASSOCIATION_REQUEST_AUTO_APPROVAL:true} - PYTEST_FLAGS = {env:PYTEST_FLAGS:--ignore=tests/integration/local/gateway_local_test.py --ignore=tests/integration/local/job_test.py} + PYTEST_FLAGS = {env:PYTEST_FLAGS:--ignore=tests/integration/local/gateway_local_test.py} commands = python -c 'import syft as sy; sy.stage_protocol_changes()' @@ -658,7 +658,7 @@ commands = PYTEST_MODULES=($PYTEST_MODULES); \ for i in "${PYTEST_MODULES[@]}"; do \ echo "Starting test for $i"; date; \ - pytest tests/integration -m $i -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no $PYTEST_FLAGS; \ + pytest tests/integration -n auto -m $i -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no $PYTEST_FLAGS; \ return=$?; \ echo "Finished $i"; \ date; \ From a187f07d0157415c5bbc19e00de10e625c438df4 Mon Sep 17 00:00:00 2001 From: Aziz Berkay Yesilyurt Date: Wed, 15 May 2024 10:52:12 +0200 Subject: [PATCH 2/7] fix job tests --- tests/integration/local/job_test.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/integration/local/job_test.py b/tests/integration/local/job_test.py index e713da731df..bf63f45b85f 100644 --- a/tests/integration/local/job_test.py +++ b/tests/integration/local/job_test.py @@ -2,7 +2,6 @@ # stdlib from secrets import token_hex -import time # third party import pytest @@ -24,6 +23,7 @@ def test_job_restart(job) -> None: assert wait_until( lambda: job.fetched_status == JobStatus.PROCESSING ), "Job not started" + assert wait_until(lambda: len(job.subjobs) == 2), "Subjobs not started" assert wait_until( lambda: all( subjob.fetched_status == JobStatus.PROCESSING for subjob in job.subjobs @@ -59,6 +59,10 @@ def test_job_restart(job) -> None: == 2 ), "Subjobs not restarted" + result = job.kill() + assert isinstance(result, SyftSuccess), "Should kill job" + assert job.fetched_status == JobStatus.INTERRUPTED + @pytest.fixture def node(): @@ -87,6 +91,7 @@ def job(node): @syft_function() def process_batch(): # stdlib + import time # noqa: F811 while time.sleep(1) is None: ... @@ -96,6 +101,7 @@ def process_batch(): @syft_function_single_use() def process_all(domain): # stdlib + import time # noqa: F811 _ = domain.launch_job(process_batch) _ = domain.launch_job(process_batch) From b857a453408a0dcb69d7ef6fcc8f788e6b18d707 Mon Sep 17 00:00:00 2001 From: Aziz Berkay Yesilyurt Date: Wed, 15 May 2024 11:21:11 +0200 Subject: [PATCH 3/7] enable local_node in k8s tests --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index a47a6241d33..6dd595d4e16 100644 --- a/tox.ini +++ b/tox.ini @@ -689,7 +689,7 @@ allowlist_externals = setenv = NODE_PORT = {env:NODE_PORT:9082} GITHUB_CI = {env:GITHUB_CI:false} - PYTEST_MODULES = {env:PYTEST_MODULES:frontend network container_workload} + PYTEST_MODULES = {env:PYTEST_MODULES:frontend network container_workload local_node} DOMAIN_CLUSTER_NAME = {env:DOMAIN_CLUSTER_NAME:test-domain-1} GATEWAY_CLUSTER_NAME = {env:GATEWAY_CLUSTER_NAME:test-gateway-1} ASSOCIATION_REQUEST_AUTO_APPROVAL = {env:ASSOCIATION_REQUEST_AUTO_APPROVAL:true} From 5b77044d8e5e40de5348ffeefc7d9f2b23900909 Mon Sep 17 00:00:00 2001 From: Aziz Berkay Yesilyurt Date: Wed, 15 May 2024 11:33:15 +0200 Subject: [PATCH 4/7] Revert "enable local_node in k8s tests" This reverts commit b857a453408a0dcb69d7ef6fcc8f788e6b18d707. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 6dd595d4e16..a47a6241d33 100644 --- a/tox.ini +++ b/tox.ini @@ -689,7 +689,7 @@ allowlist_externals = setenv = NODE_PORT = {env:NODE_PORT:9082} GITHUB_CI = {env:GITHUB_CI:false} - PYTEST_MODULES = {env:PYTEST_MODULES:frontend network container_workload local_node} + PYTEST_MODULES = {env:PYTEST_MODULES:frontend network container_workload} DOMAIN_CLUSTER_NAME = {env:DOMAIN_CLUSTER_NAME:test-domain-1} GATEWAY_CLUSTER_NAME = {env:GATEWAY_CLUSTER_NAME:test-gateway-1} ASSOCIATION_REQUEST_AUTO_APPROVAL = {env:ASSOCIATION_REQUEST_AUTO_APPROVAL:true} From 92d6d0b9dae1aeb00d4e52486eb964950be14203 Mon Sep 17 00:00:00 2001 From: khoaguin Date: Fri, 17 May 2024 11:58:34 +0700 Subject: [PATCH 5/7] [UX] change tqdm progress bar color to green --- packages/hagrid/hagrid/quickstart_ui.py | 4 ++-- packages/syft/src/syft/client/domain_client.py | 2 +- packages/syft/src/syft/store/blob_storage/seaweedfs.py | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/hagrid/hagrid/quickstart_ui.py b/packages/hagrid/hagrid/quickstart_ui.py index 9d1f8fc2652..3894a04a11c 100644 --- a/packages/hagrid/hagrid/quickstart_ui.py +++ b/packages/hagrid/hagrid/quickstart_ui.py @@ -104,7 +104,7 @@ def fetch_notebooks_for_url( print(nb) overwrite_all = False - for notebook_url in tqdm(notebooks): + for notebook_url in tqdm(notebooks, colour="green"): file_path, _, overwrite_all = quickstart_download_notebook( url=notebook_url, directory=os.path.abspath(directory + os.sep + str(url_dir) + os.sep), @@ -184,7 +184,7 @@ def fetch_notebooks_from_zipfile( extracted_files = [] overwrite_all = False - for notebook in tqdm(notebooks): + for notebook in tqdm(notebooks, colour="green"): file_path, _, overwrite_all = quickstart_extract_notebook( zip_file=path, name=notebook, diff --git a/packages/syft/src/syft/client/domain_client.py b/packages/syft/src/syft/client/domain_client.py index 75ec142bfde..8f1e7cb9dc8 100644 --- a/packages/syft/src/syft/client/domain_client.py +++ b/packages/syft/src/syft/client/domain_client.py @@ -122,7 +122,7 @@ def upload_dataset(self, dataset: CreateDataset) -> SyftSuccess | SyftError: ) prompt_warning_message(message=message, confirm=True) - for asset in tqdm(dataset.asset_list): + for asset in tqdm(dataset.asset_list, colour="green"): print(f"Uploading: {asset.name}") try: twin = TwinObject( diff --git a/packages/syft/src/syft/store/blob_storage/seaweedfs.py b/packages/syft/src/syft/store/blob_storage/seaweedfs.py index e31adc18b7d..1d88fedda37 100644 --- a/packages/syft/src/syft/store/blob_storage/seaweedfs.py +++ b/packages/syft/src/syft/store/blob_storage/seaweedfs.py @@ -80,6 +80,7 @@ def write(self, data: BytesIO) -> SyftSuccess | SyftError: with tqdm( total=total_iterations, desc=f"Uploading progress", # noqa + colour="green", ) as pbar: for part_no, url in enumerate( self.urls, From f52f81a4206deea13c49fb250c7c6d61613888a8 Mon Sep 17 00:00:00 2001 From: khoaguin Date: Mon, 20 May 2024 17:23:27 +0700 Subject: [PATCH 6/7] Resolved merge conflict by deleting packages/hagrid/hagrid/quickstart_ui.py --- packages/hagrid/hagrid/quickstart_ui.py | 356 ------------------------ 1 file changed, 356 deletions(-) delete mode 100644 packages/hagrid/hagrid/quickstart_ui.py diff --git a/packages/hagrid/hagrid/quickstart_ui.py b/packages/hagrid/hagrid/quickstart_ui.py deleted file mode 100644 index 3894a04a11c..00000000000 --- a/packages/hagrid/hagrid/quickstart_ui.py +++ /dev/null @@ -1,356 +0,0 @@ -# stdlib -from dataclasses import dataclass -import os -from pathlib import Path -import sys -from urllib.parse import urlparse -import zipfile - -# third party -import click -import requests -from tqdm import tqdm - -# relative -from .cache import DEFAULT_BRANCH -from .cache import DEFAULT_REPO -from .cache import arg_cache -from .nb_output import NBOutput - -directory = os.path.expanduser("~/.hagrid/quickstart/") - - -def quickstart_download_notebook( - url: str, directory: str, reset: bool = False, overwrite_all: bool = False -) -> tuple[str, bool, bool]: - os.makedirs(directory, exist_ok=True) - file_name = os.path.basename(url).replace("%20", "_").replace(" ", "_") - file_path = directory + os.sep + file_name - file_path = os.path.abspath(file_path) - - file_exists = os.path.isfile(file_path) - if overwrite_all: - reset = True - - if file_exists and not reset: - response = click.prompt( - f"\nOverwrite {file_name}?", - prompt_suffix="(a/y/N)", - default="n", - show_default=False, - ) - if response.lower() == "a": - reset = True - overwrite_all = True - elif response.lower() == "y": - reset = True - else: - print(f"Skipping {file_name}") - reset = False - - downloaded = False - if not file_exists or file_exists and reset: - print(f"Downloading notebook: {file_name}") - r = requests.get(url, allow_redirects=True) # nosec - with open(os.path.expanduser(file_path), "wb") as f: - f.write(r.content) - downloaded = True - return file_path, downloaded, overwrite_all - - -def fetch_notebooks_for_url( - url: str, - directory: str, - reset: bool = False, - repo: str = DEFAULT_REPO, - branch: str = DEFAULT_BRANCH, - commit: str | None = None, -) -> list[str]: - downloaded_files = [] - allowed_schemes_as_url = ["http", "https"] - url_scheme = urlparse(url).scheme - # relative mode - if url_scheme not in allowed_schemes_as_url: - notebooks = get_urls_from_dir(repo=repo, branch=branch, commit=commit, url=url) - if url.endswith(".ipynb"): - file_name = os.path.basename(url) - url_parts = url.split("notebooks") - if len(url_parts) > 1: - url_dir = url_parts[-1] - else: - url_dir = url - url_dir = url_dir.replace(file_name, "") - else: - url_dir = url - notebook_files = [] - existing_count = 0 - for notebook_url in notebooks: - url_filename = os.path.basename(notebook_url) - url_dirname = os.path.dirname(notebook_url) - if ( - url_dirname.endswith(url_dir) - and os.path.isdir(directory + url_dir) - and os.path.isfile(directory + url_dir + os.sep + url_filename) - ): - notebook_files.append(url_dir + os.sep + url_filename) - existing_count += 1 - - if existing_count > 0: - plural = "s" if existing_count > 1 else "" - print( - f"You have {existing_count} existing notebook{plural} matching: {url}" - ) - for nb in notebook_files: - print(nb) - - overwrite_all = False - for notebook_url in tqdm(notebooks, colour="green"): - file_path, _, overwrite_all = quickstart_download_notebook( - url=notebook_url, - directory=os.path.abspath(directory + os.sep + str(url_dir) + os.sep), - reset=reset, - overwrite_all=overwrite_all, - ) - downloaded_files.append(file_path) - - else: - file_path, _, _ = quickstart_download_notebook( - url=url, directory=directory, reset=reset - ) - downloaded_files.append(file_path) - return downloaded_files - - -def quickstart_extract_notebook( - zip_file: str, - name: str, - directory: Path, - reset: bool = False, - overwrite_all: bool = False, -) -> tuple[str, bool, bool]: - directory.mkdir(exist_ok=True) - reset = overwrite_all - - base_name = os.path.basename(name) - file_path = directory / base_name - file_name = file_path.name - file_exists = file_path.exists() - - if file_exists and not reset: - response = click.prompt( - f"\nOverwrite {file_name}?", - prompt_suffix="(a/y/N)", - default="n", - show_default=False, - ) - if response.lower() == "a": - reset = True - overwrite_all = True - elif response.lower() == "y": - reset = True - else: - print(f"Skipping {file_name}") - reset = False - - extracted = False - if not file_exists or file_exists and reset: - print(f"Extracting notebook: {file_name}") - with zipfile.ZipFile(zip_file, "r") as zf: - zip_info = zf.getinfo(name) - zip_info.filename = base_name - zf.extract(zip_info, directory) - extracted = True - return str(file_path.absolute()), extracted, overwrite_all - - -def fetch_notebooks_from_zipfile( - path: str, directory: str, reset: bool = False -) -> list[str]: - dir_path = Path(directory) - - with zipfile.ZipFile(path, "r") as zf: - notebooks = [f for f in zf.namelist() if f.endswith(".ipynb")] - - notebook_files = [dir_path / os.path.basename(nb) for nb in notebooks] - existing_files = [nb for nb in notebook_files if nb.exists()] - - existing_count = len(existing_files) - - if existing_count > 0: - plural = "s" if existing_count > 1 else "" - print(f"You have {existing_count} existing notebook{plural}") - for nb in existing_files: - print(nb) - - extracted_files = [] - overwrite_all = False - for notebook in tqdm(notebooks, colour="green"): - file_path, _, overwrite_all = quickstart_extract_notebook( - zip_file=path, - name=notebook, - directory=dir_path, - reset=reset, - overwrite_all=overwrite_all, - ) - extracted_files.append(file_path) - - return extracted_files - - -@dataclass -class Tutorial: - filename: str - description: str - url: str - - -REPO_RAW_PATH = "https://raw.githubusercontent.com/OpenMined/PySyft" - -TUTORIALS = { - "api/0.8": Tutorial( - filename="api/0.8", - description="0.8 API Notebooks", - url="api/0.8", - ), - "hello-syft": Tutorial( - filename="tutorials/hello-syft", - description="Hello Syft", - url="tutorials/hello-syft", - ), - "data-engineer": Tutorial( - filename="tutorials/data-engineer", - description="Data Engineer", - url="tutorials/data-engineer", - ), - "data-owner": Tutorial( - filename="tutorials/data-owner", - description="Data Owner", - url="tutorials/data-owner", - ), - "data-scientist": Tutorial( - filename="tutorials/data-scientist", - description="Data Scientist", - url="tutorials/data-scientist", - ), - "pandas-cookbook": Tutorial( - filename="tutorials/pandas-cookbook", - description="Pandas Cookbook", - url="tutorials/pandas-cookbook", - ), -} - - -class QuickstartUI: - @property - def tutorials(self) -> dict[str, Tutorial]: - return TUTORIALS - - def download( - self, tutorial_name: str, reset: bool = False, branch: str = "dev" - ) -> NBOutput: - if tutorial_name not in TUTORIALS.keys(): - return NBOutput( - f'
{tutorial_name} is not a valid tutorial name.
' - ) - else: - tutorial = TUTORIALS[tutorial_name] - downloaded_files = fetch_notebooks_for_url( - url=tutorial.url, directory=directory, branch=branch - ) - html = "" - if len(downloaded_files) == 0: - html += f'
{tutorial_name} failed to download.' - else: - first = downloaded_files[0] - jupyter_path = first.replace(os.path.abspath(directory) + "/", "") - - html += f'
{tutorial_name} downloaded.' - html += f'
📖 Click to Open Tutorial
' - return NBOutput(html) - - def _repr_html_(self) -> str: - html = "" - if not arg_cache["install_wizard_complete"]: - html += "

Step 1b: Install 🧙🏽‍♂️ Wizard (Recommended)

" - html += ( - "It looks like this might be your first time running Quickstart.
" - ) - html += ( - "
Please go through the Install Wizard notebook to " - + "install Syft and optionally start a Grid server." - ) - html += ( - '
📖 Click to start ' - + "Install 🧙🏽‍♂️ Wizard
" - ) - html += "
" - - html += "

Download Tutorials

" - html += "Below is a list of tutorials to download using quickstart.
" - html += "
    " - for name, tutorial in TUTORIALS.items(): - html += ( - "
  • 📖 Tutorial Series: " - + f"{name}
    {tutorial.description}
    • " - ) - html += "
" - first = list(TUTORIALS.keys())[0] - html += ( - "
Try running:
" - + f'quickstart.download("{first}")
' - ) - - return html - - -def get_urls_from_dir( - url: str, - repo: str, - branch: str, - commit: str | None = None, -) -> list[str]: - notebooks = [] - slug = commit if commit else branch - - gh_api_call = ( - "https://api.github.com/repos/" + repo + "/git/trees/" + slug + "?recursive=1" - ) - r = requests.get(gh_api_call) # nosec - if r.status_code != 200: - print( - f"Failed to fetch notebook from: {gh_api_call}.\n" - + "Please try again with the correct parameters!" - ) - sys.exit(1) - - res = r.json() - - for file in res["tree"]: - if file["path"].startswith("notebooks/quickstart/" + url) or file[ - "path" - ].startswith("notebooks/" + url): - if file["path"].endswith(".ipynb"): - temp_url = ( - "https://raw.githubusercontent.com/" - + repo - + "/" - + slug - + "/" - + file["path"] - ) - notebooks.append(temp_url) - - if len(notebooks) == 0: - for file in res["tree"]: - if file["path"].startswith("notebooks/" + url): - if file["path"].endswith(".ipynb"): - temp_url = ( - "https://raw.githubusercontent.com/" - + repo - + "/" - + slug - + "/" - + file["path"] - ) - notebooks.append(temp_url) - return notebooks From b53f88d8282c9d660dda9ca1bd670a1d7c7613ad Mon Sep 17 00:00:00 2001 From: Aziz Berkay Yesilyurt Date: Tue, 21 May 2024 13:11:19 +0200 Subject: [PATCH 7/7] Revert "enable job tests in integration tests" --- tests/integration/local/job_test.py | 8 +------- tox.ini | 4 ++-- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/tests/integration/local/job_test.py b/tests/integration/local/job_test.py index bf63f45b85f..e713da731df 100644 --- a/tests/integration/local/job_test.py +++ b/tests/integration/local/job_test.py @@ -2,6 +2,7 @@ # stdlib from secrets import token_hex +import time # third party import pytest @@ -23,7 +24,6 @@ def test_job_restart(job) -> None: assert wait_until( lambda: job.fetched_status == JobStatus.PROCESSING ), "Job not started" - assert wait_until(lambda: len(job.subjobs) == 2), "Subjobs not started" assert wait_until( lambda: all( subjob.fetched_status == JobStatus.PROCESSING for subjob in job.subjobs @@ -59,10 +59,6 @@ def test_job_restart(job) -> None: == 2 ), "Subjobs not restarted" - result = job.kill() - assert isinstance(result, SyftSuccess), "Should kill job" - assert job.fetched_status == JobStatus.INTERRUPTED - @pytest.fixture def node(): @@ -91,7 +87,6 @@ def job(node): @syft_function() def process_batch(): # stdlib - import time # noqa: F811 while time.sleep(1) is None: ... @@ -101,7 +96,6 @@ def process_batch(): @syft_function_single_use() def process_all(domain): # stdlib - import time # noqa: F811 _ = domain.launch_job(process_batch) _ = domain.launch_job(process_batch) diff --git a/tox.ini b/tox.ini index 69f98046aef..c79fa513cb1 100644 --- a/tox.ini +++ b/tox.ini @@ -477,7 +477,7 @@ allowlist_externals = setenv = PYTEST_MODULES = {env:PYTEST_MODULES:local_node} ASSOCIATION_REQUEST_AUTO_APPROVAL = {env:ASSOCIATION_REQUEST_AUTO_APPROVAL:true} - PYTEST_FLAGS = {env:PYTEST_FLAGS:--ignore=tests/integration/local/gateway_local_test.py} + PYTEST_FLAGS = {env:PYTEST_FLAGS:--ignore=tests/integration/local/gateway_local_test.py --ignore=tests/integration/local/job_test.py} commands = python -c 'import syft as sy; sy.stage_protocol_changes()' @@ -486,7 +486,7 @@ commands = PYTEST_MODULES=($PYTEST_MODULES); \ for i in "${PYTEST_MODULES[@]}"; do \ echo "Starting test for $i"; date; \ - pytest tests/integration -n auto -m $i -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no $PYTEST_FLAGS; \ + pytest tests/integration -m $i -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no $PYTEST_FLAGS; \ return=$?; \ echo "Finished $i"; \ date; \