From 0c12c42ffb6e36423906abe818b3f32fb28216ad Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Mon, 9 Oct 2023 18:04:26 +0530 Subject: [PATCH 1/8] added url mapping in template --- packages/hagrid/hagrid/cli.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/packages/hagrid/hagrid/cli.py b/packages/hagrid/hagrid/cli.py index 3d7a5b92044..9158e62917a 100644 --- a/packages/hagrid/hagrid/cli.py +++ b/packages/hagrid/hagrid/cli.py @@ -1358,23 +1358,30 @@ def create_launch_cmd( and parsed_kwargs["template"] is None and parsed_kwargs["tag"] not in ["local", "0.7.0"] ): + # third party + from packaging import version + # TODO: we need to redo this so that pypi and docker mappings are in a single # file inside dev if parsed_kwargs["tag"] == "latest": parsed_kwargs["template"] = LATEST_STABLE_SYFT parsed_kwargs["tag"] = LATEST_STABLE_SYFT elif parsed_kwargs["tag"] == "beta": - parsed_kwargs["template"] = "dev" + beta_version = version.parse(LATEST_BETA_SYFT) + parsed_kwargs[ + "template" + ] = f"https://github.com/OpenMined/PySyft/releases/download/v{str(beta_version)}/manifest.yml" parsed_kwargs["tag"] = LATEST_BETA_SYFT else: - template = parsed_kwargs["tag"] - # 🟡 TODO: Revert to use tags once, we have tag branches with beta - # versions also. - if "b" in template: - template = "dev" - # if template == "beta": - # template = "dev" - parsed_kwargs["template"] = template + tag = parsed_kwargs["tag"] + + if "b" in tag: + beta_version = version.parse(tag) + parsed_kwargs[ + "template" + ] = f"https://github.com/OpenMined/PySyft/releases/download/v{str(beta_version)}/manifest.yml" + else: + raise Exception(f"Not a valid beta version: {tag}") if host in ["docker"] and parsed_kwargs["template"] and host is not None: # Setup the files from the manifest_template.yml From 3bee6c9e179db896e8298641e8348c8a4e6c4df7 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Mon, 9 Oct 2023 18:26:01 +0530 Subject: [PATCH 2/8] updated url --- packages/hagrid/hagrid/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/hagrid/hagrid/cli.py b/packages/hagrid/hagrid/cli.py index 9158e62917a..6d089941a71 100644 --- a/packages/hagrid/hagrid/cli.py +++ b/packages/hagrid/hagrid/cli.py @@ -1370,7 +1370,7 @@ def create_launch_cmd( beta_version = version.parse(LATEST_BETA_SYFT) parsed_kwargs[ "template" - ] = f"https://github.com/OpenMined/PySyft/releases/download/v{str(beta_version)}/manifest.yml" + ] = f"https://github.com/OpenMined/PySyft/releases/download/v{str(beta_version)}/manifest_template.yml" parsed_kwargs["tag"] = LATEST_BETA_SYFT else: tag = parsed_kwargs["tag"] @@ -1379,7 +1379,7 @@ def create_launch_cmd( beta_version = version.parse(tag) parsed_kwargs[ "template" - ] = f"https://github.com/OpenMined/PySyft/releases/download/v{str(beta_version)}/manifest.yml" + ] = f"https://github.com/OpenMined/PySyft/releases/download/v{str(beta_version)}/manifest_template.yml" else: raise Exception(f"Not a valid beta version: {tag}") From 4d3c836fb24fa8c90d72016f180d108f132fba34 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Tue, 10 Oct 2023 07:59:02 +0530 Subject: [PATCH 3/8] improved exception handling at edge cases --- packages/hagrid/hagrid/cli.py | 46 +++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/packages/hagrid/hagrid/cli.py b/packages/hagrid/hagrid/cli.py index 6d089941a71..884abcf1a8a 100644 --- a/packages/hagrid/hagrid/cli.py +++ b/packages/hagrid/hagrid/cli.py @@ -1366,22 +1366,38 @@ def create_launch_cmd( if parsed_kwargs["tag"] == "latest": parsed_kwargs["template"] = LATEST_STABLE_SYFT parsed_kwargs["tag"] = LATEST_STABLE_SYFT - elif parsed_kwargs["tag"] == "beta": - beta_version = version.parse(LATEST_BETA_SYFT) - parsed_kwargs[ - "template" - ] = f"https://github.com/OpenMined/PySyft/releases/download/v{str(beta_version)}/manifest_template.yml" - parsed_kwargs["tag"] = LATEST_BETA_SYFT - else: - tag = parsed_kwargs["tag"] + elif parsed_kwargs["tag"] == "beta" or "b" in parsed_kwargs["tag"]: + tag = ( + LATEST_BETA_SYFT + if parsed_kwargs["tag"] == "beta" + else parsed_kwargs["tag"] + ) - if "b" in tag: - beta_version = version.parse(tag) - parsed_kwargs[ - "template" - ] = f"https://github.com/OpenMined/PySyft/releases/download/v{str(beta_version)}/manifest_template.yml" - else: - raise Exception(f"Not a valid beta version: {tag}") + # Currently, manifest_template.yml is only supported for beta versions >= 0.8.2b34 + beta_version = version.parse(tag) + MINIMUM_BETA_VERSION = "0.8.2b34" + if beta_version < version.parse(MINIMUM_BETA_VERSION): + raise Exception( + f"Minimum beta version tag supported is {MINIMUM_BETA_VERSION}" + ) + + # Check if the beta version is available + template_url = f"https://github.com/OpenMined/PySyft/releases/download/v{str(beta_version)}/manifest_template.yml" + response = requests.get(template_url) # nosec + if response.status_code != 200: + raise Exception( + f"Tag {parsed_kwargs['tag']} is not available" + + " \n for download. Please check the available tags at: " + + "\n https://github.com/OpenMined/PySyft/releases" + ) + + parsed_kwargs["template"] = template_url + parsed_kwargs["tag"] = tag + else: + raise Exception( + f"Not a valid tag: {parsed_kwargs['tag']}" + + "\nValid tags: latest, beta, beta version(ex: 0.8.2b35)" + ) if host in ["docker"] and parsed_kwargs["template"] and host is not None: # Setup the files from the manifest_template.yml From ffbd2e7388ea0177bbe91e9d122397b737a6d00b Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Tue, 10 Oct 2023 11:28:21 +0530 Subject: [PATCH 4/8] refactored version deployment --- packages/hagrid/hagrid/cli.py | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/packages/hagrid/hagrid/cli.py b/packages/hagrid/hagrid/cli.py index 884abcf1a8a..4e8e15b2c93 100644 --- a/packages/hagrid/hagrid/cli.py +++ b/packages/hagrid/hagrid/cli.py @@ -130,8 +130,10 @@ def get_compose_src_path( **kwargs: TypeDict[str, Any], ) -> str: grid_path = GRID_SRC_PATH() - tag = kwargs.get("tag", None) - if EDITABLE_MODE and template_location is None or tag == "0.7.0": # type: ignore + tag = kwargs["tag"] + # Use local compose files if in editable mode and + # template_location is None and (tag is None or tag is local) + if EDITABLE_MODE and template_location is None and (tag is None or tag == "local"): # type: ignore path = grid_path else: path = deployment_dir(node_name) @@ -1356,11 +1358,24 @@ def create_launch_cmd( if ( parsed_kwargs["tag"] is not None and parsed_kwargs["template"] is None - and parsed_kwargs["tag"] not in ["local", "0.7.0"] + and parsed_kwargs["tag"] not in ["local"] ): # third party from packaging import version + pattern = r"[0-9].[0-9].[0-9]" + input_tag = parsed_kwargs["tag"] + if ( + not re.match(pattern, input_tag) + and input_tag != "latest" + and input_tag != "beta" + and "b" not in input_tag + ): + raise Exception( + f"Not a valid tag: {parsed_kwargs['tag']}" + + "\nValid tags: latest, beta, beta version(ex: 0.8.2b35),[0-9].[0-9].[0-9]" + ) + # TODO: we need to redo this so that pypi and docker mappings are in a single # file inside dev if parsed_kwargs["tag"] == "latest": @@ -1394,10 +1409,13 @@ def create_launch_cmd( parsed_kwargs["template"] = template_url parsed_kwargs["tag"] = tag else: - raise Exception( - f"Not a valid tag: {parsed_kwargs['tag']}" - + "\nValid tags: latest, beta, beta version(ex: 0.8.2b35)" - ) + MINIMUM_TAG_VERSION = version.parse("0.8.0") + tag = version.parse(parsed_kwargs["tag"]) + if tag < MINIMUM_TAG_VERSION: + raise Exception( + f"Minimum supported stable tag version is {MINIMUM_TAG_VERSION}" + ) + parsed_kwargs["template"] = parsed_kwargs["tag"] if host in ["docker"] and parsed_kwargs["template"] and host is not None: # Setup the files from the manifest_template.yml From 91bea9d857b0e873e4c8abdd58b77dcb7f7323a1 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Tue, 10 Oct 2023 12:08:56 +0530 Subject: [PATCH 5/8] Added dev mode flag --- packages/hagrid/hagrid/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/hagrid/hagrid/cli.py b/packages/hagrid/hagrid/cli.py index 4e8e15b2c93..35852e6521f 100644 --- a/packages/hagrid/hagrid/cli.py +++ b/packages/hagrid/hagrid/cli.py @@ -132,8 +132,8 @@ def get_compose_src_path( grid_path = GRID_SRC_PATH() tag = kwargs["tag"] # Use local compose files if in editable mode and - # template_location is None and (tag is None or tag is local) - if EDITABLE_MODE and template_location is None and (tag is None or tag == "local"): # type: ignore + # template_location is None and (kwargs["dev"] is True or tag is local) + if EDITABLE_MODE and template_location is None and (kwargs["dev"] is True or tag == "local"): # type: ignore path = grid_path else: path = deployment_dir(node_name) From b9dbe4e34dc4905f5a6d4effd9336028be19b071 Mon Sep 17 00:00:00 2001 From: Ionesio Junior Date: Tue, 10 Oct 2023 09:05:09 -0300 Subject: [PATCH 6/8] Fix indexing for search results --- packages/syft/src/syft/util/notebook_ui/notebook_addons.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/syft/src/syft/util/notebook_ui/notebook_addons.py b/packages/syft/src/syft/util/notebook_ui/notebook_addons.py index d0d1d463fc9..7bb952b1d84 100644 --- a/packages/syft/src/syft/util/notebook_ui/notebook_addons.py +++ b/packages/syft/src/syft/util/notebook_ui/notebook_addons.py @@ -556,6 +556,7 @@ resetById${uid}('table${uid}'); resetById${uid}('pag${uid}'); result = paginate${uid}(result, page_size${uid}) + paginatedElements${uid} = result buildGrid${uid}(result,pageIndex${uid}); buildPaginationContainer${uid}(result); } From 93b9bc52985cc0456dc75ae6155bbf33d41b3fd9 Mon Sep 17 00:00:00 2001 From: Madhava Jay Date: Wed, 11 Oct 2023 12:38:51 +1000 Subject: [PATCH 7/8] Disabled some of the mongo thread tests on MacOS - These tests regularly flap, its likely this is related to the FileLock / Thread issues we have on MacOS --- .../syft/stores/mongo_document_store_test.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/syft/tests/syft/stores/mongo_document_store_test.py b/packages/syft/tests/syft/stores/mongo_document_store_test.py index 599d387bb9e..f8bad27165a 100644 --- a/packages/syft/tests/syft/stores/mongo_document_store_test.py +++ b/packages/syft/tests/syft/stores/mongo_document_store_test.py @@ -623,7 +623,7 @@ def _kv_cbk(tid: int) -> None: @pytest.mark.skipif( - sys.platform == "win32", reason="pytest_mock_resources + docker issues on Windows" + sys.platform != "linux", reason="pytest_mock_resources + docker issues on Windows" ) def test_mongo_store_partition_permissions_collection( mongo_store_partition: MongoStorePartition, @@ -638,7 +638,7 @@ def test_mongo_store_partition_permissions_collection( @pytest.mark.skipif( - sys.platform == "win32", reason="pytest_mock_resources + docker issues on Windows" + sys.platform != "linux", reason="pytest_mock_resources + docker issues on Windows" ) def test_mongo_store_partition_add_remove_permission( root_verify_key: SyftVerifyKey, mongo_store_partition: MongoStorePartition @@ -729,7 +729,7 @@ def test_mongo_store_partition_add_remove_permission( @pytest.mark.skipif( - sys.platform == "win32", reason="pytest_mock_resources + docker issues on Windows" + sys.platform != "linux", reason="pytest_mock_resources + docker issues on Windows" ) def test_mongo_store_partition_add_permissions( root_verify_key: SyftVerifyKey, @@ -781,7 +781,7 @@ def test_mongo_store_partition_add_permissions( @pytest.mark.skipif( - sys.platform == "win32", reason="pytest_mock_resources + docker issues on Windows" + sys.platform != "linux", reason="pytest_mock_resources + docker issues on Windows" ) @pytest.mark.parametrize("permission", PERMISSIONS) def test_mongo_store_partition_has_permission( @@ -830,7 +830,7 @@ def test_mongo_store_partition_has_permission( @pytest.mark.skipif( - sys.platform == "win32", reason="pytest_mock_resources + docker issues on Windows" + sys.platform != "linux", reason="pytest_mock_resources + docker issues on Windows" ) @pytest.mark.parametrize("permission", PERMISSIONS) def test_mongo_store_partition_take_ownership( @@ -885,7 +885,7 @@ def test_mongo_store_partition_take_ownership( @pytest.mark.skipif( - sys.platform == "win32", reason="pytest_mock_resources + docker issues on Windows" + sys.platform != "linux", reason="pytest_mock_resources + docker issues on Windows" ) def test_mongo_store_partition_permissions_set( root_verify_key: SyftVerifyKey, @@ -931,7 +931,7 @@ def test_mongo_store_partition_permissions_set( @pytest.mark.skipif( - sys.platform == "win32", reason="pytest_mock_resources + docker issues on Windows" + sys.platform != "linux", reason="pytest_mock_resources + docker issues on Windows" ) def test_mongo_store_partition_permissions_get_all( root_verify_key: SyftVerifyKey, @@ -964,7 +964,7 @@ def test_mongo_store_partition_permissions_get_all( @pytest.mark.skipif( - sys.platform == "win32", reason="pytest_mock_resources + docker issues on Windows" + sys.platform != "linux", reason="pytest_mock_resources + docker issues on Windows" ) def test_mongo_store_partition_permissions_delete( root_verify_key: SyftVerifyKey, @@ -1018,7 +1018,7 @@ def test_mongo_store_partition_permissions_delete( @pytest.mark.skipif( - sys.platform == "win32", reason="pytest_mock_resources + docker issues on Windows" + sys.platform != "linux", reason="pytest_mock_resources + docker issues on Windows" ) def test_mongo_store_partition_permissions_update( root_verify_key: SyftVerifyKey, From c1c4875d59abf2ce106a2a2a73f8f1b7d72dddb6 Mon Sep 17 00:00:00 2001 From: alfred-openmined-bot <145415986+alfred-openmined-bot@users.noreply.github.com> Date: Wed, 11 Oct 2023 04:43:46 +0000 Subject: [PATCH 8/8] [hagrid] bump version --- packages/hagrid/.bumpversion.cfg | 2 +- packages/hagrid/hagrid/manifest_template.yml | 4 ++-- packages/hagrid/hagrid/version.py | 2 +- packages/hagrid/setup.py | 2 +- scripts/hagrid_hash | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/hagrid/.bumpversion.cfg b/packages/hagrid/.bumpversion.cfg index 502fe7effad..1d006900387 100644 --- a/packages/hagrid/.bumpversion.cfg +++ b/packages/hagrid/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.3.76 +current_version = 0.3.77 tag = False tag_name = {new_version} commit = True diff --git a/packages/hagrid/hagrid/manifest_template.yml b/packages/hagrid/hagrid/manifest_template.yml index c1671ab4912..4de4a9a3173 100644 --- a/packages/hagrid/hagrid/manifest_template.yml +++ b/packages/hagrid/hagrid/manifest_template.yml @@ -1,9 +1,9 @@ manifestVersion: 0.1 -hagrid_version: 0.3.76 +hagrid_version: 0.3.77 syft_version: 0.8.2-beta.35 dockerTag: 0.8.2-beta.35 baseUrl: https://raw.githubusercontent.com/OpenMined/PySyft/ -hash: 816fedf28e3636e12a6093e6c4be39f4dc000b10 +hash: 7cc9072e300872e02ae8bfcecf7bb000cdea6a17 target_dir: ~/.hagrid/PySyft/ files: grid: diff --git a/packages/hagrid/hagrid/version.py b/packages/hagrid/hagrid/version.py index 62b4c4f8dcc..e4119daf658 100644 --- a/packages/hagrid/hagrid/version.py +++ b/packages/hagrid/hagrid/version.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # HAGrid Version -__version__ = "0.3.76" +__version__ = "0.3.77" if __name__ == "__main__": print(__version__) diff --git a/packages/hagrid/setup.py b/packages/hagrid/setup.py index 6454c6ebc54..4ba9940fcd7 100644 --- a/packages/hagrid/setup.py +++ b/packages/hagrid/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages from setuptools import setup -__version__ = "0.3.76" +__version__ = "0.3.77" DATA_FILES = {"img": ["hagrid/img/*.png"], "hagrid": ["*.yml"]} diff --git a/scripts/hagrid_hash b/scripts/hagrid_hash index 5f6869eb1f9..ecc0626ddd9 100644 --- a/scripts/hagrid_hash +++ b/scripts/hagrid_hash @@ -1 +1 @@ -e36222a225f8ce9ad3579d7c2805784c +88589c42b08774c056bb2578a4b6c26d