From 275eb5b58e0a67f16bc059c89d35bd3c25d0d781 Mon Sep 17 00:00:00 2001 From: Scott Black Date: Mon, 3 May 2021 07:11:37 -0600 Subject: [PATCH 1/7] fix absolute paths for windows --- hsclient/hydroshare.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/hsclient/hydroshare.py b/hsclient/hydroshare.py index d00c55c..f59b8f4 100644 --- a/hsclient/hydroshare.py +++ b/hsclient/hydroshare.py @@ -5,7 +5,7 @@ import tempfile import time from datetime import datetime -from posixpath import basename, dirname, join as urljoin, splitext +from posixpath import join as urljoin, splitext from typing import Dict, List, Union from urllib.parse import urlparse, quote, unquote from zipfile import ZipFile @@ -45,7 +45,7 @@ def path(self) -> str: @property def name(self) -> str: """The filename""" - return basename(self) + return os.path.basename(self) @property def extension(self) -> str: @@ -55,7 +55,7 @@ def extension(self) -> str: @property def folder(self) -> str: """The folder the file is in""" - return dirname(self) + return os.path.dirname(self) @property def checksum(self): @@ -526,7 +526,7 @@ def file_zip(self, path: str, zip_name: str = None, remove_file: bool = True) -> :param zip_name: The name of the zipped file :param remove_file: Defaults to True, set to False to not delete the file that was zipped """ - zip_name = basename(path) + ".zip" if not zip_name else zip_name + zip_name = os.path.basename(path) + ".zip" if not zip_name else zip_name data = {"input_coll_path": path, "output_zip_file_name": zip_name, "remove_original_after_zip": remove_file} zip_path = urljoin(self._hsapi_path, "functions", "zip") self._hs_session.post(zip_path, status_code=200, data=data) @@ -557,7 +557,7 @@ def file_aggregate(self, path, agg_type: AggregationType): if agg_type == AggregationType.SingleFileAggregation: type_value = 'SingleFile' if agg_type == AggregationType.FileSetAggregation: - relative_path = dirname(path) + relative_path = os.path.dirname(path) data = {"folder_path": relative_path} url = urljoin(self._hsapi_path, "functions", "set-file-type", path, type_value) @@ -578,7 +578,7 @@ def file_upload(self, *files: str, destination_path: str = "") -> None: zipped_file = os.path.join(tmpdir, 'files.zip') with ZipFile(zipped_file, 'w') as zipped: for file in files: - zipped.write(file, basename(file)) + zipped.write(file, os.path.basename(file)) self._upload(zipped_file, destination_path=destination_path) unzip_path = urljoin( self._hsapi_path, "functions", "unzip", "data", "contents", destination_path, 'files.zip' From 0bb9dec10dc39ba01aad2fdd44d27849d32ab56b Mon Sep 17 00:00:00 2001 From: Scott Black Date: Mon, 3 May 2021 08:00:32 -0600 Subject: [PATCH 2/7] add tests for absolute paths --- tests/data/another.txt | 1 + tests/test_functional.py | 22 +++++++++++++++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) create mode 100644 tests/data/another.txt diff --git a/tests/data/another.txt b/tests/data/another.txt new file mode 100644 index 0000000..65763bf --- /dev/null +++ b/tests/data/another.txt @@ -0,0 +1 @@ +another hello dude \ No newline at end of file diff --git a/tests/test_functional.py b/tests/test_functional.py index 349f217..d4df802 100644 --- a/tests/test_functional.py +++ b/tests/test_functional.py @@ -47,10 +47,26 @@ def timeseries_resource(new_resource): "ODM2_Multi_Site_One_Variable_meta.xml", ] root_path = "data/test_resource_metadata_files/" - new_resource.file_upload(*[root_path + file for file in files]) + new_resource.file_upload(*[os.path.join(root_path, file) for file in files]) return new_resource +def test_absolute_path_multiple_file_upload(new_resource): + files = [ + "other.txt", + "another.txt", + ] + root_path = "data" + new_resource.file_upload(*[os.path.abspath(os.path.join(root_path, file)) for file in files]) + assert len(new_resource.files()) == 2 + + +def test_absolute_path_single_file_upload(new_resource): + rel_path = os.path.join("data", "other.txt") + new_resource.file_upload(os.path.abspath(rel_path)) + assert len(new_resource.files()) == 1 + + def test_filtering_aggregations(timeseries_resource): assert len(timeseries_resource.aggregations(type=AggregationType.TimeSeriesAggregation)) == 1 timeseries = timeseries_resource.aggregation(type=AggregationType.TimeSeriesAggregation) @@ -387,7 +403,7 @@ def test_user_info(hydroshare): def test_aggregations(new_resource, files): root_path = "data/test_resource_metadata_files/" file_count = len(files) - 2 # exclude rdf/xml file - new_resource.file_upload(*[root_path + file for file in files]) + new_resource.file_upload(*[os.path.join(root_path, file) for file in files]) assert len(new_resource.aggregations()) == 1 assert len(new_resource.files()) == 0 agg = new_resource.aggregations()[0] @@ -425,7 +441,7 @@ def test_aggregation_fileset(new_resource, files): root_path = "data/test_resource_metadata_files/" file_count = len(files) - 2 # exclude rdf/xml file new_resource.folder_create("asdf") - new_resource.file_upload(*[root_path + file for file in files], destination_path="asdf") + new_resource.file_upload(*[os.path.join(root_path, file) for file in files], destination_path="asdf") assert len(new_resource.aggregations()) == 1 assert len(new_resource.files()) == 0 agg = new_resource.aggregations()[0] From 796200ae1732c8684f34f195fe3122a52d52f515 Mon Sep 17 00:00:00 2001 From: Scott Black Date: Mon, 3 May 2021 12:50:01 -0600 Subject: [PATCH 3/7] use posixpath for paths on hydroshare --- hsclient/hydroshare.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/hsclient/hydroshare.py b/hsclient/hydroshare.py index f59b8f4..6ab46c4 100644 --- a/hsclient/hydroshare.py +++ b/hsclient/hydroshare.py @@ -5,7 +5,7 @@ import tempfile import time from datetime import datetime -from posixpath import join as urljoin, splitext +from posixpath import join as urljoin, splitext, basename, dirname from typing import Dict, List, Union from urllib.parse import urlparse, quote, unquote from zipfile import ZipFile @@ -45,7 +45,7 @@ def path(self) -> str: @property def name(self) -> str: """The filename""" - return os.path.basename(self) + return basename(self) @property def extension(self) -> str: @@ -55,7 +55,7 @@ def extension(self) -> str: @property def folder(self) -> str: """The folder the file is in""" - return os.path.dirname(self) + return dirname(self) @property def checksum(self): @@ -526,7 +526,7 @@ def file_zip(self, path: str, zip_name: str = None, remove_file: bool = True) -> :param zip_name: The name of the zipped file :param remove_file: Defaults to True, set to False to not delete the file that was zipped """ - zip_name = os.path.basename(path) + ".zip" if not zip_name else zip_name + zip_name = basename(path) + ".zip" if not zip_name else zip_name data = {"input_coll_path": path, "output_zip_file_name": zip_name, "remove_original_after_zip": remove_file} zip_path = urljoin(self._hsapi_path, "functions", "zip") self._hs_session.post(zip_path, status_code=200, data=data) @@ -557,7 +557,7 @@ def file_aggregate(self, path, agg_type: AggregationType): if agg_type == AggregationType.SingleFileAggregation: type_value = 'SingleFile' if agg_type == AggregationType.FileSetAggregation: - relative_path = os.path.dirname(path) + relative_path = dirname(path) data = {"folder_path": relative_path} url = urljoin(self._hsapi_path, "functions", "set-file-type", path, type_value) From 0bc0172fdedf95d851eb8491705bc1cc7bd0b7bb Mon Sep 17 00:00:00 2001 From: Scott Black Date: Mon, 3 May 2021 13:28:08 -0600 Subject: [PATCH 4/7] increment version and add windows image to automated testing --- .github/workflows/python-package.yml | 4 +++- Makefile | 4 ++-- setup.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index c36c15f..8d5cb55 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -12,10 +12,12 @@ on: jobs: build: - runs-on: ubuntu-latest strategy: matrix: python-version: [3.7] + platform: [ubuntu-latest, windows-latest] + + runs-on: { { matrix.platform } } steps: - uses: actions/checkout@v2 diff --git a/Makefile b/Makefile index 5104854..e7dc507 100644 --- a/Makefile +++ b/Makefile @@ -21,8 +21,8 @@ docs-serve: .PHONY: test test: - pytest -n 8 tests + pytest -n 4 tests .PHONY: test-cov test-cov: - pytest -n 8 --cov=hsclient --cov-report html + pytest -n 4 --cov=hsclient --cov-report html diff --git a/setup.py b/setup.py index d914c4e..68f1c2e 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='hsclient', - version='0.1.6', + version='0.1.7', packages=find_packages(include=['hsclient', 'hsclient.*'], exclude=("tests",)), install_requires=[ From bc4b37641e3def2e123a43721c6dd2c3c9f47056 Mon Sep 17 00:00:00 2001 From: Scott Black Date: Mon, 3 May 2021 13:33:28 -0600 Subject: [PATCH 5/7] fix github actions runs-on platform matrix --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 8d5cb55..3fdcd82 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -17,7 +17,7 @@ jobs: python-version: [3.7] platform: [ubuntu-latest, windows-latest] - runs-on: { { matrix.platform } } + runs-on: ${{ matrix.platform }} steps: - uses: actions/checkout@v2 From f3bf24980cc58442ab48ad2ec2eefffea2af8a4c Mon Sep 17 00:00:00 2001 From: Scott Black Date: Mon, 3 May 2021 13:47:19 -0600 Subject: [PATCH 6/7] drop test concurrency down to 2 --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index e7dc507..b41845a 100644 --- a/Makefile +++ b/Makefile @@ -21,8 +21,8 @@ docs-serve: .PHONY: test test: - pytest -n 4 tests + pytest -n 2 tests .PHONY: test-cov test-cov: - pytest -n 4 --cov=hsclient --cov-report html + pytest -n 2 --cov=hsclient --cov-report html From b400a5429b14093b397df2d3a8a90de123949daf Mon Sep 17 00:00:00 2001 From: Scott Black Date: Mon, 3 May 2021 14:05:19 -0600 Subject: [PATCH 7/7] only test on ubuntu --- .github/workflows/python-package.yml | 2 +- Makefile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 3fdcd82..721d88b 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -15,7 +15,7 @@ jobs: strategy: matrix: python-version: [3.7] - platform: [ubuntu-latest, windows-latest] + platform: [ubuntu-latest] runs-on: ${{ matrix.platform }} diff --git a/Makefile b/Makefile index b41845a..e7dc507 100644 --- a/Makefile +++ b/Makefile @@ -21,8 +21,8 @@ docs-serve: .PHONY: test test: - pytest -n 2 tests + pytest -n 4 tests .PHONY: test-cov test-cov: - pytest -n 2 --cov=hsclient --cov-report html + pytest -n 4 --cov=hsclient --cov-report html