diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 259c6230..71ee4625 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -10,7 +10,8 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: [3.7, 3.8, 3.9] # NOTE: 3.6 no longer supported + python-version: [3.7, 3.8, 3.9] + POETRY_EXTRAS: ["-E swift", "-E dx", "", "-E swift -E dx"] steps: - uses: actions/checkout@v2 @@ -30,7 +31,9 @@ jobs: pip install poetry==${{env.POETRY_VERSION}} poetry config --local virtualenvs.in-project true - name: Install venv - run: make venv + run: | + export POETRY_EXTRAS="${{matrix.POETRY_EXTRAS}}" + make venv - name: List versions - pip run: | mkdir htmlcov diff --git a/AUTHORS b/AUTHORS index a3534f6d..7987eee1 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,5 +1,6 @@ Wes Kendall @wesleykendall (wesleykendall@gmail.com) -Jeff Tratner @jtratner +Jeff Tratner @jtratner (jeffrey.tratner@gmail.com) Stephanie Huang @phanieste Kyle Beauchamp @kyleabeauchamp Piotr Kaleta @pkaleta +Salika Dunatunga @scdunatun \ No newline at end of file diff --git a/Makefile b/Makefile index 646f837a..049ab072 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ WITH_PBR=$(WITH_VENV) PBR_REQUIREMENTS_FILES=requirements-pbr.txt venv: $(VENV_ACTIVATE) $(VENV_ACTIVATE): poetry.lock - poetry install + poetry install $(POETRY_EXTRAS) touch $@ develop: venv diff --git a/README.rst b/README.rst index 0e963eee..853f0c2e 100644 --- a/README.rst +++ b/README.rst @@ -20,7 +20,7 @@ Quickstart :: - pip install stor + pip install stor[dx,swift] ``stor`` provides both a CLI and a Python library for manipulating Posix and OBS with a single, cross-compatible API. diff --git a/docs/installation.rst b/docs/installation.rst index 12ddc48f..62a64fa6 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -3,12 +3,13 @@ Installation To install the latest release, type:: - pip install stor + pip install stor[dx,swift] To install the latest code directly from source, type:: pip install git+git://github.com/counsyl/stor.git +You can omit the ``dx`` or ``swift`` extras if you do not need to use those storage providers. .. _cli_tab_completion_installation: diff --git a/docs/release_notes.rst b/docs/release_notes.rst index e79ba32e..4dfa6e2d 100644 --- a/docs/release_notes.rst +++ b/docs/release_notes.rst @@ -1,6 +1,24 @@ Release Notes ============= +v5.0 - Optional dxpy and swift! +------------------------------- + +This release is almost entirely backwards compatible, but removes `dxpy` and `swiftclient` related requirements as direct dependencies. +For most users, this should be a transparent change (because they already depend on `dxpy` or `swiftclient` separately). However, this +would break workflows where stor was passively pulling in `dxpy` and `swiftclient`. To simplify updates, simply change your `requirements.txt` +file to explicitly specify the dxpy and swift extras: + +``` +# Before +stor < 5 +# After +stor[dxpy,swift] >= 5 +``` + + +* Guard dxpy imports in utils so you can use stor without either dxpy or swift installed. (#150) + v4.1.0 ------ * Replace ``multiprocessing.ThreadPool`` with ``concurrent.futures.ThreadPoolExecutor`` diff --git a/pyproject.toml b/pyproject.toml index 90501510..287bec4f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "stor" -version = "4.1.0" +version = "4.2.0" description = "Cross-compatible API for accessing Posix and OBS storage systems" authors = ["Counsyl Inc. "] license = "MIT" @@ -18,9 +18,13 @@ python = "^3.6" requests = ">=2.20.0" boto3 = ">=1.7.0" cached-property = ">=1.5.1" -dxpy = ">=0.278.0" -python-keystoneclient = ">=1.8.1" -python-swiftclient = ">=3.6.0" +dxpy = {version = ">=0.278.0", optional = true } +python-keystoneclient = {version = ">=1.8.1", optional = true} +python-swiftclient = {version = ">=3.6.0", optional = true} + +[tool.poetry.extras] +swift = ["python-keystoneclient", "python-swiftclient"] +dx = ["dxpy"] [tool.poetry.dev-dependencies] flake8 = "^3.7.9" diff --git a/stor/dx.py b/stor/dx.py index abdb97dc..b2a42ff8 100644 --- a/stor/dx.py +++ b/stor/dx.py @@ -6,9 +6,6 @@ from cached_property import cached_property from contextlib import contextmanager -import dxpy -from dxpy.exceptions import DXError -from dxpy.exceptions import DXSearchError from stor import exceptions as stor_exceptions from stor import Path @@ -20,6 +17,14 @@ import stor.settings +try: + import dxpy + from dxpy.exceptions import DXError + from dxpy.exceptions import DXSearchError +except ImportError as e: + raise utils.missing_storage_library_exception("dx", e) from e + + logger = logging.getLogger(__name__) progress_logger = logging.getLogger('%s.progress' % __name__) diff --git a/stor/obs.py b/stor/obs.py index a071f4d8..7c30f6a8 100644 --- a/stor/obs.py +++ b/stor/obs.py @@ -3,10 +3,6 @@ import posixpath import sys -import dxpy -from swiftclient.service import SwiftError -from swiftclient.service import SwiftUploadObject - from stor.base import Path from stor.posix import PosixPath from stor import utils @@ -28,6 +24,17 @@ def wrapper(self, *args, **kwargs): return wrapper +try: + from swiftclient.service import SwiftUploadObject +except ImportError: # pragma: no cover + class SwiftUploadObject(object): + """Give 90% of the utility of SwiftUploadObject class without swiftclient!""" + def __init__(self, source, object_name=None, options=None): + self.source = source + self.object_name = object_name + self.options = options + + class OBSUploadObject(SwiftUploadObject): """ An upload object similar to swiftclient's SwiftUploadObject that allows the user @@ -41,6 +48,8 @@ def __init__(self, source, object_name, options=None): source (str): A path that specifies a source file. dest (str): A path that specifies a destination file name (full key) """ + from swiftclient.service import SwiftError + try: super(OBSUploadObject, self).__init__(source, object_name=object_name, options=options) except SwiftError as exc: @@ -452,6 +461,8 @@ def close(self): self.closed = True def _wait_on_close(self): + import dxpy + if isinstance(self._path, stor.dx.DXPath): wait_on_close = stor.settings.get()['dx']['wait_on_close'] if wait_on_close: diff --git a/stor/swift.py b/stor/swift.py index 71dfb00d..4efb25e8 100644 --- a/stor/swift.py +++ b/stor/swift.py @@ -42,10 +42,6 @@ import threading from urllib import parse -from swiftclient import exceptions as swift_exceptions -from swiftclient import service as swift_service -from swiftclient import client as swift_client -from swiftclient.utils import generate_temp_url from stor import exceptions as stor_exceptions from stor import is_swift_path @@ -57,6 +53,14 @@ from stor.posix import PosixPath from stor.third_party.backoff import with_backoff +try: + from swiftclient import exceptions as swift_exceptions + from swiftclient import service as swift_service + from swiftclient import client as swift_client + from swiftclient.utils import generate_temp_url +except ImportError as e: + raise utils.missing_storage_library_exception("swift", e) from e + logger = logging.getLogger(__name__) progress_logger = logging.getLogger('%s.progress' % __name__) diff --git a/stor/tests/test_utils.py b/stor/tests/test_utils.py index aade0f14..dcc69ae9 100644 --- a/stor/tests/test_utils.py +++ b/stor/tests/test_utils.py @@ -460,3 +460,15 @@ def test_path_no_perms(self): self.mock_copy.side_effect = stor.exceptions.FailedUploadError('foo') self.assertFalse(utils.is_writeable('s3://stor-test/foo/bar')) self.assertFalse(self.mock_remove.called) + +class TestStorageLibraryException: + def test_generates_appropriate_text(self): + try: + import thisshouldnotexistatall + except ImportError as e: + exc = utils.missing_storage_library_exception("dx", e) + else: + assert False, "this should have raised an import error" + assert "dx" in str(exc) + assert "thisshouldnotexistatall" in str(exc) + assert "To use a 'dx' path, stor needs an additional python library" in str(exc) \ No newline at end of file diff --git a/stor/utils.py b/stor/utils.py index 01a6f073..c0997cfc 100644 --- a/stor/utils.py +++ b/stor/utils.py @@ -7,9 +7,7 @@ import shutil from subprocess import check_call import tempfile - -from dxpy.bindings import verify_string_dxid -from dxpy.exceptions import DXError +import warnings from stor import exceptions @@ -190,8 +188,7 @@ def is_swift_path(p): Returns: bool: True if p is a Swift path, False otherwise. """ - from stor.swift import SwiftPath - return p.startswith(SwiftPath.drive) + return p.startswith('swift://') def is_filesystem_path(p): @@ -217,8 +214,7 @@ def is_s3_path(p): Returns bool: True if p is a S3 path, False otherwise. """ - from stor.s3 import S3Path - return p.startswith(S3Path.drive) + return p.startswith('s3://') def is_obs_path(p): @@ -244,8 +240,7 @@ def is_dx_path(p): Returns bool: True if p is a DX path, False otherwise. """ - from stor.dx import DXPath - return p.startswith(DXPath.drive) + return p.startswith('dx://') def is_valid_dxid(dxid, expected_classes): @@ -257,6 +252,9 @@ def is_valid_dxid(dxid, expected_classes): Returns bool: Whether given dxid is a valid path of one of expected_classes """ + from dxpy.bindings import verify_string_dxid + from dxpy.exceptions import DXError + try: return verify_string_dxid(dxid, expected_classes) is None except DXError: @@ -744,3 +742,23 @@ def add_result(self, result): progress_msg = self.get_progress_message() if progress_msg: # pragma: no cover self.logger.log(self.level, progress_msg) + +def missing_storage_library_exception(module: str, exc: Exception): + """Generate a helpful error for user about why their import failed. + + Meant to be used as: + + try: + ... + except Import Error as e: + raise missing_storage_library_exception('dx') from e + """ + return ImportError( + f"{type(exc).__name__}: {exc}\n" + f"To use a '{module}' path, stor needs an additional python library. " + f"Please specify it as an extra in the installation.\n" + f"i.e.,: `pip install stor[{module}]` or `stor[{module}] >= 5` " + f"in requirements.txt or `poetry add stor[{module}]`.\n" + f"Alternatively, change an existing " + f'pyproject.toml file to specify `stor = {{version="5.0", extras = {module}}}`' + ) \ No newline at end of file