Move swift and dxpy into extras

.github/workflows/pythonpackage.yml

@@ -10,7 +10,8 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9]  # NOTE: 3.6 no longer supported
+        python-version: [3.7, 3.8, 3.9]
+        POETRY_EXTRAS: ["-E swift", "-E dx", "", "-E swift -E dx"]
 
     steps:
     - uses: actions/checkout@v2
@@ -30,7 +31,9 @@ jobs:
         pip install poetry==${{env.POETRY_VERSION}}
         poetry config --local virtualenvs.in-project true
     - name: Install venv
-      run: make venv
+      run: |
+        export POETRY_EXTRAS="${{matrix.POETRY_EXTRAS}}"
+        make venv
     - name: List versions - pip
       run: |
         mkdir htmlcov

AUTHORS

@@ -1,5 +1,6 @@
 Wes Kendall @wesleykendall ([email protected])
-Jeff Tratner @jtratner
+Jeff Tratner @jtratner ([email protected])
 Stephanie Huang @phanieste
 Kyle Beauchamp @kyleabeauchamp
 Piotr Kaleta @pkaleta
+Salika Dunatunga @scdunatun

Makefile

@@ -24,7 +24,7 @@ WITH_PBR=$(WITH_VENV) PBR_REQUIREMENTS_FILES=requirements-pbr.txt
 venv: $(VENV_ACTIVATE)
 
 $(VENV_ACTIVATE): poetry.lock
-	poetry install
+	poetry install $(POETRY_EXTRAS)
 	touch $@
 
 develop: venv

README.rst

@@ -20,7 +20,7 @@ Quickstart
 
 ::
 
-    pip install stor
+    pip install stor[dx,swift]
 
 ``stor`` provides both a CLI and a Python library for manipulating Posix
 and OBS with a single, cross-compatible API.

docs/installation.rst

@@ -3,12 +3,13 @@ Installation
 
 To install the latest release, type::
 
-    pip install stor
+    pip install stor[dx,swift]
 
 To install the latest code directly from source, type::
 
     pip install git+git://github.com/counsyl/stor.git
 
+You can omit the ``dx`` or ``swift`` extras if you do not need to use those storage providers.
 
 ..  _cli_tab_completion_installation:
 

docs/release_notes.rst

@@ -1,6 +1,24 @@
 Release Notes
 =============
 
+v5.0 - Optional dxpy and swift!
+-------------------------------
+
+This release is almost entirely backwards compatible, but removes `dxpy` and `swiftclient` related requirements as direct dependencies.
+For most users, this should be a transparent change (because they already depend on `dxpy` or `swiftclient` separately). However, this
+would break workflows where stor was passively pulling in `dxpy` and `swiftclient`. To simplify updates, simply change your `requirements.txt` 
+file to explicitly specify the dxpy and swift extras:
+
+```
+# Before
+stor < 5
+# After
+stor[dxpy,swift] >= 5
+```
+
+
+* Guard dxpy imports in utils so you can use stor without either dxpy or swift installed. (#150)
+
 v4.1.0
 ------
 * Replace ``multiprocessing.ThreadPool`` with ``concurrent.futures.ThreadPoolExecutor``

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "stor"
-version = "4.1.0"
+version = "4.2.0"
 description = "Cross-compatible API for accessing Posix and OBS storage systems"
 authors = ["Counsyl Inc. <[email protected]>"]
 license = "MIT"
@@ -18,9 +18,13 @@ python = "^3.6"
 requests = ">=2.20.0"
 boto3 = ">=1.7.0"
 cached-property = ">=1.5.1"
-dxpy = ">=0.278.0"
-python-keystoneclient = ">=1.8.1"
-python-swiftclient = ">=3.6.0"
+dxpy = {version = ">=0.278.0", optional = true }
+python-keystoneclient = {version = ">=1.8.1", optional = true}
+python-swiftclient = {version = ">=3.6.0", optional = true}
+
+[tool.poetry.extras]
+swift = ["python-keystoneclient", "python-swiftclient"]
+dx = ["dxpy"]
 
 [tool.poetry.dev-dependencies]
 flake8 = "^3.7.9"

stor/dx.py

@@ -6,9 +6,6 @@
 
 from cached_property import cached_property
 from contextlib import contextmanager
-import dxpy
-from dxpy.exceptions import DXError
-from dxpy.exceptions import DXSearchError
 
 from stor import exceptions as stor_exceptions
 from stor import Path
@@ -20,6 +17,14 @@
 import stor.settings
 
 
+try:
+    import dxpy
+    from dxpy.exceptions import DXError
+    from dxpy.exceptions import DXSearchError
+except ImportError as e:
+    raise utils.missing_storage_library_exception("dx", e) from e
+
+
 logger = logging.getLogger(__name__)
 progress_logger = logging.getLogger('%s.progress' % __name__)
 

stor/obs.py

@@ -3,10 +3,6 @@
 import posixpath
 import sys
 
-import dxpy
-from swiftclient.service import SwiftError
-from swiftclient.service import SwiftUploadObject
-
 from stor.base import Path
 from stor.posix import PosixPath
 from stor import utils
@@ -28,6 +24,17 @@ def wrapper(self, *args, **kwargs):
     return wrapper
 
 
+try:
+    from swiftclient.service import SwiftUploadObject
+except ImportError:  # pragma: no cover
+    class SwiftUploadObject(object):
+        """Give 90% of the utility of SwiftUploadObject class without swiftclient!"""
+        def __init__(self, source, object_name=None, options=None):
+            self.source = source
+            self.object_name = object_name
+            self.options = options
+
+
 class OBSUploadObject(SwiftUploadObject):
     """
     An upload object similar to swiftclient's SwiftUploadObject that allows the user
@@ -41,6 +48,8 @@ def __init__(self, source, object_name, options=None):
             source (str): A path that specifies a source file.
             dest (str): A path that specifies a destination file name (full key)
         """
+        from swiftclient.service import SwiftError
+
         try:
             super(OBSUploadObject, self).__init__(source, object_name=object_name, options=options)
         except SwiftError as exc:
@@ -452,6 +461,8 @@ def close(self):
         self.closed = True
 
     def _wait_on_close(self):
+        import dxpy
+
         if isinstance(self._path, stor.dx.DXPath):
             wait_on_close = stor.settings.get()['dx']['wait_on_close']
             if wait_on_close:

stor/swift.py

@@ -42,10 +42,6 @@
 import threading
 
 from urllib import parse
-from swiftclient import exceptions as swift_exceptions
-from swiftclient import service as swift_service
-from swiftclient import client as swift_client
-from swiftclient.utils import generate_temp_url
 
 from stor import exceptions as stor_exceptions
 from stor import is_swift_path
@@ -57,6 +53,14 @@
 from stor.posix import PosixPath
 from stor.third_party.backoff import with_backoff
 
+try:
+    from swiftclient import exceptions as swift_exceptions
+    from swiftclient import service as swift_service
+    from swiftclient import client as swift_client
+    from swiftclient.utils import generate_temp_url
+except ImportError as e:
+    raise utils.missing_storage_library_exception("swift", e) from e
+
 
 logger = logging.getLogger(__name__)
 progress_logger = logging.getLogger('%s.progress' % __name__)

stor/tests/test_utils.py

@@ -460,3 +460,15 @@ def test_path_no_perms(self):
         self.mock_copy.side_effect = stor.exceptions.FailedUploadError('foo')
         self.assertFalse(utils.is_writeable('s3://stor-test/foo/bar'))
         self.assertFalse(self.mock_remove.called)
+
+class TestStorageLibraryException:
+    def test_generates_appropriate_text(self):
+        try:
+            import thisshouldnotexistatall
+        except ImportError as e:
+            exc = utils.missing_storage_library_exception("dx", e)
+        else:
+            assert False, "this should have raised an import error"
+        assert "dx" in str(exc)
+        assert "thisshouldnotexistatall" in str(exc)
+        assert "To use a 'dx' path, stor needs an additional python library" in str(exc)

stor/utils.py

@@ -7,9 +7,7 @@
 import shutil
 from subprocess import check_call
 import tempfile
-
-from dxpy.bindings import verify_string_dxid
-from dxpy.exceptions import DXError
+import warnings
 
 from stor import exceptions
 
@@ -190,8 +188,7 @@ def is_swift_path(p):
     Returns:
         bool: True if p is a Swift path, False otherwise.
     """
-    from stor.swift import SwiftPath
-    return p.startswith(SwiftPath.drive)
+    return p.startswith('swift://')
 
 
 def is_filesystem_path(p):
@@ -217,8 +214,7 @@ def is_s3_path(p):
     Returns
         bool: True if p is a S3 path, False otherwise.
     """
-    from stor.s3 import S3Path
-    return p.startswith(S3Path.drive)
+    return p.startswith('s3://')
 
 
 def is_obs_path(p):
@@ -244,8 +240,7 @@ def is_dx_path(p):
     Returns
         bool: True if p is a DX path, False otherwise.
     """
-    from stor.dx import DXPath
-    return p.startswith(DXPath.drive)
+    return p.startswith('dx://')
 
 
 def is_valid_dxid(dxid, expected_classes):
@@ -257,6 +252,9 @@ def is_valid_dxid(dxid, expected_classes):
     Returns
         bool: Whether given dxid is a valid path of one of expected_classes
     """
+    from dxpy.bindings import verify_string_dxid
+    from dxpy.exceptions import DXError
+
     try:
         return verify_string_dxid(dxid, expected_classes) is None
     except DXError:
@@ -744,3 +742,23 @@ def add_result(self, result):
             progress_msg = self.get_progress_message()
             if progress_msg:  # pragma: no cover
                 self.logger.log(self.level, progress_msg)
+
+def missing_storage_library_exception(module: str, exc: Exception):
+    """Generate a helpful error for user about why their import failed.
+
+    Meant to be used as:
+
+        try:
+            ...
+        except Import Error as e:
+            raise missing_storage_library_exception('dx') from e
+    """
+    return ImportError(
+        f"{type(exc).__name__}: {exc}\n"
+        f"To use a '{module}' path, stor needs an additional python library. "
+        f"Please specify it as an extra in the installation.\n"
+        f"i.e.,: `pip install stor[{module}]` or `stor[{module}] >= 5` "
+        f"in requirements.txt or `poetry add stor[{module}]`.\n"
+        f"Alternatively, change an existing "
+        f'pyproject.toml file to specify `stor = {{version="5.0", extras = {module}}}`'
+    )