Sqlalchemy compositional

.github/workflows/test.yml

@@ -72,8 +72,7 @@ jobs:
 
       - name: Initialize mypy
         run: |
-          mypy . > /dev/null || true
-          mypy --install-types --non-interactive
+          mypy --install-types --non-interactive . || true
 
       - name: Run tests
         run: |

setup.py

@@ -78,7 +78,7 @@
     ],
     extras_require={
         'test': [
-            'pytest',
+            'pytest<7.0',
             'pytest-cov',
             'pytest-mypy',
             'pytest-flake8',

terracotta/drivers/__init__.py

@@ -8,22 +8,24 @@
 import urllib.parse as urlparse
 from pathlib import Path
 
-from terracotta.drivers.base import Driver
+from terracotta.drivers.base_classes import MetaStore
+from terracotta.drivers.terracotta_driver import TerracottaDriver
+from terracotta.drivers.geotiff_raster_store import GeoTiffRasterStore
 
 URLOrPathType = Union[str, Path]
 
 
-def load_driver(provider: str) -> Type[Driver]:
+def load_driver(provider: str) -> Type[MetaStore]:
     if provider == 'sqlite-remote':
-        from terracotta.drivers.sqlite_remote import RemoteSQLiteDriver
+        from terracotta.drivers.sqlite_remote_meta_store import RemoteSQLiteDriver
         return RemoteSQLiteDriver
 
     if provider == 'mysql':
-        from terracotta.drivers.mysql import MySQLDriver
+        from terracotta.drivers.mysql_meta_store import MySQLDriver
         return MySQLDriver
 
     if provider == 'sqlite':
-        from terracotta.drivers.sqlite import SQLiteDriver
+        from terracotta.drivers.sqlite_meta_store import SQLiteDriver
         return SQLiteDriver
 
     raise ValueError(f'Unknown database provider {provider}')
@@ -42,10 +44,10 @@ def auto_detect_provider(url_or_path: Union[str, Path]) -> str:
     return 'sqlite'
 
 
-_DRIVER_CACHE: Dict[Tuple[URLOrPathType, str, int], Driver] = {}
+_DRIVER_CACHE: Dict[Tuple[URLOrPathType, str, int], TerracottaDriver] = {}
 
 
-def get_driver(url_or_path: URLOrPathType, provider: str = None) -> Driver:
+def get_driver(url_or_path: URLOrPathType, provider: str = None) -> TerracottaDriver:
     """Retrieve Terracotta driver instance for the given path.
 
     This function always returns the same instance for identical inputs.
@@ -66,12 +68,12 @@ def get_driver(url_or_path: URLOrPathType, provider: str = None) -> Driver:
 
         >>> import terracotta as tc
         >>> tc.get_driver('tc.sqlite')
-        SQLiteDriver('/home/terracotta/tc.sqlite')
+        TerracottaDriver(meta_store=SQLiteDriver('/home/terracotta/tc.sqlite'))
         >>> tc.get_driver('mysql://root@localhost/tc')
-        MySQLDriver('mysql://root@localhost:3306/tc')
+        TerracottaDriver(meta_store=MySQLDriver('mysql://root@localhost:3306/tc'))
         >>> # pass provider if path is given in a non-standard way
         >>> tc.get_driver('root@localhost/tc', provider='mysql')
-        MySQLDriver('mysql://root@localhost:3306/tc')
+        TerracottaDriver(meta_store=MySQLDriver('mysql://root@localhost:3306/tc'))
 
     """
     if provider is None:  # try and auto-detect
@@ -85,6 +87,10 @@ def get_driver(url_or_path: URLOrPathType, provider: str = None) -> Driver:
     cache_key = (normalized_path, provider, os.getpid())
 
     if cache_key not in _DRIVER_CACHE:
-        _DRIVER_CACHE[cache_key] = DriverClass(url_or_path)
+        driver = TerracottaDriver(
+            meta_store=DriverClass(url_or_path),
+            raster_store=GeoTiffRasterStore()
+        )
+        _DRIVER_CACHE[cache_key] = driver
 
     return _DRIVER_CACHE[cache_key]

terracotta/drivers/base.py → terracotta/drivers/base_classes.py

@@ -1,4 +1,4 @@
-"""drivers/base.py
+"""drivers/base_classes.py
 
 Base class for drivers.
 """
@@ -7,9 +7,11 @@
 import functools
 from abc import ABC, abstractmethod
 from collections import OrderedDict
-from typing import (Any, Callable, Dict, List, Mapping, Sequence, Tuple,
-                    TypeVar, Union)
+from typing import (Any, Callable, Dict, List, Mapping, Optional, Sequence,
+                    Tuple, TypeVar, Union)
 
+KeysType = Mapping[str, str]
+MultiValueKeysType = Mapping[str, Union[str, List[str]]]
 Number = TypeVar('Number', int, float)
 T = TypeVar('T')
 
@@ -22,17 +24,18 @@ def requires_connection(
         return functools.partial(requires_connection, verify=verify)
 
     @functools.wraps(fun)
-    def inner(self: Driver, *args: Any, **kwargs: Any) -> T:
+    def inner(self: MetaStore, *args: Any, **kwargs: Any) -> T:
+        assert fun is not None
         with self.connect(verify=verify):
-            # Apparently mypy thinks fun might still be None, hence the ignore:
-            return fun(self, *args, **kwargs)  # type: ignore
+            return fun(self, *args, **kwargs)
+
     return inner
 
 
-class Driver(ABC):
-    """Abstract base class for all Terracotta data backends.
+class MetaStore(ABC):
+    """Abstract base class for all Terracotta metadata backends.
 
-    Defines a common interface for all drivers.
+    Defines a common interface for all metadata backends.
     """
     _RESERVED_KEYS = ('limit', 'page')
 
@@ -105,14 +108,14 @@ def get_keys(self) -> OrderedDict:
         pass
 
     @abstractmethod
-    def get_datasets(self, where: Mapping[str, Union[str, List[str]]] = None,
+    def get_datasets(self, where: MultiValueKeysType = None,
                      page: int = 0, limit: int = None) -> Dict[Tuple[str, ...], Any]:
         # Get all known dataset key combinations matching the given constraints,
-        # and a handle to retrieve the data (driver dependent)
+        # and a path to retrieve the data (driver dependent)
         pass
 
     @abstractmethod
-    def get_metadata(self, keys: Union[Sequence[str], Mapping[str, str]]) -> Dict[str, Any]:
+    def get_metadata(self, keys: KeysType) -> Optional[Dict[str, Any]]:
         """Return all stored metadata for given keys.
 
         Arguments:
@@ -136,19 +139,53 @@ def get_metadata(self, keys: Union[Sequence[str], Mapping[str, str]]) -> Dict[st
         """
         pass
 
+    @abstractmethod
+    def insert(self, keys: KeysType,
+               path: Any, **kwargs: Any) -> None:
+        """Register a new dataset. Used to populate metadata database.
+
+        Arguments:
+
+            keys: Keys of the dataset. Can either be given as a sequence of key values, or
+                as a mapping ``{key_name: key_value}``.
+            path: Path to access dataset (driver dependent).
+
+        """
+        pass
+
+    @abstractmethod
+    def delete(self, keys: KeysType) -> None:
+        """Remove a dataset from the metadata database.
+
+        Arguments:
+
+            keys:  Keys of the dataset. Can either be given as a sequence of key values, or
+                as a mapping ``{key_name: key_value}``.
+
+        """
+        pass
+
+    def __repr__(self) -> str:
+        return f'{self.__class__.__name__}(\'{self.path}\')'
+
+
+class RasterStore(ABC):
+    """Abstract base class for all Terracotta raster backends.
+
+    Defines a common interface for all raster backends."""
+
     @abstractmethod
     # TODO: add accurate signature if mypy ever supports conditional return types
-    def get_raster_tile(self, keys: Union[Sequence[str], Mapping[str, str]], *,
+    def get_raster_tile(self, path: str, *,
                         tile_bounds: Sequence[float] = None,
                         tile_size: Sequence[int] = (256, 256),
                         preserve_values: bool = False,
                         asynchronous: bool = False) -> Any:
-        """Load a raster tile with given keys and bounds.
+        """Load a raster tile with given path and bounds.
 
         Arguments:
 
-            keys: Keys of the requested dataset. Can either be given as a sequence of key values,
-                or as a mapping ``{key_name: key_value}``.
+            path: Path of the requested dataset.
             tile_bounds: Physical bounds of the tile to read, in Web Mercator projection (EPSG3857).
                 Reads the whole dataset if not given.
             tile_size: Shape of the output array to return. Must be two-dimensional.
@@ -168,39 +205,10 @@ def get_raster_tile(self, keys: Union[Sequence[str], Mapping[str, str]], *,
         """
         pass
 
-    @staticmethod
     @abstractmethod
-    def compute_metadata(data: Any, *,
+    def compute_metadata(self, path: str, *,
                          extra_metadata: Any = None,
-                         **kwargs: Any) -> Dict[str, Any]:
+                         use_chunks: bool = None,
+                         max_shape: Sequence[int] = None) -> Dict[str, Any]:
         # Compute metadata for a given input file (driver dependent)
         pass
-
-    @abstractmethod
-    def insert(self, keys: Union[Sequence[str], Mapping[str, str]],
-               handle: Any, **kwargs: Any) -> None:
-        """Register a new dataset. Used to populate metadata database.
-
-        Arguments:
-
-            keys: Keys of the dataset. Can either be given as a sequence of key values, or
-                as a mapping ``{key_name: key_value}``.
-            handle: Handle to access dataset (driver dependent).
-
-        """
-        pass
-
-    @abstractmethod
-    def delete(self, keys: Union[Sequence[str], Mapping[str, str]]) -> None:
-        """Remove a dataset from the metadata database.
-
-        Arguments:
-
-            keys:  Keys of the dataset. Can either be given as a sequence of key values, or
-                as a mapping ``{key_name: key_value}``.
-
-        """
-        pass
-
-    def __repr__(self) -> str:
-        return f'{self.__class__.__name__}(\'{self.path}\')'