Complex domain detection for collections of data items with CLI support

python/lib/client/dmod/client/__init__.py

@@ -1 +1 @@
-name = 'client'
+name = 'client'

python/lib/client/dmod/client/__main__.py

@@ -3,13 +3,15 @@
 import datetime
 import json
 from dmod.core.execution import AllocationParadigm
+from dmod.core.exception import DmodRuntimeError
 from . import name as package_name
-from .dmod_client import ClientConfig, DmodClient
+from .dmod_client import ClientConfig, DmodClient, run_domain_detection
+from .domain_detectors import ClientUniversalItemDomainDetector
 from dmod.communication.client import get_or_create_eventloop
-from dmod.core.meta_data import ContinuousRestriction, DataCategory, DataDomain, DataFormat, DiscreteRestriction, \
-    TimeRange
+from dmod.core.meta_data import (ContinuousRestriction, DataCategory, DataDomain, DataFormat, DiscreteRestriction,
+                                 TimeRange)
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Tuple, Type
+from typing import Any, List, Optional
 
 
 DEFAULT_CLIENT_CONFIG_BASENAME = '.dmod_client_config.json'
@@ -230,6 +232,17 @@ def _handle_data_service_action_args(parent_subparsers_container):
     parser_delete = action_subparsers.add_parser('delete', description="Delete a specified (entire) dataset.")
     parser_delete.add_argument('name', help='Specify the name of the dataset to delete.')
 
+    # Nested parser for the 'domain' action, with required argument for path to the data to detect over
+    parser_domain = action_subparsers.add_parser('domain', description="Ops related to DataDomains and detection.")
+    domain_command_subparsers = parser_domain.add_subparsers(dest="domain_command")
+    detect_domain_parser = domain_command_subparsers.add_parser('detect',
+                                                                description="Detect DataDomain for local data for a dataset.")
+    detect_domain_parser.add_argument('path', type=Path,
+                                      help="Specify a data file or path containing several data files.")
+
+    show_detectors = domain_command_subparsers.add_parser('list_detectors',
+                                                          description="List the domain detector subclasses that are available.")
+
     # Nested parser for the 'upload' action, with required args for dataset name and files to upload
     parser_upload = action_subparsers.add_parser('upload', description="Upload local files to a dataset.")
     parser_upload.add_argument('--data-root', dest='data_root', type=Path,
@@ -322,6 +335,23 @@ def _handle_args():
     return parser.parse_args()
 
 
+def _run_domain_command(args):
+    if args.domain_command == 'detect':
+        try:
+            domain = run_domain_detection(paths=args.path)
+            print({"success": True, "domain": f"{domain.to_json()}"})
+        except DmodRuntimeError as e:
+            print({"success": False, "reason": f"{e.__class__.__name__}", "message": f"{e!s}"})
+        except Exception as e:
+            print(f"ERROR - Encountered {e.__class__.__name__} detecting domain: {e!s}")
+            exit(1)
+    elif args.domain_command == "list_detectors":
+        all_names = [d.__name__ for d in ClientUniversalItemDomainDetector.get_default_detectors()]
+        print({"success": True, "detector_names": all_names})
+    else:
+        raise NotImplementedError(f"Unrecognized domain command '{args.domain_command!s}'")
+
+
 def find_client_config(basenames: Optional[List[str]] = None, dirs: Optional[List[Path]] = None) -> Optional[Path]:
     """
     Search locations for the client config of given basenames, falling back to defaults, and returning path if found.
@@ -354,19 +384,19 @@ def find_client_config(basenames: Optional[List[str]] = None, dirs: Optional[Lis
 
 
 def execute_dataset_command(args, client: DmodClient):
-    async_loop = get_or_create_eventloop()
-    try:
-        result = async_loop.run_until_complete(client.data_service_action(**(vars(args))))
-        print(result)
-    except ValueError as e:
-        print(str(e))
-        sys.exit(1)
-    except NotImplementedError as e:
-        print(str(e))
-        sys.exit(1)
-    except Exception as e:
-        print("ERROR: Encountered {} - {}".format(e.__class__.__name__, str(e)))
-        sys.exit(1)
+    if args.action == 'domain':
+        _run_domain_command(args)
+    else:
+        async_loop = get_or_create_eventloop()
+        try:
+            result = async_loop.run_until_complete(client.data_service_action(**(vars(args))))
+            print(result)
+        except (ValueError, NotImplementedError) as e:
+            print(str(e))
+            sys.exit(1)
+        except Exception as e:
+            print("ERROR: Encountered {} - {}".format(e.__class__.__name__, str(e)))
+            sys.exit(1)
 
 
 def execute_config_command(parsed_args, client: DmodClient):

python/lib/client/dmod/client/_version.py

@@ -1 +1 @@
-__version__ = '0.6.0'
+__version__ = '0.6.1'

python/lib/client/dmod/client/dmod_client.py

@@ -1,13 +1,18 @@
 import json
+import logging
 
 from dmod.communication import AuthClient, TransportLayerClient, WebSocketClient
 from dmod.core.common import get_subclasses
+from dmod.core.exception import DmodRuntimeError
 from dmod.core.serializable import BasicResultIndicator, ResultIndicator
-from dmod.core.meta_data import DataDomain
+from dmod.core.meta_data import DataDomain, DiscreteRestriction, StandardDatasetIndex
 from .request_clients import DataServiceClient, JobClient
 from .client_config import ClientConfig
 from pathlib import Path
-from typing import Type
+from typing import List, Optional, Type, Union
+
+from functools import reduce
+from .domain_detectors import ClientDataCollectionDomainDetector, ClientUniversalItemDomainDetector
 
 
 def determine_transport_client_type(protocol: str,
@@ -48,6 +53,45 @@ def _get_subclasses(class_val):
     raise RuntimeError(f"No subclass of `{TransportLayerClient.__name__}` found supporting protocol '{protocol}'")
 
 
+def run_domain_detection(paths: Union[Path, List[Path]], data_id: Optional[str] = None) -> DataDomain:
+    """
+    Run domain detection.
+
+    Parameters
+    ----------
+    paths: Union[Path, List[Path]]
+        One or more paths to files or directories.
+    data_id: Optional[str]
+        Hypothetical data_id for a dataset containing this data and having the returned domain, useful in situations
+        where data_id is itself an index of the domain against which constraints are compared.
+
+    Returns
+    -------
+    DataDomain
+        The detected domain.
+
+    Raises
+    ------
+    DmodRuntimeError
+        If detection is unsuccessful.
+    """
+    def _detect(p: Path):
+        if p.is_dir():
+            return ClientDataCollectionDomainDetector(data_collection=p, collection_name=data_id).detect()
+        else:
+            return ClientUniversalItemDomainDetector(item=p).detect()
+
+    if isinstance(paths, Path):
+        return _detect(paths)
+    else:
+        domain = reduce(DataDomain.merge_domains, (_detect(path) for path in paths))
+        # It's possible that for many different individual file paths, data_id won't get set, so ...
+        idx = StandardDatasetIndex.DATA_ID
+        if data_id is not None and idx in domain.data_format.indices_to_fields().keys() and idx not in domain.discrete_restrictions:
+            domain.discrete_restrictions[idx] = DiscreteRestriction(variable=idx, values=[data_id])
+        return domain
+
+
 class DmodClient:
 
     def __init__(self, client_config: ClientConfig, bypass_request_service: bool = False, *args, **kwargs):
@@ -115,11 +159,12 @@ def _extract_dataset_domain(**kwargs) -> DataDomain:
                 raise ValueError(f"Could not deserialize JSON in 'domain_file' `{domain_file!s}` to domain object")
             else:
                 return domain
-        else:
-            try:
-                return DataDomain(**kwargs)
-            except Exception as e:
-                raise RuntimeError(f"Could not inflate keyword params to object due to {e.__class__.__name__} - {e!s}")
+        try:
+            return DataDomain(**kwargs)
+        except Exception:
+            logging.debug(f"No {DataDomain.__name__} provided; trying auto detection (params were {kwargs!s}")
+        return run_domain_detection(paths=kwargs.get('upload_paths'), data_id=kwargs.get('name'))
+
 
     def _get_transport_client(self, **kwargs) -> TransportLayerClient:
         # TODO: later add support for multiplexing capabilities and spawning wrapper clients
@@ -152,6 +197,8 @@ async def data_service_action(self, action: str, **kwargs) -> ResultIndicator:
                 except TypeError as e:
                     return BasicResultIndicator(success=False, reason="No Dataset Domain Provided",
                                                 message=f"Invalid type provided for 'domain' param: {e!s} ")
+                except DmodRuntimeError as e:
+                    return BasicResultIndicator(success=False, reason="Domain Detection Failed", message=f"{e!s}")
                 except (ValueError, RuntimeError) as e:
                     return BasicResultIndicator(success=False, reason="No Dataset Domain Provided", message=f"{e!s}")
                 return await self.data_service_client.create_dataset(domain=domain, **kwargs)

python/lib/client/dmod/client/domain_detectors.py

@@ -0,0 +1,76 @@
+from dmod.core.data_domain_detectors import (AbstractUniversalItemDomainDetector, AbstractDataCollectionDomainDetector,
+                                             DataItem, ItemDataDomainDetector)
+from dmod.modeldata.data.item_domain_detector import (AorcCsvFileDomainDetector, GeoPackageHydrofabricDomainDetector,
+                                                      RealizationConfigDomainDetector)
+from typing import Any, Callable, Dict, List, Optional, Set, Type
+
+
+class ClientUniversalItemDomainDetector(AbstractUniversalItemDomainDetector):
+    """
+    Concrete implementation of :class:`AbstractUniversalItemDomainDetector`, with some default detector subclass types.
+
+
+    """
+    _default_detector_types: Set[Type[ItemDataDomainDetector]] = {
+        AorcCsvFileDomainDetector,
+        GeoPackageHydrofabricDomainDetector,
+        RealizationConfigDomainDetector
+    }
+    """ Default detector subclasses always associated with instances of this type. """
+
+    @classmethod
+    def get_default_detectors(cls) -> List[Type[ItemDataDomainDetector]]:
+        return [d for d in cls._default_detector_types]
+
+    def __init__(self,
+                 item: DataItem,
+                 item_name: Optional[str] = None,
+                 decode_format: str = 'utf-8',
+                 short_on_success: bool = False,
+                 type_sort_func: Optional[Callable[[Type[ItemDataDomainDetector]], Any]] = None):
+        """
+        Initialize an instance.
+
+        Parameters
+        ----------
+        item: DataItem
+            The data item for which a domain will be detected.
+        item_name: Optional[str]
+            The name for the item, which includes important domain metadata in some situations.
+        decode_format: str
+            The decoder format when decoding byte strings (``utf-8`` by default).
+        short_on_success: Optional[bool]
+            Indication of whether :method:`detect` should short circuit and return the 1st successful detection, rather
+            than try all subclasses and risk multiple detections, and thus an error condition (default: ``False``).
+        type_sort_func: Optional[Callable[[Type[ItemDataDomainDetector]], Any]]
+            Optional function necessary for calls to usage of the built-in ``sorted`` function to sort detector
+            subclasses during various instance operations, and serving as the ``key`` argument to ``sorted``; note that
+            sorting is performed in such places IFF this is validly set, as the subclass themselves - i.e., the
+            :class:`type` objects - do not implement `<`.
+        """
+        super().__init__(item=item,
+                         item_name=item_name,
+                         decode_format=decode_format,
+                         detector_types=self._default_detector_types,
+                         short_on_success=short_on_success,
+                         type_sort_func=type_sort_func)
+
+
+class ClientDataCollectionDomainDetector(AbstractDataCollectionDomainDetector[ClientUniversalItemDomainDetector]):
+    """
+    Concrete implementation relying on :class:`UniversalItemDomainDetector` to detect items.
+    """
+
+    def get_item_detectors(self) -> Dict[str, ClientUniversalItemDomainDetector]:
+        """
+        Get initialized detection objects, keyed by item names, for items within this instance's data collection.
+
+        Returns
+        -------
+        Dict[str, ClientUniversalItemDomainDetector]
+            Dictionary of per-item initialize detection objects, keyed by item name.
+        """
+        detectors = dict()
+        for item_name in self.get_item_names():
+            detectors[item_name] = ClientUniversalItemDomainDetector(item=self.get_item(item_name), item_name=item_name)
+        return detectors

python/lib/client/setup.py

@@ -22,7 +22,7 @@
     license='',
     include_package_data=True,
     #install_requires=['websockets', 'jsonschema'],vi
-    install_requires=['dmod-core>=0.15.0', 'websockets>=8.1', 'pydantic>=1.10.8,~=1.10', 'dmod-communication>=0.17.0',
-                      'dmod-externalrequests>=0.6.0'],
+    install_requires=['dmod-core>=0.15.2', 'websockets>=8.1', 'pydantic>=1.10.8,~=1.10', 'dmod-communication>=0.17.0',
+                      'dmod-externalrequests>=0.6.0', 'dmod-modeldata>=0.11.1'],
     packages=find_namespace_packages(include=['dmod.*'], exclude=['dmod.test'])
 )

python/lib/core/dmod/core/_version.py

@@ -1 +1 @@
-__version__ = '0.15.1'
+__version__ = '0.15.2'