Add WebServiceStore to query/search in external databases

examples/mappings/NeuroMorpho/mappings/DictionaryMapping/NeuronMorphology.hjson

@@ -4,7 +4,6 @@
         NeuronMorphology
         Entity
     ]
-    id: forge.format('identifier', 'neuronmorphologies/neuromorpho', f'{x.bbpID}')
     brainLocation: {
         type: BrainLocation
         brainRegion: forge.resolve(x.brain_region[0], scope='ontology', strategy='EXACT_CASE_INSENSITIVE_MATCH')

kgforge/core/archetypes/dataset_store.py

@@ -15,6 +15,7 @@
 from abc import abstractmethod, ABC
 
 from typing import Optional, Union, List, Type, Dict
+from kgforge.core.archetypes.model import Model
 from kgforge.core.resource import Resource
 from kgforge.core.archetypes.read_only_store import ReadOnlyStore
 from kgforge.core.archetypes.resolver import Resolver
@@ -28,6 +29,9 @@
 class DatasetStore(ReadOnlyStore):
     """A class to link to external databases, query and search directly on datasets. """
 
+    def __init__(self, model: Optional[Model] = None) -> None:
+        super().__init__(model)
+
     @property
     @abstractmethod
     def mapper(self) -> Type[Mapper]:
@@ -77,13 +81,14 @@ def types(self) -> Optional[List[str]]:
         return list(self.model.mappings(self.model.source, False).keys())
 
     def search(
-            self, filters: List[Union[Dict, Filter]], resolvers: Optional[List[Resolver]] = None,
+            self, resolvers: Optional[List[Resolver]],
+            filters: List[Union[Dict, Filter]],
             **params
     ) -> Optional[List[Resource]]:
         """Search within the database.
         :param map: bool
         """
-        unmapped_resources = self._search(filters, resolvers, **params)
+        unmapped_resources = self._search(resolvers, filters, **params)
 
         if not params.pop('map', True):
             return unmapped_resources
@@ -105,7 +110,7 @@ def sparql(
         """Use SPARQL within the database.
         :param map: bool
         """
-        unmapped_resources = super(ReadOnlyStore, self).sparql(
+        unmapped_resources = super().sparql(
             query, debug, limit, offset, **params
         )
 

kgforge/core/archetypes/read_only_store.py

@@ -63,7 +63,7 @@ def _context_to_dict(context: Context):
 
     def get_context_prefix_vocab(self) -> Tuple[Optional[Dict], Optional[Dict], Optional[str]]:
         return (
-            ReadOnlyStore._context_to_dict(self.model_context().document),
+            ReadOnlyStore._context_to_dict(self.model_context()),
             self.model_context().prefixes,
             self.model_context().vocab
         )
@@ -260,7 +260,6 @@ def elastic(
     def _initialize_service(
             self,
             endpoint: Optional[str],
-            bucket: Optional[str],
             token: Optional[str],
             searchendpoints: Optional[Dict] = None,
             **store_config,

kgforge/core/archetypes/store.py

@@ -14,13 +14,14 @@
 import json
 from abc import abstractmethod
 from pathlib import Path
-from typing import Any, Dict, List,  Optional, Union, Type, Match
+from typing import Any, Dict, List,  Optional, Union, Type
 
 from kgforge.core.archetypes.read_only_store import ReadOnlyStore, DEFAULT_LIMIT, DEFAULT_OFFSET
 from kgforge.core.archetypes.model import Model
 from kgforge.core.commons import Context
 from kgforge.core.resource import Resource
 from kgforge.core.archetypes.mapping import Mapping
+from kgforge.core.archetypes.resolver import Resolver
 from kgforge.core.archetypes.mapper import Mapper
 from kgforge.core.commons.attributes import repr_class
 from kgforge.core.commons.es_query_builder import ESQueryBuilder
@@ -32,7 +33,7 @@
     UpdatingError,
     UploadingError
 )
-from kgforge.core.commons.execution import not_supported, run
+from kgforge.core.commons.execution import run
 
 
 class Store(ReadOnlyStore):
@@ -66,7 +67,7 @@ def __init__(
             if file_resource_mapping else None
 
         self.service: Any = self._initialize_service(
-            self.endpoint, self.bucket, self.token, searchendpoints, **store_config
+            self.endpoint, self.token, searchendpoints, **store_config
         )
 
     def __repr__(self) -> str:
@@ -238,6 +239,8 @@ def _deprecate_one(self, resource: Resource) -> None:
         # POLICY Resource _store_metadata should be set using wrappers.dict.wrap_dict().
         ...
 
+    # Querying
+
     def elastic(
             self, query: str, debug: bool, limit: int = DEFAULT_LIMIT, offset: int = DEFAULT_OFFSET
     ) -> List[Resource]:

kgforge/core/commons/execution.py

@@ -15,7 +15,7 @@
 import inspect
 import traceback
 from functools import wraps
-from typing import Any, Callable, List, Optional, Tuple, Union, Type
+from typing import Any, Callable, List, Optional, Tuple, Union, Type, Dict
 import requests
 
 from kgforge.core.resource import Resource
@@ -107,6 +107,18 @@ def catch_http_error(
         raise error_to_throw(error_message_formatter(e)) from e
 
 
+def format_message(msg):
+    return "".join([msg[0].lower(), msg[1:-1], msg[-1] if msg[-1] != "." else ""])
+
+
+def error_message(error: Union[requests.HTTPError, Dict]) -> str:
+    try:
+        error_text = error.response.text() if isinstance(error, requests.HTTPError) else str(error)
+        return format_message(error_text)
+    except Exception:
+        return format_message(str(error))
+
+
 def run(fun_one: Callable, fun_many: Optional[Callable], data: Union[Resource, List[Resource]],
         exception: Type[RunException], id_required: bool = False,
         required_synchronized: Optional[bool] = None, execute_actions: bool = False,

kgforge/core/commons/parser.py

@@ -14,6 +14,7 @@
 from typing import Any
 
 import datetime
+import json
 
 
 def _parse_type(value: Any, parse_str: bool = False):
@@ -42,3 +43,19 @@ def _parse_type(value: Any, parse_str: bool = False):
         return _type, value
     except Exception:
         return _type, value
+
+
+def _process_types(values):
+    """Assign correct data type to values from a request response"""
+    if values['type'] == 'literal' and 'datatype' in values and values['datatype'] == \
+            'http://www.w3.org/2001/XMLSchema#boolean':
+
+        return json.loads(str(values["value"]).lower())
+
+    elif values['type'] == 'literal' and 'datatype' in values and values['datatype'] == \
+            'http://www.w3.org/2001/XMLSchema#integer':
+
+        return int(values["value"])
+
+    else:
+        return values["value"]

kgforge/core/commons/sparql_query_builder.py

@@ -28,7 +28,7 @@
 from kgforge.core.archetypes.resolver import Resolver
 from kgforge.core.commons.context import Context
 from kgforge.core.commons.files import is_valid_url
-from kgforge.core.commons.parser import _parse_type
+from kgforge.core.commons.parser import _parse_type, _process_types
 from kgforge.core.commons.query_builder import QueryBuilder
 from kgforge.core.wrappings.paths import Filter
 
@@ -232,23 +232,8 @@ def triples_to_resource(iri, triples):
 
     @staticmethod
     def build_resource_from_select_query(results: List) -> List[Resource]:
-
-        def process_v(v):
-            if v['type'] == 'literal' and 'datatype' in v and v['datatype'] == \
-                    'http://www.w3.org/2001/XMLSchema#boolean':
-
-                return json.loads(str(v["value"]).lower())
-
-            elif v['type'] == 'literal' and 'datatype' in v and v['datatype'] == \
-                    'http://www.w3.org/2001/XMLSchema#integer':
-
-                return int(v["value"])
-
-            else:
-                return v["value"]
-
         return [
-            Resource(**{k: process_v(v) for k, v in x.items()})
+            Resource(**{k: _process_types(v) for k, v in x.items()})
             for x in results
         ]
 

kgforge/specializations/stores/__init__.py

@@ -14,3 +14,5 @@
 
 from .bluebrain_nexus import BlueBrainNexus
 from .demo_store import DemoStore
+from .sparql_store import SPARQLStore
+from .web_service_store import WebServiceStore

kgforge/specializations/stores/bluebrain_nexus.py

@@ -929,7 +929,6 @@ def _elastic(self, query: str) -> List[Resource]:
     def _initialize_service(
             self,
             endpoint: Optional[str],
-            bucket: Optional[str],
             token: Optional[str],
             searchendpoints: Optional[Dict] = None,
             **store_config,

kgforge/specializations/stores/demo_store.py

@@ -165,8 +165,8 @@ def _elastic(self, query: str) -> Optional[Union[List[Resource], Resource]]:
     # Utils.
 
     def _initialize_service(
-            self, endpoint: Optional[str], bucket: Optional[str],
-            token: Optional[str], searchendpoints: Optional[Dict] = None, **store_config,
+            self, endpoint: Optional[str], token: Optional[str],
+            searchendpoints: Optional[Dict] = None, **store_config,
     ):
         return StoreLibrary()
 

kgforge/specializations/stores/nexus/service.py

@@ -37,6 +37,7 @@
     LazyAction,
 )
 from kgforge.core.commons.exceptions import ConfigurationError
+from kgforge.core.commons.execution import error_message, format_message
 from kgforge.core.commons.context import Context
 from kgforge.core.conversions.rdf import (
     _from_jsonld_one,
@@ -597,10 +598,8 @@ def to_resource(
 
 
 def _error_message(error: Union[HTTPError, Dict]) -> str:
-    def format_message(msg):
-        return "".join([msg[0].lower(), msg[1:-1], msg[-1] if msg[-1] != "." else ""])
-
     try:
+        # Error from Nexus
         error_json = error.response.json() if isinstance(error, HTTPError) else error
         messages = []
         reason = error_json.get("reason", None)
@@ -612,9 +611,5 @@ def format_message(msg):
         messages = messages if reason or details else [str(error)]
         return ". ".join(messages)
     except Exception:
-        pass
-    try:
-        error_text = error.response.text() if isinstance(error, HTTPError) else str(error)
-        return format_message(error_text)
-    except Exception:
-        return format_message(str(error))
+        # Return general HTTPError
+        return error_message(error)

kgforge/specializations/stores/sparql_store.py

@@ -60,17 +60,6 @@ def _download_one(
             content_type: str,
             bucket: str
     ) -> None:
-        # path: FilePath.
-        # TODO define downloading method
-        # POLICY Should notify of failures with exception DownloadingError including a message.
-        raise not_supported()
-
-    def retrieve(
-            self, id_: str, version: Optional[Union[int, str]], cross_bucket: bool = False, **params
-    ) -> Optional[Resource]:
-        raise not_supported()
-
-    def _retrieve_filename(self, id: str) -> str:
         raise not_supported()
 
     def _search(
@@ -130,7 +119,7 @@ def _search(
             )
 
         query_statements, query_filters = SPARQLQueryBuilder.build(
-                schema=None, resolvers=resolvers, context=self.model_context(), filters=filters
+                schema=None, resolvers=resolvers, context=self.model_context, filters=filters
             )
         statements = ";\n ".join(query_statements)
         _filters = (".\n".join(query_filters) + '\n') if len(filters) > 0 else ""
@@ -155,7 +144,27 @@ def _sparql(self, query: str) -> Optional[Union[List[Resource], Resource]]:
 
         data = response.json()
 
-        return SPARQLQueryBuilder.build_resource_from_response(query, data, self.model_context())
+        return SPARQLQueryBuilder.build_resource_from_response(query, data, self.model_context)
+
+    def elastic(
+            self, query: str, debug: bool, limit: int = None, offset: int = None, **params
+    ) -> Optional[Union[List[Resource], Resource]]:
+        raise not_supported()
+
+    def _prepare_download_one(self, url: str, store_metadata: Optional[DictWrapper],
+                              cross_bucket: bool) -> Tuple[str, str]:
+        raise not_supported()
+
+    def retrieve(
+        self, id: str, version: Optional[Union[int, str]], cross_bucket: bool, **params
+    ) -> Resource:
+        not_supported()
+
+    def _retrieve_filename(self, id: str) -> str:
+        not_supported()
+
+    def rewrite_uri(self, uri: str, context: Context, **kwargs) -> str:
+        raise not_supported()
 
     # Utils.
 
@@ -180,21 +189,9 @@ def _initialize_service(
             raise ValueError(f"Store configuration error: {ve}") from ve
 
         return SPARQLService(
-            endpoint=endpoint, model_context=self.model_context(),
+            endpoint=endpoint, model_context=self.model_context,
             store_context=store_context, max_connection=max_connection,
             searchendpoints=searchendpoints,
             content_type=content_type,
             accept=accept, **params
         )
-
-    def elastic(
-            self, query: str, debug: bool, limit: int = None, offset: int = None, **params
-    ) -> Optional[Union[List[Resource], Resource]]:
-        raise not_supported()
-
-    def _prepare_download_one(self, url: str, store_metadata: Optional[DictWrapper],
-                              cross_bucket: bool) -> Tuple[str, str]:
-        raise not_supported()
-
-    def rewrite_uri(self, uri: str, context: Context, **kwargs) -> str:
-        raise not_supported()