Merge pull request #5 from linkml/stashed-stuff

implements linkml-dataops

Author: cmungall

Makefile

@@ -1,4 +1,4 @@
-RUN = pipenv run
+RUN = poetry run
 LSOLR = $(RUN) lsolr
 
 all: build test
@@ -8,8 +8,11 @@ clean:
 
 build: linkml_solr/solrschema.py
 
+# Running make test
 # TODO: sometimes load starts before docker environment complete
-test: test-docker-env
+test: test-docker-env test_main
+
+test_main:
 	$(RUN) python -m unittest
 
 test_data: tests/test_models/kitchen_sink.context.jsonld tests/test_models/kitchen_sink.py
@@ -27,6 +30,8 @@ tests/test_models/%.context.jsonld: $(TESTMODELS_SRC)
 	$(RUN) gen-jsonld-context $< > $@
 tests/test_models/%.py: $(TESTMODELS_SRC)
 	$(RUN) gen-python $< > $@
+tests/test_models/%_api.py: $(TESTMODELS_SRC)
+	$(RUN) gen-python-api $< > $@
 tests/test_models/%.ttl: $(TESTMODELS_SRC)
 	$(RUN) gen-rdf $< > $@
 tests/test_models/amigo.yaml: linkml_solr/utils/golr_schema_utils.py
@@ -39,7 +44,7 @@ test-docker-env: test-server wait-10 test-load
 start-solr-server:
 	$(LSOLR) start-server
 test-server:
-	$(LSOLR) start-server -C books -s tests/test_models/books.yaml
+	$(LSOLR) start-server -C books --sleep 10 -s tests/test_models/books.yaml
 
 add-test-core: add-core-test
 add-core-%:

Pipfile

@@ -0,0 +1,23 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+linkml = "*"
+linkml-runtime = ">=1.1.24"
+linkml-dataops = "*"
+pyparsing = "~=2.4"
+sparqlwrapper = "*"
+pysolr = "*"
+requests = "*"
+flatten-dict = "*"
+linkml-solr = {editable = true, path = "."}
+json-flattener = ">= 0.1.7"
+
+[dev-packages]
+tox = "*"
+linkml-solr = {editable = true, path = "."}
+
+[pipenv]
+allow_prereleases = true

linkml_solr/cli.py

@@ -1,9 +1,10 @@
 from typing import List
 import click
+import time
 import logging
 import subprocess
-from linkml.generators.yamlgen import YAMLGenerator
-from linkml_runtime.linkml_model.meta import SchemaDefinition, SlotDefinitionName
+
+from linkml_runtime.linkml_model import SchemaDefinition
 from linkml_runtime.loaders import yaml_loader
 from linkml_solr import SolrQueryEngine, SolrEndpoint, DEFAULT_CORE, DEFAULT_SOLR_URL
 from linkml_solr.utils.solr_bulkload import bulkload_file
@@ -75,6 +76,10 @@ def bulkload(files, format, schema, url, core):
               default='8983',
               show_default=True,
               help='http port number on which solr rules.')
+@click.option('--sleep',
+              default=10,
+              show_default=True,
+              help='Number of seconds to sleep after initiating server start before exiting.')
 @click.option('--create-schema/--no-create-schema',
               default=True,
               show_default=True,
@@ -87,7 +92,7 @@ def bulkload(files, format, schema, url, core):
               default=DEFAULT_SOLR_URL,
               show_default=True,
               help='solr url.')
-def start_server(schema, kill, container, url, core, port, create_schema):
+def start_server(schema, kill, container, url, core, port, sleep: int, create_schema):
     """
     Starts a solr server (via Docker)
     """
@@ -105,7 +110,12 @@ def start_server(schema, kill, container, url, core, port, create_schema):
         'solr-precreate',
         core]
     subprocess.Popen(command)
+    print(f'Sleeping for {sleep} seconds to allow server time to initialize...')
+    time.sleep(sleep)
+    print(f'Done sleeping!')
     if create_schema and schema:
+        # TODO: use SchemaView
+        from linkml.generators.yamlgen import YAMLGenerator
         schema_obj = YAMLGenerator(schema).schema
         qe = SolrQueryEngine(schema=schema_obj,
                              endpoint=SolrEndpoint(url=f'{url}/{core}'))
@@ -157,6 +167,8 @@ def create_schema(schema, url, core):
     """
     Creates a solr schema from a LinkML schema
     """
+    # TODO: use schemaview
+    from linkml.generators.yamlgen import YAMLGenerator
     schema_obj = YAMLGenerator(schema).schema
     qe = SolrQueryEngine(schema=schema_obj,
                          endpoint=SolrEndpoint(url=f'{url}/{core}'))

linkml_solr/mapper.py

@@ -1,9 +1,8 @@
 import logging
+from typing import Union
 
 from linkml_runtime.utils.formatutils import underscore
 from linkml_runtime.linkml_model.meta import SchemaDefinition, ClassDefinition, YAMLRoot, ElementName, SlotDefinition
-from rdflib import BNode, URIRef, Literal
-from rdflib.term import Node
 
 from linkml_solr.solrmodel import *
 
@@ -34,8 +33,11 @@ def _get_slot(self, sn: str) -> Optional[SlotDefinition]:
     def _get_python_field_for_slot(self, slot: SlotDefinition) -> str:
         return underscore(slot.name) # TODO: map to pythongen
 
-    def pyval_to_solr_atom(self, v: Any, range: ElementName = None, query: SolrQuery = None) -> str:
-        return str(v)
+    def pyval_to_solr_atom(self, v: Any, range: ElementName = None, query: SolrQuery = None) -> Union[str, List[str]]:
+        if isinstance(v, list):
+            return [str(v1) for v1 in v]
+        else:
+            return str(v)
 
     def _get_linkml_class(self, in_obj: Dict) -> str:
         if ASSERTED_TYPE_FIELD in in_obj:

linkml_solr/query.py

@@ -5,15 +5,15 @@
 from dataclasses import dataclass
 import json
 import requests
+from linkml_dataops.query.query_model import AbstractQuery, FetchById
+from linkml_dataops.query.queryengine import QueryEngine
 
 from linkml_runtime.dumpers import json_dumper
 from linkml_runtime.utils.formatutils import underscore
 from linkml_runtime.linkml_model.meta import SchemaDefinition, ClassDefinition, YAMLRoot, ElementName, SlotDefinition, SlotDefinitionName
 
 from linkml_solr.solrmodel import SolrEndpoint, SolrQuery, SolrQueryResult, RawSolrResult, FIELD
-from linkml_solr.solrschema import Transaction
 from linkml_solr.solrschemagen import SolrSchemaGenerator
-
 from linkml_solr.mapper import LinkMLMapper
 
 # https://stackoverflow.com/questions/1176136/convert-string-to-python-class-object
@@ -25,17 +25,17 @@ def class_for_name(module_name, class_name):
     return c
 
 
-# TODO: inherit from linkml_runtime_api.QueryEngine
 @dataclass
-class SolrQueryEngine(object):
+class SolrQueryEngine(QueryEngine):
     """
     ORM wrapper for SOLR endpoint
     """
 
-    endpoint: SolrEndpoint
-    schema: SchemaDefinition
+    endpoint: SolrEndpoint = None
+    schema: SchemaDefinition = None
     mapper: LinkMLMapper = None
     discriminator_field: SlotDefinitionName = None
+    python_classes: List[Type[YAMLRoot]] = None
 
     def __post_init__(self):
         # TODO: use schemaview
@@ -147,6 +147,33 @@ def fetch_object(self, row: Dict,
         logging.debug(new_obj)
         return cls(**new_obj)
 
+    def fetch_by_id(self, q: AbstractQuery) -> YAMLRoot:
+        if isinstance(q, FetchById):
+            tgt_classes = [c for c in self.python_classes if c.class_name == q.target_class]
+            id_field = None
+            for c in self.schema.classes.values():
+                if c.name == q.target_class:
+                    for s in c.slots:
+                        if self.schema.slots[s].identifier:
+                            id_field = s
+            if not id_field:
+                raise ValueError(f'No ID found for {q.target_class}')
+            params = {'target_class': tgt_classes[0],
+                     id_field: q.id}
+            result = self.search(**params)
+            return result.items
+
+    def simple_query(self, target_class: str, **kwargs) -> List[YAMLRoot]:
+        tgt_classes = [c for c in self.python_classes if c.class_name == target_class]
+        if len(tgt_classes) != 1:
+            raise ValueError(f'Class: {target_class}')
+        py_class = tgt_classes[0]
+        params = {k: v for k, v in kwargs.items() if v is not None}
+        result = self.search(target_class=py_class, **params)
+        return result.items
+
+
+
     def execute(self, query: SolrQuery) -> RawSolrResult:
         """
         Execute a solr query on endpoint

linkml_solr/solrmodel.py

@@ -32,7 +32,14 @@ def http_params(self) -> dict:
                 params[k] = v
         if self.filter_query is None:
             self.filter_query = {}
-        params['fq'] = [f'{k}:{_quote(v)}' for (k,v) in self.filter_query.items()]
+        fq = []
+        for k, v in self.filter_query.items():
+            if isinstance(v, list):
+                for v1 in v:
+                    fq.append(f'{k}:{_quote(v1)}')
+            else:
+                fq.append(f'{k}:{_quote(v)}')
+        params['fq'] = fq
         if self.facet_fields is not None:
             params['facet'] = 'on'
             params['facet.field'] = self.facet_fields

linkml_solr/solrschemagen.py

@@ -6,8 +6,6 @@
 from linkml_runtime.linkml_model.meta import SchemaDefinition, ClassDefinition, SlotDefinition, ClassDefinitionName
 from linkml_runtime.dumpers import json_dumper
 from linkml_runtime.utils.schemaview import SchemaView
-from linkml_runtime.utils.yamlutils import YAMLRoot, as_json_object
-from linkml_runtime.utils.formatutils import camelcase, be, underscore
 
 from linkml.utils.generator import Generator, shared_arguments
 

linkml_solr/utils/solr_bulkload.py

@@ -1,10 +1,7 @@
 from typing import List
 import logging
 import subprocess
-from linkml.generators.yamlgen import YAMLGenerator
 from linkml_runtime.linkml_model.meta import SchemaDefinition, SlotDefinitionName
-from linkml_runtime.loaders import yaml_loader
-from linkml_solr import SolrQueryEngine, SolrEndpoint
 
 def _get_multivalued_slots(schema: SchemaDefinition) -> List[SlotDefinitionName]:
     return [s.name for s in schema.slots.values() if s.multivalued]

poetry.lock

@@ -10,12 +10,13 @@ lsolr = "linkml_solr.cli:main"
 [tool.poetry.dependencies]
 python = "^3.7.4"
 linkml = "^1.1.15"
-linkml-runtime = "^1.1.12"
+linkml-runtime = "^1.2.7"
 pyparsing = "2.4"
 SPARQLWrapper = "^1.8.5"
 pysolr = "^3.9.0"
 flatten-dict = "^0.4.2"
 json-flattener = "^0.1.9"
+linkml-dataops = "^0.1.0"
 
 [tool.poetry.dev-dependencies]
 tox = "^3.24.5"