From 9bc532a6ee89163d68cff7ce898a41638c581210 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Mon, 11 Apr 2016 23:17:09 -0400 Subject: [PATCH 1/2] implement CSW-T for GeoNode backends --- pycsw/core/metadata.py | 4 +- pycsw/plugins/repository/geonode/geonode_.py | 75 +++++++++++++++++++- 2 files changed, 77 insertions(+), 2 deletions(-) diff --git a/pycsw/core/metadata.py b/pycsw/core/metadata.py index 37ff3f51a..0a5f30195 100644 --- a/pycsw/core/metadata.py +++ b/pycsw/core/metadata.py @@ -104,7 +104,9 @@ def parse_record(context, record, repos=None, def _set(context, obj, name, value): ''' convenience method to set values ''' - setattr(obj, context.md_core_model['mappings'][name], value) + objname = context.md_core_model['mappings'][name] + if name is not None and objname is not None and value is not None: + setattr(obj, objname, value) def _parse_metadata(context, repos, record): """parse metadata formats""" diff --git a/pycsw/plugins/repository/geonode/geonode_.py b/pycsw/plugins/repository/geonode/geonode_.py index c7a642e97..81e0231f5 100644 --- a/pycsw/plugins/repository/geonode/geonode_.py +++ b/pycsw/plugins/repository/geonode/geonode_.py @@ -28,13 +28,17 @@ # # ================================================================= +from datetime import datetime from django.db import models from django.db import connection from django.db.models import Avg, Max, Min, Count from django.conf import settings from pycsw.core import util -from geonode.base.models import ResourceBase +from geonode.base.models import (GenericResource, ResourceBase, Region, + SpatialRepresentationType, TopicCategory) +from geonode.layers.metadata import set_metadata +from geonode.layers.utils import resolve_regions class GeoNodeRepository(object): ''' Class to interact with underlying repository ''' @@ -78,6 +82,11 @@ def __init__(self, context, repo_filter=None): self.queryables['_all'].update(self.queryables[qbl]) self.queryables['_all'].update(self.context.md_core_model['mappings']) + + def dataset(self): + ''' Stub to mock a pycsw dataset object for Transactions''' + return type('GenericResource', (object,), {}) + def query_ids(self, ids): ''' Query by list of identifiers ''' return self._get_repo_filter(ResourceBase.objects).filter(uuid__in=ids).all() @@ -143,8 +152,72 @@ def query(self, constraint, sortby=None, typenames=None, else: # no sort return [str(total), query.all()[startposition:startposition+int(maxrecords)]] + + def insert(self, record, source, insert_date): + ''' Insert a record into the repository ''' + + return self._insert_or_update(record, source, insert_date, mode='insert') + + + def _insert_or_update(self, record, source, insert_date, mode='insert'): + ''' Insert or update a record in the repository ''' + + defaults, keywords = _prepare_resource(record, source, insert_date) + + metadata_record, created = GenericResource.objects.get_or_create(uuid=record.uuid, defaults=defaults) + if not created and mode == 'insert': + raise RuntimeError('Record already exists') + + metadata_record.save() + + keywords = list(set(keywords)) + if keywords: + if len(keywords) > 0: + metadata_record.keywords.add(*keywords) + + def _get_repo_filter(self, query): ''' Apply repository wide side filter / mask query ''' if self.filter is not None: return query.extra(where=[self.filter]) return query + +def _prepare_resource(recobj, source, insert_date): + ''' + Prepare parsed record object to align with GeoNode internal fields + Given some pycsw GeoNode mappings are not internal fields (but + properties or functions), clean/update these fields which cannot + be applied to saving the model + ''' + + defaults = {} + + print(dir(recobj)) + # get model properties from XML + identifier, vals, regions, keywords = set_metadata(recobj.metadata_xml) + + vals['metadata_uploaded_preserve'] = True + vals['metadata_xml'] = recobj.metadata_xml + vals['uuid'] = identifier + vals['remote'] = True + vals['csw_mdsource'] = source + vals['date'] = datetime.strptime(insert_date, '%Y-%m-%dT%H:%M:%SZ') + vals['csw_schema'] = recobj.csw_schema + vals['csw_typename'] = recobj.csw_typename + vals['csw_anytext'] = recobj.csw_anytext + + for key, value in vals.items(): + if key == 'spatial_representation_type': + value = SpatialRepresentationType(identifier=value) + elif key == 'topic_category': + value, created = TopicCategory.objects.get_or_create( + identifier=value.lower(), + defaults={'description': '', 'gn_description': value}) + key = 'category' + defaults[key] = value + else: + defaults[key] = value + + keywords = list(set(keywords)) + + return [defaults, keywords] From 06722156974889d3c1df6e9af273af5fb3793146 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Sun, 26 Jun 2016 19:25:28 -0400 Subject: [PATCH 2/2] test for transaction support --- pycsw/plugins/repository/geonode/geonode_.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/pycsw/plugins/repository/geonode/geonode_.py b/pycsw/plugins/repository/geonode/geonode_.py index 81e0231f5..f070a7181 100644 --- a/pycsw/plugins/repository/geonode/geonode_.py +++ b/pycsw/plugins/repository/geonode/geonode_.py @@ -40,6 +40,11 @@ from geonode.layers.metadata import set_metadata from geonode.layers.utils import resolve_regions +GEONODE_RESOURCETYPES = [ + 'http://www.opengis.net/cat/csw/2.0.2', + 'http://www.opengis.net/wms' +] + class GeoNodeRepository(object): ''' Class to interact with underlying repository ''' def __init__(self, context, repo_filter=None): @@ -48,6 +53,8 @@ def __init__(self, context, repo_filter=None): self.context = context self.filter = repo_filter self.fts = False + self.label = 'GeoNode' + self.local_ingest = True self.dbtype = settings.DATABASES['default']['ENGINE'].split('.')[-1] @@ -82,6 +89,10 @@ def __init__(self, context, repo_filter=None): self.queryables['_all'].update(self.queryables[qbl]) self.queryables['_all'].update(self.context.md_core_model['mappings']) + if 'Harvest' in self.context.model['operations'] and 'Transaction' in self.context.model['operations']: + self.context.model['operations']['Harvest']['parameters']['ResourceType']['values'] = GEONODE_RESOURCETYPES + self.context.model['operations']['Transaction']['parameters']['TransactionSchemas']['values'] = GEONODE_RESOURCETYPES + def dataset(self): ''' Stub to mock a pycsw dataset object for Transactions''' @@ -119,7 +130,7 @@ def query_insert(self, direction='max'): def query_source(self, source): ''' Query by source ''' - return self._get_repo_filter(ResourceBase.objects).filter(source=source) + return self._get_repo_filter(ResourceBase.objects).filter(csw_mdsource=source) def query(self, constraint, sortby=None, typenames=None, maxrecords=10, startposition=0):