From 3a6f572c444d5ea489ade6709488f3bb999dd3ba Mon Sep 17 00:00:00 2001 From: Max Burnette Date: Tue, 30 Jul 2024 09:08:09 -0500 Subject: [PATCH] Update geoshp for projections --- preview.geoshp/gsclient.py | 19 +++++++-- preview.geoshp/ncsa.geo.shp.py | 75 ++++++++++++++++++++++----------- preview.geoshp/requirements.txt | 3 +- preview.geoshp/zipshputils.py | 2 +- 4 files changed, 70 insertions(+), 29 deletions(-) diff --git a/preview.geoshp/gsclient.py b/preview.geoshp/gsclient.py index 9e62cea..3cb91d0 100644 --- a/preview.geoshp/gsclient.py +++ b/preview.geoshp/gsclient.py @@ -1,6 +1,7 @@ import errno from geoserver.catalog import Catalog +from pyproj import Transformer import requests import os.path import tempfile @@ -12,7 +13,7 @@ class Client: def __init__(self, geoserver, username, password): self.restserver = urlparse.urljoin(geoserver, 'rest/') - self.wmsserver = urlparse.urljoin(geoserver, 'wms') + self.wmsserver = urlparse.urljoin(geoserver, 'ows') self.username = username self.password = password self.catalog = Catalog(self.restserver, self.username, self.password) @@ -71,14 +72,26 @@ def getLayerByResource(self, resource): self.layer = layers[0] return self.layer - def mintMetadataWithoutGeoserver(self, workspace, filename, extent): + def mintMetadataWithoutGeoserver(self, workspace, filename, extent, epsg): self.logger.debug("Creating wms metadata ... ") metadata = {} wmsLayerName = workspace + ':' + filename metadata['WMS Layer Name'] = wmsLayerName metadata['WMS Service URL'] = self.wmsserver + + # TODO: Extent should be in source coordinate system (e.g. latlon = 4326) + if epsg == "EPSG:4326": + # Initialize the transformer from EPSG:4326 to EPSG:3857 + transformer = Transformer.from_crs("EPSG:3857", "EPSG:4326", always_xy=True) + bbox = extent.split(",") + # Convert the coordinates + min_x, min_y = transformer.transform(bbox[0], bbox[1]) + max_x, max_y = transformer.transform(bbox[2], bbox[3]) + extent = f"{min_x},{min_y},{max_x},{max_y}" + print(extent) + metadata[ - 'WMS Layer URL'] = self.wmsserver + '?request=GetMap&layers=' + wmsLayerName + '&bbox=' + extent + '&width=640&height=480&srs=EPSG:3857&format=image%2Fpng' + 'WMS Layer URL'] = self.wmsserver + '?service=WFS&version=1.0.0&request=GetFeature&typeName=' + wmsLayerName + '&bbox=' + extent + '&srsname=' + epsg + '&outputFormat=application%2Fjson' self.logger.debug('[DONE]') return metadata diff --git a/preview.geoshp/ncsa.geo.shp.py b/preview.geoshp/ncsa.geo.shp.py index a094e4a..ddd0233 100755 --- a/preview.geoshp/ncsa.geo.shp.py +++ b/preview.geoshp/ncsa.geo.shp.py @@ -2,6 +2,7 @@ import logging import os +import json import tempfile import subprocess @@ -105,34 +106,60 @@ def process_message(self, connector, host, secret_key, resource, parameters): except: self.logger.debug("There is no channel or headr is zip shp") else: - # Context URL - context_url = "https://clowder.ncsa.illinois.edu/contexts/metadata.jsonld" - - metadata = { - "@context": [ - context_url, - { - 'WMS Layer Name': 'http://clowder.ncsa.illinois.edu/metadata/ncsa.geoshp.preview#WMS Layer Name', - 'WMS Service URL': 'http://clowder.ncsa.illinois.edu/metadata/ncsa.geoshp.preview#WMS Service URL', - 'WMS Layer URL': 'http://clowder.ncsa.illinois.edu/metadata/ncsa.geoshp.preview#WMS Layer URL' - } - ], - 'attachedTo': {'resourceType': 'file', 'id': parameters["id"]}, - 'agent': { - '@type': 'cat:extractor', - 'extractor_id': 'https://clowder.ncsa.illinois.edu/clowder/api/extractors/' + self.extractorName}, - 'content': { + self.logger.info("metadata checkpoint") + metadata_geo_host = os.getenv("EXTERNAL_GEOSERVER_URL", "") + if len(metadata_geo_host) > 0: + internal_geo_host = os.getenv("GEOSERVER_URL", "") + result = { + 'WMS Layer Name': result['WMS Layer Name'], + 'WMS Service URL': result['WMS Service URL'].replace(internal_geo_host, metadata_geo_host), + 'WMS Layer URL': result['WMS Layer URL'].replace(internal_geo_host, metadata_geo_host) + } + else: + result = { 'WMS Layer Name': result['WMS Layer Name'], 'WMS Service URL': result['WMS Service URL'], 'WMS Layer URL': result['WMS Layer URL'] } - } - # register geoshp preview - (_, ext) = os.path.splitext(inputfile) - (_, tmpfile) = tempfile.mkstemp(suffix=ext) - pyclowder.files.upload_metadata(connector, host, secret_key, fileid, metadata) - self.logger.debug("upload previewer") + # Context URL + self.logger.info('[%s] : %s', fileid, "Starting the metadata upload", extra={'fileid': fileid}) + metadata = self.get_metadata(result, 'file', fileid, host) + + self.logger.info(str(metadata)) + + # register geotiff WMS layers with Clowder + CLOWDER_VERSION = os.getenv("CLOWDER_VERSION", 1) + if int(CLOWDER_VERSION) == 2: + # upload visualization URL + payload = json.dumps({ + "resource": { + "collection": "files", + "resource_id": fileid + }, + "client": host, + "parameters": result, + "visualization_mimetype": "application/zip", + "visualization_component_id": "geoserver-viewer-component" + }) + headers = { + "X-API-KEY": secret_key, + "Content-Type": "application/json" + } + host = os.getenv("CLOWDER_URL", host) + self.logger.info("Visualizations going out") + self.logger.info("Visualizations going out") + self.logger.info('%sapi/v2/visualizations/config' % host) + self.logger.info(str(payload)) + connector.post('%sapi/v2/visualizations/config' % host, headers=headers, data=payload, + verify=connector.ssl_verify if connector else True) + else: + # register geoshp preview + (_, ext) = os.path.splitext(inputfile) + (_, tmpfile) = tempfile.mkstemp(suffix=ext) + host = os.getenv("CLOWDER_URL", host) + pyclowder.files.upload_metadata(connector, host, secret_key, fileid, metadata) + self.logger.debug("upload previewer") except Exception: self.logger.exception("Could not upload zipfile") @@ -210,7 +237,7 @@ def extractZipShp(self, inputfile, fileid, filename, secret_key): if success: self.logger.debug("uploading shapefile to geoserver ---->success") - metadata = gsclient.mintMetadataWithoutGeoserver(self.gs_workspace, combined_name, zipshp.getExtent()) + metadata = gsclient.mintMetadataWithoutGeoserver(self.gs_workspace, combined_name, zipshp.getExtent(), epsg) # metadata = gsclient.mintMetadata(self.gs_workspace, combined_name, zipshp.getExtent()) self.logger.debug("created metadata from geoserver") # TODO: create thumbnail and upload it to Medici diff --git a/preview.geoshp/requirements.txt b/preview.geoshp/requirements.txt index 41230a2..8dc5696 100644 --- a/preview.geoshp/requirements.txt +++ b/preview.geoshp/requirements.txt @@ -2,4 +2,5 @@ gsconfig-py3==1.0.8 pika>=1.0.0 requests>=2.10.0 wheel>=0.24.0 -pyclowder==2.5.1 +pyclowder==3.0.1 +pyproj==3.5.0 diff --git a/preview.geoshp/zipshputils.py b/preview.geoshp/zipshputils.py index 09ac0fa..7b72cf5 100644 --- a/preview.geoshp/zipshputils.py +++ b/preview.geoshp/zipshputils.py @@ -65,7 +65,7 @@ def __init__(self, shpzipfile, zipUtil="/usr/bin/7z"): self.zipShpProp['epsg'] = 'UNKNOWN' self.zipShpProp['hasError'] = True else: - if epsg_code != 'None': + if epsg_code != 'None' and epsg_code != None: self.zipShpProp['epsg'] = epsg_code else: self.zipShpProp['epsg'] = 'UNKNOWN'