From 035fb9ed332c5e276b5b4d1e36f6b914096d87ed Mon Sep 17 00:00:00 2001 From: amercader Date: Fri, 9 Jun 2023 17:11:35 +0200 Subject: [PATCH] Drop Python 2 support --- .github/workflows/test.yml | 9 +- README.rst | 2 + bin/ckan_pycsw.py | 2 +- ckanext/spatial/harvested_metadata.py | 6 +- ckanext/spatial/harvesters/base.py | 28 +-- ckanext/spatial/harvesters/csw.py | 162 ++++++++++-------- ckanext/spatial/harvesters/gemini.py | 26 +-- ckanext/spatial/harvesters/waf.py | 12 +- ckanext/spatial/lib/__init__.py | 4 +- ckanext/spatial/lib/csw_client.py | 4 +- ckanext/spatial/lib/report.py | 8 +- ckanext/spatial/plugin/__init__.py | 8 +- ckanext/spatial/postgis/model.py | 4 +- .../spatial/tests/functional/test_package.py | 120 +++++-------- .../tests/postgis/test_package_extent.py | 8 +- ckanext/spatial/util.py | 13 +- pip-requirements-py2.txt | 1 - requirements-py2.txt | 13 -- requirements.txt | 1 - 19 files changed, 206 insertions(+), 225 deletions(-) delete mode 120000 pip-requirements-py2.txt delete mode 100644 requirements-py2.txt diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ca970f89..7abee67e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,13 +22,8 @@ jobs: include: - ckan-version: "2.10" solr-image: "2.10-spatial" - requirements-file: 'requirements.txt' - ckan-version: 2.9 solr-image: 2.9-solr8-spatial - requirements-file: 'requirements.txt' - - ckan-version: 2.9-py2 - solr-image: 2.9-py2-solr8-spatial - requirements-file: 'requirements-py2.txt' fail-fast: false name: CKAN ${{ matrix.ckan-version }}, Solr ${{ matrix.solr-image }} @@ -89,10 +84,10 @@ jobs: restore-keys: | ${{ runner.os }}-spatial-ckan-${{ matrix.ckan-version }}-${{ hashFiles('*requirements*.txt') }} - - name: Install dependencies from ${{ matrix.requirements-file }} + - name: Install dependencies if: steps.cache.outputs.cache-hit != 'true' run: | - pip install -r ${{ matrix.requirements-file }} + pip install -r requirements.txt - name: Install harvester if: steps.cache.outputs.cache-hit != 'true' diff --git a/README.rst b/README.rst index ab9f3cf9..715554de 100644 --- a/README.rst +++ b/README.rst @@ -27,6 +27,8 @@ Supported Versions ------------------ ckanext-spatial >= 2.0.0 supports CKAN 2.9 and CKAN 2.10. +ckanext-spatial >= 2.1.0 supports Python 3 only. + Check the [tested enviroments](https://github.com/ckan/ckanext-spatial/blob/master/.github/workflows/test.yml) for more details. diff --git a/bin/ckan_pycsw.py b/bin/ckan_pycsw.py index 47e171b1..05479e2c 100644 --- a/bin/ckan_pycsw.py +++ b/bin/ckan_pycsw.py @@ -4,7 +4,7 @@ import io import os import argparse -from six.moves.configparser import SafeConfigParser +from configparser import SafeConfigParser import requests from lxml import etree diff --git a/ckanext/spatial/harvested_metadata.py b/ckanext/spatial/harvested_metadata.py index a7225185..b9778e4f 100644 --- a/ckanext/spatial/harvested_metadata.py +++ b/ckanext/spatial/harvested_metadata.py @@ -1,7 +1,7 @@ +import logging + from lxml import etree -import six -import logging log = logging.getLogger(__name__) @@ -38,7 +38,7 @@ def read_value(self, name): def get_xml_tree(self): if self.xml_tree is None: parser = etree.XMLParser(remove_blank_text=True) - xml_str = six.ensure_str(self.xml_str) + xml_str = str(self.xml_str) self.xml_tree = etree.fromstring(xml_str, parser=parser) return self.xml_tree diff --git a/ckanext/spatial/harvesters/base.py b/ckanext/spatial/harvesters/base.py index 947defc4..312cbb2b 100644 --- a/ckanext/spatial/harvesters/base.py +++ b/ckanext/spatial/harvesters/base.py @@ -1,6 +1,6 @@ -import six -from six.moves.urllib.parse import urlparse -from six.moves.urllib.request import urlopen + +from urllib.parse import urlparse +from urllib.request import urlopen import re import cgitb @@ -300,7 +300,7 @@ def get_package_dict(self, context, data_dict): if package is None or package.title != iso_values['title']: name = self._gen_new_name(iso_values['title']) if not name: - name = self._gen_new_name(six.text_type(iso_values['guid'])) + name = self._gen_new_name(str(iso_values['guid'])) if not name: raise Exception('Could not generate a unique name from the title or the GUID. Please choose a more unique title.') package_dict['name'] = name @@ -414,7 +414,7 @@ def _extract_first_license_url(licences): ymin = float(bbox['south']) ymax = float(bbox['north']) except ValueError as e: - self._save_object_error('Error parsing bounding box value: {0}'.format(six.text_type(e)), + self._save_object_error('Error parsing bounding box value: {0}'.format(str(e)), harvest_object, 'Import') else: # Construct a GeoJSON extent so ckanext-spatial can register the extent geometry @@ -472,7 +472,7 @@ def _extract_first_license_url(licences): log.debug('Processing extra %s', key) if not key in extras or override_extras: # Look for replacement strings - if isinstance(value,six.string_types): + if isinstance(value, str): value = value.format(harvest_source_id=harvest_object.job.source.id, harvest_source_url=harvest_object.job.source.url.strip('/'), harvest_source_title=harvest_object.job.source.title, @@ -576,7 +576,7 @@ def import_stage(self, harvest_object): iso_parser = ISODocument(harvest_object.content) iso_values = iso_parser.read_values() except Exception as e: - self._save_object_error('Error parsing ISO document for object {0}: {1}'.format(harvest_object.id, six.text_type(e)), + self._save_object_error('Error parsing ISO document for object {0}: {1}'.format(harvest_object.id, str(e)), harvest_object, 'Import') return False @@ -646,7 +646,7 @@ def import_stage(self, harvest_object): # The default package schema does not like Upper case tags tag_schema = logic.schema.default_tags_schema() - tag_schema['name'] = [not_empty, six.text_type] + tag_schema['name'] = [not_empty, str] # Flag this object as the current one harvest_object.current = True @@ -659,8 +659,8 @@ def import_stage(self, harvest_object): # We need to explicitly provide a package ID, otherwise ckanext-spatial # won't be be able to link the extent to the package. - package_dict['id'] = six.text_type(uuid.uuid4()) - package_schema['id'] = [six.text_type] + package_dict['id'] = str(uuid.uuid4()) + package_schema['id'] = [str] # Save reference to the package on the object harvest_object.package_id = package_dict['id'] @@ -675,7 +675,7 @@ def import_stage(self, harvest_object): package_id = p.toolkit.get_action('package_create')(context, package_dict) log.info('Created new package %s with guid %s', package_id, harvest_object.guid) except p.toolkit.ValidationError as e: - self._save_object_error('Validation Error: %s' % six.text_type(e.error_summary), harvest_object, 'Import') + self._save_object_error('Validation Error: %s' % str(e.error_summary), harvest_object, 'Import') return False elif status == 'change': @@ -721,7 +721,7 @@ def import_stage(self, harvest_object): package_id = p.toolkit.get_action('package_update')(context, package_dict) log.info('Updated package %s with guid %s', package_id, harvest_object.guid) except p.toolkit.ValidationError as e: - self._save_object_error('Validation Error: %s' % six.text_type(e.error_summary), harvest_object, 'Import') + self._save_object_error('Validation Error: %s' % str(e.error_summary), harvest_object, 'Import') return False model.Session.commit() @@ -738,7 +738,7 @@ def _is_wms(self, url): s = wms.WebMapService(url) return isinstance(s.contents, dict) and s.contents != {} except Exception as e: - log.error('WMS check for %s failed with exception: %s' % (url, six.text_type(e))) + log.error('WMS check for %s failed with exception: %s' % (url, str(e))) return False def _get_object_extra(self, harvest_object, key): @@ -881,7 +881,7 @@ def _validate_document(self, document_string, harvest_object, validator=None): try: xml = etree.fromstring(document_string) except etree.XMLSyntaxError as e: - self._save_object_error('Could not parse XML file: {0}'.format(six.text_type(e)), harvest_object, 'Import') + self._save_object_error('Could not parse XML file: {0}'.format(str(e)), harvest_object, 'Import') return False, None, [] valid, profile, errors = validator.is_valid(xml) diff --git a/ckanext/spatial/harvesters/csw.py b/ckanext/spatial/harvesters/csw.py index c27824be..2e317698 100644 --- a/ckanext/spatial/harvesters/csw.py +++ b/ckanext/spatial/harvesters/csw.py @@ -1,6 +1,5 @@ import re -import six -from six.moves.urllib.parse import urlparse, urlunparse, urlencode +from urllib.parse import urlparse, urlunparse, urlencode import logging @@ -17,53 +16,51 @@ class CSWHarvester(SpatialHarvester, SingletonPlugin): - ''' + """ A Harvester for CSW servers - ''' + """ + implements(IHarvester) csw = None def info(self): return { - 'name': 'csw', - 'title': 'CSW Server', - 'description': 'A server that implements OGC\'s Catalog Service for the Web (CSW) standard' - } + "name": "csw", + "title": "CSW Server", + "description": "A server that implements OGC's Catalog Service for the Web (CSW) standard", + } def get_original_url(self, harvest_object_id): - obj = model.Session.query(HarvestObject).\ - filter(HarvestObject.id==harvest_object_id).\ - first() + obj = ( + model.Session.query(HarvestObject) + .filter(HarvestObject.id == harvest_object_id) + .first() + ) parts = urlparse(obj.source.url) params = { - 'SERVICE': 'CSW', - 'VERSION': '2.0.2', - 'REQUEST': 'GetRecordById', - 'OUTPUTSCHEMA': 'http://www.isotc211.org/2005/gmd', - 'OUTPUTFORMAT':'application/xml' , - 'ID': obj.guid + "SERVICE": "CSW", + "VERSION": "2.0.2", + "REQUEST": "GetRecordById", + "OUTPUTSCHEMA": "http://www.isotc211.org/2005/gmd", + "OUTPUTFORMAT": "application/xml", + "ID": obj.guid, } - url = urlunparse(( - parts.scheme, - parts.netloc, - parts.path, - None, - urlencode(params), - None - )) + url = urlunparse( + (parts.scheme, parts.netloc, parts.path, None, urlencode(params), None) + ) return url def output_schema(self): - return 'gmd' + return "gmd" def gather_stage(self, harvest_job): - log = logging.getLogger(__name__ + '.CSW.gather') - log.debug('CswHarvester gather_stage for job: %r', harvest_job) + log = logging.getLogger(__name__ + ".CSW.gather") + log.debug("CswHarvester gather_stage for job: %r", harvest_job) # Get source URL url = harvest_job.source.url @@ -72,12 +69,16 @@ def gather_stage(self, harvest_job): try: self._setup_csw_client(url) except Exception as e: - self._save_gather_error('Error contacting the CSW server: %s' % e, harvest_job) + self._save_gather_error( + "Error contacting the CSW server: %s" % e, harvest_job + ) return None - query = model.Session.query(HarvestObject.guid, HarvestObject.package_id).\ - filter(HarvestObject.current==True).\ - filter(HarvestObject.harvest_source_id==harvest_job.source.id) + query = ( + model.Session.query(HarvestObject.guid, HarvestObject.package_id) + .filter(HarvestObject.current == True) + .filter(HarvestObject.harvest_source_id == harvest_job.source.id) + ) guid_to_package_id = {} for guid, package_id in query: @@ -86,26 +87,36 @@ def gather_stage(self, harvest_job): guids_in_db = set(guid_to_package_id.keys()) # extract cql filter if any - cql = self.source_config.get('cql') + cql = self.source_config.get("cql") - log.debug('Starting gathering for %s' % url) + log.debug("Starting gathering for %s" % url) guids_in_harvest = set() try: - for identifier in self.csw.getidentifiers(page=10, outputschema=self.output_schema(), cql=cql): + for identifier in self.csw.getidentifiers( + page=10, outputschema=self.output_schema(), cql=cql + ): try: - log.info('Got identifier %s from the CSW', identifier) + log.info("Got identifier %s from the CSW", identifier) if identifier is None: - log.error('CSW returned identifier %r, skipping...' % identifier) + log.error( + "CSW returned identifier %r, skipping..." % identifier + ) continue guids_in_harvest.add(identifier) except Exception as e: - self._save_gather_error('Error for the identifier %s [%r]' % (identifier,e), harvest_job) + self._save_gather_error( + "Error for the identifier %s [%r]" % (identifier, e), + harvest_job, + ) continue except Exception as e: - log.error('Exception: %s' % text_traceback()) - self._save_gather_error('Error gathering the identifiers from the CSW server [%s]' % six.text_type(e), harvest_job) + log.error("Exception: %s" % text_traceback()) + self._save_gather_error( + "Error gathering the identifiers from the CSW server [%s]" % str(e), + harvest_job, + ) return None new = guids_in_harvest - guids_in_db @@ -114,78 +125,95 @@ def gather_stage(self, harvest_job): ids = [] for guid in new: - obj = HarvestObject(guid=guid, job=harvest_job, - extras=[HOExtra(key='status', value='new')]) + obj = HarvestObject( + guid=guid, job=harvest_job, extras=[HOExtra(key="status", value="new")] + ) obj.save() ids.append(obj.id) for guid in change: - obj = HarvestObject(guid=guid, job=harvest_job, - package_id=guid_to_package_id[guid], - extras=[HOExtra(key='status', value='change')]) + obj = HarvestObject( + guid=guid, + job=harvest_job, + package_id=guid_to_package_id[guid], + extras=[HOExtra(key="status", value="change")], + ) obj.save() ids.append(obj.id) for guid in delete: - obj = HarvestObject(guid=guid, job=harvest_job, - package_id=guid_to_package_id[guid], - extras=[HOExtra(key='status', value='delete')]) - model.Session.query(HarvestObject).\ - filter_by(guid=guid).\ - update({'current': False}, False) + obj = HarvestObject( + guid=guid, + job=harvest_job, + package_id=guid_to_package_id[guid], + extras=[HOExtra(key="status", value="delete")], + ) + model.Session.query(HarvestObject).filter_by(guid=guid).update( + {"current": False}, False + ) obj.save() ids.append(obj.id) if len(ids) == 0: - self._save_gather_error('No records received from the CSW server', harvest_job) + self._save_gather_error( + "No records received from the CSW server", harvest_job + ) return None return ids - def fetch_stage(self,harvest_object): + def fetch_stage(self, harvest_object): # Check harvest object status - status = self._get_object_extra(harvest_object, 'status') + status = self._get_object_extra(harvest_object, "status") - if status == 'delete': + if status == "delete": # No need to fetch anything, just pass to the import stage return True - log = logging.getLogger(__name__ + '.CSW.fetch') - log.debug('CswHarvester fetch_stage for object: %s', harvest_object.id) + log = logging.getLogger(__name__ + ".CSW.fetch") + log.debug("CswHarvester fetch_stage for object: %s", harvest_object.id) url = harvest_object.source.url try: self._setup_csw_client(url) except Exception as e: - self._save_object_error('Error contacting the CSW server: %s' % e, - harvest_object) + self._save_object_error( + "Error contacting the CSW server: %s" % e, harvest_object + ) return False identifier = harvest_object.guid try: - record = self.csw.getrecordbyid([identifier], outputschema=self.output_schema()) + record = self.csw.getrecordbyid( + [identifier], outputschema=self.output_schema() + ) except Exception as e: - self._save_object_error('Error getting the CSW record with GUID %s' % identifier, harvest_object) + self._save_object_error( + "Error getting the CSW record with GUID %s" % identifier, harvest_object + ) return False if record is None: - self._save_object_error('Empty record for GUID %s' % identifier, - harvest_object) + self._save_object_error( + "Empty record for GUID %s" % identifier, harvest_object + ) return False try: # Save the fetch contents in the HarvestObject # Contents come from csw_client already declared and encoded as utf-8 # Remove original XML declaration - content = re.sub('<\?xml(.*)\?>', '', record['xml']) + content = re.sub("<\?xml(.*)\?>", "", record["xml"]) harvest_object.content = content.strip() harvest_object.save() except Exception as e: - self._save_object_error('Error saving the harvest object for GUID %s [%r]' % \ - (identifier, e), harvest_object) + self._save_object_error( + "Error saving the harvest object for GUID %s [%r]" % (identifier, e), + harvest_object, + ) return False - log.debug('XML content saved (len %s)', len(record['xml'])) + log.debug("XML content saved (len %s)", len(record["xml"])) return True def _setup_csw_client(self, url): diff --git a/ckanext/spatial/harvesters/gemini.py b/ckanext/spatial/harvesters/gemini.py index 07fa0b9d..f905944d 100644 --- a/ckanext/spatial/harvesters/gemini.py +++ b/ckanext/spatial/harvesters/gemini.py @@ -8,9 +8,9 @@ - GeminiWafHarvester - An index page with links to GEMINI resources ''' -import six + import os -from six.moves.urllib.parse import urlparse +from urllib.parse import urlparse from datetime import datetime from numbers import Number import uuid @@ -73,10 +73,10 @@ def import_stage(self, harvest_object): return True except Exception as e: log.error('Exception during import: %s' % text_traceback()) - if not six.text_type(e).strip(): + if not str(e).strip(): self._save_object_error('Error importing Gemini document.', harvest_object, 'Import') else: - self._save_object_error('Error importing Gemini document: %s' % six.text_type(e), harvest_object, 'Import') + self._save_object_error('Error importing Gemini document: %s' % str(e), harvest_object, 'Import') raise if debug_exception_mode: raise @@ -275,7 +275,7 @@ def write_package_from_gemini_string(self, content): if package is None or package.title != gemini_values['title']: name = self.gen_new_name(gemini_values['title']) if not name: - name = self.gen_new_name(six.text_type(gemini_guid)) + name = self.gen_new_name(str(gemini_guid)) if not name: raise Exception('Could not generate a unique name from the title or the GUID. Please choose a more unique title.') package_dict['name'] = name @@ -320,7 +320,7 @@ def write_package_from_gemini_string(self, content): extras_as_dict = [] for key,value in extras.items(): - if isinstance(value, six.string_types + (Number,)): + if isinstance(value, (str, Number,)): extras_as_dict.append({'key':key,'value':value}) else: extras_as_dict.append({'key':key,'value':json.dumps(value)}) @@ -413,8 +413,8 @@ def gen_new_name(self, title): else: counter = 1 while counter < 101: - if name+six.text_type(counter) not in taken: - return name+six.text_type(counter) + if name+str(counter) not in taken: + return name+str(counter) counter = counter + 1 return None @@ -454,7 +454,7 @@ def _create_package_from_data(self, package_dict, package = None): # The default package schema does not like Upper case tags tag_schema = logic.schema.default_tags_schema() - tag_schema['name'] = [not_empty,six.text_type] + tag_schema['name'] = [not_empty,str] package_schema['tags'] = tag_schema # TODO: user @@ -467,8 +467,8 @@ def _create_package_from_data(self, package_dict, package = None): if not package: # We need to explicitly provide a package ID, otherwise ckanext-spatial # won't be be able to link the extent to the package. - package_dict['id'] = six.text_type(uuid.uuid4()) - package_schema['id'] = [six.text_type] + package_dict['id'] = str(uuid.uuid4()) + package_schema['id'] = [str] action_function = get_action('package_create') else: @@ -478,7 +478,7 @@ def _create_package_from_data(self, package_dict, package = None): try: package_dict = action_function(context, package_dict) except ValidationError as e: - raise Exception('Validation Error: %s' % six.text_type(e.error_summary)) + raise Exception('Validation Error: %s' % str(e.error_summary)) if debug_exception_mode: raise @@ -572,7 +572,7 @@ def gather_stage(self, harvest_job): except Exception as e: log.error('Exception: %s' % text_traceback()) - self._save_gather_error('Error gathering the identifiers from the CSW server [%s]' % six.text_type(e), harvest_job) + self._save_gather_error('Error gathering the identifiers from the CSW server [%s]' % str(e), harvest_job) return None if len(ids) == 0: diff --git a/ckanext/spatial/harvesters/waf.py b/ckanext/spatial/harvesters/waf.py index 4ea3247f..d8232333 100644 --- a/ckanext/spatial/harvesters/waf.py +++ b/ckanext/spatial/harvesters/waf.py @@ -1,7 +1,6 @@ from __future__ import print_function -import six -from six.moves.urllib.parse import urljoin +from urllib.parse import urljoin import logging import hashlib @@ -9,7 +8,6 @@ import pyparsing as parse import requests from sqlalchemy.orm import aliased -from sqlalchemy.exc import DataError from ckan import model @@ -98,7 +96,7 @@ def gather_stage(self,harvest_job,collection_package_id=None): url_to_modified_harvest = {} ## mapping of url to last_modified in harvest try: - for url, modified_date in _extract_waf(six.text_type(content),source_url,scraper): + for url, modified_date in _extract_waf(str(content),source_url,scraper): url_to_modified_harvest[url] = modified_date except Exception as e: msg = 'Error extracting URLs from %s, error was %s' % (source_url, e) @@ -316,16 +314,16 @@ def _extract_waf(content, base_url, scraper, results = None, depth=0): response = requests.get(new_url) content = response.content except Exception as e: - print(six.text_type(e)) + print(str(e)) continue - _extract_waf(six.text_type(content), new_url, scraper, results, new_depth) + _extract_waf(str(content), new_url, scraper, results, new_depth) continue if not url.endswith('.xml'): continue date = record.date if date: try: - date = six.text_type(dateutil.parser.parse(date)) + date = str(dateutil.parser.parse(date)) except Exception as e: raise date = None diff --git a/ckanext/spatial/lib/__init__.py b/ckanext/spatial/lib/__init__.py index 2e3fcaa4..7b326c48 100644 --- a/ckanext/spatial/lib/__init__.py +++ b/ckanext/spatial/lib/__init__.py @@ -1,5 +1,5 @@ import logging -import six + import ckantoolkit as tk config = tk.config @@ -47,7 +47,7 @@ def normalize_bbox(bbox_values): If there are any problems parsing the input it returns None. """ - if isinstance(bbox_values, six.string_types): + if isinstance(bbox_values, str): bbox_values = bbox_values.split(",") if len(bbox_values) != 4: diff --git a/ckanext/spatial/lib/csw_client.py b/ckanext/spatial/lib/csw_client.py index 0ac075e5..eb86bbc8 100644 --- a/ckanext/spatial/lib/csw_client.py +++ b/ckanext/spatial/lib/csw_client.py @@ -2,7 +2,7 @@ Some very thin wrapper classes around those in OWSLib for convenience. """ -import six + import logging from owslib.etree import etree @@ -33,7 +33,7 @@ def _xmd(self, obj): pass elif callable(val): pass - elif isinstance(val, six.string_types): + elif isinstance(val, str): md[attr] = val elif isinstance(val, int): md[attr] = val diff --git a/ckanext/spatial/lib/report.py b/ckanext/spatial/lib/report.py index d1165d68..e4c54f21 100644 --- a/ckanext/spatial/lib/report.py +++ b/ckanext/spatial/lib/report.py @@ -3,9 +3,9 @@ and then saved as a CSV. ''' -from six import text_type, StringIO import datetime import csv +from io import StringIO class ReportTable(object): @@ -52,9 +52,9 @@ def get_csv(self): if isinstance(cell, datetime.datetime): cell = cell.strftime('%Y-%m-%d %H:%M') elif isinstance(cell, int): - cell = text_type(cell) + cell = str(cell) elif isinstance(cell, (list, tuple)): - cell = text_type(cell) + cell = str(cell) elif cell is None: cell = '' else: @@ -63,6 +63,6 @@ def get_csv(self): try: csvwriter.writerow(row_formatted) except Exception as e: - raise Exception("%s: %s, %s"%(e, row, row_formatted)) + raise Exception("%s: %s, %s" % (e, row, row_formatted)) csvout.seek(0) return csvout.read() diff --git a/ckanext/spatial/plugin/__init__.py b/ckanext/spatial/plugin/__init__.py index a3c7dd74..99a486e6 100644 --- a/ckanext/spatial/plugin/__init__.py +++ b/ckanext/spatial/plugin/__init__.py @@ -2,7 +2,7 @@ import mimetypes from logging import getLogger -import six + import geojson import shapely.geometry @@ -138,10 +138,10 @@ def check_spatial_extra(self, dataset_dict, update=False): try: log.debug("Received geometry: {}".format(geometry)) - geometry = geojson.loads(six.text_type(geometry)) + geometry = geojson.loads(str(geometry)) except ValueError as e: error_dict = { - "spatial": ["Error decoding JSON object: {}".format(six.text_type(e))]} + "spatial": ["Error decoding JSON object: {}".format(str(e))]} raise tk.ValidationError(error_dict) if not hasattr(geometry, "is_valid") or not geometry.is_valid: @@ -158,7 +158,7 @@ def check_spatial_extra(self, dataset_dict, update=False): except Exception as e: if bool(os.getenv('DEBUG')): raise - error_dict = {"spatial": ["Error: {}".format(six.text_type(e))]} + error_dict = {"spatial": ["Error: {}".format(str(e))]} raise tk.ValidationError(error_dict) # ITemplateHelpers diff --git a/ckanext/spatial/postgis/model.py b/ckanext/spatial/postgis/model.py index 5a89e1d7..8b43a380 100644 --- a/ckanext/spatial/postgis/model.py +++ b/ckanext/spatial/postgis/model.py @@ -1,7 +1,7 @@ import logging from string import Template -import six + from sqlalchemy import Table, Column, types, func from geoalchemy2.elements import WKTElement @@ -228,7 +228,7 @@ def bbox_query_ordered(bbox, srid=None): input_geometry = _bbox_2_wkt(bbox, srid) params = { - "query_bbox": six.text_type(input_geometry), + "query_bbox": str(input_geometry), "query_srid": input_geometry.srid, } diff --git a/ckanext/spatial/tests/functional/test_package.py b/ckanext/spatial/tests/functional/test_package.py index 542618bf..6244fd14 100644 --- a/ckanext/spatial/tests/functional/test_package.py +++ b/ckanext/spatial/tests/functional/test_package.py @@ -6,94 +6,68 @@ import ckan.tests.helpers as helpers -@pytest.mark.usefixtures("with_plugins", "with_request_context", "clean_db", "clean_index", "harvest_setup") +@pytest.fixture +def env(): + if tk.check_ckan_version(min_version="2.10"): + user = factories.SysadminWithToken() + env = {"Authorization": user["token"]} + else: + user = factories.User() + env = {"REMOTE_USER": user["name"]} + + return env + + +@pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index", "harvest_setup") class TestSpatialExtra(SpatialTestBase): - def test_spatial_extra_base(self, app): + def test_spatial_extra_base(self, app, env): - user = factories.User() - env = {"REMOTE_USER": user["name"].encode("ascii")} - dataset = factories.Dataset(user=user) - - if tk.check_ckan_version(min_version="2.9"): - offset = tk.url_for("dataset.edit", id=dataset["id"]) - else: - offset = tk.url_for(controller="package", action="edit", id=dataset["id"]) - - if tk.check_ckan_version(min_version="2.9"): - data = { - "name": dataset["name"], - "extras__0__key": u"spatial", - "extras__0__value": self.geojson_examples["point"], - } - res = app.post(offset, environ_overrides=env, data=data) - else: - form = res.forms[1] - form["extras__0__key"] = u"spatial" - form["extras__0__value"] = self.geojson_examples["point"] - - res = app.get(offset, extra_environ=env) - res = helpers.submit_and_follow(app, form, env, "save") - - assert "Error" not in res, res + dataset = factories.Dataset() + + offset = tk.url_for("dataset.edit", id=dataset["id"]) + + data = { + "name": dataset["name"], + "extras__0__key": u"spatial", + "extras__0__value": self.geojson_examples["point"], + } + + app.post(offset, headers=env, data=data, follow_redirects=False) dataset_dict = tk.get_action("package_show")({}, {"id": dataset["id"]}) assert dataset_dict["extras"][0]["key"] == "spatial" assert dataset_dict["extras"][0]["value"] == self.geojson_examples["point"] - def test_spatial_extra_bad_json(self, app): + def test_spatial_extra_bad_json(self, app, env): - user = factories.User() - env = {"REMOTE_USER": user["name"].encode("ascii")} - dataset = factories.Dataset(user=user) - - if tk.check_ckan_version(min_version="2.9"): - offset = tk.url_for("dataset.edit", id=dataset["id"]) - else: - offset = tk.url_for(controller="package", action="edit", id=dataset["id"]) - res = app.get(offset, extra_environ=env) - - if tk.check_ckan_version(min_version="2.9"): - data = { - "name": dataset["name"], - "extras__0__key": u"spatial", - "extras__0__value": u'{"Type":Bad Json]', - } - res = app.post(offset, environ_overrides=env, data=data) - else: - form = res.forms[1] - form["extras__0__key"] = u"spatial" - form["extras__0__value"] = u'{"Type":Bad Json]' - res = helpers.webtest_submit(form, extra_environ=env, name="save") + dataset = factories.Dataset() + + offset = tk.url_for("dataset.edit", id=dataset["id"]) + + data = { + "name": dataset["name"], + "extras__0__key": u"spatial", + "extras__0__value": u'{"Type":Bad Json]', + } + res = app.post(offset, environ_overrides=env, data=data) assert "Error" in res, res assert "Spatial" in res assert "Error decoding JSON object" in res - def test_spatial_extra_bad_geojson(self, app): + def test_spatial_extra_bad_geojson(self, app, env): - user = factories.User() - env = {"REMOTE_USER": user["name"].encode("ascii")} - dataset = factories.Dataset(user=user) - - if tk.check_ckan_version(min_version="2.9"): - offset = tk.url_for("dataset.edit", id=dataset["id"]) - else: - offset = tk.url_for(controller="package", action="edit", id=dataset["id"]) - res = app.get(offset, extra_environ=env) - - if tk.check_ckan_version(min_version="2.9"): - data = { - "name": dataset["name"], - "extras__0__key": u"spatial", - "extras__0__value": u'{"Type":"Bad_GeoJSON","a":2}', - } - res = app.post(offset, environ_overrides=env, data=data) - else: - form = res.forms[1] - form["extras__0__key"] = u"spatial" - form["extras__0__value"] = u'{"Type":"Bad_GeoJSON","a":2}' - res = helpers.webtest_submit(form, extra_environ=env, name="save") + dataset = factories.Dataset() + + offset = tk.url_for("dataset.edit", id=dataset["id"]) + + data = { + "name": dataset["name"], + "extras__0__key": u"spatial", + "extras__0__value": u'{"Type":"Bad_GeoJSON","a":2}', + } + res = app.post(offset, environ_overrides=env, data=data) assert "Error" in res, res assert "Spatial" in res diff --git a/ckanext/spatial/tests/postgis/test_package_extent.py b/ckanext/spatial/tests/postgis/test_package_extent.py index 1d4ba1bc..68444bdb 100644 --- a/ckanext/spatial/tests/postgis/test_package_extent.py +++ b/ckanext/spatial/tests/postgis/test_package_extent.py @@ -1,5 +1,5 @@ import pytest -import six + import time import random @@ -151,7 +151,7 @@ def create_package(**package_dict): ) class TestCompareGeometries(SpatialTestBase): def _get_extent_object(self, geometry): - if isinstance(geometry, six.string_types): + if isinstance(geometry, str): geometry = json.loads(geometry) geom_obj = shape(geometry) return PackageExtent(package_id="xxx", the_geom=WKTElement(geom_obj.wkt, 4326)) @@ -190,8 +190,8 @@ def initial_data(self): bbox = self.x_values_to_bbox(fixture_x) bbox_geojson = bbox_2_geojson(bbox) create_package( - name=munge_title_to_name(six.text_type(fixture_x)), - title=six.text_type(fixture_x), + name=munge_title_to_name(str(fixture_x)), + title=str(fixture_x), extras=[{"key": "spatial", "value": bbox_geojson}], ) diff --git a/ckanext/spatial/util.py b/ckanext/spatial/util.py index 6a4f08aa..be86dc36 100644 --- a/ckanext/spatial/util.py +++ b/ckanext/spatial/util.py @@ -3,8 +3,7 @@ from __future__ import print_function import os import sys - -import six +from io import StringIO from pkg_resources import resource_stream import logging @@ -32,7 +31,7 @@ def report(pkg=None): if pkg: - package_ref = six.text_type(pkg) + package_ref = str(pkg) pkg = model.Package.get(package_ref) if not pkg: print('Package ref "%s" not recognised' % package_ref) @@ -96,7 +95,7 @@ def report_csv(csv_filepath): def initdb(srid=None): if srid: - srid = six.text_type(srid) + srid = str(srid) from ckanext.spatial.postgis.model import setup as db_setup @@ -125,10 +124,10 @@ def update_extents(): count += 1 except ValueError as e: errors.append(u'Package %s - Error decoding JSON object: %s' % - (package.id, six.text_type(e))) + (package.id, str(e))) except TypeError as e: errors.append(u'Package %s - Error decoding JSON object: %s' % - (package.id, six.text_type(e))) + (package.id, str(e))) save_package_extent(package.id, geometry) @@ -203,7 +202,7 @@ def transform_to_html(content, xslt_package=None, xslt_path=None): style_xml = etree.parse(style) transformer = etree.XSLT(style_xml) - xml = etree.parse(six.StringIO(content and six.text_type(content))) + xml = etree.parse(StringIO(content and str(content))) html = transformer(xml) result = etree.tostring(html, pretty_print=True) diff --git a/pip-requirements-py2.txt b/pip-requirements-py2.txt deleted file mode 120000 index 983ca39d..00000000 --- a/pip-requirements-py2.txt +++ /dev/null @@ -1 +0,0 @@ -requirements-py2.txt \ No newline at end of file diff --git a/requirements-py2.txt b/requirements-py2.txt deleted file mode 100644 index cdd3536c..00000000 --- a/requirements-py2.txt +++ /dev/null @@ -1,13 +0,0 @@ -ckantoolkit -Shapely>=1.2.13,<2.0.0 -pyproj==2.2.2 -OWSLib==0.18.0 -lxml>=2.3 -argparse -pyparsing>=2.1.10 -requests>=1.1.0 -six -geojson==2.5.0 -# Harvest extension install 1.3 which try to install -# setuptools>=61.2 which is not compatible with python 2.7 -pika==1.1.0 diff --git a/requirements.txt b/requirements.txt index fa3e6989..3b0418f8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,6 @@ lxml>=2.3 argparse pyparsing>=2.1.10 requests>=1.1.0 -six pyproj==2.6.1; python_version < '3.9' pyproj==3.4.1; python_version >= '3.9'