diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..08d3374a --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,76 @@ +name: Tests +on: [push, pull_request] +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: '3.6' + - name: Install requirements + run: pip install flake8 pycodestyle + - name: Check syntax + run: flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan + + test: + needs: lint + strategy: + matrix: + ckan-version: [2.9, 2.9-py2, 2.8] + fail-fast: false + + name: CKAN ${{ matrix.ckan-version }} + runs-on: ubuntu-latest + container: + image: openknowledge/ckan-dev:${{ matrix.ckan-version }} + services: + solr: + image: ckan/ckan-solr-dev:${{ matrix.ckan-version }} + postgres: + image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }} + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + redis: + image: redis:3 + env: + CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test + CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test + CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test + CKAN_SOLR_URL: http://solr:8983/solr/ckan + CKAN_REDIS_URL: redis://redis:6379/1 + + steps: + - uses: actions/checkout@v2 + - name: Install requirements (py3) + if: ${{ matrix.ckan-version != '2.7' && matrix.ckan-version != '2.8' && matrix.ckan-version != '2.9-py2' }} + run: | + pip install -r dev-requirements.txt + - name: Install requirements (py2) + if: ${{ matrix.ckan-version == '2.7' || matrix.ckan-version == '2.8' || matrix.ckan-version == '2.9-py2' }} + run: | + pip install -r dev-requirements-py2.txt + - name: Install requirements (common) + run: | + pip install -r requirements.txt + pip install -e . + # Replace default path to CKAN core config file with the one on the container + sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini + - name: Setup extension (CKAN >= 2.9) + if: ${{ matrix.ckan-version != '2.7' && matrix.ckan-version != '2.8' }} + run: | + ckan -c test.ini db init + - name: Setup extension (CKAN < 2.9) + if: ${{ matrix.ckan-version == '2.7' || matrix.ckan-version == '2.8' }} + run: | + paster --plugin=ckan db init -c test.ini + - name: Run tests + run: pytest --ckan-ini=test.ini --cov=ckanext.validation --cov-report=xml --cov-append --disable-warnings ckanext/validation/tests -vv + + - name: Upload coverage report to codecov + uses: codecov/codecov-action@v1 + with: + file: ./coverage.xml diff --git a/README.md b/README.md index 099814d9..df59caab 100644 --- a/README.md +++ b/README.md @@ -50,12 +50,10 @@ If you are eager to get started, jump to the [Installation](#installation) and [ ## Versions supported and requirements -This extension has been tested with CKAN 2.4 to 2.7. +This extension is currently tested in CKAN 2.8 and CKAN 2.9. It is strongly recommended to use it alongside [ckanext-scheming](https://github.com/ckan/ckanext-scheming) to define the necessary extra fields in the default CKAN schema. -If you want to use [asynchronous validation](#asynchronous-validation) with background jobs and are using CKAN 2.6 or lower, [ckanext-rq](https://github.com/ckan/ckanext-rq) is also needed. Please refer to both READMEs for installation instructions. - ## Installation diff --git a/bin/travis-build.bash b/bin/travis-build.bash deleted file mode 100644 index e14c720f..00000000 --- a/bin/travis-build.bash +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/bash -set -e - -echo "This is travis-build.bash..." - -echo "Postgres version" -psql --version - -echo "Installing the packages that CKAN requires..." -sudo apt-get update -qq -sudo apt-get install solr-jetty - -echo "Installing CKAN and its Python dependencies..." -git clone https://github.com/ckan/ckan -cd ckan -if [ $CKAN_VERSION != 'master' ] -then - git checkout $CKAN_VERSION -fi -# Unpin CKAN's psycopg2 dependency get an important bugfix -# https://stackoverflow.com/questions/47044854/error-installing-psycopg2-2-6-2 -sed -i '/psycopg2/c\psycopg2' requirements.txt -python setup.py develop -pip install -r requirements.txt -pip install -r dev-requirements.txt -cd - - -echo "Creating the PostgreSQL user and database..." -sudo -u postgres psql -c "CREATE USER ckan_default WITH PASSWORD 'pass';" -sudo -u postgres psql -c 'CREATE DATABASE ckan_test WITH OWNER ckan_default;' - -echo "SOLR config..." -# Solr is multicore for tests on ckan master, but it's easier to run tests on -# Travis single-core. See https://github.com/ckan/ckan/issues/2972 -sed -i -e 's/solr_url.*/solr_url = http:\/\/127.0.0.1:8983\/solr/' ckan/test-core.ini - -echo "Initialising the database..." -cd ckan -paster db init -c test-core.ini -cd - - -echo "Installing other extensions requirements..." -git clone https://github.com/ckan/ckanext-scheming -cd ckanext-scheming -pip install -r requirements.txt -python setup.py develop -cd - -if [ $CKAN_VERSION == 'dev-v2.6' ] || [ $CKAN_VERSION == 'release-v2.5-latest' ] || [ $CKAN_VERSION == 'release-v2.4-latest' ] -then - git clone https://github.com/ckan/ckanext-rq - cd ckanext-rq - pip install -r requirements.txt - pip install -r dev-requirements.txt - python setup.py develop - cd - - # Enable the rq plugin - sed -i -e 's/ckan.plugins = /ckan.plugins = rq /' test.ini -fi - -echo "Installing ckanext-validation and its requirements..." -python setup.py develop -pip install -r requirements.txt -pip install -r dev-requirements.txt -paster --plugin=ckanext-validation validation init-db -c ckan/test-core.ini - -echo "Moving test.ini into a subdir..." -mkdir subdir -mv test.ini subdir - -echo "travis-build.bash is done." diff --git a/bin/travis-run.sh b/bin/travis-run.sh deleted file mode 100644 index f5aa2131..00000000 --- a/bin/travis-run.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh -e - -echo "NO_START=0\nJETTY_HOST=127.0.0.1\nJETTY_PORT=8983\nJAVA_HOME=$JAVA_HOME" | sudo tee /etc/default/jetty -sudo cp ckan/ckan/config/solr/schema.xml /etc/solr/conf/schema.xml -sudo service jetty restart - -nosetests --ckan \ - --nologcapture \ - --with-pylons=subdir/test.ini \ - --with-coverage \ - --cover-package=ckanext.validation \ - --cover-inclusive \ - --cover-erase \ - --cover-tests diff --git a/ckanext/validation/blueprints.py b/ckanext/validation/blueprints.py new file mode 100644 index 00000000..3ec0dc34 --- /dev/null +++ b/ckanext/validation/blueprints.py @@ -0,0 +1,46 @@ +# encoding: utf-8 + +from flask import Blueprint + +from ckantoolkit import c, NotAuthorized, ObjectNotFound, abort, _, render, get_action + +validation = Blueprint("validation", __name__) + + +def read(id, resource_id): + + try: + validation = get_action(u"resource_validation_show")( + {u"user": c.user}, {u"resource_id": resource_id} + ) + + resource = get_action(u"resource_show")({u"user": c.user}, {u"id": resource_id}) + + dataset = get_action(u"package_show")( + {u"user": c.user}, {u"id": resource[u"package_id"]} + ) + + # Needed for core resource templates + c.package = c.pkg_dict = dataset + c.resource = resource + + return render( + u"validation/validation_read.html", + extra_vars={ + u"validation": validation, + u"resource": resource, + u"dataset": dataset, + u"pkg_dict": dataset, + }, + ) + + except NotAuthorized: + abort(403, _(u"Unauthorized to read this validation report")) + except ObjectNotFound: + + abort(404, _(u"No validation report exists for this resource")) + + +validation.add_url_rule( + "/dataset//resource//validation", view_func=read +) diff --git a/ckanext/validation/cli.py b/ckanext/validation/cli.py new file mode 100644 index 00000000..3b8a0570 --- /dev/null +++ b/ckanext/validation/cli.py @@ -0,0 +1,22 @@ +import sys + +import click + +from ckanext.validation.model import create_tables, tables_exist + + +@click.group() +def validation(): + """Harvests remotely mastered metadata.""" + pass + + +@validation.command() +def init_db(): + """Creates the necessary tables in the database.""" + if tables_exist(): + print(u"Validation tables already exist") + sys.exit(0) + + create_tables() + print(u"Validation tables created") diff --git a/ckanext/validation/helpers.py b/ckanext/validation/helpers.py index ad5c57d8..fbedb352 100644 --- a/ckanext/validation/helpers.py +++ b/ckanext/validation/helpers.py @@ -2,7 +2,7 @@ import json from ckan.lib.helpers import url_for_static -from ckantoolkit import url_for, _, config, asbool, literal +from ckantoolkit import url_for, _, config, asbool, literal, h def get_validation_badge(resource, in_listing=False): @@ -89,3 +89,7 @@ def bootstrap_version(): return '3' else: return '2' + + +def use_webassets(): + return int(h.ckan_version().split('.')[1]) >= 9 diff --git a/ckanext/validation/jobs.py b/ckanext/validation/jobs.py index 8f7a608b..a02c3236 100644 --- a/ckanext/validation/jobs.py +++ b/ckanext/validation/jobs.py @@ -5,6 +5,7 @@ import json import re +import six import requests from sqlalchemy.orm.exc import NoResultFound from goodtables import validate @@ -45,7 +46,7 @@ def run_validation_job(resource): options = {} resource_options = resource.get(u'validation_options') - if resource_options and isinstance(resource_options, basestring): + if resource_options and isinstance(resource_options, six.string_types): resource_options = json.loads(resource_options) if resource_options: options.update(resource_options) @@ -77,7 +78,7 @@ def run_validation_job(resource): source = resource[u'url'] schema = resource.get(u'schema') - if schema and isinstance(schema, basestring): + if schema and isinstance(schema, six.string_types): if schema.startswith('http'): r = requests.get(schema) schema = r.json() diff --git a/ckanext/validation/logic.py b/ckanext/validation/logic.py index 0da6b684..bc6db3ee 100644 --- a/ckanext/validation/logic.py +++ b/ckanext/validation/logic.py @@ -4,6 +4,7 @@ import logging import json +import six from sqlalchemy.orm.exc import NoResultFound import ckan.plugins as plugins @@ -262,14 +263,14 @@ def resource_validation_run_batch(context, data_dict): count_resources = 0 dataset_ids = data_dict.get('dataset_ids') - if isinstance(dataset_ids, basestring): + if isinstance(dataset_ids, six.string_types): try: dataset_ids = json.loads(dataset_ids) except ValueError as e: dataset_ids = [dataset_ids] search_params = data_dict.get('query') - if isinstance(search_params, basestring): + if isinstance(search_params, six.string_types): try: search_params = json.loads(search_params) except ValueError as e: @@ -467,7 +468,7 @@ def resource_create(context, data_dict): context['use_cache'] = False t.get_action('package_update')(context, pkg_dict) context.pop('defer_commit') - except t.ValidationError, e: + except t.ValidationError as e: try: raise t.ValidationError(e.error_dict['resources'][-1]) except (KeyError, IndexError): @@ -480,7 +481,6 @@ def resource_create(context, data_dict): uploader.get_max_resource_size()) # Custom code starts - if get_create_mode_from_config() == u'sync': run_validation = True @@ -588,7 +588,7 @@ def resource_update(context, data_dict): context['use_cache'] = False updated_pkg_dict = t.get_action('package_update')(context, pkg_dict) context.pop('defer_commit') - except t.ValidationError, e: + except t.ValidationError as e: try: raise t.ValidationError(e.error_dict['resources'][-1]) except (KeyError, IndexError): diff --git a/ckanext/validation/model.py b/ckanext/validation/model.py index 35eebc46..7f1448b4 100644 --- a/ckanext/validation/model.py +++ b/ckanext/validation/model.py @@ -4,6 +4,7 @@ import uuid import logging +import six from sqlalchemy import Column, Unicode, DateTime from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.dialects.postgresql import JSON @@ -14,7 +15,7 @@ def make_uuid(): - return unicode(uuid.uuid4()) + return six.text_type(uuid.uuid4()) Base = declarative_base(metadata=metadata) diff --git a/ckanext/validation/plugin.py b/ckanext/validation/plugin/__init__.py similarity index 90% rename from ckanext/validation/plugin.py rename to ckanext/validation/plugin/__init__.py index a2f6643a..bbca635e 100644 --- a/ckanext/validation/plugin.py +++ b/ckanext/validation/plugin/__init__.py @@ -4,6 +4,8 @@ import cgi import json +import six +from werkzeug.datastructures import FileStorage as FlaskFileStorage import ckan.plugins as p import ckantoolkit as t @@ -22,6 +24,7 @@ validation_extract_report_from_errors, dump_json_value, bootstrap_version, + use_webassets, ) from ckanext.validation.validators import ( resource_schema_validator, @@ -33,14 +36,19 @@ ) from ckanext.validation.interfaces import IDataValidation +if t.check_ckan_version(min_version="2.9"): + from ckanext.validation.plugin.flask_plugin import ValidationMixin +else: + from ckanext.validation.plugin.pylons_plugin import ValidationMixin + +ALLOWED_UPLOAD_TYPES = (cgi.FieldStorage, FlaskFileStorage) log = logging.getLogger(__name__) -class ValidationPlugin(p.SingletonPlugin): +class ValidationPlugin(ValidationMixin): p.implements(p.IConfigurer) p.implements(p.IActions) - p.implements(p.IRoutes, inherit=True) p.implements(p.IAuthFunctions) p.implements(p.IResourceController, inherit=True) p.implements(p.IPackageController, inherit=True) @@ -59,22 +67,9 @@ def update_config(self, config_): else: log.debug(u'Validation tables exist') - t.add_template_directory(config_, u'templates') - t.add_public_directory(config_, u'public') - t.add_resource(u'fanstatic', 'ckanext-validation') - - # IRoutes - - def before_map(self, map_): - - controller = u'ckanext.validation.controller:ValidationController' - - map_.connect( - u'validation_read', - u'/dataset/{id}/resource/{resource_id}/validation', - controller=controller, action=u'validation') - - return map_ + t.add_template_directory(config_, u'../templates') + t.add_public_directory(config_, u'../public') + t.add_resource(u'../webassets', 'ckanext-validation') # IActions @@ -111,6 +106,7 @@ def get_helpers(self): u'validation_extract_report_from_errors': validation_extract_report_from_errors, u'dump_json_value': dump_json_value, u'bootstrap_version': bootstrap_version, + u'use_webassets': use_webassets, } # IResourceController @@ -132,11 +128,12 @@ def _process_schema_fields(self, data_dict): schema_upload = data_dict.pop(u'schema_upload', None) schema_url = data_dict.pop(u'schema_url', None) schema_json = data_dict.pop(u'schema_json', None) - - if isinstance(schema_upload, cgi.FieldStorage): - data_dict[u'schema'] = schema_upload.file.read() + if isinstance(schema_upload, ALLOWED_UPLOAD_TYPES): + uploaded_file = _get_underlying_file(schema_upload) + data_dict[u'schema'] = uploaded_file.read() elif schema_url: - if (not isinstance(schema_url, basestring) or + + if (not isinstance(schema_url, six.string_types) or not schema_url.lower()[:4] == u'http'): raise t.ValidationError({u'schema_url': 'Must be a valid URL'}) data_dict[u'schema'] = schema_url @@ -304,3 +301,9 @@ def _run_async_validation(resource_id): log.warning( u'Could not run validation for resource {}: {}'.format( resource_id, str(e))) + +def _get_underlying_file(wrapper): + if isinstance(wrapper, FlaskFileStorage): + return wrapper.stream + return wrapper.file + diff --git a/ckanext/validation/plugin/flask_plugin.py b/ckanext/validation/plugin/flask_plugin.py new file mode 100644 index 00000000..6e576a15 --- /dev/null +++ b/ckanext/validation/plugin/flask_plugin.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- + +from ckan import plugins as p + +from ckanext.validation import blueprints, cli + + +class ValidationMixin(p.SingletonPlugin): + p.implements(p.IBlueprint) + p.implements(p.IClick) + + # IBlueprint + + def get_blueprint(self): + return [blueprints.validation] + + # IClick + + def get_commands(self): + return [cli.validation] diff --git a/ckanext/validation/plugin/pylons_plugin.py b/ckanext/validation/plugin/pylons_plugin.py new file mode 100644 index 00000000..2f538d76 --- /dev/null +++ b/ckanext/validation/plugin/pylons_plugin.py @@ -0,0 +1,18 @@ +from ckan import plugins as p + + +class ValidationMixin(p.SingletonPlugin): + p.implements(p.IRoutes, inherit=True) + + # IRoutes + + def before_map(self, map_): + + controller = u'ckanext.validation.controller:ValidationController' + + map_.connect( + u'validation_read', + u'/dataset/{id}/resource/{resource_id}/validation', + controller=controller, action=u'validation') + + return map_ diff --git a/ckanext/validation/templates/package/resource_read.html b/ckanext/validation/templates/package/resource_read.html index 7d11e120..1aebef1a 100644 --- a/ckanext/validation/templates/package/resource_read.html +++ b/ckanext/validation/templates/package/resource_read.html @@ -6,6 +6,11 @@

{{ h.resource_display_name(res) | truncate(50) }} {{ h.get_validation_badge(res)|safe }}

- {% resource 'ckanext-validation/main' %} + {% if h.use_webassets() %} + {% snippet 'validation/snippets/validation_asset.html', name='ckanext-validation/validation-css' %} + {% else %} + {% snippet 'validation/snippets/validation_resource.html', name='ckanext-validation/main' %} + {% endif %} + {% endblock %} diff --git a/ckanext/validation/templates/package/snippets/resource_item.html b/ckanext/validation/templates/package/snippets/resource_item.html index 4d31fd38..f0913960 100644 --- a/ckanext/validation/templates/package/snippets/resource_item.html +++ b/ckanext/validation/templates/package/snippets/resource_item.html @@ -4,8 +4,12 @@ {{ super() }} {{ h.get_validation_badge(res, in_listing=True)|safe }} + {% if h.use_webassets() %} + {% snippet 'validation/snippets/validation_asset.html', name='ckanext-validation/validation-css' %} + {% else %} + {% snippet 'validation/snippets/validation_resource.html', name='ckanext-validation/main' %} + {% endif %} -{% resource 'ckanext-validation/main' %} {% endblock %} diff --git a/ckanext/validation/templates/scheming/form_snippets/resource_schema.html b/ckanext/validation/templates/scheming/form_snippets/resource_schema.html index 08815301..55a92146 100644 --- a/ckanext/validation/templates/scheming/form_snippets/resource_schema.html +++ b/ckanext/validation/templates/scheming/form_snippets/resource_schema.html @@ -60,7 +60,10 @@ {% set existing_value = h.scheming_display_json_value(value, indent=None) if is_json else value %} - {% resource 'ckanext-validation/resource-schema-form' %} + {% if h.use_webassets() %} + {% snippet 'validation/snippets/validation_asset.html', name='ckanext-validation/resource-schema-form' %} + {% else %} + {% snippet 'validation/snippets/validation_resource.html', name='ckanext-validation/resource-schema-form' %} + {% endif %} - diff --git a/ckanext/validation/templates/validation/snippets/validation_asset.html b/ckanext/validation/templates/validation/snippets/validation_asset.html new file mode 100644 index 00000000..211c76b4 --- /dev/null +++ b/ckanext/validation/templates/validation/snippets/validation_asset.html @@ -0,0 +1 @@ +{% asset name %} diff --git a/ckanext/validation/templates/validation/snippets/validation_report_dialog.html b/ckanext/validation/templates/validation/snippets/validation_report_dialog.html index 095973e7..812b04ab 100644 --- a/ckanext/validation/templates/validation/snippets/validation_report_dialog.html +++ b/ckanext/validation/templates/validation/snippets/validation_report_dialog.html @@ -14,5 +14,9 @@

-{% resource 'ckanext-validation/report-form' %} - +{% if h.use_webassets() %} + {% snippet 'validation/snippets/validation_asset.html', name='ckanext-validation/report-form-css' %} + {% snippet 'validation/snippets/validation_asset.html', name='ckanext-validation/report-form-js' %} +{% else %} + {% snippet 'validation/snippets/validation_resource.html', name='ckanext-validation/report-form' %} +{% endif %} diff --git a/ckanext/validation/templates/validation/snippets/validation_report_dialog_bs2.html b/ckanext/validation/templates/validation/snippets/validation_report_dialog_bs2.html index b5c2de4f..1d1ac758 100644 --- a/ckanext/validation/templates/validation/snippets/validation_report_dialog_bs2.html +++ b/ckanext/validation/templates/validation/snippets/validation_report_dialog_bs2.html @@ -7,6 +7,9 @@

{{ _('Data Validation Report') }}

- -{% resource 'ckanext-validation/report-form' %} - +{% if h.use_webassets() %} + {% snippet 'validation/snippets/validation_asset.html', name='ckanext-validation/report-form-css' %} + {% snippet 'validation/snippets/validation_asset.html', name='ckanext-validation/report-form-js' %} +{% else %} + {% snippet 'validation/snippets/validation_resource.html', name='ckanext-validation/report-form' %} +{% endif %} diff --git a/ckanext/validation/templates/validation/snippets/validation_resource.html b/ckanext/validation/templates/validation/snippets/validation_resource.html new file mode 100644 index 00000000..252e6ca8 --- /dev/null +++ b/ckanext/validation/templates/validation/snippets/validation_resource.html @@ -0,0 +1 @@ +{% resource name %} diff --git a/ckanext/validation/templates/validation/validation_read.html b/ckanext/validation/templates/validation/validation_read.html index d845c64c..227ba9d3 100644 --- a/ckanext/validation/templates/validation/validation_read.html +++ b/ckanext/validation/templates/validation/validation_read.html @@ -7,7 +7,7 @@ {% block breadcrumb_content %} {{ super() }} -
  • {{ h.resource_display_name(resource)|truncate(30) }}
    • +
  • {{ h.resource_display_name(resource)|truncate(30) }}
  • Validation Report
    • {% endblock %} @@ -34,11 +34,14 @@

    {{ h.resource_display_name(resource) | truncate(50) }}
    {% endif %} - - - {% resource 'ckanext-validation/report' %} +{% if h.use_webassets() %} + {% snippet 'validation/snippets/validation_asset.html', name='ckanext-validation/report-css' %} + {% snippet 'validation/snippets/validation_asset.html', name='ckanext-validation/report-js' %} +{% else %} + {% snippet 'validation/snippets/validation_resource.html', name='ckanext-validation/report' %} +{% endif %} {% endblock %} diff --git a/ckanext/validation/tests/fixtures.py b/ckanext/validation/tests/fixtures.py new file mode 100644 index 00000000..965056e0 --- /dev/null +++ b/ckanext/validation/tests/fixtures.py @@ -0,0 +1,9 @@ +import pytest + +from ckanext.validation.model import create_tables, tables_exist + + +@pytest.fixture +def validation_setup(): + if not tables_exist(): + create_tables() diff --git a/ckanext/validation/tests/helpers.py b/ckanext/validation/tests/helpers.py index 4ea04d6f..44d251a1 100644 --- a/ckanext/validation/tests/helpers.py +++ b/ckanext/validation/tests/helpers.py @@ -1,11 +1,14 @@ -import __builtin__ as builtins +from six.moves import builtins import cgi import functools import mock +import six +from six import StringIO, BytesIO from pyfakefs import fake_filesystem import ckan.lib.uploader +from ckan.plugins import toolkit from ckan.tests.helpers import change_config @@ -140,11 +143,33 @@ def wrapper(*args, **kwargs): return wrapper -class MockFieldStorage(cgi.FieldStorage): +if toolkit.check_ckan_version(min_version="2.9"): - def __init__(self, fp, filename): + from werkzeug.datastructures import FileStorage - self.file = fp - self.filename = filename - self.name = 'upload' - self.list = None + class MockFieldStorage(FileStorage): + pass +else: + + class MockFieldStorage(cgi.FieldStorage): + + def __init__(self, fp, filename): + + self.file = fp + self.filename = filename + self.name = 'upload' + self.list = None + + def __bool__(self): + return self.file is not None + + +def get_mock_file(contents): + if six.PY3: + mock_file = BytesIO() + mock_file.write(contents.encode('utf8')) + else: + mock_file = StringIO() + mock_file.write(contents) + + return mock_file diff --git a/ckanext/validation/tests/test_form.py b/ckanext/validation/tests/test_form.py index 29fc7b19..34d54ba0 100644 --- a/ckanext/validation/tests/test_form.py +++ b/ckanext/validation/tests/test_form.py @@ -3,25 +3,58 @@ import mock import datetime -from nose.tools import assert_in, assert_equals +import pytest +import six +import ckantoolkit as t from ckantoolkit.tests.factories import Sysadmin, Dataset from ckantoolkit.tests.helpers import ( - FunctionalTestBase, submit_and_follow, webtest_submit, call_action, - reset_db + call_action, _get_test_app ) -from ckanext.validation.model import create_tables, tables_exist -from ckanext.validation.tests.helpers import ( - VALID_CSV, INVALID_CSV, mock_uploads -) +from ckanext.validation.tests.helpers import VALID_CSV, INVALID_CSV, mock_uploads + +is_ckan29_or_higher = t.check_ckan_version(min_version="2.9") + + +def _post(app, url, extra_environ=None, data=None, upload=None): + ''' Submit a POST request to 'app', + using either webtest or Flask syntax. + ''' + if is_ckan29_or_higher: + if upload: + for entry in upload: + data[entry[0]] = (io.BytesIO(entry[2]), entry[1]) + app.post( + url=url, extra_environ=extra_environ, data=data) + else: + app.post( + url, data, extra_environ=extra_environ, upload_files=upload) + + +def _new_resource_url(dataset_id): + + if is_ckan29_or_higher: + url = "/dataset/{}/resource/new".format(dataset_id) + else: + url = "/dataset/new_resource/{}".format(dataset_id) + return url + + +def _edit_resource_url(dataset_id, resource_id): + + if is_ckan29_or_higher: + url = "/dataset/{}/resource/{}/edit".format(dataset_id, resource_id) + else: + url = "/dataset/{}/resource_edit/{}".format(dataset_id, resource_id) + return url def _get_resource_new_page_as_sysadmin(app, id): user = Sysadmin() - env = {'REMOTE_USER': user['name'].encode('ascii')} + env = {"REMOTE_USER": user["name"].encode("ascii")} response = app.get( - url='/dataset/new_resource/{}'.format(id), + url="/dataset/new_resource/{}".format(id), extra_environ=env, ) return env, response @@ -29,516 +62,499 @@ def _get_resource_new_page_as_sysadmin(app, id): def _get_resource_update_page_as_sysadmin(app, id, resource_id): user = Sysadmin() - env = {'REMOTE_USER': user['name'].encode('ascii')} + env = {"REMOTE_USER": user["name"].encode("ascii")} response = app.get( - url='/dataset/{}/resource_edit/{}'.format(id, resource_id), + url="/dataset/{}/resource_edit/{}".format(id, resource_id), extra_environ=env, ) return env, response -class TestResourceSchemaForm(FunctionalTestBase): - - def setup(self): - reset_db() - if not tables_exist(): - create_tables() - - def test_resource_form_includes_json_fields(self): +@pytest.mark.usefixtures("clean_db", "validation_setup", "with_plugins") +class TestResourceSchemaForm(object): + def test_resource_form_includes_json_fields(self, app): dataset = Dataset() + env, response = _get_resource_new_page_as_sysadmin(app, dataset["id"]) + assert '=1.12.0 ckantoolkit>=0.0.3 goodtables==1.5.1 -e git+https://github.com/ckan/ckanext-scheming.git#egg=ckanext-scheming diff --git a/setup.py b/setup.py index 7a729246..0df348f5 100644 --- a/setup.py +++ b/setup.py @@ -38,6 +38,9 @@ # Specify the Python versions you support here. In particular, ensure # that you indicate whether you support Python 2, Python 3 or both. 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ],