CKAN 2.9 / Python 3

.github/workflows/test.yml

@@ -0,0 +1,76 @@
+name: Tests
+on: [push, pull_request]
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.6'
+      - name: Install requirements
+        run: pip install flake8 pycodestyle
+      - name: Check syntax
+        run: flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan
+
+  test:
+    needs: lint
+    strategy:
+      matrix:
+        ckan-version: [2.9, 2.9-py2, 2.8]
+      fail-fast: false
+
+    name: CKAN ${{ matrix.ckan-version }}
+    runs-on: ubuntu-latest
+    container:
+      image: openknowledge/ckan-dev:${{ matrix.ckan-version }}
+    services:
+      solr:
+        image: ckan/ckan-solr-dev:${{ matrix.ckan-version }}
+      postgres:
+        image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }}
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+      redis:
+          image: redis:3
+    env:
+      CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test
+      CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test
+      CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test
+      CKAN_SOLR_URL: http://solr:8983/solr/ckan
+      CKAN_REDIS_URL: redis://redis:6379/1
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install requirements (py3)
+      if: ${{ matrix.ckan-version != '2.7' && matrix.ckan-version != '2.8' && matrix.ckan-version != '2.9-py2' }}
+      run: |
+        pip install -r dev-requirements.txt
+    - name: Install requirements (py2)
+      if: ${{ matrix.ckan-version == '2.7' || matrix.ckan-version == '2.8' || matrix.ckan-version == '2.9-py2' }}
+      run: |
+        pip install -r dev-requirements-py2.txt
+    - name: Install requirements (common)
+      run: |
+        pip install -r requirements.txt
+        pip install -e .
+        # Replace default path to CKAN core config file with the one on the container
+        sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini
+    - name: Setup extension (CKAN >= 2.9)
+      if: ${{ matrix.ckan-version != '2.7' && matrix.ckan-version != '2.8' }}
+      run: |
+        ckan -c test.ini db init
+    - name: Setup extension (CKAN < 2.9)
+      if: ${{ matrix.ckan-version == '2.7' || matrix.ckan-version == '2.8' }}
+      run: |
+        paster --plugin=ckan db init -c test.ini
+    - name: Run tests
+      run: pytest --ckan-ini=test.ini --cov=ckanext.validation --cov-report=xml --cov-append --disable-warnings ckanext/validation/tests -vv
+
+    - name: Upload coverage report to codecov
+      uses: codecov/codecov-action@v1
+      with:
+        file: ./coverage.xml

README.md

@@ -50,12 +50,10 @@ If you are eager to get started, jump to the [Installation](#installation) and [
 
 ## Versions supported and requirements
 
-This extension has been tested with CKAN 2.4 to 2.7.
+This extension is currently tested in CKAN 2.8 and CKAN 2.9.
 
 It is strongly recommended to use it alongside [ckanext-scheming](https://github.com/ckan/ckanext-scheming) to define the necessary extra fields in the default CKAN schema.
 
-If you want to use [asynchronous validation](#asynchronous-validation) with background jobs and are using CKAN 2.6 or lower, [ckanext-rq](https://github.com/ckan/ckanext-rq) is also needed. Please refer to both READMEs for installation instructions.
-
 
 ## Installation
 

ckanext/validation/blueprints.py

@@ -0,0 +1,46 @@
+# encoding: utf-8
+
+from flask import Blueprint
+
+from ckantoolkit import c, NotAuthorized, ObjectNotFound, abort, _, render, get_action
+
+validation = Blueprint("validation", __name__)
+
+
+def read(id, resource_id):
+
+    try:
+        validation = get_action(u"resource_validation_show")(
+            {u"user": c.user}, {u"resource_id": resource_id}
+        )
+
+        resource = get_action(u"resource_show")({u"user": c.user}, {u"id": resource_id})
+
+        dataset = get_action(u"package_show")(
+            {u"user": c.user}, {u"id": resource[u"package_id"]}
+        )
+
+        # Needed for core resource templates
+        c.package = c.pkg_dict = dataset
+        c.resource = resource
+
+        return render(
+            u"validation/validation_read.html",
+            extra_vars={
+                u"validation": validation,
+                u"resource": resource,
+                u"dataset": dataset,
+                u"pkg_dict": dataset,
+            },
+        )
+
+    except NotAuthorized:
+        abort(403, _(u"Unauthorized to read this validation report"))
+    except ObjectNotFound:
+
+        abort(404, _(u"No validation report exists for this resource"))
+
+
+validation.add_url_rule(
+    "/dataset/<id>/resource/<resource_id>/validation", view_func=read
+)

ckanext/validation/cli.py

@@ -0,0 +1,22 @@
+import sys
+
+import click
+
+from ckanext.validation.model import create_tables, tables_exist
+
+
+@click.group()
+def validation():
+    """Harvests remotely mastered metadata."""
+    pass
+
+
+@validation.command()
+def init_db():
+    """Creates the necessary tables in the database."""
+    if tables_exist():
+        print(u"Validation tables already exist")
+        sys.exit(0)
+
+    create_tables()
+    print(u"Validation tables created")

ckanext/validation/helpers.py

@@ -2,7 +2,7 @@
 import json
 
 from ckan.lib.helpers import url_for_static
-from ckantoolkit import url_for, _, config, asbool, literal
+from ckantoolkit import url_for, _, config, asbool, literal, h
 
 
 def get_validation_badge(resource, in_listing=False):
@@ -89,3 +89,7 @@ def bootstrap_version():
         return '3'
     else:
         return '2'
+
+
+def use_webassets():
+    return int(h.ckan_version().split('.')[1]) >= 9

ckanext/validation/jobs.py

@@ -5,6 +5,7 @@
 import json
 import re
 
+import six
 import requests
 from sqlalchemy.orm.exc import NoResultFound
 from goodtables import validate
@@ -45,7 +46,7 @@ def run_validation_job(resource):
         options = {}
 
     resource_options = resource.get(u'validation_options')
-    if resource_options and isinstance(resource_options, basestring):
+    if resource_options and isinstance(resource_options, six.string_types):
         resource_options = json.loads(resource_options)
     if resource_options:
         options.update(resource_options)
@@ -77,7 +78,7 @@ def run_validation_job(resource):
         source = resource[u'url']
 
     schema = resource.get(u'schema')
-    if schema and isinstance(schema, basestring):
+    if schema and isinstance(schema, six.string_types):
         if schema.startswith('http'):
             r = requests.get(schema)
             schema = r.json()

ckanext/validation/logic.py

@@ -4,6 +4,7 @@
 import logging
 import json
 
+import six
 from sqlalchemy.orm.exc import NoResultFound
 
 import ckan.plugins as plugins
@@ -262,14 +263,14 @@ def resource_validation_run_batch(context, data_dict):
     count_resources = 0
 
     dataset_ids = data_dict.get('dataset_ids')
-    if isinstance(dataset_ids, basestring):
+    if isinstance(dataset_ids, six.string_types):
         try:
             dataset_ids = json.loads(dataset_ids)
         except ValueError as e:
             dataset_ids = [dataset_ids]
 
     search_params = data_dict.get('query')
-    if isinstance(search_params, basestring):
+    if isinstance(search_params, six.string_types):
         try:
             search_params = json.loads(search_params)
         except ValueError as e:
@@ -467,7 +468,7 @@ def resource_create(context, data_dict):
         context['use_cache'] = False
         t.get_action('package_update')(context, pkg_dict)
         context.pop('defer_commit')
-    except t.ValidationError, e:
+    except t.ValidationError as e:
         try:
             raise t.ValidationError(e.error_dict['resources'][-1])
         except (KeyError, IndexError):
@@ -480,7 +481,6 @@ def resource_create(context, data_dict):
                   uploader.get_max_resource_size())
 
     # Custom code starts
-
     if get_create_mode_from_config() == u'sync':
 
         run_validation = True
@@ -588,7 +588,7 @@ def resource_update(context, data_dict):
         context['use_cache'] = False
         updated_pkg_dict = t.get_action('package_update')(context, pkg_dict)
         context.pop('defer_commit')
-    except t.ValidationError, e:
+    except t.ValidationError as e:
         try:
             raise t.ValidationError(e.error_dict['resources'][-1])
         except (KeyError, IndexError):

ckanext/validation/model.py

@@ -4,6 +4,7 @@
 import uuid
 import logging
 
+import six
 from sqlalchemy import Column, Unicode, DateTime
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.dialects.postgresql import JSON
@@ -14,7 +15,7 @@
 
 
 def make_uuid():
-    return unicode(uuid.uuid4())
+    return six.text_type(uuid.uuid4())
 
 
 Base = declarative_base(metadata=metadata)