Add support for CKAN 2.10

.github/workflows/test.yml

@@ -0,0 +1,79 @@
+name: Tests
+on: [push, pull_request]
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.6'
+      - name: Install requirements
+        run: pip install flake8 pycodestyle
+      - name: Check syntax
+        run: flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan
+
+  test:
+    needs: lint
+    strategy:
+      matrix:
+        ckan-version: ["2.10", 2.9, 2.9-py2, 2.8, 2.7]
+      fail-fast: false
+
+    name: CKAN ${{ matrix.ckan-version }}
+    runs-on: ubuntu-latest
+    container:
+      image: openknowledge/ckan-dev:${{ matrix.ckan-version }}
+    services:
+      solr:
+        image: ckan/ckan-solr:${{ matrix.ckan-version }}
+      postgres:
+        image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }}
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+      redis:
+          image: redis:3
+    env:
+      CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test
+      CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test
+      CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test
+      CKAN_SOLR_URL: http://solr:8983/solr/ckan
+      CKAN_REDIS_URL: redis://redis:6379/1
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Install py3 requirements
+      if: ${{ matrix.ckan-version == '2.10' || matrix.ckan-version == '2.9' }}
+      run: pip install -r requirements.txt
+    - name: Install py2 requirements  
+      if: ${{ matrix.ckan-version == '2.7' || matrix.ckan-version == '2.8' || matrix.ckan-version == '2.9-py2' }}
+      run: pip install -r requirements-py2.txt
+    - name: Install requirements
+      run: |
+        pip install -r dev-requirements.txt
+        pip install -e .
+        # Replace default path to CKAN core config file with the one on the container
+        sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini
+    - name: Setup extension (CKAN >= 2.9)
+      if: ${{ matrix.ckan-version != '2.7' && matrix.ckan-version != '2.8' }}
+      run: |
+        ckan -c test.ini db init
+        ckan -c test.ini archiver init
+        ckan -c test.ini report initdb
+        ckan -c test.ini qa init
+    - name: Setup extension (CKAN < 2.9)
+      if: ${{ matrix.ckan-version == '2.7' || matrix.ckan-version == '2.8' }}
+      run: |
+        paster --plugin=ckan db init -c test.ini
+        paster --plugin=ckanext-archiver archiver init --config=test.ini
+        paster --plugin=ckanext-report report initdb --config=test.ini
+        paster --plugin=ckanext-qa qa init --config=test.ini
+    - name: Run tests
+      run: pytest --ckan-ini=test.ini --cov=ckanext.qa --cov-report=xml --cov-append --disable-warnings ckanext/qa/tests
+    - name: Upload coverage report to codecov
+      uses: codecov/codecov-action@v1
+      with:
+        file: ./coverage.xml

CHANGELOG.md

@@ -0,0 +1,7 @@
+# ckanext-qa versions
+
+## 3.0 (2022-04-15)
+
+New features:
+ - Add Python 3 compatibility
+ - Remove Travis an start using GitHub Actions

README.rst

@@ -2,8 +2,8 @@
    these badges work. The necessary Travis and Coverage config files have been
    generated for you.
 
-.. image:: https://travis-ci.org/ckan/ckanext-qa.svg?branch=master
-    :target: https://travis-ci.org/ckan/ckanext-qa
+[![Tests](https://github.com/ckan/ckanext-qa/workflows/Tests/badge.svg?branch=master)](https://github.com/ckan/ckanext-qa/actions)
+[![Code Coverage](http://codecov.io/github/ckan/ckanext-qa/coverage.svg?branch=master)](http://codecov.io/github/ckan/ckanext-qa?branch=master)
 
 CKAN QA Extension (Quality Assurance)
 =====================================
@@ -31,7 +31,7 @@ Requirements
 
 Before installing ckanext-qa, make sure that you have installed the following:
 
-* CKAN 2.1+
+* CKAN 2.1+ (tests are only running for CKAN 2.7+)
 * ckanext-archiver 2.0+ (https://github.com/ckan/ckanext-archiver)
 * ckanext-report (https://github.com/datagovuk/ckanext-report) for reporting
 

ckanext/qa/__init__.py

@@ -6,4 +6,4 @@
     import pkgutil
     __path__ = pkgutil.extend_path(__path__, __name__)
 
-__version__ = '2.0'
+__version__ = '3.0'

ckanext/qa/bin/common.py

@@ -1,3 +1,4 @@
+from __future__ import print_function
 import os
 
 
@@ -48,5 +49,5 @@ def get_resources(state='active', publisher_ref=None, resource_id=None, dataset_
         resources = resources.filter(model.Resource.id == resource_id)
         criteria.append('Resource:%s' % resource_id)
     resources = resources.all()
-    print '%i resources (%s)' % (len(resources), ' '.join(criteria))
+    print('%i resources (%s)' % (len(resources), ' '.join(criteria)))
     return resources

ckanext/qa/bin/migrate_task_status.py

@@ -5,12 +5,11 @@
 information - resources that are no longer available but had the format
 detected in the past.
 '''
-
+from __future__ import print_function
 from optparse import OptionParser
 import logging
 import json
 import datetime
-
 import common
 from running_stats import StatsList
 
@@ -59,7 +58,7 @@ def migrate(options):
         # time, so some timezone nonesense going on. Can't do much.
         archival = Archival.get_for_resource(res.id)
         if not archival:
-            print add_stat('QA but no Archival data', res, stats)
+            print(add_stat('QA but no Archival data', res, stats))
             continue
         archival_date = archival.updated
         # the state of the resource was as it was archived on the date of
@@ -112,10 +111,10 @@ def migrate(options):
                 model.Session.add(qa)
             add_stat('Added to QA table', res, stats)
 
-    print 'Summary\n', stats.report()
+    print('Summary\n', stats.report())
     if options.write:
         model.repo.commit_and_remove()
-        print 'Written'
+        print('Written')
 
 
 def add_stat(outcome, res, stats, extra_info=None):
@@ -154,10 +153,10 @@ def date_str_to_datetime_or_none(date_str):
     if len(args) != 1:
         parser.error('Wrong number of arguments (%i)' % len(args))
     config_ini = args[0]
-    print 'Loading CKAN config...'
+    print('Loading CKAN config...')
     common.load_config(config_ini)
     common.register_translator()
-    print 'Done'
+    print('Done')
     # Setup logging to print debug out for local only
     rootLogger = logging.getLogger()
     rootLogger.setLevel(logging.WARNING)

ckanext/qa/bin/running_stats.py

@@ -14,7 +14,7 @@
         package_stats.increment('deleted')
     else:
         package_stats.increment('not deleted')
-print package_stats.report()
+print(package_stats.report())
 > deleted: 30
 > not deleted: 70
 
@@ -26,14 +26,15 @@
         package_stats.add('deleted', package.name)
     else:
         package_stats.add('not deleted' package.name)
-print package_stats.report()
+print(package_stats.report())
 > deleted: 30 pollution-uk, flood-regions, river-quality, ...
 > not deleted: 70 spending-bristol, ...
 
 '''
-
+from __future__ import print_function
 import copy
 import datetime
+import six
 
 
 class StatsCount(dict):
@@ -68,9 +69,9 @@ def report(self, indent=1, order_by_title=False, show_time_taken=True):
             report_dict[category] = self.report_value(category)
 
         if order_by_title:
-            items = sorted(report_dict.iteritems())
+            items = sorted(six.iteritems(report_dict))
         else:
-            items = sorted(report_dict.iteritems(),
+            items = sorted(six.iteritems(report_dict),
                            key=lambda x: -x[1][1])
 
         for category, value_tuple in items:
@@ -110,6 +111,6 @@ def report_value(self, category):
     package_stats.add('Success', 'good3')
     package_stats.add('Success', 'good4')
     package_stats.add('Failure', 'bad1')
-    print package_stats.report()
+    print(package_stats.report())
 
-    print StatsList().report()
+    print(StatsList().report())

ckanext/qa/cli.py

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+
+import click
+import logging
+import ckanext.qa.utils as utils
+
+
+log = logging.getLogger(__name__)
+
+
+def get_commands():
+    return [qa]
+
+
+@click.group()
+def qa():
+    pass
+
+
+@qa.command()
+def init():
+    """Creates necessary db tables"""
+    utils.init_db()
+
+
+@qa.command()
+@click.argument('ids', nargs=-1)
+@click.option('-q', '--queue', default=None)
+def update(ids, queue):
+    """Creates necessary db tables"""
+    log.info('QA update: ids:%s queue:%s' % (ids, queue))
+    if ids:
+        utils.update(*ids, queue=queue)
+    else:
+        utils.update(queue=queue)