Merge pull request #26 from qld-gov-au/develop

Develop to master

.ahoy.yml

@@ -109,6 +109,13 @@ commands:
       ahoy cli "flake8 ${@:-ckanext}" || \
       [ "${ALLOW_LINT_FAIL:-0}" -eq 1 ]
 
+  copy-local-files:
+    usage: Update files from local repo.
+    cmd: |
+      docker cp . $(docker-compose ps -q ckan):/app/
+      docker cp .docker/scripts/ckan_cli $(docker-compose ps -q ckan):/app/ckan/default/bin/
+      ahoy cli 'chmod u+x $VENV_DIR/bin/ckan_cli; cp .docker/test.ini $CKAN_INI'
+
   test-unit:
     usage: Run unit tests.
     cmd: |
@@ -120,9 +127,14 @@ commands:
     cmd: |
       ahoy start-ckan-job-workers
       ahoy start-mailmock &
+      if (echo "$CKAN_VERSION" | grep '^ckan-2.8'); then
+        BEHAVE_TAGS="--tags=-ckan29"
+      fi
       sleep 5 &&
       ahoy cli "behave -k ${*:-test/features}" --tags @smoke && \
-      ahoy cli "behave -k ${*:-test/features}" || \
+      ahoy cli "behave -k ${*:-test/features} --tags=-reporting $BEHAVE_TAGS" || \
+      [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ]
+      ahoy cli "behave -k ${*:-test/features} --tags=reporting" || \
       [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ]
       ahoy stop-mailmock
       ahoy stop-ckan-job-workers

.docker/Dockerfile.ckan

@@ -12,7 +12,7 @@ WORKDIR "${APP_DIR}"
 
 ENV DOCKERIZE_VERSION v0.6.1
 RUN apk add --no-cache curl build-base postgresql-client \
-    && curl -s -L -O https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
+    && curl -sLO https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
     && tar -C /usr/local/bin -xzvf dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
     && rm dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz
 
@@ -32,7 +32,6 @@ RUN . ${VENV_DIR}/bin/activate \
 
 COPY .docker/test.ini $CKAN_INI
 
-# Add current extension and files.
 COPY . ${APP_DIR}/
 
 COPY .docker/scripts ${APP_DIR}/scripts

.docker/scripts/ckan_cli

@@ -57,15 +57,15 @@ else
 fi
 
 if [ "$COMMAND" = "ckan" ]; then
-    echo "Using 'ckan' command from $ENV_DIR with config ${CKAN_INI}..." >&2
     # adjust args to match ckan expectations
     COMMAND=$(echo "$1" | sed -e 's/create-test-data/seed/')
+    echo "Using 'ckan' command from $ENV_DIR with config ${CKAN_INI} to run $COMMAND..." >&2
     shift
     exec $ENV_DIR/ckan -c ${CKAN_INI} $COMMAND "$@" $CLICK_ARGS
 elif [ "$COMMAND" = "paster" ]; then
-    echo "Using 'paster' command from $ENV_DIR with config ${CKAN_INI}..." >&2
     # adjust args to match paster expectations
     COMMAND=$1
+    echo "Using 'paster' command from $ENV_DIR with config ${CKAN_INI} to run $COMMAND..." >&2
     shift
     if [ "$1" = "show" ]; then shift; fi
     exec $ENV_DIR/paster --plugin=$PASTER_PLUGIN $COMMAND "$@" -c ${CKAN_INI}

.docker/scripts/create-test-data.sh

@@ -95,14 +95,22 @@ add_user_if_needed foodie "Foodie" foodie@localhost
 add_user_if_needed group_admin "Group Admin" group_admin@localhost
 add_user_if_needed walker "Walker" walker@localhost
 
-# Create test dataset with our standard fields
+# Create private test dataset with our standard fields
 curl -LsH "Authorization: ${API_KEY}" \
-    --data '{"name": "test-dataset", "owner_org": "'"${TEST_ORG_ID}"'",
+    --data '{"name": "test-dataset", "owner_org": "'"${TEST_ORG_ID}"'", "private": true,
 "update_frequency": "monthly", "author_email": "admin@localhost", "version": "1.0",
 "license_id": "other-open", "data_driven_application": "NO", "security_classification": "PUBLIC",
 "notes": "test", "de_identified_data": "NO"}' \
     ${CKAN_ACTION_URL}/package_create
 
+# Create public test dataset with our standard fields
+curl -LsH "Authorization: ${API_KEY}" \
+    --data '{"name": "public-test-dataset", "owner_org": "'"${TEST_ORG_ID}"'",
+"update_frequency": "monthly", "author_email": "[email protected]", "version": "1.0",
+"license_id": "other-open", "data_driven_application": "NO", "security_classification": "PUBLIC",
+"notes": "test", "de_identified_data": "NO"}' \
+    ${CKAN_ACTION_URL}/package_create
+
 # Datasets need to be assigned to an organisation
 
 echo "Assigning test Datasets to Organisation..."
@@ -208,6 +216,13 @@ curl -LsH "Authorization: ${API_KEY}" \
     --data "id=${DR_ORG_ID}&object=dr_member&object_type=user&capacity=member" \
     ${CKAN_ACTION_URL}/member_create
 
+echo "Creating test dataset for data request organisation:"
+
+curl -LsH "Authorization: ${API_KEY}" \
+    --data '{"name": "data_request_dataset", "description": "Dataset for data requests", "owner_org": "'"${DR_ORG_ID}"'",
+"update_frequency": "near-realtime", "author_email": "dr_admin@localhost", "version": "1.0", "license_id": "cc-by-4",
+"data_driven_application": "NO", "security_classification": "PUBLIC", "notes": "test", "de_identified_data": "NO"}'\
+    ${CKAN_ACTION_URL}/package_create
 
 echo "Creating test Data Request:"
 
@@ -269,12 +284,29 @@ curl -LsH "Authorization: ${API_KEY}" \
 # END.
 #
 
+##
+# BEGIN: Add sysadmin config values.
+# This needs to be done before closing datarequests as they require the below config values
+#
+echo "Adding ckan.datarequests.closing_circumstances:"
+
+curl -LsH "Authorization: ${API_KEY}" \
+    --data '{"ckan.comments.profanity_list": "", "ckan.datarequests.closing_circumstances":
+        "Released as open data|nominate_dataset\r\nOpen dataset already exists|nominate_dataset\r\nPartially released|nominate_dataset\r\nTo be released as open data at a later date|nominate_approximate_date\r\nData openly available elsewhere\r\nNot suitable for release as open data\r\nRequested data not available/cannot be compiled\r\nRequestor initiated closure"}' \
+    ${CKAN_ACTION_URL}/config_option_update
+
 echo "Creating config value for resource formats:"
 
 curl -LsH "Authorization: ${API_KEY}" \
     --data '{"ckanext.data_qld.resource_formats": "CSV\r\nHTML\r\nJSON\r\nRDF\r\nTXT\r\nXLS"}' \
     ${CKAN_ACTION_URL}/config_option_update
 
+echo "Creating config value for excluded display name words:"
+
+curl -LsH "Authorization: ${API_KEY}" \
+    --data '{"ckanext.data_qld.excluded_display_name_words": "gov"}' \
+    ${CKAN_ACTION_URL}/config_option_update
+
 if [ "$VENV_DIR" != "" ]; then
   deactivate
 fi

.docker/scripts/init-ext.sh

@@ -13,7 +13,6 @@ pip install -r "$VENV_DIR/src/ckanext-archiver/requirements.txt"
 pip install -r "$VENV_DIR/src/ckanext-dcat/requirements.txt"
 pip install -r "$VENV_DIR/src/ckanext-qa/requirements.txt"
 pip install -r "$VENV_DIR/src/ckanext-qgov/requirements.txt"
-pip install -r "$VENV_DIR/src/ckanext-scheming/requirements.txt"
 pip install -r "$VENV_DIR/src/ckanext-validation/requirements.txt"
 pip install -r "$VENV_DIR/src/ckanext-xloader/requirements.txt"
 pip install -r "$VENV_DIR/src/ckanext-ytp-comments/requirements.txt"

.docker/scripts/init.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env sh
 ##
-# Initialise CKAN instance.
+# Initialise CKAN data for testing.
 #
 set -e
 

.docker/test.ini

@@ -97,7 +97,7 @@ ckan.redis.url = redis://redis:6379
 #       Add ``datapusher`` to enable DataPusher
 #       Add ``resource_proxy`` to enable resource proxying and get around the
 #       same origin policy
-ckan.plugins = stats text_view image_view recline_view datastore data_qld data_qld_google_analytics datarequests scheming_datasets validation dcat qa archiver report ytp_comments xloader csrf_filter resource_type_validation harvest harvester_data_qld_geoscience qgovext
+ckan.plugins = stats text_view image_view recline_view datastore data_qld data_qld_google_analytics scheming_datasets validation dcat qa archiver report qgovext ytp_comments datarequests xloader csrf_filter resource_type_validation ssm_config odi_certificates harvest harvester_data_qld_geoscience
 
 # Define which views should be created by default
 # (plugins must be loaded in ckan.plugins)
@@ -192,19 +192,9 @@ ckanext.validation.formats = csv xlsx xls
 ##You can also provide validation options that will be used by default when running the validation:
 
 ## ckanext-scheming settings
-#   module-path:file to schemas being used
+# see https://github.com/ckan/ckanext-scheming#configuration
 scheming.dataset_schemas = ckanext.data_qld:ckan_dataset.json ckanext.harvester_data_qld_geoscience:geoscience_dataset.json
-#   will try to load "spatialx_schema.json" and "spatialxy_schema.json"
-#   as dataset schemas
-#
-#   URLs may also be used, e.g:
-#
-# scheming.dataset_schemas = http://example.com/spatialx_schema.json
-
-#   Preset files may be included as well. The default preset setting is:
 scheming.presets = ckanext.scheming:presets.json ckanext.data_qld:presets.json
-
-#   The is_fallback setting may be changed as well. Defaults to false:
 scheming.dataset_fallback = false
 
 ## ckanext-datarequests settings

.github/workflows/test.yml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ckan-version: [ckan-2.8.8-qgov.5, qgov-master-2.9.4]
+        ckan-version: [ckan-2.8.8-qgov.5, ckan-2.9.5-qgov.7]
 
     name: Continuous Integration build on CKAN ${{ matrix.ckan-version }}
     runs-on: ubuntu-latest
@@ -30,7 +30,7 @@ jobs:
 
       - name: Test
         run: .circleci/test.sh
-        timeout-minutes: 25
+        timeout-minutes: 30
 
       - name: Retrieve screenshots
         if: failure()

README.md

@@ -1,3 +1,3 @@
-# ckanext-data-qld-harvester
+# ckanext-harvester-data-qld-geoscience
 
 A CKAN extension to implement custom harvesting requirements for Data.Qld.

ckanext/harvester_data_qld_geoscience/plugin.py

@@ -1,11 +1,11 @@
 import datetime
 import logging
 import six
-import urllib
+from six.moves.urllib.parse import urlencode
 import ckan.plugins as plugins
 import ckantoolkit as toolkit
 
-import helpers
+from . import helpers
 
 from ckanext.harvest.harvesters.ckanharvester import CKANHarvester, ContentFetchError, SearchError
 from ckan.lib.helpers import json
@@ -298,7 +298,7 @@ def _search_for_datasets(self, harvest_job, remote_ckan_base_url, fq_terms=None)
         pkg_ids = set()
         previous_content = None
         while True:
-            url = base_search_url + '?' + urllib.urlencode(params)
+            url = base_search_url + '?' + urlencode(params)
             log.debug('Searching for CKAN datasets: %s', url)
             try:
                 content = self._get_content(url)

dev-requirements.txt

@@ -14,17 +14,19 @@ python-magic==0.4.18
 messytables==0.15.2
 progressbar==2.5
 
--e git+https://github.com/ckan/ckanext-dcat.git#egg=ckanext-dcat
+-e git+https://github.com/ckan/ckanext-dcat.git@v1.1.3#egg=ckanext-dcat
 -e git+https://github.com/ckan/[email protected]#egg=ckanext-harvest
--e git+https://github.com/ckan/ckanext-scheming.git@release-1.2.0#egg=ckanext-scheming
--e git+https://github.com/qld-gov-au/ckan-ex-qgov.git@4.0.2#egg=ckanext-qgov
+-e git+https://github.com/ckan/ckanext-scheming.git@release-2.1.0#egg=ckanext-scheming
+-e git+https://github.com/qld-gov-au/ckan-ex-qgov.git@5.0.2#egg=ckanext-qgov
 -e git+https://github.com/qld-gov-au/[email protected]#egg=ckanext-archiver
--e git+https://github.com/qld-gov-au/[email protected].3#egg=ckanext-csrf-filter
--e git+https://github.com/qld-gov-au/ckanext-data-qld.git@6.0.5#egg=ckanext-data-qld
+-e git+https://github.com/qld-gov-au/[email protected].4#egg=ckanext-csrf-filter
+-e git+https://github.com/qld-gov-au/ckanext-data-qld.git@6.3.4#egg=ckanext-data-qld
 -e git+https://github.com/qld-gov-au/[email protected]#egg=ckanext-datarequests
+-e git+https://github.com/qld-gov-au/[email protected]#egg=ckanext-odi-certificates
 -e git+https://github.com/qld-gov-au/[email protected]#egg=ckanext-qa
 -e git+https://github.com/qld-gov-au/[email protected]#egg=ckanext-report
 -e git+https://github.com/qld-gov-au/[email protected]#egg=ckanext-resource-type-validation
+-e git+https://github.com/qld-gov-au/[email protected]#egg=ckanext-ssm-config
 -e git+https://github.com/qld-gov-au/[email protected]#egg=ckanext-validation
--e git+https://github.com/qld-gov-au/ckanext-xloader.git@0.9.0-qgov.1#egg=ckanext-xloader
+-e git+https://github.com/qld-gov-au/ckanext-xloader.git@0.10.0-qgov.1#egg=ckanext-xloader
 -e git+https://github.com/qld-gov-au/[email protected]#egg=ckanext-ytp-comments

requirements.txt

@@ -1,5 +1,5 @@
 ckantoolkit>=0.0.4
-pika>=1.1.0
+pika>=1.1.0,<1.3.0
 pyOpenSSL>=18.0.0
 redis
 requests>=2.11.1

setup.cfg

@@ -1,21 +1,21 @@
 [extract_messages]
 keywords = translate isPlural
 add_comments = TRANSLATORS:
-output_file = ckanext/data-qld-harvester/i18n/ckanext-data-qld-harvester.pot
+output_file = ckanext/harvester-data-qld-geoscience/i18n/ckanext-harvester-data-qld-geoscience.pot
 width = 80
 
 [init_catalog]
-domain = ckanext-data-qld-harvester
-input_file = ckanext/data-qld-harvester/i18n/ckanext-data-qld-harvester.pot
-output_dir = ckanext/data-qld-harvester/i18n
+domain = ckanext-harvester-data-qld-geoscience
+input_file = ckanext/harvester-data-qld-geoscience/i18n/ckanext-harvester-data-qld-geoscience.pot
+output_dir = ckanext/harvester-data-qld-geoscience/i18n
 
 [update_catalog]
-domain = ckanext-data-qld-harvester
-input_file = ckanext/data-qld-harvester/i18n/ckanext-data-qld-harvester.pot
-output_dir = ckanext/data-qld-harvester/i18n
+domain = ckanext-harvester-data-qld-geoscience
+input_file = ckanext/harvester-data-qld-geoscience/i18n/ckanext-harvester-data-qld-geoscience.pot
+output_dir = ckanext/harvester-data-qld-geoscience/i18n
 previous = true
 
 [compile_catalog]
-domain = ckanext-data-qld-harvester
-directory = ckanext/data-qld-harvester/i18n
-statistics = true
+domain = ckanext-harvester-data-qld-geoscience
+directory = ckanext/harvester-data-qld-geoscience/i18n
+statistics = true

setup.py

@@ -10,7 +10,7 @@
     long_description = f.read()
 
 setup(
-    name='ckanext-data-qld-harvester',
+    name='ckanext-harvester-data-qld-geoscience',
 
     # Versions should comply with PEP440.  For a discussion on single-sourcing
     # the version across setup.py and the project code, see

test/features/admin_reporting.feature

@@ -10,9 +10,9 @@ Feature: AdminReporting
         Then I should see an element with id "organisation"
         When I press the element with xpath "//button[contains(string(), 'Show')]"
         Then I should see "Organisation: Test Organisation" within 1 seconds
-        Then I should see an element with xpath "//tr/th[string()='Criteria' and position()=1]"
-        Then I should see an element with xpath "//tr/th[string()='Figure' and position()=2]"
-
+        And I should see an element with xpath "//tr/th[string()='Criteria' and position()=1]"
+        And I should see an element with xpath "//tr/th[string()='Figure' and position()=2]"
+        And I should be able to download via the element with xpath "//a[contains(string(), 'Export')]"
 
     Scenario: As an editor user of my organisation, I can view 'My Reports' tab in the dashboard but I cannot view the 'Admin Report' link
         Given "TestOrgEditor" as the persona
@@ -38,4 +38,36 @@ Feature: AdminReporting
         And I click the link with text that contains "Admin Report"
         And I press the element with xpath "//button[contains(string(), 'Show')]"
         Then I should see an element with xpath "//tr[@id='overdue-datasets']/td[contains(@class, 'metric-title') and contains(string(), 'Overdue Datasets') and position()=1]"
-        Then I should see an element with xpath "//tr[@id='overdue-datasets']/td[contains(@class, 'metric-data') and position()=2]"
+        And I should see an element with xpath "//tr[@id='overdue-datasets']/td[contains(@class, 'metric-data') and position()=2]"
+
+    Scenario: As an admin user of my organisation, when I view my admin report, I can verify that datasets without groups are identified
+        Given "Organisation Admin" as the persona
+        When I log in
+        And I go to my reports page
+        And I click the link with text that contains "Admin Report"
+        And I press the element with xpath "//button[contains(string(), 'Show')]"
+        Then I should see an element with xpath "//tr[@id='datasets_no_groups']/td[contains(@class, 'metric-title') and position()=1]/a[contains(@href, 'datasets_no_groups?report_type=admin') and contains(string(), 'Datasets not added to group')]"
+        And I should see an element with xpath "//tr[@id='datasets_no_groups']/td[contains(@class, 'metric-data') and position()=2]/a[contains(@href, 'datasets_no_groups?report_type=admin')]"
+
+        When I click the link with text that contains "Datasets not added to group/s"
+        Then I should see "Admin Report: Datasets not added to group/s: Department of Health"
+        And I should see "Department of Health Spend Data"
+        When I click the link with text that contains "Department of Health Spend Data"
+        Then I should see "Department of Health Spend Data"
+        And I should see "Data and Resources"
+
+    Scenario: As an admin user of my organisation, when I view my admin report, I can verify that datasets without tags are identified
+        Given "Organisation Admin" as the persona
+        When I log in
+        And I go to my reports page
+        And I click the link with text that contains "Admin Report"
+        And I press the element with xpath "//button[contains(string(), 'Show')]"
+        Then I should see an element with xpath "//tr[@id='datasets_no_tags']/td[contains(@class, 'metric-title') and position()=1]/a[contains(@href, 'datasets_no_tags?report_type=admin') and contains(string(), 'Datasets with no tags')]"
+        And I should see an element with xpath "//tr[@id='datasets_no_tags']/td[contains(@class, 'metric-data') and position()=2]/a[contains(@href, 'datasets_no_tags?report_type=admin')]"
+
+        When I click the link with text that contains "Datasets with no tags"
+        Then I should see "Admin Report: Datasets with no tags: Department of Health"
+        And I should see "Department of Health Spend Data"
+        When I click the link with text that contains "Department of Health Spend Data"
+        Then I should see "Department of Health Spend Data"
+        And I should see "Data and Resources"