Merge pull request #261 from qld-gov-au/develop

Develop to master

.ahoy.yml

@@ -115,6 +115,13 @@ commands:
       ahoy cli "flake8 ${@:-roles}" || \
       [ "${ALLOW_LINT_FAIL:-0}" -eq 1 ]
 
+  copy-local-files:
+    usage: Update files from local repo.
+    cmd: |
+      docker cp . $(docker-compose ps -q ckan):/app/
+      docker cp .docker/scripts/ckan_cli $(docker-compose ps -q ckan):/app/ckan/default/bin/
+      ahoy cli 'chmod u+x $VENV_DIR/bin/ckan_cli; cp .docker/test-$VARS_TYPE.ini $CKAN_INI'
+
   test-unit:
     usage: Run unit tests.
     cmd: |
@@ -128,10 +135,15 @@ commands:
       ahoy start-mailmock &
       sleep 5 &&
       # Run generic tests
-      ahoy cli "behave -k --tags ~OpenData --tags ~Publications ${*:-test/features}" || \
+      BEHAVE_COMMAND="behave -k --tags=-OpenData --tags=-Publications ${*:-test/features}"
+      echo "Args: ${*:-test/features}, command: $BEHAVE_COMMAND"
+      ahoy cli "$BEHAVE_COMMAND" || \
       [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ]
       # Run app-specific tests
-      ahoy cli "behave -k --tags $VARS_TYPE ${*:-test/features}" || \
+      ahoy cli "behave -k --tags $VARS_TYPE --tags=-reporting ${*:-test/features}" || \
+      [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ]
+      # Ensure reporting tests run last so there's no race condition on the numbers
+      ahoy cli "behave -k --tags $VARS_TYPE --tags=reporting ${*:-test/features}" || \
       [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ]
       ahoy stop-mailmock
       ahoy stop-ckan-job-workers

.docker/Dockerfile.ckan

@@ -16,7 +16,7 @@ WORKDIR "${APP_DIR}"
 
 ENV DOCKERIZE_VERSION v0.6.1
 RUN apk add --no-cache curl build-base postgresql-client \
-    && curl -s -L -O https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
+    && curl -sLO https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
     && tar -C /usr/local/bin -xzvf dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
     && rm dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz
 

.docker/scripts/ckan_cli

@@ -57,15 +57,15 @@ else
 fi
 
 if [ "$COMMAND" = "ckan" ]; then
-    echo "Using 'ckan' command from $ENV_DIR with config ${CKAN_INI}..." >&2
     # adjust args to match ckan expectations
     COMMAND=$(echo "$1" | sed -e 's/create-test-data/seed/')
+    echo "Using 'ckan' command from $ENV_DIR with config ${CKAN_INI} to run $COMMAND..." >&2
     shift
     exec $ENV_DIR/ckan -c ${CKAN_INI} $COMMAND "$@" $CLICK_ARGS
 elif [ "$COMMAND" = "paster" ]; then
-    echo "Using 'paster' command from $ENV_DIR with config ${CKAN_INI}..." >&2
     # adjust args to match paster expectations
     COMMAND=$1
+    echo "Using 'paster' command from $ENV_DIR with config ${CKAN_INI} to run $COMMAND..." >&2
     shift
     if [ "$1" = "show" ]; then shift; fi
     exec $ENV_DIR/paster --plugin=$PASTER_PLUGIN $COMMAND "$@" -c ${CKAN_INI}

.docker/scripts/create-test-data-OpenData.sh

@@ -9,14 +9,22 @@ if [ "$VENV_DIR" != "" ]; then
   . ${VENV_DIR}/bin/activate
 fi
 
-# Create test dataset with our standard fields
+# Create private test dataset with our standard fields
 curl -LsH "Authorization: ${API_KEY}" \
     --data '{"name": "test-dataset", "owner_org": "'"${TEST_ORG_ID}"'", "private": true,
 "update_frequency": "monthly", "author_email": "admin@localhost", "version": "1.0",
 "license_id": "other-open", "data_driven_application": "NO", "security_classification": "PUBLIC",
 "notes": "test", "de_identified_data": "NO"}' \
     ${CKAN_ACTION_URL}/package_create
 
+# Create public test dataset with our standard fields
+curl -LsH "Authorization: ${API_KEY}" \
+    --data '{"name": "public-test-dataset", "owner_org": "'"${TEST_ORG_ID}"'",
+"update_frequency": "monthly", "author_email": "[email protected]", "version": "1.0",
+"license_id": "other-open", "data_driven_application": "NO", "security_classification": "PUBLIC",
+"notes": "test", "de_identified_data": "NO"}' \
+    ${CKAN_ACTION_URL}/package_create
+
 if [ "$VENV_DIR" != "" ]; then
   deactivate
 fi

.docker/scripts/create-test-data-Publications.sh

@@ -14,11 +14,17 @@ curl -LsH "Authorization: ${API_KEY}" \
     --data '{"name": "publishing-standards-publications-qld-gov-au", "owner_org": "'"${TEST_ORG_ID}"'"}' \
     ${CKAN_ACTION_URL}/package_create
 
-# Create test dataset with our standard fields
+# Create private test dataset with our standard fields
 curl -LsH "Authorization: ${API_KEY}" \
     --data '{"name": "test-dataset", "owner_org": "'"${TEST_ORG_ID}"'", "private": true}' \
     ${CKAN_ACTION_URL}/package_create
 
+# Create public test dataset with our standard fields
+curl -LsH "Authorization: ${API_KEY}" \
+    --data '{"name": "public-test-dataset", "owner_org": "'"${TEST_ORG_ID}"'"}' \
+    ${CKAN_ACTION_URL}/package_create
+
+
 if [ "$VENV_DIR" != "" ]; then
   deactivate
 fi

.docker/scripts/create-test-data.sh

@@ -190,6 +190,13 @@ curl -LsH "Authorization: ${API_KEY}" \
     --data '{"id": "'"${DR_ORG_ID}"'", "object": "dr_member", "object_type": "user", "capacity": "member"}' \
     ${CKAN_ACTION_URL}/member_create
 
+echo "Creating test dataset for data request organisation:"
+
+curl -LsH "Authorization: ${API_KEY}" \
+    --data '{"name": "data_request_dataset", "description": "Dataset for data requests", "owner_org": "'"${DR_ORG_ID}"'",
+"update_frequency": "near-realtime", "author_email": "dr_admin@localhost", "version": "1.0", "license_id": "cc-by-4",
+"data_driven_application": "NO", "security_classification": "PUBLIC", "notes": "test", "de_identified_data": "NO"}'\
+    ${CKAN_ACTION_URL}/package_create
 
 echo "Creating test Data Request:"
 
@@ -258,7 +265,7 @@ curl -LsH "Authorization: ${API_KEY}" \
 echo "Adding ckan.datarequests.closing_circumstances:"
 
 curl -LsH "Authorization: ${API_KEY}" \
-    --data '{"ckan.datarequests.closing_circumstances":
+    --data '{"ckan.comments.profanity_list": "", "ckan.datarequests.closing_circumstances":
         "Released as open data|nominate_dataset\r\nOpen dataset already exists|nominate_dataset\r\nPartially released|nominate_dataset\r\nTo be released as open data at a later date|nominate_approximate_date\r\nData openly available elsewhere\r\nNot suitable for release as open data\r\nRequested data not available/cannot be compiled\r\nRequestor initiated closure"}' \
     ${CKAN_ACTION_URL}/config_option_update
 
@@ -268,6 +275,12 @@ curl -LsH "Authorization: ${API_KEY}" \
     --data '{"ckanext.data_qld.resource_formats": "CSV\r\nHTML\r\nJSON\r\nRDF\r\nTXT\r\nXLS"}' \
     ${CKAN_ACTION_URL}/config_option_update
 
+echo "Creating config value for excluded display name words:"
+
+curl -LsH "Authorization: ${API_KEY}" \
+    --data '{"ckanext.data_qld.excluded_display_name_words": "gov"}' \
+    ${CKAN_ACTION_URL}/config_option_update
+
 if [ "$VENV_DIR" != "" ]; then
   deactivate
 fi

.docker/test-OpenData.ini

@@ -97,7 +97,7 @@ ckan.redis.url = redis://redis:6379
 #       Add ``datapusher`` to enable DataPusher
 #       Add ``resource_proxy`` to enable resource proxying and get around the
 #       same origin policy
-ckan.plugins = stats text_view image_view recline_view datastore data_qld datarequests scheming_datasets validation dcat qa archiver report data_qld_reporting data_qld_google_analytics ytp_comments harvester_data_qld_geoscience harvest qgovext xloader csrf_filter resource_type_validation
+ckan.plugins = stats text_view image_view recline_view datastore data_qld scheming_datasets validation dcat qa archiver report data_qld_reporting data_qld_google_analytics harvester_data_qld_geoscience harvest qgovext ytp_comments datarequests xloader csrf_filter resource_type_validation
 
 # Define which views should be created by default
 # (plugins must be loaded in ckan.plugins)
@@ -249,6 +249,30 @@ handlers = console
 qualname = ckanext
 propagate = 0
 
+[logger_ckanext_archiver]
+level = ERROR
+handlers = console
+qualname = ckanext.archiver
+propagate = 0
+
+[logger_ckanext_qa]
+level = ERROR
+handlers = console
+qualname = ckanext.qa
+propagate = 0
+
+[logger_ckanext_xloader]
+level = ERROR
+handlers = console
+qualname = ckanext.xloader
+propagate = 0
+
+[logger_ckanext_validation]
+level = ERROR
+handlers = console
+qualname = ckanext.validation
+propagate = 0
+
 [handler_console]
 class = StreamHandler
 args = (sys.stderr,)

.github/workflows/test.yml

@@ -1,6 +1,4 @@
 ---
-#based on https://raw.githubusercontent.com/ckan/ckanext-scheming/master/.github/workflows/test.yml
-# alternative https://github.com/ckan/ckan/blob/master/contrib/cookiecutter/ckan_extension/%7B%7Bcookiecutter.project%7D%7D/.github/workflows/test.yml
 name: Tests
 on:
   push:

requirements-dev.txt

@@ -6,6 +6,8 @@ ckantoolkit>=0.0.4
 factory-boy==2.12.0
 flake8==3.8.3
 mock
+pika>=1.1.0,<1.3.0
 pytest-ckan
 PyYAML>=1.0
+setuptools==44.1.1
 splinter>=0.13.0,<0.17

test/features/admin_reporting.feature

@@ -11,9 +11,9 @@ Feature: AdminReporting
         Then I should see an element with id "organisation"
         When I press the element with xpath "//button[contains(string(), 'Show')]"
         Then I should see "Organisation: Test Organisation" within 1 seconds
-        Then I should see an element with xpath "//tr/th[string()='Criteria' and position()=1]"
-        Then I should see an element with xpath "//tr/th[string()='Figure' and position()=2]"
-
+        And I should see an element with xpath "//tr/th[string()='Criteria' and position()=1]"
+        And I should see an element with xpath "//tr/th[string()='Figure' and position()=2]"
+        And I should be able to download via the element with xpath "//a[contains(string(), 'Export')]"
 
     Scenario: As an editor user of my organisation, I can view 'My Reports' tab in the dashboard but I cannot view the 'Admin Report' link
         Given "TestOrgEditor" as the persona
@@ -39,4 +39,36 @@ Feature: AdminReporting
         And I click the link with text that contains "Admin Report"
         And I press the element with xpath "//button[contains(string(), 'Show')]"
         Then I should see an element with xpath "//tr[@id='overdue-datasets']/td[contains(@class, 'metric-title') and contains(string(), 'Overdue Datasets') and position()=1]"
-        Then I should see an element with xpath "//tr[@id='overdue-datasets']/td[contains(@class, 'metric-data') and position()=2]"
+        And I should see an element with xpath "//tr[@id='overdue-datasets']/td[contains(@class, 'metric-data') and position()=2]"
+
+    Scenario: As an admin user of my organisation, when I view my admin report, I can verify that datasets without groups are identified
+        Given "Organisation Admin" as the persona
+        When I log in
+        And I go to my reports page
+        And I click the link with text that contains "Admin Report"
+        And I press the element with xpath "//button[contains(string(), 'Show')]"
+        Then I should see an element with xpath "//tr[@id='datasets_no_groups']/td[contains(@class, 'metric-title') and position()=1]/a[contains(@href, 'datasets_no_groups?report_type=admin') and contains(string(), 'Datasets not added to group')]"
+        And I should see an element with xpath "//tr[@id='datasets_no_groups']/td[contains(@class, 'metric-data') and position()=2]/a[contains(@href, 'datasets_no_groups?report_type=admin')]"
+
+        When I click the link with text that contains "Datasets not added to group/s"
+        Then I should see "Admin Report: Datasets not added to group/s: Department of Health"
+        And I should see "Department of Health Spend Data"
+        When I click the link with text that contains "Department of Health Spend Data"
+        Then I should see "Department of Health Spend Data"
+        And I should see "Data and Resources"
+
+    Scenario: As an admin user of my organisation, when I view my admin report, I can verify that datasets without tags are identified
+        Given "Organisation Admin" as the persona
+        When I log in
+        And I go to my reports page
+        And I click the link with text that contains "Admin Report"
+        And I press the element with xpath "//button[contains(string(), 'Show')]"
+        Then I should see an element with xpath "//tr[@id='datasets_no_tags']/td[contains(@class, 'metric-title') and position()=1]/a[contains(@href, 'datasets_no_tags?report_type=admin') and contains(string(), 'Datasets with no tags')]"
+        And I should see an element with xpath "//tr[@id='datasets_no_tags']/td[contains(@class, 'metric-data') and position()=2]/a[contains(@href, 'datasets_no_tags?report_type=admin')]"
+
+        When I click the link with text that contains "Datasets with no tags"
+        Then I should see "Admin Report: Datasets with no tags: Department of Health"
+        And I should see "Department of Health Spend Data"
+        When I click the link with text that contains "Department of Health Spend Data"
+        Then I should see "Department of Health Spend Data"
+        And I should see "Data and Resources"

test/features/comments.feature

@@ -91,7 +91,7 @@ Feature: Comments
         Then I should see "This is a reply" within 10 seconds
 
     @comment-delete
-    Scenario: When an Org Admin visits a dataset belonging to their organisation, they can delete a comment and should see deletion text for the user responsible.
+    Scenario: When an admin visits a dataset belonging to their organisation, they can delete a comment and should see deletion text for the user responsible.
         Given "TestOrgAdmin" as the persona
         When I log in
         Then I go to dataset "warandpeace" comments
@@ -102,7 +102,7 @@ Feature: Comments
         And I should see "Comment deleted by Test Admin." within 2 seconds
 
     @comment-delete @datarequest
-    Scenario: When an Org Admin visits a data request belonging to their organisation, they can delete a comment and should see deletion text for the user responsible.
+    Scenario: When an admin visits a data request belonging to their organisation, they can delete a comment and should see deletion text for the user responsible.
         Given "TestOrgAdmin" as the persona
         When I log in
         And I go to data request "Test Request" comments

test/features/config.feature

@@ -5,7 +5,8 @@ Feature: Config
         Given "SysAdmin" as the persona
         When I log in and go to admin config page
         Then I should see "Intro Text"
-        And I should see an element with id "field-ckanext.data_qld.excluded_display_name_words"
+        And I should see "Excluded display name words:"
+        And I should see an element with xpath "//textarea[@id='field-ckanext.data_qld.excluded_display_name_words' and contains(string(), 'gov')]"
         And I should not see an element with id "field-ckan-main-css"
         And I should not see an element with id "field-ckan-site-custom-css"
 

test/features/data_qld_theme.feature

@@ -85,6 +85,11 @@ Feature: Theme customisations
         Then I should see an element with xpath "//a[contains(@href, '/datastore/dump/') and contains(@href, 'format=json') and contains(string(), 'JSON')]"
         Then I should see an element with xpath "//a[contains(@href, '/datastore/dump/') and contains(@href, 'format=xml') and contains(string(), 'XML')]"
 
+    @Publications
+    Scenario: Homepage 'Publish in the Gazettes' link is correct
+        When I go to homepage
+        Then I should see an element with xpath "//a[@href='https://www.forgov.qld.gov.au/information-and-communication-technology/communication-and-publishing/website-and-digital-publishing/queensland-government-gazette/publish-in-the-gazette' and string()='Publish in the Gazettes']"
+
     @Publications
     Scenario: Menu items are present and correct
         When I go to "/dataset"
@@ -132,3 +137,10 @@ Feature: Theme customisations
         Then I should see an element with xpath "//a[@href='/user/login' and contains(string(), 'Log in')]"
         And I should see an element with xpath "//a[@href='/user/register' and contains(string(), 'Register')]"
         And I should not see "not found"
+
+    @unauthenticated
+    Scenario: When I go to the robots file, I can see a custom disallow block
+        Given "Unauthenticated" as the persona
+        When I go to "/robots.txt"
+        Then I should see "Disallow: /"
+        And I should not see "Allow:"

test/features/data_request.feature → test/features/datarequest.feature

@@ -1,5 +1,5 @@
-@config
 @OpenData
+@datarequest
 Feature: Data Request
 
     @unauthenticated
@@ -17,7 +17,7 @@ Feature: Data Request
     Scenario: Data requests submitted without a description will produce an error message
         Given "SysAdmin" as the persona
         When I log in and go to the data requests page
-        And I click the link with text that contains "Add data request"
+        And I press the element with xpath "//a[contains(@class, 'btn-primary')]"
         And I fill in "title" with "Test data request"
         And I press the element with xpath "//button[contains(@class, 'btn-primary')]"
         Then I should see an element with the css selector "div.error-explanation.alert.alert-error" within 2 seconds
@@ -60,7 +60,7 @@ Feature: Data Request
         When I log in and create a datarequest
         And I press the element with xpath "//a[contains(string(), 'Close')]"
         And I select "Requestor initiated closure" from "close_circumstance"
-        And I press the element with xpath "//button[contains(string(), 'Close data request')]"
+        And I press the element with xpath "//button[contains(@class, 'btn-danger')]"
         Then I should see an element with xpath "//i[contains(@class, 'icon-lock')]"
         And I should not see an element with xpath "//a[contains(string(), 'Close')]"
 
@@ -69,7 +69,7 @@ Feature: Data Request
         When I log in and create a datarequest
         And I press the element with xpath "//a[contains(string(), 'Close')]"
         And I select "Requestor initiated closure" from "close_circumstance"
-        And I press the element with xpath "//button[contains(string(), 'Close data request')]"
+        And I press the element with xpath "//button[contains(@class, 'btn-danger') and @name='close']"
         Then I should see an element with xpath "//a[contains(string(), 'Re-open')]"
         When I press the element with xpath "//a[contains(string(), 'Re-open')]"
         Then I should see an element with xpath "//i[contains(@class, 'icon-unlock')]"