Merge pull request #377 from qld-gov-au/QOLDEV-312-data-theme-CKAN-2.10

QOLDEV-312 update Open Data theme to prepare for CKAN 2.10

.ahoy.yml

@@ -66,7 +66,9 @@ commands:
 
   logs:
     usage: Show Docker logs.
-    cmd: sh bin/docker-compose.sh logs "$@"
+    cmd: |
+        ahoy title "Output logs"
+        sh bin/docker-compose.sh logs "$@"
 
   pull:
     usage: Pull latest docker images.
@@ -181,28 +183,28 @@ commands:
     usage: Starts default CKAN background job worker
     cmd: |
       ahoy title 'Starting default CKAN background job worker'
-      ahoy cli "ckan_cli jobs clear && \
-                ckan_cli jobs worker"
+      ahoy cli "ckan_cli jobs worker"
 
   start-ckan-job-worker-priority:
     usage: Starts CKAN background job worker for priority tasks
     cmd: |
       ahoy title 'Starting CKAN background job worker priority'
-      ahoy cli "ckan_cli jobs clear priority && \
-                ckan_cli jobs worker priority"
+      ahoy cli "ckan_cli jobs worker priority"
 
   start-ckan-job-worker-bulk:
     usage: Starts CKAN background job worker for low-priority tasks
     cmd: |
       ahoy title 'Starting CKAN background job worker bulk'
-      ahoy cli "ckan_cli jobs clear bulk && \
-                ckan_cli jobs worker bulk"
+      ahoy cli "ckan_cli jobs worker bulk"
 
   stop-ckan-job-workers:
     usage: Stops CKAN background job worker
     cmd: |
       ahoy title 'Stopping CKAN background job worker'
-      ahoy cli "pkill -f 'jobs worker'"
+      ahoy cli "ckan_cli jobs clear; \
+                ckan_cli jobs clear priority; \
+                ckan_cli jobs clear bulk;
+                (pgrep 'jobs worker' && pkill -f 'jobs worker') || true"
 
   # Utilities.
   title:

.flake8

@@ -18,4 +18,6 @@ max-line-length = 127
 
 # List ignore rules one per line.
 ignore =
+    C901
+    E501
     W503

.github/workflows/test.yml

@@ -46,7 +46,7 @@ jobs:
       - name: Retrieve logs
         if: always()
         run: bin/get-logs.sh
-        timeout-minutes: 5
+        timeout-minutes: 1
 
       - name: Retrieve screenshots
         if: always()

bin/serve.sh

@@ -17,7 +17,7 @@ done
 
 . ${APP_DIR}/bin/activate
 if (which ckan > /dev/null); then
-    ckan -c ${CKAN_INI} run --disable-reloader  --threaded
+    ckan -c ${CKAN_INI} run --disable-reloader --threaded
 else
     paster serve ${CKAN_INI}
 fi

test/features/comments.feature

@@ -32,7 +32,8 @@ Feature: Comments
     Scenario: When a logged-in user submits a comment on a Data Request the comment should then be visible on the Comments tab of the Data Request
         Given "CKANUser" as the persona
         When I log in
-        And I go to data request "Test Request" comments
+        And I create a datarequest
+        And I go to data request "$last_generated_title" comments
         Then I should see an element with xpath "//h3[contains(string(), 'Add a comment')]"
         When I submit a comment with subject "Test subject" and comment "This is a test comment"
         Then I should see "This is a test comment" within 10 seconds
@@ -41,11 +42,12 @@ Feature: Comments
     Scenario: When a logged-in user submits a comment on a Data Request the email should contain title and comment
         Given "CKANUser" as the persona
         When I log in
-        And I go to data request "Test Request" comments
+        And I create a datarequest
+        And I go to data request "$last_generated_title" comments
         Then I should see an element with xpath "//h3[contains(string(), 'Add a comment')]"
-        When I submit a comment with subject "Test Request" and comment "This is a test data request comment"
+        When I submit a comment with subject "Testing Data Request comment" and comment "This is a test data request comment"
         And I wait for 5 seconds
-        Then I should receive a base64 email at "test_org_admin@localhost" containing both "Data request subject: Test Request" and "Comment: This is a test data request comment"
+        Then I should receive a base64 email at "dr_admin@localhost" containing both "Data request subject: Test Title" and "Comment: This is a test data request comment"
 
     @comment-add @comment-profane
     Scenario: When a logged-in user submits a comment containing profanity on a Dataset they should receive an error message and the comment will not appear
@@ -71,7 +73,8 @@ Feature: Comments
     Scenario: When a logged-in user submits a comment containing profanity on a Data Request they should receive an error message and the comment will not appear
         Given "CKANUser" as the persona
         When I log in
-        And I go to data request "Test Request" comments
+        And I create a datarequest
+        And I go to data request "$last_generated_title" comments
         Then I should see an element with xpath "//h3[contains(string(), 'Add a comment')]"
         When I submit a comment with subject "Test subject" and comment "He had sheep, and oxen, and he asses, and menservants, and maidservants, and she asses, and camels."
         Then I should see "Comment blocked due to profanity" within 5 seconds
@@ -86,21 +89,26 @@ Feature: Comments
         When I submit a comment with subject "Testing flags" and comment "Test"
         And I press the element with xpath "//a[contains(@class, 'flag-comment')][1]"
         And I confirm the dialog containing "comment has been flagged as inappropriate" if present
-        Then I should see "Reported" within 5 seconds
+        And I wait for 5 seconds
+        And I go to dataset "$last_generated_name" comments
+        Then I should see "Reported"
         And I should receive a base64 email at "test_org_admin@localhost" containing "This comment has been flagged as inappropriate by a user"
 
     @comment-report @datarequest @email
     Scenario: When a logged-in user reports a comment on a Data Request the comment should be marked as reported and an email notification sent to the organisation admins
         Given "CKANUser" as the persona
         When I log in
-        And I go to data request "Test Request" comments
+        And I create a datarequest
+        And I go to data request "$last_generated_title" comments
         And I submit a comment with subject "Test reporting" and comment "Testing comment reporting"
         Then I should see "Testing comment reporting" within 10 seconds
 
         When I press the element with xpath "//a[contains(@class, 'flag-comment')][1]"
         And I confirm the dialog containing "comment has been flagged as inappropriate" if present
-        Then I should see "Reported" within 5 seconds
-        And I should receive a base64 email at "test_org_admin@localhost" containing "This comment has been flagged as inappropriate by a user"
+        And I wait for 5 seconds
+        And I go to data request "$last_generated_title" comments
+        Then I should see "Reported"
+        And I should receive a base64 email at "dr_admin@localhost" containing "This comment has been flagged as inappropriate by a user"
 
     @comment-reply
     Scenario: When a logged-in user submits a reply comment on a Dataset, the comment should display within 10 seconds
@@ -128,13 +136,15 @@ Feature: Comments
 
     @comment-delete @datarequest
     Scenario: When an admin visits a data request belonging to their organisation, they can delete a comment and should see deletion text for the user responsible.
-        Given "TestOrgAdmin" as the persona
+        Given "DataRequestOrgAdmin" as the persona
         When I log in
-        And I go to data request "Test Request" comments
+        And I create a datarequest
+        And I go to data request "$last_generated_title" comments
+        When I submit a comment with subject "Testing deletion" and comment "Test"
         And I press the element with xpath "//a[@title='Delete comment']"
         And I confirm the dialog containing "Are you sure you want to delete this comment?" if present
         Then I should not see "This comment was deleted." within 2 seconds
-        And I should see "Comment deleted by Test Admin." within 2 seconds
+        And I should see "Comment deleted by Data Request Admin." within 2 seconds
 
     @comment-tab
     @unauthenticated

test/features/datarequest_circumstances.feature

@@ -123,7 +123,8 @@ Feature: Datarequest-circumstances
         And I select "Open dataset already exists" from "close_circumstance"
         And I wait for 1 seconds
         # Have to use JS to change the selected value as the behaving framework does not work with autocomplete dropdown
-        And I execute the script "$('#field-accepted_dataset_id').val($('#field-accepted_dataset_id option:eq(1)').attr('value'))"
+        And I execute the script "$('#field-accepted_dataset_id option:contains("Dataset for data requests")').attr('selected', 'selected')"
+        And I take a debugging screenshot
         And I press the element with xpath "//button[contains(@class, 'btn-danger') and @name='close']"
         Then I should see "Accepted dataset" within 1 seconds
         And I should see "Dataset for data requests" within 1 seconds

test/features/engagement_reporting.feature

@@ -30,14 +30,14 @@ Feature: Engagement Reporting
         And I press "Engagement Report"
         And I press the element with xpath "//button[contains(string(), 'Show')]"
         Then I should see an element with xpath "//tr[@id='datarequests-total']/td[contains(@class, 'metric-title') and string()='Data requests' and position()=1]"
-        And I should see an element with xpath "//tr[@id='datarequests-total']/td[contains(@class, 'metric-data') and string()='25' and position()=2]"
+        And I should see an element with xpath "//tr[@id='datarequests-total']/td[contains(@class, 'metric-data') and string()='30' and position()=2]"
 
         When I create a datarequest
         And I go to my reports page
         And I press "Engagement Report"
         And I press the element with xpath "//button[contains(string(), 'Show')]"
         Then I should see an element with xpath "//tr[@id='datarequests-total']/td[contains(@class, 'metric-title') and string()='Data requests' and position()=1]"
-        And I should see an element with xpath "//tr[@id='datarequests-total']/td[contains(@class, 'metric-data') and string()='26' and position()=2]"
+        And I should see an element with xpath "//tr[@id='datarequests-total']/td[contains(@class, 'metric-data') and string()='31' and position()=2]"
 
     Scenario: As an admin user of my organisation, when I view my engagement report, I can verify the numbers are correct and increment
         Given "ReportingOrgAdmin" as the persona
@@ -55,7 +55,9 @@ Feature: Engagement Reporting
         And I should see an element with xpath "//tr[contains(@class, 'closing-circumstance')]/td[position()=2]/a[contains(@href, '/closed?') and string()='0']"
 
         When I create a dataset and resource with key-value parameters "notes=Dataset for engagement reporting" and "url=default"
-        And I press the element with xpath "//a[@class='btn btn-success' and contains(string(), 'Follow')]"
+        And I take a debugging screenshot
+        And I press "Follow"
+        And I take a debugging screenshot
         And I go to dataset "$last_generated_name" comments
         And I submit a comment with subject "Test subject" and comment "This is a test comment"
         And I go to data request "Reporting Request" comments

test/features/resource_align_default_schema.feature

@@ -10,17 +10,20 @@ Feature: Resource align_default_schema field
         Then I should see an element with xpath "//th[@class="dataset-label" and string()="Default data schema"]/following-sibling::td[contains(string(), "[blank]")]"
 
         When I go to the first resource in the dataset
-        And I press the element with xpath "//a[contains(string(),'Manage')]"
+        And I press "Manage"
         Then I should not see "Align this data schema with the dataset default"
-        And I should see "Upload"
-        And I should see "Link"
-        And I should see "JSON"
+        And I should see an element with xpath "//div[@id='resource-schema-buttons']/label[contains(string(), 'Data Schema')]"
+        When I show the non-JavaScript schema fields
+        Then I should see "Upload Data Schema"
+        And I should see "Data Schema URL"
+        And I should see "Data Schema JSON definition"
 
         When I open the new resource form for dataset "$last_generated_name"
         Then I should not see "Align this data schema with the dataset default"
-        And I should see "Upload"
-        And I should see "Link"
-        And I should see "JSON"
+        When I show the non-JavaScript schema fields
+        Then I should see "Upload Data Schema"
+        And I should see "Data Schema URL"
+        And I should see "Data Schema JSON definition"
 
     Scenario: Create resource with schema not aligned to default schema
         Given "TestOrgEditor" as the persona
@@ -39,65 +42,47 @@ Feature: Resource align_default_schema field
         And I go to the first resource in the dataset
         Then I should see an element with xpath "//th[string()='Aligned with default data schema']/following-sibling::td[string()='FALSE']"
 
-    Scenario: Edit a resource in the GUI where default schema exists and the existing schema value does not match the default
+    Scenario: Edit a resource in the GUI where default schema exists
         Given "TestOrgEditor" as the persona
         When I log in
         And I create a dataset and resource with key-value parameters "schema_json=default" and "name=another-resource::schema=default::align_default_schema=False"
         Then I should see an element with xpath "//th[@class="dataset-label" and string()="Default data schema"]/following::a[contains(string(), "View Schema File")]"
 
+        # Default and resource schema are different
+
         When I go to the first resource in the dataset
         Then I should see an element with xpath "//th[string()='Aligned with default data schema']/following-sibling::td[string()='FALSE']"
         When I click the link with text "View Schema File"
         Then I should see an element with xpath "//body[contains(string(), '"Resource schema"')]"
 
         When I go back
-        And I press the element with xpath "//a[contains(string(), 'Manage')]"
-        Then I should see an element with xpath "//input[@type='checkbox' and @name='align_default_schema' and not(@checked)]/following-sibling::label[@for='field-align_default_schema' and contains(string(),'Align this data schema with the dataset default')]"
+        And I press "Manage"
+        Then I should see "Align this data schema with the dataset default"
+        And I should see an element with xpath "//input[@name='align_default_schema' and not(@checked)]"
 
-        When I check "align_default_schema"
-        And I press the element with xpath "//button[string()='Update Resource']"
-        Then I should see an element with xpath "//th[string()='Aligned with default data schema']/following-sibling::td[string()='TRUE']"
+        When I set the resource schema to the dataset default
+        And I press "Update Resource"
+        Then I should see an element with xpath "//th[string()='Aligned with default data schema']/following-sibling::td[translate(string(), 'true', 'TRUE')='TRUE']"
         When I click the link with text "View Schema File"
         Then I should see an element with xpath "//body[contains(string(), '"Default schema"')]"
 
-    Scenario: Edit resource in the GUI where default schema exists and the existing schema value matches the default
-        Given "TestOrgEditor" as the persona
-        When I log in
-        And I create a dataset and resource with key-value parameters "schema_json=default" and "name=another-resource::schema=::align_default_schema=False"
-        Then I should see an element with xpath "//th[@class="dataset-label" and string()="Default data schema"]/following::a[contains(string(), "View Schema File")]"
-
-        When I go to the first resource in the dataset
-        Then I should see an element with xpath "//th[string()='Aligned with default data schema']/following-sibling::td[string()='FALSE']"
-        When I press the element with xpath "//a[contains(string(), 'Manage')]"
-        Then I should see "Align this data schema with the dataset default"
-
-        When I execute the script "document.getElementById('field-schema').value='{"fields":[{"format": "default","name": "Game Number","type": "integer"},{"format": "default","name": "Game Length","type": "integer"}],"missingValues": ["Default schema"]}'"
-        And I press the element with xpath "//button[string()='Update Resource']"
-        Then I should see an element with xpath "//th[string()='Aligned with default data schema']/following-sibling::td[string()='TRUE']"
-
         # now default and resource schema are the same
-        When I press the element with xpath "//a[contains(string(), 'Manage')]"
-        Then I should not see "Align this data schema with the dataset default"
-
-        # now default and resource schema are different
-        When I press the element with xpath "//textarea[@id='field-schema-json']/preceding-sibling::a[string()='Clear']"
-        And I execute the script "document.getElementById('field-schema').value='{"fields":[{"format": "default","name": "Game Number","type": "integer"},{"format": "default","name": "Game Length","type": "integer"}], "missingValues": ["Resource schema"]}'"
 
-        When I press the element with xpath "//button[string()='Update Resource']"
-        And I press the element with xpath "//a[contains(string(), 'Manage')]"
+        When I go back
+        And I press "Manage"
         Then I should see "Align this data schema with the dataset default"
+        And I should see an element with xpath "//input[@name='align_default_schema' and @checked]"
 
-        When I press the element with xpath "//button[string()='Update Resource']"
+        When I uncheck "align_default_schema"
+        And I execute the script "$('#field-schema-json ~ a.btn-remove-url').click()"
+        And I execute the script "$('#field-schema-json').val('{"fields": [{"format": "default", "name": "Foo", "type": "string"}], "missingValues": ["Baz"]}')"
+        And I take a debugging screenshot
+        And I press "Update Resource"
         And I click the link with text "View Schema File"
-        Then I should see an element with xpath "//body[contains(string(), '"Resource schema"')]"
-
+        Then I should see an element with xpath "//body[contains(string(), '"Baz"')]"
         When I go back
-        And I press the element with xpath "//a[contains(string(), 'Manage')]"
-        Then I should see an element with xpath "//input[@type='checkbox' and @name='align_default_schema' and not(@checked)]/following-sibling::label[@for='field-align_default_schema' and contains(string(), 'Align this data schema with the dataset default')]"
-
-        When I check "align_default_schema"
-        And I press the element with xpath "//button[string()='Update Resource']"
-        Then I should see an element with xpath "//th[string()='Aligned with default data schema']/following-sibling::td[string()='TRUE']"
-
-        When I click the link with text "View Schema File"
-        Then I should see an element with xpath "//body[contains(string(), '"Default schema"')]"
+        Then I should see an element with xpath "//th[string()='Aligned with default data schema']/following-sibling::td[translate(string(), 'false', 'FALSE')='FALSE']"
+        When I press "Manage"
+        And I check "align_default_schema"
+        And I press "Update Resource"
+        Then I should see an element with xpath "//th[string()='Aligned with default data schema']/following-sibling::td[translate(string(), 'true', 'TRUE')='TRUE']"