Merge branch 'snyk-fix-0995087860516ab0b701f45f410d88bb' of https://g…

…ithub.com/qld-gov-au/ckanext-data-qld into snyk-fix-0cbb4fac9a523cd415b6e6754acdae6b

.github/workflows/test.yml

@@ -12,8 +12,8 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 5
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.x'
       - name: Install requirements
@@ -31,13 +31,17 @@ jobs:
 
     name: Run ${{ matrix.behave-tag }} tests on CKAN ${{ matrix.ckan-version }}
     runs-on: ubuntu-latest
-    container: drevops/ci-builder:23.7.0
+    container: drevops/ci-runner:23.12.0
     env:
       CKAN_VERSION: ${{ matrix.ckan-version }}
       BEHAVE_TAG: ${{ matrix.behave-tag }}
 
     steps:
-      - uses: actions/checkout@v3
+      # Patch https://github.com/actions/runner/issues/863
+      - name: Preserve $HOME set in the container
+        run: echo HOME=/root >> "$GITHUB_ENV"
+
+      - uses: actions/checkout@v4
         timeout-minutes: 2
 
       - name: Build
@@ -65,7 +69,7 @@ jobs:
 
       - name: Upload screenshots
         if: failure()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: CKAN ${{ matrix.ckan-version }} ${{ matrix.behave-tag }} screenshots
           path: /tmp/artifacts/behave/screenshots

ckanext/data_qld/ckan_dataset.json

@@ -172,6 +172,14 @@
                     "value": "annually",
                     "label": "Annually"
                 },
+                {
+                    "value": "biennially",
+                    "label": "Biennially"
+                },
+                {
+                    "value": "quadrennially",
+                    "label": "Quadrennially"
+                },
                 {
                     "value": "non-regular",
                     "label": "Non-regular"

ckanext/data_qld/plugin.py

@@ -19,11 +19,14 @@
 from .resource_freshness import validation as resource_freshness_validator
 from .resource_freshness.logic.actions import get as resource_freshness_get_actions
 
-if ' qa' in tk.config.get('ckan.plugins', ''):
+try:
     from ckanext.qa.interfaces import IQA
     import ckanext.qa.lib as qa_lib
     import ckanext.qa.tasks as qa_tasks
     import os
+    qa_present = True
+except ImportError:
+    qa_present = False
 
 
 log = logging.getLogger(__name__)
@@ -43,7 +46,7 @@ class DataQldPlugin(plugins.SingletonPlugin):
     plugins.implements(plugins.IBlueprint)
     plugins.implements(plugins.IClick)
 
-    if ' qa' in tk.config.get('ckan.plugins', ''):
+    if qa_present:
         plugins.implements(IQA)
 
     # IConfigurer

ckanext/data_qld/resource_freshness/helpers/helpers.py

@@ -14,11 +14,14 @@
 
 log = logging.getLogger(__name__)
 
+# Also update ckanext-data-qld/ckanext/data_qld/ckan_dataset.json
 update_frequencies = {
     "monthly": 30,
     "quarterly": 91,
     "half-yearly": 182,
-    "annually": 365
+    "annually": 365,
+    "biennially": 730,
+    "quadrennially": 1460
 }
 
 

ckanext/data_qld/templates/header.html

@@ -109,10 +109,10 @@
           {% set dataset_search_route = 'dataset.search' %}
           <li {% if h.current_url() != '/' %}class="active"{% endif %}><a href="{{ h.url_for(dataset_search_route) }}">Data</a></li>
           <li>
-            <a href="/article/news-and-events">News and events</a>
+            <a href="/visualisations">Visualisations</a>
           </li>
           <li>
-            <a href="/article/case-studies">Case studies</a>
+            <a href="/news-and-case-studies">News and Case Studies</a>
           </li>
           <li>
             <a href="/article/standards-and-guidance">Standards and guidance</a>

dev-requirements-2.9.txt

@@ -14,3 +14,4 @@ python-magic==0.4.18
 progressbar==2.5
 
 pika>=1.1.0,<1.3.0 # resolve harvester conflict
+zipp>=3.19.1 # not directly required, pinned by Snyk to avoid a vulnerability

dev-requirements.txt

@@ -13,4 +13,5 @@ xlrd==1.2.0
 python-magic==0.4.18
 progressbar==2.5
 
+typer<0.11 # We still need Click 7 for now
 pika>=1.1.0,<1.3.0 # resolve harvester conflict

test/features/data_qld_theme.feature

@@ -129,3 +129,10 @@ Feature: Theme customisations
         When I go to "/robots.txt"
         Then I should see "Disallow: /"
         And I should not see "Allow:"
+
+    @unauthenticated
+    Scenario: When I go to the home page, I can see Visualisations and News and Case Studies in the navbar
+        Given "Unauthenticated" as the persona
+        When I go to homepage
+        Then I should see an element with xpath "//a[string()='Visualisations']"
+        And I should see an element with xpath "//a[string()='News and Case Studies']"

test/features/data_usability_rating.feature

@@ -20,3 +20,14 @@ Feature: Data usability rating
             | CSV    | csv_resource.csv   | 3     |
             | JSON   | json_resource.json | 3     |
             | RDF    | rdf_resource.rdf   | 4     |
+
+    Scenario: As a publisher, when I create an open resource with a matching schema, I can verify the score is upgraded from 3 to 4
+        Given "TestOrgEditor" as the persona
+        When I log in
+        And I create a dataset and resource with key-value parameters "license=other-open" and "format=CSV::upload=test_game_data.csv::schema=default"
+        And I press the element with xpath "//ol[contains(@class, 'breadcrumb')]//a[starts-with(@href, '/dataset/')]"
+        And I reload page every 3 seconds until I see an element with xpath "//div[contains(@class, 'qa') and contains(@class, 'openness-')]" but not more than 10 times
+        Then I should not see an element with xpath "//div[contains(@class, 'openness-3')]"
+        And I should see an element with xpath "//div[contains(@class, 'openness-4')]"
+        When I press "Test Resource"
+        Then I should see an element with xpath "//div[contains(@class, 'qa openness-4')]"

test/features/resource_freshness.feature

@@ -14,18 +14,18 @@ Feature: Resource freshness
             | TestOrgAdmin  |
             | TestOrgEditor |
 
-    Scenario Outline: An editor, admin or sysadmin user, when I go to the dataset new page and select 'monthly' update frequency, then the text 'Next update due' should be visible
+    Scenario Outline: An editor, admin or sysadmin user, when I go to the dataset new page and select an update frequency, then the text 'Next update due' should be visible
         Given "<User>" as the persona
         When I log in
         And I go to "/dataset/new"
-        And I select "monthly" from "update_frequency"
+        And I select "<Frequency>" from "update_frequency"
         Then I should see "Next update due"
 
-        Examples: Users
-            | User          |
-            | SysAdmin      |
-            | TestOrgAdmin  |
-            | TestOrgEditor |
+        Examples: Frequencies
+            | User          | Frequency     |
+            | SysAdmin      | monthly       |
+            | TestOrgAdmin  | biennially    |
+            | TestOrgEditor | quadrennially |
 
     Scenario Outline: As a user with editing privileges, when I set a 'monthly' update frequently, I should still be able to update the dataset via the API
         Given "<User>" as the persona

test/features/steps/steps.py

@@ -294,14 +294,22 @@ def fill_in_default_link_resource_fields(context):
 @when(u'I upload "{file_name}" of type "{file_format}" to resource')
 def upload_file_to_resource(context, file_name, file_format):
     context.execute_steps(u"""
-        When I execute the script "$('#resource-upload-button').trigger(click);"
+        When I execute the script "$('#resource-upload-button').trigger('click');"
         And I attach the file "{file_name}" to "upload"
         # Don't quote the injected string since it can have trailing spaces
         And I execute the script "document.getElementById('field-format').value='{file_format}'"
         And I fill in "size" with "1024" if present
     """.format(file_name=file_name, file_format=file_format))
 
 
+@when(u'I upload schema file "{file_name}" to resource')
+def upload_schema_file_to_resource(context, file_name):
+    context.execute_steps(u"""
+        When I execute the script "$('#field-schema-json ~ a.btn-remove-url').trigger('click');"
+        And I attach the file "{file_name}" to "schema_upload"
+    """.format(file_name=file_name))
+
+
 @when(u'I go to group page')
 def go_to_group_page(context):
     context.execute_steps(u"""
@@ -527,6 +535,12 @@ def create_resource_from_params(context, resource_params):
             if value == "default":
                 value = resource_default_schema
             _enter_manual_schema(context, value)
+        elif key == "schema_upload":
+            if value == "default":
+                value = "test-resource_schemea.json"
+            context.execute_steps(u"""
+                When I upload schema file "{0}" to resource
+            """.format(value))
         else:
             context.execute_steps(u"""
                 When I fill in "{0}" with "{1}" if present

test/fixtures/csv_resource_with_schema.csv

@@ -0,0 +1,2 @@
+id,location,date,measurement,observations
+1,A,01/01/1970,12345,"Lorem ipsum"