Merge pull request #269 from qld-gov-au/develop

Develop to master

.ahoy.yml

@@ -86,7 +86,7 @@ commands:
     usage: Install a site.
     cmd: |
       ahoy title "Installing a fresh site"
-      ahoy cli "./scripts/init.sh"
+      ahoy cli '$APP_DIR/scripts/init.sh'
 
   clean:
     usage: Remove containers and all build files.

.docker/Dockerfile.ckan → .docker/Dockerfile-py2.ckan

@@ -2,15 +2,17 @@ FROM amazeeio/python:2.7-ckan-21.8.0
 
 ARG SITE_URL=http://ckan:3000/
 ARG CKAN_REPO=qld-gov-au/ckan
-ARG CKAN_VERSION=ckan-2.8.8-qgov.5
+ARG CKAN_VERSION=ckan-2.9.5-qgov.8
 ARG VARS_TYPE
 ARG DEPLOY_ENV
 ENV VARS_TYPE="${VARS_TYPE}"
+ENV PYTHON_VERSION=py2
 ENV SITE_URL="${SITE_URL}"
 ENV DEPLOY_ENV="${DEPLOY_ENV}"
 ENV VENV_DIR=/app/ckan/default
+ENV SRC_DIR=/app/ckan/default/src
 ENV APP_DIR=/app
-ENV CKAN_INI=/app/ckan/default/production.ini
+ENV CKAN_INI=/app/test.ini
 
 WORKDIR "${APP_DIR}"
 
@@ -25,11 +27,11 @@ RUN apk add --no-cache curl build-base postgresql-client \
 RUN . ${VENV_DIR}/bin/activate \
     && pip install setuptools==36.1 \
     && pip install -e "git+https://github.com/${CKAN_REPO}.git@${CKAN_VERSION}#egg=ckan" \
-    && sed -i "s/psycopg2==2.4.5/psycopg2==2.7.7/g" "${VENV_DIR}/src/ckan/requirements.txt" \
-    && ((test -f "${VENV_DIR}/src/ckan/requirements-py2.txt" && \
-        pip install -r "${VENV_DIR}/src/ckan/requirements-py2.txt") || \
-        pip install -r "${VENV_DIR}/src/ckan/requirements.txt") \
-    && ln -s "${VENV_DIR}/src/ckan/who.ini" "${VENV_DIR}/who.ini" \
+    && ((test -f "${SRC_DIR}/ckan/requirements-py2.txt" && \
+        pip install -r "${SRC_DIR}/ckan/requirements-py2.txt") || \
+        pip install -r "${SRC_DIR}/ckan/requirements.txt") \
+    && pip install -r "${SRC_DIR}/ckan/dev-requirements.txt" \
+    && ln -s "${SRC_DIR}/ckan/who.ini" "$APP_DIR/who.ini" \
     && deactivate \
     && ln -s ${APP_DIR}/ckan /usr/lib/ckan \
     && fix-permissions ${APP_DIR}/ckan

.docker/Dockerfile-py3.ckan

@@ -0,0 +1,55 @@
+FROM amazeeio/python:3.8-22.7.0
+
+ARG SITE_URL=http://ckan:3000/
+ARG CKAN_REPO=qld-gov-au/ckan
+ARG CKAN_VERSION=ckan-2.9.5-qgov.8
+ARG VARS_TYPE
+ARG DEPLOY_ENV
+ENV VARS_TYPE="${VARS_TYPE}"
+ENV PYTHON_VERSION=py3
+ENV SITE_URL="${SITE_URL}"
+ENV DEPLOY_ENV="${DEPLOY_ENV}"
+ENV VENV_DIR=/app/ckan/default
+ENV SRC_DIR=/app/ckan/default/src
+ENV APP_DIR=/app
+ENV CKAN_INI=/app/test.ini
+
+WORKDIR "${APP_DIR}"
+
+ENV DOCKERIZE_VERSION v0.6.1
+RUN apk add --no-cache curl build-base git libffi-dev libxml2-dev libxslt-dev postgresql postgresql-client postgresql-dev \
+    && curl -sLO https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
+    && tar -C /usr/local/bin -xzvf dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
+    && rm dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz
+
+# Install CKAN.
+
+RUN mkdir -p ${VENV_DIR} \
+    && virtualenv ${VENV_DIR} \
+    && . ${VENV_DIR}/bin/activate \
+    && pip install setuptools==44.1.0 \
+    && pip install -e "git+https://github.com/${CKAN_REPO}.git@${CKAN_VERSION}#egg=ckan" \
+    && pip install -r "${SRC_DIR}/ckan/requirements.txt" \
+                   -r "${SRC_DIR}/ckan/dev-requirements.txt" \
+    && ln -s "${SRC_DIR}/ckan/who.ini" "$APP_DIR/who.ini" \
+    && deactivate \
+    && ln -s ${APP_DIR}/ckan /usr/lib/ckan \
+    && fix-permissions ${APP_DIR}/ckan
+
+COPY .docker/test-$VARS_TYPE.ini $CKAN_INI
+
+COPY . ${APP_DIR}/
+
+COPY .docker/scripts ${APP_DIR}/scripts
+
+COPY .docker/scripts/ckan_cli ${VENV_DIR}/bin/
+
+COPY vars/shared-$VARS_TYPE.var.yml $APP_DIR/scripts/extensions.yml
+
+RUN chmod +x ${APP_DIR}/scripts/*.sh ${VENV_DIR}/bin/ckan_cli
+
+# Init current extension.
+RUN ${APP_DIR}/scripts/init-ext.sh
+
+ENTRYPOINT ["/sbin/tini", "--", "/lagoon/entrypoints.sh"]
+CMD ["/app/scripts/serve.sh"]

.docker/scripts/create-test-data.sh

@@ -6,15 +6,14 @@ set -e
 set -x
 
 CKAN_ACTION_URL=http://ckan:3000/api/action
+CKAN_USER_NAME="${CKAN_USER_NAME:-admin}"
+CKAN_DISPLAY_NAME="${CKAN_DISPLAY_NAME:-Administrator}"
+CKAN_USER_EMAIL="${CKAN_USER_EMAIL:-admin@localhost}"
 
 if [ "$VENV_DIR" != "" ]; then
   . ${VENV_DIR}/bin/activate
 fi
 
-CKAN_USER_NAME="${CKAN_USER_NAME:-admin}"
-CKAN_DISPLAY_NAME="${CKAN_DISPLAY_NAME:-Administrator}"
-CKAN_USER_EMAIL="${CKAN_USER_EMAIL:-admin@localhost}"
-
 add_user_if_needed () {
     echo "Adding user '$2' ($1) with email address [$3]"
     ckan_cli user show "$1" | grep "$1" || ckan_cli user add "$1"\
@@ -26,7 +25,6 @@ add_user_if_needed () {
 add_user_if_needed "$CKAN_USER_NAME" "$CKAN_DISPLAY_NAME" "$CKAN_USER_EMAIL"
 ckan_cli sysadmin add "${CKAN_USER_NAME}"
 
-# We know the "admin" sysadmin account exists, so we'll use her API KEY to create further data
 API_KEY=$(ckan_cli user show "${CKAN_USER_NAME}" | tr -d '\n' | sed -r 's/^(.*)apikey=(\S*)(.*)/\2/')
 if [ "$API_KEY" = "None" ]; then
     echo "No API Key found on ${CKAN_USER_NAME}, generating API Token..."
@@ -46,17 +44,17 @@ add_user_if_needed test_org_admin "Test Admin" test_org_admin@localhost
 add_user_if_needed test_org_editor "Test Editor" test_org_editor@localhost
 add_user_if_needed test_org_member "Test Member" test_org_member@localhost
 
-echo "Creating ${TEST_ORG_TITLE} Organisation:"
+echo "Creating ${TEST_ORG_TITLE} organisation:"
 
 TEST_ORG=$( \
     curl -LsH "Authorization: ${API_KEY}" \
     --data '{"name": "'"${TEST_ORG_NAME}"'", "title": "'"${TEST_ORG_TITLE}"'"}' \
     ${CKAN_ACTION_URL}/organization_create
 )
 
-TEST_ORG_ID=$(echo $TEST_ORG | sed -r 's/^(.*)"id": "(.*)",(.*)/\2/')
+TEST_ORG_ID=$(echo $TEST_ORG | python $APP_DIR/scripts/extract-id.py)
 
-echo "Assigning test users to ${TEST_ORG_TITLE} Organisation:"
+echo "Assigning test users to '${TEST_ORG_TITLE}' organisation (${TEST_ORG_ID}):"
 
 curl -LsH "Authorization: ${API_KEY}" \
     --data '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_admin", "object_type": "user", "capacity": "admin"}' \
@@ -174,7 +172,7 @@ DR_ORG=$( \
     ${CKAN_ACTION_URL}/organization_create
 )
 
-DR_ORG_ID=$(echo $DR_ORG | sed -r 's/^(.*)"id": "(.*)",(.*)/\2/')
+DR_ORG_ID=$(echo $DR_ORG | python $APP_DIR/scripts/extract-id.py)
 
 echo "Assigning test users to ${DR_ORG_TITLE} Organisation:"
 
@@ -228,7 +226,7 @@ REPORT_ORG=$( \
     ${CKAN_ACTION_URL}/organization_create
 )
 
-REPORT_ORG_ID=$(echo $REPORT_ORG | sed -r 's/^(.*)"id": "(.*)",(.*)/\2/')
+REPORT_ORG_ID=$(echo $REPORT_ORG | python $APP_DIR/scripts/extract-id.py)
 
 echo "Assigning test users to ${REPORT_ORG_TITLE} Organisation:"
 

.docker/scripts/extract-id.py

@@ -0,0 +1,5 @@
+# encoding: utf-8
+import json
+import sys
+
+print(json.loads(sys.stdin.read())['result']['id'])

.docker/scripts/init-ext.sh

@@ -4,16 +4,37 @@
 #
 set -e
 
+install_requirements () {
+    PROJECT_DIR=$1
+    shift
+    # Identify the best match requirements file, ignore the others.
+    # If there is one specific to our Python version, use that.
+    for filename_pattern in "$@"; do
+        filename="$PROJECT_DIR/${filename_pattern}-$PYTHON_VERSION.txt"
+        if [ -f "$filename" ]; then
+            pip install -r "$filename"
+            return 0
+        fi
+    done
+    for filename_pattern in "$@"; do
+        filename="$PROJECT_DIR/$filename_pattern.txt"
+        if [ -f "$filename" ]; then
+            pip install -r "$filename"
+            return 0
+        fi
+    done
+}
+
 if [ "$VENV_DIR" != "" ]; then
   . ${VENV_DIR}/bin/activate
 fi
-pip install -r "requirements.txt"
-pip install -r "requirements-dev.txt"
+install_requirements . dev-requirements requirements-dev
 EXTENSIONS_FILE=$APP_DIR/scripts/extensions.yml python $(dirname $0)/generate-ext-requirements.py
 pip install --force-reinstall -r "/tmp/requirements-ext.txt"
-for req in `ls $VENV_DIR/src/ckanext-*/requirements.txt`; do
-    pip install -r "$req"
+for extension in . `ls -d $SRC_DIR/ckanext-*`; do
+    install_requirements $extension requirements pip-requirements
 done
+install_requirements . dev-requirements requirements-dev
 
 if [ "$VENV_DIR" != "" ]; then
   deactivate

.docker/test-OpenData.ini

@@ -97,7 +97,7 @@ ckan.redis.url = redis://redis:6379
 #       Add ``datapusher`` to enable DataPusher
 #       Add ``resource_proxy`` to enable resource proxying and get around the
 #       same origin policy
-ckan.plugins = stats text_view image_view recline_view datastore data_qld scheming_datasets validation dcat qa archiver report data_qld_reporting data_qld_google_analytics harvester_data_qld_geoscience harvest qgovext ytp_comments datarequests xloader csrf_filter resource_type_validation
+ckan.plugins = stats text_view image_view recline_view datastore data_qld data_qld_google_analytics scheming_datasets validation dcat qa archiver report harvester_data_qld_geoscience harvest qgovext ytp_comments datarequests xloader csrf_filter resource_type_validation ssm_config odi_certificates
 
 # Define which views should be created by default
 # (plugins must be loaded in ckan.plugins)
@@ -142,6 +142,8 @@ ckan.storage_path = /app/filestore
 #ckan.datapusher.url = http://127.0.0.1:8800/
 #ckan.datapusher.assume_task_stale_after = 3600
 
+ckanext.xloader.jobs_db.uri = postgresql://ckan:ckan@postgres/ckan?sslmode=disable
+
 # Resource Proxy settings
 # Preview size limit, default: 1MB
 #ckan.resource_proxy.max_file_size = 1048576

.env

@@ -15,11 +15,11 @@ PROJECT="ckan-qld-infrastructure"
 # Docker Compose project name. All containers will have this name.
 COMPOSE_PROJECT_NAME="$PROJECT"
 
-# Flag to allow code linting failures.
+# Flag to allow code linting failures. 0=enforce, 1=ignore
 ALLOW_LINT_FAIL=0
 
-# Flag to allow unit tests failures.
+# Flag to allow unit tests failures. 0=enforce, 1=ignore
 ALLOW_UNIT_FAIL=0
 
-# Flag to allow BDD tests failures.
+# Flag to allow BDD tests failures. 0=enforce, 1=ignore
 ALLOW_BDD_FAIL=0

.flake8

@@ -10,6 +10,7 @@ format = pylint
 
 # Show the source of errors.
 show_source = True
+statistics = True
 
 max-complexity = 10
 max-line-length = 127

.github/workflows/test-staging.yml

@@ -13,16 +13,18 @@ jobs:
     strategy:
       matrix:
         application: [OpenData, Publications]
+        python-version: [py2, py3]
         target_environment: [STAGING]
       fail-fast: true
 
-    name: ${{ matrix.application }} ${{ matrix.target_environment }} infrastructure build
+    name: ${{ matrix.application }} ${{ matrix.target_environment }} ${{ matrix.python-version }} infrastructure build
     runs-on: ubuntu-latest
     container: integratedexperts/ci-builder
     env:
       VARS_TYPE: ${{ matrix.application }}
       DEPLOY_ENV: ${{ matrix.target_environment }}
       CKAN_REPO: qld-gov-au/ckan
+      PYTHON_VERSION: ${{ matrix.python-version }}
 
     steps:
       - uses: actions/checkout@v2
@@ -45,6 +47,6 @@ jobs:
         if: failure()
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ matrix.application }} ${{ matrix.target_environment }} screenshots
+          name: ${{ matrix.application }} ${{ matrix.target_environment }} ${{ matrix.python-version }} screenshots
           path: /tmp/artifacts/behave/screenshots
         timeout-minutes: 3

.github/workflows/test.yml

@@ -10,16 +10,18 @@ jobs:
     strategy:
       matrix:
         application: [OpenData, Publications]
+        python-version: [py2, py3]
         target_environment: [DEV]
       fail-fast: false
 
-    name: ${{ matrix.application }} ${{ matrix.target_environment }} infrastructure build
+    name: ${{ matrix.application }} ${{ matrix.target_environment }} ${{ matrix.python-version }} infrastructure build
     runs-on: ubuntu-latest
     container: integratedexperts/ci-builder
     env:
       VARS_TYPE: ${{ matrix.application }}
       DEPLOY_ENV: ${{ matrix.target_environment }}
       CKAN_REPO: qld-gov-au/ckan
+      PYTHON_VERSION: ${{ matrix.python-version }}
 
     steps:
       - uses: actions/checkout@v2
@@ -42,6 +44,6 @@ jobs:
         if: failure()
         uses: actions/upload-artifact@v2
         with:
-          name: ${{ matrix.application }} ${{ matrix.target_environment }} screenshots
+          name: ${{ matrix.application }} ${{ matrix.target_environment }} ${{ matrix.python-version }} screenshots
           path: /tmp/artifacts/behave/screenshots
         timeout-minutes: 1

docker-compose.yml

@@ -21,7 +21,7 @@ services:
   ckan:
     build:
       context: .
-      dockerfile: .docker/Dockerfile.ckan
+      dockerfile: .docker/Dockerfile-${PYTHON_VERSION}.ckan
       args:
         SITE_URL: "http://${PROJECT}.docker.amazee.io"
         VARS_TYPE:

requirements-dev.txt

@@ -8,6 +8,6 @@ flake8==3.8.3
 mock
 pika>=1.1.0,<1.3.0
 pytest-ckan
-PyYAML>=1.0
+pyyaml==5.4.1
 setuptools==44.1.1
 splinter>=0.13.0,<0.17

test/features/data_qld_theme.feature

@@ -85,11 +85,6 @@ Feature: Theme customisations
         Then I should see an element with xpath "//a[contains(@href, '/datastore/dump/') and contains(@href, 'format=json') and contains(string(), 'JSON')]"
         Then I should see an element with xpath "//a[contains(@href, '/datastore/dump/') and contains(@href, 'format=xml') and contains(string(), 'XML')]"
 
-    @Publications
-    Scenario: Homepage 'Publish in the Gazettes' link is correct
-        When I go to homepage
-        Then I should see an element with xpath "//a[@href='https://www.forgov.qld.gov.au/information-and-communication-technology/communication-and-publishing/website-and-digital-publishing/queensland-government-gazette/publish-in-the-gazette' and string()='Publish in the Gazettes']"
-
     @Publications
     Scenario: Menu items are present and correct
         When I go to "/dataset"

test/features/data_usability_rating.feature

@@ -17,4 +17,5 @@ Feature: Data usability rating
             | TXT    | txt_resource.txt   | 1     |
             | XLS    | xls_resource.xls   | 2     |
             | CSV    | csv_resource.csv   | 3     |
+            | JSON   | json_resource.json | 3     |
             | RDF    | rdf_resource.rdf   | 4     |

test/features/datarequest.feature

@@ -17,7 +17,7 @@ Feature: Data Request
     Scenario: Data requests submitted without a description will produce an error message
         Given "SysAdmin" as the persona
         When I log in and go to the data requests page
-        And I press the element with xpath "//a[contains(@class, 'btn-primary')]"
+        And I press the element with xpath "//a[contains(@class, 'btn-primary') and contains(string(), 'Add data request')]"
         And I fill in "title" with "Test data request"
         And I press the element with xpath "//button[contains(@class, 'btn-primary')]"
         Then I should see an element with the css selector "div.error-explanation.alert.alert-error" within 2 seconds
@@ -60,7 +60,7 @@ Feature: Data Request
         When I log in and create a datarequest
         And I press the element with xpath "//a[contains(string(), 'Close')]"
         And I select "Requestor initiated closure" from "close_circumstance"
-        And I press the element with xpath "//button[contains(@class, 'btn-danger')]"
+        And I press the element with xpath "//button[contains(@class, 'btn-danger') and contains(string(), 'Close data request')]"
         Then I should see an element with xpath "//i[contains(@class, 'icon-lock')]"
         And I should not see an element with xpath "//a[contains(string(), 'Close')]"
 
@@ -69,7 +69,7 @@ Feature: Data Request
         When I log in and create a datarequest
         And I press the element with xpath "//a[contains(string(), 'Close')]"
         And I select "Requestor initiated closure" from "close_circumstance"
-        And I press the element with xpath "//button[contains(@class, 'btn-danger') and @name='close']"
+        And I press the element with xpath "//button[contains(@class, 'btn-danger') and contains(string(), 'Close data request')]"
         Then I should see an element with xpath "//a[contains(string(), 'Re-open')]"
         When I press the element with xpath "//a[contains(string(), 'Re-open')]"
         Then I should see an element with xpath "//i[contains(@class, 'icon-unlock')]"

test/features/engagement_reporting.feature

@@ -67,7 +67,7 @@ Feature: Engagement Reporting
         And I press the element with xpath "//a[contains(string(), 'Close')]"
         And I select "To be released as open data at a later date" from "close_circumstance"
         And I fill in "approx_publishing_date" with "01/01/1970"
-        And I press the element with xpath "//button[contains(@class, 'btn-danger') and @name='close']"
+        And I press the element with xpath "//button[contains(@class, 'btn-danger') and @name='close' and contains(string(), 'Close data request')]"
         And I should see an element with xpath "//i[contains(@class, 'icon-lock')]"
 
         When I go to my reports page