diff --git a/.ahoy.yml b/.ahoy.yml index 2f88b115..99af6a52 100644 --- a/.ahoy.yml +++ b/.ahoy.yml @@ -86,7 +86,7 @@ commands: usage: Install a site. cmd: | ahoy title "Installing a fresh site" - ahoy cli "./scripts/init.sh" + ahoy cli '$APP_DIR/scripts/init.sh' clean: usage: Remove containers and all build files. diff --git a/.docker/Dockerfile.ckan b/.docker/Dockerfile-py2.ckan similarity index 75% rename from .docker/Dockerfile.ckan rename to .docker/Dockerfile-py2.ckan index b664b2fc..fc06fbfd 100644 --- a/.docker/Dockerfile.ckan +++ b/.docker/Dockerfile-py2.ckan @@ -2,15 +2,17 @@ FROM amazeeio/python:2.7-ckan-21.8.0 ARG SITE_URL=http://ckan:3000/ ARG CKAN_REPO=qld-gov-au/ckan -ARG CKAN_VERSION=ckan-2.8.8-qgov.5 +ARG CKAN_VERSION=ckan-2.9.5-qgov.8 ARG VARS_TYPE ARG DEPLOY_ENV ENV VARS_TYPE="${VARS_TYPE}" +ENV PYTHON_VERSION=py2 ENV SITE_URL="${SITE_URL}" ENV DEPLOY_ENV="${DEPLOY_ENV}" ENV VENV_DIR=/app/ckan/default +ENV SRC_DIR=/app/ckan/default/src ENV APP_DIR=/app -ENV CKAN_INI=/app/ckan/default/production.ini +ENV CKAN_INI=/app/test.ini WORKDIR "${APP_DIR}" @@ -25,11 +27,11 @@ RUN apk add --no-cache curl build-base postgresql-client \ RUN . ${VENV_DIR}/bin/activate \ && pip install setuptools==36.1 \ && pip install -e "git+https://github.com/${CKAN_REPO}.git@${CKAN_VERSION}#egg=ckan" \ - && sed -i "s/psycopg2==2.4.5/psycopg2==2.7.7/g" "${VENV_DIR}/src/ckan/requirements.txt" \ - && ((test -f "${VENV_DIR}/src/ckan/requirements-py2.txt" && \ - pip install -r "${VENV_DIR}/src/ckan/requirements-py2.txt") || \ - pip install -r "${VENV_DIR}/src/ckan/requirements.txt") \ - && ln -s "${VENV_DIR}/src/ckan/who.ini" "${VENV_DIR}/who.ini" \ + && ((test -f "${SRC_DIR}/ckan/requirements-py2.txt" && \ + pip install -r "${SRC_DIR}/ckan/requirements-py2.txt") || \ + pip install -r "${SRC_DIR}/ckan/requirements.txt") \ + && pip install -r "${SRC_DIR}/ckan/dev-requirements.txt" \ + && ln -s "${SRC_DIR}/ckan/who.ini" "$APP_DIR/who.ini" \ && deactivate \ && ln -s ${APP_DIR}/ckan /usr/lib/ckan \ && fix-permissions ${APP_DIR}/ckan diff --git a/.docker/Dockerfile-py3.ckan b/.docker/Dockerfile-py3.ckan new file mode 100644 index 00000000..1245c241 --- /dev/null +++ b/.docker/Dockerfile-py3.ckan @@ -0,0 +1,55 @@ +FROM amazeeio/python:3.8-22.7.0 + +ARG SITE_URL=http://ckan:3000/ +ARG CKAN_REPO=qld-gov-au/ckan +ARG CKAN_VERSION=ckan-2.9.5-qgov.8 +ARG VARS_TYPE +ARG DEPLOY_ENV +ENV VARS_TYPE="${VARS_TYPE}" +ENV PYTHON_VERSION=py3 +ENV SITE_URL="${SITE_URL}" +ENV DEPLOY_ENV="${DEPLOY_ENV}" +ENV VENV_DIR=/app/ckan/default +ENV SRC_DIR=/app/ckan/default/src +ENV APP_DIR=/app +ENV CKAN_INI=/app/test.ini + +WORKDIR "${APP_DIR}" + +ENV DOCKERIZE_VERSION v0.6.1 +RUN apk add --no-cache curl build-base git libffi-dev libxml2-dev libxslt-dev postgresql postgresql-client postgresql-dev \ + && curl -sLO https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \ + && tar -C /usr/local/bin -xzvf dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \ + && rm dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz + +# Install CKAN. + +RUN mkdir -p ${VENV_DIR} \ + && virtualenv ${VENV_DIR} \ + && . ${VENV_DIR}/bin/activate \ + && pip install setuptools==44.1.0 \ + && pip install -e "git+https://github.com/${CKAN_REPO}.git@${CKAN_VERSION}#egg=ckan" \ + && pip install -r "${SRC_DIR}/ckan/requirements.txt" \ + -r "${SRC_DIR}/ckan/dev-requirements.txt" \ + && ln -s "${SRC_DIR}/ckan/who.ini" "$APP_DIR/who.ini" \ + && deactivate \ + && ln -s ${APP_DIR}/ckan /usr/lib/ckan \ + && fix-permissions ${APP_DIR}/ckan + +COPY .docker/test-$VARS_TYPE.ini $CKAN_INI + +COPY . ${APP_DIR}/ + +COPY .docker/scripts ${APP_DIR}/scripts + +COPY .docker/scripts/ckan_cli ${VENV_DIR}/bin/ + +COPY vars/shared-$VARS_TYPE.var.yml $APP_DIR/scripts/extensions.yml + +RUN chmod +x ${APP_DIR}/scripts/*.sh ${VENV_DIR}/bin/ckan_cli + +# Init current extension. +RUN ${APP_DIR}/scripts/init-ext.sh + +ENTRYPOINT ["/sbin/tini", "--", "/lagoon/entrypoints.sh"] +CMD ["/app/scripts/serve.sh"] diff --git a/.docker/scripts/create-test-data.sh b/.docker/scripts/create-test-data.sh index e910a764..47bca0f1 100644 --- a/.docker/scripts/create-test-data.sh +++ b/.docker/scripts/create-test-data.sh @@ -6,15 +6,14 @@ set -e set -x CKAN_ACTION_URL=http://ckan:3000/api/action +CKAN_USER_NAME="${CKAN_USER_NAME:-admin}" +CKAN_DISPLAY_NAME="${CKAN_DISPLAY_NAME:-Administrator}" +CKAN_USER_EMAIL="${CKAN_USER_EMAIL:-admin@localhost}" if [ "$VENV_DIR" != "" ]; then . ${VENV_DIR}/bin/activate fi -CKAN_USER_NAME="${CKAN_USER_NAME:-admin}" -CKAN_DISPLAY_NAME="${CKAN_DISPLAY_NAME:-Administrator}" -CKAN_USER_EMAIL="${CKAN_USER_EMAIL:-admin@localhost}" - add_user_if_needed () { echo "Adding user '$2' ($1) with email address [$3]" ckan_cli user show "$1" | grep "$1" || ckan_cli user add "$1"\ @@ -26,7 +25,6 @@ add_user_if_needed () { add_user_if_needed "$CKAN_USER_NAME" "$CKAN_DISPLAY_NAME" "$CKAN_USER_EMAIL" ckan_cli sysadmin add "${CKAN_USER_NAME}" -# We know the "admin" sysadmin account exists, so we'll use her API KEY to create further data API_KEY=$(ckan_cli user show "${CKAN_USER_NAME}" | tr -d '\n' | sed -r 's/^(.*)apikey=(\S*)(.*)/\2/') if [ "$API_KEY" = "None" ]; then echo "No API Key found on ${CKAN_USER_NAME}, generating API Token..." @@ -46,7 +44,7 @@ add_user_if_needed test_org_admin "Test Admin" test_org_admin@localhost add_user_if_needed test_org_editor "Test Editor" test_org_editor@localhost add_user_if_needed test_org_member "Test Member" test_org_member@localhost -echo "Creating ${TEST_ORG_TITLE} Organisation:" +echo "Creating ${TEST_ORG_TITLE} organisation:" TEST_ORG=$( \ curl -LsH "Authorization: ${API_KEY}" \ @@ -54,9 +52,9 @@ TEST_ORG=$( \ ${CKAN_ACTION_URL}/organization_create ) -TEST_ORG_ID=$(echo $TEST_ORG | sed -r 's/^(.*)"id": "(.*)",(.*)/\2/') +TEST_ORG_ID=$(echo $TEST_ORG | python $APP_DIR/scripts/extract-id.py) -echo "Assigning test users to ${TEST_ORG_TITLE} Organisation:" +echo "Assigning test users to '${TEST_ORG_TITLE}' organisation (${TEST_ORG_ID}):" curl -LsH "Authorization: ${API_KEY}" \ --data '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_admin", "object_type": "user", "capacity": "admin"}' \ @@ -174,7 +172,7 @@ DR_ORG=$( \ ${CKAN_ACTION_URL}/organization_create ) -DR_ORG_ID=$(echo $DR_ORG | sed -r 's/^(.*)"id": "(.*)",(.*)/\2/') +DR_ORG_ID=$(echo $DR_ORG | python $APP_DIR/scripts/extract-id.py) echo "Assigning test users to ${DR_ORG_TITLE} Organisation:" @@ -228,7 +226,7 @@ REPORT_ORG=$( \ ${CKAN_ACTION_URL}/organization_create ) -REPORT_ORG_ID=$(echo $REPORT_ORG | sed -r 's/^(.*)"id": "(.*)",(.*)/\2/') +REPORT_ORG_ID=$(echo $REPORT_ORG | python $APP_DIR/scripts/extract-id.py) echo "Assigning test users to ${REPORT_ORG_TITLE} Organisation:" diff --git a/.docker/scripts/extract-id.py b/.docker/scripts/extract-id.py new file mode 100644 index 00000000..ae9cf79d --- /dev/null +++ b/.docker/scripts/extract-id.py @@ -0,0 +1,5 @@ +# encoding: utf-8 +import json +import sys + +print(json.loads(sys.stdin.read())['result']['id']) diff --git a/.docker/scripts/init-ext.sh b/.docker/scripts/init-ext.sh index 50d4035a..d6fda09a 100755 --- a/.docker/scripts/init-ext.sh +++ b/.docker/scripts/init-ext.sh @@ -4,16 +4,37 @@ # set -e +install_requirements () { + PROJECT_DIR=$1 + shift + # Identify the best match requirements file, ignore the others. + # If there is one specific to our Python version, use that. + for filename_pattern in "$@"; do + filename="$PROJECT_DIR/${filename_pattern}-$PYTHON_VERSION.txt" + if [ -f "$filename" ]; then + pip install -r "$filename" + return 0 + fi + done + for filename_pattern in "$@"; do + filename="$PROJECT_DIR/$filename_pattern.txt" + if [ -f "$filename" ]; then + pip install -r "$filename" + return 0 + fi + done +} + if [ "$VENV_DIR" != "" ]; then . ${VENV_DIR}/bin/activate fi -pip install -r "requirements.txt" -pip install -r "requirements-dev.txt" +install_requirements . dev-requirements requirements-dev EXTENSIONS_FILE=$APP_DIR/scripts/extensions.yml python $(dirname $0)/generate-ext-requirements.py pip install --force-reinstall -r "/tmp/requirements-ext.txt" -for req in `ls $VENV_DIR/src/ckanext-*/requirements.txt`; do - pip install -r "$req" +for extension in . `ls -d $SRC_DIR/ckanext-*`; do + install_requirements $extension requirements pip-requirements done +install_requirements . dev-requirements requirements-dev if [ "$VENV_DIR" != "" ]; then deactivate diff --git a/.docker/test-OpenData.ini b/.docker/test-OpenData.ini index 7971196e..93bba37b 100644 --- a/.docker/test-OpenData.ini +++ b/.docker/test-OpenData.ini @@ -97,7 +97,7 @@ ckan.redis.url = redis://redis:6379 # Add ``datapusher`` to enable DataPusher # Add ``resource_proxy`` to enable resource proxying and get around the # same origin policy -ckan.plugins = stats text_view image_view recline_view datastore data_qld scheming_datasets validation dcat qa archiver report data_qld_reporting data_qld_google_analytics harvester_data_qld_geoscience harvest qgovext ytp_comments datarequests xloader csrf_filter resource_type_validation +ckan.plugins = stats text_view image_view recline_view datastore data_qld data_qld_google_analytics scheming_datasets validation dcat qa archiver report harvester_data_qld_geoscience harvest qgovext ytp_comments datarequests xloader csrf_filter resource_type_validation ssm_config odi_certificates # Define which views should be created by default # (plugins must be loaded in ckan.plugins) @@ -142,6 +142,8 @@ ckan.storage_path = /app/filestore #ckan.datapusher.url = http://127.0.0.1:8800/ #ckan.datapusher.assume_task_stale_after = 3600 +ckanext.xloader.jobs_db.uri = postgresql://ckan:ckan@postgres/ckan?sslmode=disable + # Resource Proxy settings # Preview size limit, default: 1MB #ckan.resource_proxy.max_file_size = 1048576 diff --git a/.env b/.env index 169401ae..126de260 100644 --- a/.env +++ b/.env @@ -15,11 +15,11 @@ PROJECT="ckan-qld-infrastructure" # Docker Compose project name. All containers will have this name. COMPOSE_PROJECT_NAME="$PROJECT" -# Flag to allow code linting failures. +# Flag to allow code linting failures. 0=enforce, 1=ignore ALLOW_LINT_FAIL=0 -# Flag to allow unit tests failures. +# Flag to allow unit tests failures. 0=enforce, 1=ignore ALLOW_UNIT_FAIL=0 -# Flag to allow BDD tests failures. +# Flag to allow BDD tests failures. 0=enforce, 1=ignore ALLOW_BDD_FAIL=0 diff --git a/.flake8 b/.flake8 index 27ae6526..8412d46e 100644 --- a/.flake8 +++ b/.flake8 @@ -10,6 +10,7 @@ format = pylint # Show the source of errors. show_source = True +statistics = True max-complexity = 10 max-line-length = 127 diff --git a/.github/workflows/test-staging.yml b/.github/workflows/test-staging.yml index ab859c83..3a64676e 100644 --- a/.github/workflows/test-staging.yml +++ b/.github/workflows/test-staging.yml @@ -13,16 +13,18 @@ jobs: strategy: matrix: application: [OpenData, Publications] + python-version: [py2, py3] target_environment: [STAGING] fail-fast: true - name: ${{ matrix.application }} ${{ matrix.target_environment }} infrastructure build + name: ${{ matrix.application }} ${{ matrix.target_environment }} ${{ matrix.python-version }} infrastructure build runs-on: ubuntu-latest container: integratedexperts/ci-builder env: VARS_TYPE: ${{ matrix.application }} DEPLOY_ENV: ${{ matrix.target_environment }} CKAN_REPO: qld-gov-au/ckan + PYTHON_VERSION: ${{ matrix.python-version }} steps: - uses: actions/checkout@v2 @@ -45,6 +47,6 @@ jobs: if: failure() uses: actions/upload-artifact@v2 with: - name: ${{ matrix.application }} ${{ matrix.target_environment }} screenshots + name: ${{ matrix.application }} ${{ matrix.target_environment }} ${{ matrix.python-version }} screenshots path: /tmp/artifacts/behave/screenshots timeout-minutes: 3 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ed40b786..5dd86798 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,16 +10,18 @@ jobs: strategy: matrix: application: [OpenData, Publications] + python-version: [py2, py3] target_environment: [DEV] fail-fast: false - name: ${{ matrix.application }} ${{ matrix.target_environment }} infrastructure build + name: ${{ matrix.application }} ${{ matrix.target_environment }} ${{ matrix.python-version }} infrastructure build runs-on: ubuntu-latest container: integratedexperts/ci-builder env: VARS_TYPE: ${{ matrix.application }} DEPLOY_ENV: ${{ matrix.target_environment }} CKAN_REPO: qld-gov-au/ckan + PYTHON_VERSION: ${{ matrix.python-version }} steps: - uses: actions/checkout@v2 @@ -42,6 +44,6 @@ jobs: if: failure() uses: actions/upload-artifact@v2 with: - name: ${{ matrix.application }} ${{ matrix.target_environment }} screenshots + name: ${{ matrix.application }} ${{ matrix.target_environment }} ${{ matrix.python-version }} screenshots path: /tmp/artifacts/behave/screenshots timeout-minutes: 1 diff --git a/docker-compose.yml b/docker-compose.yml index 878e1e6d..be0306f1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,7 +21,7 @@ services: ckan: build: context: . - dockerfile: .docker/Dockerfile.ckan + dockerfile: .docker/Dockerfile-${PYTHON_VERSION}.ckan args: SITE_URL: "http://${PROJECT}.docker.amazee.io" VARS_TYPE: diff --git a/requirements-dev.txt b/requirements-dev.txt index 3b4638f6..59d7c960 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,6 +8,6 @@ flake8==3.8.3 mock pika>=1.1.0,<1.3.0 pytest-ckan -PyYAML>=1.0 +pyyaml==5.4.1 setuptools==44.1.1 splinter>=0.13.0,<0.17 diff --git a/test/features/data_qld_theme.feature b/test/features/data_qld_theme.feature index e88077cb..e65eb598 100644 --- a/test/features/data_qld_theme.feature +++ b/test/features/data_qld_theme.feature @@ -85,11 +85,6 @@ Feature: Theme customisations Then I should see an element with xpath "//a[contains(@href, '/datastore/dump/') and contains(@href, 'format=json') and contains(string(), 'JSON')]" Then I should see an element with xpath "//a[contains(@href, '/datastore/dump/') and contains(@href, 'format=xml') and contains(string(), 'XML')]" - @Publications - Scenario: Homepage 'Publish in the Gazettes' link is correct - When I go to homepage - Then I should see an element with xpath "//a[@href='https://www.forgov.qld.gov.au/information-and-communication-technology/communication-and-publishing/website-and-digital-publishing/queensland-government-gazette/publish-in-the-gazette' and string()='Publish in the Gazettes']" - @Publications Scenario: Menu items are present and correct When I go to "/dataset" diff --git a/test/features/data_usability_rating.feature b/test/features/data_usability_rating.feature index 7cd5633f..7f69dd2a 100644 --- a/test/features/data_usability_rating.feature +++ b/test/features/data_usability_rating.feature @@ -17,4 +17,5 @@ Feature: Data usability rating | TXT | txt_resource.txt | 1 | | XLS | xls_resource.xls | 2 | | CSV | csv_resource.csv | 3 | + | JSON | json_resource.json | 3 | | RDF | rdf_resource.rdf | 4 | diff --git a/test/features/datarequest.feature b/test/features/datarequest.feature index 1782185c..2e3c1da8 100644 --- a/test/features/datarequest.feature +++ b/test/features/datarequest.feature @@ -17,7 +17,7 @@ Feature: Data Request Scenario: Data requests submitted without a description will produce an error message Given "SysAdmin" as the persona When I log in and go to the data requests page - And I press the element with xpath "//a[contains(@class, 'btn-primary')]" + And I press the element with xpath "//a[contains(@class, 'btn-primary') and contains(string(), 'Add data request')]" And I fill in "title" with "Test data request" And I press the element with xpath "//button[contains(@class, 'btn-primary')]" Then I should see an element with the css selector "div.error-explanation.alert.alert-error" within 2 seconds @@ -60,7 +60,7 @@ Feature: Data Request When I log in and create a datarequest And I press the element with xpath "//a[contains(string(), 'Close')]" And I select "Requestor initiated closure" from "close_circumstance" - And I press the element with xpath "//button[contains(@class, 'btn-danger')]" + And I press the element with xpath "//button[contains(@class, 'btn-danger') and contains(string(), 'Close data request')]" Then I should see an element with xpath "//i[contains(@class, 'icon-lock')]" And I should not see an element with xpath "//a[contains(string(), 'Close')]" @@ -69,7 +69,7 @@ Feature: Data Request When I log in and create a datarequest And I press the element with xpath "//a[contains(string(), 'Close')]" And I select "Requestor initiated closure" from "close_circumstance" - And I press the element with xpath "//button[contains(@class, 'btn-danger') and @name='close']" + And I press the element with xpath "//button[contains(@class, 'btn-danger') and contains(string(), 'Close data request')]" Then I should see an element with xpath "//a[contains(string(), 'Re-open')]" When I press the element with xpath "//a[contains(string(), 'Re-open')]" Then I should see an element with xpath "//i[contains(@class, 'icon-unlock')]" diff --git a/test/features/engagement_reporting.feature b/test/features/engagement_reporting.feature index 33e54c79..3fd6bb79 100644 --- a/test/features/engagement_reporting.feature +++ b/test/features/engagement_reporting.feature @@ -67,7 +67,7 @@ Feature: Engagement Reporting And I press the element with xpath "//a[contains(string(), 'Close')]" And I select "To be released as open data at a later date" from "close_circumstance" And I fill in "approx_publishing_date" with "01/01/1970" - And I press the element with xpath "//button[contains(@class, 'btn-danger') and @name='close']" + And I press the element with xpath "//button[contains(@class, 'btn-danger') and @name='close' and contains(string(), 'Close data request')]" And I should see an element with xpath "//i[contains(@class, 'icon-lock')]" When I go to my reports page diff --git a/test/features/groups.feature b/test/features/groups.feature index 9333bbe1..f48b0f51 100644 --- a/test/features/groups.feature +++ b/test/features/groups.feature @@ -5,7 +5,7 @@ Feature: Group APIs Given "" as the persona When I log in And I view the "silly-walks" group API "including" users - Then I should see an element with xpath "//*[contains(string(), '"success": true,') and contains(string(), '"name": "group_admin"') and contains(string(), '"name": "walker"')]" + Then I should see an element with xpath "//*[contains(string(), '"success": true') and contains(string(), '"name": "group_admin"') and contains(string(), '"name": "walker"')]" Examples: Admins | Persona | @@ -16,7 +16,7 @@ Feature: Group APIs Given "" as the persona When I log in And I view the "silly-walks" group API "including" users - Then I should see an element with xpath "//*[contains(string(), '"success": false,') and contains(string(), 'Authorization Error')]" + Then I should see an element with xpath "//*[contains(string(), '"success": false') and contains(string(), 'Authorization Error')]" Examples: Non-admin users | Persona | @@ -27,7 +27,7 @@ Feature: Group APIs Scenario: Group membership is not accessible anonymously Given "Unauthenticated" as the persona When I view the "silly-walks" group API "including" users - Then I should see an element with xpath "//*[contains(string(), '"success": false,') and contains(string(), 'Authorization Error')]" + Then I should see an element with xpath "//*[contains(string(), '"success": false') and contains(string(), 'Authorization Error')]" @unauthenticated Scenario: Group overview is accessible to everyone @@ -38,4 +38,4 @@ Feature: Group APIs And I should see an element with xpath "//a[contains(@href, '/group/silly-walks')]" When I view the "silly-walks" group API "not including" users - Then I should see an element with xpath "//*[contains(string(), '"success": true,') and contains(string(), '"name": "silly-walks"')]" + Then I should see an element with xpath "//*[contains(string(), '"success": true') and contains(string(), '"name": "silly-walks"')]" diff --git a/test/features/organisations.feature b/test/features/organisations.feature index 60ed3574..e8c2490d 100644 --- a/test/features/organisations.feature +++ b/test/features/organisations.feature @@ -5,7 +5,7 @@ Feature: Organization APIs Given "" as the persona When I log in And I view the "department-of-health" organisation API "including" users - Then I should see an element with xpath "//*[contains(string(), '"success": true,') and contains(string(), '"name": "organisation_admin"') and contains(string(), '"name": "editor"')]" + Then I should see an element with xpath "//*[contains(string(), '"success": true') and contains(string(), '"name": "organisation_admin"') and contains(string(), '"name": "editor"')]" Examples: Admins | Persona | @@ -16,7 +16,7 @@ Feature: Organization APIs Given "" as the persona When I log in And I view the "department-of-health" organisation API "including" users - Then I should see an element with xpath "//*[contains(string(), '"success": false,') and contains(string(), 'Authorization Error')]" + Then I should see an element with xpath "//*[contains(string(), '"success": false') and contains(string(), 'Authorization Error')]" Examples: Non-admin users | Persona | @@ -28,7 +28,7 @@ Feature: Organization APIs Scenario: Organisation membership is not accessible anonymously Given "Unauthenticated" as the persona When I view the "department-of-health" organisation API "including" users - Then I should see an element with xpath "//*[contains(string(), '"success": false,') and contains(string(), 'Authorization Error')]" + Then I should see an element with xpath "//*[contains(string(), '"success": false') and contains(string(), 'Authorization Error')]" @unauthenticated Scenario: Organisation overview is accessible to everyone @@ -39,4 +39,4 @@ Feature: Organization APIs And I should see an element with xpath "//a[contains(@href, '/organization/department-of-health')]" When I view the "department-of-health" organisation API "not including" users - Then I should see an element with xpath "//*[contains(string(), '"success": true,') and contains(string(), '"name": "department-of-health"')]" + Then I should see an element with xpath "//*[contains(string(), '"success": true') and contains(string(), '"name": "department-of-health"')]" diff --git a/test/features/resource_freshness.feature b/test/features/resource_freshness.feature index f9451662..888d5551 100644 --- a/test/features/resource_freshness.feature +++ b/test/features/resource_freshness.feature @@ -30,6 +30,7 @@ Feature: Resource freshness Scenario Outline: As a user with editing privileges, when I set a 'monthly' update frequently, I should still be able to update the dataset via the API Given "" as the persona When I log in + Then I should be able to patch dataset "test-dataset" via the API And I go to "/dataset/edit/test-dataset" And I select "monthly" from "update_frequency" Then I should see "Next update due" diff --git a/test/features/search_facets.feature b/test/features/search_facets.feature index a16bbe5e..c594a09e 100644 --- a/test/features/search_facets.feature +++ b/test/features/search_facets.feature @@ -2,6 +2,6 @@ Feature: Search facets Scenario: When I go to the dataset list page, I can see the 'Data Portals' facet - When I go to "/dataset" + When I go to dataset page Then I should see "Data Portals" And I should see an element with xpath "//a[contains(@href, '?dataset_type=dataset')]/span[contains(@class, 'item-label') and contains(string(), 'data.qld.gov.au')]" diff --git a/test/features/steps/steps.py b/test/features/steps/steps.py index 50ba609c..c313e15a 100644 --- a/test/features/steps/steps.py +++ b/test/features/steps/steps.py @@ -7,6 +7,7 @@ import quopri import requests import uuid +import six @step(u'I get the current URL') @@ -97,6 +98,7 @@ def add_resource(context, name, url): And I fill in "name" with "{name}" And I fill in "description" with "description" And I fill in "size" with "1024" if present + And I execute the script "document.getElementById('field-format').value='HTML'" And I press the element with xpath "//form[contains(@class, 'resource-form')]//button[contains(@class, 'btn-primary')]" """.format(name=name, url=url)) @@ -244,7 +246,11 @@ def filter_contents(mail): payload_bytes = quopri.decodestring(payload) if len(payload_bytes) > 0: payload_bytes += b'=' # do fix the padding error issue - decoded_payload = payload_bytes.decode('base64') + if six.PY2: + decoded_payload = payload_bytes.decode('base64') + else: + import base64 + decoded_payload = six.ensure_text(base64.b64decode(six.ensure_binary(payload_bytes))) print('decoded_payload: ', decoded_payload) return text in decoded_payload and (not text2 or text2 in decoded_payload) diff --git a/test/features/user_creation.feature b/test/features/user_creation.feature index eda029ab..e29db5b8 100644 --- a/test/features/user_creation.feature +++ b/test/features/user_creation.feature @@ -1,6 +1,14 @@ @user_creation Feature: User creation + Scenario: SysAdmin can create 'Excluded display name words' in ckan admin config + Given "SysAdmin" as the persona + When I log in + Then I go to "/ckan-admin/config" + Then I should see "Excluded display name words" + Then I fill in "ckanext.data_qld.excluded_display_name_words" with "gov" + And I press the element with xpath "//button[contains(@class, 'btn-primary')]" + Scenario: SysAdmin create a new user to the site. Given "SysAdmin" as the persona When I log in diff --git a/test/features/users.feature b/test/features/users.feature index dfe44d3c..39236cd1 100644 --- a/test/features/users.feature +++ b/test/features/users.feature @@ -31,7 +31,7 @@ Feature: User APIs Given "" as the persona When I log in And I go to the user list API - Then I should see an element with xpath "//*[contains(string(), '"success": true,') and contains(string(), '"name": "admin"')]" + Then I should see an element with xpath "//*[contains(string(), '"success": true') and contains(string(), '"name": "admin"')]" Examples: Admins | Persona | @@ -43,19 +43,19 @@ Feature: User APIs Given "Publisher" as the persona When I log in And I go to the user list API - Then I should see an element with xpath "//*[contains(string(), '"success": false,') and contains(string(), 'Authorization Error')]" + Then I should see an element with xpath "//*[contains(string(), '"success": false') and contains(string(), 'Authorization Error')]" @unauthenticated Scenario: User list is not accessible anonymously Given "Unauthenticated" as the persona When I go to the user list API - Then I should see an element with xpath "//*[contains(string(), '"success": false,') and contains(string(), 'requires an authenticated user')]" + Then I should see an element with xpath "//*[contains(string(), '"success": false') and contains(string(), 'requires an authenticated user')]" Scenario Outline: User detail is accessible to admins Given "" as the persona When I log in And I go to the "admin" user API - Then I should see an element with xpath "//*[contains(string(), '"success": true,') and contains(string(), '"name": "admin"')]" + Then I should see an element with xpath "//*[contains(string(), '"success": true') and contains(string(), '"name": "admin"')]" Examples: Admins | Persona | @@ -67,19 +67,19 @@ Feature: User APIs Given "Publisher" as the persona When I log in And I go to the "editor" user API - Then I should see an element with xpath "//*[contains(string(), '"success": true,') and contains(string(), '"name": "editor"')]" + Then I should see an element with xpath "//*[contains(string(), '"success": true') and contains(string(), '"name": "editor"')]" Scenario: Non-self user detail is not accessible to non-admins Given "Publisher" as the persona When I log in And I go to the "admin" user API - Then I should see an element with xpath "//*[contains(string(), '"success": false,') and contains(string(), 'Authorization Error')]" + Then I should see an element with xpath "//*[contains(string(), '"success": false') and contains(string(), 'Authorization Error')]" @unauthenticated Scenario: User detail is not accessible anonymously Given "Unauthenticated" as the persona When I go to the "editor" user API - Then I should see an element with xpath "//*[contains(string(), '"success": false,') and contains(string(), 'Authorization Error')]" + Then I should see an element with xpath "//*[contains(string(), '"success": false') and contains(string(), 'Authorization Error')]" Scenario Outline: User profile page is accessible to admins Given "" as the persona diff --git a/test/fixtures/json_resource.json b/test/fixtures/json_resource.json new file mode 100644 index 00000000..1e0ca3d4 --- /dev/null +++ b/test/fixtures/json_resource.json @@ -0,0 +1,3 @@ +{ + "foo": "baz" +} diff --git a/vars/CKAN-Stack.var.yml b/vars/CKAN-Stack.var.yml index d625c46e..b610a485 100644 --- a/vars/CKAN-Stack.var.yml +++ b/vars/CKAN-Stack.var.yml @@ -5,7 +5,7 @@ NonProductionAnalyticsId: "{{ lookup('aws_ssm', '/config/CKAN/GaIdNonProduction' ProductionAnalyticsId: "{{ lookup('aws_ssm', '/config/CKAN/GaIdProduction', region=region) }}" solr7: "http://archive.apache.org/dist/lucene/solr/7.7.2/solr-7.7.2.zip" -ckan_tag: "ckan-2.9.5-qgov.7" +ckan_tag: "ckan-2.9.5-qgov.8" ckan_qgov_branch: "qgov-master-2.9.5" common_stack: &common_stack @@ -45,7 +45,7 @@ common_stack: &common_stack EnableDataStore: "{{ enable_datastore | default('no') }}" SSMKey: "{{ SSMKey | default('') }}" DefaultEC2Key: "{{ lookup('aws_ssm', '/config/CKAN/ec2KeyPair', region=region) }}" - CookbookRevision: "{{ CookbookRevision | default('5.2.14') }}" + CookbookRevision: "{{ CookbookRevision | default('5.3.0') }}" LogBucketName: "{{ lookup('aws_ssm', '/config/CKAN/s3LogsBucket', region=region) }}" AttachmentsBucketName: "{{ lookup('aws_ssm', '/config/CKAN/' + Environment + '/app/' + service_name_lower + '/s3AttachmentBucket', region=region) }}" #/config/CKAN/PROD/app/opendata/s3AttachmentBucket SolrSource: "{{ solr7 }}" diff --git a/vars/shared-CKANTest.var.yml b/vars/shared-CKANTest.var.yml index db103142..67845e9e 100644 --- a/vars/shared-CKANTest.var.yml +++ b/vars/shared-CKANTest.var.yml @@ -22,8 +22,8 @@ extensions: shortname: "ckanext-qgov" description: "CKAN Extension for Queensland Government Open Data" type: "git" - url: "https://github.com/qld-gov-au/ckan-ex-qgov.git" - version: "5.0.2" + url: "https://github.com/qld-gov-au/ckanext-qgov.git" + version: "5.0.4" CKANExtS3Filestore: &CKANExtS3Filestore name: "ckanext-s3filestore-{{ Environment }}" @@ -31,7 +31,7 @@ extensions: description: "CKAN Extension to keep uploaded files in S3" type: "git" url: "https://github.com/qld-gov-au/ckanext-s3filestore.git" - version: "0.7.7-qgov.2" + version: "0.7.7-qgov.3" CKANExtSSMConfig: &CKANExtSSMConfig name: "ckanext-ssm-config-{{ Environment }}" @@ -47,7 +47,7 @@ extensions: description: "CKAN Extension for DCAT" type: "git" url: "https://github.com/ckan/ckanext-dcat.git" - version: "v1.1.3" + version: "v1.2.0" CKANExtScheming: &CKANExtScheming name: "ckanext-scheming-{{ Environment }}" @@ -63,7 +63,7 @@ extensions: description: "CKAN Extension for validating Data Packages" type: "git" url: "https://github.com/qld-gov-au/ckanext-validation.git" - version: "v0.0.8-qgov.4" + version: "v0.0.8-qgov.5" CKANExtDataQld: &CKANExtDataQld name: "ckanext-data-qld-{{ Environment }}" @@ -71,7 +71,7 @@ extensions: description: "CKAN Extension for Queensland Government Open Data" type: "git" url: "https://github.com/qld-gov-au/ckanext-data-qld.git" - version: "6.3.4" + version: "6.3.5" CKANExtODICertificates: &CKANExtODICertificates name: "ckanext-odi-certificates-{{ Environment }}" @@ -87,7 +87,7 @@ extensions: description: "CKAN Extension for Data Requests" type: "git" url: "https://github.com/qld-gov-au/ckanext-datarequests.git" - version: "2.2.1-qgov" + version: "2.2.1-qgov.3" CKANExtYTP: &CKANExtYTP name: "ckanext-ytp-comments-{{ Environment }}" @@ -95,7 +95,7 @@ extensions: description: "CKAN Extension for YTP Comments" type: "git" url: "https://github.com/qld-gov-au/ckanext-ytp-comments.git" - version: "2.5.0-qgov.2" + version: "2.5.0-qgov.8" CKANExtHarvest: &CKANExtHarvest name: "ckanext-harvest-{{ Environment }}" @@ -111,7 +111,7 @@ extensions: description: "CKAN Extension for Data Qld Harvesting Enhancements for Geoscience" type: "git" url: "https://github.com/qld-gov-au/ckanext-harvester-data-qld-geoscience" - version: "v0.0.5" + version: "v0.0.6" CKANExtReport: &CKANExtReport name: "ckanext-report-{{ Environment }}" @@ -119,7 +119,7 @@ extensions: description: "CKAN Extension for Reporting" type: "git" url: "https://github.com/qld-gov-au/ckanext-report.git" - version: "0.3" + version: "0.3.1-qgov.1" CKANExtArchiver: &CKANExtArchiver name: "ckanext-archiver-{{ Environment }}" @@ -127,7 +127,7 @@ extensions: description: "CKAN Extension for Archiving needed for ckanext-qa" type: "git" url: "https://github.com/qld-gov-au/ckanext-archiver.git" - version: "2.1.1-qgov.8" + version: "2.1.1-qgov.11" CKANExtQa: &CKANExtQa name: "ckanext-qa-{{ Environment }}" @@ -135,7 +135,7 @@ extensions: description: "CKAN Extension for Quality Assurance" type: "git" url: "https://github.com/qld-gov-au/ckanext-qa.git" - version: "2.0.3-qgov.4" + version: "2.0.3-qgov.6" CKANExtResourceTypeValidation: &CKANExtResourceTypeValidation name: "ckanext-resource-type-validation-{{ Environment }}" @@ -143,7 +143,7 @@ extensions: description: "CKAN Extension for checking uploaded resource types" type: "git" url: "https://github.com/qld-gov-au/ckanext-resource-type-validation.git" - version: "1.0.2" + version: "1.0.3" CKANExtCSRFFilter: &CKANExtCSRFFilter name: "ckanext-csrf-filter-{{ Environment }}" @@ -151,7 +151,7 @@ extensions: description: "CKAN Extension for preventing Cross-Site Request Forgery attacks" type: "git" url: "https://github.com/qld-gov-au/ckanext-csrf-filter.git" - version: "1.1.4" + version: "1.1.5" PROD: <<: *default_extensions @@ -186,7 +186,7 @@ extensions: CKANExtHarvestDataQldGeoScience: <<: *CKANExtHarvestDataQldGeoScience - version: "develop" + version: "master" CKANExtODICertificates: <<: *CKANExtODICertificates diff --git a/vars/shared-OpenData.var.yml b/vars/shared-OpenData.var.yml index a6b4c810..876f3fae 100644 --- a/vars/shared-OpenData.var.yml +++ b/vars/shared-OpenData.var.yml @@ -22,8 +22,8 @@ extensions: shortname: "ckanext-qgov" description: "CKAN Extension for Queensland Government Open Data" type: "git" - url: "https://github.com/qld-gov-au/ckan-ex-qgov.git" - version: "5.0.2" + url: "https://github.com/qld-gov-au/ckanext-qgov.git" + version: "5.0.4" CKANExtS3Filestore: &CKANExtS3Filestore name: "ckanext-s3filestore-{{ Environment }}" @@ -31,7 +31,7 @@ extensions: description: "CKAN Extension to keep uploaded files in S3" type: "git" url: "https://github.com/qld-gov-au/ckanext-s3filestore.git" - version: "0.7.7-qgov.2" + version: "0.7.7-qgov.3" CKANExtSSMConfig: &CKANExtSSMConfig name: "ckanext-ssm-config-{{ Environment }}" @@ -47,7 +47,7 @@ extensions: description: "CKAN Extension for DCAT" type: "git" url: "https://github.com/ckan/ckanext-dcat.git" - version: "v1.1.3" + version: "v1.2.0" CKANExtScheming: &CKANExtScheming name: "ckanext-scheming-{{ Environment }}" @@ -63,7 +63,7 @@ extensions: description: "CKAN Extension for validating Data Packages" type: "git" url: "https://github.com/qld-gov-au/ckanext-validation.git" - version: "v0.0.8-qgov.4" + version: "v0.0.8-qgov.5" CKANExtDataQld: &CKANExtDataQld name: "ckanext-data-qld-{{ Environment }}" @@ -71,7 +71,7 @@ extensions: description: "CKAN Extension for Queensland Government Open Data" type: "git" url: "https://github.com/qld-gov-au/ckanext-data-qld.git" - version: "6.3.4" + version: "6.3.5" CKANExtODICertificates: &CKANExtODICertificates name: "ckanext-odi-certificates-{{ Environment }}" @@ -87,7 +87,7 @@ extensions: description: "CKAN Extension for Data Requests" type: "git" url: "https://github.com/qld-gov-au/ckanext-datarequests.git" - version: "2.2.1-qgov" + version: "2.2.1-qgov.3" CKANExtYTP: &CKANExtYTP name: "ckanext-ytp-comments-{{ Environment }}" @@ -95,7 +95,7 @@ extensions: description: "CKAN Extension for YTP Comments" type: "git" url: "https://github.com/qld-gov-au/ckanext-ytp-comments.git" - version: "2.5.0-qgov.2" + version: "2.5.0-qgov.8" CKANExtHarvest: &CKANExtHarvest name: "ckanext-harvest-{{ Environment }}" @@ -111,8 +111,7 @@ extensions: description: "CKAN Extension for Data Qld Harvesting Enhancements" type: "git" url: "https://github.com/qld-gov-au/ckanext-harvester-data-qld-geoscience" - version: "v0.0.5" - + version: "v0.0.6" CKANExtReport: &CKANExtReport name: "ckanext-report-{{ Environment }}" @@ -120,7 +119,7 @@ extensions: description: "CKAN Extension for Reporting" type: "git" url: "https://github.com/qld-gov-au/ckanext-report.git" - version: "0.3" + version: "0.3.1-qgov.1" CKANExtArchiver: &CKANExtArchiver name: "ckanext-archiver-{{ Environment }}" @@ -128,7 +127,7 @@ extensions: description: "CKAN Extension for Archiving needed for ckanext-qa" type: "git" url: "https://github.com/qld-gov-au/ckanext-archiver.git" - version: "2.1.1-qgov.8" + version: "2.1.1-qgov.11" CKANExtQa: &CKANExtQa name: "ckanext-qa-{{ Environment }}" @@ -136,7 +135,7 @@ extensions: description: "CKAN Extension for Quality Assurance" type: "git" url: "https://github.com/qld-gov-au/ckanext-qa.git" - version: "2.0.3-qgov.4" + version: "2.0.3-qgov.6" CKANExtResourceTypeValidation: &CKANExtResourceTypeValidation name: "ckanext-resource-type-validation-{{ Environment }}" @@ -144,7 +143,7 @@ extensions: description: "CKAN Extension for checking uploaded resource types" type: "git" url: "https://github.com/qld-gov-au/ckanext-resource-type-validation.git" - version: "1.0.2" + version: "1.0.3" CKANExtCSRFFilter: &CKANExtCSRFFilter name: "ckanext-csrf-filter-{{ Environment }}" @@ -152,7 +151,7 @@ extensions: description: "CKAN Extension for preventing Cross-Site Request Forgery attacks" type: "git" url: "https://github.com/qld-gov-au/ckanext-csrf-filter.git" - version: "1.1.4" + version: "1.1.5" PROD: <<: *default_extensions @@ -225,6 +224,10 @@ extensions: <<: *CKANExtCSRFFilter version: "develop" + CKANExtHarvestDataQldGeoScience: + <<: *CKANExtHarvestDataQldGeoScience + version: "develop" + basic_facts: - Environment: PROD PublicStackZoneTLD: "{{ public_tld }}" diff --git a/vars/shared-Publications.var.yml b/vars/shared-Publications.var.yml index f83daa5e..7ab3ebc6 100644 --- a/vars/shared-Publications.var.yml +++ b/vars/shared-Publications.var.yml @@ -13,8 +13,8 @@ extensions: shortname: "ckanext-qgov" description: "CKAN Extension for Queensland Government sites" type: "git" - url: "https://github.com/qld-gov-au/ckan-ex-qgov.git" - version: "5.0.2" + url: "https://github.com/qld-gov-au/ckanext-qgov.git" + version: "5.0.4" CKANExtS3Filestore: &CKANExtS3Filestore name: "ckanext-s3filestore-{{ Environment }}" @@ -22,7 +22,7 @@ extensions: description: "CKAN Extension to keep uploaded files in S3" type: "git" url: "https://github.com/qld-gov-au/ckanext-s3filestore.git" - version: "0.7.7-qgov.2" + version: "0.7.7-qgov.3" CKANExtSSMConfig: &CKANExtSSMConfig name: "ckanext-ssm-config-{{ Environment }}" @@ -38,7 +38,7 @@ extensions: type: "git" description: "CKAN Extension for Queensland Government Publications theme" url: "https://github.com/qld-gov-au/ckanext-publications-qld-theme.git" - version: "1.1.5" + version: "1.1.6" CKANExtResourceTypeValidation: &CKANExtResourceTypeValidation name: "ckanext-resource-type-validation-{{ Environment }}" @@ -46,7 +46,7 @@ extensions: description: "CKAN Extension for checking uploaded resource types" type: "git" url: "https://github.com/qld-gov-au/ckanext-resource-type-validation.git" - version: "1.0.2" + version: "1.0.3" CKANExtCSRFFilter: &CKANExtCSRFFilter name: "ckanext-csrf-filter-{{ Environment }}" @@ -54,7 +54,7 @@ extensions: description: "CKAN Extension for preventing Cross-Site Request Forgery attacks" type: "git" url: "https://github.com/qld-gov-au/ckanext-csrf-filter.git" - version: "1.1.4" + version: "1.1.5" PROD: <<: *default_extensions