diff --git a/.ahoy.yml b/.ahoy.yml index ca64cc6d..bcfb53ad 100644 --- a/.ahoy.yml +++ b/.ahoy.yml @@ -38,7 +38,7 @@ commands: ahoy title "Building and starting Docker containers" sh bin/docker-compose.sh up -d "$@" echo "Initialising database schema" - ahoy cli '$APP_DIR/bin/init.sh' + ahoy cli '"${APP_DIR}"/bin/init.sh' echo "Waiting for containers to start listening..." ahoy cli "dockerize -wait tcp://ckan:5000 -timeout 1m" if sh bin/docker-compose.sh logs | grep -q "\[Error\]"; then exit 1; fi @@ -79,9 +79,9 @@ commands: cmd: | CKAN_CONTAINER=$(sh bin/docker-compose.sh ps -q ckan) if [ "${#}" -ne 0 \]; then - docker exec $CKAN_CONTAINER sh -c '. ${APP_DIR}/bin/activate; cd $APP_DIR;'" $*" + docker exec $CKAN_CONTAINER sh -c '. "${APP_DIR}"/bin/activate; cd $APP_DIR;'" $*" else - docker exec $CKAN_CONTAINER sh -c '. ${APP_DIR}/bin/activate && cd $APP_DIR && sh' + docker exec $CKAN_CONTAINER sh -c '. "${APP_DIR}"/bin/activate && cd $APP_DIR && sh' fi doctor: @@ -93,7 +93,7 @@ commands: cmd: | ahoy title "Installing a fresh site" ahoy start-ckan-job-workers - ahoy cli '$APP_DIR/bin/init.sh && $APP_DIR/bin/create-test-data.sh' + ahoy cli '"${APP_DIR}"/bin/init.sh && "${APP_DIR}"/bin/create-test-data.sh' ahoy stop-ckan-job-workers clean: @@ -130,13 +130,13 @@ commands: cmd: | docker cp . $(sh bin/docker-compose.sh ps -q ckan):/srv/app/ docker cp bin/ckan_cli $(sh bin/docker-compose.sh ps -q ckan):/usr/bin/ - ahoy cli 'chmod -v u+x /usr/bin/ckan_cli $APP_DIR/bin/*; cp -v .docker/test-$VARS_TYPE.ini $CKAN_INI; $APP_DIR/bin/process-config.sh' + ahoy cli 'chmod -v u+x /usr/bin/ckan_cli "${APP_DIR}"/bin/*; cp -v .docker/test-$VARS_TYPE.ini $CKAN_INI; "${APP_DIR}"/bin/process-config.sh' test-unit: usage: Run unit tests. cmd: | ahoy title 'Run unit tests' - ahoy cli 'pytest --ckan-ini=${CKAN_INI} $APP_DIR/test' || \ + ahoy cli 'pytest --ckan-ini=${CKAN_INI} "${APP_DIR}"/test' || \ [ "${ALLOW_UNIT_FAIL:-0}" -eq 1 ] test-bdd: @@ -172,7 +172,7 @@ commands: usage: Starts email mock server used for email BDD tests cmd: | ahoy title 'Starting mailmock' - ahoy cli 'mailmock -p 8025 -o ${APP_DIR}/test/emails' # for debugging mailmock email output remove --no-stdout + ahoy cli 'mailmock -p 8025 -o "${APP_DIR}"/test/emails' # for debugging mailmock email output remove --no-stdout stop-mailmock: usage: Stops email mock server used for email BDD tests diff --git a/.docker/Dockerfile-template.ckan b/.docker/Dockerfile-template.ckan index 2a7f8ea0..d813985f 100644 --- a/.docker/Dockerfile-template.ckan +++ b/.docker/Dockerfile-template.ckan @@ -1,4 +1,8 @@ -FROM openknowledge/ckan-dev:{CKAN_VERSION} +FROM ckan/ckan-dev:{CKAN_VERSION} + +# swap between root and unprivileged user +ARG ORIGINAL_USER +RUN ORIGINAL_USER=$(id -un) ARG SITE_URL=http://ckan:5000/ ENV DEPLOY_ENV={DEPLOY_ENV} @@ -10,9 +14,20 @@ ENV PYTHON={PYTHON} WORKDIR "${APP_DIR}" -ENV DOCKERIZE_VERSION v0.6.1 -RUN apk add --no-cache build-base \ - && curl -sL https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \ +COPY .docker/test-$VARS_TYPE.ini $CKAN_INI + +COPY . "${APP_DIR}"/ + +USER root + +COPY bin/ckan_cli /usr/bin/ + +COPY vars/shared-{VARS_TYPE}.var.yml "${APP_DIR}"/bin/extensions.yml + +RUN chmod +x "${APP_DIR}"/bin/*.sh /usr/bin/ckan_cli + +ENV DOCKERIZE_VERSION=v0.6.1 +RUN wget -O - https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \ | tar -C /usr/local/bin -xzvf - # Install CKAN. @@ -24,17 +39,11 @@ RUN cd $SRC_DIR/ckan \ && git reset --hard && git clean -f \ && git checkout '{CKAN_GIT_VERSION}' -COPY .docker/test-$VARS_TYPE.ini $CKAN_INI - -COPY . ${APP_DIR}/ - -COPY bin/ckan_cli /usr/bin/ - -COPY vars/shared-{VARS_TYPE}.var.yml $APP_DIR/bin/extensions.yml +RUN which ps || apt-get install -y procps -RUN chmod +x ${APP_DIR}/bin/*.sh /usr/bin/ckan_cli +USER "$ORIGINAL_USER" # Init current extension. -RUN ${APP_DIR}/bin/init-ext.sh +RUN "${APP_DIR}"/bin/init-ext.sh CMD ["/srv/app/bin/serve.sh"] diff --git a/bin/create-test-data-OpenData.sh b/bin/create-test-data-OpenData.sh index 53974369..5de30d5b 100755 --- a/bin/create-test-data-OpenData.sh +++ b/bin/create-test-data-OpenData.sh @@ -7,36 +7,30 @@ set -ex ## # BEGIN: Add sysadmin config values. # -curl -LsH "Authorization: ${API_KEY}" \ - --data '{ +api_call '{ "ckan.comments.profanity_list": "", "ckan.datarequests.closing_circumstances": "Released as open data|nominate_dataset\r\nOpen dataset already exists|nominate_dataset\r\nPartially released|nominate_dataset\r\nTo be released as open data at a later date|nominate_approximate_date\r\nData openly available elsewhere\r\nNot suitable for release as open data\r\nRequested data not available/cannot be compiled\r\nRequestor initiated closure", "ckanext.data_qld.resource_formats": "CSV\r\nHTML\r\nJSON\r\nRDF\r\nTXT\r\nXLS" - }' \ - ${CKAN_ACTION_URL}/config_option_update + }' config_option_update ## # END. # # Create private test dataset with our standard fields -curl -LsH "Authorization: ${API_KEY}" \ - --data '{"name": "test-dataset", "owner_org": "'"${TEST_ORG_ID}"'", "private": true, +api_call '{"name": "test-dataset", "owner_org": "'"${TEST_ORG_ID}"'", "private": true, "update_frequency": "monthly", "author_email": "admin@localhost", "version": "1.0", "license_id": "other-open", "data_driven_application": "NO", "security_classification": "PUBLIC", -"notes": "private test", "de_identified_data": "NO"}' \ - ${CKAN_ACTION_URL}/package_create +"notes": "private test", "de_identified_data": "NO"}' package_create # Create public test dataset with our standard fields -curl -LsH "Authorization: ${API_KEY}" \ - --data '{"name": "public-test-dataset", "owner_org": "'"${TEST_ORG_ID}"'", +api_call '{"name": "public-test-dataset", "owner_org": "'"${TEST_ORG_ID}"'", "update_frequency": "monthly", "author_email": "admin@example.com", "version": "1.0", "license_id": "other-open", "data_driven_application": "NO", "security_classification": "PUBLIC", "notes": "public test", "de_identified_data": "NO", "resources": [ {"name": "test-resource", "description": "Test resource description", "url": "https://example.com/foo", "format": "HTML", "size": 1024} -]}' \ - ${CKAN_ACTION_URL}/package_create +]}' package_create # Populate Archiver data for test dataset ckan_cli archiver update-test public-test-dataset @@ -56,36 +50,26 @@ add_user_if_needed dr_editor "Data Request Editor" dr_editor@localhost echo "Creating ${DR_ORG_TITLE} Organisation:" DR_ORG=$( \ - curl -LsH "Authorization: ${API_KEY}" \ - --data '{"name": "'"${DR_ORG_NAME}"'", "title": "'"${DR_ORG_TITLE}"'"}' \ - ${CKAN_ACTION_URL}/organization_create + api_call '{"name": "'"${DR_ORG_NAME}"'", "title": "'"${DR_ORG_TITLE}"'"}' organization_create ) DR_ORG_ID=$(echo $DR_ORG | $PYTHON $APP_DIR/bin/extract-id.py) echo "Assigning test users to ${DR_ORG_TITLE} Organisation:" -curl -LsH "Authorization: ${API_KEY}" \ - --data '{"id": "'"${DR_ORG_ID}"'", "object": "dr_admin", "object_type": "user", "capacity": "admin"}' \ - ${CKAN_ACTION_URL}/member_create +api_call '{"id": "'"${DR_ORG_ID}"'", "object": "dr_admin", "object_type": "user", "capacity": "admin"}' member_create -curl -LsH "Authorization: ${API_KEY}" \ - --data '{"id": "'"${DR_ORG_ID}"'", "object": "dr_editor", "object_type": "user", "capacity": "editor"}' \ - ${CKAN_ACTION_URL}/member_create +api_call '{"id": "'"${DR_ORG_ID}"'", "object": "dr_editor", "object_type": "user", "capacity": "editor"}' member_create echo "Creating test dataset for data request organisation:" -curl -LsH "Authorization: ${API_KEY}" \ - --data '{"name": "data_request_dataset", "title": "Dataset for data requests", "owner_org": "'"${DR_ORG_ID}"'", +api_call '{"name": "data_request_dataset", "title": "Dataset for data requests", "owner_org": "'"${DR_ORG_ID}"'", "update_frequency": "near-realtime", "author_email": "dr_admin@localhost", "version": "1.0", "license_id": "cc-by-4", -"data_driven_application": "NO", "security_classification": "PUBLIC", "notes": "test", "de_identified_data": "NO"}'\ - ${CKAN_ACTION_URL}/package_create +"data_driven_application": "NO", "security_classification": "PUBLIC", "notes": "test", "de_identified_data": "NO"}'package_create echo "Creating test Data Request:" -curl -LsH "Authorization: ${API_KEY}" \ - --data '{"title": "Test Request", "description": "This is an example", "organization_id": "'"${TEST_ORG_ID}"'"}' \ - ${CKAN_ACTION_URL}/create_datarequest +api_call '{"title": "Test Request", "description": "This is an example", "organization_id": "'"${TEST_ORG_ID}"'"}' create_datarequest ## # END. @@ -105,24 +89,18 @@ add_user_if_needed report_admin "Reporting Admin" report_admin@localhost echo "Creating ${REPORT_ORG_TITLE} Organisation:" REPORT_ORG=$( \ - curl -LsH "Authorization: ${API_KEY}" \ - --data '{"name": "'"${REPORT_ORG_NAME}"'", "title": "'"${REPORT_ORG_TITLE}"'"}' \ - ${CKAN_ACTION_URL}/organization_create + api_call '{"name": "'"${REPORT_ORG_NAME}"'", "title": "'"${REPORT_ORG_TITLE}"'"}' organization_create ) REPORT_ORG_ID=$(echo $REPORT_ORG | $PYTHON $APP_DIR/bin/extract-id.py) echo "Assigning admin user to ${REPORT_ORG_TITLE} Organisation:" -curl -LsH "Authorization: ${API_KEY}" \ - --data '{"id": "'"${REPORT_ORG_ID}"'", "object": "report_admin", "object_type": "user", "capacity": "admin"}' \ - ${CKAN_ACTION_URL}/member_create +api_call '{"id": "'"${REPORT_ORG_ID}"'", "object": "report_admin", "object_type": "user", "capacity": "admin"}' member_create echo "Creating test Data Request for reporting:" -curl -LsH "Authorization: ${API_KEY}" \ - --data '{"title": "Reporting Request", "description": "Data Request for reporting", "organization_id": "'"${REPORT_ORG_ID}"'"}' \ - ${CKAN_ACTION_URL}/create_datarequest +api_call '{"title": "Reporting Request", "description": "Data Request for reporting", "organization_id": "'"${REPORT_ORG_ID}"'"}' create_datarequest ## # END. diff --git a/bin/create-test-data-Publications.sh b/bin/create-test-data-Publications.sh index f5f17a19..0f29f659 100755 --- a/bin/create-test-data-Publications.sh +++ b/bin/create-test-data-Publications.sh @@ -6,22 +6,16 @@ set -e set -x # Create publishing standards dataset -curl -LsH "Authorization: ${API_KEY}" \ - --data '{"name": "publishing-standards-publications-qld-gov-au", "owner_org": "'"${TEST_ORG_ID}"'"}' \ - ${CKAN_ACTION_URL}/package_create +api_call '{"name": "publishing-standards-publications-qld-gov-au", "owner_org": "'"${TEST_ORG_ID}"'"}' package_create # Create private test dataset with our standard fields -curl -LsH "Authorization: ${API_KEY}" \ - --data '{"name": "test-dataset", "owner_org": "'"${TEST_ORG_ID}"'", "private": true, -"notes": "private test"}' \ - ${CKAN_ACTION_URL}/package_create +api_call '{"name": "test-dataset", "owner_org": "'"${TEST_ORG_ID}"'", "private": true, +"notes": "private test"}' package_create # Create public test dataset with our standard fields -curl -LsH "Authorization: ${API_KEY}" \ - --data '{"name": "public-test-dataset", "owner_org": "'"${TEST_ORG_ID}"'", +api_call '{"name": "public-test-dataset", "owner_org": "'"${TEST_ORG_ID}"'", "notes": "public test", "resources": [ {"name": "test-resource", "description": "Test resource description", "url": "https://example.com", "format": "HTML", "size": 1024} -]}' \ - ${CKAN_ACTION_URL}/package_create +]}' package_create diff --git a/bin/create-test-data.sh b/bin/create-test-data.sh index 7511f783..0ff36faf 100755 --- a/bin/create-test-data.sh +++ b/bin/create-test-data.sh @@ -9,7 +9,7 @@ CKAN_USER_NAME="${CKAN_USER_NAME:-admin}" CKAN_DISPLAY_NAME="${CKAN_DISPLAY_NAME:-Administrator}" CKAN_USER_EMAIL="${CKAN_USER_EMAIL:-admin@localhost}" -. ${APP_DIR}/bin/activate +. "${APP_DIR}"/bin/activate add_user_if_needed () { echo "Adding user '$2' ($1) with email address [$3]" @@ -19,6 +19,10 @@ add_user_if_needed () { password="${4:-Password123!}" } +api_call () { + wget -O - --header="Authorization: ${API_KEY}" --post-data "$1" ${CKAN_ACTION_URL}/$2 +} + add_user_if_needed "$CKAN_USER_NAME" "$CKAN_DISPLAY_NAME" "$CKAN_USER_EMAIL" ckan_cli sysadmin add "${CKAN_USER_NAME}" @@ -33,9 +37,7 @@ fi # echo "Adding sysadmin config:" -curl -LsH "Authorization: ${API_KEY}" \ - --data '{"ckanext.data_qld.excluded_display_name_words": "gov"}' \ - ${CKAN_ACTION_URL}/config_option_update +api_call '{"ckanext.data_qld.excluded_display_name_words": "gov"}' config_option_update ## # END. @@ -57,27 +59,20 @@ add_user_if_needed test_org_member "Test Member" test_org_member@localhost echo "Creating ${TEST_ORG_TITLE} organisation:" TEST_ORG=$( \ - curl -LsH "Authorization: ${API_KEY}" \ - --data '{"name": "'"${TEST_ORG_NAME}"'", "title": "'"${TEST_ORG_TITLE}"'", - "description": "Organisation for testing issues"}' \ - ${CKAN_ACTION_URL}/organization_create + api_call '{"name": "'"${TEST_ORG_NAME}"'", "title": "'"${TEST_ORG_TITLE}"'", + "description": "Organisation for testing issues"}' organization_create ) -TEST_ORG_ID=$(echo $TEST_ORG | $PYTHON ${APP_DIR}/bin/extract-id.py) +TEST_ORG_ID=$(echo $TEST_ORG | $PYTHON "${APP_DIR}"/bin/extract-id.py) echo "Assigning test users to '${TEST_ORG_TITLE}' organisation (${TEST_ORG_ID}):" -curl -LsH "Authorization: ${API_KEY}" \ - --data '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_admin", "object_type": "user", "capacity": "admin"}' \ - ${CKAN_ACTION_URL}/member_create +api_call '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_admin", "object_type": "user", "capacity": "admin"}' member_create + +api_call '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_editor", "object_type": "user", "capacity": "editor"}' member_create -curl -LsH "Authorization: ${API_KEY}" \ - --data '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_editor", "object_type": "user", "capacity": "editor"}' \ - ${CKAN_ACTION_URL}/member_create +api_call '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_member", "object_type": "user", "capacity": "member"}' member_create -curl -LsH "Authorization: ${API_KEY}" \ - --data '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_member", "object_type": "user", "capacity": "member"}' \ - ${CKAN_ACTION_URL}/member_create ## # END. # @@ -85,9 +80,7 @@ curl -LsH "Authorization: ${API_KEY}" \ # Creating test data hierarchy which creates organisations assigned to datasets echo "Creating food-standards-agency organisation:" organisation_create=$( \ - curl -LsH "Authorization: ${API_KEY}" \ - --data "name=food-standards-agency&title=Food%20Standards%20Agency" \ - ${CKAN_ACTION_URL}/organization_create + api_call "name=food-standards-agency&title=Food%20Standards%20Agency" organization_create ) echo ${organisation_create} @@ -96,25 +89,19 @@ add_user_if_needed walker "Walker" walker@localhost echo "Creating non-organisation group:" group_create=$( \ - curl -LsH "Authorization: ${API_KEY}" \ - --data '{"name": "silly-walks", "title": "Silly walks", "description": "The Ministry of Silly Walks"}' \ - ${CKAN_ACTION_URL}/group_create + api_call '{"name": "silly-walks", "title": "Silly walks", "description": "The Ministry of Silly Walks"}' group_create ) echo ${group_create} echo "Updating group_admin to have admin privileges in the silly-walks group:" group_admin_update=$( \ - curl -LsH "Authorization: ${API_KEY}" \ - --data '{"id": "silly-walks", "username": "group_admin", "role": "admin"}' \ - ${CKAN_ACTION_URL}/group_member_create + api_call '{"id": "silly-walks", "username": "group_admin", "role": "admin"}' group_member_create ) echo ${group_admin_update} echo "Updating walker to have editor privileges in the silly-walks group:" walker_update=$( \ - curl -LsH "Authorization: ${API_KEY}" \ - --data '{"id": "silly-walks", "username": "walker", "role": "editor"}' \ - ${CKAN_ACTION_URL}/group_member_create + api_call '{"id": "silly-walks", "username": "walker", "role": "editor"}' group_member_create ) echo ${walker_update} @@ -122,5 +109,5 @@ echo ${walker_update} # END. # -. ${APP_DIR}/bin/create-test-data-$VARS_TYPE.sh -. ${APP_DIR}/bin/deactivate +. "${APP_DIR}"/bin/create-test-data-$VARS_TYPE.sh +. "${APP_DIR}"/bin/deactivate diff --git a/bin/docker-compose.sh b/bin/docker-compose.sh index 405bb7cd..82f4a054 100755 --- a/bin/docker-compose.sh +++ b/bin/docker-compose.sh @@ -1,13 +1,15 @@ #!/bin/sh +set -x + # Pass commands to Docker Compose v1 or v2 depending on what is present -if (which docker-compose >/dev/null); then - # Docker Compose v1 - docker-compose $* -elif (docker compose ls >/dev/null); then +if (docker compose ls >/dev/null); then # Docker Compose v2 docker compose $* +elif (which docker-compose >/dev/null); then + # Docker Compose v1 + docker-compose $* else # Docker Compose not found exit 1 diff --git a/bin/init-ext.sh b/bin/init-ext.sh index b6cd35df..d9a51da9 100755 --- a/bin/init-ext.sh +++ b/bin/init-ext.sh @@ -32,8 +32,10 @@ install_requirements () { done } -. ${APP_DIR}/bin/activate - +. "${APP_DIR}"/bin/activate +if [ "$CKAN_VERSION" = "2.9" ]; then + pip install "setuptools>=44.1.0,<71" +fi install_requirements . dev-requirements requirements-dev EXTENSIONS_FILE=$APP_DIR/bin/extensions.yml $PYTHON $(dirname $0)/generate-ext-requirements.py pip install -r "/tmp/requirements-ext.txt" @@ -41,8 +43,8 @@ for extension in . `ls -d $SRC_DIR/ckanext-*`; do install_requirements $extension requirements pip-requirements done # force version that declares itself to be incompatible but actually works -pip install click==7.1.2 +pip install click==8.1.7 install_requirements . dev-requirements requirements-dev -. $APP_DIR/bin/process-config.sh -. ${APP_DIR}/bin/deactivate +. "${APP_DIR}"/bin/process-config.sh +. "${APP_DIR}"/bin/deactivate diff --git a/bin/init.sh b/bin/init.sh index b314d96b..936a1a09 100755 --- a/bin/init.sh +++ b/bin/init.sh @@ -4,7 +4,7 @@ # set -e -. ${APP_DIR}/bin/activate +. "${APP_DIR}"/bin/activate CLICK_ARGS="--yes" ckan_cli db clean ckan_cli db init . $APP_DIR/bin/init-${VARS_TYPE}.sh diff --git a/bin/serve.sh b/bin/serve.sh index 531b22b7..a4a5a4bc 100755 --- a/bin/serve.sh +++ b/bin/serve.sh @@ -1,21 +1,7 @@ #!/usr/bin/env sh set -e -dockerize -wait tcp://postgres:5432 -timeout 1m -dockerize -wait tcp://solr:8983 -timeout 1m -dockerize -wait tcp://redis:6379 -timeout 1m - -for i in `seq 1 60`; do - if (PGPASSWORD=pass psql -h postgres -U ckan_default -d ckan_test -c "\q"); then - echo "Database became ready on attempt $i" - break - else - echo "Database not yet ready, retrying (attempt $i)..." - sleep 1 - fi -done - -. ${APP_DIR}/bin/activate +. "${APP_DIR}"/bin/activate if (which ckan > /dev/null); then ckan -c ${CKAN_INI} run --disable-reloader --threaded else diff --git a/docker-compose.yml b/docker-compose.yml index ac9e4453..59717c08 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,6 +27,8 @@ services: condition: service_healthy solr: condition: service_started + redis: + condition: service_started networks: - amazeeio-network - default @@ -73,7 +75,7 @@ services: - default solr: - image: ckan/ckan-solr:${CKAN_VERSION} + image: ckan/ckan-solr:${CKAN_VERSION}-solr9 ports: - "8983" environment: