Skip to content

Commit

Permalink
[QOLDEV-954] replace obsolete test containers
Browse files Browse the repository at this point in the history
  • Loading branch information
ThrawnCA committed Nov 27, 2024
1 parent ae74558 commit 529c157
Show file tree
Hide file tree
Showing 10 changed files with 86 additions and 126 deletions.
14 changes: 7 additions & 7 deletions .ahoy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ commands:
ahoy title "Building and starting Docker containers"
sh bin/docker-compose.sh up -d "$@"
echo "Initialising database schema"
ahoy cli '$APP_DIR/bin/init.sh'
ahoy cli '"${APP_DIR}"/bin/init.sh'
echo "Waiting for containers to start listening..."
ahoy cli "dockerize -wait tcp://ckan:5000 -timeout 1m"
if sh bin/docker-compose.sh logs | grep -q "\[Error\]"; then exit 1; fi
Expand Down Expand Up @@ -79,9 +79,9 @@ commands:
cmd: |
CKAN_CONTAINER=$(sh bin/docker-compose.sh ps -q ckan)
if [ "${#}" -ne 0 \]; then
docker exec $CKAN_CONTAINER sh -c '. ${APP_DIR}/bin/activate; cd $APP_DIR;'" $*"
docker exec $CKAN_CONTAINER sh -c '. "${APP_DIR}"/bin/activate; cd $APP_DIR;'" $*"
else
docker exec $CKAN_CONTAINER sh -c '. ${APP_DIR}/bin/activate && cd $APP_DIR && sh'
docker exec $CKAN_CONTAINER sh -c '. "${APP_DIR}"/bin/activate && cd $APP_DIR && sh'
fi
doctor:
Expand All @@ -93,7 +93,7 @@ commands:
cmd: |
ahoy title "Installing a fresh site"
ahoy start-ckan-job-workers
ahoy cli '$APP_DIR/bin/init.sh && $APP_DIR/bin/create-test-data.sh'
ahoy cli '"${APP_DIR}"/bin/init.sh && "${APP_DIR}"/bin/create-test-data.sh'
ahoy stop-ckan-job-workers
clean:
Expand Down Expand Up @@ -130,13 +130,13 @@ commands:
cmd: |
docker cp . $(sh bin/docker-compose.sh ps -q ckan):/srv/app/
docker cp bin/ckan_cli $(sh bin/docker-compose.sh ps -q ckan):/usr/bin/
ahoy cli 'chmod -v u+x /usr/bin/ckan_cli $APP_DIR/bin/*; cp -v .docker/test-$VARS_TYPE.ini $CKAN_INI; $APP_DIR/bin/process-config.sh'
ahoy cli 'chmod -v u+x /usr/bin/ckan_cli "${APP_DIR}"/bin/*; cp -v .docker/test-$VARS_TYPE.ini $CKAN_INI; "${APP_DIR}"/bin/process-config.sh'
test-unit:
usage: Run unit tests.
cmd: |
ahoy title 'Run unit tests'
ahoy cli 'pytest --ckan-ini=${CKAN_INI} $APP_DIR/test' || \
ahoy cli 'pytest --ckan-ini=${CKAN_INI} "${APP_DIR}"/test' || \
[ "${ALLOW_UNIT_FAIL:-0}" -eq 1 ]
test-bdd:
Expand Down Expand Up @@ -172,7 +172,7 @@ commands:
usage: Starts email mock server used for email BDD tests
cmd: |
ahoy title 'Starting mailmock'
ahoy cli 'mailmock -p 8025 -o ${APP_DIR}/test/emails' # for debugging mailmock email output remove --no-stdout
ahoy cli 'mailmock -p 8025 -o "${APP_DIR}"/test/emails' # for debugging mailmock email output remove --no-stdout
stop-mailmock:
usage: Stops email mock server used for email BDD tests
Expand Down
35 changes: 22 additions & 13 deletions .docker/Dockerfile-template.ckan
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
FROM openknowledge/ckan-dev:{CKAN_VERSION}
FROM ckan/ckan-dev:{CKAN_VERSION}

# swap between root and unprivileged user
ARG ORIGINAL_USER
RUN ORIGINAL_USER=$(id -un)

ARG SITE_URL=http://ckan:5000/
ENV DEPLOY_ENV={DEPLOY_ENV}
Expand All @@ -10,9 +14,20 @@ ENV PYTHON={PYTHON}

WORKDIR "${APP_DIR}"

ENV DOCKERIZE_VERSION v0.6.1
RUN apk add --no-cache build-base \
&& curl -sL https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
COPY .docker/test-$VARS_TYPE.ini $CKAN_INI

COPY . "${APP_DIR}"/

USER root

COPY bin/ckan_cli /usr/bin/

COPY vars/shared-{VARS_TYPE}.var.yml "${APP_DIR}"/bin/extensions.yml

RUN chmod +x "${APP_DIR}"/bin/*.sh /usr/bin/ckan_cli

ENV DOCKERIZE_VERSION=v0.6.1
RUN wget -O - https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
| tar -C /usr/local/bin -xzvf -

# Install CKAN.
Expand All @@ -24,17 +39,11 @@ RUN cd $SRC_DIR/ckan \
&& git reset --hard && git clean -f \
&& git checkout '{CKAN_GIT_VERSION}'

COPY .docker/test-$VARS_TYPE.ini $CKAN_INI

COPY . ${APP_DIR}/

COPY bin/ckan_cli /usr/bin/

COPY vars/shared-{VARS_TYPE}.var.yml $APP_DIR/bin/extensions.yml
RUN which ps || apt-get install -y procps

RUN chmod +x ${APP_DIR}/bin/*.sh /usr/bin/ckan_cli
USER "$ORIGINAL_USER"

# Init current extension.
RUN ${APP_DIR}/bin/init-ext.sh
RUN "${APP_DIR}"/bin/init-ext.sh

CMD ["/srv/app/bin/serve.sh"]
52 changes: 15 additions & 37 deletions bin/create-test-data-OpenData.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,36 +7,30 @@ set -ex
##
# BEGIN: Add sysadmin config values.
#
curl -LsH "Authorization: ${API_KEY}" \
--data '{
api_call '{
"ckan.comments.profanity_list": "",
"ckan.datarequests.closing_circumstances": "Released as open data|nominate_dataset\r\nOpen dataset already exists|nominate_dataset\r\nPartially released|nominate_dataset\r\nTo be released as open data at a later date|nominate_approximate_date\r\nData openly available elsewhere\r\nNot suitable for release as open data\r\nRequested data not available/cannot be compiled\r\nRequestor initiated closure",
"ckanext.data_qld.resource_formats": "CSV\r\nHTML\r\nJSON\r\nRDF\r\nTXT\r\nXLS"
}' \
${CKAN_ACTION_URL}/config_option_update
}' config_option_update

##
# END.
#

# Create private test dataset with our standard fields
curl -LsH "Authorization: ${API_KEY}" \
--data '{"name": "test-dataset", "owner_org": "'"${TEST_ORG_ID}"'", "private": true,
api_call '{"name": "test-dataset", "owner_org": "'"${TEST_ORG_ID}"'", "private": true,
"update_frequency": "monthly", "author_email": "admin@localhost", "version": "1.0",
"license_id": "other-open", "data_driven_application": "NO", "security_classification": "PUBLIC",
"notes": "private test", "de_identified_data": "NO"}' \
${CKAN_ACTION_URL}/package_create
"notes": "private test", "de_identified_data": "NO"}' package_create

# Create public test dataset with our standard fields
curl -LsH "Authorization: ${API_KEY}" \
--data '{"name": "public-test-dataset", "owner_org": "'"${TEST_ORG_ID}"'",
api_call '{"name": "public-test-dataset", "owner_org": "'"${TEST_ORG_ID}"'",
"update_frequency": "monthly", "author_email": "[email protected]", "version": "1.0",
"license_id": "other-open", "data_driven_application": "NO", "security_classification": "PUBLIC",
"notes": "public test", "de_identified_data": "NO", "resources": [
{"name": "test-resource", "description": "Test resource description",
"url": "https://example.com/foo", "format": "HTML", "size": 1024}
]}' \
${CKAN_ACTION_URL}/package_create
]}' package_create

# Populate Archiver data for test dataset
ckan_cli archiver update-test public-test-dataset
Expand All @@ -56,36 +50,26 @@ add_user_if_needed dr_editor "Data Request Editor" dr_editor@localhost
echo "Creating ${DR_ORG_TITLE} Organisation:"

DR_ORG=$( \
curl -LsH "Authorization: ${API_KEY}" \
--data '{"name": "'"${DR_ORG_NAME}"'", "title": "'"${DR_ORG_TITLE}"'"}' \
${CKAN_ACTION_URL}/organization_create
api_call '{"name": "'"${DR_ORG_NAME}"'", "title": "'"${DR_ORG_TITLE}"'"}' organization_create
)

DR_ORG_ID=$(echo $DR_ORG | $PYTHON $APP_DIR/bin/extract-id.py)

echo "Assigning test users to ${DR_ORG_TITLE} Organisation:"

curl -LsH "Authorization: ${API_KEY}" \
--data '{"id": "'"${DR_ORG_ID}"'", "object": "dr_admin", "object_type": "user", "capacity": "admin"}' \
${CKAN_ACTION_URL}/member_create
api_call '{"id": "'"${DR_ORG_ID}"'", "object": "dr_admin", "object_type": "user", "capacity": "admin"}' member_create

curl -LsH "Authorization: ${API_KEY}" \
--data '{"id": "'"${DR_ORG_ID}"'", "object": "dr_editor", "object_type": "user", "capacity": "editor"}' \
${CKAN_ACTION_URL}/member_create
api_call '{"id": "'"${DR_ORG_ID}"'", "object": "dr_editor", "object_type": "user", "capacity": "editor"}' member_create

echo "Creating test dataset for data request organisation:"

curl -LsH "Authorization: ${API_KEY}" \
--data '{"name": "data_request_dataset", "title": "Dataset for data requests", "owner_org": "'"${DR_ORG_ID}"'",
api_call '{"name": "data_request_dataset", "title": "Dataset for data requests", "owner_org": "'"${DR_ORG_ID}"'",
"update_frequency": "near-realtime", "author_email": "dr_admin@localhost", "version": "1.0", "license_id": "cc-by-4",
"data_driven_application": "NO", "security_classification": "PUBLIC", "notes": "test", "de_identified_data": "NO"}'\
${CKAN_ACTION_URL}/package_create
"data_driven_application": "NO", "security_classification": "PUBLIC", "notes": "test", "de_identified_data": "NO"}'package_create

echo "Creating test Data Request:"

curl -LsH "Authorization: ${API_KEY}" \
--data '{"title": "Test Request", "description": "This is an example", "organization_id": "'"${TEST_ORG_ID}"'"}' \
${CKAN_ACTION_URL}/create_datarequest
api_call '{"title": "Test Request", "description": "This is an example", "organization_id": "'"${TEST_ORG_ID}"'"}' create_datarequest

##
# END.
Expand All @@ -105,24 +89,18 @@ add_user_if_needed report_admin "Reporting Admin" report_admin@localhost
echo "Creating ${REPORT_ORG_TITLE} Organisation:"

REPORT_ORG=$( \
curl -LsH "Authorization: ${API_KEY}" \
--data '{"name": "'"${REPORT_ORG_NAME}"'", "title": "'"${REPORT_ORG_TITLE}"'"}' \
${CKAN_ACTION_URL}/organization_create
api_call '{"name": "'"${REPORT_ORG_NAME}"'", "title": "'"${REPORT_ORG_TITLE}"'"}' organization_create
)

REPORT_ORG_ID=$(echo $REPORT_ORG | $PYTHON $APP_DIR/bin/extract-id.py)

echo "Assigning admin user to ${REPORT_ORG_TITLE} Organisation:"

curl -LsH "Authorization: ${API_KEY}" \
--data '{"id": "'"${REPORT_ORG_ID}"'", "object": "report_admin", "object_type": "user", "capacity": "admin"}' \
${CKAN_ACTION_URL}/member_create
api_call '{"id": "'"${REPORT_ORG_ID}"'", "object": "report_admin", "object_type": "user", "capacity": "admin"}' member_create

echo "Creating test Data Request for reporting:"

curl -LsH "Authorization: ${API_KEY}" \
--data '{"title": "Reporting Request", "description": "Data Request for reporting", "organization_id": "'"${REPORT_ORG_ID}"'"}' \
${CKAN_ACTION_URL}/create_datarequest
api_call '{"title": "Reporting Request", "description": "Data Request for reporting", "organization_id": "'"${REPORT_ORG_ID}"'"}' create_datarequest

##
# END.
Expand Down
16 changes: 5 additions & 11 deletions bin/create-test-data-Publications.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,22 +6,16 @@ set -e
set -x

# Create publishing standards dataset
curl -LsH "Authorization: ${API_KEY}" \
--data '{"name": "publishing-standards-publications-qld-gov-au", "owner_org": "'"${TEST_ORG_ID}"'"}' \
${CKAN_ACTION_URL}/package_create
api_call '{"name": "publishing-standards-publications-qld-gov-au", "owner_org": "'"${TEST_ORG_ID}"'"}' package_create

# Create private test dataset with our standard fields
curl -LsH "Authorization: ${API_KEY}" \
--data '{"name": "test-dataset", "owner_org": "'"${TEST_ORG_ID}"'", "private": true,
"notes": "private test"}' \
${CKAN_ACTION_URL}/package_create
api_call '{"name": "test-dataset", "owner_org": "'"${TEST_ORG_ID}"'", "private": true,
"notes": "private test"}' package_create

# Create public test dataset with our standard fields
curl -LsH "Authorization: ${API_KEY}" \
--data '{"name": "public-test-dataset", "owner_org": "'"${TEST_ORG_ID}"'",
api_call '{"name": "public-test-dataset", "owner_org": "'"${TEST_ORG_ID}"'",
"notes": "public test", "resources": [
{"name": "test-resource", "description": "Test resource description",
"url": "https://example.com", "format": "HTML", "size": 1024}
]}' \
${CKAN_ACTION_URL}/package_create
]}' package_create

51 changes: 19 additions & 32 deletions bin/create-test-data.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ CKAN_USER_NAME="${CKAN_USER_NAME:-admin}"
CKAN_DISPLAY_NAME="${CKAN_DISPLAY_NAME:-Administrator}"
CKAN_USER_EMAIL="${CKAN_USER_EMAIL:-admin@localhost}"

. ${APP_DIR}/bin/activate
. "${APP_DIR}"/bin/activate

add_user_if_needed () {
echo "Adding user '$2' ($1) with email address [$3]"
Expand All @@ -19,6 +19,10 @@ add_user_if_needed () {
password="${4:-Password123!}"
}

api_call () {
wget -O - --header="Authorization: ${API_KEY}" --post-data "$1" ${CKAN_ACTION_URL}/$2
}

add_user_if_needed "$CKAN_USER_NAME" "$CKAN_DISPLAY_NAME" "$CKAN_USER_EMAIL"
ckan_cli sysadmin add "${CKAN_USER_NAME}"

Expand All @@ -33,9 +37,7 @@ fi
#
echo "Adding sysadmin config:"

curl -LsH "Authorization: ${API_KEY}" \
--data '{"ckanext.data_qld.excluded_display_name_words": "gov"}' \
${CKAN_ACTION_URL}/config_option_update
api_call '{"ckanext.data_qld.excluded_display_name_words": "gov"}' config_option_update

##
# END.
Expand All @@ -57,37 +59,28 @@ add_user_if_needed test_org_member "Test Member" test_org_member@localhost
echo "Creating ${TEST_ORG_TITLE} organisation:"

TEST_ORG=$( \
curl -LsH "Authorization: ${API_KEY}" \
--data '{"name": "'"${TEST_ORG_NAME}"'", "title": "'"${TEST_ORG_TITLE}"'",
"description": "Organisation for testing issues"}' \
${CKAN_ACTION_URL}/organization_create
api_call '{"name": "'"${TEST_ORG_NAME}"'", "title": "'"${TEST_ORG_TITLE}"'",
"description": "Organisation for testing issues"}' organization_create
)

TEST_ORG_ID=$(echo $TEST_ORG | $PYTHON ${APP_DIR}/bin/extract-id.py)
TEST_ORG_ID=$(echo $TEST_ORG | $PYTHON "${APP_DIR}"/bin/extract-id.py)

echo "Assigning test users to '${TEST_ORG_TITLE}' organisation (${TEST_ORG_ID}):"

curl -LsH "Authorization: ${API_KEY}" \
--data '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_admin", "object_type": "user", "capacity": "admin"}' \
${CKAN_ACTION_URL}/member_create
api_call '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_admin", "object_type": "user", "capacity": "admin"}' member_create

api_call '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_editor", "object_type": "user", "capacity": "editor"}' member_create

curl -LsH "Authorization: ${API_KEY}" \
--data '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_editor", "object_type": "user", "capacity": "editor"}' \
${CKAN_ACTION_URL}/member_create
api_call '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_member", "object_type": "user", "capacity": "member"}' member_create

curl -LsH "Authorization: ${API_KEY}" \
--data '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_member", "object_type": "user", "capacity": "member"}' \
${CKAN_ACTION_URL}/member_create
##
# END.
#

# Creating test data hierarchy which creates organisations assigned to datasets
echo "Creating food-standards-agency organisation:"
organisation_create=$( \
curl -LsH "Authorization: ${API_KEY}" \
--data "name=food-standards-agency&title=Food%20Standards%20Agency" \
${CKAN_ACTION_URL}/organization_create
api_call "name=food-standards-agency&title=Food%20Standards%20Agency" organization_create
)
echo ${organisation_create}

Expand All @@ -96,31 +89,25 @@ add_user_if_needed walker "Walker" walker@localhost

echo "Creating non-organisation group:"
group_create=$( \
curl -LsH "Authorization: ${API_KEY}" \
--data '{"name": "silly-walks", "title": "Silly walks", "description": "The Ministry of Silly Walks"}' \
${CKAN_ACTION_URL}/group_create
api_call '{"name": "silly-walks", "title": "Silly walks", "description": "The Ministry of Silly Walks"}' group_create
)
echo ${group_create}

echo "Updating group_admin to have admin privileges in the silly-walks group:"
group_admin_update=$( \
curl -LsH "Authorization: ${API_KEY}" \
--data '{"id": "silly-walks", "username": "group_admin", "role": "admin"}' \
${CKAN_ACTION_URL}/group_member_create
api_call '{"id": "silly-walks", "username": "group_admin", "role": "admin"}' group_member_create
)
echo ${group_admin_update}

echo "Updating walker to have editor privileges in the silly-walks group:"
walker_update=$( \
curl -LsH "Authorization: ${API_KEY}" \
--data '{"id": "silly-walks", "username": "walker", "role": "editor"}' \
${CKAN_ACTION_URL}/group_member_create
api_call '{"id": "silly-walks", "username": "walker", "role": "editor"}' group_member_create
)
echo ${walker_update}

##
# END.
#

. ${APP_DIR}/bin/create-test-data-$VARS_TYPE.sh
. ${APP_DIR}/bin/deactivate
. "${APP_DIR}"/bin/create-test-data-$VARS_TYPE.sh
. "${APP_DIR}"/bin/deactivate
10 changes: 6 additions & 4 deletions bin/docker-compose.sh
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
#!/bin/sh

set -x

# Pass commands to Docker Compose v1 or v2 depending on what is present

if (which docker-compose >/dev/null); then
# Docker Compose v1
docker-compose $*
elif (docker compose ls >/dev/null); then
if (docker compose ls >/dev/null); then
# Docker Compose v2
docker compose $*
elif (which docker-compose >/dev/null); then
# Docker Compose v1
docker-compose $*
else
# Docker Compose not found
exit 1
Expand Down
Loading

0 comments on commit 529c157

Please sign in to comment.