Merge pull request #275 from qld-gov-au/QOL-9055-amazon-linux-2

QOL-9055 Prepare for Python 3 and Amazon Linux 2

.ahoy.yml

@@ -36,15 +36,10 @@ commands:
     usage: Build and start Docker containers.
     cmd: |
       echo "Preparing containers for $VARS_TYPE"
-      export CKAN_REPO
-      CKAN_VERSION=$(sh retrieve-ckan-version.sh)
-      if [ "$CKAN_VERSION" != "" ]; then
-        export CKAN_VERSION
-      fi
       docker-compose up -d "$@"
       sleep 10
       docker-compose logs
-      ahoy cli "dockerize -wait tcp://ckan:3000 -timeout 1m"
+      ahoy cli "dockerize -wait tcp://ckan:5000 -timeout 1m"
       if docker-compose logs | grep -q "\[Error\]"; then docker-compose logs; exit 1; fi
       if docker-compose logs | grep -q "Exception"; then docker-compose logs; exit 1; fi
       docker ps -a --filter name=^/${COMPOSE_PROJECT_NAME}_
@@ -76,7 +71,7 @@ commands:
 
   cli:
     usage: Start a shell inside CLI container or run a command.
-    cmd: if \[ "${#}" -ne 0 \]; then docker exec $(docker-compose ps -q ckan) sh -c '. ${VENV_DIR}/bin/activate; cd $APP_DIR;'" $*"; else docker exec $(docker-compose ps -q ckan) sh -c '. ${VENV_DIR}/bin/activate && cd $APP_DIR && sh'; fi
+    cmd: if \[ "${#}" -ne 0 \]; then docker exec $(docker-compose ps -q ckan) sh -c '. ${APP_DIR}/scripts/activate; cd $APP_DIR;'" $*"; else docker exec $(docker-compose ps -q ckan) sh -c '. ${APP_DIR}/scripts/activate && cd $APP_DIR && sh'; fi
 
   doctor:
     usage: Find problems with current project setup.
@@ -118,14 +113,15 @@ commands:
   copy-local-files:
     usage: Update files from local repo.
     cmd: |
-      docker cp . $(docker-compose ps -q ckan):/app/
-      docker cp .docker/scripts/ckan_cli $(docker-compose ps -q ckan):/app/ckan/default/bin/
-      ahoy cli 'chmod u+x $VENV_DIR/bin/ckan_cli; cp .docker/test-$VARS_TYPE.ini $CKAN_INI'
+      docker cp . $(docker-compose ps -q ckan):/srv/app/
+      docker cp .docker/scripts $(docker-compose ps -q ckan):/srv/app/
+      docker cp .docker/scripts/ckan_cli $(docker-compose ps -q ckan):/usr/bin/
+      ahoy cli 'chmod -v u+x /usr/bin/ckan_cli; cp -v .docker/test-$VARS_TYPE.ini $CKAN_INI'
 
   test-unit:
     usage: Run unit tests.
     cmd: |
-      ahoy cli 'pytest --ckan-ini=${CKAN_INI}' || \
+      ahoy cli 'pytest --ckan-ini=${CKAN_INI} $APP_DIR/test' || \
       [ "${ALLOW_UNIT_FAIL:-0}" -eq 1 ]
 
   test-bdd:

.circleci/build.sh

@@ -14,11 +14,21 @@ sed -i -e "s/##//" docker-compose.yml
 # Pull the latest images.
 ahoy pull
 
-# Disable checks used on host machine.
-export DOCTOR_CHECK_PYGMY=0
-export DOCTOR_CHECK_PORT=0
-export DOCTOR_CHECK_SSH=0
-export DOCTOR_CHECK_WEBSERVER=0
-export DOCTOR_CHECK_BOOTSTRAP=0
+QGOV_CKAN_VERSION=`sh ./retrieve-ckan-version.sh`
+CKAN_VERSION=2.9
+PYTHON=python
+if [ "$PYTHON_VERSION" = "py2" ]; then
+    CKAN_VERSION="${CKAN_VERSION}-py2"
+else
+    PYTHON="${PYTHON}3"
+fi
+export CKAN_VERSION
+
+sed "s|{CKAN_VERSION}|$CKAN_VERSION|g" .docker/Dockerfile-template.ckan \
+    | sed "s|{PYTHON_VERSION}|$PYTHON_VERSION|g" \
+    | sed "s|{DEPLOY_ENV}|$DEPLOY_ENV|g" \
+    | sed "s|{VARS_TYPE}|$VARS_TYPE|g" \
+    | sed "s|{PYTHON}|$PYTHON|g" \
+    | sed "s|{QGOV_CKAN_VERSION}|$QGOV_CKAN_VERSION|g" > .docker/Dockerfile.ckan
 
 ahoy build || (ahoy logs; exit 1)

.circleci/process-artifacts.sh

@@ -10,4 +10,4 @@ ahoy cli "mkdir -p test/screenshots"
 
 # Copy from the app container to the build host for storage.
 mkdir -p /tmp/artifacts/behave
-docker cp "$(docker-compose ps -q ckan)":/app/test/screenshots /tmp/artifacts/behave/
+docker cp "$(docker-compose ps -q ckan)":/srv/app/test/screenshots /tmp/artifacts/behave/

.circleci/test.sh

@@ -7,8 +7,8 @@ set -e
 echo "==> Lint code"
 ahoy lint
 
-#echo "==> Run Unit tests"
-#ahoy test-unit
+echo "==> Run Unit tests"
+ahoy test-unit
 
 echo "==> Run BDD tests"
 ahoy test-bdd || (ahoy logs; exit 1)

.docker/Dockerfile-template.ckan

@@ -0,0 +1,42 @@
+FROM openknowledge/ckan-dev:{CKAN_VERSION}
+
+ARG SITE_URL=http://ckan:5000/
+ENV DEPLOY_ENV={DEPLOY_ENV}
+ENV VARS_TYPE={VARS_TYPE}
+ENV PYTHON_VERSION={PYTHON_VERSION}
+ENV CKAN_SITE_URL="${SITE_URL}"
+ENV PYTHON={PYTHON}
+
+WORKDIR "${APP_DIR}"
+
+ENV DOCKERIZE_VERSION v0.6.1
+RUN apk add --no-cache build-base \
+    && curl -sL https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
+        | tar -C /usr/local/bin -xzvf -
+
+# Install CKAN.
+
+RUN cd $SRC_DIR/ckan \
+    && git remote set-url origin 'https://github.com/qld-gov-au/ckan.git' \
+    && git fetch --tags origin {QGOV_CKAN_VERSION}\
+    && git reset --hard && git clean -f \
+    && git checkout '{QGOV_CKAN_VERSION}'
+
+COPY .docker/test-$VARS_TYPE.ini $CKAN_INI
+
+RUN sed -i "s@SITE_URL@${CKAN_SITE_URL}@g" $CKAN_INI
+
+COPY . ${APP_DIR}/
+
+COPY .docker/scripts ${APP_DIR}/scripts
+
+COPY .docker/scripts/ckan_cli /usr/bin/
+
+COPY vars/shared-{VARS_TYPE}.var.yml $APP_DIR/scripts/extensions.yml
+
+RUN chmod +x ${APP_DIR}/scripts/*.sh /usr/bin/ckan_cli
+
+# Init current extension.
+RUN ${APP_DIR}/scripts/init-ext.sh
+
+CMD ["/srv/app/scripts/serve.sh"]

.docker/scripts/activate

@@ -0,0 +1,3 @@
+if [ "$VENV_DIR" != "" ]; then
+  . ${VENV_DIR}/bin/activate
+fi

.docker/scripts/create-test-data-OpenData.sh

@@ -5,9 +5,7 @@
 set -e
 set -x
 
-if [ "$VENV_DIR" != "" ]; then
-  . ${VENV_DIR}/bin/activate
-fi
+. ${APP_DIR}/scripts/activate
 
 # Create private test dataset with our standard fields
 curl -LsH "Authorization: ${API_KEY}" \
@@ -25,6 +23,4 @@ curl -LsH "Authorization: ${API_KEY}" \
 "notes": "test", "de_identified_data": "NO"}' \
     ${CKAN_ACTION_URL}/package_create
 
-if [ "$VENV_DIR" != "" ]; then
-  deactivate
-fi
+. ${APP_DIR}/scripts/deactivate

.docker/scripts/create-test-data-Publications.sh

@@ -5,9 +5,7 @@
 set -e
 set -x
 
-if [ "$VENV_DIR" != "" ]; then
-  . ${VENV_DIR}/bin/activate
-fi
+. ${APP_DIR}/scripts/activate
 
 # Create publishing standards dataset
 curl -LsH "Authorization: ${API_KEY}" \
@@ -25,6 +23,4 @@ curl -LsH "Authorization: ${API_KEY}" \
     ${CKAN_ACTION_URL}/package_create
 
 
-if [ "$VENV_DIR" != "" ]; then
-  deactivate
-fi
+. ${APP_DIR}/scripts/deactivate