diff --git a/.github/workflows/master_merge.yml b/.github/workflows/master_merge.yml index b521325..f6d9fbc 100644 --- a/.github/workflows/master_merge.yml +++ b/.github/workflows/master_merge.yml @@ -54,6 +54,23 @@ jobs: cache-from: type=local,src=/tmp/.buildx-cache-alpine-2-10 cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-alpine-2-10 + - name: Get docker tag for alpine xloader image + id: alpine-xloader + run: | + echo "IMAGE_TAG=$(awk -F '=' '/IMAGE_TAG/{print $2}' ./images/ckan/2.10/Dockerfile.xloader)" >> $GITHUB_OUTPUT + + - name: Build and push CKAN 2.10 alpine xloader + uses: docker/build-push-action@v5 + with: + context: ./images/ckan/2.10 + file: ./images/ckan/2.10/Dockerfile.xloader + push: true + tags: | + keitaro/ckan:${{ steps.alpine.outputs.IMAGE_TAG }} + ghcr.io/keitaroinc/ckan:${{ steps.alpine.outputs.IMAGE_TAG }} + cache-from: type=local,src=/tmp/.buildx-cache-alpine-2-10 + cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-alpine-2-10 + - name: Get docker tag for Ubuntu image id: ubuntu run: | diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml index 0bad5a2..2083bc1 100644 --- a/.github/workflows/pr_checks.yml +++ b/.github/workflows/pr_checks.yml @@ -39,6 +39,21 @@ jobs: cache-from: type=local,src=/tmp/.buildx-cache-alpine-2-10 cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-alpine-2-10 + - name: Get docker tag for Alpine xloader image + id: alpine-xloader + run: | + echo "IMAGE_TAG=$(awk -F '=' '/IMAGE_TAG/{print $2}' ./images/ckan/2.10/Dockerfile.xloader)" >> $GITHUB_OUTPUT + + - name: Build CKAN 2.10 alpine xloader + uses: docker/build-push-action@v5 + with: + context: ./images/ckan/2.10 + file: ./images/ckan/2.10/Dockerfile.xloader + push: false + tags: keitaro/ckan:${{ steps.alpine.outputs.IMAGE_TAG }} + cache-from: type=local,src=/tmp/.buildx-cache-alpine-2-10 + cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-alpine-2-10 + - name: Get docker tag for Ubuntu image id: ubuntu run: | diff --git a/.github/workflows/trivy_scan.yml b/.github/workflows/trivy_scan.yml index 8bce2d1..bbc4554 100644 --- a/.github/workflows/trivy_scan.yml +++ b/.github/workflows/trivy_scan.yml @@ -55,6 +55,30 @@ jobs: with: sarif_file: 'trivy-results.sarif' + scan_2_10_xloader: + name: scan_2_10_xloader + runs-on: ubuntu-20.04 + steps: + + - name: Checkout code + uses: actions/checkout@v4 + + - name: Build an image from Dockerfile + run: | + docker build -t keitaro/ckan/2.10:${{ github.sha }} -f ./images/ckan/2.10/Dockerfile.xloader ./images/ckan/2.10/ + + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: 'keitaro/ckan/2.10:${{ github.sha }}' + format: 'sarif' + output: 'trivy-results.sarif' + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: 'trivy-results.sarif' + scan_2_10_focal: name: scan_2_10_focal diff --git a/images/ckan/2.10/Dockerfile.xloader b/images/ckan/2.10/Dockerfile.xloader new file mode 100644 index 0000000..ed19c36 --- /dev/null +++ b/images/ckan/2.10/Dockerfile.xloader @@ -0,0 +1,207 @@ +################## +### Build CKAN ### +################## +FROM alpine:3.17.2 as ckanbuild + +# Used by Github Actions to tag the image with +ENV IMAGE_TAG=2.10.4 + +# Set CKAN version to build +ENV GIT_URL=https://github.com/ckan/ckan.git +ENV GIT_BRANCH=ckan-2.10.4 + +# Set src dirs +ENV SRC_DIR=/srv/app/src +ENV PIP_SRC=${SRC_DIR} + +WORKDIR ${SRC_DIR} + +# Packages to build CKAN requirements and plugins +RUN apk add --no-cache \ + python3 \ + python3-dev \ + git \ + curl \ + postgresql-dev \ + linux-headers \ + gcc \ + make \ + g++ \ + autoconf \ + automake \ + libtool \ + patch \ + musl-dev \ + pcre-dev \ + pcre \ + libffi-dev \ + libxml2-dev \ + libxslt-dev + +# Create the src directory +RUN mkdir -p ${SRC_DIR} + +# Install pip +RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ + python ${SRC_DIR}/get-pip.py + +# Downgrade setuptools so that CKAN requirements can be built +RUN pip install setuptools==44.1.0 + +# Fetch and build CKAN and requirements +RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan +# Copy patches and apply patches script +COPY ./patches ${SRC_DIR}/patches +COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh +# Apply patches +# RUN ${SRC_DIR}/apply_ckan_patches.sh +RUN rm -rf /srv/app/src/ckan/.git +RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt +RUN pip wheel --wheel-dir=/wheels uWSGI==2.0.20 gevent==22.10.2 greenlet==2.0.2 + + +########################### +### Default-Extensions #### +########################### +FROM alpine:3.17.2 as extbuild + +# Set src dirs +ENV SRC_DIR=/srv/app/src +ENV PIP_SRC=${SRC_DIR} + +# List of default extensions +ENV DEFAULT_EXTENSIONS envvars + +# Locations and tags, please use specific tags or revisions +ENV ENVVARS_GIT_URL=https://github.com/okfn/ckanext-envvars +ENV ENVVARS_GIT_BRANCH=0.0.2 + +RUN apk add --no-cache \ + python3 \ + python3-dev \ + git \ + curl + +# Create the src directory +RUN mkdir -p ${SRC_DIR} + +# Install pip +RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ + python ${SRC_DIR}/get-pip.py + +# Fetch and build the default CKAN extensions +RUN pip wheel --wheel-dir=/wheels git+${ENVVARS_GIT_URL}@${ENVVARS_GIT_BRANCH}#egg=ckanext-envvars + +############ +### MAIN ### +############ +FROM alpine:3.17.2 + +LABEL maintainer="Keitaro Inc " +LABEL org.opencontainers.image.source https://github.com/keitaroinc/docker-ckan + +ENV APP_DIR=/srv/app +ENV SRC_DIR=/srv/app/src +ENV CKAN_DIR=${SRC_DIR}/ckan +ENV DATA_DIR=/srv/app/data +ENV PIP_SRC=${SRC_DIR} +ENV CKAN_SITE_URL=http://localhost:5000 +ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore xloader + +# Install necessary packages to run CKAN +RUN apk add --no-cache \ + supervisor \ + python3 \ + bash \ + git \ + gettext \ + curl \ + postgresql-client \ + libmagic \ + pcre \ + libxslt \ + libxml2 \ + tzdata \ + apache2-utils && \ + # Create SRC_DIR + mkdir -p ${SRC_DIR} + + +# Install pip +RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ + python ${SRC_DIR}/get-pip.py + + ### XLoader ### +RUN pip3 install -e 'git+https://github.com/keitaroinc/ckanext-xloader.git@master#egg=ckanext-xloader' && \ + pip3 install -r ${SRC_DIR}/ckanext-xloader/requirements.txt && \ + pip3 install -U requests[security] + +# Get artifacts from build stages +COPY --from=ckanbuild /wheels /srv/app/wheels +COPY --from=extbuild /wheels /srv/app/ext_wheels +COPY --from=ckanbuild /srv/app/src/ckan ${CKAN_DIR} + +# Additional install steps for build stages artifacts +RUN pip install --no-index --find-links=/srv/app/wheels uWSGI==2.0.20 gevent==22.10.2 + +# Create a local user and group to run the app +RUN addgroup -g 92 -S ckan && \ + adduser -u 92 -h /srv/app -H -D -S -G ckan ckan + +WORKDIR ${CKAN_DIR} + +# Install CKAN +RUN pip install -e /srv/app/src/ckan && \ + cp who.ini ${APP_DIR} && \ + pip install --no-index --find-links=/srv/app/wheels -r requirements.txt && \ + # Install default CKAN extensions + pip install --no-index --find-links=/srv/app/ext_wheels ckanext-envvars && \ + # Create and update CKAN config + # Set timezone + echo "UTC" > /etc/timezone && \ + # Generate CKAN config + ckan generate config ${APP_DIR}/production.ini && \ + ckan config-tool ${APP_DIR}/production.ini "beaker.session.secret = " && \ + # Configure plugins + ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \ + # Create the data directory + mkdir ${DATA_DIR} && \ + # Webassets can't be loaded from env variables at runtime, it needs to be in the config so that it is created + ckan config-tool ${APP_DIR}/production.ini "ckan.webassets.path = ${DATA_DIR}/webassets" && \ + # Set the default level for extensions to INFO + ckan config-tool ${APP_DIR}/production.ini -s logger_ckanext -e level=INFO && \ + # Change ownership to app user + chown -R ckan:ckan /srv/app + +# Remove wheels +RUN rm -rf /srv/app/wheels /srv/app/ext_wheels + +# Copy necessary scripts +COPY setup/app ${APP_DIR} + +WORKDIR ${APP_DIR} + +# Create entrypoint directory for children image scripts +RUN mkdir docker-entrypoint.d + +# Create afterinit directory for children image scripts +RUN mkdir docker-afterinit.d + +RUN pip install supervisor +RUN mkdir /var/supervisor +RUN mkdir /var/log/ckan +RUN mkdir /etc/supervisor/conf.d/ -p +COPY supervisorvalidation.conf /etc/supervisor/conf.d/supervisor-ckan-worker.conf +COPY supervisord.conf /etc/supervisord.conf +COPY supervisorstart.sh /srv/app/docker-afterinit.d + +# Change ownership to app user +RUN chown -R ckan:ckan ${APP_DIR} /var/supervisor /var/log/ckan /etc/supervisor/conf.d/ + +EXPOSE 5000 + +HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1 + +USER ckan + +CMD ["/srv/app/start_ckan.sh"] \ No newline at end of file diff --git a/images/ckan/2.10/supervisord.conf b/images/ckan/2.10/supervisord.conf new file mode 100644 index 0000000..82c3d19 --- /dev/null +++ b/images/ckan/2.10/supervisord.conf @@ -0,0 +1,23 @@ +; supervisor config file +[unix_http_server] +file=/var/supervisor/supervisor.sock ; (the path to the socket file) +chmod=0700 ; sockef file mode (default 0700) +[supervisord] +logfile=/var/log/ckan/supervisord.log ; (main log file;default $CWD/supervisord.log) +;pidfile=/var/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid) +;childlogdir=/var/log/supervisor ; ('AUTO' child log dir, default $TEMP) +; the below section must remain in the config file for RPC +; (supervisorctl/web interface) to work, additional interfaces may be +; added by defining them in separate rpcinterface: sections +user=ckan +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface +[supervisorctl] +serverurl=unix:///var/supervisor/supervisor.sock ; use a unix:// URL for a unix socket +; The [include] section can just contain the "files" setting. This +; setting can list multiple files (separated by whitespace or +; newlines). It can also contain wildcards. The filenames are +; interpreted as relative to this file. Included files *cannot* +; include files themselves. +[include] +files = /etc/supervisor/conf.d/*.conf \ No newline at end of file diff --git a/images/ckan/2.10/supervisorstart.sh b/images/ckan/2.10/supervisorstart.sh new file mode 100644 index 0000000..7c4858c --- /dev/null +++ b/images/ckan/2.10/supervisorstart.sh @@ -0,0 +1,2 @@ +#!/bin/sh +supervisord \ No newline at end of file diff --git a/images/ckan/2.10/supervisorvalidation.conf b/images/ckan/2.10/supervisorvalidation.conf new file mode 100644 index 0000000..d6082cf --- /dev/null +++ b/images/ckan/2.10/supervisorvalidation.conf @@ -0,0 +1,54 @@ +; ======================================================= +; Supervisor configuration for CKAN background job worker +; ======================================================= +; 1. Copy this file to /etc/supervisor/conf.d +; 2. Make sure the paths below match your setup +[program:ckan-worker-default] +command=ckan -c /srv/app/production.ini jobs worker default +; Log files. +redirect_stderr=true +stdout_logfile=/var/log/ckan/ckan-worker.log +; Make sure that the worker is started on system start and automatically +; restarted if it crashes unexpectedly. +autostart=true +autorestart=true +; Number of seconds the process has to run before it is considered to have +; started successfully. +startsecs=10 +; Need to wait for currently executing tasks to finish at shutdown. +; Increase this if you have very long running tasks. +stopwaitsecs = 600 + +[program:ckan-worker-bulk] +command=ckan -c /srv/app/production.ini jobs worker bulk +; Log files. +redirect_stderr=true +stdout_logfile=/var/log/ckan/ckan-worker.log +; Make sure that the worker is started on system start and automatically +; restarted if it crashes unexpectedly. +autostart=true +autorestart=true +; Number of seconds the process has to run before it is considered to have +; started successfully. +startsecs=10 +; Need to wait for currently executing tasks to finish at shutdown. +; Increase this if you have very long running tasks. +stopwaitsecs = 600 + +[program:ckan-worker-priority] +command=ckan -c /srv/app/production.ini jobs worker priority +numprocs=2 +process_name=%(program_name)s-%(process_num)02d +; Log files. +redirect_stderr=true +stdout_logfile=/var/log/ckan/ckan-worker.log +; Make sure that the worker is started on system start and automatically +; restarted if it crashes unexpectedly. +autostart=true +autorestart=true +; Number of seconds the process has to run before it is considered to have +; started successfully. +startsecs=10 +; Need to wait for currently executing tasks to finish at shutdown. +; Increase this if you have very long running tasks. +stopwaitsecs = 600 \ No newline at end of file