From 33da2a1b5e7f8da8647c8620130b4d4df910dfe7 Mon Sep 17 00:00:00 2001
From: "stojanovskis1@icloud.com" <stojanovskis1@icloud.com>
Date: Mon, 10 Jun 2024 14:01:28 +0200
Subject: [PATCH 1/3] Create new tag with xloader as default plugin,
 replacement for datapusher

---
 .github/workflows/master_merge.yml         |  17 ++
 .github/workflows/pr_checks.yml            |  15 ++
 .github/workflows/trivy_scan.yml           |  24 +++
 images/ckan/2.10/Dockerfile.xloader        | 207 +++++++++++++++++++++
 images/ckan/2.10/supervisord.conf          |  23 +++
 images/ckan/2.10/supervisorstart.sh        |   2 +
 images/ckan/2.10/supervisorvalidation.conf |  54 ++++++
 7 files changed, 342 insertions(+)
 create mode 100644 images/ckan/2.10/Dockerfile.xloader
 create mode 100644 images/ckan/2.10/supervisord.conf
 create mode 100644 images/ckan/2.10/supervisorstart.sh
 create mode 100644 images/ckan/2.10/supervisorvalidation.conf

diff --git a/.github/workflows/master_merge.yml b/.github/workflows/master_merge.yml
index b521325..1a200fa 100644
--- a/.github/workflows/master_merge.yml
+++ b/.github/workflows/master_merge.yml
@@ -54,6 +54,23 @@ jobs:
           cache-from: type=local,src=/tmp/.buildx-cache-alpine-2-10
           cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-alpine-2-10
 
+      - name: Get docker tag for alpine xloader image
+        id: alpine
+        run: |
+          echo "IMAGE_TAG=$(awk -F '=' '/IMAGE_TAG/{print $2}' ./images/ckan/2.10/Dockerfile.xloader)" >> $GITHUB_OUTPUT
+
+      - name: Build and push CKAN 2.10 alpine xloader
+        uses: docker/build-push-action@v5
+        with:
+          context: ./images/ckan/2.10
+          file: ./images/ckan/2.10/Dockerfile.xloader
+          push: true
+          tags: |
+            keitaro/ckan:${{ steps.alpine.outputs.IMAGE_TAG }}
+            ghcr.io/keitaroinc/ckan:${{ steps.alpine.outputs.IMAGE_TAG }}
+          cache-from: type=local,src=/tmp/.buildx-cache-alpine-2-10
+          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-alpine-2-10
+
       - name: Get docker tag for Ubuntu image
         id: ubuntu
         run: |
diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml
index 0bad5a2..528d153 100644
--- a/.github/workflows/pr_checks.yml
+++ b/.github/workflows/pr_checks.yml
@@ -39,6 +39,21 @@ jobs:
           cache-from: type=local,src=/tmp/.buildx-cache-alpine-2-10
           cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-alpine-2-10
 
+      - name: Get docker tag for Alpine xloader image
+        id: alpine
+        run: |
+          echo "IMAGE_TAG=$(awk -F '=' '/IMAGE_TAG/{print $2}' ./images/ckan/2.10/Dockerfile.xloader)" >> $GITHUB_OUTPUT
+
+      - name: Build CKAN 2.10 alpine
+        uses: docker/build-push-action@v5
+        with:
+          context: ./images/ckan/2.10
+          file: ./images/ckan/2.10/Dockerfile.xloader
+          push: false
+          tags: keitaro/ckan:${{ steps.alpine.outputs.IMAGE_TAG }}
+          cache-from: type=local,src=/tmp/.buildx-cache-alpine-2-10
+          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-alpine-2-10
+
       - name: Get docker tag for Ubuntu image
         id: ubuntu
         run: |
diff --git a/.github/workflows/trivy_scan.yml b/.github/workflows/trivy_scan.yml
index 8bce2d1..bbc4554 100644
--- a/.github/workflows/trivy_scan.yml
+++ b/.github/workflows/trivy_scan.yml
@@ -55,6 +55,30 @@ jobs:
         with:
           sarif_file: 'trivy-results.sarif'
 
+  scan_2_10_xloader:
+    name: scan_2_10_xloader
+    runs-on: ubuntu-20.04
+    steps:
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Build an image from Dockerfile
+        run: |
+          docker build -t keitaro/ckan/2.10:${{ github.sha }} -f ./images/ckan/2.10/Dockerfile.xloader ./images/ckan/2.10/
+
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: 'keitaro/ckan/2.10:${{ github.sha }}'
+          format: 'sarif'
+          output: 'trivy-results.sarif'
+
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v3
+        with:
+          sarif_file: 'trivy-results.sarif'
+
 
   scan_2_10_focal:
     name: scan_2_10_focal
diff --git a/images/ckan/2.10/Dockerfile.xloader b/images/ckan/2.10/Dockerfile.xloader
new file mode 100644
index 0000000..ed19c36
--- /dev/null
+++ b/images/ckan/2.10/Dockerfile.xloader
@@ -0,0 +1,207 @@
+##################
+### Build CKAN ###
+##################
+FROM alpine:3.17.2 as ckanbuild
+
+# Used by Github Actions to tag the image with
+ENV IMAGE_TAG=2.10.4
+
+# Set CKAN version to build
+ENV GIT_URL=https://github.com/ckan/ckan.git
+ENV GIT_BRANCH=ckan-2.10.4
+
+# Set src dirs
+ENV SRC_DIR=/srv/app/src
+ENV PIP_SRC=${SRC_DIR}
+
+WORKDIR ${SRC_DIR}
+
+# Packages to build CKAN requirements and plugins
+RUN apk add --no-cache \
+        python3 \
+        python3-dev \
+        git \
+        curl \
+        postgresql-dev \
+        linux-headers \
+        gcc \
+        make \
+        g++ \
+        autoconf \
+        automake \
+        libtool \
+        patch \
+        musl-dev \
+        pcre-dev \
+        pcre \
+        libffi-dev \
+        libxml2-dev \
+        libxslt-dev
+
+# Create the src directory
+RUN mkdir -p ${SRC_DIR}
+
+# Install pip
+RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
+    python ${SRC_DIR}/get-pip.py
+    
+# Downgrade setuptools so that CKAN requirements can be built
+RUN pip install setuptools==44.1.0
+
+# Fetch and build CKAN and requirements
+RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan
+# Copy patches and apply patches script
+COPY ./patches ${SRC_DIR}/patches
+COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh
+# Apply patches
+# RUN ${SRC_DIR}/apply_ckan_patches.sh
+RUN rm -rf /srv/app/src/ckan/.git
+RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt
+RUN pip wheel --wheel-dir=/wheels uWSGI==2.0.20 gevent==22.10.2 greenlet==2.0.2
+
+
+###########################
+### Default-Extensions ####
+###########################
+FROM alpine:3.17.2 as extbuild
+
+# Set src dirs
+ENV SRC_DIR=/srv/app/src
+ENV PIP_SRC=${SRC_DIR}
+
+# List of default extensions
+ENV DEFAULT_EXTENSIONS envvars
+
+# Locations and tags, please use specific tags or revisions
+ENV ENVVARS_GIT_URL=https://github.com/okfn/ckanext-envvars
+ENV ENVVARS_GIT_BRANCH=0.0.2
+
+RUN apk add --no-cache \
+        python3 \
+        python3-dev \
+        git \
+        curl 
+
+# Create the src directory
+RUN mkdir -p ${SRC_DIR}
+
+# Install pip
+RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
+    python ${SRC_DIR}/get-pip.py
+
+# Fetch and build the default CKAN extensions
+RUN pip wheel --wheel-dir=/wheels git+${ENVVARS_GIT_URL}@${ENVVARS_GIT_BRANCH}#egg=ckanext-envvars
+
+############
+### MAIN ###
+############
+FROM alpine:3.17.2
+
+LABEL maintainer="Keitaro Inc <info@keitaro.com>"
+LABEL org.opencontainers.image.source https://github.com/keitaroinc/docker-ckan
+
+ENV APP_DIR=/srv/app
+ENV SRC_DIR=/srv/app/src
+ENV CKAN_DIR=${SRC_DIR}/ckan
+ENV DATA_DIR=/srv/app/data
+ENV PIP_SRC=${SRC_DIR}
+ENV CKAN_SITE_URL=http://localhost:5000
+ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore xloader
+
+# Install necessary packages to run CKAN
+RUN apk add --no-cache \
+        supervisor \
+        python3 \
+        bash \
+        git \
+        gettext \
+        curl \
+        postgresql-client \
+        libmagic \
+        pcre \
+        libxslt \
+        libxml2 \
+        tzdata \
+        apache2-utils && \
+    # Create SRC_DIR
+    mkdir -p ${SRC_DIR}
+
+
+# Install pip
+RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \
+    python ${SRC_DIR}/get-pip.py
+
+    ### XLoader ###
+RUN pip3 install -e 'git+https://github.com/keitaroinc/ckanext-xloader.git@master#egg=ckanext-xloader' && \ 
+   pip3 install -r ${SRC_DIR}/ckanext-xloader/requirements.txt && \
+   pip3 install -U requests[security]
+
+# Get artifacts from build stages
+COPY --from=ckanbuild /wheels /srv/app/wheels
+COPY --from=extbuild /wheels /srv/app/ext_wheels
+COPY --from=ckanbuild /srv/app/src/ckan ${CKAN_DIR}
+
+# Additional install steps for build stages artifacts
+RUN pip install --no-index --find-links=/srv/app/wheels uWSGI==2.0.20 gevent==22.10.2
+
+# Create a local user and group to run the app
+RUN addgroup -g 92 -S ckan && \
+    adduser -u 92 -h /srv/app -H -D -S -G ckan ckan
+
+WORKDIR ${CKAN_DIR}
+
+# Install CKAN
+RUN pip install -e /srv/app/src/ckan && \
+    cp who.ini ${APP_DIR} && \
+    pip install --no-index --find-links=/srv/app/wheels -r requirements.txt && \
+    # Install default CKAN extensions
+    pip install --no-index --find-links=/srv/app/ext_wheels ckanext-envvars && \
+    # Create and update CKAN config
+    # Set timezone
+    echo "UTC" >  /etc/timezone && \
+    # Generate CKAN config
+    ckan generate config ${APP_DIR}/production.ini && \
+    ckan config-tool ${APP_DIR}/production.ini "beaker.session.secret = " && \
+    # Configure plugins
+    ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \
+    # Create the data directory
+    mkdir ${DATA_DIR} && \
+    # Webassets can't be loaded from env variables at runtime, it needs to be in the config so that it is created
+    ckan config-tool ${APP_DIR}/production.ini "ckan.webassets.path = ${DATA_DIR}/webassets" && \
+    # Set the default level for extensions to INFO
+    ckan config-tool ${APP_DIR}/production.ini -s logger_ckanext -e level=INFO && \
+    # Change ownership to app user
+    chown -R ckan:ckan /srv/app
+
+# Remove wheels
+RUN rm -rf /srv/app/wheels /srv/app/ext_wheels
+
+# Copy necessary scripts
+COPY setup/app ${APP_DIR}
+
+WORKDIR ${APP_DIR}
+
+# Create entrypoint directory for children image scripts
+RUN mkdir docker-entrypoint.d
+
+# Create afterinit directory for children image scripts
+RUN mkdir docker-afterinit.d
+
+RUN pip install supervisor
+RUN mkdir /var/supervisor
+RUN mkdir /var/log/ckan
+RUN mkdir /etc/supervisor/conf.d/ -p
+COPY supervisorvalidation.conf /etc/supervisor/conf.d/supervisor-ckan-worker.conf
+COPY supervisord.conf /etc/supervisord.conf
+COPY supervisorstart.sh /srv/app/docker-afterinit.d
+
+# Change ownership to app user
+RUN chown -R ckan:ckan ${APP_DIR} /var/supervisor /var/log/ckan /etc/supervisor/conf.d/
+
+EXPOSE 5000
+
+HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1
+
+USER ckan
+
+CMD ["/srv/app/start_ckan.sh"]
\ No newline at end of file
diff --git a/images/ckan/2.10/supervisord.conf b/images/ckan/2.10/supervisord.conf
new file mode 100644
index 0000000..82c3d19
--- /dev/null
+++ b/images/ckan/2.10/supervisord.conf
@@ -0,0 +1,23 @@
+; supervisor config file
+[unix_http_server]
+file=/var/supervisor/supervisor.sock   ; (the path to the socket file)
+chmod=0700                       ; sockef file mode (default 0700)
+[supervisord]
+logfile=/var/log/ckan/supervisord.log ; (main log file;default $CWD/supervisord.log)
+;pidfile=/var/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid)
+;childlogdir=/var/log/supervisor            ; ('AUTO' child log dir, default $TEMP)
+; the below section must remain in the config file for RPC
+; (supervisorctl/web interface) to work, additional interfaces may be
+; added by defining them in separate rpcinterface: sections
+user=ckan
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+[supervisorctl]
+serverurl=unix:///var/supervisor/supervisor.sock ; use a unix:// URL  for a unix socket
+; The [include] section can just contain the "files" setting.  This
+; setting can list multiple files (separated by whitespace or
+; newlines).  It can also contain wildcards.  The filenames are
+; interpreted as relative to this file.  Included files *cannot*
+; include files themselves.
+[include]
+files = /etc/supervisor/conf.d/*.conf
\ No newline at end of file
diff --git a/images/ckan/2.10/supervisorstart.sh b/images/ckan/2.10/supervisorstart.sh
new file mode 100644
index 0000000..7c4858c
--- /dev/null
+++ b/images/ckan/2.10/supervisorstart.sh
@@ -0,0 +1,2 @@
+#!/bin/sh
+supervisord
\ No newline at end of file
diff --git a/images/ckan/2.10/supervisorvalidation.conf b/images/ckan/2.10/supervisorvalidation.conf
new file mode 100644
index 0000000..d6082cf
--- /dev/null
+++ b/images/ckan/2.10/supervisorvalidation.conf
@@ -0,0 +1,54 @@
+; =======================================================
+; Supervisor configuration for CKAN background job worker
+; =======================================================
+; 1. Copy this file to /etc/supervisor/conf.d
+; 2. Make sure the paths below match your setup
+[program:ckan-worker-default]
+command=ckan -c /srv/app/production.ini jobs worker default
+; Log files.
+redirect_stderr=true
+stdout_logfile=/var/log/ckan/ckan-worker.log
+; Make sure that the worker is started on system start and automatically
+; restarted if it crashes unexpectedly.
+autostart=true
+autorestart=true
+; Number of seconds the process has to run before it is considered to have
+; started successfully.
+startsecs=10
+; Need to wait for currently executing tasks to finish at shutdown.
+; Increase this if you have very long running tasks.
+stopwaitsecs = 600
+
+[program:ckan-worker-bulk]
+command=ckan -c /srv/app/production.ini jobs worker bulk
+; Log files.
+redirect_stderr=true
+stdout_logfile=/var/log/ckan/ckan-worker.log
+; Make sure that the worker is started on system start and automatically
+; restarted if it crashes unexpectedly.
+autostart=true
+autorestart=true
+; Number of seconds the process has to run before it is considered to have
+; started successfully.
+startsecs=10
+; Need to wait for currently executing tasks to finish at shutdown.
+; Increase this if you have very long running tasks.
+stopwaitsecs = 600
+
+[program:ckan-worker-priority]
+command=ckan -c /srv/app/production.ini jobs worker priority
+numprocs=2
+process_name=%(program_name)s-%(process_num)02d
+; Log files.
+redirect_stderr=true
+stdout_logfile=/var/log/ckan/ckan-worker.log
+; Make sure that the worker is started on system start and automatically
+; restarted if it crashes unexpectedly.
+autostart=true
+autorestart=true
+; Number of seconds the process has to run before it is considered to have
+; started successfully.
+startsecs=10
+; Need to wait for currently executing tasks to finish at shutdown.
+; Increase this if you have very long running tasks.
+stopwaitsecs = 600
\ No newline at end of file

From 8e6cebfc520c6e0e259170cb0d9b4a1e63c79f2c Mon Sep 17 00:00:00 2001
From: "stojanovskis1@icloud.com" <stojanovskis1@icloud.com>
Date: Mon, 10 Jun 2024 14:04:16 +0200
Subject: [PATCH 2/3] Change id in github actions

---
 .github/workflows/master_merge.yml | 2 +-
 .github/workflows/pr_checks.yml    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/master_merge.yml b/.github/workflows/master_merge.yml
index 1a200fa..f6d9fbc 100644
--- a/.github/workflows/master_merge.yml
+++ b/.github/workflows/master_merge.yml
@@ -55,7 +55,7 @@ jobs:
           cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-alpine-2-10
 
       - name: Get docker tag for alpine xloader image
-        id: alpine
+        id: alpine-xloader
         run: |
           echo "IMAGE_TAG=$(awk -F '=' '/IMAGE_TAG/{print $2}' ./images/ckan/2.10/Dockerfile.xloader)" >> $GITHUB_OUTPUT
 
diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml
index 528d153..749e635 100644
--- a/.github/workflows/pr_checks.yml
+++ b/.github/workflows/pr_checks.yml
@@ -40,7 +40,7 @@ jobs:
           cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-alpine-2-10
 
       - name: Get docker tag for Alpine xloader image
-        id: alpine
+        id: alpine-xloader
         run: |
           echo "IMAGE_TAG=$(awk -F '=' '/IMAGE_TAG/{print $2}' ./images/ckan/2.10/Dockerfile.xloader)" >> $GITHUB_OUTPUT
 

From fca13ab8d89b6629f3b84bd3b8dd5264d169f8ff Mon Sep 17 00:00:00 2001
From: "stojanovskis1@icloud.com" <stojanovskis1@icloud.com>
Date: Mon, 10 Jun 2024 14:07:18 +0200
Subject: [PATCH 3/3] Change name in github actions

---
 .github/workflows/pr_checks.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml
index 749e635..2083bc1 100644
--- a/.github/workflows/pr_checks.yml
+++ b/.github/workflows/pr_checks.yml
@@ -44,7 +44,7 @@ jobs:
         run: |
           echo "IMAGE_TAG=$(awk -F '=' '/IMAGE_TAG/{print $2}' ./images/ckan/2.10/Dockerfile.xloader)" >> $GITHUB_OUTPUT
 
-      - name: Build CKAN 2.10 alpine
+      - name: Build CKAN 2.10 alpine xloader
         uses: docker/build-push-action@v5
         with:
           context: ./images/ckan/2.10