From 5b8a2358fa3728d1edd1aa3e8c6a9ed5752df8c3 Mon Sep 17 00:00:00 2001 From: "filip.mihajlovski" Date: Thu, 16 Feb 2023 16:42:29 +0100 Subject: [PATCH 1/7] created ckan2.10 image for alpine --- images/ckan/2.10/Dockerfile | 190 ++++++++++++++ images/ckan/2.10/Dockerfile.focal | 243 ++++++++++++++++++ images/ckan/2.10/patches/.gitkeep | 0 .../ckan/2.10/scripts/apply_ckan_patches.sh | 5 + images/ckan/2.10/setup/app/extra_scripts.sh | 4 + .../api/3/action/status_show/index.html | 25 ++ .../2.10/setup/app/maintenance/index.html | 25 ++ .../ckan/2.10/setup/app/maintenance/serve.py | 35 +++ images/ckan/2.10/setup/app/prerun.py | 230 +++++++++++++++++ images/ckan/2.10/setup/app/start_ckan.sh | 66 +++++ images/ckan/2.10/setup/app/uwsgi.conf | 2 + images/ckan/2.10/setup/app/wsgi.py | 28 ++ 12 files changed, 853 insertions(+) create mode 100644 images/ckan/2.10/Dockerfile create mode 100644 images/ckan/2.10/Dockerfile.focal create mode 100644 images/ckan/2.10/patches/.gitkeep create mode 100755 images/ckan/2.10/scripts/apply_ckan_patches.sh create mode 100755 images/ckan/2.10/setup/app/extra_scripts.sh create mode 100644 images/ckan/2.10/setup/app/maintenance/api/3/action/status_show/index.html create mode 100644 images/ckan/2.10/setup/app/maintenance/index.html create mode 100644 images/ckan/2.10/setup/app/maintenance/serve.py create mode 100644 images/ckan/2.10/setup/app/prerun.py create mode 100755 images/ckan/2.10/setup/app/start_ckan.sh create mode 100644 images/ckan/2.10/setup/app/uwsgi.conf create mode 100644 images/ckan/2.10/setup/app/wsgi.py diff --git a/images/ckan/2.10/Dockerfile b/images/ckan/2.10/Dockerfile new file mode 100644 index 0000000..becda88 --- /dev/null +++ b/images/ckan/2.10/Dockerfile @@ -0,0 +1,190 @@ +################## +### Build CKAN ### +################## +FROM alpine:3.17.2 as ckanbuild + +# Used by Github Actions to tag the image with +ENV IMAGE_TAG=2.10.0 + +# Set CKAN version to build +ENV GIT_URL=https://github.com/ckan/ckan.git +ENV GIT_BRANCH=ckan-2.10.0 + +# Set src dirs +ENV SRC_DIR=/srv/app/src +ENV PIP_SRC=${SRC_DIR} + +WORKDIR ${SRC_DIR} + +# Packages to build CKAN requirements and plugins +RUN apk add --no-cache \ + python3 \ + python3-dev \ + git \ + curl \ + postgresql-dev \ + linux-headers \ + gcc \ + make \ + g++ \ + autoconf \ + automake \ + libtool \ + patch \ + musl-dev \ + pcre-dev \ + pcre \ + libffi-dev \ + libxml2-dev \ + libxslt-dev + +# Create the src directory +RUN mkdir -p ${SRC_DIR} + +# Install pip +RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ + python ${SRC_DIR}/get-pip.py + +# Downgrade setuptools so that CKAN requirements can be built +RUN pip install setuptools==44.1.0 + +# Fetch and build CKAN and requirements +RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan +# Copy patches and apply patches script +COPY ./patches ${SRC_DIR}/patches +COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh +# Apply patches +# RUN ${SRC_DIR}/apply_ckan_patches.sh +RUN rm -rf /srv/app/src/ckan/.git +RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt +RUN pip wheel --wheel-dir=/wheels uWSGI==2.0.20 gevent==21.12.0 greenlet==1.1.3 + + +########################### +### Default-Extensions #### +########################### +FROM alpine:3.17.2 as extbuild + +# Set src dirs +ENV SRC_DIR=/srv/app/src +ENV PIP_SRC=${SRC_DIR} + +# List of default extensions +ENV DEFAULT_EXTENSIONS envvars + +# Locations and tags, please use specific tags or revisions +ENV ENVVARS_GIT_URL=https://github.com/okfn/ckanext-envvars +ENV ENVVARS_GIT_BRANCH=0.0.2 + +RUN apk add --no-cache \ + python3 \ + python3-dev \ + git \ + curl + +# Create the src directory +RUN mkdir -p ${SRC_DIR} + +# Install pip +RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ + python ${SRC_DIR}/get-pip.py + +# Fetch and build the default CKAN extensions +RUN pip wheel --wheel-dir=/wheels git+${ENVVARS_GIT_URL}@${ENVVARS_GIT_BRANCH}#egg=ckanext-envvars + +############ +### MAIN ### +############ +FROM alpine:3.17.2 + +LABEL maintainer="Keitaro Inc " +LABEL org.opencontainers.image.source https://github.com/keitaroinc/docker-ckan + +ENV APP_DIR=/srv/app +ENV SRC_DIR=/srv/app/src +ENV CKAN_DIR=${SRC_DIR}/ckan +ENV DATA_DIR=/srv/app/data +ENV PIP_SRC=${SRC_DIR} +ENV CKAN_SITE_URL=http://localhost:5000 +ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore datapusher + +# Install necessary packages to run CKAN +RUN apk add --no-cache \ + python3 \ + bash \ + git \ + gettext \ + curl \ + postgresql-client \ + libmagic \ + pcre \ + libxslt \ + libxml2 \ + tzdata \ + apache2-utils && \ + # Create SRC_DIR + mkdir -p ${SRC_DIR} + + +# Install pip +RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ + python ${SRC_DIR}/get-pip.py + +# Get artifacts from build stages +COPY --from=ckanbuild /wheels /srv/app/wheels +COPY --from=extbuild /wheels /srv/app/ext_wheels +COPY --from=ckanbuild /srv/app/src/ckan ${CKAN_DIR} + +# Additional install steps for build stages artifacts +RUN pip install --no-index --find-links=/srv/app/wheels uWSGI==2.0.20 gevent==21.12.0 + +# Create a local user and group to run the app +RUN addgroup -g 92 -S ckan && \ + adduser -u 92 -h /srv/app -H -D -S -G ckan ckan + +WORKDIR ${CKAN_DIR} + +# Install CKAN +RUN pip install -e /srv/app/src/ckan && \ + cp who.ini ${APP_DIR} && \ + pip install --no-index --find-links=/srv/app/wheels -r requirements.txt && \ + # Install default CKAN extensions + pip install --no-index --find-links=/srv/app/ext_wheels ckanext-envvars && \ + # Create and update CKAN config + # Set timezone + echo "UTC" > /etc/timezone && \ + # Generate CKAN config + ckan generate config ${APP_DIR}/production.ini && \ + ckan config-tool ${APP_DIR}/production.ini "beaker.session.secret = " && \ + # Configure plugins + ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \ + # Create the data directory + mkdir ${DATA_DIR} && \ + # Webassets can't be loaded from env variables at runtime, it needs to be in the config so that it is created + ckan config-tool ${APP_DIR}/production.ini "ckan.webassets.path = ${DATA_DIR}/webassets" && \ + # Set the default level for extensions to INFO + ckan config-tool ${APP_DIR}/production.ini -s logger_ckanext -e level=INFO && \ + # Change ownership to app user + chown -R ckan:ckan /srv/app + +# Remove wheels +RUN rm -rf /srv/app/wheels /srv/app/ext_wheels + +# Copy necessary scripts +COPY setup/app ${APP_DIR} + +WORKDIR ${APP_DIR} + +# Create entrypoint directory for children image scripts +ONBUILD RUN mkdir docker-entrypoint.d + +# Create afterinit directory for children image scripts +ONBUILD RUN mkdir docker-afterinit.d + +EXPOSE 5000 + +HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1 + +USER ckan + +CMD ["/srv/app/start_ckan.sh"] diff --git a/images/ckan/2.10/Dockerfile.focal b/images/ckan/2.10/Dockerfile.focal new file mode 100644 index 0000000..369d7fd --- /dev/null +++ b/images/ckan/2.10/Dockerfile.focal @@ -0,0 +1,243 @@ +################## +### Build CKAN ### +################## +FROM ubuntu:focal-20210827 as ckanbuild + +# Used by Github Actions to tag the image with +ENV IMAGE_TAG=2.9.7-focal + +# Set CKAN version to build +ENV GIT_URL=https://github.com/ckan/ckan.git +ENV GIT_BRANCH=ckan-2.9.7 + +# Set timezone +ENV TZ=UTC +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +# Set Locale +ENV LC_ALL=en_US.UTF-8 + +# Set src dirs +ENV SRC_DIR=/srv/app/src +ENV PIP_SRC=${SRC_DIR} + +WORKDIR ${SRC_DIR} + +# Set the locale +RUN apt-get update +RUN apt-get install --no-install-recommends -y locales +RUN sed -i "/$LC_ALL/s/^# //g" /etc/locale.gen +RUN dpkg-reconfigure --frontend=noninteractive locales +RUN update-locale LANG=${LC_ALL} + +# Instal apt-utils +RUN apt-get install --no-install-recommends -y \ + apt-utils + +# Packages to build CKAN requirements and plugins +RUN apt-get install --no-install-recommends -y \ + git \ + curl \ + ca-certificates \ + python3 \ + libpq-dev \ + linux-headers-generic \ + gcc-10 \ + make \ + g++-10 \ + autoconf \ + automake \ + libtool \ + patch \ + libpcre3-dev \ + libpcre3 \ + python3-dev \ + libffi-dev \ + libxml2-dev \ + libxslt-dev + +# Use gcc 10 +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 --slave /usr/bin/g++ g++ /usr/bin/g++-10 --slave /usr/bin/gcov gcov /usr/bin/gcov-10 --slave /usr/bin/x86_64-linux-gnu-gcc x86_64-linux-gnu-gcc /usr/bin/x86_64-linux-gnu-gcc-10 + +# Link python to python3 +RUN ln -s /usr/bin/python3 /usr/bin/python + +# Create the src directory +RUN mkdir -p ${SRC_DIR} + +# Install pip +RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ + python ${SRC_DIR}/get-pip.py + +# Downgrade setuptools so that CKAN requirements can be built +RUN pip install setuptools==44.1.0 + +# Fetch and build CKAN and requirements +RUN pip install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan +# Copy patches and apply patches script +COPY ./patches ${SRC_DIR}/patches +COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh +# Apply patches +RUN ${SRC_DIR}/apply_ckan_patches.sh +RUN rm -rf /srv/app/src/ckan/.git +RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt +RUN pip wheel --wheel-dir=/wheels uWSGI==2.0.20 gevent==21.12.0 greenlet==1.1.3 + + +########################### +### Default-Extensions #### +########################### +FROM ubuntu:focal-20210827 as extbuild + +# Set src dirs +ENV SRC_DIR=/srv/app/src +ENV PIP_SRC=${SRC_DIR} + +# List of default extensions +ENV DEFAULT_EXTENSIONS envvars + +# Locations and tags, please use specific tags or revisions +ENV ENVVARS_GIT_URL=https://github.com/okfn/ckanext-envvars +ENV ENVVARS_GIT_BRANCH=0.0.1 + +RUN apt-get update && \ + apt-get install --no-install-recommends -y \ + git \ + curl \ + ca-certificates \ + python3 \ + python3-dev + +# Link python to python3 +RUN ln -s /usr/bin/python3 /usr/bin/python + +# Create the src directory +RUN mkdir -p ${SRC_DIR} + +# Install pip +RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ + python ${SRC_DIR}/get-pip.py + +# Downgrade setuptools so that CKAN requirements can be built +RUN pip install setuptools==44.1.0 + +# Fetch and build the default CKAN extensions +RUN pip wheel --wheel-dir=/wheels git+${ENVVARS_GIT_URL}@${ENVVARS_GIT_BRANCH}#egg=ckanext-envvars + +############ +### MAIN ### +############ +FROM ubuntu:focal-20210827 + +LABEL maintainer="Keitaro Inc " +LABEL org.opencontainers.image.source https://github.com/keitaroinc/docker-ckan + +# Set timezone +ENV TZ=UTC +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +# Set Locale +ENV LC_ALL=en_US.UTF-8 + +# Set the locale +RUN apt-get update && \ + apt-get install --no-install-recommends -y locales && \ + sed -i "/$LC_ALL/s/^# //g" /etc/locale.gen && \ + dpkg-reconfigure --frontend=noninteractive locales && \ + update-locale LANG=${LC_ALL} && \ + rm -rf /var/lib/apt/lists/* + +ENV APP_DIR=/srv/app +ENV SRC_DIR=/srv/app/src +ENV CKAN_DIR=${SRC_DIR}/ckan +ENV DATA_DIR=/srv/app/data +ENV PIP_SRC=${SRC_DIR} +ENV CKAN_SITE_URL=http://localhost:5000 +ENV CKAN__PLUGINS envvars image_view text_view recline_view datastore datapusher + +# Install necessary packages to run CKAN +RUN apt-get update && \ + apt-get install --no-install-recommends -y \ + gettext \ + curl \ + ca-certificates \ + libpq5 \ + git \ + postgresql-client \ + python3 \ + python3-distutils \ + libpython3.8 \ + libmagic1 \ + libpcre3 \ + libxslt1.1 \ + libxml2 \ + tzdata \ + apache2-utils && \ + rm -rf /var/lib/apt/lists/* && \ + # Create SRC_DIR + mkdir -p ${SRC_DIR} && \ + # Link python to python3 + ln -s /usr/bin/python3 /usr/bin/python + +# Install pip +RUN curl -o ${SRC_DIR}/get-pip.py https://bootstrap.pypa.io/get-pip.py && \ + python ${SRC_DIR}/get-pip.py + +# Downgrade setuptools so that CKAN requirements can be built +RUN pip install setuptools==44.1.0 + +# Get artifacts from build stages +COPY --from=ckanbuild /wheels /srv/app/wheels +COPY --from=extbuild /wheels /srv/app/ext_wheels +COPY --from=ckanbuild /srv/app/src/ckan ${CKAN_DIR} + +# Additional install steps for build stages artifacts +RUN pip install --no-index --find-links=/srv/app/wheels uWSGI==2.0.20 gevent==21.12.0 + +# Create a local user and group to run the app +RUN groupadd -g 92 ckan && \ + useradd -rm -d /srv/app -s /bin/bash -g ckan -u 92 ckan + +WORKDIR ${CKAN_DIR} + +# Install CKAN +RUN pip install -e /srv/app/src/ckan && \ + cp who.ini ${APP_DIR} && \ + pip install --no-index --find-links=/srv/app/wheels -r requirements.txt && \ + # Install default CKAN extensions + pip install --no-index --find-links=/srv/app/ext_wheels ckanext-envvars && \ + # Create and update CKAN config + # Generate CKAN config + ckan generate config ${APP_DIR}/production.ini && \ + # Configure plugins + ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}" && \ + # Create the data directory + mkdir ${DATA_DIR} && \ + # Webassets can't be loaded from env variables at runtime, it needs to be in the config so that it is created + ckan config-tool ${APP_DIR}/production.ini "ckan.webassets.path = ${DATA_DIR}/webassets" && \ + # Set the default level for extensions to INFO + ckan config-tool ${APP_DIR}/production.ini -s logger_ckanext -e level=INFO && \ + # Change ownership to app user + chown -R ckan:ckan /srv/app + +# Remove wheels +RUN rm -rf /srv/app/wheels /srv/app/ext_wheels + +# Copy necessary scripts +COPY setup/app ${APP_DIR} + +WORKDIR ${APP_DIR} + +# Create entrypoint directory for children image scripts +ONBUILD RUN mkdir docker-entrypoint.d + +# Create afterinit directory for children image scripts +ONBUILD RUN mkdir docker-afterinit.d + +EXPOSE 5000 + +HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit 1 + +USER ckan + +CMD ["/srv/app/start_ckan.sh"] diff --git a/images/ckan/2.10/patches/.gitkeep b/images/ckan/2.10/patches/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/images/ckan/2.10/scripts/apply_ckan_patches.sh b/images/ckan/2.10/scripts/apply_ckan_patches.sh new file mode 100755 index 0000000..a7bceb9 --- /dev/null +++ b/images/ckan/2.10/scripts/apply_ckan_patches.sh @@ -0,0 +1,5 @@ +#!/bin/bash +shopt -s nullglob +for patch in patches/*.patch; do + /usr/bin/patch -p0 -i $patch +done diff --git a/images/ckan/2.10/setup/app/extra_scripts.sh b/images/ckan/2.10/setup/app/extra_scripts.sh new file mode 100755 index 0000000..80a70ef --- /dev/null +++ b/images/ckan/2.10/setup/app/extra_scripts.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +# this is called before uwsgi is executed +# uset his to add extra scripts before ckan is started diff --git a/images/ckan/2.10/setup/app/maintenance/api/3/action/status_show/index.html b/images/ckan/2.10/setup/app/maintenance/api/3/action/status_show/index.html new file mode 100644 index 0000000..50276ec --- /dev/null +++ b/images/ckan/2.10/setup/app/maintenance/api/3/action/status_show/index.html @@ -0,0 +1,25 @@ + + + + + Maintenance + + +

Maintenance

+

Our data portal is currently in maintenance, please try in a while.

+ + diff --git a/images/ckan/2.10/setup/app/maintenance/index.html b/images/ckan/2.10/setup/app/maintenance/index.html new file mode 100644 index 0000000..50276ec --- /dev/null +++ b/images/ckan/2.10/setup/app/maintenance/index.html @@ -0,0 +1,25 @@ + + + + + Maintenance + + +

Maintenance

+

Our data portal is currently in maintenance, please try in a while.

+ + diff --git a/images/ckan/2.10/setup/app/maintenance/serve.py b/images/ckan/2.10/setup/app/maintenance/serve.py new file mode 100644 index 0000000..77f3bd4 --- /dev/null +++ b/images/ckan/2.10/setup/app/maintenance/serve.py @@ -0,0 +1,35 @@ +""" +Copyright (c) 2016 Keitaro AB + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from http.server import ThreadingHTTPServer, SimpleHTTPRequestHandler +import os + +PORT = 5000 + +web_dir = os.path.join(os.path.dirname(__file__)) +os.chdir(web_dir) + + +def run(server_class=ThreadingHTTPServer, handler_class=SimpleHTTPRequestHandler): + server_address = ("0.0.0.0", PORT) + httpd = server_class(server_address, handler_class) + print("Starting maintenance mode") + httpd.serve_forever() + + +if __name__ == "__main__": + run() + diff --git a/images/ckan/2.10/setup/app/prerun.py b/images/ckan/2.10/setup/app/prerun.py new file mode 100644 index 0000000..64a56bc --- /dev/null +++ b/images/ckan/2.10/setup/app/prerun.py @@ -0,0 +1,230 @@ +""" +Copyright (c) 2016 Keitaro AB + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import os +import sys +import subprocess +import psycopg2 +from sqlalchemy.engine.url import make_url +import urllib.request, urllib.error, urllib.parse +import re +import json + +import time + +ckan_ini = os.environ.get('CKAN_INI', '/srv/app/production.ini') + +RETRY = 5 + +def check_db_connection(retry=None): + + print('[prerun] Start check_db_connection...') + + if retry is None: + retry = RETRY + elif retry == 0: + print('[prerun] Giving up after 5 tries...') + sys.exit(1) + + conn_str = os.environ.get('CKAN_SQLALCHEMY_URL', '') + try: + db_user = make_url(conn_str).username + db_passwd = make_url(conn_str).password + db_host = make_url(conn_str).host + db_name = make_url(conn_str).database + connection = psycopg2.connect(user=db_user, + host=db_host, + password=db_passwd, + database=db_name) + + except psycopg2.Error as e: + print((str(e))) + print('[prerun] Unable to connect to the database...try again in a while.') + import time + time.sleep(10) + check_db_connection(retry = retry - 1) + else: + connection.close() + +def check_solr_connection(retry=None): + + print('[prerun] Start check_solr_connection...') + + if retry is None: + retry = RETRY + elif retry == 0: + print('[prerun] Giving up after 5 tries...') + sys.exit(1) + + url = os.environ.get('CKAN_SOLR_URL', '') + username = os.environ.get('SOLR_ADMIN_USERNAME', 'admin') + password = os.environ.get('SOLR_ADMIN_PASSWORD', 'pass') + search_url = '{url}/schema/name?wt=json'.format(url=url) + + try: + if not username: + connection = urllib.request.urlopen(search_url) + else: + passman = urllib.request.HTTPPasswordMgrWithDefaultRealm() + passman.add_password(None, search_url, username, password) + authhandler = urllib.request.HTTPBasicAuthHandler(passman) + opener = urllib.request.build_opener(authhandler) + urllib.request.install_opener(opener) + connection = urllib.request.urlopen(search_url) + except urllib.error.URLError as e: + print('[prerun] Unable to connect to solr...try again in a while.') + import time + time.sleep(10) + check_solr_connection(retry = retry - 1) + else: + import re + conn_info = connection.read() + schema_name = json.loads(conn_info) + if 'ckan' in schema_name['name']: + print('[prerun] Succesfully connected to solr and CKAN schema loaded') + else: + print('[prerun] Succesfully connected to solr, but CKAN schema not found') + sys.exit(1) + +def init_db(): + + print('[prerun] Start init_db...') + + db_command = ['ckan', '-c', ckan_ini, 'db', 'init'] + + print('[prerun] Initializing or upgrading db - start using ckan db init') + try: + # run init scripts + subprocess.check_output(db_command, stderr=subprocess.STDOUT) + + print('[prerun] Initializing or upgrading db - end') + except subprocess.CalledProcessError as e: + if 'OperationalError' in str(e.output): + print(e.output.decode('utf-8')) + print('[prerun] Database not ready, waiting a bit before exit...') + import time + time.sleep(5) + sys.exit(1) + else: + print(e.output.decode('utf-8')) + raise e + print('[prerun] Initializing or upgrading db - finish') + + +def init_datastore(): + + conn_str = os.environ.get('CKAN_DATASTORE_WRITE_URL') + if not conn_str: + print('[prerun] Skipping datastore initialization') + return + + datastore_perms_command = ['ckan', '-c', ckan_ini, 'datastore', + 'set-permissions'] + + db_user = make_url(conn_str).username + db_passwd = make_url(conn_str).password + db_host = make_url(conn_str).host + db_name = make_url(conn_str).database + connection = psycopg2.connect(user=db_user, + host=db_host, + password=db_passwd, + database=db_name) + cursor = connection.cursor() + + print('[prerun] Initializing datastore db - start') + try: + datastore_perms = subprocess.Popen( + datastore_perms_command, + stdout=subprocess.PIPE) + + perms_sql = datastore_perms.stdout.read() + perms_sql = perms_sql.decode('utf-8') + perms_sql = perms_sql.replace("@"+db_host, "") + # Remove internal pg command as psycopg2 does not like it + perms_sql = re.sub('\\\\connect \"(.*)\"', '', perms_sql) + cursor.execute(perms_sql) + for notice in connection.notices: + print(notice) + + connection.commit() + + print('[prerun] Initializing datastore db - end') + print((datastore_perms.stdout.read())) + except psycopg2.Error as e: + print('[prerun] Could not initialize datastore') + print(e.decode('utf-8')) + + except subprocess.CalledProcessError as e: + if 'OperationalError' in str(e.output): + print(e.output.decode('utf-8')) + print('[prerun] Database not ready, waiting a bit before exit...') + time.sleep(5) + sys.exit(1) + else: + print(e.output.decode('utf-8')) + raise e + finally: + cursor.close() + connection.close() + + +def create_sysadmin(): + + print('[prerun] Start create_sysadmin...') + + name = os.environ.get('CKAN_SYSADMIN_NAME') + password = os.environ.get('CKAN_SYSADMIN_PASSWORD') + email = os.environ.get('CKAN_SYSADMIN_EMAIL') + + if name and password and email: + + # Check if user exists + command = ['ckan', '-c', ckan_ini, 'user', 'show', name] + + out = subprocess.check_output(command) + if 'User:None' not in re.sub(r'\s', '', out.decode('utf-8')): + print('[prerun] Sysadmin user exists, skipping creation') + return + + # Create user + command = ['ckan', '-c', ckan_ini, 'user', 'add', + name, + 'password=' + password, + 'email=' + email] + + subprocess.call(command) + print(('[prerun] Created user {0}'.format(name))) + + # Make it sysadmin + command = ['ckan', '-c', ckan_ini, 'sysadmin', 'add', + name] + + subprocess.call(command) + print(('[prerun] Made user {0} a sysadmin'.format(name))) + +if __name__ == '__main__': + + maintenance = os.environ.get('MAINTENANCE_MODE', '').lower() == 'true' + + if maintenance: + print('[prerun] Maintenance mode, skipping setup...') + else: + check_db_connection() + check_solr_connection() + init_db() + if os.environ.get('CKAN_DATASTORE_WRITE_URL'): + init_datastore() + create_sysadmin() diff --git a/images/ckan/2.10/setup/app/start_ckan.sh b/images/ckan/2.10/setup/app/start_ckan.sh new file mode 100755 index 0000000..f1b5986 --- /dev/null +++ b/images/ckan/2.10/setup/app/start_ckan.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# Run any startup scripts provided by images extending this one +if [[ -d "${APP_DIR}/docker-entrypoint.d" ]] +then + for f in ${APP_DIR}/docker-entrypoint.d/*; do + case "$f" in + *.sh) echo "$0: Running init file $f"; . "$f" ;; + *.py) echo "$0: Running init file $f"; python "$f"; echo ;; + *) echo "$0: Ignoring $f (not an sh or py file)" ;; + esac + echo + done +fi + +if grep -E "beaker.session.secret ?= ?$" $APP_DIR/production.ini +then + echo "Setting secrets in ini file" + ckan config-tool $APP_DIR/production.ini "beaker.session.secret=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" + ckan config-tool $APP_DIR/production.ini "api_token.jwt.encode.secret=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())')" + ckan config-tool $APP_DIR/production.ini "api_token.jwt.decode.secret=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())')" +fi + +echo "Starting UWSGI with '${UWSGI_PROC_NO:-2}' workers" +UWSGI_OPTS="--socket /tmp/uwsgi.sock --uid ckan --gid ckan --http :5000 --master --enable-threads --wsgi-file /srv/app/wsgi.py --module wsgi:application --lazy-apps --gevent 2000 -p ${UWSGI_PROC_NO:-2} -L --gevent-early-monkey-patch --vacuum --harakiri 50 --callable application" + +# Run the prerun script to init CKAN and create the default admin user +python prerun.py || { echo '[CKAN prerun] FAILED. Exiting...' ; exit 1; } + +# Check if we are in maintenance mode and if yes serve the maintenance pages +if [ "$MAINTENANCE_MODE" = true ]; then PYTHONUNBUFFERED=1 python maintenance/serve.py; fi + +# Run any after prerun/init scripts provided by images extending this one +if [[ -d "${APP_DIR}/docker-afterinit.d" ]] +then + for f in ${APP_DIR}/docker-afterinit.d/*; do + case "$f" in + *.sh) echo "$0: Running after prerun init file $f"; . "$f" ;; + *.py) echo "$0: Running after prerun init file $f"; python "$f"; echo ;; + *) echo "$0: Ignoring $f (not an sh or py file)" ;; + esac + echo + done +fi + +# Check whether http basic auth password protection is enabled and enable basicauth routing on uwsgi respecfully +if [ $? -eq 0 ] +then + if [ "$PASSWORD_PROTECT" = true ] + then + if [ "$HTPASSWD_USER" ] || [ "$HTPASSWD_PASSWORD" ] + then + # Generate htpasswd file for basicauth + htpasswd -d -b -c /srv/app/.htpasswd $HTPASSWD_USER $HTPASSWD_PASSWORD + # Start uwsgi with basicauth + uwsgi --ini /srv/app/uwsgi.conf --pcre-jit $UWSGI_OPTS + else + echo "Missing HTPASSWD_USER or HTPASSWD_PASSWORD environment variables. Exiting..." + exit 1 + fi + else + # Start uwsgi + uwsgi $UWSGI_OPTS + fi +else + echo "[prerun] failed...not starting CKAN." +fi diff --git a/images/ckan/2.10/setup/app/uwsgi.conf b/images/ckan/2.10/setup/app/uwsgi.conf new file mode 100644 index 0000000..6321d6d --- /dev/null +++ b/images/ckan/2.10/setup/app/uwsgi.conf @@ -0,0 +1,2 @@ +[uwsgi] +route = ^(?!/api).*$ basicauth:Restricted,/srv/app/.htpasswd diff --git a/images/ckan/2.10/setup/app/wsgi.py b/images/ckan/2.10/setup/app/wsgi.py new file mode 100644 index 0000000..63f3a7c --- /dev/null +++ b/images/ckan/2.10/setup/app/wsgi.py @@ -0,0 +1,28 @@ +""" +Copyright (c) 2016 Keitaro AB + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +# -*- coding: utf-8 -*- + +import os +from ckan.config.middleware import make_app +from ckan.cli import CKANConfigLoader +from logging.config import fileConfig as loggingFileConfig +config_filepath = os.path.join( + os.path.dirname(os.path.abspath(__file__)), u'production.ini') +abspath = os.path.join(os.path.dirname(os.path.abspath(__file__))) +loggingFileConfig(config_filepath) +config = CKANConfigLoader(config_filepath).get_config() +application = make_app(config) From 07af9e29a086369792e94aed80273e9e72f068a8 Mon Sep 17 00:00:00 2001 From: "filip.mihajlovski" Date: Wed, 22 Feb 2023 11:20:12 +0100 Subject: [PATCH 2/7] created 2.10 compose --- compose/2.10/.ckan-env | 42 ++++++++ compose/2.10/.env | 35 +++++++ compose/2.10/docker-compose.yml | 96 +++++++++++++++++++ compose/2.10/postgresql/Dockerfile | 13 +++ .../00_create_datastore.sh | 8 ++ .../20_postgis_permissions.sql | 2 + compose/2.10/solr8/ckan_init_solr.sh | 38 ++++++++ 7 files changed, 234 insertions(+) create mode 100644 compose/2.10/.ckan-env create mode 100644 compose/2.10/.env create mode 100644 compose/2.10/docker-compose.yml create mode 100644 compose/2.10/postgresql/Dockerfile create mode 100644 compose/2.10/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh create mode 100644 compose/2.10/postgresql/docker-entrypoint-initdb.d/20_postgis_permissions.sql create mode 100755 compose/2.10/solr8/ckan_init_solr.sh diff --git a/compose/2.10/.ckan-env b/compose/2.10/.ckan-env new file mode 100644 index 0000000..5ad12d2 --- /dev/null +++ b/compose/2.10/.ckan-env @@ -0,0 +1,42 @@ +# Runtime configuration of CKAN enabled through ckanext-envvars +# Information about how it works: https://github.com/okfn/ckanext-envvars +# Note that variables here take presedence over build/up time variables in .env + +# Set to true to disable CKAN from starting and serve a maintenance page +MAINTENANCE_MODE=false + +# General Settings +CKAN_SITE_ID=default +CKAN_SITE_URL=http://localhost:5005 +CKAN_PORT=5005 +CKAN_MAX_UPLOAD_SIZE_MB=20 +CKAN___BEAKER__SESSION__SECRET=CHANGE_ME +# See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings +CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME +CKAN___API_TOKEN__JWT__DECODE__SECRET=string:CHANGE_ME +# CKAN Plugins +CKAN__PLUGINS=envvars image_view text_view recline_view datastore datapusher +# CKAN requires storage path to be set in order for filestore to be enabled +CKAN__STORAGE_PATH=/srv/app/data +CKAN__WEBASSETS__PATH=/srv/app/data/webassets +# SYSADMIN settings, a sysadmin user is created automatically with the below credentials +CKAN_SYSADMIN_NAME=sysadmin +CKAN_SYSADMIN_PASSWORD=password +CKAN_SYSADMIN_EMAIL=sysadmin@ckantest.com + +# Email settings +CKAN_SMTP_SERVER=smtp.corporateict.domain:25 +CKAN_SMTP_STARTTLS=True +CKAN_SMTP_USER=user +CKAN_SMTP_PASSWORD=pass +CKAN_SMTP_MAIL_FROM=ckan@localhost + +# Datapusher configuration +CKAN__DATAPUSHER__URL=http://datapusher:8000 +CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000/ +CKAN__DATAPUSHER__API_TOKEN=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJqdGkiOiJ6M0lGRGw4SVdFR3lVRzFYRHJ0cm56WmNmcDlzbmFuUHlUMjdqLXRVSW9nIiwiaWF0IjoxNjc3MDYwNjQ1fQ.ECCa0wU-FOOGpdE9_TTVj__2J9SfANRBD6gRClBvPk0 +# CKAN__DATAPUSHER__API_TOKEN=replace_this_with_api_token_once_ckan_starts + +# Solr configuration +CKAN_VERSION=2.9.7 +CKAN_CORE_NAME=ckan \ No newline at end of file diff --git a/compose/2.10/.env b/compose/2.10/.env new file mode 100644 index 0000000..3a722e6 --- /dev/null +++ b/compose/2.10/.env @@ -0,0 +1,35 @@ +# Variables in this file will be used as build arguments when running +# docker-compose build and docker-compose up +# Verify correct substitution with "docker-compose config" +# If variables are newly added or enabled, please delete and rebuild the images to pull in changes: +# docker-compose down -v +# docker-compose build +# docker-compose up -d + +# Database +POSTGRES_PASSWORD=ckan +POSTGRES_PORT=5432 +DATASTORE_READONLY_PASSWORD=datastore + +# CKAN +CKAN_VERSION=2.9.7 +CKAN_SITE_ID=default +CKAN_SITE_URL=http://localhost:5005 +CKAN_PORT=5005 +CKAN_MAX_UPLOAD_SIZE_MB=10 + +# Datapusher +DATAPUSHER_VERSION=0.0.17 +DATAPUSHER_MAX_CONTENT_LENGTH=10485760 +DATAPUSHER_CHUNK_SIZE=16384 +DATAPUSHER_CHUNK_INSERT_ROWS=250 +DATAPUSHER_DOWNLOAD_TIMEOUT=30 +DATAPUSHER_SSL_VERIFY=False +DATAPUSHER_REWRITE_RESOURCES=True +DATAPUSHER_REWRITE_URL=http://ckan:5005 + +# SOLR +CKAN_CORE_NAME=ckan + +# Redis +REDIS_VERSION=6.0.7 diff --git a/compose/2.10/docker-compose.yml b/compose/2.10/docker-compose.yml new file mode 100644 index 0000000..77c9126 --- /dev/null +++ b/compose/2.10/docker-compose.yml @@ -0,0 +1,96 @@ +# docker-compose build && docker-compose up -d +version: "3" + +volumes: + ckan_data: + pg_data: + solr_data: + +services: + ckan: + container_name: ckan + build: /home/filip/Desktop/Projects/localdev/docker-ckan/images/ckan/2.10/ + networks: + - frontend + - backend + depends_on: + - db + - solr + ports: + - "0.0.0.0:${CKAN_PORT}:5000" + env_file: + - ./.ckan-env + environment: + - CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan + - CKAN_DATASTORE_WRITE_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/datastore + - CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:${DATASTORE_READONLY_PASSWORD}@db/datastore + - CKAN_SOLR_URL=http://solr:8983/solr/ckan + - CKAN_REDIS_URL=redis://redis:6379/1 + - CKAN_SITE_URL=${CKAN_SITE_URL} + - CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD} + + volumes: + - ckan_data:/srv/app/data + + datapusher: + container_name: datapusher + image: ghcr.io/keitaroinc/datapusher:${DATAPUSHER_VERSION} + networks: + - frontend + - backend + ports: + - "8000:8000" + environment: + - DATAPUSHER_MAX_CONTENT_LENGTH=${DATAPUSHER_MAX_CONTENT_LENGTH} + - DATAPUSHER_CHUNK_SIZE=${DATAPUSHER_CHUNK_SIZE} + - DATAPUSHER_CHUNK_INSERT_ROWS=${DATAPUSHER_CHUNK_INSERT_ROWS} + - DATAPUSHER_DOWNLOAD_TIMEOUT=${DATAPUSHER_DOWNLOAD_TIMEOUT} + - DATAPUSHER_SSL_VERIFY=${DATA_PUSHER_SSL_VERIFY} + - DATAPUSHER_REWRITE_RESOURCES=${DATAPUSHER_REWRITE_RESOURCES} + - DATAPUSHER_REWRITE_URL=${DATAPUSHER_REWRITE_URL} + + + db: + container_name: db + build: + context: . + dockerfile: postgresql/Dockerfile + args: + - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + networks: + - backend + environment: + - DS_RO_PASS=${DATASTORE_READONLY_PASSWORD} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - PGDATA=/var/lib/postgresql/data/db + volumes: + - pg_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD", "pg_isready", "-U", "ckan"] + + solr: + container_name: solr + image: solr:8.11.1 + networks: + - backend + env_file: + - ./.ckan-env + environment: + - CKAN_CORE_NAME=${CKAN_CORE_NAME} + - CKAN_VERSION=${CKAN_VERSION} + volumes: + - solr_data:/var/solr + - ${PWD}/solr8/ckan_init_solr.sh:/docker-entrypoint-initdb.d/ckan_init_solr.sh + + redis: + container_name: redis + image: redis:${REDIS_VERSION} + networks: + - backend + +networks: + frontend: + backend: diff --git a/compose/2.10/postgresql/Dockerfile b/compose/2.10/postgresql/Dockerfile new file mode 100644 index 0000000..e5fcbb7 --- /dev/null +++ b/compose/2.10/postgresql/Dockerfile @@ -0,0 +1,13 @@ +FROM postgis/postgis:14-3.2-alpine + +# Allow connections; we don't map out any ports so only linked docker containers can connect +RUN echo "host all all 0.0.0.0/0 md5" >> /var/lib/postgresql/data/pg_hba.conf + +# Customize default user/pass/db +ENV POSTGRES_DB ckan +ENV POSTGRES_USER ckan +ARG POSTGRES_PASSWORD +ARG DS_RO_PASS + +# Include datastore setup scripts +COPY ./postgresql/docker-entrypoint-initdb.d /docker-entrypoint-initdb.d diff --git a/compose/2.10/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh b/compose/2.10/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh new file mode 100644 index 0000000..ec1b3c3 --- /dev/null +++ b/compose/2.10/postgresql/docker-entrypoint-initdb.d/00_create_datastore.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -e + +psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL + CREATE ROLE datastore_ro NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN PASSWORD '$DS_RO_PASS'; + CREATE DATABASE datastore OWNER ckan ENCODING 'utf-8'; + GRANT ALL PRIVILEGES ON DATABASE datastore TO ckan; +EOSQL \ No newline at end of file diff --git a/compose/2.10/postgresql/docker-entrypoint-initdb.d/20_postgis_permissions.sql b/compose/2.10/postgresql/docker-entrypoint-initdb.d/20_postgis_permissions.sql new file mode 100644 index 0000000..8b5348e --- /dev/null +++ b/compose/2.10/postgresql/docker-entrypoint-initdb.d/20_postgis_permissions.sql @@ -0,0 +1,2 @@ +ALTER VIEW geometry_columns OWNER TO ckan; +ALTER TABLE spatial_ref_sys OWNER TO ckan; diff --git a/compose/2.10/solr8/ckan_init_solr.sh b/compose/2.10/solr8/ckan_init_solr.sh new file mode 100755 index 0000000..8ea06c3 --- /dev/null +++ b/compose/2.10/solr8/ckan_init_solr.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# +# Initialize SOLR for CKAN by creating a ckan core +# Arguments are supplied via environment variables: CKAN_CORE_NAME CKAN_VERSION +# Example: +# CKAN_CORE_NAME=ckan +# CKAN_VERSION=2.9.7 + +set -e + +CKAN_SOLR_SCHEMA_URL=https://raw.githubusercontent.com/ckan/ckan/ckan-$CKAN_VERSION/ckan/config/solr/schema.solr8.xml + +echo "Check whether managed schema exists for CKAN $CKAN_VERSION" +if ! curl --output /dev/null --silent --head --fail "$CKAN_SOLR_SCHEMA_URL"; then + echo "Can't find CKAN SOLR schema at URL: $CKAN_SOLR_SCHEMA_URL. Exiting..." + exit 1 +fi + +echo "Check whether SOLR is initialized for CKAN" +CORESDIR=/var/solr/data + +COREDIR="$CORESDIR/$CKAN_CORE_NAME" +if [ -d "$COREDIR" ]; then + echo "SOLR already initialized, skipping initialization" +else + echo "Initializing SOLR core $CKAN_CORE_NAME for CKAN $CKAN_VERSION" + # init script for handling an empty /var/solr + /opt/docker-solr/scripts/init-var-solr + + # Precreate CKAN core + /opt/docker-solr/scripts/precreate-core $CKAN_CORE_NAME + + # Replace the managed schema with CKANs schema + echo "Adding CKAN managed schema" + curl $CKAN_SOLR_SCHEMA_URL -o /var/solr/data/$CKAN_CORE_NAME/conf/managed-schema -s + + echo "SOLR initialized" +fi From 24b95e7cad8a025e7ea7c7ca47168d085df9fc63 Mon Sep 17 00:00:00 2001 From: Kiril Poposki Date: Wed, 22 Feb 2023 11:28:33 +0100 Subject: [PATCH 3/7] fixing docker-compose build for ckan 2.10 --- compose/2.10/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/2.10/docker-compose.yml b/compose/2.10/docker-compose.yml index 77c9126..bcd3f60 100644 --- a/compose/2.10/docker-compose.yml +++ b/compose/2.10/docker-compose.yml @@ -9,7 +9,7 @@ volumes: services: ckan: container_name: ckan - build: /home/filip/Desktop/Projects/localdev/docker-ckan/images/ckan/2.10/ + build: ../../images/ckan/2.10/ networks: - frontend - backend From 0231df73f8a0982a81c6780a303f625dd08543d4 Mon Sep 17 00:00:00 2001 From: Kiril-Poposki1998 Date: Thu, 7 Dec 2023 11:52:43 +0100 Subject: [PATCH 4/7] ckan 2.10 github actions for pull request and push to master --- .github/workflows/master_merge.yml | 65 ++++++++++++++++++++++++++++++ .github/workflows/pr_checks.yml | 51 +++++++++++++++++++++++ 2 files changed, 116 insertions(+) diff --git a/.github/workflows/master_merge.yml b/.github/workflows/master_merge.yml index 37aaf2d..f43d916 100644 --- a/.github/workflows/master_merge.yml +++ b/.github/workflows/master_merge.yml @@ -6,6 +6,71 @@ on: jobs: + build-ckan-2-10: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.CR_PAT }} + + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: | + /tmp/.buildx-cache-alpine-2-10 + /tmp/.buildx-cache-ubuntu-2-10 + key: ${{ runner.os }}-buildx-2-10-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx-2-10 + + - name: Get docker tag for Alpine image + id: alpine + run: | + echo "::set-output name=IMAGE_TAG::$(awk -F '=' '/IMAGE_TAG/{print $2}' ./images/ckan/2.10/Dockerfile)" + + - name: Build and push CKAN 2.10 alpine + uses: docker/build-push-action@v2 + with: + context: ./images/ckan/2.10 + file: ./images/ckan/2.10/Dockerfile + push: true + tags: | + keitaro/ckan:${{ steps.alpine.outputs.IMAGE_TAG }} + ghcr.io/keitaroinc/ckan:${{ steps.alpine.outputs.IMAGE_TAG }} + cache-from: type=local,src=/tmp/.buildx-cache-alpine-2-10 + cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-alpine-2-10 + + - name: Get docker tag for Ubuntu image + id: ubuntu + run: | + echo "::set-output name=IMAGE_TAG::$(awk -F '=' '/IMAGE_TAG/{print $2}' ./images/ckan/2.10/Dockerfile.focal)" + + - name: Build and push CKAN 2.10 ubuntu + uses: docker/build-push-action@v2 + with: + context: ./images/ckan/2.10 + file: ./images/ckan/2.10/Dockerfile.focal + push: true + tags: | + keitaro/ckan:${{ steps.ubuntu.outputs.IMAGE_TAG }} + ghcr.io/keitaroinc/ckan:${{ steps.ubuntu.outputs.IMAGE_TAG }} + cache-from: type=local,src=/tmp/.buildx-cache-ubuntu-2-10 + cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-ubuntu-2-10 + build-ckan-2-9: runs-on: ubuntu-20.04 steps: diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml index 4619032..9990320 100644 --- a/.github/workflows/pr_checks.yml +++ b/.github/workflows/pr_checks.yml @@ -6,6 +6,57 @@ on: jobs: + build-ckan-2-10: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: | + /tmp/.buildx-cache-alpine-2-10 + /tmp/.buildx-cache-ubuntu-2-10 + key: ${{ runner.os }}-buildx-2-10-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx-2-10 + + - name: Get docker tag for Alpine image + id: alpine + run: | + echo "::set-output name=IMAGE_TAG::$(awk -F '=' '/IMAGE_TAG/{print $2}' ./images/ckan/2.10/Dockerfile)" + + - name: Build CKAN 2.10 alpine + uses: docker/build-push-action@v2 + with: + context: ./images/ckan/2.10 + file: ./images/ckan/2.10/Dockerfile + push: false + tags: keitaro/ckan:${{ steps.alpine.outputs.IMAGE_TAG }} + cache-from: type=local,src=/tmp/.buildx-cache-alpine-2-10 + cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-alpine-2-10 + + - name: Get docker tag for Ubuntu image + id: ubuntu + run: | + echo "::set-output name=IMAGE_TAG::$(awk -F '=' '/IMAGE_TAG/{print $2}' ./images/ckan/2.10/Dockerfile.focal)" + + - name: Build CKAN 2.10 ubuntu + uses: docker/build-push-action@v2 + with: + context: ./images/ckan/2.10 + file: ./images/ckan/2.10/Dockerfile.focal + push: false + tags: keitaro/ckan:${{ steps.ubuntu.outputs.IMAGE_TAG }} + cache-from: type=local,src=/tmp/.buildx-cache-ubuntu-2-10 + cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-ubuntu-2-10 + build-args: | + --progress=plain + + build-ckan-2-9: runs-on: ubuntu-20.04 steps: From 1bea0e7e6cdaaf57f65944731bf2e73d13946ebe Mon Sep 17 00:00:00 2001 From: Kiril-Poposki1998 Date: Wed, 13 Dec 2023 11:45:46 +0100 Subject: [PATCH 5/7] ckan 2.10 dockerfile update and scripts along with docker compose setup --- compose/2.10/.ckan-env | 6 +++--- compose/2.10/.env | 6 +++--- compose/2.10/docker-compose.yml | 2 +- compose/2.10/solr8/ckan_init_solr.sh | 2 +- images/ckan/2.10/Dockerfile | 8 ++++---- images/ckan/2.10/Dockerfile.focal | 7 +++---- 6 files changed, 15 insertions(+), 16 deletions(-) diff --git a/compose/2.10/.ckan-env b/compose/2.10/.ckan-env index 5ad12d2..b9f2598 100644 --- a/compose/2.10/.ckan-env +++ b/compose/2.10/.ckan-env @@ -7,8 +7,8 @@ MAINTENANCE_MODE=false # General Settings CKAN_SITE_ID=default -CKAN_SITE_URL=http://localhost:5005 -CKAN_PORT=5005 +CKAN_SITE_URL=http://localhost:5000 +CKAN_PORT=5000 CKAN_MAX_UPLOAD_SIZE_MB=20 CKAN___BEAKER__SESSION__SECRET=CHANGE_ME # See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings @@ -38,5 +38,5 @@ CKAN__DATAPUSHER__API_TOKEN=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJqdGkiOiJ6M0l # CKAN__DATAPUSHER__API_TOKEN=replace_this_with_api_token_once_ckan_starts # Solr configuration -CKAN_VERSION=2.9.7 +CKAN_VERSION=2.10 CKAN_CORE_NAME=ckan \ No newline at end of file diff --git a/compose/2.10/.env b/compose/2.10/.env index 3a722e6..b9e1c2d 100644 --- a/compose/2.10/.env +++ b/compose/2.10/.env @@ -12,10 +12,10 @@ POSTGRES_PORT=5432 DATASTORE_READONLY_PASSWORD=datastore # CKAN -CKAN_VERSION=2.9.7 +CKAN_VERSION=2.10 CKAN_SITE_ID=default -CKAN_SITE_URL=http://localhost:5005 -CKAN_PORT=5005 +CKAN_SITE_URL=http://localhost:5000 +CKAN_PORT=5000 CKAN_MAX_UPLOAD_SIZE_MB=10 # Datapusher diff --git a/compose/2.10/docker-compose.yml b/compose/2.10/docker-compose.yml index bcd3f60..ccf8daf 100644 --- a/compose/2.10/docker-compose.yml +++ b/compose/2.10/docker-compose.yml @@ -1,5 +1,5 @@ # docker-compose build && docker-compose up -d -version: "3" +version: "3.8" volumes: ckan_data: diff --git a/compose/2.10/solr8/ckan_init_solr.sh b/compose/2.10/solr8/ckan_init_solr.sh index 8ea06c3..296d52b 100755 --- a/compose/2.10/solr8/ckan_init_solr.sh +++ b/compose/2.10/solr8/ckan_init_solr.sh @@ -8,7 +8,7 @@ set -e -CKAN_SOLR_SCHEMA_URL=https://raw.githubusercontent.com/ckan/ckan/ckan-$CKAN_VERSION/ckan/config/solr/schema.solr8.xml +CKAN_SOLR_SCHEMA_URL=https://raw.githubusercontent.com/ckan/ckan/$CKAN_VERSION/ckan/config/solr/schema.xml echo "Check whether managed schema exists for CKAN $CKAN_VERSION" if ! curl --output /dev/null --silent --head --fail "$CKAN_SOLR_SCHEMA_URL"; then diff --git a/images/ckan/2.10/Dockerfile b/images/ckan/2.10/Dockerfile index becda88..b5e00cc 100644 --- a/images/ckan/2.10/Dockerfile +++ b/images/ckan/2.10/Dockerfile @@ -4,11 +4,11 @@ FROM alpine:3.17.2 as ckanbuild # Used by Github Actions to tag the image with -ENV IMAGE_TAG=2.10.0 +ENV IMAGE_TAG=2.10.2 # Set CKAN version to build ENV GIT_URL=https://github.com/ckan/ckan.git -ENV GIT_BRANCH=ckan-2.10.0 +ENV GIT_BRANCH=ckan-2.10.2 # Set src dirs ENV SRC_DIR=/srv/app/src @@ -57,7 +57,7 @@ COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh # RUN ${SRC_DIR}/apply_ckan_patches.sh RUN rm -rf /srv/app/src/ckan/.git RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt -RUN pip wheel --wheel-dir=/wheels uWSGI==2.0.20 gevent==21.12.0 greenlet==1.1.3 +RUN pip wheel --wheel-dir=/wheels uWSGI==2.0.20 gevent==22.10.2 greenlet==2.0.2 ########################### @@ -136,7 +136,7 @@ COPY --from=extbuild /wheels /srv/app/ext_wheels COPY --from=ckanbuild /srv/app/src/ckan ${CKAN_DIR} # Additional install steps for build stages artifacts -RUN pip install --no-index --find-links=/srv/app/wheels uWSGI==2.0.20 gevent==21.12.0 +RUN pip install --no-index --find-links=/srv/app/wheels uWSGI==2.0.20 gevent==22.10.2 # Create a local user and group to run the app RUN addgroup -g 92 -S ckan && \ diff --git a/images/ckan/2.10/Dockerfile.focal b/images/ckan/2.10/Dockerfile.focal index 369d7fd..7c2e867 100644 --- a/images/ckan/2.10/Dockerfile.focal +++ b/images/ckan/2.10/Dockerfile.focal @@ -8,7 +8,7 @@ ENV IMAGE_TAG=2.9.7-focal # Set CKAN version to build ENV GIT_URL=https://github.com/ckan/ckan.git -ENV GIT_BRANCH=ckan-2.9.7 +ENV GIT_BRANCH=ckan-2.10.2 # Set timezone ENV TZ=UTC @@ -81,8 +81,7 @@ COPY ./scripts/apply_ckan_patches.sh ${SRC_DIR}/apply_ckan_patches.sh RUN ${SRC_DIR}/apply_ckan_patches.sh RUN rm -rf /srv/app/src/ckan/.git RUN pip wheel --wheel-dir=/wheels -r ckan/requirements.txt -RUN pip wheel --wheel-dir=/wheels uWSGI==2.0.20 gevent==21.12.0 greenlet==1.1.3 - +RUN pip wheel --wheel-dir=/wheels uWSGI==2.0.20 gevent==22.10.2 greenlet==2.0.2 ########################### ### Default-Extensions #### @@ -192,7 +191,7 @@ COPY --from=extbuild /wheels /srv/app/ext_wheels COPY --from=ckanbuild /srv/app/src/ckan ${CKAN_DIR} # Additional install steps for build stages artifacts -RUN pip install --no-index --find-links=/srv/app/wheels uWSGI==2.0.20 gevent==21.12.0 +RUN pip install --no-index --find-links=/srv/app/wheels uWSGI==2.0.20 gevent==22.10.2 # Create a local user and group to run the app RUN groupadd -g 92 ckan && \ From efe53d5f560f7d5eae4ce9341b25a63f1f8d33cd Mon Sep 17 00:00:00 2001 From: Kiril-Poposki1998 Date: Tue, 19 Dec 2023 11:09:52 +0100 Subject: [PATCH 6/7] change image in docker compose and version in env file --- compose/2.10/.env | 2 +- compose/2.10/docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/2.10/.env b/compose/2.10/.env index b9e1c2d..ef7cf75 100644 --- a/compose/2.10/.env +++ b/compose/2.10/.env @@ -12,7 +12,7 @@ POSTGRES_PORT=5432 DATASTORE_READONLY_PASSWORD=datastore # CKAN -CKAN_VERSION=2.10 +CKAN_VERSION=2.10.2 CKAN_SITE_ID=default CKAN_SITE_URL=http://localhost:5000 CKAN_PORT=5000 diff --git a/compose/2.10/docker-compose.yml b/compose/2.10/docker-compose.yml index ccf8daf..00b40d2 100644 --- a/compose/2.10/docker-compose.yml +++ b/compose/2.10/docker-compose.yml @@ -9,7 +9,7 @@ volumes: services: ckan: container_name: ckan - build: ../../images/ckan/2.10/ + image: ghcr.io/keitaroinc/ckan:2.10.2 networks: - frontend - backend From e7711d51ab0f82cd6f0643111d2db5a7141108d0 Mon Sep 17 00:00:00 2001 From: Kiril-Poposki1998 Date: Tue, 19 Dec 2023 11:23:57 +0100 Subject: [PATCH 7/7] add beaker secrets to start ckan --- compose/2.10/.env | 2 +- images/ckan/2.10/setup/app/start_ckan.sh | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/compose/2.10/.env b/compose/2.10/.env index ef7cf75..b9e1c2d 100644 --- a/compose/2.10/.env +++ b/compose/2.10/.env @@ -12,7 +12,7 @@ POSTGRES_PORT=5432 DATASTORE_READONLY_PASSWORD=datastore # CKAN -CKAN_VERSION=2.10.2 +CKAN_VERSION=2.10 CKAN_SITE_ID=default CKAN_SITE_URL=http://localhost:5000 CKAN_PORT=5000 diff --git a/images/ckan/2.10/setup/app/start_ckan.sh b/images/ckan/2.10/setup/app/start_ckan.sh index f1b5986..abf4473 100755 --- a/images/ckan/2.10/setup/app/start_ckan.sh +++ b/images/ckan/2.10/setup/app/start_ckan.sh @@ -12,12 +12,24 @@ then done fi +# Add session secret from chart +if [[ -z $BEAKER_SESSION_SECRET || -v $BEAKER_SESSION_SECRET || -z $JWT_ENCODE_SECRET || -v $JWT_ENCODE_SECRET || -z $JWT_DECODE_SECRET || -v $JWT_DECODE_SECRET ]];then + echo "Not all environment variables are set. Generating sessions..." +else + echo "Setting session secrets from environment variables" + ckan config-tool $APP_DIR/production.ini "beaker.session.secret=$BEAKER_SESSION_SECRET" + ckan config-tool $APP_DIR/production.ini "api_token.jwt.encode.secret=$JWT_ENCODE_SECRET" + ckan config-tool $APP_DIR/production.ini "api_token.jwt.decode.secret=$JWT_DECODE_SECRET" +fi + if grep -E "beaker.session.secret ?= ?$" $APP_DIR/production.ini then echo "Setting secrets in ini file" ckan config-tool $APP_DIR/production.ini "beaker.session.secret=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" - ckan config-tool $APP_DIR/production.ini "api_token.jwt.encode.secret=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())')" - ckan config-tool $APP_DIR/production.ini "api_token.jwt.decode.secret=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())')" + ckan config-tool $APP_DIR/production.ini "WTF_CSRF_SECRET_KEY=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" + JWT_SECRET=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())') + ckan config-tool $APP_DIR/production.ini "api_token.jwt.encode.secret=$JWT_SECRET" + ckan config-tool $APP_DIR/production.ini "api_token.jwt.decode.secret=$JWT_SECRET" fi echo "Starting UWSGI with '${UWSGI_PROC_NO:-2}' workers"