Merge pull request #71 from GenomicDataInfrastructure/67-gdi-ckan-har…

…vesters-are-not-running

fix(cron): Run harvester cron job in background (run and clean up log…

ckan/Dockerfile

@@ -41,6 +41,18 @@ RUN mkdir -p /var/log/ckan/std && chown -R ckan:ckan /var/log/ckan
 COPY docker-entrypoint.d/* /docker-entrypoint.d/
 COPY config/ckan_harvesting.conf /etc/supervisord.d/ckan_harvesting.conf
 
+# Copy crontab file to the cron.d directory
+ADD config/crontab-harvester /etc/cron.d/crontab-harvester
+
+# Give execution rights on the cron job
+RUN chmod 644 /etc/cron.d/crontab-harvester
+
+# Apply cron job
+RUN crontab -u ckan /etc/cron.d/crontab-harvester
+
+# Create log file for cron
+RUN touch /var/log/cron.log && chown ckan:ckan /var/log/cron.log
+
 # Apply any patches needed to CKAN core or any of the built extensions (not the
 # runtime mounted ones)
 COPY patches ${APP_DIR}/patches

ckan/Dockerfile.dev

@@ -35,6 +35,18 @@ RUN mkdir -p /var/log/ckan/std && chown -R ckan:ckan /var/log/ckan
 COPY docker-entrypoint.d/* /docker-entrypoint.d/
 COPY config/ckan_harvesting.conf /etc/supervisord.d/ckan_harvesting.conf
 
+# Copy crontab file to the cron.d directory
+ADD config/crontab-harvester /etc/cron.d/crontab-harvester
+
+# Give execution rights on the cron job
+RUN chmod 644 /etc/cron.d/crontab-harvester
+
+# Apply cron job
+RUN crontab -u ckan /etc/cron.d/crontab-harvester
+
+# Create log file for cron
+RUN touch /var/log/cron.log && chown ckan:ckan /var/log/cron.log
+
 # Apply any patches needed to CKAN core or any of the built extensions (not the
 # runtime mounted ones)
 COPY patches ${APP_DIR}/patches

ckan/config/ckan_harvesting.conf

@@ -28,3 +28,18 @@ stderr_logfile=/var/log/ckan/std/fetch_consumer.log
 autostart=true
 autorestart=true
 startsecs=10
+
+
+[program:crond]
+
+; Full Path to executable, should be path to virtual environment,
+; Full path to config file too.
+
+command=/usr/sbin/crond -n
+
+numprocs=1
+stdout_logfile=/var/log/cron.log
+stderr_logfile=/var/log/cron.log
+autostart=true
+autorestart=true
+startsecs=10

ckan/config/crontab-harvester

@@ -0,0 +1,13 @@
+# SPDX-FileCopyrightText: Stichting Health-RI
+#
+# SPDX-License-Identifier: AGPL-3.0-only
+
+# Set the shell to bash
+SHELL=/bin/bash
+
+# Source the environment variables
+BASH_ENV=/etc/profile.d/custom_env.sh
+
+# Every 15 minutes, run the harvester and clean the harvest log every day at 5am
+*/15 * * * * /usr/bin/ckan -c /srv/app/ckan.ini harvester run
+0 5 * * * /usr/bin/ckan -c /srv/app/ckan.ini harvester clean-harvest-log

ckan/docker-entrypoint.d/cp_env_to_cron_env.sh

@@ -0,0 +1,8 @@
+# SPDX-FileCopyrightText: Stichting Health-RI
+#
+# SPDX-License-Identifier: AGPL-3.0-only
+
+#!/bin/sh
+
+# Dump environment variables to a file that will be sourced by cron
+printenv | sed 's/^\(.*\)$/export \1/g' > /etc/profile.d/custom_env.sh