Initial version (part two) incorporating review suggestions from kdj

docker/Dockerfile

@@ -1,8 +1,12 @@
-FROM debian:bookworm AS samtools_build
+FROM ubuntu:18.04 AS samtools_build
 
-ARG HTSLIB_VERSION=1.19
-ARG SAMTOOLS_VERSION=1.19
+ARG HTSLIB_VERSION=1.19.1
+ARG SAMTOOLS_VERSION=1.19.2
 ARG BCFTOOLS_VERSION=1.19
+ARG IRODS_VERSION="4.2.12"
+ARG DEFLATE_VERSION=1.19
+
+COPY irods_clients_dev /opt/docker/irods_clients_dev
 
 RUN apt-get update && \
     DEBIAN_FRONTEND=noninteractive apt-get upgrade -y && \
@@ -13,49 +17,116 @@ RUN apt-get update && \
       build-essential \
       autoconf \
       automake \
-      jq \
       libbz2-dev \
       libcurl4-openssl-dev \
       libgd-dev \
       liblzma-dev \
       libssl-dev \
       libncurses5-dev \
-      curl
+      curl \
+      dirmngr \
+      gpg \
+      gpg-agent \
+      cmake \
+      lsb-release \
+      locales && \
+      locale-gen en_GB en_GB.UTF-8 && \
+      localedef -i en_GB -c -f UTF-8 -A /usr/share/locale/locale.alias en_GB.UTF-8
+
+ENV LANG=en_GB.UTF-8 \
+    LANGUAGE=en_GB \
+    LC_ALL=en_GB.UTF-8
+
+
+RUN curl -sSL https://packages.irods.org/irods-signing-key.asc | apt-key add - && \
+    echo "deb [arch=amd64] https://packages.irods.org/apt/ $(lsb_release -sc) main" |\
+    tee /etc/apt/sources.list.d/renci-irods.list && \
+    apt-get update && \
+    apt-get install -q -y --no-install-recommends \
+    irods-dev="${IRODS_VERSION}-1~$(lsb_release -sc)" \
+    irods-runtime="${IRODS_VERSION}-1~$(lsb_release -sc)"
+
+RUN apt-key add - < /opt/docker/irods_clients_dev/keys/git-core-ppa.asc && \
+    echo "deb https://ppa.launchpadcontent.net/git-core/ppa/ubuntu $(lsb_release -sc) main" |\
+    tee /etc/apt/sources.list.d/git-core.list && \
+    apt-get update && \
+    apt-get install -q -y --no-install-recommends \
+    git
+
+RUN cd /tmp && \
+    curl -sSL -O https://github.com/ebiggers/libdeflate/releases/download/v${DEFLATE_VERSION}/libdeflate-${DEFLATE_VERSION}.tar.gz && \
+    tar xzf libdeflate-${DEFLATE_VERSION}.tar.gz && \
+    cd libdeflate-${DEFLATE_VERSION} && \
+    cmake . && \
+    cmake --build .
 
 RUN cd /tmp && \
     curl -sSL -O "https://github.com/samtools/htslib/releases/download/${HTSLIB_VERSION}/htslib-${HTSLIB_VERSION}.tar.bz2" && \
     tar xfj htslib-${HTSLIB_VERSION}.tar.bz2 && \
     cd htslib-${HTSLIB_VERSION} && \
-    ./configure --enable-plugins && \
+    CPPFLAGS=-I../libdeflate-${DEFLATE_VERSION} LDFLAGS=-L../libdeflate-${DEFLATE_VERSION} ./configure --enable-plugins --with-libdeflate && \
     make && \
     make install && \
     ldconfig
 
+RUN cd /tmp && \
+    curl -sSL -O "https://github.com/samtools/htslib-plugins/archive/refs/tags/201712.tar.gz" && \
+    tar xfz 201712.tar.gz && \
+    cd htslib-plugins-201712 && \
+    export IRODS_HOME=/usr/include/irods && \
+    make && \
+    make install
+
 RUN cd /tmp && \
     curl -sSL -O "https://github.com/samtools/samtools/releases/download/${SAMTOOLS_VERSION}/samtools-${SAMTOOLS_VERSION}.tar.bz2" && \
     tar xfj samtools-${SAMTOOLS_VERSION}.tar.bz2 && \
     cd samtools-${SAMTOOLS_VERSION} && \
-    ./configure --with-htslib=system && \
+    ./configure --with-htslib=/tmp/htslib-${HTSLIB_VERSION} && \
     make install
 
 RUN cd /tmp && \
     curl -sSL -O "https://github.com/samtools/bcftools/releases/download/${BCFTOOLS_VERSION}/bcftools-${BCFTOOLS_VERSION}.tar.bz2" && \
     tar xfj bcftools-${BCFTOOLS_VERSION}.tar.bz2 && \
     cd bcftools-${BCFTOOLS_VERSION} && \
-    ./configure --with-htslib=system && \
+    ./configure --with-htslib=/tmp/htslib-${HTSLIB_VERSION} && \
     make install
 
+#
+# Next stage: make everything smaller
+#
+FROM ubuntu:18.04
+
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get upgrade -y && \
+    apt-get install -qq -y --no-install-recommends \
+      libssl-dev \
+      gpg \
+      gpg-agent \
+      locales && \
+      locale-gen en_GB en_GB.UTF-8 && \
+      localedef -i en_GB -c -f UTF-8 -A /usr/share/locale/locale.alias en_GB.UTF-8
+
+
+COPY --from=samtools_build /usr/local/bin/samtools /usr/local/bin/samtools
+COPY --from=samtools_build /usr/local/bin/bcftools /usr/local/bin/bcftools
+COPY --from=samtools_build /usr/local/bin/htsfile /usr/local/bin/htsfile
+COPY --from=samtools_build /usr/local/bin/tabix /usr/local/bin/tabix
+COPY --from=samtools_build /usr/local/lib/lib* /usr/local/lib/
+COPY --from=samtools_build /usr/lib/lib* /usr/lib/
+COPY --from=samtools_build /usr/local/libexec/htslib /usr/local/libexec/htslib/
+COPY --from=samtools_build /opt/irods-externals /opt/irods-externals
+COPY --from=samtools_build /tmp/libdeflate-1.19/libdeflate.so* /usr/local/lib/
+
 # Copy the singularity-wrapper scripts
 ARG DOCKER_IMAGE
 ARG DOCKER_TAG
-#ENV MANIFEST_PATH="./manifest.json"
 ENV DOCKER_IMAGE="$DOCKER_IMAGE"
 ENV DOCKER_TAG="$DOCKER_TAG"
 
 RUN ldconfig
 
 COPY --from=singularity ./scripts/* /usr/local/bin/
-COPY ./manifest.json /opt/wtsi-npg/etc/manifest.json
+COPY ./manifest.txt /opt/wtsi-npg/etc/manifest.txt
 
-CMD ["/usr/bin/bash"]
+CMD ["/bin/bash"]
 

docker/irods_clients_dev/keys/git-core-ppa.asc

@@ -0,0 +1,13 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mI0ESXjaGwEEAMA26F3+mnRW8uRqASMsEa5EsmgvUpLD7EKpC7903OpiMGSvZ2sE
+34g7W6nUQY0R//AZS2iW4ZXfvdhQTQuuPlHM6Q3iUAt+nyXcf9xBlscs8Gm722u4
+jAtFtBS4BMQRhRRfWTHwJIOM6OpGIccjPe8pQfIeoRxkKJxlehzw2mU1ABEBAAG0
+KExhdW5jaHBhZCBQUEEgZm9yIFVidW50dSBHaXQgTWFpbnRhaW5lcnOItgQTAQIA
+IAUCSXjaGwIbAwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJEKFxXYjh3x8k/zMD
+/RKBMjavvFl71YBazSOGl2YfSsZiR/ANsby3+rUaULb8uxzCHXAQnlH5vdtLSPry
+aLBvzCU8C3C02qNT8jRacU2752zsCkCi1SLRSOXdI/ATJHza5aTvYV93rTITBhU4
+sJQeK9RW0CtaDRAxJsn/Dr6J3lL/c9m9cT5fFpxOIsF4
+=33dX
+-----END PGP PUBLIC KEY BLOCK-----
+

docker/manifest.txt

@@ -0,0 +1,4 @@
+bcftools
+samtools
+htsfile
+tabix

singularity/scripts/singularity-service-docker

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Running this script will start an instance of the container, if one is not 
+# Running this script will start an instance of the container, if one is not
 # already running, and then execute the command in the instance.
 #
 # Cleaning up unwanted instances afterwards is outside the scope of this script.
@@ -11,12 +11,45 @@ DOCKER_USER=${DOCKER_USER:?required, but not set}
 DOCKER_IMAGE=${DOCKER_IMAGE:?required, but was not set}
 DOCKER_TAG=${DOCKER_TAG:?required, but not set}
 
+DOCKER_URL="docker://$DOCKER_REGISTRY/$DOCKER_USER/$DOCKER_IMAGE:$DOCKER_TAG"
+
 # Colons and slashes are not allowed in Singularity instance names
 instance="$DOCKER_REGISTRY--$DOCKER_USER--$DOCKER_IMAGE--$DOCKER_TAG"
 
-if ! singularity instance list | grep "$instance" 2>&1 | logger -p user.notice -t singularity-service-docker ; then
-    singularity instance start \
-        "docker://$DOCKER_REGISTRY/$DOCKER_USER/$DOCKER_IMAGE:$DOCKER_TAG" "$instance" 2>&1 | logger -p user.notice -t singularity-service-docker
+# Requires bash >=4.1 for file descriptor management with "{<variable>}"
+LOCK_FILE="/var/lock/$instance--$USER.lock"
+LOCK_TIMEOUT=60
+
+# Get the first free file descriptor
+exec {FD}>"$LOCK_FILE"
+
+# Release the lock and close the file descriptor
+clean_up() {
+    flock --unlock $FD || {
+        echo >&2 "Failed to release lock on $LOCK_FILE (fd: $FD): $?"
+    }
+    exec {FD}>&-
+}
+
+clean_up_and_exit() {
+    clean_up
+    exit 1
+}
+
+# clean_up_and_exit on these signals
+trap clean_up_and_exit SIGINT SIGTERM SIGUSR1 SIGUSR2
+
+flock --exclusive --timeout $LOCK_TIMEOUT $FD || {
+    echo >&2 "Failed to obtain lock on $LOCK_FILE (fd: $FD): $?"
+    exit 1
+}
+
+if ! singularity instance list 2> >(logger -s -p user.err -t singularity-service-docker) |\
+        grep "$instance" 2>&1 >/dev/null
+then
+    singularity --silent instance start "$DOCKER_URL" "$instance" 2> >(logger -s -p user.err -t singularity-service-docker) >/dev/null
 fi
 
+clean_up
+
 singularity exec "instance://$instance" "$@"

singularity/scripts/singularity-wrapper

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/sh
 #
 # Copyright (C) 2023 Genome Research Ltd. All rights reserved.
 #
@@ -17,8 +17,6 @@
 #
 # Author: Keith James <[email protected]>
 
-set -euo pipefail
-
 usage() {
     cat 1>&2 << EOF
 Install singularity proxy wrappers for the executables listed in a
@@ -36,7 +34,7 @@ Usage: $0
     [-e]
     [-h]
     [-i <Docker image name>]
-    [-m <JSON manifest path>]
+    [-m <manifest path>]
     [-p <wrapper install prefix>]
     [-r <Docker registry name>]
     [-s]
@@ -73,7 +71,7 @@ EOF
 
 # Print an application manifest
 print_manifest() {
-    jq . "$MANIFEST_PATH"
+    cat "$MANIFEST_PATH"
 }
 
 # Write a bash script wrapping an application in a Docker container
@@ -113,9 +111,9 @@ install_wrappers() {
     install -d "$dir"
     cp "/usr/local/bin/$singularity_wrap_impl" "$PREFIX/bin"
 
-    for exe in "${wrappers[@]}" ; do
+    while IFS= read -r exe; do
         write_wrapper "$dir" "$exe"
-    done
+    done < "$MANIFEST_PATH"
 }
 
 DOCKER_REGISTRY=${DOCKER_REGISTRY:-ghcr.io}
@@ -124,7 +122,7 @@ DOCKER_IMAGE=${DOCKER_IMAGE:-""}
 DOCKER_TAG=${DOCKER_TAG:-latest}
 
 PREFIX=${PREFIX:-/opt/wtsi-npg}
-MANIFEST_PATH=${MANIFEST_PATH:-"$PREFIX/etc/manifest.json"}
+MANIFEST_PATH=${MANIFEST_PATH:-"$PREFIX/etc/manifest.txt"}
 
 singularity_wrap_impl="singularity-run-docker"
 
@@ -171,19 +169,12 @@ done
 
 shift $((OPTIND -1))
 
-declare -a wrappers
 if [ ! -e "$MANIFEST_PATH" ] ; then
     echo -e "\nERROR:\n  The manifest of executables at '$MANIFEST_PATH' does not exist"
     exit 4
 fi
 
-
-
-
-
-wrappers=($(jq -j '.executable[] + " "' "$MANIFEST_PATH"))
-
-operation="$@"
+operation="$1"
 if [ -z "$operation" ] ; then
     usage
     echo -e "\nERROR:\n  An operation argument is required"