diff --git a/docker/Dockerfile b/docker/Dockerfile index b873eff..e437165 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,8 +1,12 @@ -FROM debian:bookworm AS samtools_build +FROM ubuntu:18.04 AS samtools_build -ARG HTSLIB_VERSION=1.19 -ARG SAMTOOLS_VERSION=1.19 +ARG HTSLIB_VERSION=1.19.1 +ARG SAMTOOLS_VERSION=1.19.2 ARG BCFTOOLS_VERSION=1.19 +ARG IRODS_VERSION="4.2.12" +ARG DEFLATE_VERSION=1.19 + +COPY irods_clients_dev /opt/docker/irods_clients_dev RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get upgrade -y && \ @@ -13,49 +17,116 @@ RUN apt-get update && \ build-essential \ autoconf \ automake \ - jq \ libbz2-dev \ libcurl4-openssl-dev \ libgd-dev \ liblzma-dev \ libssl-dev \ libncurses5-dev \ - curl + curl \ + dirmngr \ + gpg \ + gpg-agent \ + cmake \ + lsb-release \ + locales && \ + locale-gen en_GB en_GB.UTF-8 && \ + localedef -i en_GB -c -f UTF-8 -A /usr/share/locale/locale.alias en_GB.UTF-8 + +ENV LANG=en_GB.UTF-8 \ + LANGUAGE=en_GB \ + LC_ALL=en_GB.UTF-8 + + +RUN curl -sSL https://packages.irods.org/irods-signing-key.asc | apt-key add - && \ + echo "deb [arch=amd64] https://packages.irods.org/apt/ $(lsb_release -sc) main" |\ + tee /etc/apt/sources.list.d/renci-irods.list && \ + apt-get update && \ + apt-get install -q -y --no-install-recommends \ + irods-dev="${IRODS_VERSION}-1~$(lsb_release -sc)" \ + irods-runtime="${IRODS_VERSION}-1~$(lsb_release -sc)" + +RUN apt-key add - < /opt/docker/irods_clients_dev/keys/git-core-ppa.asc && \ + echo "deb https://ppa.launchpadcontent.net/git-core/ppa/ubuntu $(lsb_release -sc) main" |\ + tee /etc/apt/sources.list.d/git-core.list && \ + apt-get update && \ + apt-get install -q -y --no-install-recommends \ + git + +RUN cd /tmp && \ + curl -sSL -O https://github.com/ebiggers/libdeflate/releases/download/v${DEFLATE_VERSION}/libdeflate-${DEFLATE_VERSION}.tar.gz && \ + tar xzf libdeflate-${DEFLATE_VERSION}.tar.gz && \ + cd libdeflate-${DEFLATE_VERSION} && \ + cmake . && \ + cmake --build . RUN cd /tmp && \ curl -sSL -O "https://github.com/samtools/htslib/releases/download/${HTSLIB_VERSION}/htslib-${HTSLIB_VERSION}.tar.bz2" && \ tar xfj htslib-${HTSLIB_VERSION}.tar.bz2 && \ cd htslib-${HTSLIB_VERSION} && \ - ./configure --enable-plugins && \ + CPPFLAGS=-I../libdeflate-${DEFLATE_VERSION} LDFLAGS=-L../libdeflate-${DEFLATE_VERSION} ./configure --enable-plugins --with-libdeflate && \ make && \ make install && \ ldconfig +RUN cd /tmp && \ + curl -sSL -O "https://github.com/samtools/htslib-plugins/archive/refs/tags/201712.tar.gz" && \ + tar xfz 201712.tar.gz && \ + cd htslib-plugins-201712 && \ + export IRODS_HOME=/usr/include/irods && \ + make && \ + make install + RUN cd /tmp && \ curl -sSL -O "https://github.com/samtools/samtools/releases/download/${SAMTOOLS_VERSION}/samtools-${SAMTOOLS_VERSION}.tar.bz2" && \ tar xfj samtools-${SAMTOOLS_VERSION}.tar.bz2 && \ cd samtools-${SAMTOOLS_VERSION} && \ - ./configure --with-htslib=system && \ + ./configure --with-htslib=/tmp/htslib-${HTSLIB_VERSION} && \ make install RUN cd /tmp && \ curl -sSL -O "https://github.com/samtools/bcftools/releases/download/${BCFTOOLS_VERSION}/bcftools-${BCFTOOLS_VERSION}.tar.bz2" && \ tar xfj bcftools-${BCFTOOLS_VERSION}.tar.bz2 && \ cd bcftools-${BCFTOOLS_VERSION} && \ - ./configure --with-htslib=system && \ + ./configure --with-htslib=/tmp/htslib-${HTSLIB_VERSION} && \ make install +# +# Next stage: make everything smaller +# +FROM ubuntu:18.04 + +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get upgrade -y && \ + apt-get install -qq -y --no-install-recommends \ + libssl-dev \ + gpg \ + gpg-agent \ + locales && \ + locale-gen en_GB en_GB.UTF-8 && \ + localedef -i en_GB -c -f UTF-8 -A /usr/share/locale/locale.alias en_GB.UTF-8 + + +COPY --from=samtools_build /usr/local/bin/samtools /usr/local/bin/samtools +COPY --from=samtools_build /usr/local/bin/bcftools /usr/local/bin/bcftools +COPY --from=samtools_build /usr/local/bin/htsfile /usr/local/bin/htsfile +COPY --from=samtools_build /usr/local/bin/tabix /usr/local/bin/tabix +COPY --from=samtools_build /usr/local/lib/lib* /usr/local/lib/ +COPY --from=samtools_build /usr/lib/lib* /usr/lib/ +COPY --from=samtools_build /usr/local/libexec/htslib /usr/local/libexec/htslib/ +COPY --from=samtools_build /opt/irods-externals /opt/irods-externals +COPY --from=samtools_build /tmp/libdeflate-1.19/libdeflate.so* /usr/local/lib/ + # Copy the singularity-wrapper scripts ARG DOCKER_IMAGE ARG DOCKER_TAG -#ENV MANIFEST_PATH="./manifest.json" ENV DOCKER_IMAGE="$DOCKER_IMAGE" ENV DOCKER_TAG="$DOCKER_TAG" RUN ldconfig COPY --from=singularity ./scripts/* /usr/local/bin/ -COPY ./manifest.json /opt/wtsi-npg/etc/manifest.json +COPY ./manifest.txt /opt/wtsi-npg/etc/manifest.txt -CMD ["/usr/bin/bash"] +CMD ["/bin/bash"] diff --git a/docker/irods_clients_dev/keys/git-core-ppa.asc b/docker/irods_clients_dev/keys/git-core-ppa.asc new file mode 100644 index 0000000..2d6e128 --- /dev/null +++ b/docker/irods_clients_dev/keys/git-core-ppa.asc @@ -0,0 +1,13 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mI0ESXjaGwEEAMA26F3+mnRW8uRqASMsEa5EsmgvUpLD7EKpC7903OpiMGSvZ2sE +34g7W6nUQY0R//AZS2iW4ZXfvdhQTQuuPlHM6Q3iUAt+nyXcf9xBlscs8Gm722u4 +jAtFtBS4BMQRhRRfWTHwJIOM6OpGIccjPe8pQfIeoRxkKJxlehzw2mU1ABEBAAG0 +KExhdW5jaHBhZCBQUEEgZm9yIFVidW50dSBHaXQgTWFpbnRhaW5lcnOItgQTAQIA +IAUCSXjaGwIbAwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJEKFxXYjh3x8k/zMD +/RKBMjavvFl71YBazSOGl2YfSsZiR/ANsby3+rUaULb8uxzCHXAQnlH5vdtLSPry +aLBvzCU8C3C02qNT8jRacU2752zsCkCi1SLRSOXdI/ATJHza5aTvYV93rTITBhU4 +sJQeK9RW0CtaDRAxJsn/Dr6J3lL/c9m9cT5fFpxOIsF4 +=33dX +-----END PGP PUBLIC KEY BLOCK----- + diff --git a/docker/manifest.json b/docker/manifest.json deleted file mode 100644 index 879b07a..0000000 --- a/docker/manifest.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "package": "samtools", - "executable": [ - "bcftools", - "samtools", - "htsfile", - "tabix" - ] -} diff --git a/docker/manifest.txt b/docker/manifest.txt new file mode 100644 index 0000000..e665aa3 --- /dev/null +++ b/docker/manifest.txt @@ -0,0 +1,4 @@ +bcftools +samtools +htsfile +tabix diff --git a/singularity/scripts/singularity-service-docker b/singularity/scripts/singularity-service-docker index 63f9fe7..9ab4460 100755 --- a/singularity/scripts/singularity-service-docker +++ b/singularity/scripts/singularity-service-docker @@ -1,6 +1,6 @@ #!/bin/bash # -# Running this script will start an instance of the container, if one is not +# Running this script will start an instance of the container, if one is not # already running, and then execute the command in the instance. # # Cleaning up unwanted instances afterwards is outside the scope of this script. @@ -11,12 +11,45 @@ DOCKER_USER=${DOCKER_USER:?required, but not set} DOCKER_IMAGE=${DOCKER_IMAGE:?required, but was not set} DOCKER_TAG=${DOCKER_TAG:?required, but not set} +DOCKER_URL="docker://$DOCKER_REGISTRY/$DOCKER_USER/$DOCKER_IMAGE:$DOCKER_TAG" + # Colons and slashes are not allowed in Singularity instance names instance="$DOCKER_REGISTRY--$DOCKER_USER--$DOCKER_IMAGE--$DOCKER_TAG" -if ! singularity instance list | grep "$instance" 2>&1 | logger -p user.notice -t singularity-service-docker ; then - singularity instance start \ - "docker://$DOCKER_REGISTRY/$DOCKER_USER/$DOCKER_IMAGE:$DOCKER_TAG" "$instance" 2>&1 | logger -p user.notice -t singularity-service-docker +# Requires bash >=4.1 for file descriptor management with "{}" +LOCK_FILE="/var/lock/$instance--$USER.lock" +LOCK_TIMEOUT=60 + +# Get the first free file descriptor +exec {FD}>"$LOCK_FILE" + +# Release the lock and close the file descriptor +clean_up() { + flock --unlock $FD || { + echo >&2 "Failed to release lock on $LOCK_FILE (fd: $FD): $?" + } + exec {FD}>&- +} + +clean_up_and_exit() { + clean_up + exit 1 +} + +# clean_up_and_exit on these signals +trap clean_up_and_exit SIGINT SIGTERM SIGUSR1 SIGUSR2 + +flock --exclusive --timeout $LOCK_TIMEOUT $FD || { + echo >&2 "Failed to obtain lock on $LOCK_FILE (fd: $FD): $?" + exit 1 +} + +if ! singularity instance list 2> >(logger -s -p user.err -t singularity-service-docker) |\ + grep "$instance" 2>&1 >/dev/null +then + singularity --silent instance start "$DOCKER_URL" "$instance" 2> >(logger -s -p user.err -t singularity-service-docker) >/dev/null fi +clean_up + singularity exec "instance://$instance" "$@" diff --git a/singularity/scripts/singularity-wrapper b/singularity/scripts/singularity-wrapper index 371af21..ead3db5 100755 --- a/singularity/scripts/singularity-wrapper +++ b/singularity/scripts/singularity-wrapper @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # # Copyright (C) 2023 Genome Research Ltd. All rights reserved. # @@ -17,8 +17,6 @@ # # Author: Keith James -set -euo pipefail - usage() { cat 1>&2 << EOF Install singularity proxy wrappers for the executables listed in a @@ -36,7 +34,7 @@ Usage: $0 [-e] [-h] [-i ] - [-m ] + [-m ] [-p ] [-r ] [-s] @@ -73,7 +71,7 @@ EOF # Print an application manifest print_manifest() { - jq . "$MANIFEST_PATH" + cat "$MANIFEST_PATH" } # Write a bash script wrapping an application in a Docker container @@ -113,9 +111,9 @@ install_wrappers() { install -d "$dir" cp "/usr/local/bin/$singularity_wrap_impl" "$PREFIX/bin" - for exe in "${wrappers[@]}" ; do + while IFS= read -r exe; do write_wrapper "$dir" "$exe" - done + done < "$MANIFEST_PATH" } DOCKER_REGISTRY=${DOCKER_REGISTRY:-ghcr.io} @@ -124,7 +122,7 @@ DOCKER_IMAGE=${DOCKER_IMAGE:-""} DOCKER_TAG=${DOCKER_TAG:-latest} PREFIX=${PREFIX:-/opt/wtsi-npg} -MANIFEST_PATH=${MANIFEST_PATH:-"$PREFIX/etc/manifest.json"} +MANIFEST_PATH=${MANIFEST_PATH:-"$PREFIX/etc/manifest.txt"} singularity_wrap_impl="singularity-run-docker" @@ -171,19 +169,12 @@ done shift $((OPTIND -1)) -declare -a wrappers if [ ! -e "$MANIFEST_PATH" ] ; then echo -e "\nERROR:\n The manifest of executables at '$MANIFEST_PATH' does not exist" exit 4 fi - - - - -wrappers=($(jq -j '.executable[] + " "' "$MANIFEST_PATH")) - -operation="$@" +operation="$1" if [ -z "$operation" ] ; then usage echo -e "\nERROR:\n An operation argument is required"