Merge pull request #114 from wtsi-npg/devel

Merge devel to master

.gitattributes

@@ -0,0 +1 @@
+*.bz2 -diff

.travis.yml

@@ -0,0 +1,40 @@
+
+dist: xenial
+language: python
+python: 3.7
+
+services: docker
+
+env:
+  - >
+    IRODS_IMAGE=wsinpg/ub-12.04-conda-irods-4.1:0.3
+    CONDA_IMAGE=wsinpg/ub-12.04-conda:0.3
+    CONDA_CHANNEL=https://dnap.cog.sanger.ac.uk/npg/conda/devel/generic
+    COMPARE_BRANCH=origin/devel
+
+before_install:
+  - docker --version
+  # We should rarely need to pull the iRODS build image, so we don't
+  # do it preemptively here
+  - docker pull $CONDA_IMAGE
+
+install:
+  - pip install jinja2 networkx packaging pyaml rfc3987 setuptools
+  # Ensure we have a complete repository to compare diffs with other
+  # branches
+  - git config remote.origin.fetch +refs/heads/*:refs/remotes/origin/*
+  - git fetch
+
+script:
+  - mkdir -p $TRAVIS_BUILD_DIR/conda-artefacts
+  - while sleep 300; do echo $SECONDS sec elapsed, still building ... ; done &
+  - >
+    ./scripts/package_sort.py --changes $COMPARE_BRANCH | ./scripts/build.py
+    --recipes-dir $TRAVIS_BUILD_DIR
+    --artefacts-dir $TRAVIS_BUILD_DIR/conda-artefacts
+    --irods-build-image $IRODS_IMAGE
+    --conda-build-image $CONDA_IMAGE
+    --build-channel $CONDA_CHANNEL
+    --verbose
+  - kill %1
+  - ls -l $TRAVIS_BUILD_DIR/conda-artefacts/linux-64

ChangeLog

@@ -0,0 +1,5 @@
+
+ensure that libmaus2 and scramble are built with a particular version of libstaden-read. These versions need to match
+because "biobambam [...] won't work with modern io_lib's until it is updated to use the resized structure elements"
+(see https://github.com/samtools/samtools/issues/980). So currently (5Aug2019) biobambam2 2.0.79 requires
+libmaus2 2.0.420 which requires libstaden-read 1.14.9.

README.md

@@ -0,0 +1,86 @@
+This repository contains [Conda](https://conda.io) recipes to build
+tools and libraries used by [WSI NPG](https://github.com/wtsi-npg).
+
+Our recipes differ from those provided by
+[Anaconda Inc.](https://github.com/AnacondaRecipes),
+[Conda Forge](https://conda-forge.org) and
+[BioConda](https://bioconda.github.io/) in order to meet our specific
+needs:
+
+* Build artefacts are separated into sub-packages. For a typical
+  package written in C and named `example`, these would be:
+
+  * `example` containing executables and their documentation, such
+    as manpages.
+
+  * `libexample` containing the example libraries (static and shared).
+
+  * `example-dev` containing the C headers, any pkg-config files,
+    build-time configuration executables and API manpages.
+
+* Recipes depend only on the Anaconda `defaults` channel
+
+* We maintain recipes for multiple versions of packages in
+  production. The recipes are located in directory hierarchy by name
+  and version.
+
+* Recipes do not support Windows or MacOS.
+
+Typical Conda recipes create a single package bundling all build
+artefacts (executables, libraries, headers, manpages etc) together, so
+installing a program that depends on a C shared library from another
+package will cause any executables in that package also to be
+installed in the target environment. This is something we specifically
+want to avoid.
+
+We avoid using the `conda-forge` and `bioconda` channels so that we
+are in complete control of the deployed package dependency graph and
+do not unexpectedly upgrade (or downgrade) packages that may affect
+data analysis.
+
+We don't use the Windows or MacOS platforms, so we simplify our
+recipes by omitting support for them.
+
+### Building the recipes ###
+
+Building from source requires Conda (we use
+[Miniconda](https://docs.conda.io/en/latest/miniconda.html)), with the
+[conda-build](https://github.com/conda/conda-build) and
+[conda-verify](https://github.com/conda/conda-verify) packages
+installed.
+
+A complete from-source build can be achieved using the
+`./scripts/package_sort.py` script which inspects the recipes,
+calculates their dependency DAG and then outputs a list sorted so that
+they are built in the correct order:
+
+    ./scripts/package_sort.py recipes/ | head -4
+    rna-seqc 1.1.8 recipes/rna-seqc/1.1.8
+    bowtie2 2.2.7 recipes/bowtie2/2.2.7
+    teepot 1.2.0 recipes/teepot/1.2.0
+    eigen 3.3.4 recipes/eigen/3.3.4
+
+The list of recipe paths may be passed directly to `conda-build`:
+
+    ./scripts/package_sort.py recipes | awk '{print $3}' | xargs conda-build 
+
+which will build everything using just the Anaconda defaults channel
+and the local channel for dependencies.
+
+
+### Naming new recipes ###
+
+The rules are:
+
+1. The package containing the executables should be named after the
+commonly used name for the software (e.g. bwa, minimap2, curl)
+
+2. If 1. is not possible e.g. because the executables are in
+sub-package, the Conda meta-package is renamed [package name]-pkg and
+the executables sub-package keeps the common name
+(e.g. curl-pkg,curl,libcurl,libcurl-dev)
+
+3. If 2. is not possible because e.g. the common name for the software
+is a library name and the software also provides executables then the
+executables package is renamed [package]-bin
+(e.g. libml2-pkg,libxml2-bin,libxml2,libxml2-dev)

conda_build_config.yaml

@@ -0,0 +1,11 @@
+
+CFLAGS: >-
+  -O2
+  -fPIC
+  -ffunction-sections
+  -fno-plt
+  -fstack-protector-strong
+  -ftree-vectorize
+  -march=x86-64
+  -mtune=generic
+  -pipe

config/conda_build_config.yaml

@@ -0,0 +1,10 @@
+# This is the default build config which uses -march=x86-64 to create
+# programs that are portable across all of our CPU architectures.
+# Replacing this with e.g. -march=native will use instructions that
+# may not be poratble to other types of machine. See
+# https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html#index-march-13
+
+CFLAGS: >-
+  -march=x86-64 -mtune=generic
+  -fPIC -O2
+  -ftree-vectorize -fstack-protector-strong -ffunction-sections -pipe

recipes/bam_stats/1.13.0/Makefile.patch

@@ -0,0 +1,39 @@
+diff --git c/Makefile c/Makefile
+index 5b6474f..b2592d6 100644
+--- c/Makefile
++++ c/Makefile
+@@ -2,14 +2,14 @@
+ VERSION=$(shell perl -I../lib -MPCAP -e 'print PCAP->VERSION;')
+
+ #Compiler
+-CC = gcc -O3 -DVERSION='"$(VERSION)"' -g
++CC ?= gcc -O3 -DVERSION='"$(PKG_VERSION)"' -g
+
+ #CC = gcc -O3 -DVERSION='"$(VERSION)"' -g
+
+ #compiler flags
+ # -g adds debug info to the executable file
+ # -Wall turns on most warnings from compiler
+-CFLAGS = -Wall
++CFLAGS ?= -Wall
+
+ HTSLOC?=$(HTSLIB)
+
+@@ -64,7 +64,7 @@ all: clean make_htslib_tmp $(BAM_STATS_TARGET) test remove_htslib_tmp
+ 	@echo  bam_stats compiled.
+
+ $(BAM_STATS_TARGET): $(OBJS)
+-	$(CC) $(CFLAGS) $(INCLUDES) -o $(BAM_STATS_TARGET) $(OBJS) $(LFLAGS) $(LIBS) ./bam_stats.c
++	$(CC) $(CFLAGS) $(CPPFLAGS) $(INCLUDES) -o $(BAM_STATS_TARGET) $(OBJS) $(LFLAGS) $(LDFLAGS) $(LIBS) ./bam_stats.c
+
+ #Unit Tests
+ test: $(BAM_STATS_TARGET)
+@@ -98,7 +98,7 @@ valgrind:
+ # the rule(a .c file) and $@: the name of the target of the rule (a .o file)
+ # (see the gnu make manual section about automatic variables)
+ .c.o:
+-	$(CC) $(CFLAGS) $(INCLUDES) -c $<  -o $@
++	$(CC) $(CFLAGS) $(INCLUDES) $(CPPFLAGS) -c $<  -o $@
+
+ clean:
+ 	@echo clean

recipes/bam_stats/1.13.0/build.sh

@@ -1,9 +1,11 @@
 #!/bin/sh
 
-set -e
+set -ex
 
 cd ./c
-perl -i -ple 's/^(\t\$\(CC\).*?)$/$1 \$(LDFLAGS) \$(LDLIBS)/smx' Makefile
-make ../bin/bam_stats CFLAGS="-I./ -I$PREFIX/include" LDFLAGS="-L$PREFIX/lib"
+make ../bin/bam_stats CC="$GCC" \
+     CFLAGS="-O3 -DVERSION='\"$PKG_VERSION\"' -g" \
+     CPPFLAGS="-I$PREFIX/include" \
+     LDFLAGS="-L$PREFIX/lib"
 
 cp ../bin/bam_stats "$PREFIX/bin/"

recipes/bam_stats/1.13.0/meta.yaml

@@ -10,23 +10,22 @@ about:
   summary: "Legacy, see cancerit/PCAP-core: NGS reference implementations and helper code for the IGCG/TCGA Pan-Cancer Analysis Project."
 
 build:
-  number: 0
+  number: 1
 
 source:
   git_url: https://github.com/ICGC-TCGA-PanCancer/PCAP-core.git
   git_rev: b8201e219ac7c7a7d968a23ea836b708a133deb8
-
-features:
-  - irods
+  patches:
+    - Makefile.patch
 
 requirements:
   build:
-    - gcc_npg >=7.3
-    - htslib >=1.7
-
+    - {{ compiler("c") }}
+    - make
+  host:
+    - libhts-dev
   run:
-    - htslib >=1.7
-    - zlib
+    - libhts
 
 test:
   commands:

recipes/bambi/0.11.1/ax_with_htslib.patch

@@ -0,0 +1,12 @@
+index 40e8f48..1b8bafc 100644
+--- m4/ax_with_htslib.m4
++++ m4/ax_with_htslib.m4
+@@ -71,7 +71,7 @@ AC_DEFUN([AX_WITH_HTSLIB], [
+                    LDFLAGS="-L$HTSLIB_HOME/lib $LDFLAGS"
+                ])
+
+-               LIBS="$HTSLIB_HOME/lib/libhts.a -lz -ldl -lbz2 -llzma -lpthread -lcurl -lcrypto"
++               LIBS="-lhts -lz -ldl -lbz2 -llzma -lpthread -lcurl -lcrypto"
+
+                AC_MSG_CHECKING([checking htslib version])
+                AC_RUN_IFELSE([AC_LANG_PROGRAM([

recipes/bambi/0.11.1/build.sh

@@ -1,12 +1,13 @@
 #!/bin/sh
 
-set -e
+set -ex
 
 autoreconf -fi
 
-n=`expr $CPU_COUNT / 4 \| 1`
+n="$CPU_COUNT"
 
-./configure --prefix="$PREFIX" --with-htslib="$PREFIX" CPPFLAGS="-I$PREFIX/include" LDFLAGS="-L$PREFIX/lib"
+CPPFLAGS="-I$PREFIX/include" LDFLAGS="-Wl,-rpath,$PREFIX/lib -L$PREFIX/lib" \
+        ./configure --prefix="$PREFIX" --with-htslib="$PREFIX"
 
 make -j $n CPPFLAGS="-I$PREFIX/include" LDFLAGS="-L$PREFIX/lib"
 make install prefix="$PREFIX"