Merge branch 'release/v3.5.0'

cancerit · May 21, 2021 · b5ba69c · b5ba69c
2 parents caa3ce0 + 33db56c
commit b5ba69c
Show file tree

Hide file tree

Showing 39 changed files with 163 additions and 200 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,9 +1,12 @@
-# 1. Set your default branch to develop
+# 1. Set your GitHub default branch to dev/develop
 # 2. In circleci project settings->advanced set:
 #    - GitHub Status Updates (this should be on already)
 #    - Only build pull requests
 #    - Auto-cancel redundant builds
 #    - Free and Open Source (this should be on already)
+# 3. If existing project with quay.io, remove the quay build triggers.
+#    - as circleci is pusing the image
+#    - reevaluate when/if quay.io adds ability to authenticate pulls
 
 version: 2.1
 jobs:
@@ -19,10 +22,10 @@ jobs:
           name: Build Docker image
           command: |
             set -e
-            BRANCH_OR_TAG="${CIRCLE_TAG:-$CLEAN_BRANCH}"
+            BRANCH_OR_TAG="${CIRCLE_TAG:-$CIRCLE_BRANCH}"
             CLEAN_BRANCH=$(echo $BRANCH_OR_TAG | tr / -)
-            docker login -u "$DOCKERHUB_USER" -p "$DOCKERHUB_PASSWORD"
-            docker login -u "$QUAYIO_USER" -p "$QUAYIO_PASSWORD" quay.io
+            echo "$DOCKERHUB_PASSWORD" | docker login -u "$DOCKERHUB_USER" --password-stdin
+            echo "$QUAYIO_PASSWORD" | docker login -u "$QUAYIO_USER" --password-stdin quay.io
             docker build -t $IMAGE_NAME:$CLEAN_BRANCH .
             echo 'Verify program(s) from each inherited base image - cgpbigwig'
             docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH bwjoin --version
@@ -59,50 +62,3 @@ workflows:
           filters:
             tags:
               only: /.+/
-
-
-# # Use the latest 2.1 version of CircleCI pipeline process engine. See: https://circleci.com/docs/2.0/configuration-reference
-# version: 2.1
-
-# workflows:
-#   build:
-#     jobs:
-#       - build:
-#           context:
-#             - dockerhub-casmservice
-#       - publish:
-#           context:
-#             - quayio-casmservice
-
-# jobs:
-#  build:
-#    machine: true
-#    steps:
-#      - checkout
-#       # build the image from our area, use docker login to protect against future rate-limiting
-#      - run: |
-#           set -e
-#           echo $CIRCLE_BRANCH
-#           docker login -u $DOCKERHUB_USER -p $DOCKERHUB_PASSWORD
-#           docker build -t quay.io/wtsicgp/cgppindel:$CIRCLE_BRANCH .
-#           docker images | grep -c cgppindel
-#           echo 'Verify program(s) from each inherited base image - dockstore-cgpbigwig'
-#           docker run -t --rm quay.io/wtsicgp/cgppindel:$CIRCLE_BRANCH bwjoin --version
-#           echo 'Verify program(s) from each inherited base image - dockstore-cgpmap'
-#           docker run -t --rm quay.io/wtsicgp/cgppindel:$CIRCLE_BRANCH bwa_mem.pl -version
-#           docker run -t --rm quay.io/wtsicgp/cgppindel:$CIRCLE_BRANCH bammarkduplicates2 --version
-#           docker run -t --rm quay.io/wtsicgp/cgppindel:$CIRCLE_BRANCH samtools --version
-#           docker run -t --rm quay.io/wtsicgp/cgppindel:$CIRCLE_BRANCH bash -c 'bwa 2>&1 | grep Version'
-#           echo 'Verify program(s) from this repo'
-#           docker run -t --rm quay.io/wtsicgp/cgppindel:$CIRCLE_BRANCH pindel.pl --version
-#           docker run -t --rm quay.io/wtsicgp/cgppindel:$CIRCLE_BRANCH pindel_input_gen.pl --version
-#           docker run -t --rm quay.io/wtsicgp/cgppindel:$CIRCLE_BRANCH pindel_2_combined_vcf.pl --version
-#           docker run -t --rm quay.io/wtsicgp/cgppindel:$CIRCLE_BRANCH FlagVcf.pl --version
-#           docker run -t --rm quay.io/wtsicgp/cgppindel:$CIRCLE_BRANCH pindel_merge_vcf_bam.pl --version
-#           docker run -t --rm quay.io/wtsicgp/cgppindel:$CIRCLE_BRANCH pindel_np_from_vcf.pl --version
-#           docker run -t --rm quay.io/wtsicgp/cgppindel:$CIRCLE_BRANCH pindel_germ_bed.pl --version
-#           docker run -t --rm quay.io/wtsicgp/cgppindel:$CIRCLE_BRANCH which pindel
-#           docker run -t --rm quay.io/wtsicgp/cgppindel:$CIRCLE_BRANCH which filter_pindel_reads
-
-#      ## deploy the image
-#      #- run: docker push company/app:$CIRCLE_BRANCH
diff --git a/.dockerignore b/.dockerignore
@@ -9,3 +9,4 @@
 /perl/docs
 /perl/docs.tar.gz
 /python/env
+/install_tmp
diff --git a/README.md b/README.md
@@ -2,11 +2,7 @@
 
 cgpPindel contains the Cancer Genome Projects workflow for [Pindel][pindel-core].
 
-[![Quay Badge][quay-status]][quay-repo]
-
-| Master                                        | Develop                                         |
-| --------------------------------------------- | ----------------------------------------------- |
-| [![Master Badge][travis-master]][travis-base] | [![Develop Badge][travis-develop]][travis-base] |
+[![cancerit](https://circleci.com/gh/cancerit/cgpPindel.svg?style=svg)](https://circleci.com/gh/cancerit/cgpPindel)
 
 The is a lightly modified version of pindel v2.0 with CGP specific processing for:
 
@@ -18,25 +14,24 @@ The is a lightly modified version of pindel v2.0 with CGP specific processing fo
 
 Contents:
 
-<!-- TOC depthFrom:2 depthTo:6 withLinks:1 updateOnSave:1 orderedList:0 -->
-
 * [Docker, Singularity and Dockstore](#docker-singularity-and-dockstore)
 * [Dependencies/Install](#dependenciesinstall)
 * [Creating a release](#creating-a-release)
   * [Preparation](#preparation)
   * [Release process](#release-process)
     * [Code changes](#code-changes)
-    * [Docker image](#docker-image)
+    * [Testing](#testing)
+      * [Regression CI](#regression-ci)
+      * [Public CI](#public-ci)
     * [Cutting the release](#cutting-the-release)
 * [LICENCE](#licence)
 
-<!-- /TOC -->
-
 ## Docker, Singularity and Dockstore
 
-There are pre-built images containing this codebase on quay.io.
+There are pre-built images containing this codebase on quay.io.  When pulling an image you must specify
+the version there is no `latest`.
 
-* [cgpPindel][cgpPindel-git]: Contained within this repository
+* [cgpPindel quay.io][quay-repo]: Contained within this repository
   * Smallest build required to use cgpPindel
   * Not linked to Dockstore (yet)
   * Updated most frequently
@@ -49,7 +44,7 @@ The docker images are known to work correctly after import into a singularity im
 
 ## Dependencies/Install
 
-Please install the following first:
+When doing a native install please install the following first:
 
 * [PCAP-core v2.0+][pcap-core-rel]
 * [cgpVcf v2.0+][cgpvcf-rel]
@@ -77,37 +72,48 @@ Please be aware that this expects basic C compilation libraries and tools to be
 
 ### Release process
 
-This project is maintained using HubFlow.
+This project is maintained using the [HubFlow][hubflow-docs] methodology.
 
 #### Code changes
 
 1. Make appropriate changes
 2. Update `perl/lib/Sanger/CGP/Pindel.pm` to the correct version (adding rc/beta to end if applicable).
 3. Update `CHANGES.md` to show major items.
-4. Run `./prerelease.sh`
-5. Check all tests and coverage reports are acceptable.
-6. Commit the updated docs and updated module/version.
-7. Push commits.
+4. Commit the updated docs and updated module/version.
+5. Push commits.
+
+#### Testing
+
+##### Regression CI
 
-#### Docker image
+An internal CI system is used to validate each release using real, large scale datasets.
 
-1. Use the GitHub tools to draft a release.
-2. Build image locally
-3. Run example inputs and verify any changes are acceptable
-4. Bump version in `Dockerfile`
-5. Push changes
+##### Public CI
+
+Circleci is used to:
+
+* Build Docker image (unit tests are part of build)
+* Validate expected tools exist
+* For tags only: push image to quay.io
+
+CI only runs for:
+
+* Branches with pull-requests
+* Default branch (`dev`)
+* Tags
 
 #### Cutting the release
 
-1. Check state on Travis
+Internal regression CI processes must be completed prior to this.
+
+1. Check state on [Circleci][circle-repo]
 2. Generate the release (add notes to GitHub)
 3. Confirm that image has been built on [quay.io][quay-builds]
-4. Update the [dockstore][dockstore-cgpPindel] entry, see [their docs][dockstore-get-started].
 
 ## LICENCE
 
 ```
-Copyright (c) 2014-2018 Genome Research Ltd.
+Copyright (c) 2014-2021 Genome Research Ltd.
 
 Author: CASM/Cancer IT <[email protected]>
 
@@ -142,15 +148,15 @@ identical to a statement that reads ‘Copyright (c) 2005, 2006, 2007, 2008,
 [pcap-core-rel]: https://github.com/cancerit/PCAP-core/releases
 [ds-cgpwxs-git]: https://github.com/cancerit/dockstore-cgpwxs
 [ds-cgpwgs-git]: https://github.com/cancerit/dockstore-cgpwgs
-[cgpPindel-git]: https://github.com/cancerit/cgpPindel
 [pindel-core]: http://gmt.genome.wustl.edu/pindel/current
+[hubflow-docs]: https://datasift.github.io/gitflow/
 
-<!-- Travis -->
-[travis-base]: https://travis-ci.org/cancerit/cgpPindel
-[travis-master]: https://travis-ci.org/cancerit/cgpPindel.svg?branch=master
-[travis-develop]: https://travis-ci.org/cancerit/cgpPindel.svg?branch=dev
+<!-- Circleci -->
+[circle-repo]: https://app.circleci.com/pipelines/github/cancerit/cgpPindel
+[circle-badge-svg]: https://circleci.com/gh/cancerit/cgpPindel.svg?style=svg
+<!-- point this at the default branch -->
+[circle-badge-link]: https://travis-ci.org/cancerit/cgpPindel.svg?branch=dev
 
 <!-- Quay.io -->
-[quay-status]: https://quay.io/repository/wtsicgp/cgppindel/status
 [quay-repo]: https://quay.io/repository/wtsicgp/cgppindel
 [quay-builds]: https://quay.io/repository/wtsicgp/cgppindel?tab=builds
diff --git a/c++/pindel.cpp b/c++/pindel.cpp
@@ -1216,7 +1216,7 @@ if (Analyze_TD_INV_LI_Others) {
                                      }
 
 									 {
-										 if (Reads[ReadIndex].IndelSize >= MIN_IndelSize_NT && Reads[ReadIndex].IndelSize > Reads[ReadIndex].NT_size)
+										 if (Reads[ReadIndex].IndelSize >= MIN_IndelSize_NT)
 										 {
 											 DI[Reads[ReadIndex].BPLeft / BoxSize].push_back(ReadIndex);
 											 Reads[ReadIndex].Used = true;
@@ -1274,7 +1274,7 @@ if (Analyze_TD_INV_LI_Others) {
                                      }
 
 									 {
-										 if (Reads[ReadIndex].IndelSize >= MIN_IndelSize_NT && Reads[ReadIndex].IndelSize > Reads[ReadIndex].NT_size)
+										 if (Reads[ReadIndex].IndelSize >= MIN_IndelSize_NT)
 										 {
 											 DI[Reads[ReadIndex].BPLeft / BoxSize].push_back(ReadIndex);
 											 Reads[ReadIndex].Used = true;
@@ -2266,14 +2266,14 @@ void CheckLeft_Close(const SPLIT_READ & OneRead,
 		for (short i = 0; i <= OneRead.MAX_SNP_ERROR; i++) {
 			if (Left_PD[i].size() == 1 && CurrentLength >= BP_Left_Start + i) {
 				Sum = 0;
-				if (ADDITIONAL_MISMATCH)                               
+				if (ADDITIONAL_MISMATCH)
 				   for (short j = 0; j <= i+ADDITIONAL_MISMATCH; j++)
 			   		  Sum += Left_PD[j].size();                      //Maybe the previous SNPS still exist; revise in 2018.11.26 by Liang Hao
-				if (Sum == 1 && i <= (short)(CurrentLength * Seq_Error_Rate + 1)) {       
+				if (Sum == 1 && i <= (short)(CurrentLength * Seq_Error_Rate + 1)) {
 					UniquePoint TempOne;
 					TempOne.LengthStr = CurrentLength;
 					TempOne.AbsLoc = Left_PD[i][0];
-					
+
 					TempOne.Direction = FORWARD;
 					TempOne.Strand = ANTISENSE;
 					TempOne.Mismatches = i;
@@ -2356,7 +2356,7 @@ void CheckRight_Close(const SPLIT_READ & OneRead,
 	   for (short i = 0; i <= OneRead.MAX_SNP_ERROR; i++) {
 		   if (Right_PD[i].size() == 1 && CurrentLength >= BP_Right_Start + i) {
 				Sum = 0;
-				if (ADDITIONAL_MISMATCH)                               
+				if (ADDITIONAL_MISMATCH)
 				   for (short j = 0; j <= i+ADDITIONAL_MISMATCH; j++)
 			   		  Sum += Right_PD[j].size();                      //Maybe the previous SNPS still exist; revise in 2018.11.26 by Liang Hao
 				if (Sum == 1 && i <= (short)(CurrentLength * Seq_Error_Rate + 1)) {
@@ -2446,10 +2446,10 @@ void CheckLeft_Far(const SPLIT_READ & OneRead,
 		for (short i = 0; i <= OneRead.MAX_SNP_ERROR; i++) {
 			if (Left_PD[i].size() == 1 && CurrentLength >= BP_Left_Start + i) {
 				Sum = 0;
-				if (ADDITIONAL_MISMATCH)                               
+				if (ADDITIONAL_MISMATCH)
 				   for (short j = 0; j <= i+ADDITIONAL_MISMATCH; j++)
 			   		  Sum += Left_PD[j].size();                      //Maybe the previous SNPS still exist; revise in 2018.11.26 by Liang Hao
-				if (Sum == 1 && i <= (short)(CurrentLength * Seq_Error_Rate + 1)) {  
+				if (Sum == 1 && i <= (short)(CurrentLength * Seq_Error_Rate + 1)) {
 					UniquePoint TempOne;
 					TempOne.LengthStr = CurrentLength;
 					TempOne.AbsLoc = Left_PD[i][0];
@@ -2565,7 +2565,7 @@ void CheckRight_Far(const SPLIT_READ & OneRead,
 	   for (short i = 0; i <= OneRead.MAX_SNP_ERROR; i++) {
 		   if (Right_PD[i].size() == 1 && CurrentLength >= BP_Right_Start + i) {
 				Sum = 0;
-				if (ADDITIONAL_MISMATCH)                               
+				if (ADDITIONAL_MISMATCH)
 				   for (short j = 0; j <= i+ADDITIONAL_MISMATCH; j++)
 			   		  Sum += Right_PD[j].size();                      //Maybe the previous SNPS still exist; revise in 2018.11.26 by Liang Hao
 				if (Sum == 1 && i <= (short)(CurrentLength * Seq_Error_Rate + 1)) {
@@ -4648,7 +4648,7 @@ void GetFarEnd_SingleStrandDownStreamInsertions(const string & CurrentChr, SPLIT
 
 		End = Temp_One_Read.UP_Close[0].AbsLoc + Temp_One_Read.UP_Close[0].LengthStr;
 		if (End > SpacerBeforeAfter + Temp_One_Read.InsertSize * 2 + DSizeArray[RangeIndex])
-		   Start = End - DSizeArray[RangeIndex] - Temp_One_Read.InsertSize * 2; 
+		   Start = End - DSizeArray[RangeIndex] - Temp_One_Read.InsertSize * 2;
 		else Start = SpacerBeforeAfter;
 
 

diff --git a/perl/Makefile.PL b/perl/Makefile.PL
@@ -1,7 +1,7 @@
 #!/usr/bin/perl
 
 ########## LICENCE ##########
-# Copyright (c) 2014-2019 Genome Research Ltd.
+# Copyright (c) 2014-2021 Genome Research Ltd.
 #
 # Author: CASM/Cancer IT <[email protected]>
 #

diff --git a/perl/bin/FlagVcf.pl b/perl/bin/FlagVcf.pl
@@ -1,7 +1,7 @@
 #!/usr/bin/perl
 
 ########## LICENCE ##########
-# Copyright (c) 2014-2018 Genome Research Ltd.
+# Copyright (c) 2014-2021 Genome Research Ltd.
 #
 # Author: CASM/Cancer IT <[email protected]>
 #

diff --git a/perl/bin/pindel.pl b/perl/bin/pindel.pl
@@ -1,7 +1,7 @@
 #!/usr/bin/perl
 
 ########## LICENCE ##########
-# Copyright (c) 2014-2018 Genome Research Ltd.
+# Copyright (c) 2014-2021 Genome Research Ltd.
 #
 # Author: CASM/Cancer IT <[email protected]>
 #

diff --git a/perl/bin/pindel_2_combined_vcf.pl b/perl/bin/pindel_2_combined_vcf.pl
@@ -1,7 +1,7 @@
 #!/usr/bin/perl
 
 ########## LICENCE ##########
-# Copyright (c) 2014-2018 Genome Research Ltd.
+# Copyright (c) 2014-2021 Genome Research Ltd.
 #
 # Author: CASM/Cancer IT <[email protected]>
 #

diff --git a/perl/bin/pindel_germ_bed.pl b/perl/bin/pindel_germ_bed.pl
@@ -1,7 +1,7 @@
 #!/usr/bin/perl
 
 ########## LICENCE ##########
-# Copyright (c) 2014-2018 Genome Research Ltd.
+# Copyright (c) 2014-2021 Genome Research Ltd.
 #
 # Author: CASM/Cancer IT <[email protected]>
 #

diff --git a/perl/bin/pindel_input_gen.pl b/perl/bin/pindel_input_gen.pl
@@ -1,7 +1,7 @@
 #!/usr/bin/perl
 
 ########## LICENCE ##########
-# Copyright (c) 2014-2019 Genome Research Ltd.
+# Copyright (c) 2014-2021 Genome Research Ltd.
 #
 # Author: CASM/Cancer IT <[email protected]>
 #

diff --git a/perl/bin/pindel_merge_vcf_bam.pl b/perl/bin/pindel_merge_vcf_bam.pl
@@ -1,7 +1,7 @@
 #!/usr/bin/perl
 
 ########## LICENCE ##########
-# Copyright (c) 2014-2018 Genome Research Ltd.
+# Copyright (c) 2014-2021 Genome Research Ltd.
 #
 # Author: CASM/Cancer IT <[email protected]>
 #

diff --git a/perl/bin/pindel_np_from_vcf.pl b/perl/bin/pindel_np_from_vcf.pl
@@ -1,7 +1,7 @@
 #!/usr/bin/perl
 
 ########## LICENCE ##########
-# Copyright (c) 2014-2018 Genome Research Ltd.
+# Copyright (c) 2014-2021 Genome Research Ltd.
 #
 # Author: CASM/Cancer IT <[email protected]>
 #

diff --git a/perl/lib/Sanger/CGP/Pindel.pm b/perl/lib/Sanger/CGP/Pindel.pm
@@ -1,7 +1,7 @@
 package Sanger::CGP::Pindel;
 
 ########## LICENCE ##########
-# Copyright (c) 2014-2020 Genome Research Ltd.
+# Copyright (c) 2014-2021 Genome Research Ltd.
 #
 # Author: CASM/Cancer IT <[email protected]>
 #
@@ -26,7 +26,7 @@ use strict;
 use Const::Fast qw(const);
 
 use base 'Exporter';
-our $VERSION = '3.4.1';
+our $VERSION = '3.5.0';
 our @EXPORT = qw($VERSION);
 
 1;
diff --git a/perl/lib/Sanger/CGP/Pindel/Implement.pm b/perl/lib/Sanger/CGP/Pindel/Implement.pm
@@ -1,7 +1,7 @@
 package Sanger::CGP::Pindel::Implement;
 
 ########## LICENCE ##########
-# Copyright (c) 2014-2018 Genome Research Ltd.
+# Copyright (c) 2014-2021 Genome Research Ltd.
 #
 # Author: CASM/Cancer IT <[email protected]>
 #