Merge branch 'develop' into np_jw_test_illumina_genotyping_arrays

beta-pipelines/skylab/slidetags/SlideTags.wdl

@@ -2,17 +2,42 @@ version 1.0
 
 import "scripts/spatial-count.wdl" as SpatialCount
 import "scripts/positioning.wdl" as Positioning
+import "../../../pipelines/skylab/optimus/Optimus.wdl" as optimus
 
 workflow SlideTags {
 
     String pipeline_version = "1.0.0"
 
     input {
+        # slide-tags inputs
         String id
         Array[String] fastq_paths
         Array[String] pucks
         Array[String] rna_paths
         String sb_path
+
+        # Optimus Inputs
+        String cloud_provider = "gcp"
+        Boolean is_slidetags = true
+        String input_id
+        Int expected_cells = 3000 ## copied from Multiome ?
+        String counting_mode = "sn_rna"
+        Array[File] gex_r1_fastq
+        Array[File] gex_r2_fastq
+        Array[File]? gex_i1_fastq        
+        File tar_star_reference
+        File annotations_gtf
+        File? mt_genes
+        Int tenx_chemistry_version = 3
+        Int emptydrops_lower = 100
+        Boolean force_no_check = false
+        Boolean ignore_r1_read_length = false
+        String star_strand_mode = "Reverse"
+        Boolean count_exons = false
+        File gex_whitelist
+        String? soloMultiMappers
+        String? gex_nhash_id
+
         String docker = "us.gcr.io/broad-gotc-prod/slide-tags:1.1.0"
      }
 
@@ -21,6 +46,33 @@ workflow SlideTags {
         pucks: "Array of paths to puck files"
         docker: "Docker image to use"
     }
+
+    # Call the Optimus workflow
+    call optimus.Optimus as Optimus {
+        input:
+            cloud_provider = cloud_provider,
+            #disk_starsolo = disk_starsolo,
+            counting_mode = counting_mode,
+            r1_fastq = gex_r1_fastq,
+            r2_fastq = gex_r2_fastq,
+            i1_fastq = gex_i1_fastq,
+            input_id = input_id + "_gex",
+            output_bam_basename = input_id + "_gex",
+            gex_nhash_id = gex_nhash_id,
+            tar_star_reference = tar_star_reference,
+            annotations_gtf = annotations_gtf,
+            mt_genes = mt_genes,
+            tenx_chemistry_version = tenx_chemistry_version,
+            whitelist = gex_whitelist,
+            emptydrops_lower = emptydrops_lower,
+            force_no_check = force_no_check,
+            ignore_r1_read_length = ignore_r1_read_length,
+            star_strand_mode = star_strand_mode,
+            count_exons = count_exons,
+            soloMultiMappers = soloMultiMappers,
+            gex_expected_cells = expected_cells,
+            is_slidetags = is_slidetags
+    } 
 
     call SpatialCount.count as spatial_count {
         input:

pipeline_versions.txt

@@ -27,14 +27,14 @@ BroadInternalImputation	 1.1.14	2024-11-04
 BroadInternalArrays	 1.1.14	2024-11-04 
 BroadInternalRNAWithUMIs	 1.0.36	2024-11-04 
 RNAWithUMIsPipeline	 1.0.18	2024-11-04 
-Multiome	 5.9.4	2024-12-05 
-MultiSampleSmartSeq2SingleNucleus	 2.0.6	2024-11-15 
+Multiome	 5.9.5	2024-12-12 
+MultiSampleSmartSeq2SingleNucleus	 2.0.7	2024-12-12 
 BuildIndices	 3.1.0	2024-11-26 
-SlideSeq	 3.4.7	2024-12-3 
-PairedTag	 1.9.0	2024-12-05 
+SlideSeq	 3.4.8	2024-12-12 
+PairedTag	 1.9.1	2024-12-12 
 atac	 2.5.3	2024-11-22 
 scATAC	 1.3.2	2023-08-03 
 snm3C	 4.0.4	2024-08-06 
-Optimus	 7.9.0	2024-12-05 
+Optimus	 7.9.1	2024-12-12 
 MultiSampleSmartSeq2	 2.2.22	2024-09-11 
 SmartSeq2SingleSample	 5.1.21	2024-09-11 

pipelines/skylab/multiome/Multiome.changelog.md

@@ -1,3 +1,8 @@
+# 5.9.5
+2024-12-12 (Date of Last Commit)
+
+* Added a boolean variable is_slidetags; default is false but it is set to true if the Slide-Tags pipeline is calling Optimus
+
 # 5.9.4
 2024-12-05 (Date of Last Commit)
 

pipelines/skylab/multiome/Multiome.wdl

@@ -7,7 +7,7 @@ import "../../../tasks/broad/Utilities.wdl" as utils
 
 workflow Multiome {
 
-    String pipeline_version = "5.9.4"
+    String pipeline_version = "5.9.5"
 
     input {
         String cloud_provider

pipelines/skylab/optimus/Optimus.changelog.md

@@ -1,3 +1,8 @@
+# 7.9.1
+2024-12-12 (Date of Last Commit)
+
+* Added a boolean variable is_slidetags; set to false by default, but set to true if the Slide-Tags pipeline is calling Optimus
+
 # 7.9.0
 2024-12-05 (Date of Last Commit)
 

pipelines/skylab/optimus/Optimus.wdl

@@ -64,18 +64,23 @@ workflow Optimus {
     # Set to Forward, Reverse, or Unstranded to account for stranded library preparations (per STARsolo documentation)
     String star_strand_mode = "Forward"
 
-# Set to true to count reads aligned to exonic regions in sn_rna mode
+    # Set to true to count reads aligned to exonic regions in sn_rna mode
     Boolean count_exons = false
 
+    # Set starsolo disk size as adjustable parameter
+    # Int disk_starsolo
+
     # this pipeline does not set any preemptible varibles and only relies on the task-level preemptible settings
     # you could override the tasklevel preemptible settings by passing it as one of the workflows inputs
     # for example: `"Optimus.StarAlign.preemptible": 3` will let the StarAlign task, which by default disables the
     # usage of preemptible machines, attempt to request for preemptible instance up to 3 times. 
+
+    # Set to true if slide-tags calls optimus, other wise set to false
+    Boolean is_slidetags = false
   }
 
   # version of this pipeline
-  String pipeline_version = "7.9.0"
-
+  String pipeline_version = "7.9.1"
 
   # this is used to scatter matched [r1_fastq, r2_fastq, i1_fastq] arrays
   Array[Int] indices = range(length(r1_fastq))
@@ -98,7 +103,6 @@ workflow Optimus {
   String star_merge_docker = "star-merge-npz:1.3.0"
   String samtools_star = "samtools-star:1.0.0-1.11-2.7.11a-1731516196"
 
-
   #TODO how do we handle these?
   String alpine_docker = "alpine-bash@sha256:965a718a07c700a5204c77e391961edee37477634ce2f9cf652a8e4c2db858ff"
   String gcp_alpine_docker_prefix = "bashell/"
@@ -187,7 +191,8 @@ workflow Optimus {
         count_exons = count_exons,
         output_bam_basename = output_bam_basename + "_" + idx,
         soloMultiMappers = soloMultiMappers,
-        samtools_star_docker_path = docker_prefix + samtools_star
+        samtools_star_docker_path = docker_prefix + samtools_star,
+        is_slidetags = is_slidetags
     }
   }
   call Merge.MergeSortBamFiles as MergeBam {
@@ -356,6 +361,8 @@ workflow Optimus {
     File? aligner_metrics = MergeStarOutputs.cell_reads_out
     File library_metrics = final_library_metrics
     File? mtx_files = MergeStarOutputs.mtx_files
+    File? filtered_mtx_files = MergeStarOutputs.filtered_mtx_files
+
     Array[File?] multimappers_EM_matrix = STARsoloFastq.multimappers_EM_matrix
     Array[File?] multimappers_Uniform_matrix = STARsoloFastq.multimappers_Uniform_matrix
     Array[File?] multimappers_Rescue_matrix = STARsoloFastq.multimappers_Rescue_matrix
@@ -375,4 +382,4 @@ workflow Optimus {
     String? output_directory = CellBender.output_dir
     File? summary_pdf = CellBender.pdf
   }
-}
+}

pipelines/skylab/paired_tag/PairedTag.changelog.md

@@ -1,3 +1,8 @@
+# 1.9.1
+2024-12-12 (Date of Last Commit)
+
+* Added a boolean variable is_slidetags. Set to true if Slide-Tags pipeline calling Optimus, otherwise false. 
+
 # 1.9.0
 2024-12-05 (Date of Last Commit)
 

pipelines/skylab/paired_tag/PairedTag.wdl

@@ -8,7 +8,7 @@ import "../../../tasks/broad/Utilities.wdl" as utils
 
 workflow PairedTag {
 
-    String pipeline_version = "1.9.0"
+    String pipeline_version = "1.9.1"
 
 
     input {

pipelines/skylab/slideseq/SlideSeq.changelog.md

@@ -1,3 +1,8 @@
+# 3.4.8
+2024-12-12 (Date of Last Commit)
+
+*  Added a boolean variable is_slidetags; this does not affect the outputs of the pipeline
+
 # 3.4.7
 2024-12-3 (Date of Last Commit)
 

pipelines/skylab/slideseq/SlideSeq.wdl

@@ -25,7 +25,7 @@ import "../../../tasks/broad/Utilities.wdl" as utils
 
 workflow SlideSeq {
 
-    String pipeline_version = "3.4.7"
+    String pipeline_version = "3.4.8"
 
     input {
         Array[File] r1_fastq

pipelines/skylab/smartseq2_single_nucleus_multisample/MultiSampleSmartSeq2SingleNucleus.changelog.md

@@ -1,3 +1,8 @@
+# 2.0.7
+2024-12-12 (Date of Last Commit)
+
+*  Added a boolean variable is_slidetags; this does not affect the outputs of the pipeline
+
 # 2.0.6
 2024-11-15 (Date of Last Commit)
 

pipelines/skylab/smartseq2_single_nucleus_multisample/MultiSampleSmartSeq2SingleNucleus.wdl

@@ -57,7 +57,7 @@ workflow MultiSampleSmartSeq2SingleNucleus {
   }
 
   # Version of this pipeline
-  String pipeline_version = "2.0.6"
+  String pipeline_version = "2.0.7"
 
   if (false) {
      String? none = "None"

tasks/skylab/StarAlign.wdl

@@ -229,10 +229,16 @@ task STARsoloFastq {
     String samtools_star_docker_path
     Int machine_mem_mb = 64000
     Int cpu = 8
-    # multiply input size by 2.2 to account for output bam file + 20% overhead, add size of reference.
-    Int disk = ceil((size(tar_star_reference, "Gi") * 3)) + ceil(size(r1_fastq, "Gi") * 20) +  ceil(size(r2_fastq, "Gi") * 20)
-    # by default request non preemptible machine to make sure the slow star alignment step completes
+   # by default request non preemptible machine to make sure the slow star alignment step completes
     Int preemptible = 3
+
+    # if slide_tags true set disk to 1000 otherwise dynamic allocation based on input size
+    # dynamic allocation multiplies input size by 2.2 to account for output bam file + 20% overhead, add size of reference.
+    Boolean is_slidetags
+    Int disk = if is_slidetags then 1000 else 
+    ceil(size(tar_star_reference, "Gi") * 3) + 
+    ceil(size(r1_fastq, "Gi") * 20) + 
+    ceil(size(r2_fastq, "Gi") * 20)
   }
 
   meta {
@@ -252,7 +258,8 @@ task STARsoloFastq {
   }
 
   command <<<
-       set -e
+    set -euo pipefail
+    set -x
 
     UMILen=10
     CBLen=16
@@ -490,7 +497,9 @@ task MergeStarOutput {
   }
 
   command <<<
-    set -e
+    set -euo pipefail
+    set -x 
+
     declare -a barcodes_files=(~{sep=' ' barcodes})
     declare -a features_files=(~{sep=' ' features})
     declare -a matrix_files=(~{sep=' ' matrix})
@@ -522,11 +531,8 @@ task MergeStarOutput {
     #list files
     echo "listing files"
     ls
-
-
-
-    if [ -f "${cell_reads_files[0]}" ]; then
-
+    # if theres a file in cell_reads_files --  check if non empty
+    if [ -n "${cell_reads_files[*]}" ]; then
       # Destination file for cell reads
       dest="~{input_id}_cell_reads.txt"
 
@@ -609,6 +615,11 @@ task MergeStarOutput {
         --features ${features_files[@]} \
         --matrix ${matrix_files[@]} \
         --input_id ~{input_id}
+
+    # tar up filtered matrix outputbarcodes.tsv, outputfeatures.tsv, outputmatrix.mtx
+    echo "Tarring up filtered matrix files"
+    tar -cvf ~{input_id}_filtered_mtx_files.tar outputbarcodes.tsv outputfeatures.tsv outputmatrix.mtx
+    echo "Done"
   >>>
 
   runtime {
@@ -627,6 +638,7 @@ task MergeStarOutput {
     File? cell_reads_out = "~{input_id}.star_metrics.tar"
     File? library_metrics="~{input_id}_library_metrics.csv"
     File? mtx_files ="~{input_id}.mtx_files.tar"
+    File? filtered_mtx_files = "~{input_id}_filtered_mtx_files.tar"
     File? outputbarcodes = "outputbarcodes.tsv"
   }
 }
@@ -791,4 +803,4 @@ task STARGenomeRefVersion {
     disk: disk + " GB" # TES
     cpu:"1"
   }
-}
+}

verification/test-wdls/scripts/requirements.txt

@@ -1 +1 @@
-jinja2==3.1.4
+jinja2==3.1.5

website/docs/Pipelines/PairedTag_Pipeline/README.md

@@ -2,13 +2,13 @@
 sidebar_position: 1
 slug: /Pipelines/PairedTag_Pipeline/README
 ---
-
 # Paired-Tag Overview
 
 |                          Pipeline Version                           | Date Updated | Documentation Author | Questions or Feedback |
 |:---:| :---: | :---: | :---: |
 | [PairedTag_v1.8.2](https://github.com/broadinstitute/warp/releases) | November, 2024 | WARP Pipelines | Please [file an issue in WARP](https://github.com/broadinstitute/warp/issues). |
 
+![pairedtag_diagram](pairedtag_diagram.png)
 
 ## Introduction to the Paired-Tag workflow
 

website/docs/Pipelines/snM3C/snm3c.methods.md

@@ -1,3 +1,6 @@
+---
+sidebar_position: 3
+---
 # snM3C v4.0.1 Methods
 
 # Methods