Merge pull request #642 from AlexsLemonade/allyhawkins/0.7.1-release

Sync changes for v0.7.1 release to `main`

bin/add_celltypes_to_sce.R

@@ -42,6 +42,12 @@ option_list <- list(
     type = "character",
     help = "Name of marker by cell type reference file used with CellAssign.
             File name is expected to be in form: `<ref_name>_<source>_<version>.tsv`"
+  ),
+  make_option(
+    opt_str = c("--celltype_ref_metafile"),
+    type = "character",
+    help = "Metadata TSV file containing cell type reference metadata.
+            This file is used to obtain a list of organs used for CellAssign annotation and is only required if CellAssign results provided as input."
   )
 )
 
@@ -177,6 +183,10 @@ if (!is.null(opt$cellassign_predictions)) {
     stop("CellAssign reference filename must be provided")
   }
 
+  if (is.null(opt$celltype_ref_metafile)) {
+    stop("Cell type reference metadata filename must be provided")
+  }
+
   predictions <- readr::read_tsv(opt$cellassign_predictions)
 
   # get cell type with maximum prediction value for each cell
@@ -216,6 +226,17 @@ if (!is.null(opt$cellassign_predictions)) {
   # note that if `metadata(sce)$celltype_methods` doesn't exist yet, this will
   #  come out to just the string "cellassign"
   metadata(sce)$celltype_methods <- c(metadata(sce)$celltype_methods, "cellassign")
+
+  # add cellassign reference organs to metadata
+  cellassign_organs <- opt$celltype_ref_metafile |>
+    readr::read_tsv() |>
+    dplyr::filter(celltype_ref_name == cellassign_ref_info[["ref_name"]]) |>
+    dplyr::pull(organs)
+
+  if (cellassign_organs == "" | is.na(cellassign_organs)) {
+    stop("Failed to obtain CellAssign reference organ list.")
+  }
+  metadata(sce)$cellassign_reference_organs <- cellassign_organs
 }
 
 # export annotated object with cellassign assignments

external-instructions.md

@@ -78,12 +78,12 @@ Using the above command will run the workflow from the `main` branch of the work
 To update to the latest released version you can run `nextflow pull AlexsLemonade/scpca-nf` before the `nextflow run` command.
 
 To be sure that you are using a consistent version, you can specify use of a release tagged version of the workflow, set below with the `-r` flag.
-The command below will pull the `scpca-nf` workflow directly from Github using the `v0.7.0` version.
+The command below will pull the `scpca-nf` workflow directly from Github using the `v0.7.1` version.
 Released versions can be found on the [`scpca-nf` repository releases page](https://github.com/AlexsLemonade/scpca-nf/releases).
 
 ```sh
 nextflow run AlexsLemonade/scpca-nf \
-  -r v0.7.0 \
+  -r v0.7.1 \
   -config <path to config file>  \
   -profile <name of profile>
 ```
@@ -312,7 +312,7 @@ If you will be analyzing spatial expression data, you will also need the Cell Ra
 
 If your compute nodes do not have internet access, you will likely have to pre-pull the required container images as well.
 When doing this, it is important to be sure that you also specify the revision (version tag) of the `scpca-nf` workflow that you are using.
-For example, if you would run `nextflow run AlexsLemonade/scpca-nf -r v0.7.0`, then you will want to set `-r v0.7.0` for `get_refs.py` as well to be sure you have the correct containers.
+For example, if you would run `nextflow run AlexsLemonade/scpca-nf -r v0.7.1`, then you will want to set `-r v0.7.1` for `get_refs.py` as well to be sure you have the correct containers.
 By default, `get_refs.py` will download files and images associated with the latest release.
 
 If your system uses Docker, you can add the `--docker` flag:

internal-instructions.md

@@ -82,7 +82,7 @@ Please refer to our [`CONTRIBUTING.md`](CONTRIBUTING.md#stub-workflows) for more
 When running the workflow for a project or group of samples that is ready to be released on ScPCA portal, please use the tag for the latest release:
 
 ```
-nextflow run AlexsLemonade/scpca-nf -r v0.7.0 -profile ccdl,batch --project SCPCP000000
+nextflow run AlexsLemonade/scpca-nf -r v0.7.1 -profile ccdl,batch --project SCPCP000000
 ```
 
 ### Processing example data

lib/Utils.groovy

@@ -1,6 +1,8 @@
 
 import groovy.json.JsonSlurper
 import groovy.json.JsonOutput
+import nextflow.script.WorkflowMetadata
+import nextflow.NextflowMeta
 
 /**
  * Utility functions for scpca-nf
@@ -24,10 +26,12 @@ class Utils {
    * Write a metadata map to a JSON string
    *
    * @param meta A metadata map
+   * @param updates A map of updates to the metadata
    * @return A JSON string
    */
-  static def makeJson(meta) {
-    def meta_json = JsonOutput.toJson(meta)
+  static def makeJson(meta, updates = [:]) {
+    def meta_out = meta + updates // merge/overwrite the metadata with any updates
+    def meta_json = JsonOutput.toJson(meta_out)
     meta_json = JsonOutput.prettyPrint(meta_json)
     return(meta_json)
   }
@@ -64,4 +68,19 @@ class Utils {
       ''
     }
   }
+
+
+  /**
+   * Make a map of versions for the workflow and nextflow, for adding to metadata
+   *
+   * @param workflow A nextflow WorkflowMetadata object containing the revision and manifest
+   * @param nextflow A NextflowMeta object (created by the nextflow runtime)
+   * @return A map with keys "scpca_version" and "nextflow_version"
+   */
+  static def getVersions(WorkflowMetadata workflow, NextflowMeta nextflow){
+    [
+      scpca_version : workflow.revision ?: workflow.manifest.version,
+      nextflow_version : nextflow.version.toString()
+    ]
+  }
 }

main.nf

@@ -63,6 +63,18 @@ if (!resolution_strategies.contains(params.af_resolution)) {
   param_error = true
 }
 
+// cell type annotation file checks
+if (params.perform_celltyping) {
+  if (!file(params.project_celltype_metafile).exists()) {
+    log.error("The 'project_celltype_metafile' file '${params.project_celltype_metafile}' can not be found.")
+    param_error = true
+  }
+  if (!file(params.celltype_ref_metadata).exists()) {
+    log.error("The 'celltype_ref_metadata' file '${params.celltype_ref_metadata}' can not be found.")
+    param_error = true
+  }
+}
+
 if(param_error){
   System.exit(1)
 }
@@ -118,7 +130,7 @@ workflow {
         t2g_bulk_path: params.ref_rootdir + "/" + sample_refs["t2g_bulk_path"],
         cellranger_index: params.ref_rootdir + "/" + sample_refs["cellranger_index"],
         star_index: params.ref_rootdir + "/" + sample_refs["star_index"],
-        scpca_version: workflow.manifest.version,
+        scpca_version: workflow.revision ?: workflow.manifest.version,
         nextflow_version: nextflow.version.toString()
       ]
     }

modules/af-features.nf

@@ -53,6 +53,8 @@ process alevin_feature{
     tech_version = meta.technology.split('_').last()
     umi_geom = umi_geom_map[tech_version]
     // get meta to write as file
+    // make sure workflow version strings are correct
+    meta += Utils.getVersions(workflow, nextflow)
     meta_json = Utils.makeJson(meta)
     """
     mkdir -p ${run_dir}
@@ -74,6 +76,7 @@ process alevin_feature{
     """
   stub:
     run_dir = "${meta.run_id}-features"
+    meta += Utils.getVersions(workflow, nextflow)
     meta_json = Utils.makeJson(meta)
     """
     mkdir -p ${run_dir}
@@ -95,6 +98,7 @@ process fry_quant_feature{
   script:
     quant_dir = rad_dir + '_quant'
     // get meta to write as file
+    meta += Utils.getVersions(workflow, nextflow)
     meta_json = Utils.makeJson(meta)
     """
     alevin-fry generate-permit-list \
@@ -124,6 +128,7 @@ process fry_quant_feature{
     """
   stub:
     quant_dir = rad_dir + '_quant'
+    meta += Utils.getVersions(workflow, nextflow)
     meta_json = Utils.makeJson(meta)
     """
     mkdir -p '${quant_dir}'

modules/af-rna.nf

@@ -23,6 +23,7 @@ process alevin_rad{
       '10Xv3.1': '--chromiumV3'
     ]
     // get meta to write as file
+    meta += Utils.getVersions(workflow, nextflow)
     meta_json = Utils.makeJson(meta)
     // run alevin like normal with the --rad flag
     // creates output directory with RAD file needed for alevin-fry
@@ -43,6 +44,7 @@ process alevin_rad{
     """
   stub:
     rad_dir = file(meta.rad_dir).name
+    meta += Utils.getVersions(workflow, nextflow)
     meta_json = Utils.makeJson(meta)
     """
     mkdir -p ${rad_dir}
@@ -66,6 +68,7 @@ process fry_quant_rna{
   script:
     quant_dir = rad_dir + '_quant'
     // get meta to write as file
+    meta += Utils.getVersions(workflow, nextflow)
     meta_json = Utils.makeJson(meta)
     """
     alevin-fry generate-permit-list \
@@ -95,6 +98,7 @@ process fry_quant_rna{
     """
   stub:
     quant_dir = rad_dir + '_quant'
+    meta += Utils.getVersions(workflow, nextflow)
     meta_json = Utils.makeJson(meta)
     """
     mkdir -p '${quant_dir}'

modules/bulk-salmon.nf

@@ -39,6 +39,7 @@ process salmon{
   script:
     salmon_dir = file(meta.salmon_results_dir).name
     // get meta to write as file
+    meta += Utils.getVersions(workflow, nextflow)
     meta_json = Utils.makeJson(meta)
     """
     salmon quant -i ${index} \
@@ -56,6 +57,7 @@ process salmon{
     """
   stub:
     salmon_dir = file(meta.salmon_results_dir).name
+    meta += Utils.getVersions(workflow, nextflow)
     meta_json = Utils.makeJson(meta)
     """
     mkdir -p ${salmon_dir}

modules/cellsnp.nf

@@ -51,6 +51,7 @@ process vireo{
     tuple val(meta), path(outdir)
   script:
     outdir = file(meta.vireo_dir).name
+    meta += Utils.getVersions(workflow, nextflow)
     meta_json = Utils.makeJson(meta)
     """
     vireo \
@@ -63,6 +64,7 @@ process vireo{
     """
   stub:
     outdir = file(meta.vireo_dir).name
+    meta += Utils.getVersions(workflow, nextflow)
     meta_json = Utils.makeJson(meta)
     """
     mkdir -p ${outdir}
@@ -99,4 +101,3 @@ workflow cellsnp_vireo {
   emit:
     vireo.out
 }
-

modules/classify-celltypes.nf

@@ -15,6 +15,8 @@ process classify_singler {
     tuple val(meta.library_id), path(singler_dir)
   script:
     singler_dir = file(meta.singler_dir).name
+    meta += Utils.getVersions(workflow, nextflow)
+    meta_json = Utils.makeJson(meta)
     """
     # create output directory
     mkdir "${singler_dir}"
@@ -27,13 +29,15 @@ process classify_singler {
       --threads ${task.cpus}
 
     # write out meta file
-    echo '${Utils.makeJson(meta)}' > "${singler_dir}/scpca-meta.json"
+    echo '${meta_json}' > "${singler_dir}/scpca-meta.json"
     """
   stub:
     singler_dir = file(meta.singler_dir).name
+    meta += Utils.getVersions(workflow, nextflow)
+    meta_json = Utils.makeJson(meta)
     """
     mkdir "${singler_dir}"
-    echo '${Utils.makeJson(meta)}' > "${singler_dir}/scpca-meta.json"
+    echo '${meta_json}' > "${singler_dir}/scpca-meta.json"
     """
 }
 
@@ -54,7 +58,8 @@ process classify_cellassign {
     tuple val(meta.library_id), path(cellassign_dir)
   script:
     cellassign_dir = file(meta.cellassign_dir).name
-
+    meta += Utils.getVersions(workflow, nextflow)
+    meta_json = Utils.makeJson(meta)
     """
     # create output directory
     mkdir "${cellassign_dir}"
@@ -73,13 +78,15 @@ process classify_cellassign {
       --threads ${task.cpus}
 
     # write out meta file
-    echo '${Utils.makeJson(meta)}' > "${cellassign_dir}/scpca-meta.json"
+    echo '${meta_json}' > "${cellassign_dir}/scpca-meta.json"
     """
   stub:
     cellassign_dir = file(meta.cellassign_dir).name
+    meta += Utils.getVersions(workflow, nextflow)
+    meta_json = Utils.makeJson(meta)
     """
     mkdir "${cellassign_dir}"
-    echo '${Utils.makeJson(meta)}' > "${cellassign_dir}/scpca-meta.json"
+    echo '${meta_json}' > "${cellassign_dir}/scpca-meta.json"
     """
 }
 
@@ -90,6 +97,7 @@ process add_celltypes_to_sce {
   tag "${meta.library_id}"
   input:
     tuple val(meta), path(processed_rds), path(singler_dir), path(cellassign_dir)
+    path(celltype_ref_metadata) // TSV file of references metadata needed for CellAssign only
   output:
     tuple val(meta), path(annotated_rds)
   script:
@@ -105,7 +113,8 @@ process add_celltypes_to_sce {
       ${singler_present ? "--singler_results  ${singler_results}" : ''} \
       ${singler_present ? "--singler_model_file ${meta.singler_model_file}" : ''} \
       ${cellassign_present ? "--cellassign_predictions  ${cellassign_predictions}" : ''} \
-      ${cellassign_present ? "--cellassign_ref_file ${meta.cellassign_reference_file}" : ''}
+      ${cellassign_present ? "--cellassign_ref_file ${meta.cellassign_reference_file}" : ''} \
+      ${cellassign_present ? "--celltype_ref_metafile ${celltype_ref_metadata}" : ''}
     """
   stub:
     annotated_rds = "${meta.library_id}_processed_annotated.rds"
@@ -183,7 +192,7 @@ workflow annotate_celltypes {
       .mix(classify_singler.out)
 
     // create cellassign input channel: [meta, processed sce, cellassign reference file]
-      cellassign_input_ch = celltype_input_ch
+    cellassign_input_ch = celltype_input_ch
       // add in cellassign reference
       .map{it.toList() + [file(it[0].cellassign_reference_file ?: empty_file)]}
       // skip if no cellassign reference file or reference name is not defined
@@ -227,7 +236,10 @@ workflow annotate_celltypes {
 
 
     // incorporate annotations into SCE object
-    add_celltypes_to_sce(assignment_input_ch.add_celltypes)
+    add_celltypes_to_sce(
+      assignment_input_ch.add_celltypes,
+      file(params.celltype_ref_metadata) // file with CellAssign reference organs
+    )
 
     // mix in libraries without new celltypes
     // result is [meta, proccessed rds]

modules/spaceranger.nf

@@ -14,6 +14,7 @@ process spaceranger{
     tuple val(meta), path(out_id)
   script:
     out_id = file(meta.spaceranger_results_dir).name
+    meta += Utils.getVersions(workflow, nextflow)
     meta_json = Utils.makeJson(meta)
     """
     spaceranger count \
@@ -35,6 +36,7 @@ process spaceranger{
     """
   stub:
     out_id = file(meta.spaceranger_results_dir).name
+    meta += Utils.getVersions(workflow, nextflow)
     meta_json = Utils.makeJson(meta)
     """
     mkdir -p ${out_id}/outs

nextflow.config

@@ -5,7 +5,7 @@ manifest{
   homePage = 'https://github.com/AlexsLemonade/scpca-nf'
   mainScript = 'main.nf'
   defaultBranch = 'main'
-  version = 'v0.7.0'
+  version = 'v0.7.1'
 }
 
 // global parameters for workflows