Merge pull request #736 from wtsi-npg/devel

pull from devel to master to create release 95.0.0

Build.PL

@@ -129,6 +129,7 @@ my $builder = $class->new(
                 'Config::Auto'                    => 0,
                 'Crypt::CBC'                      => 0,
                 'Cwd'                             => 0,
+                'Data::UUID'                      => 0,
                 'Date::Parse'                     => '2.27',
                 'DateTime'                        => '0.5',
                 'DateTime::TimeZone'              => 0,
@@ -208,6 +209,7 @@ my $builder = $class->new(
                 'SQL::Translator'                 => '0.11006',
                 'Sys::Hostname'                   => '1.11',
                 'Template'                        => '2.19',
+                'Text::CSV'                       => '0',
                 'Try::Tiny'                       => 0,
                 'URI::Escape'                     => 0,
                 'utf8'                            => 0,
@@ -218,6 +220,7 @@ my $builder = $class->new(
 
                 'WTSI::DNAP::Utilities::Loggable'  => '>= 0.5.3',
                 'WTSI::DNAP::Utilities::Timestamp' => '0',
+                'WTSI::DNAP::Warehouse::Schema'    => '0',
             },
          );
 

Changes

@@ -1,5 +1,11 @@
 LIST OF CHANGES
 
+release 95.0.0
+ - Remove singularity recipe and related files
+ - Import mlwh drivers
+ - Reimplemented waiting for the onboard analysis
+ - Added a samplesheet generator for NovaSeqX
+
 release 94.0.1
  - Fixed bugs in the staging area daemon script:
    - an incorrect method name

MANIFEST

@@ -163,6 +163,7 @@ lib/npg_tracking/report/event2followers.pm
 lib/npg_tracking/report/event2subscribers.pm
 lib/npg_tracking/report/events.pm
 lib/npg/samplesheet.pm
+lib/npg/samplesheet/novaseq_xseries.pm
 lib/npg/util.pm
 lib/npg/util/mailer.pm
 lib/npg/view.pm
@@ -276,6 +277,10 @@ lib/st/api/base.pm
 lib/st/api/batch.pm
 lib/st/api/lims.pm
 lib/st/api/lims/samplesheet.pm
+lib/st/api/lims/ml_warehouse/driver.pm
+lib/st/api/lims/ml_warehouse.pm
+lib/st/api/lims/ml_warehouse_auto.pm 
+lib/st/api/lims/ml_warehouse_fc_cache.pm
 lib/st/api/lims/xml.pm
 lib/st/api/project.pm
 lib/st/api/sample.pm
@@ -403,13 +408,15 @@ t/40-st-base.t
 t/40-st-batch.t
 t/40-st-lims-insert_size.t
 t/40-st-lims-mlwarehouse.t
+t/40-st-lims-ml_warehouse-drivers.t
 t/40-st-lims-samplesheet.t
 t/40-st-lims.t
 t/40-st-project.t
 t/40-st-sample.t
 t/40-st-study.t
 t/45-st-api-lims-traversal.t
 t/47-samplesheet.t
+t/47-npg_samplesheet_novaseq_xseries.t
 t/50-controller.t
 t/50-decorator.t
 t/60-illumina-runfolder.t
@@ -482,6 +489,11 @@ t/data/fixtures_lims_wh/100-IseqRunLaneMetric.yml
 t/data/fixtures_lims_wh/100-IseqRunStatus.yml
 t/data/fixtures_lims_wh/100-StudyUser.yml
 t/data/fixtures_lims_wh/200-IseqProductMetric.yml
+t/data/fixtures_stlims_wh/000-Sample.yml
+t/data/fixtures_stlims_wh/000-Study.yml
+t/data/fixtures_stlims_wh/100-IseqFlowcell.yml
+t/data/fixtures_stlims_wh/100-StudyUser.yml
+t/data/fixtures_stlims_wh/200-IseqProductMetric.yml
 t/data/gaii/staging/IL3/incoming/100622_IL3_01234/this_run_is_not_complete_yet
 t/data/gaii/staging/IL5/incoming/not_a_dir
 t/data/gaii/staging/IL999/incoming/100622_IL3_01234/Recipe_GA2-PEM_2x76Cycle_v7.7.xml

bin/npg_samplesheet_generator_NovaSeqXSeries

@@ -0,0 +1,118 @@
+#!/usr/bin/env perl
+
+use strict;
+use warnings;
+use FindBin qw($Bin);
+use lib ( -d "$Bin/../lib/perl5" ? "$Bin/../lib/perl5" : "$Bin/../lib" );
+
+use npg::samplesheet::novaseq_xseries;
+
+our $VERSION = '0';
+
+npg::samplesheet::novaseq_xseries->new_with_options()->process();
+
+1;
+
+=head1 NAME
+
+npg_samplesheet_generator_NovaSeqXSeries
+
+=head1 USAGE
+
+  npg_samplesheet_generator_NovaSeqXSeries [options]
+
+=head1 REQUIRED ARGUMENTS
+
+=head1 OPTIONS
+
+  C<help>     - displays help message and exists
+
+  C<varcall>  - variant calling mode, defaults to C<None>, other valid options
+                C<SmallVariantCaller> and C<AllVariantCallers>
+
+  C<batch_id> - LIMS batch identifier, optional
+
+  C<id_run>   - NPG run ID, optional; if supplied, the record for this run
+                should exists in the run tracking database
+  C<align>    - a boolean option, false by default; if set, the DRAGEN
+                germline analysis section is added to the file if suitable
+                samples are present
+  C<keep_fastq> - an option to keep FASTQ files for aligned data, false
+                  default
+  C<dragen_max_number_of_configs> - DRAGEN analysis can deal with a limited
+                  number of distinct configurations. Set this attribute if
+                  processing not on-board.
+
+=head1 DESCRIPTION
+
+Samplesheet generation to initiate DRAGEN analysis of data sequenced
+on the NovaSeq Series X Illumina instrument.
+
+=head1 EXIT STATUS
+
+0
+
+=head1 DIAGNOSTICS
+
+=head1 CONFIGURATION
+
+Access to the ml warehouse database is required.
+
+=head1 DEPENDENCIES
+
+=over
+
+=item strict
+
+=item warnings
+
+=item Carp
+
+=item Text::CSV
+
+=item Readonly
+
+=item List::MoreUtils
+
+=item List::Util
+
+=item Getopt::Long
+
+=item Pod::Usage
+
+=item DateTime
+
+=item Data::UUID
+
+=item st::api::lims
+
+=item npg_tracking::Schema
+
+=back
+
+=head1 BUGS AND LIMITATIONS
+
+=head1 INCOMPATIBILITIES
+
+=head1 AUTHOR
+
+Marina Gourtovaia E<lt>[email protected]<gt>
+
+=head1 LICENSE AND COPYRIGHT
+
+Copyright (C) 2023 Genome Research Ltd
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+=cut

bin/staging_area_monitor

@@ -160,7 +160,7 @@ sub check_path { ##### Function to deal with one run folder
 
                 # Check if we are waiting for the onboard analysis to finish.
                 if ($folder->is_onboard_analysis_planned() &&
-                    !$folder->has_onboard_analysis_finished()) {
+                    !$folder->is_onboard_analysis_output_copied()) {
                     return;
                 }
 

lib/Monitor/RunFolder/Staging.pm

@@ -30,6 +30,13 @@ Readonly::Scalar my $MODE_INDEX           => 2;
 
 Readonly::Scalar my $RTA_COMPLETE_FN      => q[RTAComplete.txt];
 Readonly::Scalar my $COPY_COMPLETE_FN     => q[CopyComplete.txt];
+Readonly::Scalar my $DEFAULT_ONBOARD_ANALYSIS_DN => q[1];
+# The file with the name below flags the completion of a particular
+# DRAGEN analysys. However, it's the CopyComplete.txt file in the particular
+# analysis directory (/Analysis/1/, /Analysis/2/, etc.) that flags the
+# completion of the analysis output to staging. In future we might track
+# the DRAGEN analysis timeline, so keep this variable, though it is not
+# used at the moment.
 Readonly::Scalar my $ONBOARD_ANALYSIS_COMPLETE_FN =>
                                              q[Secondary_Analysis_Complete.txt];
 Readonly::Scalar my $ONBOARD_ANALYSIS_SAMPLESHEET_FN => q[SampleSheet.csv]; 
@@ -107,13 +114,15 @@ sub is_onboard_analysis_planned {
     return $planned;
 }
 
-sub has_onboard_analysis_finished {
+sub is_onboard_analysis_output_copied {
+
     my $self = shift;
 
     my $analysis_dir = $self->dragen_analysis_path();
     my $found = 0;
     if (-d $analysis_dir) {
-        my $file = join q[/], $analysis_dir, $ONBOARD_ANALYSIS_COMPLETE_FN;
+        my $file = join q[/], $analysis_dir,
+                              $DEFAULT_ONBOARD_ANALYSIS_DN, $COPY_COMPLETE_FN;
         $found = -f $file;
         carp sprintf '%s is%sfound', $file, $found ? q[ ] : q[ not ];         
     } else {
@@ -415,10 +424,11 @@ Returns true if the run folder for the NovaSeq SeriesX run contains a
 samplesheet with a section for the bcl on board data conversion. Always returns
 false for other instrument types.
 
-=head2 has_onboard_analysis_finished
+=head2 is_onboard_analysis_output_copied
 
-Returns true if the file that inticates that the onboard analysis has finished
-is present in the run folder.
+Returns true if the file that flags the end of the transfer of the onboard
+analysis output to staging is present in the default DRAGEN analysis directory
+([RUNFOLDER_NAME]/Analysis/1/ at the moment).
 
 =head1 CONFIGURATION AND ENVIRONMENT