Merge pull request #725 from wtsi-npg/devel

pull from devel to master to create release 94.0.0
wtsi-npg · Jun 2, 2023 · 686b2a6 · 686b2a6
2 parents 9d1fcfd + 198d5a0
commit 686b2a6
Show file tree

Hide file tree

Showing 12 changed files with 293 additions and 461 deletions.
diff --git a/Changes b/Changes
@@ -1,5 +1,20 @@
 LIST OF CHANGES
 
+release 94.0.0
+ - Added checks for on-board analysis in progress
+ - Improved logging for a long wait case
+ - Simplified run complete and mirrored detection
+ - Simplified the staging area daemon.
+   Bring the code in accordance with the current practice:
+   - consider only one staging area,
+   - narrow down the glob pattern for areas below the staging
+     area to 'IL*'.
+ - Removed roles which defined attributes that were used
+   in one class only.
+ - Changed the way the connection to the database is established
+   by the staging monitor. Set up the connection explicitly in
+   the script rather than inside one of the classess.
+
 release 93.0.0
  - Introduced a new instrument format database record for NovaSeqX,
    updated test data and one of the tests. 

diff --git a/MANIFEST b/MANIFEST
@@ -119,8 +119,6 @@ htdocs/js/wtsi/prototype-1.6.0.3.js
 htdocs/js/wtsi/sorttable.js
 htdocs/js/wtsi/zebra.js
 INSTALL
-lib/Monitor/Roles/Schema.pm
-lib/Monitor/Roles/Username.pm
 lib/Monitor/RunFolder.pm
 lib/Monitor/RunFolder/Staging.pm
 lib/Monitor/Staging.pm

diff --git a/bin/staging_area_monitor b/bin/staging_area_monitor
@@ -11,6 +11,7 @@ use Readonly;
 use Try::Tiny;
 use List::MoreUtils qw(any);
 
+use npg_tracking::Schema;
 use Monitor::Staging;
 use Monitor::RunFolder::Staging;
 
@@ -49,6 +50,9 @@ sub check_path { ##### Function to deal with one run folder
     my $id_run = $folder->tracking_run()->id_run;
 
     _log("Run $id_run status $run_status");
+    if ($folder->platform_is_NovaSeqX()) {
+        _log("... run on NovaSeqX instrument");
+    }
 
     my $db_flowcell_barcode      = $folder->tracking_run()->flowcell_id;
     my $staging_flowcell_barcode = $folder->run_flowcell; # from RunParameters.xml
@@ -134,8 +138,7 @@ sub check_path { ##### Function to deal with one run folder
             $folder->update_run_record(); # Inspect and update cycles,
                                           # including expected cycle count!
 
-            if ( $folder->mirroring_complete($run_path) &&
-                 $folder->check_tiles($run_path) ) {
+            if ( $folder->check_tiles($run_path) ) {
 
                 my $previous_size = $previous_size_of->{$run_path};
                 my ( $current_size, $latest_mod ) = $folder->monitor_stats();
@@ -154,6 +157,12 @@ sub check_path { ##### Function to deal with one run folder
                     return;
                 }
 
+                # Check if we are waiting for the onboard analysis to finish.
+                if ($folder->is_onboard_analysis_planned() &&
+                    !$folder->has_onboard_analysis_finished()) {
+                    return;
+                }
+
                 # Set status to 'run mirrored' and move run folder
                 # from /incoming/ to /analysis/
                 $folder->update_run_status('run mirrored');
@@ -171,14 +180,14 @@ sub check_path { ##### Function to deal with one run folder
 } ###### End of function to deal with one run folder
 
 ###########################################################
-###############   Main daemon loop  #######################
+###############   Main daemon loop.  ######################
 ###########################################################
 
 sub main {
 
-    my $monitor   = Monitor::Staging->new();
-    my @area_list = $monitor->validate_areas(@ARGV);
-    @area_list or croak 'No valid arguments';
+    my $schema  = npg_tracking::Schema->connect();
+    my $monitor = Monitor::Staging->new(schema => $schema);
+    my $staging_area = $monitor->validate_areas(@ARGV);
 
     my $time = sub {
         return strftime '%a %b %e %T %Y%n', localtime;
@@ -190,21 +199,19 @@ sub main {
 
         _log_noindent($time->());
 
-        foreach my $area (@area_list) {
-            foreach my $run_path ( $monitor->find_live($area) ) {
-                _log_noindent("Considering $run_path");
-                my $done;
-                try {
-                    $done = check_path($run_path, $monitor->schema, $previous_size_of);
-                } finally {
-                    if (@_) {
-                        _log("ERROR: Execution for $run_path died with: @_\n");
-                    } else {
-                        _log("Succesfully processed $run_path");
-                    }
-                };
-                $done and delete $previous_size_of->{$run_path};
-            }
+        foreach my $run_path ( $monitor->find_live($staging_area) ) {
+            _log_noindent("Considering $run_path");
+            my $done;
+            try {
+                $done = check_path($run_path, $schema, $previous_size_of);
+            } finally {
+                if (@_) {
+                    _log("ERROR: Execution for $run_path died with: @_\n");
+                } else {
+                    _log("Succesfully processed $run_path");
+                }
+            };
+            $done and delete $previous_size_of->{$run_path};
         }
 
         _log_noindent("Gone to sleep for $SLEEP_INTERVAL seconds at " . $time->());
@@ -224,11 +231,14 @@ staging_area_monitor
 
 =head1 SYNOPSIS
 
-    staging_area_monitor /nfs/sf10 [/nfs/sf8] ...
+  # Assuming staging_area_path is an existing path
+
+  staging_area_monitor staging_area_path 
 
 =head1 DESCRIPTION
 
-Monitor one or more staging areas supplied as command-line arguments.
+Monitors one staging area. The staging area path is supplied as a
+command-line argument.
 
 The script runs an infinite loop so do not call it via a crontab.
 

diff --git a/lib/Monitor/Roles/Schema.pm b/lib/Monitor/Roles/Schema.pm
diff --git a/lib/Monitor/Roles/Username.pm b/lib/Monitor/Roles/Username.pm
diff --git a/lib/Monitor/RunFolder.pm b/lib/Monitor/RunFolder.pm
@@ -2,16 +2,18 @@ package Monitor::RunFolder;
 
 use Moose;
 use Carp;
+use Readonly;
 
 extends 'npg_tracking::illumina::runfolder';
-with qw/ Monitor::Roles::Username /;
 
 our $VERSION = '0';
 
+Readonly::Scalar my $USERNAME => 'pipeline';
+
 sub update_run_status {
   my ($self, $status_description) = @_;
   $self->tracking_run()
-       ->update_run_status($status_description, $self->username());
+       ->update_run_status($status_description, $USERNAME);
   return;
 }
 
@@ -36,7 +38,7 @@ sub set_instrument_side {
     my $db_iside = $self->tracking_run()->instrument_side || q[];
     if ($db_iside ne $li_iside) {
       my $is_set = $self->tracking_run()
-                        ->set_instrument_side($li_iside, $self->username());
+                        ->set_instrument_side($li_iside, $USERNAME);
       if ($is_set) {
         return $li_iside;
       }
@@ -52,7 +54,7 @@ sub set_workflow_type {
     my $db_wftype = $self->tracking_run()->workflow_type || q[];
     if ($db_wftype ne $li_wftype) {
       my $is_set = $self->tracking_run()
-                        ->set_workflow_type($li_wftype, $self->username());
+                        ->set_workflow_type($li_wftype, $USERNAME);
       if ($is_set) {
         return $li_wftype;
       }
@@ -65,11 +67,11 @@ sub set_run_tags {
   my $self = shift;
 
   $self->is_paired_read()
-    ? $self->tracking_run()->set_tag( $self->username, 'paired_read' )
-    : $self->tracking_run()->set_tag( $self->username, 'single_read' );
+    ? $self->tracking_run()->set_tag( $USERNAME, 'paired_read' )
+    : $self->tracking_run()->set_tag( $USERNAME, 'single_read' );
 
   $self->is_indexed()
-    ? $self->tracking_run()->set_tag( $self->username, 'multiplex' )
+    ? $self->tracking_run()->set_tag( $USERNAME, 'multiplex' )
     : $self->tracking_run()->unset_tag( 'multiplex' );
 
   return;
@@ -203,14 +205,14 @@ finding the run folder. Updates extected cycle count value if needed.
 
 =item Carp
 
+=item Readonly
+
 =back
 
 =head1 INCOMPATIBILITIES
 
 =head1 BUGS AND LIMITATIONS
 
-Please inform the author of any found.
-
 =head1 AUTHOR
 
 =over