From c8dca48ab07374e4f0466b6495d16a243c135334 Mon Sep 17 00:00:00 2001
From: Marina Gourtovaia <mg8@sanger.ac.uk>
Date: Fri, 19 May 2023 19:16:05 +0100
Subject: [PATCH 1/9] Fixed warnings in the test.

---
 lib/Monitor/Roles/Schema.pm   | 88 -----------------------------------
 lib/Monitor/Roles/Username.pm | 88 -----------------------------------
 t/10-bait.t                   |  5 +-
 3 files changed, 2 insertions(+), 179 deletions(-)
 delete mode 100644 lib/Monitor/Roles/Schema.pm
 delete mode 100644 lib/Monitor/Roles/Username.pm

diff --git a/lib/Monitor/Roles/Schema.pm b/lib/Monitor/Roles/Schema.pm
deleted file mode 100644
index 00eea210e..000000000
--- a/lib/Monitor/Roles/Schema.pm
+++ /dev/null
@@ -1,88 +0,0 @@
-#########
-# Author:        jo3
-# Created:       2010-04-28
-
-package Monitor::Roles::Schema;
-
-use Moose::Role;
-use npg_tracking::Schema;
-
-our $VERSION = '0';
-
-has _schema => (
-    reader     => 'schema',
-    is         => 'ro',
-    isa        => 'npg_tracking::Schema',
-    lazy_build => 1,
-);
-
-sub _build__schema {
-    my ($self) = @_;
-    return npg_tracking::Schema->connect();
-}
-
-1;
-
-
-__END__
-
-
-=head1 NAME
-
-Monitor::Roles::Schema - provide the DBIx npg_tracking schema.
-
-=head1 VERSION
-
-=head1 SYNOPSIS
-
-    C<<use Moose;
-       with 'Monitor::Roles::Schema';>>
-
-=head1 DESCRIPTION
-
-Create a DBIx schema.
-
-=head1 SUBROUTINES/METHODS
-
-
-=head1 CONFIGURATION AND ENVIRONMENT
-
-Will decide whether to use live or dev on the basis of, in order, a 'dev'
-constructor attribute, $ENV{dev}, or the default value 'live';
-
-=head1 DEPENDENCIES
-
-=over
-
-=item Moose::Role
-
-=back
-
-=head1 INCOMPATIBILITIES
-
-
-=head1 BUGS AND LIMITATIONS
-
-
-=head1 AUTHOR
-
-John O'Brien, E<lt>jo3@sanger.ac.ukE<gt>
-
-=head1 LICENSE AND COPYRIGHT
-
-Copyright (C) 2010 GRL, by John O'Brien
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-=cut
diff --git a/lib/Monitor/Roles/Username.pm b/lib/Monitor/Roles/Username.pm
deleted file mode 100644
index fa36d07d9..000000000
--- a/lib/Monitor/Roles/Username.pm
+++ /dev/null
@@ -1,88 +0,0 @@
-#########
-# Author:        jo3
-# Created:       2010-04-28
-
-package Monitor::Roles::Username;
-
-use Moose::Role;
-use Readonly;
-
-our $VERSION = '0';
-
-Readonly::Scalar my $USERNAME => 'pipeline';
-
-has _username => (
-    reader     => 'username',
-    is         => 'ro',
-    isa        => 'Str',
-    default    => $USERNAME,
-    documentation => 'The username to attach to database updates',
-);
-
-
-1;
-
-__END__
-
-
-=head1 NAME
-
-Monitor::Roles::Username - username to use when writing run statuses etc to
-the npg tracking database.
-
-=head1 VERSION
-
-
-=head1 SYNOPSIS
-
-    C<<use Moose;
-       with 'Monitor::Roles::Username';>>
-
-=head1 DESCRIPTION
-
-Right now all it does is return a username. Potentionally SSO stuff could be
-added here.
-
-=head1 SUBROUTINES/METHODS
-
-
-=head1 CONFIGURATION AND ENVIRONMENT
-
-=head1 DEPENDENCIES
-
-=over
-
-=item Moose::Role
-
-=item Readonly
-
-=back
-
-=head1 INCOMPATIBILITIES
-
-
-=head1 BUGS AND LIMITATIONS
-
-
-=head1 AUTHOR
-
-John O'Brien, E<lt>jo3@sanger.ac.ukE<gt>
-
-=head1 LICENSE AND COPYRIGHT
-
-Copyright (C) 2010 GRL, by John O'Brien
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-=cut
diff --git a/t/10-bait.t b/t/10-bait.t
index be6810af8..0956fde59 100644
--- a/t/10-bait.t
+++ b/t/10-bait.t
@@ -63,8 +63,8 @@ use_ok('npg_tracking::data::bait');
    'bait path not found where bait name is all white space';
 
   # There is no bait name for this lane
-  my %ref = (id_run => 7753, position => 2, tag_index => 1);
-  my $test =npg_tracking::data::bait->new(repository => $repos, %ref,
+  %ref = (id_run => 7753, position => 2, tag_index => 1);
+  $test =npg_tracking::data::bait->new(repository => $repos, %ref,
     lims => st::api::lims->new(%ref));
   lives_and { is $test->bait_path, undef } 'bait path undefined';
   is($test->bait_intervals_path, undef, 'bait CTR file undefined');
@@ -72,4 +72,3 @@ use_ok('npg_tracking::data::bait');
 }
 
 1;
-

From 2d657858f6fd9b2ce508e05a909b864b1a5535e4 Mon Sep 17 00:00:00 2001
From: Marina Gourtovaia <mg8@sanger.ac.uk>
Date: Fri, 19 May 2023 19:22:49 +0100
Subject: [PATCH 2/9] Removed superfluous Moose roles.

Removed roles which defined attributes that were used
in one class only.
Chaned the way the connection to the database is established
in the staging monitor. Set up the connection explicitly in
the script rather than inside one of the classess.
---
 MANIFEST                         |  2 --
 bin/staging_area_monitor         |  6 ++++--
 lib/Monitor/RunFolder.pm         | 20 ++++++++++---------
 lib/Monitor/RunFolder/Staging.pm |  2 +-
 lib/Monitor/Staging.pm           | 33 ++++++++++++++++----------------
 t/34-monitor-runfolder.t         |  6 +-----
 t/34-monitor-staging.t           |  6 +++---
 7 files changed, 37 insertions(+), 38 deletions(-)

diff --git a/MANIFEST b/MANIFEST
index 176cdb84d..b9ed0878a 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -119,8 +119,6 @@ htdocs/js/wtsi/prototype-1.6.0.3.js
 htdocs/js/wtsi/sorttable.js
 htdocs/js/wtsi/zebra.js
 INSTALL
-lib/Monitor/Roles/Schema.pm
-lib/Monitor/Roles/Username.pm
 lib/Monitor/RunFolder.pm
 lib/Monitor/RunFolder/Staging.pm
 lib/Monitor/Staging.pm
diff --git a/bin/staging_area_monitor b/bin/staging_area_monitor
index 9c780fa59..f19c5ad0a 100755
--- a/bin/staging_area_monitor
+++ b/bin/staging_area_monitor
@@ -11,6 +11,7 @@ use Readonly;
 use Try::Tiny;
 use List::MoreUtils qw(any);
 
+use npg_tracking::Schema;
 use Monitor::Staging;
 use Monitor::RunFolder::Staging;
 
@@ -176,7 +177,8 @@ sub check_path { ##### Function to deal with one run folder
 
 sub main {
 
-    my $monitor   = Monitor::Staging->new();
+    my $schema    = npg_tracking::Schema->connect();
+    my $monitor   = Monitor::Staging->new(schema => $schema);
     my @area_list = $monitor->validate_areas(@ARGV);
     @area_list or croak 'No valid arguments';
 
@@ -195,7 +197,7 @@ sub main {
                 _log_noindent("Considering $run_path");
                 my $done;
                 try {
-                    $done = check_path($run_path, $monitor->schema, $previous_size_of);
+                    $done = check_path($run_path, $schema, $previous_size_of);
                 } finally {
                     if (@_) {
                         _log("ERROR: Execution for $run_path died with: @_\n");
diff --git a/lib/Monitor/RunFolder.pm b/lib/Monitor/RunFolder.pm
index 68e6400d1..5d9dcb258 100644
--- a/lib/Monitor/RunFolder.pm
+++ b/lib/Monitor/RunFolder.pm
@@ -2,16 +2,18 @@ package Monitor::RunFolder;
 
 use Moose;
 use Carp;
+use Readonly;
 
 extends 'npg_tracking::illumina::runfolder';
-with qw/ Monitor::Roles::Username /;
 
 our $VERSION = '0';
 
+Readonly::Scalar my $USERNAME => 'pipeline';
+
 sub update_run_status {
   my ($self, $status_description) = @_;
   $self->tracking_run()
-       ->update_run_status($status_description, $self->username());
+       ->update_run_status($status_description, $USERNAME);
   return;
 }
 
@@ -36,7 +38,7 @@ sub set_instrument_side {
     my $db_iside = $self->tracking_run()->instrument_side || q[];
     if ($db_iside ne $li_iside) {
       my $is_set = $self->tracking_run()
-                        ->set_instrument_side($li_iside, $self->username());
+                        ->set_instrument_side($li_iside, $USERNAME);
       if ($is_set) {
         return $li_iside;
       }
@@ -52,7 +54,7 @@ sub set_workflow_type {
     my $db_wftype = $self->tracking_run()->workflow_type || q[];
     if ($db_wftype ne $li_wftype) {
       my $is_set = $self->tracking_run()
-                        ->set_workflow_type($li_wftype, $self->username());
+                        ->set_workflow_type($li_wftype, $USERNAME);
       if ($is_set) {
         return $li_wftype;
       }
@@ -65,11 +67,11 @@ sub set_run_tags {
   my $self = shift;
 
   $self->is_paired_read()
-    ? $self->tracking_run()->set_tag( $self->username, 'paired_read' )
-    : $self->tracking_run()->set_tag( $self->username, 'single_read' );
+    ? $self->tracking_run()->set_tag( $USERNAME, 'paired_read' )
+    : $self->tracking_run()->set_tag( $USERNAME, 'single_read' );
 
   $self->is_indexed()
-    ? $self->tracking_run()->set_tag( $self->username, 'multiplex' )
+    ? $self->tracking_run()->set_tag( $USERNAME, 'multiplex' )
     : $self->tracking_run()->unset_tag( 'multiplex' );
 
   return;
@@ -203,14 +205,14 @@ finding the run folder. Updates extected cycle count value if needed.
 
 =item Carp
 
+=item Readonly
+
 =back
 
 =head1 INCOMPATIBILITIES
 
 =head1 BUGS AND LIMITATIONS
 
-Please inform the author of any found.
-
 =head1 AUTHOR
 
 =over
diff --git a/lib/Monitor/RunFolder/Staging.pm b/lib/Monitor/RunFolder/Staging.pm
index c70947a74..9d4bd8e60 100644
--- a/lib/Monitor/RunFolder/Staging.pm
+++ b/lib/Monitor/RunFolder/Staging.pm
@@ -222,7 +222,7 @@ sub move_to_analysis {
         }
         if ($self->status_update) {
             my $status = 'analysis pending';
-            $self->tracking_run()->update_run_status($status, $self->username() );
+            $self->tracking_run()->update_run_status($status);
             push @ms, "Updated Run Status to $status";
         }
     }
diff --git a/lib/Monitor/Staging.pm b/lib/Monitor/Staging.pm
index 4010dc357..807671e29 100644
--- a/lib/Monitor/Staging.pm
+++ b/lib/Monitor/Staging.pm
@@ -9,18 +9,19 @@ use MooseX::StrictConstructor;
 use npg_tracking::illumina::run::folder::validation;
 use npg_tracking::illumina::run::folder::location;
 
-with 'Monitor::Roles::Schema';
-
-
 our $VERSION = '0';
 
-
 has known_areas => (
     is      => 'ro',
     isa     => 'ArrayRef',
     default => sub { [ @npg_tracking::illumina::run::folder::location::STAGING_AREAS ] },
 );
 
+has schema => (
+    is         => 'ro',
+    required   => 1,
+    isa        => 'npg_tracking::Schema',
+);
 
 sub validate_areas {
     my ( $self, @arguments ) = @_;
@@ -107,19 +108,19 @@ sub find_live {
                   my $db_external_name = $run_row->instrument->external_name();
                   if (! $db_external_name ){
                       warn qq[No instrument external_name for '$id_run' found in database, skipping\n];
-		      next;
-	          }
+                      next;
+                  }
                   my $staging_instrument_name;
                   try {
                       $staging_instrument_name = $check->instrument_name(); # from RunInfo.xml
-		      warn "Retrieved instrument_name $staging_instrument_name\n";
-		  } catch {
+                      warn "Retrieved instrument_name $staging_instrument_name\n";
+                  } catch {
                       warn "error retrieving instrument_name from RunInfo.xml\n";
-		  };
-		  if ( ! defined $staging_instrument_name ) {
-                     warn "instrument_name is undefined, skipping $run_dir\n";
-		     next;
-	          } else {
+                  };
+                  if ( ! defined $staging_instrument_name ) {
+                      warn "instrument_name is undefined, skipping $run_dir\n";
+                      next;
+                  } else {
                       if ($db_external_name ne $staging_instrument_name) {
                           warn "Skipping $run_dir - instrument name mismatch" .
                              " for run $id_run, '$staging_instrument_name' on staging," .
@@ -127,7 +128,7 @@ sub find_live {
                           next;
                       }
                       $run_row->update({'folder_name' => $run_folder}); # or validation will fail
-		  }
+                  }
                 }    
                 if ( npg_tracking::illumina::run::folder::validation->new(
                          run_folder          => $run_folder,
@@ -189,7 +190,7 @@ short read sequencer.
 =head1 DESCRIPTION
 
 This class gets various bits of information from the staging area for a short
-read sequencer (GA-II and HiSeq).
+read sequencer.
 
 =head1 SUBROUTINES/METHODS
 
@@ -238,7 +239,7 @@ they match /staging_area/machine/{incoming, analysis}/run_folder
 
 =head1 LICENSE AND COPYRIGHT
 
-Copyright (C) 2013,2014,2015,2018,2019,2020 Genome Research Ltd.
+Copyright (C) 2013,2014,2015,2018,2019,2020,2023 Genome Research Ltd.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
diff --git a/t/34-monitor-runfolder.t b/t/34-monitor-runfolder.t
index 30bf53676..a30fef2f6 100644
--- a/t/34-monitor-runfolder.t
+++ b/t/34-monitor-runfolder.t
@@ -2,7 +2,7 @@ use strict;
 use warnings;
 use English qw(-no_match_vars);
 use File::Copy;
-use Test::More tests => 35;
+use Test::More tests => 34;
 use Test::Exception;
 use Test::Warn;
 use File::Temp qw/ tempdir /;
@@ -34,10 +34,6 @@ my $dir4rf = tempdir( CLEANUP => 1 );
 
     is( $test->tracking_run()->current_run_status_description(), 'analysis pending',
         'Retrieve current run status' );
-
-    # Test Monitor::Roles::Username
-    is( $test->username(), 'pipeline',
-        'Retrieve default username for updates' );
 }
 
 {
diff --git a/t/34-monitor-staging.t b/t/34-monitor-staging.t
index ac63844ca..868f44210 100644
--- a/t/34-monitor-staging.t
+++ b/t/34-monitor-staging.t
@@ -18,7 +18,7 @@ my $schema = t::dbic_util->new->test_schema(fixture_path => q[t/data/dbic_fixtur
 
 {
     my $test;
-    lives_ok { $test = Monitor::Staging->new( _schema => $schema ) }
+    lives_ok { $test = Monitor::Staging->new( schema => $schema ) }
          'Object creation ok';
     throws_ok { $test->find_live() }
               qr/Top[ ]level[ ]staging[ ]path[ ]required/msx,
@@ -38,7 +38,7 @@ my $schema = t::dbic_util->new->test_schema(fixture_path => q[t/data/dbic_fixtur
       mkdir $dir;
       push @staging_areas, $dir;
     }
-    my $mtest = Monitor::Staging->new( _schema => $schema, known_areas => \@staging_areas);
+    my $mtest = Monitor::Staging->new( schema => $schema, known_areas => \@staging_areas);
     my @val_area_output;
 
     warning_is { @val_area_output = $mtest->validate_areas() }
@@ -72,7 +72,7 @@ my $schema = t::dbic_util->new->test_schema(fixture_path => q[t/data/dbic_fixtur
     my $run = $schema->resultset('Run')->search({id_run => 5222})->next;
     $run->update_run_status('run cancelled');
 
-    my $test = Monitor::Staging->new( _schema => $schema );
+    my $test = Monitor::Staging->new( schema => $schema );
     my $root = tempdir( CLEANUP => 1 );
     my @path = map { $root . $_ }
                qw(/IL5/outgoing/100713_IL24_0433

From 8c0365eace648359177fda661ff1e45ba13a3274 Mon Sep 17 00:00:00 2001
From: Marina Gourtovaia <mg8@sanger.ac.uk>
Date: Mon, 22 May 2023 12:32:59 +0100
Subject: [PATCH 3/9] Simplified the staging area daemon.

Bring the code in accordance with the current practice:
  - consider only one staging area,
  - narrow down the glob pattern for areas below the staging
    area to 'IL*'.
---
 bin/staging_area_monitor | 44 ++++++++++++-------------
 lib/Monitor/Staging.pm   | 71 +++++++++++++++-------------------------
 t/34-monitor-staging.t   | 58 ++++++++++++--------------------
 3 files changed, 69 insertions(+), 104 deletions(-)

diff --git a/bin/staging_area_monitor b/bin/staging_area_monitor
index f19c5ad0a..694ac80ec 100755
--- a/bin/staging_area_monitor
+++ b/bin/staging_area_monitor
@@ -172,15 +172,14 @@ sub check_path { ##### Function to deal with one run folder
 } ###### End of function to deal with one run folder
 
 ###########################################################
-###############   Main daemon loop  #######################
+###############   Main daemon loop.  ######################
 ###########################################################
 
 sub main {
 
-    my $schema    = npg_tracking::Schema->connect();
-    my $monitor   = Monitor::Staging->new(schema => $schema);
-    my @area_list = $monitor->validate_areas(@ARGV);
-    @area_list or croak 'No valid arguments';
+    my $schema  = npg_tracking::Schema->connect();
+    my $monitor = Monitor::Staging->new(schema => $schema);
+    my $staging_area = $monitor->validate_areas(@ARGV);
 
     my $time = sub {
         return strftime '%a %b %e %T %Y%n', localtime;
@@ -192,21 +191,19 @@ sub main {
 
         _log_noindent($time->());
 
-        foreach my $area (@area_list) {
-            foreach my $run_path ( $monitor->find_live($area) ) {
-                _log_noindent("Considering $run_path");
-                my $done;
-                try {
-                    $done = check_path($run_path, $schema, $previous_size_of);
-                } finally {
-                    if (@_) {
-                        _log("ERROR: Execution for $run_path died with: @_\n");
-                    } else {
-                        _log("Succesfully processed $run_path");
-                    }
-                };
-                $done and delete $previous_size_of->{$run_path};
-            }
+        foreach my $run_path ( $monitor->find_live($staging_area) ) {
+            _log_noindent("Considering $run_path");
+            my $done;
+            try {
+                $done = check_path($run_path, $schema, $previous_size_of);
+            } finally {
+                if (@_) {
+                    _log("ERROR: Execution for $run_path died with: @_\n");
+                } else {
+                    _log("Succesfully processed $run_path");
+                }
+            };
+            $done and delete $previous_size_of->{$run_path};
         }
 
         _log_noindent("Gone to sleep for $SLEEP_INTERVAL seconds at " . $time->());
@@ -226,11 +223,14 @@ staging_area_monitor
 
 =head1 SYNOPSIS
 
-    staging_area_monitor /nfs/sf10 [/nfs/sf8] ...
+  # Assuming staging_area_path is an existing path
+
+  staging_area_monitor staging_area_path 
 
 =head1 DESCRIPTION
 
-Monitor one or more staging areas supplied as command-line arguments.
+Monitors one staging area. The staging area path is supplied as a
+command-line argument.
 
 The script runs an infinite loop so do not call it via a crontab.
 
diff --git a/lib/Monitor/Staging.pm b/lib/Monitor/Staging.pm
index 807671e29..cfb0c0895 100644
--- a/lib/Monitor/Staging.pm
+++ b/lib/Monitor/Staging.pm
@@ -1,22 +1,16 @@
 package Monitor::Staging;
 
 use Moose;
-use Monitor::RunFolder;
+use namespace::autoclean;
 use Carp;
 use Try::Tiny;
 use MooseX::StrictConstructor;
 
 use npg_tracking::illumina::run::folder::validation;
-use npg_tracking::illumina::run::folder::location;
+use Monitor::RunFolder;
 
 our $VERSION = '0';
 
-has known_areas => (
-    is      => 'ro',
-    isa     => 'ArrayRef',
-    default => sub { [ @npg_tracking::illumina::run::folder::location::STAGING_AREAS ] },
-);
-
 has schema => (
     is         => 'ro',
     required   => 1,
@@ -24,36 +18,20 @@ has schema => (
 );
 
 sub validate_areas {
-    my ( $self, @arguments ) = @_;
-
-    carp 'Empty argument list' if !@arguments;
-    my $known_areas = $self->known_areas();
-
-    my %check;
-    foreach (@arguments) {
-        my $area = $_;
-
-        if (m/^ \d+ $/msx) {
-            if ( !defined $known_areas->[ $_ + 0 ] ) {
-                carp "Parameter out of bounds: $_";
-                next;
-            }
-            $area = $known_areas->[$_];
-        }
-
-        if ( !-d $area ) {
-            carp "Staging directory not found: $area";
-            next;
-        }
-
-        carp "$area specified twice" if $check{$area};
+    my ( $self, @staging_areas ) = @_;
 
-        $check{$area}++;
+    if (!@staging_areas) {
+        croak 'Empty argument list of staging areas';
     }
-
-    my @validated = sort { $a cmp $b } keys %check;
-
-    return @validated;
+    if (@staging_areas > 1) {
+        croak 'Multiple staging areas cannot be processed';
+    }
+    my $staging_area = $staging_areas[0];
+    if ( !-d $staging_area ) {
+        croak "Staging directory not found: $staging_area";
+    }
+   
+    return $staging_area;
 }
 
 
@@ -65,7 +43,7 @@ sub find_live {
 
     my %path_for;
 
-    foreach my $run_dir ( glob $staging_area . q{/{IL,HS}*/{incoming,analysis}/*} ) {
+    foreach my $run_dir ( glob $staging_area . q{/IL*/{incoming,analysis}/*} ) {
 
         warn "\n\nFound $run_dir\n";
 
@@ -168,15 +146,15 @@ sub find_live {
     return values %path_for;
 }
 
-no Moose;
 __PACKAGE__->meta->make_immutable();
+
 1;
 
 __END__
 
 =head1 NAME
 
-Monitor::Staging - interrogate the staging area of an Illumina
+Monitor::Staging - interrogate the staging area designated to an Illumina
 short read sequencer.
 
 =head1 VERSION
@@ -186,7 +164,6 @@ short read sequencer.
     C<<use Monitor::Staging;
        my $stage_poll = Monitor::Staging->new();>>
 
-
 =head1 DESCRIPTION
 
 This class gets various bits of information from the staging area for a short
@@ -196,15 +173,17 @@ read sequencer.
 
 =head2 validate_areas
 
-Check if the parameters passed to the method are valid staging areas. They
-may be passed as absolute paths or as integer indices of some (hard-coded)
-external array.
+Check if the argument passed to the method is a valid staging areas,
+ie an existing directory. Error if no argument or multiple arguments
+is/are given.
 
 =head2 find_live
 
 Take a staging area path as a required argument and return a list of all run
-directories (no tests or repeats) found in incoming and analysis folders below it. I.e.
-they match /staging_area/machine/{incoming, analysis}/run_folder
+directories (no tests or repeats) found in incoming and analysis folders below
+it.
+
+The path pattern should match /[staging_area]/IL*/{incoming, analysis}/[run_folder]
 
 
 =head1 CONFIGURATION AND ENVIRONMENT
@@ -215,6 +194,8 @@ they match /staging_area/machine/{incoming, analysis}/run_folder
 
 =item Moose
 
+=item namespace::autoclean
+
 =item Carp
 
 =item Try::Tiny
diff --git a/t/34-monitor-staging.t b/t/34-monitor-staging.t
index 868f44210..5e468d8a4 100644
--- a/t/34-monitor-staging.t
+++ b/t/34-monitor-staging.t
@@ -1,9 +1,7 @@
 use strict;
 use warnings;
-use Test::More tests => 12;
-use Test::Deep;
+use Test::More tests => 10;
 use Test::Exception;
-use Test::Warn;
 use File::Temp qw/ tempdir /;
 use File::Path qw/ make_path /;
 
@@ -31,41 +29,24 @@ my $schema = t::dbic_util->new->test_schema(fixture_path => q[t/data/dbic_fixtur
 {
     my $tdir = tempdir( CLEANUP => 1 );
     my @staging_areas = ();
-    foreach my $id (qw(18 22 33 36)) {
-      my $dir = join q[/], $tdir, q[nfs];
-      mkdir $dir;
-      $dir = join q[/], $dir, q[sf] . $id;
-      mkdir $dir;
-      push @staging_areas, $dir;
+    foreach my $id (qw(18 22)) {
+        my $dir = join q[/], $tdir, q[nfs], q[sf] . $id;
+        make_path $dir;
+        push @staging_areas, $dir;
     }
-    my $mtest = Monitor::Staging->new( schema => $schema, known_areas => \@staging_areas);
-    my @val_area_output;
+    my $mtest = Monitor::Staging->new( schema => $schema );
 
-    warning_is { @val_area_output = $mtest->validate_areas() }
-               'Empty argument list',
-               'Warn for no argument...';
-    is( scalar @val_area_output, 0, '...and return an empty list' );
-
-    warning_like {
-                   @val_area_output = $mtest->validate_areas( 1, 100_000_000 )
-                 }
-                 qr/Parameter[ ]out[ ]of[ ]bounds:[ ]/msx,
-                 'Warn for out of bounds index...';
-
-    warning_like {
-                   push @val_area_output,
-                        $mtest->validate_areas( 2, '/no/such/dir' )
-                 }
-                 qr/Staging[ ]directory[ ]not[ ]found:[ ]/msx,
-                 '   ...warn for directory not found...';
-
-    warning_like { push @val_area_output, $mtest->validate_areas( 3, 3 ) }
-                 qr/[ ]specified[ ]twice/msx,
-                 '   ...warn for duplicate area...';
-
-    cmp_bag( \@val_area_output, [ @{$mtest->known_areas}[1..3], ],
-             '   ...causes of warnings filtered from output'
-    );
+    throws_ok { $mtest->validate_areas() } qr/Empty argument list/,
+        'error for no argument';
+    throws_ok { $mtest->validate_areas(@staging_areas) }
+        qr/Multiple staging areas cannot be processed/,
+        'error for multiple staging directories';
+    throws_ok { $mtest->validate_areas('/no/such/dir') }
+        qr/Staging[ ]directory[ ]not[ ]found:[ ]/msx,
+        'error for the non-existing staging directory';
+    
+    is ($mtest->validate_areas($staging_areas[0]), $staging_areas[0],
+        'directory path is returned');
 }
 
 {
@@ -73,6 +54,7 @@ my $schema = t::dbic_util->new->test_schema(fixture_path => q[t/data/dbic_fixtur
     $run->update_run_status('run cancelled');
 
     my $test = Monitor::Staging->new( schema => $schema );
+
     my $root = tempdir( CLEANUP => 1 );
     my @path = map { $root . $_ }
                qw(/IL5/outgoing/100713_IL24_0433
@@ -88,7 +70,9 @@ my $schema = t::dbic_util->new->test_schema(fixture_path => q[t/data/dbic_fixtur
     shift @path; # not in analysis | incoming
     shift @path; # run cancelled
     shift @path; # runfolder name mismatch against the db
-    is( join(q[ ],sort @live_incoming), join(q[ ],sort @path), 'runfolders found in incoming and analysis');
+    pop @path;   # looking at IL* only
+    is( join(q[ ],sort @live_incoming), join(q[ ],sort @path),
+        'runfolders found in incoming and analysis');
 }
 
 1;

From 69d5fc69cd22f5ee0afaa5ba68d0db79f330d00c Mon Sep 17 00:00:00 2001
From: Marina Gourtovaia <mg8@sanger.ac.uk>
Date: Mon, 22 May 2023 20:51:45 +0100
Subject: [PATCH 4/9] Simplified run complete and mirrored detection.

Removed exessive complexity in finding RTACpmplete.txt
and CopyComplete.txt files.

Removed a second round of examining the above files
at the stage of deciding whether the mirroring has
been completed.

Removed examination of the content of the Error log
file since it is not present in the runfolders created
by the contemporary instruments.
---
 bin/staging_area_monitor         |  3 +-
 lib/Monitor/RunFolder/Staging.pm | 80 +++++---------------------------
 t/34-monitor-runfolder-staging.t | 72 ++--------------------------
 3 files changed, 18 insertions(+), 137 deletions(-)

diff --git a/bin/staging_area_monitor b/bin/staging_area_monitor
index f19c5ad0a..fd1762813 100755
--- a/bin/staging_area_monitor
+++ b/bin/staging_area_monitor
@@ -135,8 +135,7 @@ sub check_path { ##### Function to deal with one run folder
             $folder->update_run_record(); # Inspect and update cycles,
                                           # including expected cycle count!
 
-            if ( $folder->mirroring_complete($run_path) &&
-                 $folder->check_tiles($run_path) ) {
+            if ( $folder->check_tiles($run_path) ) {
 
                 my $previous_size = $previous_size_of->{$run_path};
                 my ( $current_size, $latest_mod ) = $folder->monitor_stats();
diff --git a/lib/Monitor/RunFolder/Staging.pm b/lib/Monitor/RunFolder/Staging.pm
index 9d4bd8e60..152b4ae0e 100644
--- a/lib/Monitor/RunFolder/Staging.pm
+++ b/lib/Monitor/RunFolder/Staging.pm
@@ -6,9 +6,7 @@ use English qw(-no_match_vars);
 use File::Copy;
 use File::Find;
 use File::Basename;
-use IO::All;
 use List::Util qw(max);
-use Perl6::Slurp;
 use Readonly;
 use List::MoreUtils qw(any);
 use Try::Tiny;
@@ -25,63 +23,43 @@ Readonly::Scalar my $SECONDS_PER_MINUTE   => 60;
 Readonly::Scalar my $MINUTES_PER_HOUR     => 60;
 Readonly::Scalar my $SECONDS_PER_HOUR     => $SECONDS_PER_MINUTE * $MINUTES_PER_HOUR;
 Readonly::Scalar my $MAX_COMPLETE_WAIT    => 6 * $SECONDS_PER_HOUR;
-Readonly::Scalar my $RTA_COMPLETE         => 10 * $SECONDS_PER_MINUTE;
 Readonly::Scalar my $INTENSITIES_DIR_PATH => q[Data/Intensities];
 Readonly::Scalar my $BASECALLS_DIR_PATH   => qq[$INTENSITIES_DIR_PATH/BaseCalls];
 Readonly::Scalar my $MODE_INDEX           => 2;
 
-Readonly::Scalar my $RTA_COMPLETE_FN      => q[RTAComplete\.txt];
-Readonly::Scalar my $COPY_COMPLETE_FN     => q[CopyComplete\.txt];
-
-has 'rta_complete_wait' => (isa          => 'Int',
-                            is           => 'ro',
-                            default      => $RTA_COMPLETE,
-                           );
+Readonly::Scalar my $RTA_COMPLETE_FN      => q[RTAComplete.txt];
+Readonly::Scalar my $COPY_COMPLETE_FN     => q[CopyComplete.txt];
 
 has 'status_update' => (isa          => 'Bool',
                         is           => 'ro',
                         default      => 1,
                        );
 
-sub _find_files {
-    my ($self, $filename) = @_;
-    my $run_path = $self->runfolder_path();
-
-    my @file_list;
-
-    # The trailing slash forces IO::All to cope with symlinks.
-    eval { @file_list = io("$run_path/")->all_files(); 1; }
-        or do { carp $EVAL_ERROR; return 0; };
-
-    my @markers = map {q().$_} grep { basename($_) =~ m/\A $filename \Z/msx } @file_list;
-
-    if ( scalar @markers > 1 ) {
-      carp qq[Unexpected to find multiple files matching pattern '$filename' in staging.];
-    }
-
-    return @markers;
+sub _find_file {
+    my ($self, $file_path) = @_;
+    my $file = join q[/], $self->runfolder_path(), $file_path;
+    return -f $file ? $file : q();
 }
 
 sub is_run_complete {
     my ($self) = @_;
     
-    my @markers = $self->_find_files($RTA_COMPLETE_FN);
-    my $has_rta_complete_file = scalar @markers;
-    my $has_copy_complete_file = $self->_find_files($COPY_COMPLETE_FN);
+    my $rta_complete_file = $self->_find_file($RTA_COMPLETE_FN);
+    my $copy_complete_file = $self->_find_file($COPY_COMPLETE_FN);
  
-    if ( $has_rta_complete_file ) {
+    if ( $rta_complete_file ) {
         if ( $self->platform_NovaSeq() or $self->platform_NovaSeqX()) {
-            if ( $has_copy_complete_file ) {
+            if ( $copy_complete_file ) {
                 return 1;
             } else {
-                my $mtime = ( stat $markers[0] )[$MTIME_INDEX];
+                my $mtime = ( stat $rta_complete_file )[$MTIME_INDEX];
                 my $last_modified = time() - $mtime;
                 return $last_modified > $MAX_COMPLETE_WAIT;
             }
         }
         return 1;
     } else {
-        if ( $has_copy_complete_file ) {
+        if ( $copy_complete_file ) {
             my $rf = $self->runfolder_path();
             carp "Runfolder '$rf' with CopyComplete but not RTAComplete";
         }
@@ -89,30 +67,6 @@ sub is_run_complete {
     return 0;
 }
 
-sub mirroring_complete {
-    my ($self) = @_;
-
-    print {*STDERR} "\tChecking for mirroring complete.\n" or carp $OS_ERROR;
-
-    my $run_path = $self->runfolder_path();
-
-    my @markers = $self->_find_files($RTA_COMPLETE_FN);
-
-    my $mtime = ( scalar @markers )
-                ? ( stat $markers[0] )[$MTIME_INDEX]
-                : time;
-    my $last_modified = time() - $mtime;
-
-    my $events_file  = $self->runfolder_path() . q{/Events.log};
-    my $events_log   = ( -e $events_file ) ? slurp($events_file) : q{};
-    my $events_regex =
-        qr{Copying[ ]logs[ ]to[ ]network[ ]run[ ]folder\s* \Z }msx;
-
-    return ( $last_modified > $self->rta_complete_wait ) ? 1
-         : ( $events_log =~ $events_regex )       ? 1
-         :                                          0;
-}
-
 sub monitor_stats {
     my ($self) = @_;
 
@@ -374,12 +328,6 @@ but has been there for longer than a timeout limit.
 Perform a series of checks to make sure the run really is complete. Return 0
 if any of them fails. If all pass return 1.
 
-=head2 mirroring_complete
-
-Determines if mirroring is complete by checking for the presence and last
-modification time of certain files. Return 0 if the tests fail (mirroring is
-*not* complete), otherwise return 1.
-
 =head2 monitor_stats
 
 Returns the sum of all file sizes in the tree below $self->runfolder_path(), and
@@ -422,12 +370,8 @@ Move the run folder from 'analysis' to 'outgoing'.
 
 =item File::Basename
 
-=item IO::All
-
 =item List::Util
 
-=item Perl6::Slurp
-
 =item Readonly
 
 =item Try::Tiny
diff --git a/t/34-monitor-runfolder-staging.t b/t/34-monitor-runfolder-staging.t
index 2e52a86cc..a3bf5b69a 100644
--- a/t/34-monitor-runfolder-staging.t
+++ b/t/34-monitor-runfolder-staging.t
@@ -1,9 +1,8 @@
 use strict;
 use warnings;
-use Test::More tests => 52;
+use Test::More tests => 49;
 use Test::Exception;
 use Test::Warn;
-use Test::Deep;
 use File::Copy;
 use File::Find;
 use File::Temp qw(tempdir);
@@ -84,7 +83,7 @@ ENDXML
 }
 
 subtest 'updating run data from filesystem' => sub {
-    plan tests => 6;
+    plan tests => 5;
 
     my $basedir = tempdir( CLEANUP => 1 );
 
@@ -109,7 +108,6 @@ subtest 'updating run data from filesystem' => sub {
     my $run_folder = Monitor::RunFolder::Staging->new(runfolder_path      => $fs_run_folder,
                                                       npg_tracking_schema => $schema);
     isa_ok($run_folder, 'Monitor::RunFolder::Staging');
-    is($run_folder->rta_complete_wait, 600, 'default rta complete wait time');
 
     is( $run_folder->_get_folder_path_glob, qq[$basedir/IL3/*/],
         'internal glob correct' );
@@ -121,28 +119,6 @@ subtest 'updating run data from filesystem' => sub {
         '  folder path glob updated' );
 };
 
-{
-    my $tmpdir = tempdir( CLEANUP => 1 );
-
-    system('cp',  '-rp', $MOCK_STAGING . '/ILorHSany_sf20/incoming/100914_HS3_05281_A_205MBABXX', $tmpdir);
-    sleep 5;
-
-    my $mock_path = $tmpdir . '/100914_HS3_05281_A_205MBABXX';
-
-    my $test = Monitor::RunFolder::Staging->new(runfolder_path      => $mock_path,
-                                                npg_tracking_schema => $schema,
-                                                rta_complete_wait   => 15);
-
-    rename "$mock_path/RTAComplete.txt", "$mock_path/RTA_renamed_Complete.txt";
-
-    ok( !$test->mirroring_complete(),
-        'Mirroring is not complete' );
-
-    rename "$mock_path/RTA_renamed_Complete.txt", "$mock_path/RTAComplete.txt";
-
-    ok( $test->mirroring_complete(),
-        'Mirroring is complete' );
-}
 
 sub touch_file {
     my ($path) = @_;
@@ -151,48 +127,10 @@ sub touch_file {
     close $fh;
 }
 
-subtest 'folder identifies copy complete for HiSeq (Non-NovaSeq)' => sub {
-    plan tests => 7;
-    my $basedir = tempdir( CLEANUP => 1 );
-
-    my $fs_run_folder = qq[$basedir/IL3/incoming/100622_IL3_01234];
-    make_path($fs_run_folder);
-
-    my $id_run = 1234;
-    my $run_data = {
-      id_run => $id_run,
-      id_instrument => 67,
-      id_instrument_format => 10,
-      team => 'A',
-      expected_cycle_count => 310,
-      actual_cycle_count => 0, # So there is no lag
-    };
-
-    write_run_files($id_run, $fs_run_folder);
-
-    my $run = $schema->resultset('Run')->find($id_run);
-    $run->update($run_data);
-
-    my $run_folder = Monitor::RunFolder::Staging->new(runfolder_path      => $fs_run_folder,
-                                                      npg_tracking_schema => $schema);
-
-    ok(!$run_folder->is_run_complete(), 'Run is not complete');
-
-    for my $file_name (qw[ RTAComplete RTAcomplete rtacomplete RTAComplete.tsv RTAComplete_old.txt ]) {
-      note $file_name;
-      my $path_to_complete = qq[$fs_run_folder/$file_name];
-      touch_file($path_to_complete);
-      ok(!$run_folder->is_run_complete(), 'Run is not complete');
-      unlink $path_to_complete or die "Could not delete file $path_to_complete: $!";
-    }
-
-    my $path_to_complete = qq[$fs_run_folder/RTAComplete.txt];
-    touch_file($path_to_complete);
-    ok($run_folder->is_run_complete(), 'Run is complete');
-};
 
 subtest 'folder identifies copy complete for NovaSeq' => sub {
-    plan tests => 11;
+    plan tests => 10;
+
     my $basedir = tempdir( CLEANUP => 1 );
 
     my $fs_run_folder = qq[$basedir/IL3/incoming/100622_IL3_01234];
@@ -225,7 +163,7 @@ subtest 'folder identifies copy complete for NovaSeq' => sub {
     touch_file($path_to_rta_complete);
     ok(!$run_folder->is_run_complete(), 'Only RTAComplete is not enough for NovaSeq');
 
-    for my $file_name (qw[ CopyComplete Copycomplete copycomplete CopyComplete.tsv CopyComplete_old.txt ]) {
+    for my $file_name (qw[ CopyComplete Copycomplete copycomplete CopyComplete_old.txt ]) {
       note $file_name;
       my $path_to_wrong_copy_complete = qq[$fs_run_folder/$file_name];
       touch_file($path_to_wrong_copy_complete);

From 112b5f520d14fdeec64a7b2cba78b30ea62a3953 Mon Sep 17 00:00:00 2001
From: Marina Gourtovaia <mg8@sanger.ac.uk>
Date: Tue, 23 May 2023 10:38:20 +0100
Subject: [PATCH 5/9] Improved logging for a long wait case.

---
 lib/Monitor/RunFolder/Staging.pm | 29 ++++++++++++++++++++++-------
 t/34-monitor-runfolder-staging.t | 25 ++++++++++++++++++++-----
 2 files changed, 42 insertions(+), 12 deletions(-)

diff --git a/lib/Monitor/RunFolder/Staging.pm b/lib/Monitor/RunFolder/Staging.pm
index 152b4ae0e..f74ddf610 100644
--- a/lib/Monitor/RunFolder/Staging.pm
+++ b/lib/Monitor/RunFolder/Staging.pm
@@ -46,25 +46,40 @@ sub is_run_complete {
     
     my $rta_complete_file = $self->_find_file($RTA_COMPLETE_FN);
     my $copy_complete_file = $self->_find_file($COPY_COMPLETE_FN);
- 
+    my $is_run_complete = 0;
+
     if ( $rta_complete_file ) {
         if ( $self->platform_NovaSeq() or $self->platform_NovaSeqX()) {
             if ( $copy_complete_file ) {
-                return 1;
+                $is_run_complete = 1;
             } else {
                 my $mtime = ( stat $rta_complete_file )[$MTIME_INDEX];
                 my $last_modified = time() - $mtime;
-                return $last_modified > $MAX_COMPLETE_WAIT;
+                if  ($last_modified > $MAX_COMPLETE_WAIT) {
+                    carp sprintf q[Runfolder '%s' with %s but not %s],
+                        $self->runfolder_path(),
+                        $RTA_COMPLETE_FN,
+                        $COPY_COMPLETE_FN;
+                    # Do we ever wait for this long and proceed regardless?
+                    # Log to find out.
+                    carp qq[Has waited for over $MAX_COMPLETE_WAIT secs, ] .
+                        q[consider copied];
+                    $is_run_complete = 1;
+                }
             }
+        } else {
+            $is_run_complete = 1;
         }
-        return 1;
     } else {
         if ( $copy_complete_file ) {
-            my $rf = $self->runfolder_path();
-            carp "Runfolder '$rf' with CopyComplete but not RTAComplete";
+            carp sprintf q[Runfolder '%s' with %s but not %s],
+                $self->runfolder_path(),
+                $COPY_COMPLETE_FN,
+                $RTA_COMPLETE_FN;
         }
     }
-    return 0;
+
+    return $is_run_complete;
 }
 
 sub monitor_stats {
diff --git a/t/34-monitor-runfolder-staging.t b/t/34-monitor-runfolder-staging.t
index a3bf5b69a..e12438181 100644
--- a/t/34-monitor-runfolder-staging.t
+++ b/t/34-monitor-runfolder-staging.t
@@ -129,7 +129,7 @@ sub touch_file {
 
 
 subtest 'folder identifies copy complete for NovaSeq' => sub {
-    plan tests => 10;
+    plan tests => 12;
 
     my $basedir = tempdir( CLEANUP => 1 );
 
@@ -174,7 +174,11 @@ subtest 'folder identifies copy complete for NovaSeq' => sub {
     unlink $path_to_rta_complete or die "Could not delete file $path_to_rta_complete: $!";
 
     touch_file($path_to_copy_complete);
-    ok(!$run_folder->is_run_complete(), 'Only CopyComplete is not enough for NovaSeq');
+    my $complete = 1;
+    warning_like { $complete = $run_folder->is_run_complete() }
+        { carped => qr/with CopyComplete\.txt but not RTAComplete\.txt/ },
+        'Missing RTAComplete.txt is logged';
+    is($complete, 0, 'Only CopyComplete.txt file is not enough for NovaSeq');
 
     touch_file($path_to_rta_complete);
     ok($run_folder->is_run_complete(), 'RTAComplete + CopyComplete is enough for NovaSeq');
@@ -195,7 +199,12 @@ subtest 'folder identifies copy complete for NovaSeq' => sub {
 
     utime($atime, $mtime, $path_to_rta_complete)
         or die "couldn't backdate $path_to_rta_complete, $!";
-    ok($run_folder->is_run_complete(), 'RTAComplete + long wait time is enough for NovaSeq');
+    $complete = 0;
+    warnings_like { $complete = $run_folder->is_run_complete() } [
+        { carped => qr/with RTAComplete\.txt but not CopyComplete\.txt/ },
+        { carped => qr/Has waited for over 21600 secs, consider copied/ }
+    ], 'Missing CopyComplete.txt and end of the wait are logged';
+    is($complete, 1, 'RTAComplete + long wait time is enough for NovaSeq');
 };
 
 subtest 'counting cycles and tiles' => sub {
@@ -423,7 +432,7 @@ subtest 'counting cycles and tiles' => sub {
 }
 
 subtest 'run completion for NovaSeqX' => sub {
-    plan tests => 4;
+    plan tests => 5;
 
     my $tmpdir = tempdir( CLEANUP => 1 );
     copy('t/data/run_params/RunParameters.novaseqx.xml',"$tmpdir/RunParameters.xml")
@@ -447,7 +456,13 @@ subtest 'run completion for NovaSeqX' => sub {
     $mtime = $atime;
     utime($atime, $mtime, $rta_complete_file)
         or die "couldn't backdate $rta_complete_file, $!";
-    ok( $monitor->is_run_complete(),
+
+    my $complete = 0;
+    warnings_like { $complete = $monitor->is_run_complete() } [
+        { carped => qr/with RTAComplete\.txt but not CopyComplete\.txt/ },
+        { carped => qr/Has waited for over 21600 secs, consider copied/ }
+    ], 'Missing CopyComplete.txt and end of the wait are logged';
+    ok( $complete,
         'RTAComplete + long wait time is enough for NovaSeqX');
 };
 

From 1cc5b0778df821d8f1b18b86c055f56bc199d36c Mon Sep 17 00:00:00 2001
From: Marina Gourtovaia <mg8@sanger.ac.uk>
Date: Thu, 25 May 2023 17:12:13 +0100
Subject: [PATCH 6/9] Added checks for on-board analysis in progress

---
 bin/staging_area_monitor         |  9 +++++
 lib/Monitor/RunFolder/Staging.pm | 59 ++++++++++++++++++++++++++++--
 t/34-monitor-runfolder-staging.t | 63 ++++++++++++++++++++++++++++++--
 3 files changed, 125 insertions(+), 6 deletions(-)

diff --git a/bin/staging_area_monitor b/bin/staging_area_monitor
index fd1762813..971d57d2a 100755
--- a/bin/staging_area_monitor
+++ b/bin/staging_area_monitor
@@ -50,6 +50,9 @@ sub check_path { ##### Function to deal with one run folder
     my $id_run = $folder->tracking_run()->id_run;
 
     _log("Run $id_run status $run_status");
+    if ($folder->platform_is_NovaSeqX()) {
+        _log("... run on NovaSeqX instrument");
+    }
 
     my $db_flowcell_barcode      = $folder->tracking_run()->flowcell_id;
     my $staging_flowcell_barcode = $folder->run_flowcell; # from RunParameters.xml
@@ -154,6 +157,12 @@ sub check_path { ##### Function to deal with one run folder
                     return;
                 }
 
+                # Check if we are waiting for the onboard analysis to finish.
+                if ($folder->is_onboard_analysis_planned() &&
+                    !$folder->has_onboard_analysis_finished()) {
+                    return;
+                }
+
                 # Set status to 'run mirrored' and move run folder
                 # from /incoming/ to /analysis/
                 $folder->update_run_status('run mirrored');
diff --git a/lib/Monitor/RunFolder/Staging.pm b/lib/Monitor/RunFolder/Staging.pm
index f74ddf610..4ebe7b6d0 100644
--- a/lib/Monitor/RunFolder/Staging.pm
+++ b/lib/Monitor/RunFolder/Staging.pm
@@ -6,6 +6,7 @@ use English qw(-no_match_vars);
 use File::Copy;
 use File::Find;
 use File::Basename;
+use File::Slurp;
 use List::Util qw(max);
 use Readonly;
 use List::MoreUtils qw(any);
@@ -29,6 +30,9 @@ Readonly::Scalar my $MODE_INDEX           => 2;
 
 Readonly::Scalar my $RTA_COMPLETE_FN      => q[RTAComplete.txt];
 Readonly::Scalar my $COPY_COMPLETE_FN     => q[CopyComplete.txt];
+Readonly::Scalar my $ONBOARD_ANALYSIS_COMPLETE_FN =>
+                                             q[Secondary_Analysis_Complete.txt];
+Readonly::Scalar my $ONBOARD_ANALYSIS_SAMPLESHEET_FN => q[SampleSheet.csv]; 
 
 has 'status_update' => (isa          => 'Bool',
                         is           => 'ro',
@@ -82,6 +86,43 @@ sub is_run_complete {
     return $is_run_complete;
 }
 
+
+
+sub is_onboard_analysis_planned {
+    my $self = shift;
+
+    my $planned = 0;
+    if ($self->platform_NovaSeqX()) {
+        my $ss = join q[/],
+            $self->runfolder_path(), $ONBOARD_ANALYSIS_SAMPLESHEET_FN;
+        my $have_ss = -f $ss;
+        carp sprintf 'Samplesheet %s for the onboard analysis%sfound',
+            $ss, $have_ss ? q[ ] : q[ not ];
+        if ($have_ss) {
+            $planned = any { $_ =~ /^\[BCLConvert/smx } read_file($ss);
+            carp sprintf 'Samplesheet %s %s the BCLConvert section',
+                $ss, $planned ? q[has] : q[dooen't have];
+        }
+    }
+    return $planned;
+}
+
+sub has_onboard_analysis_finished {
+    my $self = shift;
+
+    my $analysis_dir = $self->dragen_analysis_path();
+    my $found = 0;
+    if (-d $analysis_dir) {
+        my $file = join q[/], $analysis_dir, $ONBOARD_ANALYSIS_COMPLETE_FN;
+        $found = -f $file;
+        carp sprintf '%s is%sfound', $file, $found ? q[ ] : q[ not ];         
+    } else {
+        carp "No DRAGEN analysis directory $analysis_dir";
+    }
+
+    return $found;
+}
+
 sub monitor_stats {
     my ($self) = @_;
 
@@ -345,8 +386,8 @@ if any of them fails. If all pass return 1.
 
 =head2 monitor_stats
 
-Returns the sum of all file sizes in the tree below $self->runfolder_path(), and
-also the highest epoch time found.
+Returns the sum of all file sizes in the tree below $self->runfolder_path()
+and the highest epoch time found.
 
 =head2 get_latest_cycle
 
@@ -361,12 +402,24 @@ Move the run folder from 'incoming' to 'analysis'. Then set the run status to
 
 =head2 is_in_analysis
 
-Returns true if the runfolder is in analysis upstream directory and false othenrwise
+Returns true if the runfolder is in analysis upstream directory and
+false otherwise.
 
 =head2 move_to_outgoing
 
 Move the run folder from 'analysis' to 'outgoing'.
 
+=head2 is_onboard_analysis_planned
+
+Returns true if the run folder for the NovaSeq SeriesX run contains a
+samplesheet with a section for the bcl on board data conversion. Always returns
+false for other instrument types.
+
+=head2 has_onboard_analysis_finished
+
+Returns true if the file that inticates that the onboard analysis has finished
+is present in the run folder.
+
 =head1 CONFIGURATION AND ENVIRONMENT
 
 =head1 DEPENDENCIES
diff --git a/t/34-monitor-runfolder-staging.t b/t/34-monitor-runfolder-staging.t
index e12438181..379ff26ec 100644
--- a/t/34-monitor-runfolder-staging.t
+++ b/t/34-monitor-runfolder-staging.t
@@ -1,6 +1,6 @@
 use strict;
 use warnings;
-use Test::More tests => 49;
+use Test::More tests => 51;
 use Test::Exception;
 use Test::Warn;
 use File::Copy;
@@ -119,7 +119,6 @@ subtest 'updating run data from filesystem' => sub {
         '  folder path glob updated' );
 };
 
-
 sub touch_file {
     my ($path) = @_;
 
@@ -127,7 +126,6 @@ sub touch_file {
     close $fh;
 }
 
-
 subtest 'folder identifies copy complete for NovaSeq' => sub {
     plan tests => 12;
 
@@ -466,4 +464,63 @@ subtest 'run completion for NovaSeqX' => sub {
         'RTAComplete + long wait time is enough for NovaSeqX');
 };
 
+subtest 'onboard analysis completion for NovaSeqX' => sub {
+    plan tests => 6;
+
+    my $tmpdir = tempdir( CLEANUP => 1 );
+    copy('t/data/run_params/RunParameters.novaseqx.xml',"$tmpdir/RunParameters.xml")
+      or die "Copy failed: $!";
+    copy('t/data/run_info/runInfo.novaseqx.xml',"$tmpdir/RunInfo.xml")
+      or die "Copy failed: $!";
+
+    my $monitor = Monitor::RunFolder::Staging->new(runfolder_path => $tmpdir);
+    
+    ok (!$monitor->is_onboard_analysis_planned(),
+        'onboard analysis is not planned');
+    ok (!$monitor->has_onboard_analysis_finished(),
+        'onboard analysis has not finished');
+
+    my $ss_file = "$tmpdir/SampleSheet.csv";
+    open my $fh, '>', $ss_file or die 'Cannot open a file for writing';
+    print $fh "[Header]\n[Cloud_BCLConvert_Settings]\n" or die 'Cannot print';
+    close $fh or die 'Cannot close the file handle';
+    ok (!$monitor->is_onboard_analysis_planned(),
+        'onboard analysis is not planned');
+
+    mkdir "$tmpdir/Analysis";
+    ok (!$monitor->has_onboard_analysis_finished(),
+        'onboard analysis has not finished');
+
+    open $fh, '>>', $ss_file or die 'Cannot open a file for appending';
+    print $fh "[BCLConvert_Settings]\n" or die 'Cannot print';
+    close $fh or die 'Cannot close the file handle';
+    ok ($monitor->is_onboard_analysis_planned(), 'onboard analysis is planned');
+
+    `touch $tmpdir/Analysis/Secondary_Analysis_Complete.txt`;
+    ok ($monitor->has_onboard_analysis_finished(),
+        'onboard analysis has finished');     
+};
+
+subtest 'no onboard analysis planned for NovaSeq' => sub {
+    plan tests => 2;
+
+    my $tmpdir = tempdir( CLEANUP => 1 );
+    copy('t/data/run_params/RunParameters.novaseq.xml',"$tmpdir/RunParameters.xml")
+      or die "Copy failed: $!";
+    copy('t/data/run_info/runInfo.novaseq.xml',"$tmpdir/RunInfo.xml")
+      or die "Copy failed: $!";
+    my $monitor = Monitor::RunFolder::Staging->new(runfolder_path => $tmpdir);
+    
+    ok (!$monitor->is_onboard_analysis_planned(),
+        'onboard analysis is not planned');
+
+    my $ss_file = "$tmpdir/SampleSheet.csv";
+    open my $fh, '>', $ss_file or die 'Cannot open a file for writing';
+    print $fh "[Header]\n[BCLConvert_Settings]\n" or die 'Cannot print';
+    close $fh or die 'Cannot close the file handle';
+    ok (!$monitor->is_onboard_analysis_planned(),
+        'onboard analysis is not planned');
+};
+
+
 1;

From f70d771ffd8f26cdde0b31bd397cd000b2757279 Mon Sep 17 00:00:00 2001
From: Marina Gourtovaia <mg8@sanger.ac.uk>
Date: Thu, 1 Jun 2023 09:17:49 +0100
Subject: [PATCH 7/9] Fixed a typo in the warning message

---
 lib/Monitor/RunFolder/Staging.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/Monitor/RunFolder/Staging.pm b/lib/Monitor/RunFolder/Staging.pm
index 4ebe7b6d0..1ca23d7c2 100644
--- a/lib/Monitor/RunFolder/Staging.pm
+++ b/lib/Monitor/RunFolder/Staging.pm
@@ -101,7 +101,7 @@ sub is_onboard_analysis_planned {
         if ($have_ss) {
             $planned = any { $_ =~ /^\[BCLConvert/smx } read_file($ss);
             carp sprintf 'Samplesheet %s %s the BCLConvert section',
-                $ss, $planned ? q[has] : q[dooen't have];
+                $ss, $planned ? q[has] : q[doesn't have];
         }
     }
     return $planned;

From 6f35b2897d8b69775d464b8e4570a81a598c992b Mon Sep 17 00:00:00 2001
From: jmtcsngr <jmtc@sanger.ac.uk>
Date: Fri, 2 Jun 2023 12:06:38 +0100
Subject: [PATCH 8/9] prep release 94.0.0

---
 Changes | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/Changes b/Changes
index a87ffe93a..6c16e5c92 100644
--- a/Changes
+++ b/Changes
@@ -1,5 +1,20 @@
 LIST OF CHANGES
 
+release 94.0.0
+ - Added checks for on-board analysis in progress
+ - Improved logging for a long wait case
+ - Simplified run complete and mirrored detection
+ - Simplified the staging area daemon.
+   Bring the code in accordance with the current practice:
+   - consider only one staging area,
+   - narrow down the glob pattern for areas below the staging
+     area to 'IL*'.
+ - Removed roles which defined attributes that were used
+   in one class only.
+ - Chaned the way the connection to the database is established
+   in the staging monitor. Set up the connection explicitly in
+   the script rather than inside one of the classess.
+
 release 93.0.0
  - Introduced a new instrument format database record for NovaSeqX,
    updated test data and one of the tests. 

From 198d5a0221e971c7340bb4d57ae878ac07aa18ae Mon Sep 17 00:00:00 2001
From: mgcam <mg8@sanger.ac.uk>
Date: Fri, 2 Jun 2023 12:24:20 +0100
Subject: [PATCH 9/9] Fix the type in the Changes file

---
 Changes | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Changes b/Changes
index 6c16e5c92..ea6072d31 100644
--- a/Changes
+++ b/Changes
@@ -11,8 +11,8 @@ release 94.0.0
      area to 'IL*'.
  - Removed roles which defined attributes that were used
    in one class only.
- - Chaned the way the connection to the database is established
-   in the staging monitor. Set up the connection explicitly in
+ - Changed the way the connection to the database is established
+   by the staging monitor. Set up the connection explicitly in
    the script rather than inside one of the classess.
 
 release 93.0.0