From a5862ce1122e7e6a3f58da65b554577d352cd1f5 Mon Sep 17 00:00:00 2001
From: Andrew Paxley <andrew.paxley@silverstripe.com>
Date: Tue, 19 Oct 2021 12:08:24 +1300
Subject: [PATCH] FIX releaseJobLock when marking a job as broken (#354)

FIX releaseJobLock when marking a job as broken
---
 src/Services/QueuedJob.php        | 22 +++++++++++-----------
 src/Services/QueuedJobService.php | 17 ++++++++++++++---
 2 files changed, 25 insertions(+), 14 deletions(-)

diff --git a/src/Services/QueuedJob.php b/src/Services/QueuedJob.php
index bbaffb6e..896ce895 100644
--- a/src/Services/QueuedJob.php
+++ b/src/Services/QueuedJob.php
@@ -14,32 +14,32 @@ interface QueuedJob
      * Always run immediate jobs as soon as possible
      * @var string
      */
-    const IMMEDIATE = '1';
+    public const IMMEDIATE = '1';
 
     /**
      * Queued jobs may have some processing to do, but should be pretty quick
      * @var string
      */
-    const QUEUED = '2';
+    public const QUEUED = '2';
 
     /**
      * Large jobs will take minutes, not seconds to run
      * @var string
      */
-    const LARGE = '3';
+    public const LARGE = '3';
 
     /**
      * Statuses
      * @var string
      */
-    const STATUS_NEW = 'New';
-    const STATUS_INIT = 'Initialising';
-    const STATUS_RUN = 'Running';
-    const STATUS_WAIT = 'Waiting';
-    const STATUS_COMPLETE = 'Complete';
-    const STATUS_PAUSED = 'Paused';
-    const STATUS_CANCELLED = 'Cancelled';
-    const STATUS_BROKEN = 'Broken';
+    public const STATUS_NEW = 'New';
+    public const STATUS_INIT = 'Initialising';
+    public const STATUS_RUN = 'Running';
+    public const STATUS_WAIT = 'Waiting';
+    public const STATUS_COMPLETE = 'Complete';
+    public const STATUS_PAUSED = 'Paused';
+    public const STATUS_CANCELLED = 'Cancelled';
+    public const STATUS_BROKEN = 'Broken';
 
     /**
      * Gets a title for the job that can be used in listings
diff --git a/src/Services/QueuedJobService.php b/src/Services/QueuedJobService.php
index 7e1625ad..9d0d0088 100644
--- a/src/Services/QueuedJobService.php
+++ b/src/Services/QueuedJobService.php
@@ -911,7 +911,7 @@ public function runJob($jobId)
                             $job->process();
                         } catch (\Throwable $e) {
                             $logger->error($e->getMessage(), ['exception' => $e]);
-                            $jobDescriptor->JobStatus =  QueuedJob::STATUS_BROKEN;
+                            $this->markJobAsBroken($jobDescriptor);
                             $this->extend('updateJobDescriptorAndJobOnException', $jobDescriptor, $job, $e);
                         }
 
@@ -930,7 +930,7 @@ public function runJob($jobId)
                                 'Job stalled after {attempts} attempts - please check',
                                 ['attempts' => $stallCount]
                             ));
-                            $jobDescriptor->JobStatus = QueuedJob::STATUS_BROKEN;
+                            $this->markJobAsBroken($jobDescriptor);
                         }
 
                         // now we'll be good and check our memory usage. If it is too high, we'll set the job to
@@ -1056,7 +1056,7 @@ protected function handleBrokenJobException(QueuedJobDescriptor $jobDescriptor,
                 'exception' => $e,
             ]
         );
-        $jobDescriptor->JobStatus =  QueuedJob::STATUS_BROKEN;
+        $this->markJobAsBroken($jobDescriptor);
         $this->extend('updateJobDescriptorAndJobOnException', $jobDescriptor, $job, $e);
         $jobDescriptor->write();
     }
@@ -1516,6 +1516,17 @@ protected function releaseJobLock(QueuedJobDescriptor $descriptor): void
         $descriptor->Worker = null;
     }
 
+    /**
+     * Mark a Job as Broken and release the lock so it can be resumed
+     *
+     * @param QueuedJobDescriptor $descriptor
+     */
+    protected function markJobAsBroken(QueuedJobDescriptor $descriptor): void
+    {
+        $descriptor->JobStatus = QueuedJob::STATUS_BROKEN;
+        $this->releaseJobLock($descriptor);
+    }
+
     /**
      * @return string
      */