Process jobs with expired claims

app/models/disqualified/record.rb

@@ -15,8 +15,12 @@ class Disqualified::Record < Disqualified::BaseRecord
     joins("LEFT OUTER JOIN disqualified_sequences ds ON ds.uuid = sequence_uuid AND ds.current_step = sequence_step")
       .where("ds.uuid = sequence_uuid OR (ds.uuid IS NULL AND sequence_uuid IS NULL)")
   }
-  scope :pending, -> { where(finished_at: nil, run_at: (..Time.now), locked_by: nil) }
-  scope :runnable, -> { with_sequence.pending }
+  scope :runnable, -> do
+    with_sequence
+      .where(finished_at: nil, run_at: ..Time.now)
+      .and(where(locked_by: nil).or(where(locked_at: ..10.minutes.ago)))
+      .order(run_at: :asc)
+  end
 
   sig do
     params(id: T.nilable(T.any(Integer, String))).returns(Disqualified::Record)
@@ -26,7 +30,6 @@ def self.claim_one!(id: nil)
     association =
       Disqualified::Record
         .runnable
-        .order(run_at: :asc)
         .limit(1)
 
     if id

test/models/disqualified/record_test.rb

@@ -54,6 +54,15 @@ def perform
     end
   end
 
+  test ".runnable returns non-finished jobs with expired claim" do
+    travel_to(2.days.ago) do
+      NoArgJob.perform_async
+      Disqualified::Record.claim_one!
+      assert_equal(0, Disqualified::Record.runnable.size)
+    end
+    assert_equal(1, Disqualified::Record.runnable.size)
+  end
+
   test "#run! doesn't run ran jobs" do
     NoArgJob.perform_async
     record = Disqualified::Record.runnable.first
@@ -87,4 +96,24 @@ def perform
       end
     end
   end
+
+  test "#run! runs jobs with expired claims" do
+    claimed_id = nil
+    travel_to(2.days.ago) do
+      NoArgJob.perform_async
+      record = Disqualified::Record.claim_one!
+      claimed_id = record.id
+    end
+    expired = Disqualified::Record.first
+    expired.run!
+    assert_equal(claimed_id, expired.id)
+  end
+
+  test "#run! doesn't run currently claimed jobs" do
+    NoArgJob.perform_async
+    record = Disqualified::Record.claim_one!
+    assert_raise(Disqualified::Error::NoClaimableJob) do
+      Disqualified::Record.find(record.id).run!
+    end
+  end
 end