feat(timer): Allow limiting the work done by dispatch

Sometimes it is required that TimerQueue::dispatch finishes fast in a
bounded amount of time while still making progress on the queued timers.
In which case, you want dispatch to process a small number of queued
timers and then return back to caller.

Furthermore, limit the work done by low priority timer queue to just 1
in the reactor. This prevents too much time being taken by low priority
timers if there are a lot of them eligible to be dispatched.
They are low priority timers, so its ok if they are delayed by a little
bit of time.

SDB-8174

toolbox/io/Reactor.cpp

@@ -95,7 +95,7 @@ int Reactor::poll(CyclTime now, Duration timeout)
     work += dispatch(now, buf, n);
     // Low priority timers are only dispatched during empty cycles.
     if (work == 0) {
-        work += tqs_[Low].dispatch(now);
+        work += tqs_[Low].dispatch(now, 1);
     }
     // End of cycle hooks.
     if (work > 0) {

toolbox/io/Timer.cpp

@@ -76,25 +76,25 @@ Timer TimerQueue::insert(MonoTime expiry, Duration interval, TimerSlot slot)
     return tmr;
 }
 
-int TimerQueue::dispatch(CyclTime now)
+int TimerQueue::dispatch(CyclTime now, int max_work)
 {
-    int work{};
-    while (!heap_.empty()) {
-
+    int timers_processed{0};
+    for (int i = 0; (i < max_work) && (!heap_.empty()); i++) {
         // If not pending, then must have been cancelled.
         if (!heap_.front().pending()) {
             pop();
             --cancelled_;
             assert(cancelled_ >= 0);
         } else if (heap_.front().expiry() <= now.mono_time()) {
             expire(now);
-            ++work;
+            ++timers_processed;
         } else {
             break;
         }
     }
+
     gc();
-    return work;
+    return timers_processed;
 }
 
 Timer TimerQueue::allocate(MonoTime expiry, Duration interval, TimerSlot slot)

toolbox/io/Timer.hpp

@@ -17,6 +17,7 @@
 #ifndef TOOLBOX_IO_TIMER_HPP
 #define TOOLBOX_IO_TIMER_HPP
 
+#include <limits>
 #include <toolbox/sys/Time.hpp>
 #include <toolbox/util/Slot.hpp>
 
@@ -167,7 +168,7 @@ class TOOLBOX_API TimerQueue {
     }
     // clang-format on
 
-    int dispatch(CyclTime now);
+    int dispatch(CyclTime now, int max_work = std::numeric_limits<int>::max());
 
   private:
     Timer allocate(MonoTime expiry, Duration interval, TimerSlot slot);