From 85de82e8ba711890b521080e3a3ccd8251ec1b07 Mon Sep 17 00:00:00 2001
From: "bodong.yang" <bodong.yang@tier4.jp>
Date: Thu, 5 Dec 2024 03:13:16 +0000
Subject: [PATCH] _status_monitor: still record the exception during shm write
 with burst_limited_logger; use global shutdown cleanup

---
 src/otaclient/_status_monitor.py | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/src/otaclient/_status_monitor.py b/src/otaclient/_status_monitor.py
index 5626ec357..757fda592 100644
--- a/src/otaclient/_status_monitor.py
+++ b/src/otaclient/_status_monitor.py
@@ -17,7 +17,6 @@
 from __future__ import annotations
 
 import atexit
-import contextlib
 import logging
 import queue
 import time
@@ -36,8 +35,19 @@
     UpdateTiming,
 )
 from otaclient._utils import SharedOTAClientStatusWriter
+from otaclient_common.logging import BurstSuppressFilter
 
 logger = logging.getLogger(__name__)
+burst_suppressed_logger = logging.getLogger(f"{__name__}.shm_push")
+# NOTE: for request_error, only allow max 6 lines of logging per 30 seconds
+burst_suppressed_logger.addFilter(
+    BurstSuppressFilter(
+        f"{__name__}.shm_push",
+        upper_logger_name=__name__,
+        burst_round_length=30,
+        burst_max=6,
+    )
+)
 
 _status_report_queue: queue.Queue | None = None
 
@@ -232,6 +242,7 @@ def _on_update_meta(status_storage: OTAClientStatus, payload: SetUpdateMetaRepor
 
 
 class OTAClientStatusCollector:
+    """NOTE: status_monitor should only be started once during whole otaclient lifecycle!"""
 
     def __init__(
         self,
@@ -245,6 +256,9 @@ def __init__(
         self.shm_push_interval = shm_push_interval
 
         self._input_queue = msg_queue
+        global _status_report_queue
+        _status_report_queue = msg_queue
+
         self._status = None
         self._shm_status = shm_status
 
@@ -295,9 +309,13 @@ def _status_collector_thread(self) -> None:
 
                 # ------ push status on load_report ------ #
                 if self.load_report(report) and self._status and _now > _next_shm_push:
-                    with contextlib.suppress(Exception):
+                    try:
                         self._shm_status.write_msg(self._status)
                         _next_shm_push = _now + self.shm_push_interval
+                    except Exception as e:
+                        burst_suppressed_logger.warning(
+                            f"failed to push status to shm: {e!r}"
+                        )
             except queue.Empty:
                 time.sleep(self.min_collect_interval)