Skip to content

Commit

Permalink
_status_monitor: still record the exception during shm write with bur…
Browse files Browse the repository at this point in the history
…st_limited_logger; use global shutdown cleanup
  • Loading branch information
Bodong-Yang committed Dec 5, 2024
1 parent b771c3f commit 85de82e
Showing 1 changed file with 20 additions and 2 deletions.
22 changes: 20 additions & 2 deletions src/otaclient/_status_monitor.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
from __future__ import annotations

import atexit
import contextlib
import logging
import queue
import time
Expand All @@ -36,8 +35,19 @@
UpdateTiming,
)
from otaclient._utils import SharedOTAClientStatusWriter
from otaclient_common.logging import BurstSuppressFilter

logger = logging.getLogger(__name__)
burst_suppressed_logger = logging.getLogger(f"{__name__}.shm_push")
# NOTE: for request_error, only allow max 6 lines of logging per 30 seconds
burst_suppressed_logger.addFilter(
BurstSuppressFilter(
f"{__name__}.shm_push",
upper_logger_name=__name__,
burst_round_length=30,
burst_max=6,
)
)

_status_report_queue: queue.Queue | None = None

Expand Down Expand Up @@ -232,6 +242,7 @@ def _on_update_meta(status_storage: OTAClientStatus, payload: SetUpdateMetaRepor


class OTAClientStatusCollector:
"""NOTE: status_monitor should only be started once during whole otaclient lifecycle!"""

def __init__(
self,
Expand All @@ -245,6 +256,9 @@ def __init__(
self.shm_push_interval = shm_push_interval

self._input_queue = msg_queue
global _status_report_queue
_status_report_queue = msg_queue

self._status = None
self._shm_status = shm_status

Expand Down Expand Up @@ -295,9 +309,13 @@ def _status_collector_thread(self) -> None:

# ------ push status on load_report ------ #
if self.load_report(report) and self._status and _now > _next_shm_push:
with contextlib.suppress(Exception):
try:
self._shm_status.write_msg(self._status)
_next_shm_push = _now + self.shm_push_interval
except Exception as e:
burst_suppressed_logger.warning(
f"failed to push status to shm: {e!r}"
)
except queue.Empty:
time.sleep(self.min_collect_interval)

Expand Down

0 comments on commit 85de82e

Please sign in to comment.