refactor(v3.6.x): cleanup iot-logger related

otaclient/app/log_setting.py

@@ -12,9 +12,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
+from __future__ import annotations
+
 import logging
 import os
+import time
 import yaml
+from queue import Queue, Empty
+from threading import Thread
+from urllib.parse import urljoin
+
+import requests
 
 from otaclient import otaclient_package_name
 from .configs import config as cfg
@@ -37,7 +46,53 @@ def get_logger(name: str, loglevel: int) -> logging.Logger:
     return logger
 
 
-def configure_logging(loglevel: int, *, http_logging_url: str):
+class _LogTeeHandler(logging.Handler):
+    """Implementation of teeing local logs to a remote otaclient-iot-logger server."""
+
+    def __init__(
+        self,
+        upload_interval: int = 30,
+        max_entries_per_upload: int = 512,
+        max_backlog: int = 2048,
+    ) -> None:
+        super().__init__()
+        self._queue: Queue[str] = Queue(maxsize=max_backlog)
+        self._max_entries_per_upload = max_entries_per_upload
+        self._upload_interval = upload_interval
+
+    def emit(self, record: logging.LogRecord) -> None:
+        try:
+            self._queue.put_nowait(self.format(record))
+        except Exception:
+            pass
+
+    def start_upload_thread(self, endpoint_url: str):
+        _queue, _max_per_upload, _interval = (
+            self._queue,
+            self._max_entries_per_upload,
+            self._upload_interval,
+        )
+
+        def _thread_main():
+            _session = requests.Session()
+
+            while True:
+                for _ in range(_max_per_upload):
+                    try:
+                        _entry = _queue.get_nowait()
+                        _session.post(endpoint_url, data=_entry)
+                    except Empty:
+                        break
+                    except Exception:
+                        pass
+                time.sleep(_interval)
+
+        _thread = Thread(target=_thread_main, daemon=True)
+        _thread.start()
+        return _thread
+
+
+def configure_logging(loglevel: int, *, ecu_id: str):
     """Configure logging with http handler."""
     # configure the root logger
     # NOTE: force to reload the basicConfig, this is for overriding setting
@@ -49,15 +104,17 @@ def configure_logging(loglevel: int, *, http_logging_url: str):
     _otaclient_logger = logging.getLogger(otaclient_package_name)
     _otaclient_logger.setLevel(loglevel)
 
-    # if http_logging is enabled, attach the http handler to
-    # the otaclient package root logger
-    if http_logging_host := os.environ.get("HTTP_LOGGING_SERVER"):
-        from otaclient.aws_iot_log_server import CustomHttpHandler
+    if iot_logger_url := os.environ.get("HTTP_LOGGING_SERVER"):
+        iot_logger_url = f"{iot_logger_url.rstrip('/')}/"
+        log_upload_endpoint = urljoin(iot_logger_url, ecu_id)
 
-        ch = CustomHttpHandler(host=http_logging_host, url=http_logging_url)
+        ch = _LogTeeHandler()
         fmt = logging.Formatter(fmt=cfg.LOG_FORMAT)
         ch.setFormatter(fmt)
 
+        # star the logging thread
+        ch.start_upload_thread(log_upload_endpoint)
+
         # NOTE: "otaclient" logger will be the root logger for all loggers name
         #       starts with "otaclient.", and the settings will affect its child loggers.
         #       For example, settings for "otaclient" logger will also be effective to

otaclient/app/main.py

@@ -27,7 +27,7 @@
 from .ota_client_service import launch_otaclient_grpc_server
 
 # configure logging before any code being executed
-configure_logging(loglevel=cfg.DEFAULT_LOG_LEVEL, http_logging_url=get_ecu_id())
+configure_logging(loglevel=cfg.DEFAULT_LOG_LEVEL, ecu_id=get_ecu_id())
 logger = logging.getLogger(__name__)
 
 

otaclient/app/ota_client_stub.py

@@ -90,7 +90,7 @@ def _subprocess_init(self):
         #       to CRITICAL to filter out third_party libs' logging(requests, urllib3, etc.),
         #       and then set the otaclient.ota_proxy logger to DEFAULT_LOG_LEVEL
         log_setting.configure_logging(
-            loglevel=logging.CRITICAL, http_logging_url=log_setting.get_ecu_id()
+            loglevel=logging.CRITICAL, ecu_id=log_setting.get_ecu_id()
         )
         otaproxy_logger = logging.getLogger("otaclient.ota_proxy")
         otaproxy_logger.setLevel(cfg.DEFAULT_LOG_LEVEL)