diff --git a/otaclient/app/boot_control/_cboot.py b/otaclient/app/boot_control/_cboot.py index 512789b3b..cb760076f 100644 --- a/otaclient/app/boot_control/_cboot.py +++ b/otaclient/app/boot_control/_cboot.py @@ -13,6 +13,7 @@ # limitations under the License. +import logging import os import re from pathlib import Path @@ -21,7 +22,7 @@ from typing import Generator, Optional -from .. import log_setting, errors as ota_errors +from .. import errors as ota_errors from ..common import ( copytree_identical, read_str_from_file, @@ -44,9 +45,7 @@ from .firmware import Firmware -logger = log_setting.get_logger( - __name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL) -) +logger = logging.getLogger(__name__) class NvbootctrlError(Exception): @@ -178,7 +177,9 @@ def _init_dev_info(self): self.current_slot: str = Nvbootctrl.get_current_slot() self.current_rootfs_dev: str = CMDHelperFuncs.get_current_rootfs_dev() # NOTE: boot dev is always emmc device now - self.current_boot_dev: str = f"/dev/{Nvbootctrl.EMMC_DEV}p{Nvbootctrl.SLOTID_PARTID_MAP[self.current_slot]}" + self.current_boot_dev: str = ( + f"/dev/{Nvbootctrl.EMMC_DEV}p{Nvbootctrl.SLOTID_PARTID_MAP[self.current_slot]}" + ) self.standby_slot: str = Nvbootctrl.CURRENT_STANDBY_FLIP[self.current_slot] standby_partid = Nvbootctrl.SLOTID_PARTID_MAP[self.standby_slot] diff --git a/otaclient/app/boot_control/_common.py b/otaclient/app/boot_control/_common.py index be3444ae8..0ba392b35 100644 --- a/otaclient/app/boot_control/_common.py +++ b/otaclient/app/boot_control/_common.py @@ -14,6 +14,7 @@ r"""Shared utils for boot_controller.""" +import logging import os import shutil import sys @@ -28,7 +29,6 @@ MountFailedReason, ) -from .. import log_setting from ..configs import config as cfg from ..common import ( read_str_from_file, @@ -39,9 +39,7 @@ from ..proto import wrapper -logger = log_setting.get_logger( - __name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL) -) +logger = logging.getLogger(__name__) class CMDHelperFuncs: diff --git a/otaclient/app/boot_control/_grub.py b/otaclient/app/boot_control/_grub.py index 8c3bf8971..f4678f248 100644 --- a/otaclient/app/boot_control/_grub.py +++ b/otaclient/app/boot_control/_grub.py @@ -31,6 +31,7 @@ """ +import logging import re import shutil from dataclasses import dataclass @@ -39,7 +40,7 @@ from pathlib import Path from pprint import pformat -from .. import log_setting, errors as ota_errors +from .. import errors as ota_errors from ..common import ( re_symlink_atomic, read_str_from_file, @@ -59,9 +60,7 @@ from .protocol import BootControllerProtocol -logger = log_setting.get_logger( - __name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL) -) +logger = logging.getLogger(__name__) class _GrubBootControllerError(Exception): diff --git a/otaclient/app/boot_control/_rpi_boot.py b/otaclient/app/boot_control/_rpi_boot.py index 7317a48de..46c4a2bdd 100644 --- a/otaclient/app/boot_control/_rpi_boot.py +++ b/otaclient/app/boot_control/_rpi_boot.py @@ -14,13 +14,14 @@ """Boot control support for Raspberry pi 4 Model B.""" +import logging import os import re from string import Template from pathlib import Path from typing import Generator -from .. import log_setting, errors as ota_errors +from .. import errors as ota_errors from ..proto import wrapper from ..common import replace_atomic, subprocess_call @@ -33,9 +34,7 @@ from .configs import rpi_boot_cfg as cfg from .protocol import BootControllerProtocol -logger = log_setting.get_logger( - __name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL) -) +logger = logging.getLogger(__name__) _FSTAB_TEMPLATE_STR = ( "LABEL=${rootfs_fslabel}\t/\text4\tdiscard,x-systemd.growfs\t0\t1\n" diff --git a/otaclient/app/boot_control/firmware.py b/otaclient/app/boot_control/firmware.py index 67a819f63..2384c4bca 100644 --- a/otaclient/app/boot_control/firmware.py +++ b/otaclient/app/boot_control/firmware.py @@ -13,16 +13,13 @@ # limitations under the License. +import logging import yaml import zstandard from pathlib import Path from typing import Dict, Callable -from ..configs import config as cfg -from .. import log_setting -logger = log_setting.get_logger( - __name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL) -) +logger = logging.getLogger(__name__) class Firmware: diff --git a/otaclient/app/boot_control/selecter.py b/otaclient/app/boot_control/selecter.py index 2a4edfa97..ab4f91224 100644 --- a/otaclient/app/boot_control/selecter.py +++ b/otaclient/app/boot_control/selecter.py @@ -13,6 +13,7 @@ # limitations under the License. +import logging import platform from pathlib import Path from typing import Type @@ -21,13 +22,9 @@ from ._errors import BootControlError from .protocol import BootControllerProtocol -from ..configs import config as cfg from ..common import read_str_from_file -from .. import log_setting -logger = log_setting.get_logger( - __name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL) -) +logger = logging.getLogger(__name__) def detect_bootloader(raise_on_unknown=True) -> BootloaderType: diff --git a/otaclient/app/common.py b/otaclient/app/common.py index 6351ebcd9..4355f48c2 100644 --- a/otaclient/app/common.py +++ b/otaclient/app/common.py @@ -16,6 +16,7 @@ from __future__ import annotations import itertools +import logging import os import shlex import shutil @@ -42,7 +43,6 @@ ) from urllib.parse import urljoin -from .log_setting import get_logger from .configs import config as cfg from otaclient._utils.linux import ( @@ -52,7 +52,7 @@ ParsedPasswd, ) -logger = get_logger(__name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL)) +logger = logging.getLogger(__name__) def get_backoff(n: int, factor: float, _max: float) -> float: diff --git a/otaclient/app/create_standby/__init__.py b/otaclient/app/create_standby/__init__.py index f90c07972..2b885b1c9 100644 --- a/otaclient/app/create_standby/__init__.py +++ b/otaclient/app/create_standby/__init__.py @@ -13,15 +13,13 @@ # limitations under the License. +import logging from typing import Type from .interface import StandbySlotCreatorProtocol -from ..configs import CreateStandbyMechanism, config as cfg -from .. import log_setting +from ..configs import CreateStandbyMechanism -logger = log_setting.get_logger( - __name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL) -) +logger = logging.getLogger(__name__) def get_standby_slot_creator( diff --git a/otaclient/app/create_standby/common.py b/otaclient/app/create_standby/common.py index ae28dcd7e..04b74d7dd 100644 --- a/otaclient/app/create_standby/common.py +++ b/otaclient/app/create_standby/common.py @@ -11,9 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +r"""Common used helpers, classes and functions for different bank creating methods.""" -r"""Common used helpers, classes and functions for different bank creating methods.""" +import logging import os import random import time @@ -43,16 +44,13 @@ from ..configs import config as cfg from ..ota_metadata import OTAMetadata, MetafilesV1 from ..proto.wrapper import RegularInf, DirectoryInf -from .. import log_setting from ..update_stats import ( OTAUpdateStatsCollector, RegProcessOperation, RegInfProcessedStats, ) -logger = log_setting.get_logger( - __name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL) -) +logger = logging.getLogger(__name__) class _WeakRef: diff --git a/otaclient/app/create_standby/rebuild_mode.py b/otaclient/app/create_standby/rebuild_mode.py index f4e7950e1..44e4b9b3c 100644 --- a/otaclient/app/create_standby/rebuild_mode.py +++ b/otaclient/app/create_standby/rebuild_mode.py @@ -13,6 +13,7 @@ # limitations under the License. +import logging import os import shutil import time @@ -29,14 +30,11 @@ RegProcessOperation, ) from ..proto.wrapper import RegularInf -from .. import log_setting from .common import HardlinkRegister, DeltaGenerator, DeltaBundle from .interface import StandbySlotCreatorProtocol -logger = log_setting.get_logger( - __name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL) -) +logger = logging.getLogger(__name__) class RebuildMode(StandbySlotCreatorProtocol): diff --git a/otaclient/app/downloader.py b/otaclient/app/downloader.py index 29e75d879..88317b044 100644 --- a/otaclient/app/downloader.py +++ b/otaclient/app/downloader.py @@ -14,6 +14,7 @@ import errno +import logging import os import requests import threading @@ -50,11 +51,8 @@ from otaclient.ota_proxy import OTAFileCacheControl from .configs import config as cfg from .common import wait_with_backoff -from . import log_setting -logger = log_setting.get_logger( - __name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL) -) +logger = logging.getLogger(__name__) EMPTY_FILE_SHA256 = r"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" CACHE_CONTROL_HEADER = OTAFileCacheControl.HEADER_LOWERCASE diff --git a/otaclient/app/ecu_info.py b/otaclient/app/ecu_info.py index e89f7a4e9..0adb78134 100644 --- a/otaclient/app/ecu_info.py +++ b/otaclient/app/ecu_info.py @@ -13,20 +13,18 @@ # limitations under the License. r"""ECU metadatas definition.""" + +import logging import yaml from copy import deepcopy from dataclasses import dataclass, field, fields, MISSING from pathlib import Path from typing import Iterator, NamedTuple, Union, Dict, List, Any -from . import log_setting -from .configs import config as cfg, server_cfg +from .configs import server_cfg from .boot_control import BootloaderType -logger = log_setting.get_logger( - __name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL) -) - +logger = logging.getLogger(__name__) DEFAULT_ECU_INFO = { "format_version": 1, # current version is 1 diff --git a/otaclient/app/log_setting.py b/otaclient/app/log_setting.py index 5d1b4a26c..b4dab0582 100644 --- a/otaclient/app/log_setting.py +++ b/otaclient/app/log_setting.py @@ -18,37 +18,16 @@ import atexit import logging import os -import weakref import yaml from queue import Queue -from threading import Thread +from threading import Event, Thread from urllib.parse import urljoin -from typing import MutableMapping import requests from otaclient import otaclient_package_name from .configs import config as cfg -_logging_running = True -_logging_upload_thread: MutableMapping[ - Thread, Queue[str | None] -] = weakref.WeakKeyDictionary() - - -def _python_exit(): - """Let the log upload thread exit at python exit.""" - global _logging_running - _logging_running = False - - # unblock the thread - for t, q in _logging_upload_thread.items(): - q.put_nowait(None) - t.join(timeout=3) - - -atexit.register(_python_exit) - # NOTE: EcuInfo imports this log_setting so independent get_ecu_id are required. def get_ecu_id(): @@ -60,13 +39,6 @@ def get_ecu_id(): return "autoware" -def get_logger(name: str, loglevel: int) -> logging.Logger: - """Helper method to get logger with name and loglevel.""" - logger = logging.getLogger(name) - logger.setLevel(loglevel) - return logger - - class _LogTeeHandler(logging.Handler): """Implementation of teeing local logs to a remote otaclient-iot-logger server.""" @@ -81,14 +53,14 @@ def emit(self, record: logging.LogRecord) -> None: pass def start_upload_thread(self, endpoint_url: str): - _queue = self._queue + log_queue = self._queue + stop_logging_upload = Event() def _thread_main(): _session = requests.Session() - global _logging_running - while _logging_running: - entry = _queue.get() + while not stop_logging_upload.is_set(): + entry = log_queue.get() if entry is None: return # stop signal if not entry: @@ -99,15 +71,18 @@ def _thread_main(): except Exception: pass - _thread = Thread(target=_thread_main, daemon=True) - _thread.start() + log_upload_thread = Thread(target=_thread_main, daemon=True) + log_upload_thread.start() - # register the logging upload thread - global _logging_upload_thread - _logging_upload_thread[_thread] = _queue + def _thread_exit(): + stop_logging_upload.set() + log_queue.put_nowait(None) + log_upload_thread.join(6) + atexit.register(_thread_exit) -def configure_logging(loglevel: int, *, ecu_id: str): + +def configure_logging(): """Configure logging with http handler.""" # configure the root logger # NOTE: force to reload the basicConfig, this is for overriding setting @@ -117,7 +92,12 @@ def configure_logging(loglevel: int, *, ecu_id: str): logging.basicConfig(level=logging.CRITICAL, format=cfg.LOG_FORMAT, force=True) # NOTE: set the to the otaclient package root logger _otaclient_logger = logging.getLogger(otaclient_package_name) - _otaclient_logger.setLevel(loglevel) + _otaclient_logger.setLevel(cfg.DEFAULT_LOG_LEVEL) + + # configure each sub loggers + for _module_name, _log_level in cfg.LOG_LEVEL_TABLE.items(): + _logger = logging.getLogger(_module_name) + _logger.setLevel(_log_level) # NOTE(20240306): for only god knows reason, although in proxy_info.yaml, # the logging_server field is assigned with an URL, and otaclient @@ -137,7 +117,7 @@ def configure_logging(loglevel: int, *, ecu_id: str): ch.setFormatter(fmt) # star the logging thread - log_upload_endpoint = urljoin(iot_logger_url, ecu_id) + log_upload_endpoint = urljoin(iot_logger_url, get_ecu_id()) ch.start_upload_thread(log_upload_endpoint) # NOTE: "otaclient" logger will be the root logger for all loggers name diff --git a/otaclient/app/main.py b/otaclient/app/main.py index 08296cfe1..58c0b3ab0 100644 --- a/otaclient/app/main.py +++ b/otaclient/app/main.py @@ -23,11 +23,11 @@ from .proto import wrapper, v2, v2_grpc, ota_metafiles # noqa: F401 from .common import read_str_from_file, write_str_to_file_sync from .configs import config as cfg, EXTRA_VERSION_FILE -from .log_setting import configure_logging, get_ecu_id +from .log_setting import configure_logging from .ota_client_service import launch_otaclient_grpc_server # configure logging before any code being executed -configure_logging(loglevel=cfg.DEFAULT_LOG_LEVEL, ecu_id=get_ecu_id()) +configure_logging() logger = logging.getLogger(__name__) diff --git a/otaclient/app/ota_client.py b/otaclient/app/ota_client.py index eb3f144d8..1c1baace6 100644 --- a/otaclient/app/ota_client.py +++ b/otaclient/app/ota_client.py @@ -17,6 +17,7 @@ import asyncio import gc import json +import logging import time import threading from concurrent.futures import ThreadPoolExecutor @@ -46,7 +47,6 @@ RegInfProcessedStats, RegProcessOperation, ) -from . import log_setting from otaclient._utils import get_file_size from otaclient._utils.linux import create_swapfile @@ -56,9 +56,7 @@ except ImportError: __version__ = "unknown" -logger = log_setting.get_logger( - __name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL) -) +logger = logging.getLogger(__name__) class OTAClientControlFlags: diff --git a/otaclient/app/ota_client_stub.py b/otaclient/app/ota_client_stub.py index cc1444ab9..0afb9d45a 100644 --- a/otaclient/app/ota_client_stub.py +++ b/otaclient/app/ota_client_stub.py @@ -42,9 +42,7 @@ ) -logger = log_setting.get_logger( - __name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL) -) +logger = logging.getLogger(__name__) class _OTAProxyContext(OTAProxyContextProto): @@ -89,9 +87,7 @@ def _subprocess_init(self): # NOTE: on otaproxy subprocess, we first set log level of the root logger # to CRITICAL to filter out third_party libs' logging(requests, urllib3, etc.), # and then set the otaclient.ota_proxy logger to DEFAULT_LOG_LEVEL - log_setting.configure_logging( - loglevel=logging.CRITICAL, ecu_id=log_setting.get_ecu_id() - ) + log_setting.configure_logging() otaproxy_logger = logging.getLogger("otaclient.ota_proxy") otaproxy_logger.setLevel(cfg.DEFAULT_LOG_LEVEL) self.logger = otaproxy_logger diff --git a/otaclient/app/ota_metadata.py b/otaclient/app/ota_metadata.py index a69d945c4..33856bd9d 100644 --- a/otaclient/app/ota_metadata.py +++ b/otaclient/app/ota_metadata.py @@ -40,6 +40,7 @@ from __future__ import annotations import base64 import json +import logging import re import shutil import time @@ -85,11 +86,8 @@ SymbolicLinkInf, ) from .proto.streamer import Uint32LenDelimitedMsgReader, Uint32LenDelimitedMsgWriter -from . import log_setting -logger = log_setting.get_logger( - __name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL) -) +logger = logging.getLogger(__name__) CACHE_CONTROL_HEADER = OTAFileCacheControl.HEADER_LOWERCASE diff --git a/otaclient/app/ota_status.py b/otaclient/app/ota_status.py index 3ce9d699f..53ff85c8f 100644 --- a/otaclient/app/ota_status.py +++ b/otaclient/app/ota_status.py @@ -13,13 +13,11 @@ # limitations under the License. -from .configs import config as cfg +import logging + from .proto import wrapper -from . import log_setting -logger = log_setting.get_logger( - __name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL) -) +logger = logging.getLogger(__name__) class LiveOTAStatus: diff --git a/otaclient/app/proxy_info.py b/otaclient/app/proxy_info.py index 010ccbd83..6c33b2026 100644 --- a/otaclient/app/proxy_info.py +++ b/otaclient/app/proxy_info.py @@ -11,26 +11,23 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - - """Proxy setting parsing. check docs/README.md for more details. """ + + +import logging import yaml import warnings from dataclasses import dataclass, fields from typing import Any, ClassVar, Dict from pathlib import Path -from . import log_setting from .configs import config as cfg from .configs import server_cfg -logger = log_setting.get_logger( - __name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL) -) - +logger = logging.getLogger(__name__) PRE_DEFINED_PROXY_INFO_YAML = """ enable_local_ota_proxy: true diff --git a/otaclient/app/update_stats.py b/otaclient/app/update_stats.py index 6b5c820ee..70da00afe 100644 --- a/otaclient/app/update_stats.py +++ b/otaclient/app/update_stats.py @@ -13,6 +13,7 @@ # limitations under the License. +import logging import time from dataclasses import dataclass from enum import Enum @@ -21,14 +22,11 @@ from threading import Event, Lock, Thread from typing import Generator, List -from . import log_setting from .configs import config as cfg from .proto.wrapper import UpdateStatus -logger = log_setting.get_logger( - __name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL) -) +logger = logging.getLogger(__name__) class RegProcessOperation(Enum):