feat: re-implement otaclient configs, add support for dynamic root an…

…d runtime configurable settings (#398)

Re-implement the otaclient.app.configs into otaclient.configs, add support for dynamic root and runtime configurable settings.
The dynamic root feature is needed for enabling otaclient-in-container in the future.
Runtime configurable settings feature allows the user to configure the otaclient's behavior via environmental variables. 

Other changes include:
1. now importing otaclient.configs module and its sub modules with _ prefix will not have side-effect(i.e., ecu_info.yaml being loaded, etc.), the configs loading are done by the otaclient.configs.cfg module.
2. cleanup, re-arrange, merge and simplify some of the settings.
3. ecu_info and proxy_info config objects now are exposed via otaclient.configs.cfg.

src/otaclient/app/configs.py

@@ -17,8 +17,10 @@
 from logging import INFO
 from typing import Dict, Tuple
 
-from otaclient.configs.ecu_info import ecu_info  # noqa
-from otaclient.configs.proxy_info import proxy_info  # noqa
+from otaclient.configs.cfg import (
+    ecu_info,  # noqa
+    proxy_info,  # noqa
+)
 
 
 class CreateStandbyMechanism(Enum):

src/otaclient/app/ota_client_stub.py

@@ -32,13 +32,14 @@
 from ota_proxy import config as local_otaproxy_cfg
 from otaclient import log_setting
 from otaclient.boot_control._common import CMDHelperFuncs
-from otaclient.configs.ecu_info import ECUContact
+from otaclient.configs import ECUContact
+from otaclient.configs.cfg import ecu_info, proxy_info
 from otaclient_api.v2 import types as api_types
 from otaclient_api.v2.api_caller import ECUNoResponse, OTAClientCall
 from otaclient_common.common import ensure_otaproxy_start
 
 from .configs import config as cfg
-from .configs import ecu_info, proxy_info, server_cfg
+from .configs import server_cfg
 from .ota_client import OTAClientControlFlags, OTAServicer
 
 logger = logging.getLogger(__name__)

src/otaclient/boot_control/configs.py

@@ -18,7 +18,7 @@
 from dataclasses import dataclass
 
 from otaclient.app.configs import BaseConfig
-from otaclient.configs.ecu_info import BootloaderType
+from otaclient.configs import BootloaderType
 
 
 @dataclass

src/otaclient/configs/__init__.py

@@ -12,3 +12,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """otaclient configs package."""
+
+from otaclient.configs._cfg_configurable import (
+    ENV_PREFIX,
+    ConfigurableSettings,
+    set_configs,
+)
+from otaclient.configs._cfg_consts import Consts, CreateStandbyMechanism, dynamic_root
+from otaclient.configs._ecu_info import BootloaderType, ECUContact, ECUInfo
+from otaclient.configs._proxy_info import ProxyInfo
+
+__all__ = [
+    "ENV_PREFIX",
+    "ConfigurableSettings",
+    "Consts",
+    "CreateStandbyMechanism",
+    "BootloaderType",
+    "ECUContact",
+    "ECUInfo",
+    "ProxyInfo",
+    "set_configs",
+    "dynamic_root",
+]

src/otaclient/configs/_cfg_configurable.py

@@ -0,0 +1,134 @@
+# Copyright 2022 TIER IV, INC. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Runtime configurable configs for otaclient."""
+
+
+from __future__ import annotations
+
+import logging
+from typing import Dict, Literal
+
+from pydantic import BaseModel
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+from otaclient.configs._cfg_consts import cfg_consts
+
+logger = logging.getLogger(__name__)
+
+ENV_PREFIX = "OTACLIENT_"
+LOG_LEVEL_LITERAL = Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
+CREATE_STANDBY_METHOD_LTIERAL = Literal["REBUILD", "IN_PLACE"]
+
+
+class _OTAClientSettings(BaseModel):
+    #
+    # ------ logging settings ------ #
+    #
+    DEFAULT_LOG_LEVEL: LOG_LEVEL_LITERAL = "INFO"
+    LOG_LEVEL_TABLE: Dict[str, LOG_LEVEL_LITERAL] = {
+        "ota_metadata": "INFO",
+        "otaclient": "INFO",
+        "otaclient_api": "INFO",
+        "otaclient_common": "INFO",
+        "otaproxy": "INFO",
+    }
+    LOG_FORMAT: str = (
+        "[%(asctime)s][%(levelname)s]-%(name)s:%(funcName)s:%(lineno)d,%(message)s"
+    )
+
+    #
+    # ------ downloading settings ------ #
+    #
+    DOWNLOAD_RETRY_PRE_REQUEST: int = 3
+    DOWNLOAD_BACKOFF_MAX: int = 3  # seconds
+    DOWNLOAD_BACKOFF_FACTOR: float = 0.1  # seconds
+
+    DOWNLOAD_THREADS: int = 6
+    MAX_CONCURRENT_DOWNLOAD_TASKS: int = 128
+    DOWNLOAD_INACTIVE_TIMEOUT: int = 5 * 60  # seconds
+
+    #
+    # ------ create standby settings ------ #
+    #
+    CREATE_STANDBY_METHOD: CREATE_STANDBY_METHOD_LTIERAL = "REBUILD"
+    MAX_CONCURRENT_PROCESS_FILE_TASKS: int = 512
+    MAX_PROCESS_FILE_THREAD: int = 6
+    CREATE_STANDBY_RETRY_MAX: int = 1024
+
+    #
+    # ------ debug flags ------ #
+    #
+    DEBUG_ENABLE_FAILURE_TRACEBACK_IN_STATUS_RESP: bool = False
+    DEBUG_DISABLE_OTAPROXY_HTTPS_VERIFY: bool = False
+    DEBUG_DISABLE_OTAMETA_CERT_CHECK: bool = False
+    DEBUG_DISABLE_OTAMETA_SIGN_CHECK: bool = False
+
+
+class _MultipleECUSettings(BaseModel):
+    # The timeout of waiting sub ECU acks the OTA request.
+    WAITING_SUBECU_ACK_REQ_TIMEOUT: int = 6
+
+    # The timeout of waiting sub ECU responds to status API request
+    QUERYING_SUBECU_STATUS_TIMEOUT: int = 6
+
+    # The ECU status storage will summarize the stored ECUs' status report
+    # and generate overall status report for all ECUs every <INTERVAL> seconds.
+    OVERALL_ECUS_STATUS_UPDATE_INTERVAL: int = 6  # seconds
+
+    # If ECU has been disconnected longer than <TIMEOUT> seconds, it will be
+    # treated as UNREACHABLE, and will not be counted when generating overall
+    # ECUs status report.
+    # NOTE: unreachable_timeout should be larger than
+    #       downloading_group timeout
+    ECU_UNREACHABLE_TIMEOUT: int = 20 * 60  # seconds
+
+    # Otaproxy should not be shutdowned with less than <INTERVAL> seconds
+    # after it just starts to prevent repeatedly start/stop cycle.
+    OTAPROXY_MINIMUM_SHUTDOWN_INTERVAL: int = 1 * 60  # seconds
+
+    # When any ECU acks update request, this ECU will directly set the overall ECU status
+    # to any_in_update=True, any_requires_network=True, all_success=False, to prevent
+    # pre-mature overall ECU status changed caused by child ECU delayed ack to update request.
+    #
+    # This pre-set overall ECU status will be kept for <KEEP_TIME> seconds.
+    # This value is expected to be larger than the time cost for subECU acks the OTA request.
+    PAUSED_OVERALL_ECUS_STATUS_CHANGE_ON_UPDATE_REQ_ACKED: int = 60  # seconds
+
+
+class _OTAProxySettings(BaseModel):
+    OTAPROXY_ENABLE_EXTERNAL_CACHE: bool = True
+    EXTERNAL_CACHE_DEV_FSLABEL: str = "ota_cache_src"
+    EXTERNAL_CACHE_DEV_MOUNTPOINT: str = f"{cfg_consts.MOUNT_SPACE}/external_cache"
+    EXTERNAL_CACHE_SRC_PATH: str = f"{EXTERNAL_CACHE_DEV_MOUNTPOINT}/data"
+
+
+class ConfigurableSettings(_OTAClientSettings, _MultipleECUSettings, _OTAProxySettings):
+    """otaclient runtime configuration settings."""
+
+
+def set_configs() -> ConfigurableSettings:
+    try:
+
+        class _SettingParser(ConfigurableSettings, BaseSettings):
+            model_config = SettingsConfigDict(
+                validate_default=True,
+                env_prefix=ENV_PREFIX,
+            )
+
+        _parsed_setting = _SettingParser()
+        return ConfigurableSettings.model_construct(**_parsed_setting.model_dump())
+    except Exception as e:
+        logger.error(f"failed to parse otaclient configurable settings: {e!r}")
+        logger.warning("use default settings ...")
+        return ConfigurableSettings()

src/otaclient/configs/_cfg_consts.py

@@ -0,0 +1,94 @@
+# Copyright 2022 TIER IV, INC. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""otaclient internal uses consts, should not be changed from external."""
+
+
+from __future__ import annotations
+
+from enum import Enum
+
+from otaclient_common import replace_root
+
+CANONICAL_ROOT = "/"
+
+
+class CreateStandbyMechanism(str, Enum):
+    LEGACY = "LEGACY"  # deprecated and removed
+    REBUILD = "REBUILD"  # default
+    IN_PLACE = "IN_PLACE"  # not yet implemented
+
+
+class Consts:
+
+    @property
+    def ACTIVE_ROOT(self) -> str:  # NOSONAR
+        return self._ACTIVE_ROOT
+
+    #
+    # ------ paths ------ #
+    #
+    RUN_DIR = "/run/otaclient"
+    OTACLIENT_PID_FILE = "/run/otaclient.pid"
+
+    # runtime folder for holding ota related files
+    RUNTIME_OTA_SESSION = "/run/otaclient/ota"
+
+    MOUNT_SPACE = "/run/otaclient/mnt"
+    ACTIVE_SLOT_MNT = "/run/otaclient/mnt/active_slot"
+    STANDBY_SLOT_MNT = "/run/otaclient/mnt/standby_slot"
+
+    OTA_TMP_STORE = "/.ota-tmp"
+    """tmp store for local copy, located at standby slot."""
+
+    OPT_OTA_DPATH = "/opt/ota"
+    OTACLIENT_INSTALLATION = "/opt/ota/client"
+    CERT_DPATH = "/opt/ota/client/certs"
+    IMAGE_META_DPATH = "/opt/ota/image-meta"
+
+    BOOT_DPATH = "/boot"
+    OTA_DPATH = "/boot/ota"
+    ECU_INFO_FPATH = "/boot/ota/ecu_info.yaml"
+    PROXY_INFO_FPATH = "/boot/ota/proxy_info.yaml"
+
+    ETC_DPATH = "/etc"
+    PASSWD_FPATH = "/etc/passwd"
+    GROUP_FPATH = "/etc/group"
+    FSTAB_FPATH = "/etc/fstab"
+
+    #
+    # ------ consts ------ #
+    #
+    # ota status files
+    OTA_STATUS_FNAME = "status"
+    OTA_VERSION_FNAME = "version"
+    SLOT_IN_USE_FNAME = "slot_in_use"
+
+    OTA_API_SERVER_PORT = 50051
+    OTAPROXY_LISTEN_PORT = 8082
+
+    def __init__(self) -> None:
+        """For future updating the ACTIVE_ROOT."""
+
+        # TODO: detect rootfs here
+        self._ACTIVE_ROOT = CANONICAL_ROOT
+
+
+cfg_consts = Consts()
+
+
+def dynamic_root(canonical_path: str) -> str:
+    """Re-root the input path with the actual ACTIVE_ROOT."""
+    if cfg_consts.ACTIVE_ROOT == CANONICAL_ROOT:
+        return canonical_path
+    return replace_root(canonical_path, CANONICAL_ROOT, cfg_consts.ACTIVE_ROOT)

src/otaclient/configs/ecu_info.py → src/otaclient/configs/_ecu_info.py

@@ -133,8 +133,3 @@ def parse_ecu_info(ecu_info_file: StrOrPath) -> ECUInfo:
         logger.warning(f"{ecu_info_file=} is invalid: {e!r}\n{_raw_yaml_str=}")
         logger.warning(f"use default ecu_info: {DEFAULT_ECU_INFO}")
         return DEFAULT_ECU_INFO
-
-
-# NOTE(20240327): set the default as literal for now,
-#   in the future this will be app_cfg.ECU_INFO_FPATH
-ecu_info = parse_ecu_info(ecu_info_file="/boot/ota/ecu_info.yaml")

src/otaclient/configs/proxy_info.py → src/otaclient/configs/_proxy_info.py

@@ -147,8 +147,3 @@ def parse_proxy_info(proxy_info_file: StrOrPath) -> ProxyInfo:
         logger.warning(f"{proxy_info_file=} is invalid: {e!r}\n{_raw_yaml_str=}")
         logger.warning(f"use default proxy_info: {DEFAULT_PROXY_INFO}")
         return DEFAULT_PROXY_INFO
-
-
-# NOTE(20240327): set the default as literal for now,
-#   in the future this will be app_cfg.PROXY_INFO_FPATH
-proxy_info = parse_proxy_info("/boot/ota/proxy_info.yaml")

src/otaclient/configs/cfg.py

@@ -0,0 +1,61 @@
+# Copyright 2022 TIER IV, INC. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Load ecu_info, proxy_info and otaclient configs."""
+
+from typing import TYPE_CHECKING, Any
+
+from otaclient.configs._cfg_configurable import ConfigurableSettings, set_configs
+from otaclient.configs._cfg_consts import Consts
+from otaclient.configs._ecu_info import (
+    BootloaderType,
+    ECUContact,
+    ECUInfo,
+    parse_ecu_info,
+)
+from otaclient.configs._proxy_info import ProxyInfo, parse_proxy_info
+
+__all__ = [
+    "BootloaderType",
+    "ECUContact",
+    "ECUInfo",
+    "ecu_info",
+    "ProxyInfo",
+    "proxy_info",
+    "cfg",
+]
+
+cfg_configurable = set_configs()
+cfg_consts = Consts()
+
+if TYPE_CHECKING:
+
+    class _OTAClientConfigs(ConfigurableSettings, Consts):
+        """OTAClient configs."""
+
+else:
+
+    class _OTAClientConfigs:
+
+        def __getattribute__(self, name: str) -> Any:
+            for _cfg in [cfg_consts, cfg_configurable]:
+                try:
+                    return getattr(_cfg, name)
+                except AttributeError:
+                    continue
+            raise AttributeError(f"no such config field: {name=}")
+
+
+cfg = _OTAClientConfigs()
+ecu_info = parse_ecu_info(ecu_info_file=cfg.ECU_INFO_FPATH)
+proxy_info = parse_proxy_info(proxy_info_file=cfg.PROXY_INFO_FPATH)