diff --git a/otaclient/__init__.py b/otaclient/__init__.py index 6c4bf729f..dbf40f6b7 100644 --- a/otaclient/__init__.py +++ b/otaclient/__init__.py @@ -12,7 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. - -from otaclient._version import version, __version__ +try: + from otaclient._version import version, __version__ +except ImportError: + # unknown version + version = __version__ = "0.0.0" __all__ = ["version", "__version__"] diff --git a/otaclient/_utils/typing.py b/otaclient/_utils/typing.py new file mode 100644 index 000000000..874f87897 --- /dev/null +++ b/otaclient/_utils/typing.py @@ -0,0 +1,48 @@ +# Copyright 2022 TIER IV, INC. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import annotations +from pathlib import Path +from typing import Any, Callable, TypeVar, Union +from typing_extensions import Annotated, ParamSpec + +from pydantic import Field + +P = ParamSpec("P") +T = TypeVar("T") +RT = TypeVar("RT") + +StrOrPath = Union[str, Path] + +# pydantic helpers + +NetworkPort = Annotated[int, Field(ge=1, le=65535)] + + +def gen_strenum_validator(enum_type: type[T]) -> Callable[[T | str | Any], T]: + """A before validator generator that converts input value into enum + before passing it to pydantic validator. + + NOTE(20240129): as upto pydantic v2.5.3, (str, Enum) field cannot + pass strict validation if input is str. + """ + + def _inner(value: T | str | Any) -> T: + assert isinstance( + value, (enum_type, str) + ), f"{value=} should be {enum_type} or str type" + return enum_type(value) + + return _inner diff --git a/otaclient/app/boot_control/_cboot.py b/otaclient/app/boot_control/_cboot.py index cb760076f..38a3634a0 100644 --- a/otaclient/app/boot_control/_cboot.py +++ b/otaclient/app/boot_control/_cboot.py @@ -177,9 +177,7 @@ def _init_dev_info(self): self.current_slot: str = Nvbootctrl.get_current_slot() self.current_rootfs_dev: str = CMDHelperFuncs.get_current_rootfs_dev() # NOTE: boot dev is always emmc device now - self.current_boot_dev: str = ( - f"/dev/{Nvbootctrl.EMMC_DEV}p{Nvbootctrl.SLOTID_PARTID_MAP[self.current_slot]}" - ) + self.current_boot_dev: str = f"/dev/{Nvbootctrl.EMMC_DEV}p{Nvbootctrl.SLOTID_PARTID_MAP[self.current_slot]}" self.standby_slot: str = Nvbootctrl.CURRENT_STANDBY_FLIP[self.current_slot] standby_partid = Nvbootctrl.SLOTID_PARTID_MAP[self.standby_slot] diff --git a/otaclient/app/boot_control/configs.py b/otaclient/app/boot_control/configs.py index d9fb96478..1eae10525 100644 --- a/otaclient/app/boot_control/configs.py +++ b/otaclient/app/boot_control/configs.py @@ -13,39 +13,13 @@ # limitations under the License. +from __future__ import annotations from dataclasses import dataclass, field -from enum import Enum from typing import Dict from ..configs import BaseConfig - -class BootloaderType(Enum): - """Bootloaders that supported by otaclient. - - grub: generic x86_64 platform with grub - cboot: ADLink rqx-580, rqx-58g, with BSP 32.5.x - (theoretically other Nvidia jetson xavier devices using cboot are also supported) - rpi_boot: raspberry pi 4 with eeprom version newer than 2020-10-28 - """ - - UNSPECIFIED = "unspecified" - GRUB = "grub" - CBOOT = "cboot" - RPI_BOOT = "rpi_boot" - - @classmethod - def parse_str(cls, _input: str) -> "BootloaderType": - res = cls.UNSPECIFIED - try: # input is enum key(capitalized) - res = BootloaderType[_input] - except KeyError: - pass - try: # input is enum value(uncapitalized) - res = BootloaderType(_input) - except ValueError: - pass - return res +from otaclient.configs.ecu_info import BootloaderType @dataclass diff --git a/otaclient/app/boot_control/selecter.py b/otaclient/app/boot_control/selecter.py index ab4f91224..369e2835f 100644 --- a/otaclient/app/boot_control/selecter.py +++ b/otaclient/app/boot_control/selecter.py @@ -13,12 +13,13 @@ # limitations under the License. +from __future__ import annotations import logging import platform from pathlib import Path -from typing import Type +from typing_extensions import deprecated -from .configs import BootloaderType, cboot_cfg, rpi_boot_cfg +from .configs import BootloaderType from ._errors import BootControlError from .protocol import BootControllerProtocol @@ -27,7 +28,8 @@ logger = logging.getLogger(__name__) -def detect_bootloader(raise_on_unknown=True) -> BootloaderType: +@deprecated("bootloader type SHOULD be set in ecu_info.yaml instead of detecting") +def detect_bootloader() -> BootloaderType: """Detect which bootloader we are running with by some evidence. Evidence: @@ -42,12 +44,16 @@ def detect_bootloader(raise_on_unknown=True) -> BootloaderType: machine, arch = platform.machine(), platform.processor() if machine == "x86_64" or arch == "x86_64": return BootloaderType.GRUB + # distinguish between rpi and jetson xavier device if machine == "aarch64" or arch == "aarch64": + from .configs import cboot_cfg, rpi_boot_cfg + # evidence: jetson xvaier device has a special file which reveals the # tegra chip id if Path(cboot_cfg.TEGRA_CHIP_ID_PATH).is_file(): return BootloaderType.CBOOT + # evidence: rpi device has a special file which reveals the rpi model rpi_model_file = Path(rpi_boot_cfg.RPI_MODEL_FILE) if rpi_model_file.is_file(): @@ -55,24 +61,21 @@ def detect_bootloader(raise_on_unknown=True) -> BootloaderType: rpi_boot_cfg.RPI_MODEL_HINT ) != -1: return BootloaderType.RPI_BOOT - else: - logger.error( - f"detect unsupported raspberry pi platform({_model_str=}), only {rpi_boot_cfg.RPI_MODEL_HINT} is supported" - ) - # failed to detect bootloader - if raise_on_unknown: - raise ValueError(f"unsupported platform({machine=}, {arch=}) detected, abort") - return BootloaderType.UNSPECIFIED + logger.error( + f"detect unsupported raspberry pi platform({_model_str=}), " + f"only {rpi_boot_cfg.RPI_MODEL_HINT} is supported" + ) + raise ValueError(f"unsupported platform({machine=}, {arch=}) detected, abort") def get_boot_controller( bootloader_type: BootloaderType, -) -> Type[BootControllerProtocol]: +) -> type[BootControllerProtocol]: # if ecu_info doesn't specify the bootloader type, - # we try to detect by ourselves - if bootloader_type is BootloaderType.UNSPECIFIED: - bootloader_type = detect_bootloader(raise_on_unknown=True) + # we try to detect by ourselves. + if bootloader_type is BootloaderType.AUTO_DETECT: + bootloader_type = detect_bootloader() logger.info(f"use boot_controller for {bootloader_type=}") if bootloader_type == BootloaderType.GRUB: diff --git a/otaclient/app/configs.py b/otaclient/app/configs.py index 7cb9e6ab4..fbd072d38 100644 --- a/otaclient/app/configs.py +++ b/otaclient/app/configs.py @@ -17,6 +17,11 @@ from logging import INFO from typing import Dict, Tuple +from otaclient.configs.ecu_info import ecu_info +from otaclient.configs.proxy_info import proxy_info + +ecu_info, proxy_info = ecu_info, proxy_info # to prevent static check warnings + class CreateStandbyMechanism(Enum): LEGACY = 0 # deprecated and removed diff --git a/otaclient/app/ecu_info.py b/otaclient/app/ecu_info.py deleted file mode 100644 index 0adb78134..000000000 --- a/otaclient/app/ecu_info.py +++ /dev/null @@ -1,129 +0,0 @@ -# Copyright 2022 TIER IV, INC. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -r"""ECU metadatas definition.""" - - -import logging -import yaml -from copy import deepcopy -from dataclasses import dataclass, field, fields, MISSING -from pathlib import Path -from typing import Iterator, NamedTuple, Union, Dict, List, Any - -from .configs import server_cfg -from .boot_control import BootloaderType - -logger = logging.getLogger(__name__) - -DEFAULT_ECU_INFO = { - "format_version": 1, # current version is 1 - "ecu_id": "autoware", # should be unique for each ECU in vehicle -} - - -class ECUContact(NamedTuple): - ecu_id: str - host: str - port: int = server_cfg.SERVER_PORT - - -@dataclass -class ECUInfo: - """ - Version 1 scheme example: - format_vesrion: 1 - ecu_id: "autoware" - ip_addr: "0.0.0.0" - bootloader: "grub" - secondaries: - - ecu_id: "p1" - ip_addr: "0.0.0.0" - available_ecu_ids: - - "autoware" - - "p1 - """ - - ecu_id: str - ip_addr: str = "127.0.0.1" - bootloader: str = BootloaderType.UNSPECIFIED.value - available_ecu_ids: list = field(default_factory=list) # list[str] - secondaries: list = field(default_factory=list) # list[dict[str, Any]] - format_version: int = 1 - - @classmethod - def parse_ecu_info(cls, ecu_info_file: Union[str, Path]) -> "ECUInfo": - ecu_info = deepcopy(DEFAULT_ECU_INFO) - try: - ecu_info_yaml = Path(ecu_info_file).read_text() - _ecu_info = yaml.safe_load(ecu_info_yaml) - assert isinstance(_ecu_info, Dict) - ecu_info = _ecu_info - except (yaml.error.MarkedYAMLError, AssertionError) as e: - logger.warning( - f"invalid {ecu_info_yaml=}, use default config: {e!r}" # type: ignore - ) - except Exception as e: - logger.warning( - f"{ecu_info_file=} not found or unexpected err, use default config: {e!r}" - ) - logger.info(f"parsed {ecu_info=}") - - # load options - # NOTE: if option is not presented, - # this option will be set to the default value - _ecu_info_dict: Dict[str, Any] = dict() - for _field in fields(cls): - _option = ecu_info.get(_field.name) - if not isinstance(_option, _field.type): - if _option is not None: - logger.warning( - f"{_field.name} contains invalid value={_option}, " - "ignored and set to default={_field.default}" - ) - if _field.default is MISSING and _field.default_factory is MISSING: - raise ValueError( - f"required field {_field.name} is not presented, abort" - ) - _ecu_info_dict[_field.name] = ( - _field.default - if _field.default is not MISSING - else _field.default_factory() # type: ignore - ) - # parsed _option is available - else: - _ecu_info_dict[_field.name] = _option - - # initialize ECUInfo inst - return cls(**deepcopy(_ecu_info_dict)) - - def iter_direct_subecu_contact(self) -> Iterator[ECUContact]: - for subecu in self.secondaries: - try: - yield ECUContact( - ecu_id=subecu["ecu_id"], - host=subecu["ip_addr"], - port=subecu.get("port", server_cfg.SERVER_PORT), - ) - except KeyError: - raise ValueError(f"{subecu=} info is invalid") - - def get_bootloader(self) -> BootloaderType: - return BootloaderType.parse_str(self.bootloader) - - def get_available_ecu_ids(self) -> List[str]: - # onetime fix, if no availabe_ecu_id is specified, - # add my_ecu_id into the list - if len(self.available_ecu_ids) == 0: - self.available_ecu_ids.append(self.ecu_id) - return self.available_ecu_ids.copy() diff --git a/otaclient/app/log_setting.py b/otaclient/app/log_setting.py index b33d7342e..3d5a66f57 100644 --- a/otaclient/app/log_setting.py +++ b/otaclient/app/log_setting.py @@ -17,26 +17,13 @@ import atexit import logging -import os -import yaml +import requests from queue import Queue from threading import Event, Thread from urllib.parse import urljoin -import requests - import otaclient -from .configs import config as cfg - - -# NOTE: EcuInfo imports this log_setting so independent get_ecu_id are required. -def get_ecu_id(): - try: - with open(cfg.ECU_INFO_FILE) as f: - ecu_info = yaml.load(f, Loader=yaml.SafeLoader) - return ecu_info["ecu_id"] - except Exception: - return "autoware" +from .configs import config as cfg, ecu_info, proxy_info class _LogTeeHandler(logging.Handler): @@ -99,25 +86,15 @@ def configure_logging(): _logger = logging.getLogger(_module_name) _logger.setLevel(_log_level) - # NOTE(20240306): for only god knows reason, although in proxy_info.yaml, - # the logging_server field is assigned with an URL, and otaclient - # expects an URL, the run.sh from autoware_ecu_system_setup pass in - # HTTP_LOGGING_SERVER with URL schema being removed!? - # NOTE: I will do a quick fix here as I don't want to touch autoware_ecu_system_setup - # for now, leave it in the future. - if iot_logger_url := os.environ.get("HTTP_LOGGING_SERVER"): - # special treatment for not-a-URL passed in by run.sh - # note that we only support http, the proxy_info.yaml should be properly setup. - if not (iot_logger_url.startswith("http") or iot_logger_url.startswith("HTTP")): - iot_logger_url = f"http://{iot_logger_url.strip('/')}" - iot_logger_url = f"{iot_logger_url.strip('/')}/" + if iot_logger_url := proxy_info.logging_server: + iot_logger_url = f"{str(iot_logger_url).strip('/')}/" ch = _LogTeeHandler() fmt = logging.Formatter(fmt=cfg.LOG_FORMAT) ch.setFormatter(fmt) # star the logging thread - log_upload_endpoint = urljoin(iot_logger_url, get_ecu_id()) + log_upload_endpoint = urljoin(iot_logger_url, ecu_info.ecu_id) ch.start_upload_thread(log_upload_endpoint) # NOTE: "otaclient" logger will be the root logger for all loggers name diff --git a/otaclient/app/ota_client.py b/otaclient/app/ota_client.py index 1c1baace6..0006d124a 100644 --- a/otaclient/app/ota_client.py +++ b/otaclient/app/ota_client.py @@ -36,9 +36,8 @@ RetryTaskMap, RetryTaskMapInterrupted, ) -from .configs import config as cfg +from .configs import config as cfg, ecu_info from .create_standby import StandbySlotCreatorProtocol, get_standby_slot_creator -from .ecu_info import ECUInfo from .interface import OTAClientProtocol from .ota_status import LiveOTAStatus from .proto import wrapper @@ -290,6 +289,7 @@ def _update_standby_slot(self): logger.info("finished updating standby slot") def _process_persistents(self): + logger.info("start persist files handling...") standby_slot_mp = Path(cfg.MOUNT_POINT) _handler = PersistFilesHandler( @@ -317,6 +317,7 @@ def _process_persistents(self): _swapfile_size = get_file_size(_per_fpath, units="MiB") assert _swapfile_size is not None, f"{_per_fpath} doesn't exist" create_swapfile(_new_swapfile, _swapfile_size) + logger.warning(f"create {_new_swapfile} with {_swapfile_size=}") except Exception as e: logger.warning( f"failed to create swapfile {_per_fpath} to standby slot, skip: {e!r}" @@ -664,7 +665,6 @@ def __init__( self, *, control_flags: OTAClientControlFlags, - ecu_info: ECUInfo, executor: Optional[ThreadPoolExecutor] = None, otaclient_version: str = __version__, proxy: Optional[str] = None, @@ -694,7 +694,7 @@ def __init__( self._otaclient_inst: Optional[OTAClient] = None # select boot_controller and standby_slot implementations - _bootctrl_cls = get_boot_controller(ecu_info.get_bootloader()) + _bootctrl_cls = get_boot_controller(ecu_info.bootloader) _standby_slot_creator = get_standby_slot_creator(cfg.STANDBY_CREATION_MODE) # boot controller starts up diff --git a/otaclient/app/ota_client_service.py b/otaclient/app/ota_client_service.py index b8bf7502d..3a34fb5cf 100644 --- a/otaclient/app/ota_client_service.py +++ b/otaclient/app/ota_client_service.py @@ -15,8 +15,7 @@ import grpc.aio -from .configs import config as cfg, server_cfg -from .ecu_info import ECUInfo +from .configs import ecu_info, server_cfg from .proto import wrapper, v2, v2_grpc from .ota_client_stub import OTAClientServiceStub @@ -41,9 +40,7 @@ async def Status(self, request: v2.StatusRequest, context) -> v2.StatusResponse: def create_otaclient_grpc_server(): - ecu_info = ECUInfo.parse_ecu_info(cfg.ECU_INFO_FILE) - - service_stub = OTAClientServiceStub(ecu_info=ecu_info) + service_stub = OTAClientServiceStub() ota_client_service_v2 = OtaClientServiceV2(service_stub) server = grpc.aio.server() diff --git a/otaclient/app/ota_client_stub.py b/otaclient/app/ota_client_stub.py index 0afb9d45a..63298374f 100644 --- a/otaclient/app/ota_client_stub.py +++ b/otaclient/app/ota_client_stub.py @@ -13,6 +13,7 @@ # limitations under the License. +from __future__ import annotations import asyncio import logging import shutil @@ -26,15 +27,14 @@ from typing_extensions import Self from . import log_setting -from .configs import config as cfg, server_cfg +from .configs import config as cfg, server_cfg, ecu_info, proxy_info from .common import ensure_otaproxy_start from .boot_control._common import CMDHelperFuncs -from .ecu_info import ECUContact, ECUInfo from .ota_client import OTAClientControlFlags, OTAServicer from .ota_client_call import ECUNoResponse, OtaClientCall from .proto import wrapper -from .proxy_info import proxy_cfg +from otaclient.configs.ecu_info import ECUContact from otaclient.ota_proxy import ( OTAProxyContextProto, subprocess_otaproxy_launcher, @@ -51,13 +51,12 @@ class _OTAProxyContext(OTAProxyContextProto): def __init__( self, *, - upper_proxy: Optional[str], - external_cache_enabled: bool, + external_cache_enabled: bool = True, external_cache_dev_fslable: str = cfg.EXTERNAL_CACHE_DEV_FSLABEL, external_cache_dev_mp: str = cfg.EXTERNAL_CACHE_DEV_MOUNTPOINT, external_cache_path: str = cfg.EXTERNAL_CACHE_SRC_PATH, ) -> None: - self.upper_proxy = upper_proxy + self.upper_proxy = proxy_info.upper_ota_proxy self.external_cache_enabled = external_cache_enabled self._external_cache_activated = False @@ -95,7 +94,7 @@ def _subprocess_init(self): # wait for upper otaproxy if any if self.upper_proxy: otaproxy_logger.info(f"wait for {self.upper_proxy=} online...") - ensure_otaproxy_start(self.upper_proxy) + ensure_otaproxy_start(str(self.upper_proxy)) def _mount_external_cache_storage(self): # detect cache_dev on every startup @@ -165,17 +164,12 @@ class OTAProxyLauncher: """Launcher of start/stop otaproxy in subprocess.""" def __init__( - self, - *, - executor: ThreadPoolExecutor, - subprocess_ctx: OTAProxyContextProto, - _proxy_info=proxy_cfg, - _proxy_server_cfg=local_otaproxy_cfg, + self, *, executor: ThreadPoolExecutor, subprocess_ctx: OTAProxyContextProto ) -> None: - self._proxy_info = _proxy_info - self._proxy_server_cfg = _proxy_server_cfg - self.enabled = _proxy_info.enable_local_ota_proxy - self.upper_otaproxy = _proxy_info.upper_ota_proxy + self.enabled = proxy_info.enable_local_ota_proxy + self.upper_otaproxy = ( + str(proxy_info.upper_ota_proxy) if proxy_info.upper_ota_proxy else "" + ) self.subprocess_ctx = subprocess_ctx self._lock = asyncio.Lock() @@ -200,7 +194,7 @@ def cleanup_cache_dir(self): NOTE: this method should only be called when all ECUs in the cluster are in SUCCESS ota_status(overall_ecu_status.all_success==True). """ - if (cache_dir := Path(self._proxy_server_cfg.BASE_DIR)).is_dir(): + if (cache_dir := Path(local_otaproxy_cfg.BASE_DIR)).is_dir(): logger.info("cleanup ota_cache on success") shutil.rmtree(cache_dir, ignore_errors=True) @@ -214,23 +208,24 @@ async def start(self, *, init_cache: bool) -> Optional[int]: _subprocess_entry = subprocess_otaproxy_launcher( subprocess_ctx=self.subprocess_ctx ) + otaproxy_subprocess = await self._run_in_executor( partial( _subprocess_entry, - host=self._proxy_info.local_ota_proxy_listen_addr, - port=self._proxy_info.local_ota_proxy_listen_port, + host=str(proxy_info.local_ota_proxy_listen_addr), + port=proxy_info.local_ota_proxy_listen_port, init_cache=init_cache, - cache_dir=self._proxy_server_cfg.BASE_DIR, - cache_db_f=self._proxy_server_cfg.DB_FILE, + cache_dir=local_otaproxy_cfg.BASE_DIR, + cache_db_f=local_otaproxy_cfg.DB_FILE, upper_proxy=self.upper_otaproxy, - enable_cache=self._proxy_info.enable_local_ota_proxy_cache, - enable_https=self._proxy_info.gateway, + enable_cache=proxy_info.enable_local_ota_proxy_cache, + enable_https=proxy_info.gateway_otaproxy, ) ) self._otaproxy_subprocess = otaproxy_subprocess logger.info( f"otaproxy({otaproxy_subprocess.pid=}) started at " - f"{self._proxy_info.local_ota_proxy_listen_addr}:{self._proxy_info.local_ota_proxy_listen_port}" + f"{proxy_info.local_ota_proxy_listen_addr}:{proxy_info.local_ota_proxy_listen_port}" ) return otaproxy_subprocess.pid @@ -316,7 +311,7 @@ class ECUStatusStorage: # disconnected ECU will be excluded from status API response. DISCONNECTED_ECU_TIMEOUT_FACTOR = 3 - def __init__(self, ecu_info: ECUInfo) -> None: + def __init__(self) -> None: self.my_ecu_id = ecu_info.ecu_id self._writer_lock = asyncio.Lock() # ECU status storage @@ -670,7 +665,6 @@ def __init__( self, ecu_status_storage: ECUStatusStorage, *, - ecu_info: ECUInfo, otaclient_wrapper: OTAServicer, ) -> None: self._otaclient_wrapper = otaclient_wrapper # for local ECU status polling @@ -683,7 +677,7 @@ def __init__( # In normal running this event will never be set. self._debug_ecu_status_polling_shutdown_event = asyncio.Event() asyncio.create_task(self._polling_local_ecu_status()) - for ecu_contact in ecu_info.iter_direct_subecu_contact(): + for ecu_contact in ecu_info.secondaries: asyncio.create_task(self._polling_direct_subecu_status(ecu_contact)) async def _polling_direct_subecu_status(self, ecu_contact: ECUContact): @@ -692,7 +686,7 @@ async def _polling_direct_subecu_status(self, ecu_contact: ECUContact): try: _ecu_resp = await OtaClientCall.status_call( ecu_contact.ecu_id, - ecu_contact.host, + str(ecu_contact.ip_addr), ecu_contact.port, timeout=server_cfg.QUERYING_SUBECU_STATUS_TIMEOUT, request=wrapper.StatusRequest(), @@ -720,30 +714,28 @@ class OTAClientServiceStub: OTAPROXY_SHUTDOWN_DELAY = cfg.OTAPROXY_MINIMUM_SHUTDOWN_INTERVAL - def __init__(self, *, ecu_info: ECUInfo, _proxy_cfg=proxy_cfg): + def __init__(self): self._executor = ThreadPoolExecutor(thread_name_prefix="otaclient_service_stub") self._run_in_executor = partial( asyncio.get_running_loop().run_in_executor, self._executor ) - self.ecu_info = ecu_info + self.sub_ecus = ecu_info.secondaries self.listen_addr = ecu_info.ip_addr self.listen_port = server_cfg.SERVER_PORT self.my_ecu_id = ecu_info.ecu_id self._otaclient_control_flags = OTAClientControlFlags() self._otaclient_wrapper = OTAServicer( - ecu_info=ecu_info, executor=self._executor, control_flags=self._otaclient_control_flags, - proxy=_proxy_cfg.get_proxy_for_local_ota(), + proxy=proxy_info.get_proxy_for_local_ota(), ) # ecu status tracking - self._ecu_status_storage = ECUStatusStorage(ecu_info) + self._ecu_status_storage = ECUStatusStorage() self._ecu_status_tracker = _ECUTracker( self._ecu_status_storage, - ecu_info=ecu_info, otaclient_wrapper=self._otaclient_wrapper, ) @@ -754,14 +746,10 @@ def __init__(self, *, ecu_info: ECUInfo, _proxy_cfg=proxy_cfg): # allow us to stop background task without changing codes. # In normal running this event will never be set. self._debug_status_checking_shutdown_event = asyncio.Event() - if _proxy_cfg.enable_local_ota_proxy: + if proxy_info.enable_local_ota_proxy: self._otaproxy_launcher = OTAProxyLauncher( executor=self._executor, - subprocess_ctx=_OTAProxyContext( - upper_proxy=_proxy_cfg.upper_ota_proxy, - # NOTE: default enable detecting external cache storage - external_cache_enabled=True, - ), + subprocess_ctx=_OTAProxyContext(), ) asyncio.create_task(self._otaproxy_lifecycle_managing()) asyncio.create_task(self._otaclient_control_flags_managing()) @@ -835,13 +823,13 @@ async def update(self, request: wrapper.UpdateRequest) -> wrapper.UpdateResponse # first: dispatch update request to all directly connected subECUs tasks: Dict[asyncio.Task, ECUContact] = {} - for ecu_contact in self.ecu_info.iter_direct_subecu_contact(): + for ecu_contact in self.sub_ecus: if not request.if_contains_ecu(ecu_contact.ecu_id): continue _task = asyncio.create_task( OtaClientCall.update_call( ecu_contact.ecu_id, - ecu_contact.host, + str(ecu_contact.ip_addr), ecu_contact.port, request=request, timeout=server_cfg.WAITING_SUBECU_ACK_REQ_TIMEOUT, @@ -899,13 +887,13 @@ async def rollback( # first: dispatch rollback request to all directly connected subECUs tasks: Dict[asyncio.Task, ECUContact] = {} - for ecu_contact in self.ecu_info.iter_direct_subecu_contact(): + for ecu_contact in self.sub_ecus: if not request.if_contains_ecu(ecu_contact.ecu_id): continue _task = asyncio.create_task( OtaClientCall.rollback_call( ecu_contact.ecu_id, - ecu_contact.host, + str(ecu_contact.ip_addr), ecu_contact.port, request=request, timeout=server_cfg.WAITING_SUBECU_ACK_REQ_TIMEOUT, diff --git a/otaclient/app/proxy_info.py b/otaclient/app/proxy_info.py deleted file mode 100644 index 6c33b2026..000000000 --- a/otaclient/app/proxy_info.py +++ /dev/null @@ -1,135 +0,0 @@ -# Copyright 2022 TIER IV, INC. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Proxy setting parsing. - -check docs/README.md for more details. -""" - - -import logging -import yaml -import warnings -from dataclasses import dataclass, fields -from typing import Any, ClassVar, Dict -from pathlib import Path - -from .configs import config as cfg -from .configs import server_cfg - -logger = logging.getLogger(__name__) - -PRE_DEFINED_PROXY_INFO_YAML = """ -enable_local_ota_proxy: true -gateway: true -""" - - -@dataclass -class ProxyInfo: - """OTA-proxy configuration. - - NOTE 1(20221220): when proxy_info.yaml is missing/not a valid yaml, - a pre_defined proxy_info.yaml as follow will be used. - NOTE 2(20221220): when a config field is missing/assigned with invalid value, - default value will be applied. - - Attributes: - enable_local_ota_proxy: whether to launch a local ota_proxy server. - enable_local_ota_proxy_cache: enable cache mechanism on ota-proxy. - gateway: whether to use HTTPS when local ota_proxy connects to remote. - local_ota_proxy_listen_addr: ipaddr ota_proxy listens on. - local_ota_proxy_listen_port: port ota_proxy used. - upper_ota_proxy: the upper proxy used by local ota_proxy(proxy chain). - """ - - # NOTE(20221219): the default values for the following settings - # now align with v2.5.4 - gateway: bool = False - upper_ota_proxy: str = "" - enable_local_ota_proxy: bool = False - local_ota_proxy_listen_addr: str = server_cfg.OTA_PROXY_LISTEN_ADDRESS - local_ota_proxy_listen_port: int = server_cfg.OTA_PROXY_LISTEN_PORT - # NOTE: this field not presented in v2.5.4, - # for current implementation, it should be default to True. - # This field doesn't take effect if enable_local_ota_proxy is False - enable_local_ota_proxy_cache: bool = True - - # for maintaining compatibility, will be removed in the future - # Dict[str, str]: -> - # 20220526: "enable_ota_proxy" -> "enable_local_ota_proxy" - _compatibility: ClassVar[Dict[str, str]] = { - "enable_ota_proxy": "enable_local_ota_proxy" - } - - def get_proxy_for_local_ota(self) -> str: - if self.enable_local_ota_proxy: - # if local proxy is enabled, local ota client also uses it - return f"http://{self.local_ota_proxy_listen_addr}:{self.local_ota_proxy_listen_port}" - elif self.upper_ota_proxy: - # else we directly use the upper proxy - return self.upper_ota_proxy - else: - # default not using proxy - return "" - - -def parse_proxy_info(proxy_info_file: str = cfg.PROXY_INFO_FILE) -> ProxyInfo: - _loaded: Dict[str, Any] - try: - _loaded = yaml.safe_load(Path(proxy_info_file).read_text()) - assert isinstance(_loaded, Dict) - except (FileNotFoundError, AssertionError): - logger.warning( - f"failed to load {proxy_info_file=} or config file corrupted, " - "use default main ECU config" - ) - _loaded = yaml.safe_load(PRE_DEFINED_PROXY_INFO_YAML) - - # load options - # NOTE: if option is not presented, - # this option will be set to the default value - _proxy_info_dict: Dict[str, Any] = dict() - for _field in fields(ProxyInfo): - _option = _loaded.get(_field.name) - if not isinstance(_option, _field.type): - if _option is not None: - logger.warning( - f"{_field.name} contains invalid value={_option}, " - f"ignored and set to default={_field.default}" - ) - _proxy_info_dict[_field.name] = _field.default - continue - - _proxy_info_dict[_field.name] = _option - - # maintain compatiblity with old proxy_info format - for old, new in ProxyInfo._compatibility.items(): - if old in _loaded: - warnings.warn( - f"option field '{old}' is replaced by '{new}', " - f"and the support for '{old}' option might be dropped in the future. " - f"please use '{new}' instead.", - DeprecationWarning, - stacklevel=2, - ) - _proxy_info_dict[new] = _loaded[old] - - return ProxyInfo(**_proxy_info_dict) - - -proxy_cfg = parse_proxy_info() - -if __name__ == "__main__": - proxy_cfg = parse_proxy_info("./proxy_info.yaml") - logger.info(f"{proxy_cfg!r}, {proxy_cfg.get_proxy_for_local_ota()=}") diff --git a/otaclient/configs/__init__.py b/otaclient/configs/__init__.py new file mode 100644 index 000000000..55ed90a28 --- /dev/null +++ b/otaclient/configs/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022 TIER IV, INC. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""otaclient configs package.""" diff --git a/otaclient/configs/_common.py b/otaclient/configs/_common.py new file mode 100644 index 000000000..90f55bce0 --- /dev/null +++ b/otaclient/configs/_common.py @@ -0,0 +1,38 @@ +# Copyright 2022 TIER IV, INC. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import annotations +from pydantic import BaseModel, ConfigDict +from pydantic_settings import BaseSettings, SettingsConfigDict + + +# prefix for environmental vars name for configs. +ENV_PREFIX = "OTA_" + + +class BaseConfigurableConfig(BaseSettings): + """Common base for configs that are configurable via ENV.""" + + model_config = SettingsConfigDict( + env_prefix=ENV_PREFIX, + frozen=True, + validate_default=True, + ) + + +class BaseFixedConfig(BaseModel): + """Common base for configs that should be fixed and not changable.""" + + model_config = ConfigDict(frozen=True, validate_default=True) diff --git a/otaclient/configs/ecu_info.py b/otaclient/configs/ecu_info.py new file mode 100644 index 000000000..9ddd65cb1 --- /dev/null +++ b/otaclient/configs/ecu_info.py @@ -0,0 +1,137 @@ +# Copyright 2022 TIER IV, INC. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""ECU metadatas definition and parsing logic.""" + + +from __future__ import annotations +import logging +import warnings +from enum import Enum +from pathlib import Path +from typing import List +from typing_extensions import Annotated + +import yaml +from pydantic import AfterValidator, BeforeValidator, Field, IPvAnyAddress + +from otaclient._utils.typing import StrOrPath, gen_strenum_validator, NetworkPort +from otaclient.configs._common import BaseFixedConfig + +logger = logging.getLogger(__name__) + + +class BootloaderType(str, Enum): + """Bootloaders that supported by otaclient. + + auto_detect: (DEPRECATED) at runtime detecting what bootloader is in use in this ECU. + grub: generic x86_64 platform with grub. + cboot: ADLink rqx-580, rqx-58g, with BSP 32.5.x. + (theoretically other Nvidia jetson xavier devices using cboot are also supported) + rpi_boot: raspberry pi 4 with eeprom version newer than 2020-10-28(with tryboot support). + """ + + AUTO_DETECT = "auto_detect" + GRUB = "grub" + CBOOT = "cboot" + RPI_BOOT = "rpi_boot" + + @staticmethod + def deprecation_validator(value: BootloaderType) -> BootloaderType: + if value == BootloaderType.AUTO_DETECT: + _warning_msg = ( + "bootloader field in ecu_info.yaml is not set or set to auto_detect(DEPRECATED), " + "runtime bootloader type detection is UNRELIABLE and bootloader field SHOULD be " + "set in ecu_info.yaml" + ) + warnings.warn(_warning_msg, DeprecationWarning, stacklevel=2) + logger.warning(_warning_msg) + return value + + +class ECUContact(BaseFixedConfig): + ecu_id: str + ip_addr: IPvAnyAddress + # NOTE(20240327): set the default as literal for now, in the future + # this will be service_cfg.CLIENT_CALL_PORT. + port: NetworkPort = 50051 + + +class ECUInfo(BaseFixedConfig): + """ECU info configuration. + + Attributes: + format_version: the ecu_info.yaml scheme version, current is 1. + ecu_id: the unique ID of this ECU. + ip_addr: the IP address OTA servicer listening on, default is . + bootloader: the bootloader type of this ECU. + available_ecu_ids: a list of ECU IDs that should be involved in OTA campaign. + secondaries: a list of ECUContact objects for sub ECUs. + """ + + format_version: int = 1 + ecu_id: str + # NOTE(20240327): set the default as literal for now, in the future + # when app_configs are fully backported this will be replaced by + # service_cfg.DEFAULT_SERVER_ADDRESS + ip_addr: IPvAnyAddress = Field(default="127.0.0.1") + bootloader: Annotated[ + BootloaderType, + BeforeValidator(gen_strenum_validator(BootloaderType)), + AfterValidator(BootloaderType.deprecation_validator), + Field(validate_default=False), + ] = BootloaderType.AUTO_DETECT + available_ecu_ids: List[str] = Field(default_factory=list) + secondaries: List[ECUContact] = Field(default_factory=list) + + def get_available_ecu_ids(self) -> list[str]: + """ + NOTE: this method should be used instead of directly accessing the + available_ecu_ids attrs for backward compatibility reason. + """ + # onetime fix, if no availabe_ecu_id is specified, + # add my_ecu_id into the list + if len(self.available_ecu_ids) == 0: + return [self.ecu_id] + return self.available_ecu_ids.copy() + + +# NOTE: this is backward compatibility for old x1 that doesn't have +# ecu_info.yaml installed. +DEFAULT_ECU_INFO = ECUInfo( + format_version=1, # current version is 1 + ecu_id="autoware", # should be unique for each ECU in vehicle +) + + +def parse_ecu_info(ecu_info_file: StrOrPath) -> ECUInfo: + try: + _raw_yaml_str = Path(ecu_info_file).read_text() + except FileNotFoundError as e: + logger.warning(f"{ecu_info_file=} not found: {e!r}") + logger.warning(f"use default ecu_info: {DEFAULT_ECU_INFO}") + return DEFAULT_ECU_INFO + + try: + loaded_ecu_info = yaml.safe_load(_raw_yaml_str) + assert isinstance(loaded_ecu_info, dict), "not a valid yaml file" + return ECUInfo.model_validate(loaded_ecu_info, strict=True) + except Exception as e: + logger.warning(f"{ecu_info_file=} is invalid: {e!r}\n{_raw_yaml_str=}") + logger.warning(f"use default ecu_info: {DEFAULT_ECU_INFO}") + return DEFAULT_ECU_INFO + + +# NOTE(20240327): set the default as literal for now, +# in the future this will be app_cfg.ECU_INFO_FPATH +ecu_info = parse_ecu_info(ecu_info_file="/boot/ota/ecu_info.yaml") diff --git a/otaclient/configs/proxy_info.py b/otaclient/configs/proxy_info.py new file mode 100644 index 000000000..ff0357808 --- /dev/null +++ b/otaclient/configs/proxy_info.py @@ -0,0 +1,153 @@ +# Copyright 2022 TIER IV, INC. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""proxy_info.yaml definition and parsing logic.""" + + +from __future__ import annotations +import logging +import warnings +from functools import cached_property +from typing import Any, ClassVar, Optional +from pathlib import Path + +import yaml +from pydantic import AliasChoices, Field, IPvAnyAddress, AnyHttpUrl +from pydantic_core import Url + +from otaclient._utils.typing import StrOrPath, NetworkPort +from otaclient.configs._common import BaseFixedConfig + +logger = logging.getLogger(__name__) + + +class ProxyInfo(BaseFixedConfig): + """OTA-proxy configuration. + + NOTE 1(20221220): when proxy_info.yaml is missing/not a valid yaml, + a pre_defined proxy_info.yaml as follow will be used. + + Attributes: + format_version: the proxy_info.yaml scheme version, current is 1. + enable_local_ota_proxy: whether to launch a local ota_proxy server. + enable_local_ota_proxy_cache: enable cache mechanism on ota-proxy. + local_ota_proxy_listen_addr: ipaddr ota_proxy listens on. + local_ota_proxy_listen_port: port ota_proxy used. + upper_ota_proxy: the URL of upper OTA proxy used by local ota_proxy server + or otaclient(proxy chain). + logging_server: the URL of AWS IoT otaclient logs upload server. + """ + + format_version: int = 1 + # NOTE(20221219): the default values for the following settings + # now align with v2.5.4 + upper_ota_proxy: Optional[AnyHttpUrl] = None + enable_local_ota_proxy: bool = Field( + default=False, + # NOTE(20240126): "enable_ota_proxy" is superseded by "enable_local_ota_proxy". + validation_alias=AliasChoices( + "enable_local_ota_proxy", + "enable_ota_proxy", + ), + ) + # NOTE(20240327): set the default as literal for now, in the future + # this will be app_cfg.OTA_PROXY_LISTEN_ADDRESS and app_cfg.OTA_PROXY_LISTEN_PORT. + local_ota_proxy_listen_addr: IPvAnyAddress = Field(default="0.0.0.0") + local_ota_proxy_listen_port: NetworkPort = 8082 + # NOTE: this field not presented in v2.5.4, + # for current implementation, it should be default to True. + # This field doesn't take effect if enable_local_ota_proxy is False + enable_local_ota_proxy_cache: bool = True + + # NOTE(20240201): check ota_client_log_server_port var in autoware_ecu_setup + # ansible configurations. + LOGGING_SERVER_PORT: ClassVar[int] = 8083 + # NOTE: when logging_server is not configured, it implicitly means the logging server + # is located at localhost. + # check roles/ota_client/templates/run.sh.j2 in ecu_setup repo. + logging_server: Optional[AnyHttpUrl] = Url( + f"http://127.0.0.1:{LOGGING_SERVER_PORT}" + ) + + def get_proxy_for_local_ota(self) -> str | None: + """Tell local otaclient which proxy to use(or not use any).""" + if self.enable_local_ota_proxy: + # if local otaproxy is enabled, local otaclient also uses it + return f"http://{self.local_ota_proxy_listen_addr}:{self.local_ota_proxy_listen_port}" + elif self.upper_ota_proxy: + # else we directly use the upper proxy + return str(self.upper_ota_proxy) + # default not using proxy + + @cached_property + def gateway_otaproxy(self) -> bool: + """Whether this local otaproxy is a gateway otaproxy. + + Evidence is if no upper_ota_proxy, then this otaproxy should act as a gateway. + NOTE(20240202): this replaces the previous user-configurable gateway field in + the proxy_info.yaml. + """ + return not bool(self.upper_ota_proxy) + + +# deprecated field definition +# -> +_deprecated_field: dict[str, str] = {"enable_ota_proxy": "enable_local_ota_proxy"} + + +def _deprecation_check(_in: dict[str, Any]) -> None: + """ + NOTE: in the future if pydantic support deprecated field annotated, use that + mechanism instead of this function. + """ + for old_fname, _ in _in.items(): + if new_fname := _deprecated_field.get(old_fname): + warnings.warn( + f"option field '{old_fname}' is superseded by '{new_fname}', " + f"and the support for '{old_fname}' option might be dropped in the future. " + f"please use '{new_fname}' in proxy_info.yaml instead.", + DeprecationWarning, + stacklevel=2, + ) + + +# NOTE: this default is for backward compatible with old device +# that doesn't have proxy_info.yaml installed. +DEFAULT_PROXY_INFO = ProxyInfo( + format_version=1, + enable_local_ota_proxy=True, +) + + +def parse_proxy_info(proxy_info_file: StrOrPath) -> ProxyInfo: + try: + _raw_yaml_str = Path(proxy_info_file).read_text() + except FileNotFoundError as e: + logger.warning(f"{proxy_info_file=} not found: {e!r}") + logger.warning(f"use default proxy_info: {DEFAULT_PROXY_INFO}") + return DEFAULT_PROXY_INFO + + try: + loaded_proxy_info = yaml.safe_load(_raw_yaml_str) + assert isinstance(loaded_proxy_info, dict), "not a valid yaml file" + _deprecation_check(loaded_proxy_info) + return ProxyInfo.model_validate(loaded_proxy_info, strict=True) + except Exception as e: + logger.warning(f"{proxy_info_file=} is invalid: {e!r}\n{_raw_yaml_str=}") + logger.warning(f"use default proxy_info: {DEFAULT_PROXY_INFO}") + return DEFAULT_PROXY_INFO + + +# NOTE(20240327): set the default as literal for now, +# in the future this will be app_cfg.PROXY_INFO_FPATH +proxy_info = parse_proxy_info("/boot/ota/proxy_info.yaml") diff --git a/otaclient/requirements.txt b/otaclient/requirements.txt index 1f8491993..1edba4457 100644 --- a/otaclient/requirements.txt +++ b/otaclient/requirements.txt @@ -9,4 +9,6 @@ uvicorn[standard]==0.20.0 aiohttp==3.8.5 aiofiles==22.1.0 zstandard==0.18.0 -typing_extensions==4.6.3 \ No newline at end of file +typing_extensions==4.6.3 +pydantic==2.6.4 +pydantic-settings==2.2.1 \ No newline at end of file diff --git a/tests/test_create_standby.py b/tests/test_create_standby.py index 3a3a6d7f2..1e402f223 100644 --- a/tests/test_create_standby.py +++ b/tests/test_create_standby.py @@ -22,7 +22,6 @@ from otaclient.app.boot_control import BootControllerProtocol from otaclient.app.configs import config as otaclient_cfg -from otaclient.app.proto import wrapper from tests.conftest import TestConfiguration as cfg from tests.utils import SlotMeta, compare_dir @@ -65,7 +64,6 @@ def prepare_ab_slots(self, tmp_path: Path, ab_slots: SlotMeta): @pytest.fixture(autouse=True) def mock_setup(self, mocker: MockerFixture, prepare_ab_slots): - from otaclient.app.proxy_info import ProxyInfo from otaclient.app.configs import BaseConfig # ------ mock boot_controller ------ # diff --git a/tests/test_ecu_info.py b/tests/test_ecu_info.py index 7f954ddbb..7811830c9 100644 --- a/tests/test_ecu_info.py +++ b/tests/test_ecu_info.py @@ -13,24 +13,18 @@ # limitations under the License. -import yaml -import pytest +from __future__ import annotations from pathlib import Path -from typing import Any, List, Dict -from otaclient.app.boot_control import BootloaderType -from otaclient.app.ecu_info import ECUInfo +import pytest -# please refer to ecu_info.py for the DEFAULT_ECU_INFO definition -DEFAULT_ECU_INFO_OBJ = ECUInfo( - format_version=1, - ecu_id="autoware", - bootloader=BootloaderType.UNSPECIFIED.value, +from otaclient.configs.ecu_info import ( + DEFAULT_ECU_INFO, + parse_ecu_info, + ECUInfo, + ECUContact, + BootloaderType, ) -DEFAULT_ECU_INFO_YAML = """\ -format_version: 1 -ecu_id: "autoware" -""" @pytest.mark.parametrize( @@ -40,27 +34,26 @@ # case 1.1: valid yaml(empty file), invalid ecu_info ( "# this is an empty file", - DEFAULT_ECU_INFO_OBJ, + DEFAULT_ECU_INFO, ), # case 1.2: valid yaml(array), invalid ecu_info ( ("- this is an\n" "- yaml file that\n" "- contains a array\n"), - DEFAULT_ECU_INFO_OBJ, + DEFAULT_ECU_INFO, ), # case 1.2: invalid yaml ( " - \n not a \n [ valid yaml", - DEFAULT_ECU_INFO_OBJ, + DEFAULT_ECU_INFO, ), # --- case 2: single ECU --- # # case 2.1: basic single ECU ( ("format_version: 1\n" 'ecu_id: "autoware"\n' 'ip_addr: "192.168.1.1"\n'), ECUInfo( - format_version=1, ecu_id="autoware", - ip_addr="192.168.1.1", - bootloader=BootloaderType.UNSPECIFIED.value, + ip_addr="192.168.1.1", # type: ignore + bootloader=BootloaderType.AUTO_DETECT, ), ), # case 2.2: single ECU with bootloader type specified @@ -72,10 +65,9 @@ 'bootloader: "grub"\n' ), ECUInfo( - format_version=1, ecu_id="autoware", - ip_addr="192.168.1.1", - bootloader=BootloaderType.GRUB.value, + ip_addr="192.168.1.1", # type: ignore + bootloader=BootloaderType.GRUB, ), ), # --- case 3: multiple ECUs --- # @@ -93,20 +85,19 @@ ' ip_addr: "192.168.0.12"\n' ), ECUInfo( - format_version=1, ecu_id="autoware", ip_addr="192.168.1.1", - bootloader=BootloaderType.UNSPECIFIED.value, + bootloader=BootloaderType.AUTO_DETECT, available_ecu_ids=["autoware", "p1", "p2"], secondaries=[ - { - "ecu_id": "p1", - "ip_addr": "192.168.0.11", - }, - { - "ecu_id": "p2", - "ip_addr": "192.168.0.12", - }, + ECUContact( + ecu_id="p1", + ip_addr="192.168.0.11", # type: ignore + ), + ECUContact( + ecu_id="p2", + ip_addr="192.168.0.12", # type: ignore + ), ], ), ), @@ -125,20 +116,19 @@ ' ip_addr: "192.168.0.12"\n' ), ECUInfo( - format_version=1, ecu_id="autoware", ip_addr="192.168.1.1", + bootloader=BootloaderType.GRUB, available_ecu_ids=["autoware", "p1", "p2"], - bootloader=BootloaderType.GRUB.value, secondaries=[ - { - "ecu_id": "p1", - "ip_addr": "192.168.0.11", - }, - { - "ecu_id": "p2", - "ip_addr": "192.168.0.12", - }, + ECUContact( + ecu_id="p1", + ip_addr="192.168.0.11", # type: ignore + ), + ECUContact( + ecu_id="p2", + ip_addr="192.168.0.12", # type: ignore + ), ], ), ), @@ -150,7 +140,7 @@ def test_ecu_info(tmp_path: Path, ecu_info_yaml: str, expected_ecu_info: ECUInfo (ecu_info_file := ota_dir / "ecu_info.yaml").write_text(ecu_info_yaml) # --- execution --- # - loaded_ecu_info = ECUInfo.parse_ecu_info(ecu_info_file) + loaded_ecu_info = parse_ecu_info(ecu_info_file) # --- assertion --- # assert loaded_ecu_info == expected_ecu_info diff --git a/tests/test_ota_client.py b/tests/test_ota_client.py index 65eacf29f..53e55ffb0 100644 --- a/tests/test_ota_client.py +++ b/tests/test_ota_client.py @@ -29,8 +29,7 @@ from otaclient.app.create_standby import StandbySlotCreatorProtocol from otaclient.app.create_standby.common import DeltaBundle, RegularDelta from otaclient.app.configs import config as otaclient_cfg -from otaclient.app.ecu_info import ECUInfo -from otaclient.app.errors import OTAError, OTAErrorRecoverable +from otaclient.app.errors import OTAErrorRecoverable from otaclient.app.ota_client import ( OTAClient, _OTAUpdater, @@ -40,6 +39,7 @@ from otaclient.app.ota_metadata import parse_regulars_from_txt, parse_dirs_from_txt from otaclient.app.proto.wrapper import RegularInf, DirectoryInf from otaclient.app.proto import wrapper +from otaclient.configs.ecu_info import ECUInfo from tests.conftest import TestConfiguration as cfg from tests.utils import SlotMeta @@ -358,11 +358,10 @@ def test_status_in_update(self): class TestOTAServicer: - BOOTLOADER_TYPE = BootloaderType.GRUB ECU_INFO = ECUInfo( format_version=1, ecu_id="my_ecu_id", - bootloader=BOOTLOADER_TYPE.value, + bootloader=BootloaderType.GRUB, available_ecu_ids=["my_ecu_id"], secondaries=[], ) @@ -388,15 +387,16 @@ async def mock_setup(self, mocker: pytest_mock.MockerFixture): f"{cfg.OTACLIENT_MODULE_PATH}.get_standby_slot_creator", return_value=self.standby_slot_creator_cls, ) + mocker.patch(f"{cfg.OTACLIENT_MODULE_PATH}.ecu_info", self.ECU_INFO) # # ------ start OTAServicer instance ------ # self.local_use_proxy = "" self.otaclient_stub = OTAServicer( - ecu_info=self.ECU_INFO, executor=self._executor, control_flags=self.control_flags, + otaclient_version="otaclient test version", proxy=self.local_use_proxy, ) diff --git a/tests/test_ota_client_service.py b/tests/test_ota_client_service.py index 9a2482a0e..fd440d0cd 100644 --- a/tests/test_ota_client_service.py +++ b/tests/test_ota_client_service.py @@ -13,15 +13,17 @@ # limitations under the License. +from __future__ import annotations import asyncio import pytest import pytest_mock from otaclient.app.configs import server_cfg -from otaclient.app.ecu_info import ECUInfo from otaclient.app.ota_client_service import create_otaclient_grpc_server from otaclient.app.ota_client_call import OtaClientCall from otaclient.app.proto import wrapper +from otaclient.configs.ecu_info import ECUInfo + from tests.conftest import cfg from tests.utils import compare_message @@ -72,13 +74,11 @@ def setup_test(self, mocker: pytest_mock.MockerFixture): return_value=self.otaclient_service_stub, ) - ecu_info_mock = mocker.MagicMock(spec=ECUInfo) - # NOTE: mocked to use 127.0.0.1, and still use server_cfg.SERVER_PORT - ecu_info_mock.parse_ecu_info.return_value = ECUInfo( + self.ecu_info_mock = ecu_info_mock = ECUInfo( ecu_id=self.otaclient_service_stub.MY_ECU_ID, - ip_addr=self.LISTEN_ADDR, + ip_addr=self.LISTEN_ADDR, # type: ignore ) - mocker.patch(f"{cfg.OTACLIENT_SERVICE_MODULE_PATH}.ECUInfo", ecu_info_mock) + mocker.patch(f"{cfg.OTACLIENT_SERVICE_MODULE_PATH}.ecu_info", ecu_info_mock) @pytest.fixture(autouse=True) async def launch_otaclient_server(self, setup_test): diff --git a/tests/test_ota_client_stub.py b/tests/test_ota_client_stub.py index a741aeb43..226af4360 100644 --- a/tests/test_ota_client_stub.py +++ b/tests/test_ota_client_stub.py @@ -13,15 +13,16 @@ # limitations under the License. +from __future__ import annotations import asyncio +import logging import pytest from concurrent.futures import ThreadPoolExecutor from pathlib import Path from typing import Any, Dict, List, Set -import pytest_mock +from pytest_mock import MockerFixture -from otaclient.app.ecu_info import ECUInfo from otaclient.app.ota_client import OTAServicer from otaclient.app.ota_client_call import OtaClientCall from otaclient.app.ota_client_stub import ( @@ -30,15 +31,14 @@ OTAProxyLauncher, ) from otaclient.app.proto import wrapper -from otaclient.app.proxy_info import ProxyInfo +from otaclient.configs.ecu_info import ECUInfo, parse_ecu_info +from otaclient.configs.proxy_info import ProxyInfo, parse_proxy_info from otaclient.ota_proxy import OTAProxyContextProto from otaclient.ota_proxy.config import Config as otaproxyConfig from tests.utils import compare_message from tests.conftest import cfg -import logging - logger = logging.getLogger(__name__) @@ -60,6 +60,27 @@ - "p2" """ +PROXY_INFO_YAML = """\ +gateway_otaproxy: false, +enable_local_ota_proxy: true +local_ota_proxy_listen_addr: "127.0.0.1" +local_ota_proxy_listen_port: 8082 +""" + + +@pytest.fixture +def ecu_info_fixture(tmp_path: Path) -> ECUInfo: + _yaml_f = tmp_path / "ecu_info.yaml" + _yaml_f.write_text(ECU_INFO_YAML) + return parse_ecu_info(_yaml_f) + + +@pytest.fixture +def proxy_info_fixture(tmp_path: Path) -> ProxyInfo: + _yaml_f = tmp_path / "proxy_info.yaml" + _yaml_f.write_text(PROXY_INFO_YAML) + return parse_proxy_info(_yaml_f) + class _DummyOTAProxyContext(OTAProxyContextProto): def __init__(self, sentinel) -> None: @@ -80,28 +101,30 @@ def __exit__(self, __exc_type, __exc_value, __traceback): class TestOTAProxyLauncher: @pytest.fixture(autouse=True) - async def mock_setup(self, tmp_path: Path): - proxy_info = ProxyInfo( - gateway=False, - upper_ota_proxy="", - enable_local_ota_proxy=True, - local_ota_proxy_listen_addr="127.0.0.1", - local_ota_proxy_listen_port=8082, - ) - proxy_server_cfg = otaproxyConfig() - + async def mock_setup( + self, mocker: MockerFixture, tmp_path: Path, proxy_info_fixture + ): cache_base_dir = tmp_path / "ota_cache" self.sentinel_file = tmp_path / "otaproxy_sentinel" + + # ------ prepare mocked proxy_info and otaproxy_cfg ------ # + self.proxy_info = proxy_info = proxy_info_fixture + self.proxy_server_cfg = proxy_server_cfg = otaproxyConfig() proxy_server_cfg.BASE_DIR = str(cache_base_dir) # type: ignore proxy_server_cfg.DB_FILE = str(cache_base_dir / "cache_db") # type: ignore + # ------ apply cfg patches ------ # + mocker.patch(f"{cfg.OTACLIENT_STUB_MODULE_PATH}.proxy_info", proxy_info) + mocker.patch( + f"{cfg.OTACLIENT_STUB_MODULE_PATH}.local_otaproxy_cfg", + proxy_server_cfg, + ) + # init launcher inst threadpool = ThreadPoolExecutor() self.otaproxy_launcher = OTAProxyLauncher( executor=threadpool, subprocess_ctx=_DummyOTAProxyContext(str(self.sentinel_file)), - _proxy_info=proxy_info, - _proxy_server_cfg=proxy_server_cfg, ) try: @@ -137,13 +160,15 @@ class TestECUStatusStorage: SAFE_INTERVAL_FOR_PROPERTY_UPDATE = 1.2 @pytest.fixture(autouse=True) - async def setup_test(self, tmp_path: Path): - ecu_info_f = tmp_path / "ecu_info.yml" - ecu_info_f.write_text(ECU_INFO_YAML) - self.ecu_info = ECUInfo.parse_ecu_info(ecu_info_f) + async def setup_test(self, mocker: MockerFixture, ecu_info_fixture): + # ------ load test ecu_info.yaml ------ # + self.ecu_info = ecu_info = ecu_info_fixture + + # ------ apply cfg patches ------ # + mocker.patch(f"{cfg.OTACLIENT_STUB_MODULE_PATH}.ecu_info", ecu_info) # init and setup the ecu_storage - self.ecu_storage = ECUStatusStorage(self.ecu_info) + self.ecu_storage = ECUStatusStorage() # NOTE: decrease the interval for faster testing self.ecu_storage.PROPERTY_REFRESH_INTERVAL = self.PROPERTY_REFRESH_INTERVAL_FOR_TEST # type: ignore @@ -694,16 +719,17 @@ async def _subecu_accept_update_request(ecu_id, *args, **kwargs): ) @pytest.fixture(autouse=True) - async def setup_test(self, tmp_path: Path, mocker: pytest_mock.MockerFixture): + async def setup_test( + self, mocker: MockerFixture, ecu_info_fixture, proxy_info_fixture + ): threadpool = ThreadPoolExecutor() - # prepare ecu_info - ecu_info_f = tmp_path / "ecu_info.yml" - ecu_info_f.write_text(ECU_INFO_YAML) - self.ecu_info = ECUInfo.parse_ecu_info(ecu_info_f) + # ------ mock and patch ecu_info ------ # + self.ecu_info = ecu_info = ecu_info_fixture + mocker.patch(f"{cfg.OTACLIENT_STUB_MODULE_PATH}.ecu_info", ecu_info) - # init and setup the ecu_storage - self.ecu_storage = ECUStatusStorage(self.ecu_info) + # ------ init and setup the ecu_storage ------ # + self.ecu_storage = ECUStatusStorage() self.ecu_storage.on_ecus_accept_update_request = mocker.AsyncMock() # NOTE: decrease the interval to speed up testing # (used by _otaproxy_lifecycle_managing/_otaclient_control_flags_managing task) @@ -723,11 +749,10 @@ async def setup_test(self, tmp_path: Path, mocker: pytest_mock.MockerFixture): self.otaclient_call.update_call = mocker.AsyncMock( wraps=self._subecu_accept_update_request ) - # proxy_info - self.proxy_info = ProxyInfo( - enable_local_ota_proxy=True, - upper_ota_proxy="", - ) + + # ------ mock and patch proxy_info ------ # + self.proxy_info = proxy_info = proxy_info_fixture + mocker.patch(f"{cfg.OTACLIENT_STUB_MODULE_PATH}.proxy_info", proxy_info) # --- patching and mocking --- # mocker.patch( @@ -751,9 +776,7 @@ async def setup_test(self, tmp_path: Path, mocker: pytest_mock.MockerFixture): ) # --- start the OTAClientServiceStub --- # - self.otaclient_service_stub = OTAClientServiceStub( - ecu_info=self.ecu_info, _proxy_cfg=self.proxy_info - ) + self.otaclient_service_stub = OTAClientServiceStub() try: yield diff --git a/tests/test_proxy_info.py b/tests/test_proxy_info.py index 3940e091b..2a3cd8e00 100644 --- a/tests/test_proxy_info.py +++ b/tests/test_proxy_info.py @@ -13,107 +13,96 @@ # limitations under the License. +from __future__ import annotations import logging import pytest -from dataclasses import asdict from pathlib import Path -from typing import Any, Dict -logger = logging.getLogger(__name__) - -# pre-defined proxy_info.yaml for when -# proxy_info.yaml is missing/not found -PRE_DEFINED_PROXY_INFO_YAML = """ -enable_local_ota_proxy: true -gateway: true -""" -# parsed pre-defined proxy_info.yaml -PARSED_PRE_DEFINED_PROXY_INFO_DICT = { - "gateway": True, - "upper_ota_proxy": "", - "enable_local_ota_proxy": True, - "enable_local_ota_proxy_cache": True, - "local_ota_proxy_listen_addr": "0.0.0.0", - "local_ota_proxy_listen_port": 8082, -} - -PERCEPTION_ECU_PROXY_INFO_YAML = """ -gateway: false -enable_local_ota_proxy: true -upper_ota_proxy: "http://10.0.0.1:8082" -enable_local_ota_proxy_cache: true -""" +from otaclient.configs.proxy_info import ( + parse_proxy_info, + DEFAULT_PROXY_INFO, + ProxyInfo, +) -# Bad configured yaml file that contains invalid value. -# All fields are assigned with invalid value, -# all invalid fields should be replaced by default value. -BAD_CONFIGURED_PROXY_INFO_YAML = """ -enable_local_ota_proxy: dafef -gateway: 123 -upper_ota_proxy: true -enable_local_ota_proxy_cache: adfaea -local_ota_proxy_listen_addr: 123 -local_ota_proxy_listen_port: "2808" -""" -# default setting for each config fields -# NOTE: check docs/README.md for details -DEFAULT_SETTINGS_FOR_PROXY_INFO_DICT = { - "gateway": False, - "upper_ota_proxy": "", - "enable_local_ota_proxy": False, - "enable_local_ota_proxy_cache": True, - "local_ota_proxy_listen_addr": "0.0.0.0", - "local_ota_proxy_listen_port": 8082, -} +logger = logging.getLogger(__name__) @pytest.mark.parametrize( "_input_yaml, _expected", ( - # case 1: testing when proxy_info.yaml is missing + # ------ case 1: proxy_info.yaml is missing ------ # # this case is for single ECU that doesn't have proxy_info.yaml and - # can directly connect to the remote + # can directly connect to the remote. + # NOTE: this default value is for x1 backward compatibility. ( - PRE_DEFINED_PROXY_INFO_YAML, - PARSED_PRE_DEFINED_PROXY_INFO_DICT, + "# this is an empty file", + DEFAULT_PROXY_INFO, ), - # case 2: tesing typical sub ECU setting + # ------ case 2: typical sub ECU's proxy_info.yaml ------ # ( - PERCEPTION_ECU_PROXY_INFO_YAML, - { - "gateway": False, - "upper_ota_proxy": "http://10.0.0.1:8082", - "enable_local_ota_proxy": True, - "enable_local_ota_proxy_cache": True, - "local_ota_proxy_listen_addr": "0.0.0.0", - "local_ota_proxy_listen_port": 8082, - }, + ( + "enable_local_ota_proxy: true\n" + 'upper_ota_proxy: "http://10.0.0.1:8082"\n' + "enable_local_ota_proxy_cache: true\n" + 'logging_server: "http://10.0.0.1:8083"\n' + ), + ProxyInfo( + **{ + "format_version": 1, + "upper_ota_proxy": "http://10.0.0.1:8082", + "enable_local_ota_proxy": True, + "enable_local_ota_proxy_cache": True, + "local_ota_proxy_listen_addr": "0.0.0.0", + "local_ota_proxy_listen_port": 8082, + "logging_server": "http://10.0.0.1:8083", + } + ), ), - # case 3: testing invalid/corrupted proxy_info.yaml + # ------ case 3: invalid/corrupted proxy_info.yaml ------ # # If the proxy_info.yaml is not a yaml, otaclient will also treat # this case the same as proxy_info.yaml missing, the pre-defined # proxy_info.yaml will be used. ( "not a valid proxy_info.yaml", - PARSED_PRE_DEFINED_PROXY_INFO_DICT, + DEFAULT_PROXY_INFO, ), - # case 4: testing default settings against invalid fields - # all config fields are expected to be assigned with default setting. - # NOTE: if proxy_info.yaml is valid yaml but fields are invalid, - # default value settings will be applied. - # NOTE 2: not the same as [case1: proxy_info.yaml missing/corrupted]! + # ------ case 4: proxy_info.yaml is valid yaml but contains invalid fields ------ # + # in this case, default predefined default proxy_info.yaml will be loaded + # NOTE(20240126): Previous behavior is invalid field will be replaced by default value, + # and other fields will be preserved as much as possible. + # This is not proper, now the behavior changes to otaclient will treat + # the whole config file as invalid and load the default config. ( - BAD_CONFIGURED_PROXY_INFO_YAML, - DEFAULT_SETTINGS_FOR_PROXY_INFO_DICT, + # Bad configured yaml file that contains invalid value. + # This yaml file is valid, but all fields' values are invalid. + ( + "enable_local_ota_proxy: dafef\n" + "upper_ota_proxy: true\n" + "enable_local_ota_proxy_cache: adfaea\n" + "local_ota_proxy_listen_addr: 123\n" + 'local_ota_proxy_listen_port: "2808"\n' + ), + DEFAULT_PROXY_INFO, + ), + # ------ case 5: corrupted/invalid yaml ------ # + # in this case, default predefined default proxy_info.yaml will be loaded + ( + "/t/t/t/t/t/t/t/tyaml file should not contain tabs/t/t/t/", + DEFAULT_PROXY_INFO, + ), + # ------ case 6: backward compatibility test ------ # + # for superseded field, the corresponding field should be assigned. + # for removed field, it should not impact the config file loading. + ( + "enable_ota_proxy: true\ngateway: false\n", + DEFAULT_PROXY_INFO, ), ), ) -def test_proxy_info(tmp_path: Path, _input_yaml: str, _expected: Dict[str, Any]): - from otaclient.app.proxy_info import parse_proxy_info - +def test_proxy_info(tmp_path: Path, _input_yaml: str, _expected: ProxyInfo): proxy_info_file = tmp_path / "proxy_info.yml" proxy_info_file.write_text(_input_yaml) _proxy_info = parse_proxy_info(str(proxy_info_file)) - assert asdict(_proxy_info) == _expected + assert _proxy_info == _expected