diff --git a/otaclient/_utils/__init__.py b/otaclient/_utils/__init__.py index 4b7060019..965fca823 100644 --- a/otaclient/_utils/__init__.py +++ b/otaclient/_utils/__init__.py @@ -11,34 +11,29 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Callable, TypeVar -from typing_extensions import ParamSpec, Concatenate - -P = ParamSpec("P") - -def copy_callable_typehint(_source: Callable[P, Any]): - """This helper function return a decorator that can type hint the target - function as the _source function. - At runtime, this decorator actually does nothing, but just return the input function as it. - But the returned function will have the same type hint as the source function in ide. - It will not impact the runtime behavior of the decorated function. - """ - - def _decorator(target) -> Callable[P, Any]: - return target - - return _decorator +from __future__ import annotations +import os.path +from functools import cached_property +from pydantic import computed_field +from typing import Any, Callable, TypeVar +T = TypeVar("T") -RT = TypeVar("RT") +_CONTAINER_INDICATOR_FILES = [ + "/.dockerenv", + "/run/.dockerenv", + "/run/.containerenv", +] -def copy_callable_typehint_to_method(_source: Callable[P, Any]): - """Works the same as copy_callable_typehint, but omit the first arg.""" +def if_run_as_container() -> bool: + for indicator in _CONTAINER_INDICATOR_FILES: + if os.path.isfile(indicator): + return True + return False - def _decorator(target: Callable[..., RT]) -> Callable[Concatenate[Any, P], RT]: - return target # type: ignore - return _decorator +def cached_computed_field(_f: Callable[[Any], Any]) -> cached_property[Any]: + return computed_field(cached_property(_f)) diff --git a/otaclient/_utils/logging.py b/otaclient/_utils/logging.py index 6865ca30f..cbae06dee 100644 --- a/otaclient/_utils/logging.py +++ b/otaclient/_utils/logging.py @@ -61,3 +61,9 @@ def filter(self, _: logging.LogRecord) -> bool: ) self._round_warned = True return False + + +def check_loglevel(_in: int) -> int: + """Pydantic validator for logging level number.""" + assert _in in logging._levelToName, f"{_in} is not a valid logging level" + return _in diff --git a/otaclient/_utils/path.py b/otaclient/_utils/path.py new file mode 100644 index 000000000..4bd343d8a --- /dev/null +++ b/otaclient/_utils/path.py @@ -0,0 +1,35 @@ +# Copyright 2022 TIER IV, INC. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os.path +from otaclient._utils.typing import StrOrPath + + +def replace_root(path: StrOrPath, old_root: StrOrPath, new_root: StrOrPath) -> str: + """Replace a relative to to . + + For example, if path="/abc", old_root="/", new_root="/new_root", + then we will have "/new_root/abc". + """ + # normalize all the input args + path = os.path.normpath(path) + old_root = os.path.normpath(old_root) + new_root = os.path.normpath(new_root) + + if not (old_root.startswith("/") and new_root.startswith("/")): + raise ValueError(f"{old_root=} and/or {new_root=} is not valid root") + if os.path.commonpath([path, old_root]) != old_root: + raise ValueError(f"{old_root=} is not the root of {path=}") + return os.path.join(new_root, os.path.relpath(path, old_root)) diff --git a/otaclient/_utils/typing.py b/otaclient/_utils/typing.py new file mode 100644 index 000000000..a77af36e1 --- /dev/null +++ b/otaclient/_utils/typing.py @@ -0,0 +1,51 @@ +# Copyright 2022 TIER IV, INC. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from pathlib import Path +from typing import Any, Callable, TypeVar, Union +from typing_extensions import Concatenate, ParamSpec + +P = ParamSpec("P") +RT = TypeVar("RT") + +StrOrPath = Union[str, Path] + + +def copy_callable_typehint(_source: Callable[P, Any]): + """This helper function return a decorator that can type hint the target + function as the _source function. + + At runtime, this decorator actually does nothing, but just return the input function as it. + But the returned function will have the same type hint as the source function in ide. + It will not impact the runtime behavior of the decorated function. + """ + + def _decorator(target) -> Callable[P, Any]: + return target + + return _decorator + + +def copy_callable_typehint_to_method(_source: Callable[P, Any]): + """Works the same as copy_callable_typehint, but omit the first arg.""" + + def _decorator(target: Callable[..., RT]) -> Callable[Concatenate[Any, P], RT]: + return target # type: ignore + + return _decorator + + +def check_port(_in: Any) -> bool: + return isinstance(_in, int) and _in >= 0 and _in <= 65535 diff --git a/otaclient/app/boot_control/_cboot.py b/otaclient/app/boot_control/_cboot.py index 512789b3b..e7f725d51 100644 --- a/otaclient/app/boot_control/_cboot.py +++ b/otaclient/app/boot_control/_cboot.py @@ -20,7 +20,7 @@ from subprocess import CalledProcessError from typing import Generator, Optional - +from ..configs import config as cfg from .. import log_setting, errors as ota_errors from ..common import ( copytree_identical, @@ -39,14 +39,12 @@ SlotInUseMixin, VersionControlMixin, ) -from .configs import cboot_cfg as cfg +from .configs import cboot_cfg as boot_cfg from .protocol import BootControllerProtocol from .firmware import Firmware -logger = log_setting.get_logger( - __name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL) -) +logger = log_setting.get_logger(__name__) class NvbootctrlError(Exception): @@ -160,14 +158,14 @@ class _CBootControl: def __init__(self): # NOTE: only support rqx-580, rqx-58g platform right now! # detect the chip id - self.chip_id = read_str_from_file(cfg.TEGRA_CHIP_ID_PATH) - if not self.chip_id or int(self.chip_id) not in cfg.CHIP_ID_MODEL_MAP: + self.chip_id = read_str_from_file(boot_cfg.TEGRA_CHIP_ID_FPATH) + if not self.chip_id or int(self.chip_id) not in boot_cfg.CHIP_ID_MODEL_MAP: raise NotImplementedError( f"unsupported platform found (chip_id: {self.chip_id}), abort" ) self.chip_id = int(self.chip_id) - self.model = cfg.CHIP_ID_MODEL_MAP[self.chip_id] + self.model = boot_cfg.CHIP_ID_MODEL_MAP[self.chip_id] logger.info(f"{self.model=}, (chip_id={hex(self.chip_id)})") # initializing dev info @@ -300,33 +298,36 @@ def __init__(self) -> None: try: self._cboot_control: _CBootControl = _CBootControl() + # ------ prepare mount space ------ # + otaclient_ms = Path(cfg.OTACLIENT_MOUNT_SPACE_DPATH) + otaclient_ms.mkdir(exist_ok=True, parents=True) + otaclient_ms.chmod(0o700) + # load paths ## first try to unmount standby dev if possible self.standby_slot_dev = self._cboot_control.get_standby_rootfs_dev() CMDHelperFuncs.umount(self.standby_slot_dev) - self.standby_slot_mount_point = Path(cfg.MOUNT_POINT) - self.standby_slot_mount_point.mkdir(exist_ok=True) + self.standby_slot_mount_point = Path(cfg.STANDBY_SLOT_MP) + self.standby_slot_mount_point.mkdir(exist_ok=True, parents=True) ## refroot mount point - _refroot_mount_point = cfg.ACTIVE_ROOT_MOUNT_POINT - # first try to umount refroot mount point - CMDHelperFuncs.umount(_refroot_mount_point) - if not os.path.isdir(_refroot_mount_point): - os.mkdir(_refroot_mount_point) + _refroot_mount_point = cfg.ACTIVE_SLOT_MP self.ref_slot_mount_point = Path(_refroot_mount_point) + if os.path.isdir(_refroot_mount_point): + # first try to umount refroot mount point + CMDHelperFuncs.umount(_refroot_mount_point) + elif not os.path.exists(_refroot_mount_point): + self.ref_slot_mount_point.mkdir(exist_ok=True, parents=True) + ## ota-status dir ### current slot - self.current_ota_status_dir = Path(cfg.ACTIVE_ROOTFS_PATH) / Path( - cfg.OTA_STATUS_DIR - ).relative_to("/") + self.current_ota_status_dir = Path(boot_cfg.ACTIVE_BOOT_OTA_STATUS_DPATH) self.current_ota_status_dir.mkdir(parents=True, exist_ok=True) ### standby slot # NOTE: might not yet be populated before OTA update applied! - self.standby_ota_status_dir = self.standby_slot_mount_point / Path( - cfg.OTA_STATUS_DIR - ).relative_to("/") + self.standby_ota_status_dir = Path(boot_cfg.STANDBY_BOOT_OTA_STATUS_DPATH) # init ota-status self._init_boot_control() @@ -418,7 +419,7 @@ def _is_switching_boot(self) -> bool: def _populate_boot_folder_to_separate_bootdev(self): # mount the actual standby_boot_dev now - _boot_dir_mount_point = Path(cfg.SEPARATE_BOOT_MOUNT_POINT) + _boot_dir_mount_point = Path(boot_cfg.SEPARATE_BOOT_MOUNT_POINT) _boot_dir_mount_point.mkdir(exist_ok=True, parents=True) try: @@ -513,16 +514,11 @@ def pre_update(self, version: str, *, standby_as_ref: bool, erase_standby=False) def post_update(self) -> Generator[None, None, None]: try: # firmware update - firmware = Firmware( - self.standby_slot_mount_point - / Path(cfg.FIRMWARE_CONFIG).relative_to("/") - ) + firmware = Firmware(Path(boot_cfg.FIRMWARE_CFG_STANDBY_FPATH)) firmware.update(int(self._cboot_control.get_standby_slot())) # update extlinux_cfg file - _extlinux_cfg = self.standby_slot_mount_point / Path( - cfg.EXTLINUX_FILE - ).relative_to("/") + _extlinux_cfg = Path(boot_cfg.STANDBY_EXTLINUX_FPATH) self._cboot_control.update_extlinux_cfg( dst=_extlinux_cfg, ref=_extlinux_cfg, diff --git a/otaclient/app/boot_control/_common.py b/otaclient/app/boot_control/_common.py index be3444ae8..e41269c03 100644 --- a/otaclient/app/boot_control/_common.py +++ b/otaclient/app/boot_control/_common.py @@ -21,6 +21,7 @@ from subprocess import CalledProcessError from typing import List, Optional, Union, Callable +from otaclient._utils.path import replace_root from ._errors import ( BootControlError, MountError, @@ -39,9 +40,7 @@ from ..proto import wrapper -logger = log_setting.get_logger( - __name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL) -) +logger = log_setting.get_logger(__name__) class CMDHelperFuncs: @@ -798,9 +797,7 @@ def __init__( # NOTE(20230907): this will always be /boot, # in the future this attribute will not be used by # standby slot creater. - self.standby_boot_dir = self.standby_slot_mount_point / Path( - cfg.BOOT_DIR - ).relative_to("/") + self.standby_boot_dir = self.standby_slot_mount_point / "boot" def mount_standby(self, *, raise_exc: bool = True) -> bool: """Mount standby slot dev to . @@ -861,12 +858,15 @@ def preserve_ota_folder_to_standby(self): so we should preserve it for each slot, accross each update. """ logger.debug("copy /boot/ota from active to standby.") + + _src = Path(cfg.BOOT_OTA_DPATH) + _dst = Path( + replace_root(cfg.BOOT_OTA_DPATH, cfg.ACTIVE_ROOTFS, cfg.STANDBY_SLOT_MP) + ) try: - _src = self.active_slot_mount_point / Path(cfg.OTA_DIR).relative_to("/") - _dst = self.standby_slot_mount_point / Path(cfg.OTA_DIR).relative_to("/") shutil.copytree(_src, _dst, dirs_exist_ok=True) except Exception as e: - raise ValueError(f"failed to copy /boot/ota from active to standby: {e!r}") + raise ValueError(f"failed to copy {_src=} to {_dst=}: {e!r}") def umount_all(self, *, ignore_error: bool = False): logger.debug("unmount standby slot and active slot mount point...") diff --git a/otaclient/app/boot_control/_grub.py b/otaclient/app/boot_control/_grub.py index 8c3bf8971..e9cdc236b 100644 --- a/otaclient/app/boot_control/_grub.py +++ b/otaclient/app/boot_control/_grub.py @@ -40,6 +40,7 @@ from pprint import pformat from .. import log_setting, errors as ota_errors +from ..configs import config as cfg from ..common import ( re_symlink_atomic, read_str_from_file, @@ -55,13 +56,11 @@ SlotMountHelper, cat_proc_cmdline, ) -from .configs import grub_cfg as cfg +from .configs import grub_cfg as boot_cfg from .protocol import BootControllerProtocol -logger = log_setting.get_logger( - __name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL) -) +logger = log_setting.get_logger(__name__) class _GrubBootControllerError(Exception): @@ -390,17 +389,17 @@ def __init__(self) -> None: f"{self.active_slot=}@{self.active_root_dev}, {self.standby_slot=}@{self.standby_root_dev}" ) - self.boot_dir = Path(cfg.BOOT_DIR) - self.grub_file = Path(cfg.GRUB_CFG_PATH) + self.boot_dir = Path(cfg.BOOT_DPATH) + self.grub_file = Path(boot_cfg.GRUB_CFG_FPATH) - self.ota_partition_symlink = self.boot_dir / cfg.BOOT_OTA_PARTITION_FILE + self.ota_partition_symlink = self.boot_dir / boot_cfg.BOOT_OTA_PARTITION_FNAME self.active_ota_partition_folder = ( - self.boot_dir / cfg.BOOT_OTA_PARTITION_FILE + self.boot_dir / boot_cfg.BOOT_OTA_PARTITION_FNAME ).with_suffix(f".{self.active_slot}") self.active_ota_partition_folder.mkdir(exist_ok=True) self.standby_ota_partition_folder = ( - self.boot_dir / cfg.BOOT_OTA_PARTITION_FILE + self.boot_dir / boot_cfg.BOOT_OTA_PARTITION_FNAME ).with_suffix(f".{self.standby_slot}") self.standby_ota_partition_folder.mkdir(exist_ok=True) @@ -516,7 +515,7 @@ def _get_current_booted_files() -> Tuple[str, str]: # lookup the grub file and find the booted entry # NOTE(20230905): use standard way to find initrd img initrd_img = f"{GrubHelper.INITRD}{GrubHelper.FNAME_VER_SPLITTER}{kernel_ver}" - if not (Path(cfg.BOOT_DIR) / initrd_img).is_file(): + if not (Path(cfg.BOOT_DPATH) / initrd_img).is_file(): raise ValueError(f"failed to find booted initrd image({initrd_img})") return kernel_ma.group("kernel"), initrd_img @@ -556,10 +555,7 @@ def _grub_update_on_booted_slot(self, *, abort_on_standby_missed=True): 1. this method only ensures the entry existence for current booted slot. 2. this method ensures the default entry to be the current booted slot. """ - grub_default_file = Path(cfg.ACTIVE_ROOTFS_PATH) / Path( - cfg.DEFAULT_GRUB_PATH - ).relative_to("/") - + grub_default_file = Path(boot_cfg.GRUB_DEFAULT_FPATH) # NOTE: If the path points to a symlink, exists() returns # whether the symlink points to an existing file or directory. active_vmlinuz = self.boot_dir / GrubHelper.KERNEL_OTA @@ -601,9 +597,11 @@ def _grub_update_on_booted_slot(self, *, abort_on_standby_missed=True): logger.debug(f"generated grub_default: {pformat(_out)}") write_str_to_file_sync(grub_default_file, _out) - # step4: populate new active grub_file + # step4: populate new grub.cfg to active slot's ota-partition folder # update the ota.standby entry's rootfs uuid to standby slot's uuid - active_slot_grub_file = self.active_ota_partition_folder / cfg.GRUB_CFG_FNAME + active_slot_grub_file = ( + self.active_ota_partition_folder / boot_cfg.GRUB_CFG_FNAME + ) grub_cfg_content = GrubHelper.grub_mkconfig() standby_uuid_str = CMDHelperFuncs.get_uuid_str_by_dev(self.standby_root_dev) @@ -631,14 +629,14 @@ def _grub_update_on_booted_slot(self, *, abort_on_standby_missed=True): # finally, point grub.cfg to active slot's grub.cfg re_symlink_atomic( # /boot/grub/grub.cfg -> ../ota-partition/grub.cfg self.grub_file, - Path("../") / cfg.BOOT_OTA_PARTITION_FILE / "grub.cfg", + Path("../") / boot_cfg.BOOT_OTA_PARTITION_FNAME / boot_cfg.GRUB_CFG_FNAME, ) logger.info(f"update_grub for {self.active_slot} finished.") def _ensure_ota_partition_symlinks(self, active_slot: str): """Ensure /boot/{ota_partition,vmlinuz-ota,initrd.img-ota} symlinks from specified point's of view.""" - ota_partition_folder = Path(cfg.BOOT_OTA_PARTITION_FILE) # ota-partition + ota_partition_folder = Path(boot_cfg.BOOT_OTA_PARTITION_FNAME) # ota-partition re_symlink_atomic( # /boot/ota-partition -> ota-partition. self.boot_dir / ota_partition_folder, ota_partition_folder.with_suffix(f".{active_slot}"), @@ -654,7 +652,7 @@ def _ensure_ota_partition_symlinks(self, active_slot: str): def _ensure_standby_slot_boot_files_symlinks(self, standby_slot: str): """Ensure boot files symlinks for specified .""" - ota_partition_folder = Path(cfg.BOOT_OTA_PARTITION_FILE) # ota-partition + ota_partition_folder = Path(boot_cfg.BOOT_OTA_PARTITION_FNAME) # ota-partition re_symlink_atomic( # /boot/vmlinuz-ota.standby -> ota-partition./vmlinuz-ota self.boot_dir / GrubHelper.KERNEL_OTA_STANDBY, ota_partition_folder.with_suffix(f".{standby_slot}") @@ -744,12 +742,19 @@ class GrubController(BootControllerProtocol): def __init__(self) -> None: try: self._boot_control = _GrubControl() + + # ------ prepare mount space ------ # + otaclient_ms = Path(cfg.OTACLIENT_MOUNT_SPACE_DPATH) + otaclient_ms.mkdir(exist_ok=True, parents=True) + otaclient_ms.chmod(0o700) + self._mp_control = SlotMountHelper( standby_slot_dev=self._boot_control.standby_root_dev, - standby_slot_mount_point=cfg.MOUNT_POINT, + standby_slot_mount_point=cfg.STANDBY_SLOT_MP, active_slot_dev=self._boot_control.active_root_dev, - active_slot_mount_point=cfg.ACTIVE_ROOT_MOUNT_POINT, + active_slot_mount_point=cfg.ACTIVE_SLOT_MP, ) + self._ota_status_control = OTAStatusFilesControl( active_slot=self._boot_control.active_slot, standby_slot=self._boot_control.standby_slot, @@ -823,7 +828,7 @@ def _cleanup_standby_ota_partition_folder(self): cfg.OTA_STATUS_FNAME, cfg.OTA_VERSION_FNAME, cfg.SLOT_IN_USE_FNAME, - cfg.GRUB_CFG_FNAME, + boot_cfg.GRUB_CFG_FNAME, ) removes = ( f @@ -890,15 +895,9 @@ def post_update(self) -> Generator[None, None, None]: try: logger.info("grub_boot: post-update setup...") # ------ update fstab ------ # - active_fstab = self._mp_control.active_slot_mount_point / Path( - cfg.FSTAB_FILE_PATH - ).relative_to("/") - standby_fstab = self._mp_control.standby_slot_mount_point / Path( - cfg.FSTAB_FILE_PATH - ).relative_to("/") self._update_fstab( - standby_slot_fstab=standby_fstab, - active_slot_fstab=active_fstab, + standby_slot_fstab=Path(boot_cfg.STANDBY_FSTAB_FPATH), + active_slot_fstab=Path(boot_cfg.ACTIVE_FSTAB_FPATH), ) # ------ prepare boot files ------ # diff --git a/otaclient/app/boot_control/_rpi_boot.py b/otaclient/app/boot_control/_rpi_boot.py index 7317a48de..60e37c053 100644 --- a/otaclient/app/boot_control/_rpi_boot.py +++ b/otaclient/app/boot_control/_rpi_boot.py @@ -21,6 +21,7 @@ from typing import Generator from .. import log_setting, errors as ota_errors +from ..configs import config as cfg from ..proto import wrapper from ..common import replace_atomic, subprocess_call @@ -30,12 +31,10 @@ CMDHelperFuncs, write_str_to_file_sync, ) -from .configs import rpi_boot_cfg as cfg +from .configs import rpi_boot_cfg as boot_cfg from .protocol import BootControllerProtocol -logger = log_setting.get_logger( - __name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL) -) +logger = log_setting.get_logger(__name__) _FSTAB_TEMPLATE_STR = ( "LABEL=${rootfs_fslabel}\t/\text4\tdiscard,x-systemd.growfs\t0\t1\n" @@ -68,8 +67,8 @@ class _RPIBootControl: i.e., config.txt for slot_a will be config.txt_slot_a """ - SLOT_A = cfg.SLOT_A_FSLABEL - SLOT_B = cfg.SLOT_B_FSLABEL + SLOT_A = boot_cfg.SLOT_A_FSLABEL + SLOT_B = boot_cfg.SLOT_B_FSLABEL AB_FLIPS = { SLOT_A: SLOT_B, SLOT_B: SLOT_A, @@ -77,7 +76,7 @@ class _RPIBootControl: SEP_CHAR = "_" def __init__(self) -> None: - self.system_boot_path = Path(cfg.SYSTEM_BOOT_MOUNT_POINT) + self.system_boot_path = Path(boot_cfg.SYSTEM_BOOT_MOUNT_POINT) if not ( self.system_boot_path.is_dir() and CMDHelperFuncs.is_target_mounted(self.system_boot_path) @@ -94,7 +93,7 @@ def _init_slots_info(self): try: # detect active slot self._active_slot_dev = CMDHelperFuncs.get_dev_by_mount_point( - cfg.ACTIVE_ROOTFS_PATH + cfg.ACTIVE_ROOTFS ) if not ( _active_slot := CMDHelperFuncs.get_fslabel_by_dev(self._active_slot_dev) @@ -161,12 +160,13 @@ def _init_boot_files(self): """ logger.debug("checking boot files...") # boot file - self.config_txt = self.system_boot_path / cfg.CONFIG_TXT - self.tryboot_txt = self.system_boot_path / cfg.TRYBOOT_TXT + self.config_txt = self.system_boot_path / boot_cfg.CONFIG_TXT_FNAME + self.tryboot_txt = self.system_boot_path / boot_cfg.TRYBOOT_TXT_FNAME # active slot self.config_txt_active_slot = ( - self.system_boot_path / f"{cfg.CONFIG_TXT}{self.SEP_CHAR}{self.active_slot}" + self.system_boot_path + / f"{boot_cfg.CONFIG_TXT_FNAME}{self.SEP_CHAR}{self.active_slot}" ) if not self.config_txt_active_slot.is_file(): _err_msg = f"missing {self.config_txt_active_slot=}" @@ -174,22 +174,24 @@ def _init_boot_files(self): raise _RPIBootControllerError(_err_msg) self.cmdline_txt_active_slot = ( self.system_boot_path - / f"{cfg.CMDLINE_TXT}{self.SEP_CHAR}{self.active_slot}" + / f"{boot_cfg.CMDLINE_TXT_FNAME}{self.SEP_CHAR}{self.active_slot}" ) if not self.cmdline_txt_active_slot.is_file(): _err_msg = f"missing {self.cmdline_txt_active_slot=}" logger.error(_err_msg) raise _RPIBootControllerError(_err_msg) self.vmlinuz_active_slot = ( - self.system_boot_path / f"{cfg.VMLINUZ}{self.SEP_CHAR}{self.active_slot}" + self.system_boot_path + / f"{boot_cfg.VMLINUZ_FNAME}{self.SEP_CHAR}{self.active_slot}" ) self.initrd_img_active_slot = ( - self.system_boot_path / f"{cfg.INITRD_IMG}{self.SEP_CHAR}{self.active_slot}" + self.system_boot_path + / f"{boot_cfg.INITRD_IMG_FNAME}{self.SEP_CHAR}{self.active_slot}" ) # standby slot self.config_txt_standby_slot = ( self.system_boot_path - / f"{cfg.CONFIG_TXT}{self.SEP_CHAR}{self.standby_slot}" + / f"{boot_cfg.CONFIG_TXT_FNAME}{self.SEP_CHAR}{self.standby_slot}" ) if not self.config_txt_standby_slot.is_file(): _err_msg = f"missing {self.config_txt_standby_slot=}" @@ -197,18 +199,19 @@ def _init_boot_files(self): raise _RPIBootControllerError(_err_msg) self.cmdline_txt_standby_slot = ( self.system_boot_path - / f"{cfg.CMDLINE_TXT}{self.SEP_CHAR}{self.standby_slot}" + / f"{boot_cfg.CMDLINE_TXT_FNAME}{self.SEP_CHAR}{self.standby_slot}" ) if not self.cmdline_txt_standby_slot.is_file(): _err_msg = f"missing {self.cmdline_txt_standby_slot=}" logger.error(_err_msg) raise _RPIBootControllerError(_err_msg) self.vmlinuz_standby_slot = ( - self.system_boot_path / f"{cfg.VMLINUZ}{self.SEP_CHAR}{self.standby_slot}" + self.system_boot_path + / f"{boot_cfg.VMLINUZ_FNAME}{self.SEP_CHAR}{self.standby_slot}" ) self.initrd_img_standby_slot = ( self.system_boot_path - / f"{cfg.INITRD_IMG}{self.SEP_CHAR}{self.standby_slot}" + / f"{boot_cfg.INITRD_IMG_FNAME}{self.SEP_CHAR}{self.standby_slot}" ) def _update_firmware(self): @@ -228,10 +231,14 @@ def _update_firmware(self): # check if the vmlinuz and initrd.img presented in /boot/firmware(system-boot), # if so, it means that flash-kernel works and copies the kernel, inird.img from /boot, # then we rename vmlinuz and initrd.img to vmlinuz_ and initrd.img_ - if (_vmlinuz := Path(cfg.SYSTEM_BOOT_MOUNT_POINT) / cfg.VMLINUZ).is_file(): + if ( + _vmlinuz := Path(boot_cfg.SYSTEM_BOOT_MOUNT_POINT) + / boot_cfg.VMLINUZ_FNAME + ).is_file(): os.replace(_vmlinuz, self.vmlinuz_active_slot) if ( - _initrd_img := Path(cfg.SYSTEM_BOOT_MOUNT_POINT) / cfg.INITRD_IMG + _initrd_img := Path(boot_cfg.SYSTEM_BOOT_MOUNT_POINT) + / boot_cfg.INITRD_IMG_FNAME ).is_file(): os.replace(_initrd_img, self.initrd_img_active_slot) os.sync() @@ -282,7 +289,7 @@ def finalize_switching_boot(self) -> bool: """ logger.info("finalizing switch boot...") try: - _flag_file = self.system_boot_path / cfg.SWITCH_BOOT_FLAG_FILE + _flag_file = self.system_boot_path / boot_cfg.SWITCH_BOOT_FLAG_FNAME if _flag_file.is_file(): # we are just after second reboot, after firmware_update, # finalize the switch boot and then return True @@ -379,22 +386,25 @@ class RPIBootController(BootControllerProtocol): def __init__(self) -> None: try: self._rpiboot_control = _RPIBootControl() - # mount point prepare + + # ------ prepare mount space ------ # + otaclient_ms = Path(cfg.OTACLIENT_MOUNT_SPACE_DPATH) + otaclient_ms.mkdir(exist_ok=True, parents=True) + otaclient_ms.chmod(0o700) + self._mp_control = SlotMountHelper( standby_slot_dev=self._rpiboot_control.standby_slot_dev, - standby_slot_mount_point=cfg.MOUNT_POINT, + standby_slot_mount_point=cfg.STANDBY_SLOT_MP, active_slot_dev=self._rpiboot_control.active_slot_dev, - active_slot_mount_point=cfg.ACTIVE_ROOT_MOUNT_POINT, + active_slot_mount_point=cfg.ACTIVE_SLOT_MP, ) - # init ota-status files + self._ota_status_control = OTAStatusFilesControl( active_slot=self._rpiboot_control.active_slot, standby_slot=self._rpiboot_control.standby_slot, - current_ota_status_dir=Path(cfg.ACTIVE_ROOTFS_PATH) - / Path(cfg.OTA_STATUS_DIR).relative_to("/"), + current_ota_status_dir=Path(boot_cfg.ACTIVE_BOOT_OTA_STATUS_DPATH), # NOTE: might not yet be populated before OTA update applied! - standby_ota_status_dir=Path(cfg.MOUNT_POINT) - / Path(cfg.OTA_STATUS_DIR).relative_to("/"), + standby_ota_status_dir=Path(boot_cfg.STANDBY_BOOT_OTA_STATUS_DPATH), finalize_switching_boot=self._rpiboot_control.finalize_switching_boot, ) @@ -404,7 +414,8 @@ def __init__(self) -> None: wrapper.StatusOta.ROLLBACKING, ): _flag_file = ( - self._rpiboot_control.system_boot_path / cfg.SWITCH_BOOT_FLAG_FILE + self._rpiboot_control.system_boot_path + / boot_cfg.SWITCH_BOOT_FLAG_FNAME ) _flag_file.unlink(missing_ok=True) @@ -425,17 +436,20 @@ def _copy_kernel_for_standby_slot(self): logger.debug( "prepare standby slot's kernel/initrd.img to system-boot partition..." ) + vmlinuz_fname = boot_cfg.VMLINUZ_FNAME + initrd_fname = boot_cfg.INITRD_IMG_FNAME + try: # search for kernel _kernel_pa, _kernel_ver = ( - re.compile(rf"{cfg.VMLINUZ}-(?P.*)"), + re.compile(rf"{vmlinuz_fname}-(?P.*)"), None, ) # NOTE: if there is multiple kernel, pick the first one we encounted # NOTE 2: according to ota-image specification, it should only be one # version of kernel and initrd.img for _candidate in self._mp_control.standby_boot_dir.glob( - f"{cfg.VMLINUZ}-*" + f"{vmlinuz_fname}-*" ): if _ma := _kernel_pa.match(_candidate.name): _kernel_ver = _ma.group("kernel_ver") @@ -443,9 +457,9 @@ def _copy_kernel_for_standby_slot(self): if _kernel_ver is not None: _kernel, _initrd_img = ( - self._mp_control.standby_boot_dir / f"{cfg.VMLINUZ}-{_kernel_ver}", self._mp_control.standby_boot_dir - / f"{cfg.INITRD_IMG}-{_kernel_ver}", + / f"{vmlinuz_fname}-{_kernel_ver}", + self._mp_control.standby_boot_dir / f"{initrd_fname}-{_kernel_ver}", ) _kernel_sysboot, _initrd_img_sysboot = ( self._rpiboot_control.vmlinuz_standby_slot, @@ -454,7 +468,9 @@ def _copy_kernel_for_standby_slot(self): replace_atomic(_kernel, _kernel_sysboot) replace_atomic(_initrd_img, _initrd_img_sysboot) else: - raise ValueError("failed to kernel in /boot folder at standby slot") + raise ValueError( + "failed to find kernel in /boot folder at standby slot" + ) except Exception as e: _err_msg = "failed to copy kernel/initrd_img for standby slot" logger.error(_err_msg) @@ -469,9 +485,7 @@ def _write_standby_fstab(self): """ logger.debug("update standby slot fstab file...") try: - _fstab_fpath = self._mp_control.standby_slot_mount_point / Path( - cfg.FSTAB_FPATH - ).relative_to("/") + _fstab_fpath = Path(boot_cfg.STANDBY_FSTAB_FPATH) _updated_fstab_str = Template(_FSTAB_TEMPLATE_STR).substitute( rootfs_fslabel=self._rpiboot_control.standby_slot ) @@ -506,7 +520,7 @@ def pre_update(self, version: str, *, standby_as_ref: bool, erase_standby: bool) # 20230613: remove any leftover flag file if presented _flag_file = ( - self._rpiboot_control.system_boot_path / cfg.SWITCH_BOOT_FLAG_FILE + self._rpiboot_control.system_boot_path / boot_cfg.SWITCH_BOOT_FLAG_FNAME ) _flag_file.unlink(missing_ok=True) except Exception as e: diff --git a/otaclient/app/boot_control/configs.py b/otaclient/app/boot_control/configs.py index d9fb96478..fe10b954a 100644 --- a/otaclient/app/boot_control/configs.py +++ b/otaclient/app/boot_control/configs.py @@ -13,14 +13,25 @@ # limitations under the License. -from dataclasses import dataclass, field -from enum import Enum -from typing import Dict +from __future__ import annotations +import os.path +from enum import Enum, unique +from pydantic import BaseModel, ConfigDict +from typing import TYPE_CHECKING, ClassVar as _std_ClassVar, Any +from typing_extensions import Self -from ..configs import BaseConfig +from otaclient._utils import cached_computed_field +from otaclient._utils.path import replace_root +from ..configs import config as cfg +# A simple trick to make plain ClassVar work when +# __future__.annotations is activated. +if not TYPE_CHECKING: + _std_ClassVar = _std_ClassVar[Any] -class BootloaderType(Enum): + +@unique +class BootloaderType(str, Enum): """Bootloaders that supported by otaclient. grub: generic x86_64 platform with grub @@ -35,71 +46,234 @@ class BootloaderType(Enum): RPI_BOOT = "rpi_boot" @classmethod - def parse_str(cls, _input: str) -> "BootloaderType": + def parse_str(cls, _input: str) -> Self: res = cls.UNSPECIFIED try: # input is enum key(capitalized) - res = BootloaderType[_input] + res = cls[_input] except KeyError: pass try: # input is enum value(uncapitalized) - res = BootloaderType(_input) + res = cls(_input) except ValueError: pass return res -@dataclass -class GrubControlConfig(BaseConfig): +class _SeparatedBootParOTAStatusConfig(BaseModel): + """Configs for platforms that have separated boot dir. + + Currently cboot and rpi_boot platforms are included in this catagory. + 1. cboot: separated boot devices, boot dirs on each slots, + 2. rpi_boot: separated boot dirs on each slots, only share system-boot. + + grub platform shares /boot dir by sharing the same boot device. + """ + + OTA_STATUS_DNAME: _std_ClassVar = "ota-status" + + @cached_computed_field + def ACTIVE_BOOT_OTA_STATUS_DPATH(self) -> str: + """The dynamically rooted location of ota-status dir. + + Default: /boot/ota-status + """ + return os.path.join(cfg.BOOT_DPATH, self.OTA_STATUS_DNAME) + + @cached_computed_field + def STANDBY_BOOT_OTA_STATUS_DPATH(self) -> str: + """The dynamically rooted location standby slot's ota-status dir. + + NOTE: this location is relatived to standby slot's mount point. + NOTE(20231117): for platform with separated boot dev(like cboot), it is boot controller's + responsibility to copy the /boot dir from standby slot rootfs to separated boot dev. + + Default: /mnt/otaclient/standby_slot/boot/ota-status + """ + return replace_root( + self.ACTIVE_BOOT_OTA_STATUS_DPATH, cfg.ACTIVE_ROOTFS, cfg.STANDBY_SLOT_MP + ) + + +class _CommonConfig(BaseModel): + model_config = ConfigDict(frozen=True, validate_default=True) + DEFAULT_FSTAB_FPATH: _std_ClassVar = "/etc/fstab" + + @cached_computed_field + def STANDBY_FSTAB_FPATH(self) -> str: + """The dynamically rooted location of standby slot's fstab file. + + NOTE: this location is relatived to standby slot's mount point. + + Default: /mnt/otaclient/standby_slot/etc/fstab + """ + return replace_root( + self.DEFAULT_FSTAB_FPATH, cfg.DEFAULT_ACTIVE_ROOTFS, cfg.STANDBY_SLOT_MP + ) + + @cached_computed_field + def ACTIVE_FSTAB_FPATH(self) -> str: + """The dynamically rooted location of active slot's fstab file. + + Default: /etc/fstab + """ + return replace_root( + self.DEFAULT_FSTAB_FPATH, cfg.DEFAULT_ACTIVE_ROOTFS, cfg.ACTIVE_ROOTFS + ) + + +class GrubControlConfig(_CommonConfig): """x86-64 platform, with grub as bootloader.""" - BOOTLOADER: BootloaderType = BootloaderType.GRUB - FSTAB_FILE_PATH: str = "/etc/fstab" - GRUB_DIR: str = "/boot/grub" - GRUB_CFG_FNAME: str = "grub.cfg" - GRUB_CFG_PATH: str = "/boot/grub/grub.cfg" - DEFAULT_GRUB_PATH: str = "/etc/default/grub" - BOOT_OTA_PARTITION_FILE: str = "ota-partition" + BOOTLOADER: _std_ClassVar = BootloaderType.GRUB + GRUB_CFG_FNAME: _std_ClassVar = "grub.cfg" + BOOT_OTA_PARTITION_FNAME: _std_ClassVar = "ota-partition" + + @cached_computed_field + def BOOT_GRUB_DPATH(self) -> str: + """The dynamically rooted location of /boot/grub dir. + + Default: /boot/grub + """ + return os.path.join(cfg.BOOT_DPATH, "grub") + + @cached_computed_field + def GRUB_CFG_FPATH(self) -> str: + """The dynamically rooted location of /boot/grub/grub.cfg file. + + Default: /boot/grub/grub.cfg + """ + return os.path.join(self.BOOT_GRUB_DPATH, self.GRUB_CFG_FNAME) + + @cached_computed_field + def GRUB_DEFAULT_FPATH(self) -> str: + """The dynamically rooted location of /etc/default/grub file. + + Default: /etc/default/grub + """ + return os.path.join(cfg.ETC_DPATH, "default/grub") -@dataclass -class CBootControlConfig(BaseConfig): +class CBootControlConfig(_CommonConfig, _SeparatedBootParOTAStatusConfig): """arm platform, with cboot as bootloader. NOTE: only for tegraid:0x19, roscube-x platform(jetson-xavier-agx series) """ - BOOTLOADER: BootloaderType = BootloaderType.CBOOT - TEGRA_CHIP_ID_PATH: str = "/sys/module/tegra_fuse/parameters/tegra_chip_id" - CHIP_ID_MODEL_MAP: Dict[int, str] = field(default_factory=lambda: {0x19: "rqx_580"}) - OTA_STATUS_DIR: str = "/boot/ota-status" - EXTLINUX_FILE: str = "/boot/extlinux/extlinux.conf" - SEPARATE_BOOT_MOUNT_POINT: str = "/mnt/standby_boot" + BOOTLOADER: _std_ClassVar = BootloaderType.CBOOT + CHIP_ID_MODEL_MAP: _std_ClassVar = {0x19: "rqx_580"} + DEFAULT_TEGRA_CHIP_ID_FPATH: _std_ClassVar = ( + "/sys/module/tegra_fuse/parameters/tegra_chip_id" + ) + DEFAULT_EXTLINUX_DPATH: _std_ClassVar = "/boot/extlinux" + DEFAULT_FIRMWARE_CFG_FPATH: _std_ClassVar = "/opt/ota/firmwares/firmware.yaml" + EXTLINUX_CFG_FNAME: _std_ClassVar = "extlinux.conf" + + @cached_computed_field + def TEGRA_CHIP_ID_FPATH(self) -> str: + """The dynamically rooted location of tegra chip id query API. + + Default: /sys/module/tegra_fuse/parameters/tegra_chip_id + """ + return replace_root( + self.DEFAULT_TEGRA_CHIP_ID_FPATH, + cfg.DEFAULT_ACTIVE_ROOTFS, + cfg.ACTIVE_ROOTFS, + ) + + @cached_computed_field + def STANDBY_BOOT_EXTLINUX_DPATH(self) -> str: + """The dynamically rooted location of standby slot's extlinux cfg dir. + + NOTE: this location is relatived to standby slot's mount point. + + Default: /mnt/otaclient/standby_slot/boot/extlinux + """ + return replace_root( + self.DEFAULT_EXTLINUX_DPATH, + cfg.DEFAULT_ACTIVE_ROOTFS, + cfg.STANDBY_SLOT_MP, + ) + + @cached_computed_field + def STANDBY_EXTLINUX_FPATH(self) -> str: + """The dynamically rooted location of standby slot's extlinux cfg file. + + NOTE: this location is relatived to standby slot's mount point. + + Default: /mnt/otaclient/standby_slot/boot/extlinux/extlinux.conf + """ + return os.path.join(self.STANDBY_BOOT_EXTLINUX_DPATH, self.EXTLINUX_CFG_FNAME) + + @cached_computed_field + def SEPARATE_BOOT_MOUNT_POINT(self) -> str: + """The dynamically rooted location of standby slot's boot dev mount point. + + Default: /mnt/otaclient/standby_boot + """ + return os.path.join(cfg.OTACLIENT_MOUNT_SPACE_DPATH, "standby_boot") + # refer to the standby slot - FIRMWARE_CONFIG: str = "/opt/ota/firmwares/firmware.yaml" - - -@dataclass -class RPIBootControlConfig(BaseConfig): - BBOOTLOADER: BootloaderType = BootloaderType.RPI_BOOT - RPI_MODEL_FILE = "/proc/device-tree/model" - RPI_MODEL_HINT = "Raspberry Pi 4 Model B" - - # slot configuration - SLOT_A_FSLABEL = "slot_a" - SLOT_B_FSLABEL = "slot_b" - SYSTEM_BOOT_FSLABEL = "system-boot" - - # boot folders - SYSTEM_BOOT_MOUNT_POINT = "/boot/firmware" - OTA_STATUS_DIR = "/boot/ota-status" - - # boot related files - CONFIG_TXT = "config.txt" # primary boot cfg - TRYBOOT_TXT = "tryboot.txt" # tryboot boot cfg - VMLINUZ = "vmlinuz" - INITRD_IMG = "initrd.img" - CMDLINE_TXT = "cmdline.txt" - SWITCH_BOOT_FLAG_FILE = "._ota_switch_boot_finalized" + @cached_computed_field + def FIRMWARE_CFG_STANDBY_FPATH(self) -> str: + """The dynamically rooted location of standby slot's cboot firmware.yaml file. + + NOTE: this location is relatived to standby slot's mount point. + + Default: /mnt/otaclient/standby_slot/opt/ota/firmwares/firmware.yaml + """ + return replace_root( + self.DEFAULT_FIRMWARE_CFG_FPATH, + cfg.DEFAULT_ACTIVE_ROOTFS, + cfg.STANDBY_SLOT_MP, + ) + + +class RPIBootControlConfig(_CommonConfig, _SeparatedBootParOTAStatusConfig): + BBOOTLOADER: _std_ClassVar = BootloaderType.RPI_BOOT + + DEFAULT_RPI_MODEL_FPATH: _std_ClassVar = "/proc/device-tree/model" + RPI_MODEL_HINT: _std_ClassVar = "Raspberry Pi 4 Model B" + + SLOT_A_FSLABEL: _std_ClassVar = "slot_a" + SLOT_B_FSLABEL: _std_ClassVar = "slot_b" + + SYSTEM_BOOT_FSLABEL: _std_ClassVar = "system-boot" + SWITCH_BOOT_FLAG_FNAME: _std_ClassVar = "._ota_switch_boot_finalized" + + # boot files fname + CONFIG_TXT_FNAME: _std_ClassVar = "config.txt" # primary boot cfg + TRYBOOT_TXT_FNAME: _std_ClassVar = "tryboot.txt" # tryboot boot cfg + VMLINUZ_FNAME: _std_ClassVar = "vmlinuz" + INITRD_IMG_FNAME: _std_ClassVar = "initrd.img" + CMDLINE_TXT_FNAME: _std_ClassVar = "cmdline.txt" + + @cached_computed_field + def RPI_MODEL_FPATH(self) -> str: + """The dynamically rooted location of rpi model query API. + + Default: /proc/device-tree/model + """ + return replace_root( + self.DEFAULT_RPI_MODEL_FPATH, + cfg.DEFAULT_ACTIVE_ROOTFS, + cfg.ACTIVE_ROOTFS, + ) + + @cached_computed_field + def SYSTEM_BOOT_MOUNT_POINT(self) -> str: + """The dynamically rooted location of rpi system-boot partition mount point. + + Default: /boot/firmware + """ + return os.path.join(cfg.BOOT_DPATH, "firmware") + + @cached_computed_field + def SWITCH_BOOT_FLAG_FPATH(self) -> str: + """The dynamically rooted location of rpi switch boot flag file. + + Default: /boot/firmware/._ota_switch_boot_finalized + """ + return os.path.join(self.SYSTEM_BOOT_MOUNT_POINT, self.SWITCH_BOOT_FLAG_FNAME) grub_cfg = GrubControlConfig() diff --git a/otaclient/app/boot_control/firmware.py b/otaclient/app/boot_control/firmware.py index 67a819f63..d7b426347 100644 --- a/otaclient/app/boot_control/firmware.py +++ b/otaclient/app/boot_control/firmware.py @@ -17,12 +17,9 @@ import zstandard from pathlib import Path from typing import Dict, Callable -from ..configs import config as cfg from .. import log_setting -logger = log_setting.get_logger( - __name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL) -) +logger = log_setting.get_logger(__name__) class Firmware: diff --git a/otaclient/app/boot_control/selecter.py b/otaclient/app/boot_control/selecter.py index 2a4edfa97..b23d1c537 100644 --- a/otaclient/app/boot_control/selecter.py +++ b/otaclient/app/boot_control/selecter.py @@ -21,13 +21,10 @@ from ._errors import BootControlError from .protocol import BootControllerProtocol -from ..configs import config as cfg from ..common import read_str_from_file from .. import log_setting -logger = log_setting.get_logger( - __name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL) -) +logger = log_setting.get_logger(__name__) def detect_bootloader(raise_on_unknown=True) -> BootloaderType: @@ -49,10 +46,10 @@ def detect_bootloader(raise_on_unknown=True) -> BootloaderType: if machine == "aarch64" or arch == "aarch64": # evidence: jetson xvaier device has a special file which reveals the # tegra chip id - if Path(cboot_cfg.TEGRA_CHIP_ID_PATH).is_file(): + if Path(cboot_cfg.TEGRA_CHIP_ID_FPATH).is_file(): return BootloaderType.CBOOT # evidence: rpi device has a special file which reveals the rpi model - rpi_model_file = Path(rpi_boot_cfg.RPI_MODEL_FILE) + rpi_model_file = Path(rpi_boot_cfg.RPI_MODEL_FPATH) if rpi_model_file.is_file(): if (_model_str := read_str_from_file(rpi_model_file)).find( rpi_boot_cfg.RPI_MODEL_HINT diff --git a/otaclient/app/common.py b/otaclient/app/common.py index a4eb4f4f9..15c6808fb 100644 --- a/otaclient/app/common.py +++ b/otaclient/app/common.py @@ -11,9 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +r"""Utils that shared between modules are listed here.""" -r"""Utils that shared between modules are listed here.""" import itertools import os import shlex @@ -44,7 +44,7 @@ from .log_setting import get_logger from .configs import config as cfg -logger = get_logger(__name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL)) +logger = get_logger(__name__) def get_backoff(n: int, factor: float, _max: float) -> float: diff --git a/otaclient/app/configs.py b/otaclient/app/configs.py index 533915f6c..3908019db 100644 --- a/otaclient/app/configs.py +++ b/otaclient/app/configs.py @@ -11,182 +11,23 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# flake8: noqa +"""Runtime configs and consts for otaclient. +This is a virtual module that imports configs required by otaclient app. +""" -from enum import Enum, auto -from logging import INFO -from pathlib import Path -from typing import Dict, Tuple +from __future__ import annotations +from pathlib import Path from otaclient import __file__ as _otaclient__init__ - -_OTACLIENT_PACKAGE_ROOT = Path(_otaclient__init__).parent - -# NOTE: VERSION file is installed under otaclient package root -EXTRA_VERSION_FILE = str(_OTACLIENT_PACKAGE_ROOT / "version.txt") - - -class CreateStandbyMechanism(Enum): - LEGACY = 0 # deprecated and removed - REBUILD = auto() # default - IN_PLACE = auto() # not yet implemented - - -class OtaClientServerConfig: - SERVER_PORT = 50051 - WAITING_SUBECU_ACK_REQ_TIMEOUT = 6 - QUERYING_SUBECU_STATUS_TIMEOUT = 30 - LOOP_QUERYING_SUBECU_STATUS_INTERVAL = 10 - STATUS_UPDATE_INTERVAL = 1 - - # proxy server - OTA_PROXY_LISTEN_ADDRESS = "0.0.0.0" - OTA_PROXY_LISTEN_PORT = 8082 - - -class _InternalSettings: - """Common internal settings for otaclient. - - WARNING: typically the common settings SHOULD NOT be changed! - otherwise the backward compatibility will be impact. - Change the fields in BaseConfig if you want to tune the otaclient. - """ - - # ------ common paths ------ # - RUN_DIR = "/run/otaclient" - OTACLIENT_PID_FILE = "/run/otaclient.pid" - # NOTE: certs dir is located at the otaclient package root - CERTS_DIR = str(_OTACLIENT_PACKAGE_ROOT / "certs") - ACTIVE_ROOTFS_PATH = "/" - BOOT_DIR = "/boot" - OTA_DIR = "/boot/ota" - ECU_INFO_FILE = "/boot/ota/ecu_info.yaml" - PROXY_INFO_FILE = "/boot/ota/proxy_info.yaml" - PASSWD_FILE = "/etc/passwd" - GROUP_FILE = "/etc/group" - FSTAB_FPATH = "/etc/fstab" - # where the OTA image meta store for this slot - META_FOLDER = "/opt/ota/image-meta" - - # ------ device configuration files ------ # - # this files should be placed under /boot/ota folder - ECU_INFO_FNAME = "ecu_info.yaml" - PROXY_INFO_FNAME = "proxy_info.yaml" - - # ------ ota-status files ------ # - # this files should be placed under /boot/ota-status folder - OTA_STATUS_FNAME = "status" - OTA_VERSION_FNAME = "version" - SLOT_IN_USE_FNAME = "slot_in_use" - - # ------ otaclient internal used path ------ # - # standby/refroot mount points - MOUNT_POINT = "/mnt/standby" - # where active(old) image partition will be bind mounted to - ACTIVE_ROOT_MOUNT_POINT = "/mnt/refroot" - # tmp store for local copy - OTA_TMP_STORE = "/.ota-tmp" - # tmp store for standby slot OTA image meta - OTA_TMP_META_STORE = "/.ota-meta" - # compressed OTA image support - SUPPORTED_COMPRESS_ALG: Tuple[str, ...] = ("zst", "zstd") +from otaclient.configs.app_cfg import app_config as config, CreateStandbyMechanism +from otaclient.configs.debug_cfg import debug_flags +from otaclient.configs.logging_cfg import logging_config +from otaclient.configs.ota_service_cfg import service_config -class BaseConfig(_InternalSettings): - """User configurable otaclient settings.""" +OTACLIENT_PACKAGE_ROOT = Path(_otaclient__init__).parent - # ------ otaclient logging setting ------ # - DEFAULT_LOG_LEVEL = INFO - LOG_LEVEL_TABLE: Dict[str, int] = { - "otaclient.app.boot_control.cboot": INFO, - "otaclient.app.boot_control.grub": INFO, - "otaclient.app.ota_client": INFO, - "otaclient.app.ota_client_service": INFO, - "otaclient.app.ota_client_stub": INFO, - "otaclient.app.ota_metadata": INFO, - "otaclient.app.downloader": INFO, - "otaclient.app.main": INFO, - } - LOG_FORMAT = ( - "[%(asctime)s][%(levelname)s]-%(name)s:%(funcName)s:%(lineno)d,%(message)s" - ) - - # ------ otaclient behavior setting ------ # - # the following settings can be safely changed according to the - # actual environment otaclient running at. - # --- file read/write settings --- # - CHUNK_SIZE = 1 * 1024 * 1024 # 1MB - LOCAL_CHUNK_SIZE = 4 * 1024 * 1024 # 4MB - - # --- download settings for single download task --- # - DOWNLOAD_RETRY = 3 - DOWNLOAD_BACKOFF_MAX = 3 # seconds - DOWNLOAD_BACKOFF_FACTOR = 0.1 # seconds - # downloader settings - MAX_DOWNLOAD_THREAD = 7 - DOWNLOADER_CONNPOOL_SIZE_PER_THREAD = 20 - - # --- download settings for the whole download tasks group --- # - # if retry keeps failing without any success in - # DOWNLOAD_GROUP_NO_SUCCESS_RETRY_TIMEOUT time, failed the whole - # download task group and raise NETWORK OTA error. - MAX_CONCURRENT_DOWNLOAD_TASKS = 128 - DOWNLOAD_GROUP_INACTIVE_TIMEOUT = 5 * 60 # seconds - DOWNLOAD_GROUP_BACKOFF_MAX = 12 # seconds - DOWNLOAD_GROUP_BACKOFF_FACTOR = 1 # seconds - - # --- stats collector setting --- # - STATS_COLLECT_INTERVAL = 1 # second - - # --- create standby setting --- # - # now only REBUILD mode is available - STANDBY_CREATION_MODE = CreateStandbyMechanism.REBUILD - MAX_CONCURRENT_PROCESS_FILE_TASKS = 256 - CREATE_STANDBY_RETRY_MAX = 3 - CREATE_STANDBY_BACKOFF_FACTOR = 1 - CREATE_STANDBY_BACKOFF_MAX = 6 - - # --- ECU status polling setting, otaproxy dependency managing --- # - # The ECU status storage will summarize the stored ECUs' status report - # and generate overall status report for all ECUs every seconds. - OVERALL_ECUS_STATUS_UPDATE_INTERVAL = 6 # seconds - - # If ECU has been disconnected longer than seconds, it will be - # treated as UNREACHABLE, and will not be counted when generating overall - # ECUs status report. - # NOTE: unreachable_timeout should be larger than - # downloading_group timeout - ECU_UNREACHABLE_TIMEOUT = 20 * 60 # seconds - - # Otaproxy should not be shutdowned with less than seconds - # after it just starts to prevent repeatedly start/stop cycle. - OTAPROXY_MINIMUM_SHUTDOWN_INTERVAL = 1 * 60 # seconds - - # When any ECU acks update request, this ECU will directly set the overall ECU status - # to any_in_update=True, any_requires_network=True, all_success=False, to prevent - # pre-mature overall ECU status changed caused by child ECU delayed ack to update request. - # - # This pre-set overall ECU status will be kept for seconds. - # This value is expected to be larger than the time cost for subECU acks the OTA request. - KEEP_OVERALL_ECUS_STATUS_ON_ANY_UPDATE_REQ_ACKED = 60 # seconds - - # Active status polling interval, when there is active OTA update in the cluster. - ACTIVE_INTERVAL = 1 # second - - # Idle status polling interval, when ther is no active OTA updaste in the cluster. - IDLE_INTERVAL = 10 # seconds - - # --- External cache source support for otaproxy --- # - EXTERNAL_CACHE_DEV_FSLABEL = "ota_cache_src" - EXTERNAL_CACHE_DEV_MOUNTPOINT = "/mnt/external_cache_src" - EXTERNAL_CACHE_SRC_PATH = "/mnt/external_cache_src/data" - - # default version string to be reported in status API response - DEFAULT_VERSION_STR = "" - - DEBUG_MODE = False - - -# init cfgs -server_cfg = OtaClientServerConfig() -config = BaseConfig() +# NOTE: VERSION file is installed under otaclient package root +EXTRA_VERSION_FILE = str(OTACLIENT_PACKAGE_ROOT / "version.txt") diff --git a/otaclient/app/copy_tree.py b/otaclient/app/copy_tree.py index 2026f6758..8e32beb17 100644 --- a/otaclient/app/copy_tree.py +++ b/otaclient/app/copy_tree.py @@ -19,11 +19,8 @@ from pathlib import Path from . import log_setting -from .configs import config as cfg -logger = log_setting.get_logger( - __name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL) -) +logger = log_setting.get_logger(__name__) class CopyTree: diff --git a/otaclient/app/create_standby/__init__.py b/otaclient/app/create_standby/__init__.py index f90c07972..e4831c751 100644 --- a/otaclient/app/create_standby/__init__.py +++ b/otaclient/app/create_standby/__init__.py @@ -16,12 +16,11 @@ from typing import Type from .interface import StandbySlotCreatorProtocol -from ..configs import CreateStandbyMechanism, config as cfg +from ..configs import CreateStandbyMechanism from .. import log_setting -logger = log_setting.get_logger( - __name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL) -) + +logger = log_setting.get_logger(__name__) def get_standby_slot_creator( diff --git a/otaclient/app/create_standby/common.py b/otaclient/app/create_standby/common.py index ae28dcd7e..b7adc89c5 100644 --- a/otaclient/app/create_standby/common.py +++ b/otaclient/app/create_standby/common.py @@ -11,9 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +r"""Common used helpers, classes and functions for different bank creating methods.""" -r"""Common used helpers, classes and functions for different bank creating methods.""" +from __future__ import annotations import os import random import time @@ -50,9 +51,7 @@ RegInfProcessedStats, ) -logger = log_setting.get_logger( - __name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL) -) +logger = log_setting.get_logger(__name__) class _WeakRef: @@ -101,8 +100,10 @@ def subscribe_no_wait(self) -> str: class HardlinkRegister: def __init__(self): self._lock = Lock() - self._hash_ref_dict: Dict[str, _WeakRef] = WeakValueDictionary() # type: ignore - self._ref_tracker_dict: Dict[_WeakRef, _HardlinkTracker] = WeakKeyDictionary() # type: ignore + self._hash_ref_dict: WeakValueDictionary[str, _WeakRef] = WeakValueDictionary() + self._ref_tracker_dict: WeakKeyDictionary[ + _WeakRef, _HardlinkTracker + ] = WeakKeyDictionary() def get_tracker( self, _identifier: Any, path: str, nlink: int diff --git a/otaclient/app/create_standby/rebuild_mode.py b/otaclient/app/create_standby/rebuild_mode.py index 97e3fd099..1decb47c1 100644 --- a/otaclient/app/create_standby/rebuild_mode.py +++ b/otaclient/app/create_standby/rebuild_mode.py @@ -20,6 +20,7 @@ from pathlib import Path from typing import List, Set, Tuple +from otaclient._utils.path import replace_root from ..common import RetryTaskMap, get_backoff from ..configs import config as cfg from ..ota_metadata import MetafilesV1, OTAMetadata @@ -34,9 +35,7 @@ from .common import HardlinkRegister, DeltaGenerator, DeltaBundle from .interface import StandbySlotCreatorProtocol -logger = log_setting.get_logger( - __name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL) -) +logger = log_setting.get_logger(__name__) class RebuildMode(StandbySlotCreatorProtocol): @@ -59,7 +58,7 @@ def __init__( # recycle folder, files copied from referenced slot will be stored here, # also the meta files will be stored under this folder - self._ota_tmp = self.standby_slot_mp / Path(cfg.OTA_TMP_STORE).relative_to("/") + self._ota_tmp = Path(cfg.STANDBY_OTA_TMP_DPATH) self._ota_tmp.mkdir(exist_ok=True) def _cal_and_prepare_delta(self): @@ -82,13 +81,15 @@ def _process_persistents(self): """NOTE: just copy from legacy mode""" from ..copy_tree import CopyTree - _passwd_file = Path(cfg.PASSWD_FILE) - _group_file = Path(cfg.GROUP_FILE) _copy_tree = CopyTree( - src_passwd_file=_passwd_file, - src_group_file=_group_file, - dst_passwd_file=self.standby_slot_mp / _passwd_file.relative_to("/"), - dst_group_file=self.standby_slot_mp / _group_file.relative_to("/"), + src_passwd_file=Path(cfg.PASSWD_FPATH), + src_group_file=Path(cfg.GROUP_FPATH), + dst_passwd_file=Path( + replace_root(cfg.PASSWD_FPATH, cfg.ACTIVE_ROOTFS, cfg.STANDBY_SLOT_MP) + ), + dst_group_file=Path( + replace_root(cfg.GROUP_FPATH, cfg.ACTIVE_ROOTFS, cfg.STANDBY_SLOT_MP) + ), ) for _perinf in self._ota_metadata.iter_metafile(MetafilesV1.PERSISTENT_FNAME): @@ -187,7 +188,7 @@ def _process_regular(self, _input: Tuple[bytes, Set[RegularInf]]): def _save_meta(self): """Save metadata to META_FOLDER.""" - _dst = self.standby_slot_mp / Path(cfg.META_FOLDER).relative_to("/") + _dst = Path(cfg.STANDBY_IMAGE_META_DPATH) _dst.mkdir(parents=True, exist_ok=True) logger.info(f"save image meta files to {_dst}") diff --git a/otaclient/app/downloader.py b/otaclient/app/downloader.py index 29e75d879..ab1325804 100644 --- a/otaclient/app/downloader.py +++ b/otaclient/app/downloader.py @@ -46,15 +46,13 @@ from urllib3.util.retry import Retry from urllib3.response import HTTPResponse -from otaclient._utils import copy_callable_typehint +from otaclient._utils.typing import copy_callable_typehint from otaclient.ota_proxy import OTAFileCacheControl from .configs import config as cfg from .common import wait_with_backoff from . import log_setting -logger = log_setting.get_logger( - __name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL) -) +logger = log_setting.get_logger(__name__) EMPTY_FILE_SHA256 = r"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" CACHE_CONTROL_HEADER = OTAFileCacheControl.HEADER_LOWERCASE diff --git a/otaclient/app/ecu_info.py b/otaclient/app/ecu_info.py index e89f7a4e9..6cc182ddc 100644 --- a/otaclient/app/ecu_info.py +++ b/otaclient/app/ecu_info.py @@ -20,12 +20,10 @@ from typing import Iterator, NamedTuple, Union, Dict, List, Any from . import log_setting -from .configs import config as cfg, server_cfg from .boot_control import BootloaderType +from .configs import service_config -logger = log_setting.get_logger( - __name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL) -) +logger = log_setting.get_logger(__name__) DEFAULT_ECU_INFO = { @@ -37,7 +35,7 @@ class ECUContact(NamedTuple): ecu_id: str host: str - port: int = server_cfg.SERVER_PORT + port: int = service_config.CLIENT_CALL_PORT @dataclass @@ -57,7 +55,7 @@ class ECUInfo: """ ecu_id: str - ip_addr: str = "127.0.0.1" + ip_addr: str = str(service_config.DEFAULT_SERVER_ADDRESS) bootloader: str = BootloaderType.UNSPECIFIED.value available_ecu_ids: list = field(default_factory=list) # list[str] secondaries: list = field(default_factory=list) # list[dict[str, Any]] @@ -115,7 +113,7 @@ def iter_direct_subecu_contact(self) -> Iterator[ECUContact]: yield ECUContact( ecu_id=subecu["ecu_id"], host=subecu["ip_addr"], - port=subecu.get("port", server_cfg.SERVER_PORT), + port=subecu.get("port", service_config.CLIENT_CALL_PORT), ) except KeyError: raise ValueError(f"{subecu=} info is invalid") diff --git a/otaclient/app/interface.py b/otaclient/app/interface.py index 564dec98c..d7d0b3cdb 100644 --- a/otaclient/app/interface.py +++ b/otaclient/app/interface.py @@ -37,8 +37,6 @@ def update( version: str, url_base: str, cookies_json: str, - *, - fsm, # OTAUpdateFSM ) -> None: ... diff --git a/otaclient/app/log_setting.py b/otaclient/app/log_setting.py index c5bcdbb98..f9012dcce 100644 --- a/otaclient/app/log_setting.py +++ b/otaclient/app/log_setting.py @@ -17,23 +17,25 @@ import yaml from otaclient import otaclient_package_name -from .configs import config as cfg +from .configs import config as cfg, logging_config # NOTE: EcuInfo imports this log_setting so independent get_ecu_id are required. def get_ecu_id(): try: - with open(cfg.ECU_INFO_FILE) as f: + with open(cfg.ECU_INFO_FPATH) as f: ecu_info = yaml.load(f, Loader=yaml.SafeLoader) return ecu_info["ecu_id"] except Exception: return "autoware" -def get_logger(name: str, loglevel: int) -> logging.Logger: - """Helper method to get logger with name and loglevel.""" +def get_logger(name: str) -> logging.Logger: + """Helper method to get logger with name.""" logger = logging.getLogger(name) - logger.setLevel(loglevel) + logger.setLevel( + logging_config.LOG_LEVEL_TABLE.get(__name__, logging_config.LOGGING_LEVEL) + ) return logger @@ -44,7 +46,9 @@ def configure_logging(loglevel: int, *, http_logging_url: str): # when launching subprocess. # NOTE: for the root logger, set to CRITICAL to filter away logs from other # external modules unless reached CRITICAL level. - logging.basicConfig(level=logging.CRITICAL, format=cfg.LOG_FORMAT, force=True) + logging.basicConfig( + level=logging.CRITICAL, format=logging_config.LOG_FORMAT, force=True + ) # NOTE: set the to the otaclient package root logger _otaclient_logger = logging.getLogger(otaclient_package_name) _otaclient_logger.setLevel(loglevel) @@ -55,7 +59,7 @@ def configure_logging(loglevel: int, *, http_logging_url: str): from otaclient.aws_iot_log_server import CustomHttpHandler ch = CustomHttpHandler(host=http_logging_host, url=http_logging_url) - fmt = logging.Formatter(fmt=cfg.LOG_FORMAT) + fmt = logging.Formatter(fmt=logging_config.LOG_FORMAT) ch.setFormatter(fmt) # NOTE: "otaclient" logger will be the root logger for all loggers name diff --git a/otaclient/app/main.py b/otaclient/app/main.py index 1da711042..4c0c9ca0d 100644 --- a/otaclient/app/main.py +++ b/otaclient/app/main.py @@ -13,41 +13,60 @@ # limitations under the License. +from __future__ import annotations import asyncio -import logging import os +import os.path import sys from pathlib import Path -from otaclient import __version__ # type: ignore +# NOTE: preload protobuf/grpc related modules before any other component +# modules being imported. from .proto import wrapper, v2, v2_grpc, ota_metafiles # noqa: F401 + +from otaclient import __version__ # type: ignore +from otaclient._utils import if_run_as_container from .common import read_str_from_file, write_str_to_file_sync -from .configs import config as cfg, EXTRA_VERSION_FILE -from .log_setting import configure_logging, get_ecu_id +from .configs import config as cfg, logging_config, EXTRA_VERSION_FILE +from .log_setting import configure_logging, get_ecu_id, get_logger from .ota_client_service import launch_otaclient_grpc_server # configure logging before any code being executed -configure_logging(loglevel=cfg.DEFAULT_LOG_LEVEL, http_logging_url=get_ecu_id()) -logger = logging.getLogger(__name__) +configure_logging(logging_config.LOGGING_LEVEL, http_logging_url=get_ecu_id()) +logger = get_logger(__name__) def _check_other_otaclient(): """Check if there is another otaclient instance running.""" # create a lock file to prevent multiple ota-client instances start - if pid := read_str_from_file(cfg.OTACLIENT_PID_FILE): + if pid := read_str_from_file(cfg.OTACLIENT_PID_FPATH): # running process will have a folder under /proc if Path(f"/proc/{pid}").is_dir(): logger.error(f"another instance of ota-client({pid=}) is running, abort") sys.exit() else: logger.warning(f"dangling otaclient lock file({pid=}) detected, cleanup") - Path(cfg.OTACLIENT_PID_FILE).unlink(missing_ok=True) + Path(cfg.OTACLIENT_PID_FPATH).unlink(missing_ok=True) # create run dir - _run_dir = Path(cfg.RUN_DIR) + _run_dir = Path(cfg.RUN_DPATH) _run_dir.mkdir(parents=True, exist_ok=True) os.chmod(_run_dir, 0o550) # write our pid to the lock file - write_str_to_file_sync(cfg.OTACLIENT_PID_FILE, f"{os.getpid()}") + write_str_to_file_sync(cfg.OTACLIENT_PID_FPATH, f"{os.getpid()}") + + +def _check_active_rootfs(): + """Checking the ACTIVE_ROOTFS config value when in container mode.""" + active_rootfs_mp = cfg.ACTIVE_ROOTFS + if active_rootfs_mp == cfg.DEFAULT_ACTIVE_ROOTFS: + return + + assert os.path.isdir( + active_rootfs_mp + ), f"ACTIVE_ROOTFS must be a dir, get {active_rootfs_mp}" + assert os.path.isabs( + active_rootfs_mp + ), f"ACTIVE_ROOTFS must be absolute, get: {active_rootfs_mp}" def main(): @@ -56,6 +75,17 @@ def main(): logger.info(read_str_from_file(EXTRA_VERSION_FILE)) logger.info(f"otaclient version: {__version__}") - # start the otaclient grpc server + # issue a warning if otaclient detects itself is running as container, + # but config.IS_CONTAINER is not True(ACTIVE_ROOTFS is not configured). + # TODO: do more things over this unexpected condition? + if if_run_as_container() and not cfg.IS_CONTAINER: + logger.warning( + "otaclient seems to run as container, but host rootfs is not mounted into the container " + "and/or ACTIVE_ROOTFS not specified" + ) + + # do pre-start checking + _check_active_rootfs() _check_other_otaclient() + asyncio.run(launch_otaclient_grpc_server()) diff --git a/otaclient/app/ota_client.py b/otaclient/app/ota_client.py index c244044fc..c1ad63c69 100644 --- a/otaclient/app/ota_client.py +++ b/otaclient/app/ota_client.py @@ -13,6 +13,7 @@ # limitations under the License. +from __future__ import annotations import asyncio import gc import json @@ -33,7 +34,7 @@ RetryTaskMap, RetryTaskMapInterrupted, ) -from .configs import config as cfg +from .configs import config as cfg, debug_flags from .create_standby import StandbySlotCreatorProtocol, get_standby_slot_creator from .ecu_info import ECUInfo from .interface import OTAClientProtocol @@ -51,9 +52,7 @@ except ImportError: __version__ = "unknown" -logger = log_setting.get_logger( - __name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL) -) +logger = log_setting.get_logger(__name__) class OTAClientControlFlags: @@ -119,12 +118,8 @@ def __init__( self._update_stats_collector = OTAUpdateStatsCollector() # paths - self._ota_tmp_on_standby = Path(cfg.MOUNT_POINT) / Path( - cfg.OTA_TMP_STORE - ).relative_to("/") - self._ota_tmp_image_meta_dir_on_standby = Path(cfg.MOUNT_POINT) / Path( - cfg.OTA_TMP_META_STORE - ).relative_to("/") + self._ota_tmp_on_standby = Path(cfg.STANDBY_OTA_TMP_DPATH) + self._ota_tmp_image_meta_dir_on_standby = Path(cfg.STANDBY_IMAGE_META_DPATH) # helper methods @@ -211,6 +206,7 @@ def _update_standby_slot(self): """Apply OTA update to standby slot.""" # ------ pre_update ------ # # --- prepare standby slot --- # + # NOTE: erase standby slot or not based on the used StandbySlotCreator logger.debug("boot controller prepares standby slot...") self._boot_controller.pre_update( @@ -218,9 +214,10 @@ def _update_standby_slot(self): standby_as_ref=False, # NOTE: this option is deprecated and not used by bootcontroller erase_standby=self._create_standby_cls.should_erase_standby_slot(), ) + # prepare the tmp storage on standby slot after boot_controller.pre_update finished - self._ota_tmp_on_standby.mkdir(exist_ok=True) - self._ota_tmp_image_meta_dir_on_standby.mkdir(exist_ok=True) + self._ota_tmp_on_standby.mkdir(parents=True, exist_ok=True) + self._ota_tmp_image_meta_dir_on_standby.mkdir(parents=True, exist_ok=True) # --- init standby_slot creator, calculate delta --- # logger.info("start to calculate and prepare delta...") @@ -228,8 +225,8 @@ def _update_standby_slot(self): self._standby_slot_creator = self._create_standby_cls( ota_metadata=self._otameta, boot_dir=str(self._boot_controller.get_standby_boot_dir()), - standby_slot_mount_point=cfg.MOUNT_POINT, - active_slot_mount_point=cfg.ACTIVE_ROOT_MOUNT_POINT, + standby_slot_mount_point=cfg.STANDBY_SLOT_MP, + active_slot_mount_point=cfg.ACTIVE_SLOT_MP, stats_collector=self._update_stats_collector, ) try: @@ -531,7 +528,10 @@ def _on_failure(self, exc: ota_errors.OTAError, ota_status: wrapper.StatusOta): try: self.last_failure_type = exc.failure_type self.last_failure_reason = exc.get_failure_reason() - if cfg.DEBUG_MODE: + if ( + debug_flags.DEBUG_MODE + or debug_flags.DEBUG_ENABLE_TRACEBACK_IN_STATUS_API + ): self.last_failure_traceback = exc.get_failure_traceback() logger.error( @@ -668,7 +668,10 @@ def __init__( failure_reason=e.get_failure_reason(), ) - if cfg.DEBUG_MODE: + if ( + debug_flags.DEBUG_MODE + or debug_flags.DEBUG_ENABLE_TRACEBACK_IN_STATUS_API + ): self._otaclient_startup_failed_status.failure_traceback = ( e.get_failure_traceback() ) @@ -695,7 +698,10 @@ def __init__( failure_reason=e.get_failure_reason(), ) - if cfg.DEBUG_MODE: + if ( + debug_flags.DEBUG_MODE + or debug_flags.DEBUG_ENABLE_TRACEBACK_IN_STATUS_API + ): self._otaclient_startup_failed_status.failure_traceback = ( e.get_failure_traceback() ) diff --git a/otaclient/app/ota_client_call.py b/otaclient/app/ota_client_call.py index 458a7ce79..e61588bab 100644 --- a/otaclient/app/ota_client_call.py +++ b/otaclient/app/ota_client_call.py @@ -13,10 +13,12 @@ # limitations under the License. +from __future__ import annotations import grpc.aio +from typing import Optional from .proto import wrapper, v2_grpc -from .configs import server_cfg +from .configs import service_config class ECUNoResponse(Exception): @@ -28,10 +30,10 @@ class OtaClientCall: async def status_call( ecu_id: str, ecu_ipaddr: str, - ecu_port: int = server_cfg.SERVER_PORT, + ecu_port: int = service_config.CLIENT_CALL_PORT, *, request: wrapper.StatusRequest, - timeout=None, + timeout: Optional[float] = None, ) -> wrapper.StatusResponse: try: ecu_addr = f"{ecu_ipaddr}:{ecu_port}" @@ -47,10 +49,10 @@ async def status_call( async def update_call( ecu_id: str, ecu_ipaddr: str, - ecu_port: int = server_cfg.SERVER_PORT, + ecu_port: int = service_config.CLIENT_CALL_PORT, *, request: wrapper.UpdateRequest, - timeout=None, + timeout: Optional[float] = None, ) -> wrapper.UpdateResponse: try: ecu_addr = f"{ecu_ipaddr}:{ecu_port}" @@ -66,10 +68,10 @@ async def update_call( async def rollback_call( ecu_id: str, ecu_ipaddr: str, - ecu_port: int = server_cfg.SERVER_PORT, + ecu_port: int = service_config.CLIENT_CALL_PORT, *, request: wrapper.RollbackRequest, - timeout=None, + timeout: Optional[float] = None, ) -> wrapper.RollbackResponse: try: ecu_addr = f"{ecu_ipaddr}:{ecu_port}" diff --git a/otaclient/app/ota_client_service.py b/otaclient/app/ota_client_service.py index b8bf7502d..33bf08fd2 100644 --- a/otaclient/app/ota_client_service.py +++ b/otaclient/app/ota_client_service.py @@ -13,13 +13,17 @@ # limitations under the License. +from __future__ import annotations import grpc.aio -from .configs import config as cfg, server_cfg +from .configs import config as cfg, debug_flags, service_config from .ecu_info import ECUInfo +from .log_setting import get_logger from .proto import wrapper, v2, v2_grpc from .ota_client_stub import OTAClientServiceStub +logger = get_logger(__name__) + class OtaClientServiceV2(v2_grpc.OtaClientServiceServicer): def __init__(self, ota_client_stub: OTAClientServiceStub): @@ -41,7 +45,7 @@ async def Status(self, request: v2.StatusRequest, context) -> v2.StatusResponse: def create_otaclient_grpc_server(): - ecu_info = ECUInfo.parse_ecu_info(cfg.ECU_INFO_FILE) + ecu_info = ECUInfo.parse_ecu_info(cfg.ECU_INFO_FPATH) service_stub = OTAClientServiceStub(ecu_info=ecu_info) ota_client_service_v2 = OtaClientServiceV2(service_stub) @@ -50,7 +54,17 @@ def create_otaclient_grpc_server(): v2_grpc.add_OtaClientServiceServicer_to_server( server=server, servicer=ota_client_service_v2 ) - server.add_insecure_port(f"{ecu_info.ip_addr}:{server_cfg.SERVER_PORT}") + + listen_addr = ecu_info.ip_addr + if debug_flags.DEBUG_SERVER_LISTEN_ADDR: # for advanced debug use case only + logger.warning(f"{debug_flags.DEBUG_SERVER_LISTEN_ADDR=} is activated") + listen_addr = debug_flags.DEBUG_SERVER_LISTEN_ADDR + listen_port = service_config.SERVER_PORT + + listen_info = f"{listen_addr}:{listen_port}" + logger.info(f"create OTA grpc server at {listen_info}") + + server.add_insecure_port(listen_info) return server diff --git a/otaclient/app/ota_client_stub.py b/otaclient/app/ota_client_stub.py index d9affd1c6..fb048f9fa 100644 --- a/otaclient/app/ota_client_stub.py +++ b/otaclient/app/ota_client_stub.py @@ -13,6 +13,7 @@ # limitations under the License. +from __future__ import annotations import asyncio import logging import shutil @@ -26,7 +27,7 @@ from typing_extensions import Self from . import log_setting -from .configs import config as cfg, server_cfg +from .configs import config as cfg, logging_config from .common import ensure_otaproxy_start from .boot_control._common import CMDHelperFuncs from .ecu_info import ECUContact, ECUInfo @@ -42,9 +43,7 @@ ) -logger = log_setting.get_logger( - __name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL) -) +logger = log_setting.get_logger(__name__) class _OTAProxyContext(OTAProxyContextProto): @@ -93,7 +92,7 @@ def _subprocess_init(self): loglevel=logging.CRITICAL, http_logging_url=log_setting.get_ecu_id() ) otaproxy_logger = logging.getLogger("otaclient.ota_proxy") - otaproxy_logger.setLevel(cfg.DEFAULT_LOG_LEVEL) + otaproxy_logger.setLevel(logging_config.LOGGING_LEVEL) self.logger = otaproxy_logger # wait for upper otaproxy if any @@ -698,7 +697,7 @@ async def _polling_direct_subecu_status(self, ecu_contact: ECUContact): ecu_contact.ecu_id, ecu_contact.host, ecu_contact.port, - timeout=server_cfg.QUERYING_SUBECU_STATUS_TIMEOUT, + timeout=cfg.QUERYING_SUBECU_STATUS_TIMEOUT, request=wrapper.StatusRequest(), ) await self._ecu_status_storage.update_from_child_ecu(_ecu_resp) @@ -731,8 +730,6 @@ def __init__(self, *, ecu_info: ECUInfo, _proxy_cfg=proxy_cfg): ) self.ecu_info = ecu_info - self.listen_addr = ecu_info.ip_addr - self.listen_port = server_cfg.SERVER_PORT self.my_ecu_id = ecu_info.ecu_id self._otaclient_control_flags = OTAClientControlFlags() @@ -848,7 +845,7 @@ async def update(self, request: wrapper.UpdateRequest) -> wrapper.UpdateResponse ecu_contact.host, ecu_contact.port, request=request, - timeout=server_cfg.WAITING_SUBECU_ACK_REQ_TIMEOUT, + timeout=cfg.WAITING_SUBECU_ACK_REQ_TIMEOUT, ) ) tasks[_task] = ecu_contact @@ -863,7 +860,7 @@ async def update(self, request: wrapper.UpdateRequest) -> wrapper.UpdateResponse _ecu_contact = tasks[_task] logger.warning( f"{_ecu_contact} doesn't respond to update request on-time" - f"(within {server_cfg.WAITING_SUBECU_ACK_REQ_TIMEOUT}s): {e!r}" + f"(within {cfg.WAITING_SUBECU_ACK_REQ_TIMEOUT}s): {e!r}" ) # NOTE(20230517): aligns with the previous behavior that create # response with RECOVERABLE OTA error for unresponsive @@ -912,7 +909,7 @@ async def rollback( ecu_contact.host, ecu_contact.port, request=request, - timeout=server_cfg.WAITING_SUBECU_ACK_REQ_TIMEOUT, + timeout=cfg.WAITING_SUBECU_ACK_REQ_TIMEOUT, ) ) tasks[_task] = ecu_contact @@ -926,7 +923,7 @@ async def rollback( _ecu_contact = tasks[_task] logger.warning( f"{_ecu_contact} doesn't respond to rollback request on-time" - f"(within {server_cfg.WAITING_SUBECU_ACK_REQ_TIMEOUT}s): {e!r}" + f"(within {cfg.WAITING_SUBECU_ACK_REQ_TIMEOUT}s): {e!r}" ) # NOTE(20230517): aligns with the previous behavior that create # response with RECOVERABLE OTA error for unresponsive diff --git a/otaclient/app/ota_metadata.py b/otaclient/app/ota_metadata.py index a69d945c4..92b964e32 100644 --- a/otaclient/app/ota_metadata.py +++ b/otaclient/app/ota_metadata.py @@ -87,9 +87,7 @@ from .proto.streamer import Uint32LenDelimitedMsgReader, Uint32LenDelimitedMsgWriter from . import log_setting -logger = log_setting.get_logger( - __name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL) -) +logger = log_setting.get_logger(__name__) CACHE_CONTROL_HEADER = OTAFileCacheControl.HEADER_LOWERCASE @@ -599,7 +597,7 @@ class OTAMetadata: def __init__(self, *, url_base: str, downloader: Downloader) -> None: self.url_base = url_base self._downloader = downloader - self._tmp_dir = TemporaryDirectory(prefix="ota_metadata", dir=cfg.RUN_DIR) + self._tmp_dir = TemporaryDirectory(prefix="ota_metadata", dir=cfg.RUN_DPATH) self._tmp_dir_path = Path(self._tmp_dir.name) # download and parse the metadata.jwt @@ -626,7 +624,7 @@ def _process_metadata_jwt(self) -> _MetadataJWTClaimsLayout: """Download, loading and parsing metadata.jwt.""" logger.debug("process metadata.jwt...") # download and parse metadata.jwt - with NamedTemporaryFile(prefix="metadata_jwt", dir=cfg.RUN_DIR) as meta_f: + with NamedTemporaryFile(prefix="metadata_jwt", dir=cfg.RUN_DPATH) as meta_f: _downloaded_meta_f = Path(meta_f.name) self._downloader.download_retry_inf( urljoin_ensure_base(self.url_base, self.METADATA_JWT), @@ -640,13 +638,13 @@ def _process_metadata_jwt(self) -> _MetadataJWTClaimsLayout: ) _parser = _MetadataJWTParser( - _downloaded_meta_f.read_text(), certs_dir=cfg.CERTS_DIR + _downloaded_meta_f.read_text(), certs_dir=cfg.OTA_CERTS_DPATH ) # get not yet verified parsed ota_metadata _ota_metadata = _parser.get_otametadata() # download certificate and verify metadata against this certificate - with NamedTemporaryFile(prefix="metadata_cert", dir=cfg.RUN_DIR) as cert_f: + with NamedTemporaryFile(prefix="metadata_cert", dir=cfg.RUN_DPATH) as cert_f: cert_info = _ota_metadata.certificate cert_fname, cert_hash = cert_info.file, cert_info.hash cert_file = Path(cert_f.name) diff --git a/otaclient/app/ota_status.py b/otaclient/app/ota_status.py index 3ce9d699f..daa294f17 100644 --- a/otaclient/app/ota_status.py +++ b/otaclient/app/ota_status.py @@ -13,13 +13,10 @@ # limitations under the License. -from .configs import config as cfg from .proto import wrapper from . import log_setting -logger = log_setting.get_logger( - __name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL) -) +logger = log_setting.get_logger(__name__) class LiveOTAStatus: diff --git a/otaclient/app/proxy_info.py b/otaclient/app/proxy_info.py index 010ccbd83..129e9dfa7 100644 --- a/otaclient/app/proxy_info.py +++ b/otaclient/app/proxy_info.py @@ -25,11 +25,8 @@ from . import log_setting from .configs import config as cfg -from .configs import server_cfg -logger = log_setting.get_logger( - __name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL) -) +logger = log_setting.get_logger(__name__) PRE_DEFINED_PROXY_INFO_YAML = """ @@ -61,8 +58,8 @@ class ProxyInfo: gateway: bool = False upper_ota_proxy: str = "" enable_local_ota_proxy: bool = False - local_ota_proxy_listen_addr: str = server_cfg.OTA_PROXY_LISTEN_ADDRESS - local_ota_proxy_listen_port: int = server_cfg.OTA_PROXY_LISTEN_PORT + local_ota_proxy_listen_addr: str = str(cfg.OTA_PROXY_LISTEN_ADDRESS) + local_ota_proxy_listen_port: int = cfg.OTA_PROXY_LISTEN_PORT # NOTE: this field not presented in v2.5.4, # for current implementation, it should be default to True. # This field doesn't take effect if enable_local_ota_proxy is False @@ -87,7 +84,7 @@ def get_proxy_for_local_ota(self) -> str: return "" -def parse_proxy_info(proxy_info_file: str = cfg.PROXY_INFO_FILE) -> ProxyInfo: +def parse_proxy_info(proxy_info_file: str = cfg.PROXY_INFO_FPATH) -> ProxyInfo: _loaded: Dict[str, Any] try: _loaded = yaml.safe_load(Path(proxy_info_file).read_text()) diff --git a/otaclient/app/update_stats.py b/otaclient/app/update_stats.py index 6b5c820ee..e8c544e98 100644 --- a/otaclient/app/update_stats.py +++ b/otaclient/app/update_stats.py @@ -26,9 +26,7 @@ from .proto.wrapper import UpdateStatus -logger = log_setting.get_logger( - __name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL) -) +logger = log_setting.get_logger(__name__) class RegProcessOperation(Enum): diff --git a/otaclient/configs/__init__.py b/otaclient/configs/__init__.py new file mode 100644 index 000000000..55ed90a28 --- /dev/null +++ b/otaclient/configs/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2022 TIER IV, INC. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""otaclient configs package.""" diff --git a/otaclient/configs/_common.py b/otaclient/configs/_common.py new file mode 100644 index 000000000..8be08a965 --- /dev/null +++ b/otaclient/configs/_common.py @@ -0,0 +1,37 @@ +# Copyright 2022 TIER IV, INC. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import annotations +from pydantic import BaseModel, ConfigDict +from pydantic_settings import BaseSettings, SettingsConfigDict + +# prefix for environmental vars name for configs. +ENV_PREFIX = "OTA_" + + +class BaseConfigurableConfig(BaseSettings): + """Common base for configs that are configurable via ENV.""" + + model_config = SettingsConfigDict( + env_prefix=ENV_PREFIX, + frozen=True, + validate_default=True, + ) + + +class BaseFixedConfig(BaseModel): + """Common base for configs that should be fixed and not changable.""" + + model_config = ConfigDict(frozen=True, validate_default=True) diff --git a/otaclient/configs/app_cfg.py b/otaclient/configs/app_cfg.py new file mode 100644 index 000000000..71ad1bdd7 --- /dev/null +++ b/otaclient/configs/app_cfg.py @@ -0,0 +1,429 @@ +# Copyright 2022 TIER IV, INC. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""otaclient runtime configs and constants. + +There are three types of configs defined in this module: +1. fixed internal configs: static configs value, should not be changed + due to compatibility reason. +2. configurable configs(via env): configurable configs that changes + otaclient's runtime behavior, check _ConfigurableConfig and _init_config + for more details. +3. dynamically generated configs(paths consts): dynamically generated + otaclient runtime configs based on configurable configs. +""" + + +from __future__ import annotations +import os.path +from enum import Enum +from pydantic import BaseModel, Field, IPvAnyAddress +from typing import TYPE_CHECKING, Any, ClassVar as _std_ClassVar + +from otaclient._utils import cached_computed_field +from otaclient._utils.path import replace_root +from ._common import ENV_PREFIX, BaseConfigurableConfig, BaseFixedConfig + +# A simple trick to make plain ClassVar work when +# __future__.annotations is activated. +if not TYPE_CHECKING: + _std_ClassVar = _std_ClassVar[Any] + +# NOTE: ACTIVE_ROOTFS is specially treated and retrieved via HOST_ROOTFS_ENV. +HOST_ROOTFS_ENV = f"{ENV_PREFIX}HOST_ROOTFS" + + +class CreateStandbyMechanism(str, Enum): + LEGACY = "legacy" # deprecated and removed + REBUILD = "rebuild" # current default + IN_PLACE = "in_place" # not yet implemented + + +# +# ------ configs and consts definition ------ # +# + + +class _FixedInternalConfig(BaseModel): + """Fixed internal configs that should not be changed.""" + + RUN_DPATH: _std_ClassVar = "/run/otaclient" + OTACLIENT_PID_FPATH: _std_ClassVar = "/run/otaclient.pid" + SUPPORTED_COMPRESS_ALG: _std_ClassVar = ("zst", "zstd") + + # filesystem label of external cache source + EXTERNAL_CACHE_DEV_FSLABEL: _std_ClassVar = "ota_cache_src" + + +class _DynamicRootedPathsConfig(BaseModel): + """Dynamic generated internal paths config. + + Paths configured in this class are dynamically adjusted and rooted with + specified ACTIVE_ROOTFS, by default ACTIVE_ROOTFS is "/". + """ + + # + # ------ active_rootfs & container mode ------ # + # + DEFAULT_ACTIVE_ROOTFS: _std_ClassVar = "/" + + ACTIVE_ROOTFS: str = DEFAULT_ACTIVE_ROOTFS + + @cached_computed_field + def IS_CONTAINER(self) -> bool: + """Whether otaclient is running as container. + + If active rootfs is specified and not "/", otaclient will + activate the container running mode. + """ + return self.ACTIVE_ROOTFS != self.DEFAULT_ACTIVE_ROOTFS + + # + # ------ mount point placement ------ # + # + DEFAULT_OTACLIENT_MOUNT_SPACE: _std_ClassVar = "/mnt/otaclient" + + @cached_computed_field + def OTACLIENT_MOUNT_SPACE_DPATH(self) -> str: + """The dynamically rooted location to hold mount points created by otaclient. + + Default: /mnt/otaclient + """ + return replace_root( + self.DEFAULT_OTACLIENT_MOUNT_SPACE, + self.DEFAULT_ACTIVE_ROOTFS, + self.ACTIVE_ROOTFS, + ) + + @cached_computed_field + def STANDBY_SLOT_MP(self) -> str: + """The dynamically rooted location to mount standby slot partition. + + Default: /mnt/otaclient/standby_slot + """ + return os.path.join(self.OTACLIENT_MOUNT_SPACE_DPATH, "standby_slot") + + @cached_computed_field + def ACTIVE_SLOT_MP(self) -> str: + """The dynamically rooted location to mount active slot partition. + + Default: /mnt/otaclient/active_slot + """ + return os.path.join(self.OTACLIENT_MOUNT_SPACE_DPATH, "active_slot") + + # + # ------ /boot related ------ # + # + @cached_computed_field + def BOOT_DPATH(self) -> str: + """The dynamically rooted location of /boot dir. + + Default: /boot + """ + return os.path.join(self.ACTIVE_ROOTFS, "boot") + + # /boot/ota and its files + + @cached_computed_field + def BOOT_OTA_DPATH(self) -> str: + """The dynamically rooted location holding proxy_info.yaml and ecu_info.yaml. + + Default: /boot/ota + """ + return os.path.join(self.BOOT_DPATH, "ota") + + @cached_computed_field + def ECU_INFO_FPATH(self) -> str: + """The dynamically rooted location of ecu_info.yaml. + + Default: /boot/ota/ecu_info.yaml + """ + return os.path.join(self.BOOT_OTA_DPATH, "ecu_info.yaml") + + @cached_computed_field + def PROXY_INFO_FPATH(self) -> str: + """The dynamically rooted location of proxy_info.yaml. + + Default: /boot/ota/proxy_info.yaml + """ + return os.path.join(self.BOOT_OTA_DPATH, "proxy_info.yaml") + + # --- /boot/ota-status and its files --- # + # NOTE: the actual location of these files depends on each boot controller + # implementation, please refer to boot_control.configs. + + OTA_STATUS_FNAME: _std_ClassVar = "status" + OTA_VERSION_FNAME: _std_ClassVar = "version" + SLOT_IN_USE_FNAME: _std_ClassVar = "slot_in_use" + + # --- some files under /etc --- # + + @cached_computed_field + def ETC_DPATH(self) -> str: + """The dynamically rooted location of /etc folder. + + Default: /etc + """ + return os.path.join(self.ACTIVE_ROOTFS, "etc") + + @cached_computed_field + def PASSWD_FPATH(self) -> str: + """The dynamically rooted location of /etc/passwd file. + + Default: /etc/passwd + """ + return os.path.join(self.ETC_DPATH, "passwd") + + @cached_computed_field + def GROUP_FPATH(self) -> str: + """The dynamically rooted location of /etc/group file. + + Default: /etc/group + """ + return os.path.join(self.ETC_DPATH, "group") + + @cached_computed_field + def FSTAB_FPATH(self) -> str: + """The dynamically rooted location of /etc/fstab file. + + Default: /etc/fstab + """ + return os.path.join(self.ETC_DPATH, "fstab") + + # + # ------ /opt/ota paths ------ # + # + # This folder holds files and packages related to OTA functionality. + + DEFAULT_OTA_CERTS_DPATHS: _std_ClassVar = "/opt/ota/client/certs" + DEFAULT_OTA_INSTALLATION_PATH: _std_ClassVar = "/opt/ota" + + @cached_computed_field + def OTA_INSTALLATION_PATH(self) -> str: + """The dynamically rooted OTA installation path. + + Default: /opt/ota + """ + return replace_root( + self.DEFAULT_OTA_INSTALLATION_PATH, + self.DEFAULT_ACTIVE_ROOTFS, + self.ACTIVE_ROOTFS, + ) + + @cached_computed_field + def OTA_CERTS_DPATH(self) -> str: + """The dynamically rooted location of certs for OTA metadata validation. + + Default: /opt/ota/client/certs + """ + return replace_root( + self.DEFAULT_OTA_CERTS_DPATHS, + self.DEFAULT_ACTIVE_ROOTFS, + self.ACTIVE_ROOTFS, + ) + + @cached_computed_field + def OTACLIENT_INSTALLATION_PATH(self) -> str: + """The dynamically rooted location of otaclient installation path. + + Default: /opt/ota/client + """ + return os.path.join(self.OTA_INSTALLATION_PATH, "client") + + @cached_computed_field + def ACTIVE_IMAGE_META_DPATH(self) -> str: + """The dynamically rooted location of the image-meta of active slot. + + Default: /opt/ota/image-meta + """ + return os.path.join(self.OTA_INSTALLATION_PATH, "image-meta") + + @cached_computed_field + def STANDBY_IMAGE_META_DPATH(self) -> str: + """The location of save destination of standby slot. + + NOTE: this location is relatived to the standby slot's mount point. + + Default: /mnt/otaclient/standby_slot/opt/ota/image-meta + """ + return replace_root( + self.ACTIVE_IMAGE_META_DPATH, self.ACTIVE_ROOTFS, self.STANDBY_SLOT_MP + ) + + # + # ------ external OTA cache source support ------ + # + @cached_computed_field + def EXTERNAL_CACHE_DEV_MOUNTPOINT(self) -> str: + """The mount point for external OTA cache source partition. + + Default: /mnt/otaclient/external_cache_src + """ + return os.path.join(self.OTACLIENT_MOUNT_SPACE_DPATH, "external_cache_src") + + @cached_computed_field + def EXTERNAL_CACHE_SRC_PATH(self) -> str: + """The data folder of the external cache source filesystem. + + NOTE: this path is relatived to the external cache source mount point. + + Default: /mnt/otaclient/external_cache_src/data + """ + return os.path.join(self.EXTERNAL_CACHE_DEV_MOUNTPOINT, "data") + + +class _InternalConfig(_FixedInternalConfig, _DynamicRootedPathsConfig): + """Internal configs for otaclient. + + User should not change these settings, except ACTIVE_ROOTFS if running as container, + otherwise otaclient might not work properly or backward-compatibility breaks. + """ + + +class _ConfigurableConfig(BaseModel): + """User configurable otaclient settings. + + These settings can tune the runtime performance and behavior of otaclient, + configurable via environment variables, with prefix OTA. + For example, to set SERVER_ADDRESS, set env OTA_SERVER_ADDRESS=10.0.1.1 . + """ + + # name of OTA used temp folder + OTA_TMP_DNAME: str = "ota_tmp" + + # + # ------ otaproxy server config ------ # + # + OTA_PROXY_LISTEN_ADDRESS: IPvAnyAddress = IPvAnyAddress("0.0.0.0") + OTA_PROXY_LISTEN_PORT: int = Field(default=8082, ge=0, le=65535) + + # + # ------ otaclient runtime behavior setting ------ # + # + + # --- request dispatch settings --- # + WAITING_SUBECU_ACK_REQ_TIMEOUT: int = 6 + QUERYING_SUBECU_STATUS_TIMEOUT: int = 30 + LOOP_QUERYING_SUBECU_STATUS_INTERVAL: int = 10 + + # --- file I/O settings --- # + CHUNK_SIZE: int = 1 * 1024 * 1024 # 1MB + LOCAL_CHUNK_SIZE: int = 4 * 1024 * 1024 # 4MB + + # + # --- download settings for single download task --- # + # + DOWNLOAD_RETRY: int = 3 + DOWNLOAD_BACKOFF_MAX: int = 3 # seconds + DOWNLOAD_BACKOFF_FACTOR: float = 0.1 # seconds + + # + # --- downloader settings --- # + # + MAX_DOWNLOAD_THREAD: int = Field(default=7, le=32) + DOWNLOADER_CONNPOOL_SIZE_PER_THREAD: int = Field(default=20, le=64) + + # + # --- download settings for the whole download tasks group --- # + # + # if retry keeps failing without any success in + # DOWNLOAD_GROUP_NO_SUCCESS_RETRY_TIMEOUT time, failed the whole + # download task group and raise NETWORK OTA error. + MAX_CONCURRENT_DOWNLOAD_TASKS: int = Field(default=128, le=1024) + DOWNLOAD_GROUP_INACTIVE_TIMEOUT: int = 5 * 60 # seconds + DOWNLOAD_GROUP_BACKOFF_MAX: int = 12 # seconds + DOWNLOAD_GROUP_BACKOFF_FACTOR: int = 1 # seconds + + # + # --- stats collector setting --- # + # + STATS_COLLECT_INTERVAL: int = 1 # second + + # + # --- create standby setting --- # + # + # now only REBUILD mode is available + STANDBY_CREATION_MODE: CreateStandbyMechanism = CreateStandbyMechanism.REBUILD + MAX_CONCURRENT_PROCESS_FILE_TASKS: int = Field(default=256, le=2048) + CREATE_STANDBY_RETRY_MAX: int = 3 + CREATE_STANDBY_BACKOFF_FACTOR: int = 1 + CREATE_STANDBY_BACKOFF_MAX: int = 6 + + # + # --- ECU status polling setting, otaproxy dependency managing --- # + # + # The ECU status storage will summarize the stored ECUs' status report + # and generate overall status report for all ECUs every seconds. + OVERALL_ECUS_STATUS_UPDATE_INTERVAL: int = 6 # seconds + + # If ECU has been disconnected longer than seconds, it will be + # treated as UNREACHABLE, and will not be counted when generating overall + # ECUs status report. + # NOTE: unreachable_timeout should be larger than + # downloading_group timeout + ECU_UNREACHABLE_TIMEOUT: int = 20 * 60 # seconds + + # Otaproxy should not be shutdowned with less than seconds + # after it just starts to prevent repeatedly start/stop cycle. + OTAPROXY_MINIMUM_SHUTDOWN_INTERVAL: int = 1 * 60 # seconds + + # When any ECU acks update request, this ECU will directly set the overall ECU status + # to any_in_update=True, any_requires_network=True, all_success=False, to prevent + # pre-mature overall ECU status changed caused by child ECU delayed ack to update request. + # + # This pre-set overall ECU status will be kept for seconds. + # This value is expected to be larger than the time cost for subECU acks the OTA request. + KEEP_OVERALL_ECUS_STATUS_ON_ANY_UPDATE_REQ_ACKED: int = 60 # seconds + + # Active status polling interval, when there is active OTA update in the cluster. + ACTIVE_INTERVAL: int = 1 # second + + # Idle status polling interval, when ther is no active OTA updaste in the cluster. + IDLE_INTERVAL: int = 10 # seconds + + # + # --- default version str --- # + # + # The string return in status API firmware_version field if version is unknown. + DEFAULT_VERSION_STR: str = "" + + +class Config(BaseFixedConfig, _InternalConfig, _ConfigurableConfig): + @cached_computed_field + def STANDBY_OTA_TMP_DPATH(self) -> str: + """The location for holding OTA runtime files during OTA. + + NOTE: this location is relatived to standby slot's mount point. + + Default: /mnt/otaclient/standby_slot/ota_tmp + """ + return os.path.join(self.STANDBY_SLOT_MP, self.OTA_TMP_DNAME) + + +# ------ init config ------ # + + +def _init_config() -> Config: + class _ConfigureViaENV(BaseConfigurableConfig, _ConfigurableConfig): + """one-time class that parse configs from environment vars.""" + + return Config( + # especially get ACTIVE_ROOTFS via HOST_ROOTFS_ENV. + ACTIVE_ROOTFS=os.getenv( + HOST_ROOTFS_ENV, _DynamicRootedPathsConfig.DEFAULT_ACTIVE_ROOTFS + ), + **_ConfigureViaENV().model_dump(), + ) + + +app_config = _init_config() diff --git a/otaclient/configs/debug_cfg.py b/otaclient/configs/debug_cfg.py new file mode 100644 index 000000000..9d242a5cb --- /dev/null +++ b/otaclient/configs/debug_cfg.py @@ -0,0 +1,49 @@ +# Copyright 2022 TIER IV, INC. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""otaclient debug flags settings. + +These configs are configurable via environmental vars. + +Debug configs consists of two types of configs: +1. flag: which enables/disables specific feature at runtime, +2. override_config: which overrides specific config at runtime. + +If the main DEBUG_MODE flag is enable, all flag type debug configs +will be enabled. + +For override_config type config, it will be enabled if this config +has assigned correct value via environmental var. +""" + + +from __future__ import annotations +from pydantic import IPvAnyAddress +from typing import Optional +from ._common import BaseConfigurableConfig + + +class DebugFlags(BaseConfigurableConfig): + """Enable internal debug features.""" + + # main DEBUG_MODE switch, this flag will enable all debug feature. + DEBUG_MODE: bool = False + + # enable failure_traceback field in status API response. + DEBUG_ENABLE_TRACEBACK_IN_STATUS_API: bool = False + + # override otaclient grpc server listen address + DEBUG_SERVER_LISTEN_ADDR: Optional[IPvAnyAddress] = None + + +debug_flags = DebugFlags() diff --git a/otaclient/configs/logging_cfg.py b/otaclient/configs/logging_cfg.py new file mode 100644 index 000000000..6d771571d --- /dev/null +++ b/otaclient/configs/logging_cfg.py @@ -0,0 +1,47 @@ +# Copyright 2022 TIER IV, INC. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""otaclient logging configs. + +These configs are configurable via environmental vars. +""" + + +from __future__ import annotations +import logging +from pydantic import AfterValidator +from typing import Dict +from typing_extensions import Annotated + +from otaclient._utils.logging import check_loglevel +from ._common import BaseConfigurableConfig + + +class LoggingConfig(BaseConfigurableConfig): + LOGGING_LEVEL: Annotated[int, AfterValidator(check_loglevel)] = logging.INFO + LOG_LEVEL_TABLE: Dict[str, Annotated[int, AfterValidator(check_loglevel)]] = { + "otaclient.app.boot_control.cboot": LOGGING_LEVEL, + "otaclient.app.boot_control.grub": LOGGING_LEVEL, + "otaclient.app.ota_client": LOGGING_LEVEL, + "otaclient.app.ota_client_service": LOGGING_LEVEL, + "otaclient.app.ota_client_stub": LOGGING_LEVEL, + "otaclient.app.ota_metadata": LOGGING_LEVEL, + "otaclient.app.downloader": LOGGING_LEVEL, + "otaclient.app.main": LOGGING_LEVEL, + } + LOG_FORMAT: str = ( + "[%(asctime)s][%(levelname)s]-%(name)s:%(funcName)s:%(lineno)d,%(message)s" + ) + + +logging_config = LoggingConfig() diff --git a/otaclient/configs/ota_service_cfg.py b/otaclient/configs/ota_service_cfg.py new file mode 100644 index 000000000..256c649ac --- /dev/null +++ b/otaclient/configs/ota_service_cfg.py @@ -0,0 +1,38 @@ +# Copyright 2022 TIER IV, INC. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""otaclient grpc server config. + +For compatibility reason, this config is NOT configurable via env vars. +""" + + +from __future__ import annotations +from pydantic import Field, ConfigDict, IPvAnyAddress +from typing import Literal, Union +from ._common import BaseFixedConfig + + +class OTAServiceConfig(BaseFixedConfig): + """Configurable configs for OTA grpc server/client call.""" + + model_config = ConfigDict(frozen=True, validate_default=True) + + # used when listen_addr is not configured in ecu_info.yaml. + DEFAULT_SERVER_ADDRESS: Union[IPvAnyAddress, Literal["127.0.0.1"]] = "127.0.0.1" + + SERVER_PORT: int = Field(default=50051, ge=0, le=65535) + CLIENT_CALL_PORT: int = Field(default=50051, ge=0, le=65535) + + +service_config = OTAServiceConfig() diff --git a/otaclient/ota_proxy/cache_control.py b/otaclient/ota_proxy/cache_control.py index 2eb27e240..267502e15 100644 --- a/otaclient/ota_proxy/cache_control.py +++ b/otaclient/ota_proxy/cache_control.py @@ -17,7 +17,7 @@ from typing import Dict, List, ClassVar from typing_extensions import Self -from otaclient._utils import copy_callable_typehint_to_method +from otaclient._utils.typing import copy_callable_typehint_to_method _FIELDS = "_fields" diff --git a/otaclient/requirements.txt b/otaclient/requirements.txt index 226ce4622..a38ebb762 100644 --- a/otaclient/requirements.txt +++ b/otaclient/requirements.txt @@ -15,4 +15,6 @@ aiohttp==3.9.0 aiofiles==22.1.0 zstandard==0.18.0 pycurl==7.45.1 -typing_extensions==4.6.3 \ No newline at end of file +typing_extensions==4.8.0 +pydantic==2.5.2 +pydantic-settings==2.1.0 \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 9e1ce3843..30f816de1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,8 +41,11 @@ extend-exclude = '''( [tool.coverage.run] branch = false +include = [ + "otaclient/app/**/*.py", + "otaclient/ota_proxy/**/*.py", +] omit = ["**/*_pb2.py*","**/*_pb2_grpc.py*"] -source = ["otaclient.app", "otaclient.ota_proxy"] [tool.coverage.report] exclude_also = [ diff --git a/tests/conftest.py b/tests/conftest.py index 45b798d9a..6c9501e95 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -47,6 +47,8 @@ class TestConfiguration: # dummy ota-image setting OTA_IMAGE_DIR = "/ota-image" + OTA_IMAGE_DATA_DIR = "/ota-image/data" + METADATA_JWT_FNAME = "metadata.jwt" OTA_IMAGE_SERVER_ADDR = "127.0.0.1" OTA_IMAGE_SERVER_PORT = 8080 OTA_IMAGE_URL = f"http://{OTA_IMAGE_SERVER_ADDR}:{OTA_IMAGE_SERVER_PORT}" @@ -92,28 +94,28 @@ class TestConfiguration: OTA_PROXY_SERVER_PORT = 18080 -cfg = TestConfiguration() +test_cfg = TestConfiguration() @pytest.fixture(autouse=True, scope="session") def run_http_server_subprocess(): _server_p = Process( target=run_http_server, - args=[cfg.OTA_IMAGE_SERVER_ADDR, cfg.OTA_IMAGE_SERVER_PORT], - kwargs={"directory": cfg.OTA_IMAGE_DIR}, + args=[test_cfg.OTA_IMAGE_SERVER_ADDR, test_cfg.OTA_IMAGE_SERVER_PORT], + kwargs={"directory": test_cfg.OTA_IMAGE_DIR}, ) try: _server_p.start() # NOTE: wait for 2 seconds for the server to fully start time.sleep(2) - logger.info(f"start background ota-image server on {cfg.OTA_IMAGE_URL}") + logger.info(f"start background ota-image server on {test_cfg.OTA_IMAGE_URL}") yield finally: logger.info("shutdown background ota-image server") _server_p.kill() -@pytest.fixture(scope="session") +@pytest.fixture(scope="class") def ab_slots(tmp_path_factory: pytest.TempPathFactory) -> SlotMeta: """Prepare AB slots for the whole test session. @@ -129,16 +131,17 @@ def ab_slots(tmp_path_factory: pytest.TempPathFactory) -> SlotMeta: Return: A tuple includes the path to A/B slots respectly. """ - # prepare slot_a + # + # ------ prepare slot_a ------ # + # slot_a = tmp_path_factory.mktemp("slot_a") shutil.copytree( - Path(cfg.OTA_IMAGE_DIR) / "data", slot_a, dirs_exist_ok=True, symlinks=True + Path(test_cfg.OTA_IMAGE_DIR) / "data", slot_a, dirs_exist_ok=True, symlinks=True ) - # simulate the diff between versions + # simulate the diff between local running image and target OTA image shutil.move(str(slot_a / "var"), slot_a / "var_old") shutil.move(str(slot_a / "usr"), slot_a / "usr_old") - # boot dir is a separated folder, so delete the boot folder under slot_a - # shutil.rmtree(slot_a / "boot", ignore_errors=True) + # manually create symlink to kernel and initrd.img vmlinuz_symlink = slot_a / "boot" / TestConfiguration.KERNEL_PREFIX vmlinuz_symlink.symlink_to( @@ -149,19 +152,17 @@ def ab_slots(tmp_path_factory: pytest.TempPathFactory) -> SlotMeta: f"{TestConfiguration.INITRD_PREFIX}-{TestConfiguration.KERNEL_VERSION}" ) - # prepare slot_b + # + # ------ prepare slot_b ------ # + # slot_b = tmp_path_factory.mktemp("slot_b") - # boot dev + # + # ------ prepare separated boot dev ------ # + # slot_a_boot_dev = tmp_path_factory.mktemp("slot_a_boot") - slot_a_boot_dir = slot_a_boot_dev / "boot" - slot_a_boot_dir.mkdir() - shutil.copytree( - Path(cfg.OTA_IMAGE_DIR) / "data/boot", slot_a_boot_dir, dirs_exist_ok=True - ) slot_b_boot_dev = tmp_path_factory.mktemp("slot_b_boot") - slot_b_boot_dir = slot_b_boot_dev / "boot" - slot_b_boot_dir.mkdir() + return SlotMeta( slot_a=str(slot_a), slot_b=str(slot_b), diff --git a/tests/test_boot_control/test_cboot.py b/tests/test_boot_control/test_cboot.py index f0d7d8bc3..decf67a0c 100644 --- a/tests/test_boot_control/test_cboot.py +++ b/tests/test_boot_control/test_cboot.py @@ -11,8 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - - r""" CBOOT switch boot mechanism follow(normal successful case): * Assume we are at slot-0, and apply an OTA update @@ -42,25 +40,31 @@ current slot: 1, ota_status=SUCCESS, slot_in_use=1 standby slot: 0 """ -from functools import partial -from pathlib import Path + + +import logging import shutil import typing import pytest import pytest_mock +from functools import partial +from pathlib import Path + +from otaclient._utils.path import replace_root +from otaclient.app.proto import wrapper +from otaclient.configs.app_cfg import Config as otaclient_Config -import logging from tests.utils import SlotMeta, compare_dir -from tests.conftest import TestConfiguration as cfg +from tests.conftest import TestConfiguration as test_cfg logger = logging.getLogger(__name__) class CbootFSM: def __init__(self) -> None: - self.current_slot = cfg.SLOT_A_ID_CBOOT - self.standby_slot = cfg.SLOT_B_ID_CBOOT + self.current_slot = test_cfg.SLOT_A_ID_CBOOT + self.standby_slot = test_cfg.SLOT_B_ID_CBOOT self.current_slot_bootable = True self.standby_slot_bootable = True @@ -73,10 +77,10 @@ def get_standby_slot(self): return self.standby_slot def get_standby_partuuid_str(self): - if self.standby_slot == cfg.SLOT_B_ID_CBOOT: - return f"PARTUUID={cfg.SLOT_B_PARTUUID}" + if self.standby_slot == test_cfg.SLOT_B_ID_CBOOT: + return f"PARTUUID={test_cfg.SLOT_B_PARTUUID}" else: - return f"PARTUUID={cfg.SLOT_A_PARTUUID}" + return f"PARTUUID={test_cfg.SLOT_A_PARTUUID}" def is_current_slot_bootable(self): return self.current_slot_bootable @@ -103,206 +107,275 @@ class TestCBootControl: EXTLNUX_CFG_SLOT_A = Path(__file__).parent / "extlinux.conf_slot_a" EXTLNUX_CFG_SLOT_B = Path(__file__).parent / "extlinux.conf_slot_b" - def cfg_for_slot_a_as_current(self): - """ - NOTE: we always only refer to ota-status dir at the rootfs! - """ - from otaclient.app.boot_control.configs import CBootControlConfig - - _mocked_cboot_cfg = CBootControlConfig() - _mocked_cboot_cfg.MOUNT_POINT = str(self.slot_b) # type: ignore - _mocked_cboot_cfg.ACTIVE_ROOT_MOUNT_POINT = str(self.slot_a) # type: ignore - # NOTE: SEPARATE_BOOT_MOUNT_POINT is the root of the boot device! - _mocked_cboot_cfg.SEPARATE_BOOT_MOUNT_POINT = str(self.slot_b_boot_dev) - _mocked_cboot_cfg.ACTIVE_ROOTFS_PATH = str(self.slot_a) # type: ignore - return _mocked_cboot_cfg - - def cfg_for_slot_b_as_current(self): - from otaclient.app.boot_control.configs import CBootControlConfig - - _mocked_cboot_cfg = CBootControlConfig() - _mocked_cboot_cfg.MOUNT_POINT = str(self.slot_a) # type: ignore - _mocked_cboot_cfg.ACTIVE_ROOT_MOUNT_POINT = str(self.slot_b) # type: ignore - # NOTE: SEPARATE_BOOT_MOUNT_POINT is the root of the boot device! - _mocked_cboot_cfg.SEPARATE_BOOT_MOUNT_POINT = str(self.slot_a_boot_dev) - _mocked_cboot_cfg.ACTIVE_ROOTFS_PATH = str(self.slot_b) # type: ignore - return _mocked_cboot_cfg - @pytest.fixture def cboot_ab_slot(self, ab_slots: SlotMeta): """ TODO: not considering rootfs on internal storage now - boot folder structure for cboot: - boot_dir_{slot_a, slot_b}/ - ota-status/ - status - version - slot_in_use """ + self.slot_a = Path(ab_slots.slot_a) self.slot_b = Path(ab_slots.slot_b) self.slot_a_boot_dev = Path(ab_slots.slot_a_boot_dev) self.slot_b_boot_dev = Path(ab_slots.slot_b_boot_dev) - self.slot_a_uuid = cfg.SLOT_A_PARTUUID - self.slot_b_uuid = cfg.SLOT_B_PARTUUID + self.slot_a_uuid = test_cfg.SLOT_A_PARTUUID + self.slot_b_uuid = test_cfg.SLOT_B_PARTUUID + # # prepare ota_status dir for slot_a - self.slot_a_ota_status_dir = self.slot_a / Path(cfg.OTA_STATUS_DIR).relative_to( - "/" + # + self.slot_a_ota_status_dir = Path( + replace_root(test_cfg.OTA_STATUS_DIR, "/", str(self.slot_a)) ) self.slot_a_ota_status_dir.mkdir(parents=True) + slot_a_ota_status = self.slot_a_ota_status_dir / "status" - slot_a_ota_status.write_text("SUCCESS") + slot_a_ota_status.write_text(wrapper.StatusOta.SUCCESS.name) + slot_a_version = self.slot_a_ota_status_dir / "version" - slot_a_version.write_text(cfg.CURRENT_VERSION) + slot_a_version.write_text(test_cfg.CURRENT_VERSION) + slot_a_slot_in_use = self.slot_a_ota_status_dir / "slot_in_use" - slot_a_slot_in_use.write_text(cfg.SLOT_A_ID_CBOOT) - # also prepare a copy of boot folder to rootfs - shutil.copytree( - self.slot_a_boot_dev / Path(cfg.BOOT_DIR).relative_to("/"), - self.slot_a / Path(cfg.BOOT_DIR).relative_to("/"), - dirs_exist_ok=True, - ) + slot_a_slot_in_use.write_text(test_cfg.SLOT_A_ID_CBOOT) - # prepare extlinux file + # + # prepare extlinux file for slot_a + # extlinux_dir = self.slot_a / "boot/extlinux" - extlinux_dir.mkdir() + extlinux_dir.mkdir(parents=True) extlinux_cfg = extlinux_dir / "extlinux.conf" extlinux_cfg.write_text(self.EXTLNUX_CFG_SLOT_A.read_text()) + # copy the /boot folder from slot_a to slot_a_boot_dev + shutil.copytree(self.slot_a / "boot", self.slot_a_boot_dev, dirs_exist_ok=True) + + # # ota_status dir for slot_b(separate boot dev) - self.slot_b_ota_status_dir = self.slot_b / Path(cfg.OTA_STATUS_DIR).relative_to( - "/" + # + self.slot_b_ota_status_dir = Path( + replace_root(test_cfg.OTA_STATUS_DIR, "/", str(self.slot_b)) ) @pytest.fixture - def mock_setup( - self, - mocker: pytest_mock.MockerFixture, - cboot_ab_slot, - ): + def mock_setup(self, mocker: pytest_mock.MockerFixture, cboot_ab_slot): from otaclient.app.boot_control._cboot import _CBootControl from otaclient.app.boot_control._common import CMDHelperFuncs ###### start fsm ###### self._fsm = CbootFSM() - ###### mocking _CBootControl ###### - _CBootControl_mock = typing.cast( + # init config + + # config for slot_a as active rootfs + self.mocked_cfg_slot_a = otaclient_Config(ACTIVE_ROOTFS=str(self.slot_a)) + mocker.patch( + f"{test_cfg.BOOT_CONTROL_CONFIG_MODULE_PATH}.cfg", self.mocked_cfg_slot_a + ) + mocker.patch(f"{test_cfg.CBOOT_MODULE_PATH}.cfg", self.mocked_cfg_slot_a) + # NOTE: remember to also patch boot.common module + mocker.patch( + f"{test_cfg.BOOT_CONTROL_COMMON_MODULE_PATH}.cfg", self.mocked_cfg_slot_a + ) + + # config for slot_b as active rootfs + self.mocked_cfg_slot_b = otaclient_Config(ACTIVE_ROOTFS=str(self.slot_b)) + + # mocking _CBootControl + + __cbootcontrol_mock: _CBootControl = typing.cast( _CBootControl, mocker.MagicMock(spec=_CBootControl) ) # mock methods - _CBootControl_mock.get_current_slot = mocker.MagicMock( + __cbootcontrol_mock.get_current_slot = mocker.MagicMock( wraps=self._fsm.get_current_slot ) - _CBootControl_mock.get_standby_slot = mocker.MagicMock( + __cbootcontrol_mock.get_standby_slot = mocker.MagicMock( wraps=self._fsm.get_standby_slot ) - _CBootControl_mock.get_standby_rootfs_partuuid_str = mocker.MagicMock( + __cbootcontrol_mock.get_standby_rootfs_partuuid_str = mocker.MagicMock( wraps=self._fsm.get_standby_partuuid_str ) - _CBootControl_mock.mark_current_slot_boot_successful.side_effect = partial( + __cbootcontrol_mock.mark_current_slot_boot_successful.side_effect = partial( self._fsm.mark_current_slot_as, True ) - _CBootControl_mock.set_standby_slot_unbootable.side_effect = partial( + __cbootcontrol_mock.set_standby_slot_unbootable.side_effect = partial( self._fsm.mark_standby_slot_as, False ) - _CBootControl_mock.switch_boot.side_effect = self._fsm.switch_boot - _CBootControl_mock.is_current_slot_marked_successful = mocker.MagicMock( + __cbootcontrol_mock.switch_boot.side_effect = self._fsm.switch_boot + __cbootcontrol_mock.is_current_slot_marked_successful = mocker.MagicMock( wraps=self._fsm.is_current_slot_bootable ) # NOTE: we only test external rootfs - _CBootControl_mock.is_external_rootfs_enabled.return_value = True + __cbootcontrol_mock.is_external_rootfs_enabled.return_value = True # make update_extlinux_cfg as it - _CBootControl_mock.update_extlinux_cfg = mocker.MagicMock( + __cbootcontrol_mock.update_extlinux_cfg = mocker.MagicMock( wraps=_CBootControl.update_extlinux_cfg ) ###### mocking _CMDHelper ###### - _CMDHelper_mock = typing.cast( + # mock all helper methods in CMDHelper + + _cmdhelper_mock = typing.cast( CMDHelperFuncs, mocker.MagicMock(spec=CMDHelperFuncs) ) - ###### patching ###### # patch _CBootControl - _CBootControl_path = f"{cfg.CBOOT_MODULE_PATH}._CBootControl" - mocker.patch(_CBootControl_path, return_value=_CBootControl_mock) + + mocker.patch( + f"{test_cfg.CBOOT_MODULE_PATH}._CBootControl", + return_value=__cbootcontrol_mock, + ) + # patch CMDHelperFuncs + # NOTE: also remember to patch CMDHelperFuncs in common - mocker.patch(f"{cfg.CBOOT_MODULE_PATH}.CMDHelperFuncs", _CMDHelper_mock) + mocker.patch(f"{test_cfg.CBOOT_MODULE_PATH}.CMDHelperFuncs", _cmdhelper_mock) + mocker.patch( + f"{test_cfg.BOOT_CONTROL_COMMON_MODULE_PATH}.CMDHelperFuncs", + _cmdhelper_mock, + ) + mocker.patch(f"{test_cfg.CBOOT_MODULE_PATH}.Nvbootctrl") + + # patch CBootControl mounting + + def _mount_standby_slot(*args, **kwargs): + # remove the old already exists folder + Path(self.mocked_cfg_slot_a.STANDBY_SLOT_MP).rmdir() + # simulate mount slot_b into otaclient mount space on slot_a + Path(self.mocked_cfg_slot_a.STANDBY_SLOT_MP).symlink_to(self.slot_b) + + def _mount_active_slot(*args, **kwargs): + # remove the old already exists folder + Path(self.mocked_cfg_slot_a.ACTIVE_SLOT_MP).rmdir() + # simlulate mount slot_b into otaclient mount space on slot_a + Path(self.mocked_cfg_slot_a.ACTIVE_SLOT_MP).symlink_to(self.slot_a) + + mocker.patch( + f"{test_cfg.CBOOT_MODULE_PATH}.CBootController._prepare_and_mount_standby", + mocker.MagicMock(side_effect=_mount_standby_slot), + ) mocker.patch( - f"{cfg.BOOT_CONTROL_COMMON_MODULE_PATH}.CMDHelperFuncs", _CMDHelper_mock + f"{test_cfg.CBOOT_MODULE_PATH}.CBootController._mount_refroot", + mocker.MagicMock(side_effect=_mount_active_slot), ) - mocker.patch(f"{cfg.CBOOT_MODULE_PATH}.Nvbootctrl") ###### binding mocked object to test instance ###### - self._CBootControl_mock = _CBootControl_mock - self._CMDHelper_mock = _CMDHelper_mock + self.__cbootcontrol_mock = __cbootcontrol_mock + self._cmdhelper_mock = _cmdhelper_mock def test_cboot_normal_update(self, mocker: pytest_mock.MockerFixture, mock_setup): from otaclient.app.boot_control._cboot import CBootController - - _cfg_patch_path = f"{cfg.CBOOT_MODULE_PATH}.cfg" - _relative_ota_status_path = Path(cfg.OTA_STATUS_DIR).relative_to("/") + from otaclient.app.boot_control.configs import CBootControlConfig, cboot_cfg ###### stage 1 ###### - mocker.patch(_cfg_patch_path, self.cfg_for_slot_a_as_current()) - logger.info("init cboot controller...") + active_ota_status_dpath = Path(cboot_cfg.ACTIVE_BOOT_OTA_STATUS_DPATH) + standby_ota_status_dpath = Path(cboot_cfg.STANDBY_BOOT_OTA_STATUS_DPATH) + + logger.info("[TESTING] init cboot controller...") cboot_controller = CBootController() assert ( - self.slot_a / _relative_ota_status_path / "status" - ).read_text() == "SUCCESS" + active_ota_status_dpath / otaclient_Config.OTA_STATUS_FNAME + ).read_text() == wrapper.StatusOta.SUCCESS.name - # test pre-update + # test cboot pre-update cboot_controller.pre_update( - version=cfg.UPDATE_VERSION, + version=test_cfg.UPDATE_VERSION, standby_as_ref=False, # NOTE: not used erase_standby=False, # NOTE: not used ) # assert current slot ota-status assert ( - self.slot_a / _relative_ota_status_path / "status" - ).read_text() == "FAILURE" + active_ota_status_dpath / otaclient_Config.OTA_STATUS_FNAME + ).read_text() == wrapper.StatusOta.FAILURE.name + assert ( - self.slot_a / "boot/ota-status/slot_in_use" + active_ota_status_dpath / otaclient_Config.SLOT_IN_USE_FNAME ).read_text() == self._fsm.get_standby_slot() + # assert standby slot ota-status + # NOTE: after pre_update phase, standby slot should be "mounted" assert ( - self.slot_b / _relative_ota_status_path / "status" - ).read_text() == "UPDATING" + standby_ota_status_dpath / otaclient_Config.OTA_STATUS_FNAME + ).read_text() == wrapper.StatusOta.UPDATING.name + assert ( - self.slot_b / _relative_ota_status_path / "version" - ).read_text() == cfg.UPDATE_VERSION + standby_ota_status_dpath / otaclient_Config.OTA_VERSION_FNAME + ).read_text() == test_cfg.UPDATE_VERSION + assert ( - self.slot_b / _relative_ota_status_path / "slot_in_use" + standby_ota_status_dpath / otaclient_Config.SLOT_IN_USE_FNAME ).read_text() == self._fsm.get_standby_slot() - logger.info("pre-update completed, entering post-update...") + logger.info("[TESTING] pre-update completed, entering post-update...") # NOTE: standby slot's extlinux file is not yet populated(done by create_standby) # prepare it by ourself # NOTE 2: populate to standby rootfs' boot folder - standby_extlinux_dir = self.slot_b / "boot/extlinux" - standby_extlinux_dir.mkdir() - standby_extlinux_file = standby_extlinux_dir / "extlinux.conf" - standby_extlinux_file.write_text(self.EXTLNUX_CFG_SLOT_A.read_text()) + standby_slot_extlinux_dir = Path(cboot_cfg.STANDBY_BOOT_EXTLINUX_DPATH) + standby_slot_extlinux_dir.mkdir(exist_ok=True, parents=True) + Path(cboot_cfg.STANDBY_EXTLINUX_FPATH).write_text( + self.EXTLNUX_CFG_SLOT_A.read_text() + ) + + # Prepare the symlink from standby slot boot dev mountpoint to standby slot + # boot dev to simulate mounting. + # This is for post_update copies boot folders from standby slot to standby boot dev. + shutil.rmtree(cboot_cfg.SEPARATE_BOOT_MOUNT_POINT, ignore_errors=True) + Path(cboot_cfg.SEPARATE_BOOT_MOUNT_POINT).symlink_to(self.slot_b_boot_dev) - # test post-update + # test cboot post-update _post_updater = cboot_controller.post_update() next(_post_updater) next(_post_updater, None) + + # + # assertion + # + # confirm the extlinux.conf is properly updated in standby slot + assert ( + Path(cboot_cfg.STANDBY_EXTLINUX_FPATH).read_text() + == self.EXTLNUX_CFG_SLOT_B.read_text() + ) + + # confirm extlinux.cfg for standby slot is properly copied + # to standby slot's boot dev assert ( - self.slot_b_boot_dev / "boot/extlinux/extlinux.conf" - ).read_text() == self.EXTLNUX_CFG_SLOT_B.read_text() - self._CBootControl_mock.switch_boot.assert_called_once() - self._CMDHelper_mock.reboot.assert_called_once() + Path( + replace_root( + cboot_cfg.STANDBY_EXTLINUX_FPATH, + self.mocked_cfg_slot_a.STANDBY_SLOT_MP, + str(self.slot_b_boot_dev), + ) + ).read_text() + == self.EXTLNUX_CFG_SLOT_B.read_text() + ) + + self.__cbootcontrol_mock.switch_boot.assert_called_once() + self._cmdhelper_mock.reboot.assert_called_once() + # assert separate bootdev is populated correctly - compare_dir(self.slot_b / "boot", self.slot_b_boot_dev / "boot") + compare_dir( + self.slot_b / "boot", + self.slot_b_boot_dev / "boot", + ) ###### stage 2 ###### - logger.info("post-update completed, test init after first reboot...") - mocker.patch(_cfg_patch_path, self.cfg_for_slot_b_as_current()) - cboot_controller = CBootController() + + logger.info("[TESTING] cboot init on first reboot...") + + # slot_b as active slot + mocker.patch( + f"{test_cfg.BOOT_CONTROL_CONFIG_MODULE_PATH}.cfg", self.mocked_cfg_slot_b + ) + mocker.patch(f"{test_cfg.CBOOT_MODULE_PATH}.cfg", self.mocked_cfg_slot_b) + + # NOTE: old cboot_cfg's properties are cached, so create a new one + _recreated_cboot_cfg = CBootControlConfig() + mocker.patch(f"{test_cfg.CBOOT_MODULE_PATH}.boot_cfg", _recreated_cboot_cfg) + + # init cboot control again + CBootController() assert ( - self.slot_b / _relative_ota_status_path / "status" - ).read_text() == "SUCCESS" + Path(_recreated_cboot_cfg.ACTIVE_BOOT_OTA_STATUS_DPATH) + / otaclient_Config.OTA_STATUS_FNAME + ).read_text() == wrapper.StatusOta.SUCCESS.name + assert self._fsm.is_boot_switched diff --git a/tests/test_boot_control/test_grub.py b/tests/test_boot_control/test_grub.py index 180a5d6a1..bbd3cbb00 100644 --- a/tests/test_boot_control/test_grub.py +++ b/tests/test_boot_control/test_grub.py @@ -21,22 +21,25 @@ import pytest_mock from pathlib import Path +from otaclient._utils.path import replace_root +from otaclient.app.boot_control.configs import GrubControlConfig +from otaclient.configs.app_cfg import Config as otaclient_Config from otaclient.app.proto import wrapper from tests.utils import SlotMeta -from tests.conftest import TestConfiguration as cfg +from tests.conftest import TestConfiguration as test_cfg logger = logging.getLogger(__name__) class GrubFSM: def __init__(self, slot_a_mp, slot_b_mp) -> None: - self._current_slot = cfg.SLOT_A_ID_GRUB - self._standby_slot = cfg.SLOT_B_ID_GRUB + self._current_slot = test_cfg.SLOT_A_ID_GRUB + self._standby_slot = test_cfg.SLOT_B_ID_GRUB self._current_slot_mp = Path(slot_a_mp) self._standby_slot_mp = Path(slot_b_mp) - self._current_slot_dev_uuid = f"UUID={cfg.SLOT_A_UUID}" - self._standby_slot_dev_uuid = f"UUID={cfg.SLOT_B_UUID}" + self._current_slot_dev_uuid = f"UUID={test_cfg.SLOT_A_UUID}" + self._standby_slot_dev_uuid = f"UUID={test_cfg.SLOT_B_UUID}" self.current_slot_bootable = True self.standby_slot_bootable = True @@ -82,10 +85,10 @@ def switch_boot(self): self.is_boot_switched = True def cat_proc_cmdline(self): - if self._current_slot == cfg.SLOT_A_ID_GRUB: - return cfg.CMDLINE_SLOT_A + if self._current_slot == test_cfg.SLOT_A_ID_GRUB: + return test_cfg.CMDLINE_SLOT_A else: - return cfg.CMDLINE_SLOT_B + return test_cfg.CMDLINE_SLOT_B class GrubMkConfigFSM: @@ -132,36 +135,8 @@ class TestGrubControl: FSTAB_UPDATED = (Path(__file__).parent / "fstab_updated").read_text() DEFAULT_GRUB = (Path(__file__).parent / "default_grub").read_text() - def cfg_for_slot_a_as_current(self): - from otaclient.app.boot_control.configs import GrubControlConfig - - _mocked_grub_cfg = GrubControlConfig() - _mocked_grub_cfg.MOUNT_POINT = str(self.slot_b) # type: ignore - _mocked_grub_cfg.ACTIVE_ROOTFS_PATH = str(self.slot_a) # type: ignore - _mocked_grub_cfg.BOOT_DIR = str( # type: ignore - self.boot_dir - ) # unified boot dir - _mocked_grub_cfg.GRUB_DIR = str(self.boot_dir / "grub") - _mocked_grub_cfg.GRUB_CFG_PATH = str(self.boot_dir / "grub/grub.cfg") - - return _mocked_grub_cfg - - def cfg_for_slot_b_as_current(self): - from otaclient.app.boot_control.configs import GrubControlConfig - - _mocked_grub_cfg = GrubControlConfig() - _mocked_grub_cfg.MOUNT_POINT = str(self.slot_a) # type: ignore - _mocked_grub_cfg.ACTIVE_ROOTFS_PATH = str(self.slot_b) # type: ignore - _mocked_grub_cfg.BOOT_DIR = str( # type: ignore - self.boot_dir - ) # unified boot dir - _mocked_grub_cfg.GRUB_DIR = str(self.boot_dir / "grub") - _mocked_grub_cfg.GRUB_CFG_PATH = str(self.boot_dir / "grub/grub.cfg") - - return _mocked_grub_cfg - @pytest.fixture - def grub_ab_slot(self, tmp_path: Path, ab_slots: SlotMeta): + def grub_ab_slot(self, ab_slots: SlotMeta): """ NOTE: this test simulating init and updating from a non-ota-partition enabled system NOTE: boot dirs for grub are located under boot folder @@ -174,44 +149,46 @@ def grub_ab_slot(self, tmp_path: Path, ab_slots: SlotMeta): """ self._grub_mkconfig_fsm = GrubMkConfigFSM() + # + # ------ init otaclient configs ------ # + # + self.mocked_otaclient_cfg_slot_a = otaclient_Config( + ACTIVE_ROOTFS=ab_slots.slot_a + ) + self.mocked_otaclient_cfg_slot_b = otaclient_Config( + ACTIVE_ROOTFS=ab_slots.slot_b + ) + + # + # ----- setup slots ------ # + # self.slot_a = Path(ab_slots.slot_a) self.slot_b = Path(ab_slots.slot_b) - self.boot_dir = tmp_path / Path(cfg.BOOT_DIR).relative_to("/") - self.slot_b_boot_dir = self.slot_b / "boot" - self.slot_b_boot_dir.mkdir(parents=True, exist_ok=True) + # NOTE: grub uses share boot device, here we use slot_a_boot as shared boot + self.shared_boot_dir = Path(ab_slots.slot_a_boot_dev) self.slot_a_ota_partition_dir = ( - self.boot_dir / f"{cfg.OTA_PARTITION_DIRNAME}.{cfg.SLOT_A_ID_GRUB}" + self.shared_boot_dir + / f"{test_cfg.OTA_PARTITION_DIRNAME}.{test_cfg.SLOT_A_ID_GRUB}" ) self.slot_b_ota_partition_dir = ( - self.boot_dir / f"{cfg.OTA_PARTITION_DIRNAME}.{cfg.SLOT_B_ID_GRUB}" + self.shared_boot_dir + / f"{test_cfg.OTA_PARTITION_DIRNAME}.{test_cfg.SLOT_B_ID_GRUB}" ) - # copy the contents from pre-populated boot_dir to test /boot folder + + # copy the contents from slot_a's pre-populated boot_dir to test /boot folder # NOTE: check kernel version from the ota-test_base image Dockerfile shutil.copytree( - Path(ab_slots.slot_a_boot_dev) / Path(cfg.BOOT_DIR).relative_to("/"), - self.boot_dir, + self.slot_a / "boot", + self.shared_boot_dir, dirs_exist_ok=True, ) + shutil.rmtree(self.slot_a / "boot", ignore_errors=True) + # simulate boot device mounted on slot_a + (self.slot_a / "boot").symlink_to(self.shared_boot_dir) - # NOTE: dummy ota-image doesn't have grub installed, - # so we need to prepare /etc/default/grub by ourself - default_grub = self.slot_a / Path(cfg.DEFAULT_GRUB_FILE).relative_to("/") - default_grub.write_text((Path(__file__).parent / "default_grub").read_text()) - - # prepare fstab file - slot_a_fstab_file = self.slot_a / Path(cfg.FSTAB_FILE).relative_to("/") - slot_a_fstab_file.write_text(self.FSTAB_ORIGIN) - slot_b_fstab_file = self.slot_b / Path(cfg.FSTAB_FILE).relative_to("/") - slot_b_fstab_file.parent.mkdir(parents=True, exist_ok=True) - slot_b_fstab_file.write_text(self.FSTAB_ORIGIN) - - # prepare grub file for slot_a - init_grub_file = self.boot_dir / Path(cfg.GRUB_FILE).relative_to("/boot") - init_grub_file.parent.mkdir(parents=True, exist_ok=True) - init_grub_file.write_text( - self._grub_mkconfig_fsm.GRUB_CFG_SLOT_A_NON_OTAPARTITION - ) + # create slot_b's boot folder + (self.slot_b / "boot").mkdir(exist_ok=True, parents=True) @pytest.fixture(autouse=True) def mock_setup( @@ -225,8 +202,30 @@ def mock_setup( # ------ start fsm ------ # self._fsm = GrubFSM(slot_a_mp=self.slot_a, slot_b_mp=self.slot_b) + # + # ------ apply otaclient cfg patch ------ # + # + mocker.patch( + f"{test_cfg.BOOT_CONTROL_CONFIG_MODULE_PATH}.cfg", + self.mocked_otaclient_cfg_slot_a, + ) + mocker.patch( + f"{test_cfg.GRUB_MODULE_PATH}.cfg", self.mocked_otaclient_cfg_slot_a + ) + # NOTE: remember to also patch otaclient cfg in boot.common module + mocker.patch( + f"{test_cfg.BOOT_CONTROL_COMMON_MODULE_PATH}.cfg", + self.mocked_otaclient_cfg_slot_a, + ) + + # after the mocked otaclient cfg is applied, we can init rpi_boot cfg instance + self.mocked_boot_cfg_slot_a = GrubControlConfig() + + # # ------ mock SlotMountHelper ------ # + # _mocked_slot_mount_helper = mocker.MagicMock(spec=SlotMountHelper) + type(_mocked_slot_mount_helper).standby_slot_dev = mocker.PropertyMock( wraps=self._fsm.get_standby_slot_dev ) @@ -243,8 +242,16 @@ def mock_setup( wraps=self._fsm.get_standby_boot_dir ) - # ------ mock GrubABPartitionDetector ------ # + mocker.patch( + f"{test_cfg.GRUB_MODULE_PATH}.SlotMountHelper", + return_value=_mocked_slot_mount_helper, + ) + + # + # ------ patching GrubABPartitionDetector ------ # + # _mocked_ab_partition_detector = mocker.MagicMock(spec=GrubABPartitionDetector) + type(_mocked_ab_partition_detector).active_slot = mocker.PropertyMock( wraps=self._fsm.get_active_slot ) @@ -258,153 +265,218 @@ def mock_setup( wraps=self._fsm.get_standby_slot_dev ) - ###### mocking GrubHelper ###### + mocker.patch( + f"{test_cfg.GRUB_MODULE_PATH}.GrubABPartitionDetector", + return_value=_mocked_ab_partition_detector, + ) + + # + # ------ patching GrubHelper ------ # + # _grub_reboot_mock = mocker.MagicMock() mocker.patch( - f"{cfg.GRUB_MODULE_PATH}.GrubHelper.grub_reboot", _grub_reboot_mock + f"{test_cfg.GRUB_MODULE_PATH}.GrubHelper.grub_reboot", _grub_reboot_mock ) # bind to test instance self._grub_reboot_mock = _grub_reboot_mock - ###### mocking CMDHelperFuncs ###### - _CMDHelper_mock = typing.cast( + # + # ------ patching CMDHelperFuncs ------ # + # + _cmdhelper_mock = typing.cast( CMDHelperFuncs, mocker.MagicMock(spec=CMDHelperFuncs) ) - _CMDHelper_mock.reboot.side_effect = self._fsm.switch_boot - _CMDHelper_mock.get_uuid_str_by_dev = mocker.MagicMock( + + _cmdhelper_mock.reboot.side_effect = self._fsm.switch_boot + _cmdhelper_mock.get_uuid_str_by_dev = mocker.MagicMock( wraps=self._fsm.get_uuid_str_by_dev ) # bind the mocker to the test instance - self._CMDHelper_mock = _CMDHelper_mock + self._cmdhelper_mock = _cmdhelper_mock - ###### mock GrubHelper ###### - _grub_mkconfig_path = f"{cfg.GRUB_MODULE_PATH}.GrubHelper.grub_mkconfig" + # NOTE: also remember to patch CMDHelperFuncs in boot.common mocker.patch( - _grub_mkconfig_path, - wraps=self._grub_mkconfig_fsm.grub_mkconfig, + f"{test_cfg.BOOT_CONTROL_COMMON_MODULE_PATH}.CMDHelperFuncs", + _cmdhelper_mock, ) + mocker.patch(f"{test_cfg.GRUB_MODULE_PATH}.CMDHelperFuncs", _cmdhelper_mock) - ###### patching ###### - # patch CMDHelper - # NOTE: also remember to patch CMDHelperFuncs in common - _CMDHelper_at_common_path = ( - f"{cfg.BOOT_CONTROL_COMMON_MODULE_PATH}.CMDHelperFuncs" - ) - _CMDHelper_at_grub_path = f"{cfg.GRUB_MODULE_PATH}.CMDHelperFuncs" - mocker.patch(_CMDHelper_at_common_path, _CMDHelper_mock) - mocker.patch(_CMDHelper_at_grub_path, _CMDHelper_mock) - # patch _GrubABPartitionDetector - _GrubABPartitionDetector_path = ( - f"{cfg.GRUB_MODULE_PATH}.GrubABPartitionDetector" - ) + # + # ------ patching GrubHelper ------ # + # + _grub_mkconfig_path = f"{test_cfg.GRUB_MODULE_PATH}.GrubHelper.grub_mkconfig" mocker.patch( - _GrubABPartitionDetector_path, return_value=_mocked_ab_partition_detector + _grub_mkconfig_path, + wraps=self._grub_mkconfig_fsm.grub_mkconfig, ) - # patch SlotMountHelper - _SlotMountHelper_path = f"{cfg.GRUB_MODULE_PATH}.SlotMountHelper" - mocker.patch(_SlotMountHelper_path, return_value=_mocked_slot_mount_helper) + # patch reading from /proc/cmdline mocker.patch( - f"{cfg.GRUB_MODULE_PATH}.cat_proc_cmdline", + f"{test_cfg.GRUB_MODULE_PATH}.cat_proc_cmdline", mocker.MagicMock(wraps=self._fsm.cat_proc_cmdline), ) + @pytest.fixture(autouse=True) + def setup_test(self, mock_setup): + # NOTE: dummy ota-image doesn't have grub installed, + # so we need to prepare /etc/default/grub by ourself + default_grub = Path(self.mocked_boot_cfg_slot_a.GRUB_DEFAULT_FPATH) + default_grub.write_text((Path(__file__).parent / "default_grub").read_text()) + + # prepare fstab file + slot_a_fstab_file = Path(self.mocked_otaclient_cfg_slot_a.FSTAB_FPATH) + slot_a_fstab_file.write_text(self.FSTAB_ORIGIN) + + slot_b_fstab_file = Path( + replace_root(slot_a_fstab_file, self.slot_a, self.slot_b) + ) + slot_b_fstab_file.parent.mkdir(parents=True, exist_ok=True) + slot_b_fstab_file.write_text(self.FSTAB_ORIGIN) + + # prepare grub file for slot_a + init_grub_file = Path(self.mocked_boot_cfg_slot_a.GRUB_CFG_FPATH) + init_grub_file.parent.mkdir(parents=True, exist_ok=True) + init_grub_file.write_text( + self._grub_mkconfig_fsm.GRUB_CFG_SLOT_A_NON_OTAPARTITION + ) + + # + # ------ setup mount space ------ # + # + # NOTE: as we mock CMDHelpers, mount is not executed, so we prepare the mount points + # by ourselves.(In the future we can use FSM to do it.) + Path(self.mocked_otaclient_cfg_slot_a.OTACLIENT_MOUNT_SPACE_DPATH).mkdir( + parents=True, exist_ok=True + ) + Path(self.mocked_otaclient_cfg_slot_a.ACTIVE_SLOT_MP).symlink_to(self.slot_a) + Path(self.mocked_otaclient_cfg_slot_a.STANDBY_SLOT_MP).symlink_to(self.slot_b) + def test_grub_normal_update(self, mocker: pytest_mock.MockerFixture): + from otaclient.app.boot_control.configs import GrubControlConfig from otaclient.app.boot_control._grub import GrubController - _cfg_patch_path = f"{cfg.GRUB_MODULE_PATH}.cfg" - - ###### stage 1 ###### + # + # ------ stage 1 ------ # + # # test init from non-ota-partition enabled system - # mock cfg - mocker.patch(_cfg_patch_path, self.cfg_for_slot_a_as_current()) - grub_controller = GrubController() assert ( - self.slot_a_ota_partition_dir / "status" + self.slot_a_ota_partition_dir / otaclient_Config.OTA_STATUS_FNAME ).read_text() == wrapper.StatusOta.INITIALIZED.name # assert ota-partition file points to slot_a ota-partition folder assert ( - os.readlink(self.boot_dir / cfg.OTA_PARTITION_DIRNAME) - == f"{cfg.OTA_PARTITION_DIRNAME}.{cfg.SLOT_A_ID_GRUB}" + os.readlink(self.shared_boot_dir / test_cfg.OTA_PARTITION_DIRNAME) + == f"{test_cfg.OTA_PARTITION_DIRNAME}.{test_cfg.SLOT_A_ID_GRUB}" ) assert ( - self.boot_dir / "grub/grub.cfg" - ).read_text() == GrubMkConfigFSM.GRUB_CFG_SLOT_A_UPDATED + Path(self.mocked_boot_cfg_slot_a.GRUB_CFG_FPATH).read_text() + == GrubMkConfigFSM.GRUB_CFG_SLOT_A_UPDATED + ) # test pre-update + logger.info("[TESTING] execute pre_update ...") grub_controller.pre_update( - version=cfg.UPDATE_VERSION, + version=test_cfg.UPDATE_VERSION, standby_as_ref=False, # NOTE: not used erase_standby=False, # NOTE: not used ) + # update slot_b, slot_a_ota_status->FAILURE, slot_b_ota_status->UPDATING assert ( - self.slot_a_ota_partition_dir / "status" + self.slot_a_ota_partition_dir / otaclient_Config.OTA_STATUS_FNAME ).read_text() == wrapper.StatusOta.FAILURE.name assert ( - self.slot_b_ota_partition_dir / "status" + self.slot_b_ota_partition_dir / otaclient_Config.OTA_STATUS_FNAME ).read_text() == wrapper.StatusOta.UPDATING.name + # NOTE: we have to copy the new kernel files to the slot_b's boot dir # this is done by the create_standby module - _kernel = f"{cfg.KERNEL_PREFIX}-{cfg.KERNEL_VERSION}" - _initrd = f"{cfg.INITRD_PREFIX}-{cfg.KERNEL_VERSION}" - shutil.copy(self.slot_a_ota_partition_dir / _kernel, self.slot_b_boot_dir) - shutil.copy(self.slot_a_ota_partition_dir / _initrd, self.slot_b_boot_dir) + _kernel = f"{test_cfg.KERNEL_PREFIX}-{test_cfg.KERNEL_VERSION}" + _initrd = f"{test_cfg.INITRD_PREFIX}-{test_cfg.KERNEL_VERSION}" + shutil.copy(self.slot_a_ota_partition_dir / _kernel, self.slot_b / "boot") + shutil.copy(self.slot_a_ota_partition_dir / _initrd, self.slot_b / "boot") - logger.info("pre-update completed, entering post-update...") + logger.info("[TESTING] pre-update completed, entering post-update...") # test post-update _post_updater = grub_controller.post_update() next(_post_updater) next(_post_updater, None) + + # ensure the standby slot's fstab file is updated with slot_b's UUID assert ( - self.slot_b / Path(cfg.FSTAB_FILE).relative_to("/") - ).read_text().strip() == self.FSTAB_UPDATED.strip() + Path(self.mocked_otaclient_cfg_slot_b.FSTAB_FPATH).read_text().strip() + == self.FSTAB_UPDATED.strip() + ) + # ensure /boot/grub/grub.cfg is updated assert ( - self.boot_dir / "grub/grub.cfg" - ).read_text().strip() == GrubMkConfigFSM.GRUB_CFG_SLOT_A_UPDATED.strip() + Path(self.mocked_boot_cfg_slot_a.GRUB_CFG_FPATH).read_text().strip() + == GrubMkConfigFSM.GRUB_CFG_SLOT_A_UPDATED.strip() + ) # NOTE: check grub.cfg_slot_a_post_update, the target entry is 0 self._grub_reboot_mock.assert_called_once_with(0) - self._CMDHelper_mock.reboot.assert_called_once() + self._cmdhelper_mock.reboot.assert_called_once() - ###### stage 2 ###### + # + # ------ stage 2 ------ # + # + # active slot: slot_b # test init after first reboot + # simulate boot dev mounted on slot_b + shutil.rmtree(self.slot_b / "boot", ignore_errors=True) + (self.slot_b / "boot").symlink_to(self.shared_boot_dir) + # NOTE: dummy ota-image doesn't have grub installed, # so we need to prepare /etc/default/grub by ourself - default_grub = self.slot_b / Path(cfg.DEFAULT_GRUB_FILE).relative_to("/") + default_grub = self.slot_b / Path(test_cfg.DEFAULT_GRUB_FILE).relative_to("/") default_grub.parent.mkdir(parents=True, exist_ok=True) default_grub.write_text(self.DEFAULT_GRUB) - logger.info("post-update completed, test init after first reboot...") - # mock cfg - mocker.patch(_cfg_patch_path, self.cfg_for_slot_b_as_current()) + logger.info("[TESTING] post-update completed, test init after first reboot...") + + # patch otaclient cfg for slot_b + mocker.patch( + f"{test_cfg.BOOT_CONTROL_CONFIG_MODULE_PATH}.cfg", + self.mocked_otaclient_cfg_slot_b, + ) + mocker.patch( + f"{test_cfg.GRUB_MODULE_PATH}.cfg", self.mocked_otaclient_cfg_slot_b + ) + # NOTE: remember to also patch otaclient cfg in boot.common module + mocker.patch( + f"{test_cfg.BOOT_CONTROL_COMMON_MODULE_PATH}.cfg", + self.mocked_otaclient_cfg_slot_b, + ) + + # NOTE: old grub boot_cfg's properties are cached, so create a new one + _recreated_grub_ctrl_cfg = GrubControlConfig() + mocker.patch(f"{test_cfg.GRUB_MODULE_PATH}.boot_cfg", _recreated_grub_ctrl_cfg) ### test pre-init ### assert self._fsm.is_boot_switched assert ( - self.slot_b_ota_partition_dir / "status" + self.slot_b_ota_partition_dir / otaclient_Config.OTA_STATUS_FNAME ).read_text() == wrapper.StatusOta.UPDATING.name # assert ota-partition file is not yet switched before first reboot init assert ( - os.readlink(self.boot_dir / cfg.OTA_PARTITION_DIRNAME) - == f"{cfg.OTA_PARTITION_DIRNAME}.{cfg.SLOT_A_ID_GRUB}" + os.readlink(self.shared_boot_dir / test_cfg.OTA_PARTITION_DIRNAME) + == f"{test_cfg.OTA_PARTITION_DIRNAME}.{test_cfg.SLOT_A_ID_GRUB}" ) ### test first reboot init ### _ = GrubController() # assert ota-partition file switch to slot_b ota-partition folder after first reboot init assert ( - os.readlink(self.boot_dir / cfg.OTA_PARTITION_DIRNAME) - == f"{cfg.OTA_PARTITION_DIRNAME}.{cfg.SLOT_B_ID_GRUB}" + os.readlink(self.shared_boot_dir / test_cfg.OTA_PARTITION_DIRNAME) + == f"{test_cfg.OTA_PARTITION_DIRNAME}.{test_cfg.SLOT_B_ID_GRUB}" ) assert ( - self.slot_b_ota_partition_dir / "status" + self.slot_b_ota_partition_dir / otaclient_Config.OTA_STATUS_FNAME ).read_text() == wrapper.StatusOta.SUCCESS.name assert ( - self.slot_b_ota_partition_dir / "version" - ).read_text() == cfg.UPDATE_VERSION + self.slot_b_ota_partition_dir / otaclient_Config.OTA_VERSION_FNAME + ).read_text() == test_cfg.UPDATE_VERSION @pytest.mark.parametrize( diff --git a/tests/test_boot_control/test_ota_status_control.py b/tests/test_boot_control/test_ota_status_control.py index dc914b6da..fd244cd87 100644 --- a/tests/test_boot_control/test_ota_status_control.py +++ b/tests/test_boot_control/test_ota_status_control.py @@ -18,11 +18,11 @@ import pytest from functools import partial from pathlib import Path -from typing import Optional, Union +from typing import Optional +from otaclient.app.configs import config as cfg from otaclient.app.common import read_str_from_file, write_str_to_file from otaclient.app.proto import wrapper -from otaclient.app.boot_control.configs import BaseConfig as cfg from otaclient.app.boot_control._common import OTAStatusFilesControl logger = logging.getLogger(__name__) diff --git a/tests/test_boot_control/test_rpi_boot.py b/tests/test_boot_control/test_rpi_boot.py index 6c3dd1e77..dbf95240e 100644 --- a/tests/test_boot_control/test_rpi_boot.py +++ b/tests/test_boot_control/test_rpi_boot.py @@ -7,9 +7,11 @@ from string import Template from tests.utils import SlotMeta -from tests.conftest import TestConfiguration as cfg +from tests.conftest import TestConfiguration as test_cfg +from otaclient._utils.path import replace_root +from otaclient.configs.app_cfg import Config as otaclient_Config from otaclient.app.boot_control._rpi_boot import _FSTAB_TEMPLATE_STR -from otaclient.app.boot_control.configs import rpi_boot_cfg +from otaclient.app.boot_control.configs import RPIBootControlConfig from otaclient.app.proto import wrapper import logging @@ -19,8 +21,8 @@ class _RPIBootTestCfg: # slot config - SLOT_A = rpi_boot_cfg.SLOT_A_FSLABEL - SLOT_B = rpi_boot_cfg.SLOT_B_FSLABEL + SLOT_A = RPIBootControlConfig.SLOT_A_FSLABEL + SLOT_B = RPIBootControlConfig.SLOT_B_FSLABEL SLOT_A_DEV = "slot_a_dev" SLOT_B_DEV = "slot_b_dev" SEP_CHAR = "_" @@ -32,12 +34,12 @@ class _RPIBootTestCfg: CMDLINE_TXT_SLOT_B = "cmdline_txt_slot_b" # module path - rpi_boot__RPIBootControl_MODULE = f"{cfg.RPI_BOOT_MODULE_PATH}._RPIBootControl" + rpiboot_control_module_path = f"{test_cfg.RPI_BOOT_MODULE_PATH}._RPIBootControl" rpi_boot_RPIBoot_CMDHelperFuncs_MODULE = ( - f"{cfg.RPI_BOOT_MODULE_PATH}.CMDHelperFuncs" + f"{test_cfg.RPI_BOOT_MODULE_PATH}.CMDHelperFuncs" ) boot_control_common_CMDHelperFuncs_MODULE = ( - f"{cfg.BOOT_CONTROL_COMMON_MODULE_PATH}.CMDHelperFuncs" + f"{test_cfg.BOOT_CONTROL_COMMON_MODULE_PATH}.CMDHelperFuncs" ) # image version @@ -86,126 +88,223 @@ class TestRPIBootControl: """ @pytest.fixture - def rpi_boot_ab_slot(self, tmp_path: Path, ab_slots: SlotMeta): - self.slot_a_mp = Path(ab_slots.slot_a) - self.slot_b_mp = Path(ab_slots.slot_b) - - # setup ota_status dir for slot_a - self.slot_a_ota_status_dir = self.slot_a_mp / Path( - rpi_boot_cfg.OTA_STATUS_DIR - ).relative_to("/") + def rpi_boot_ab_slot(self, ab_slots: SlotMeta): + self.slot_a_pa = Path(ab_slots.slot_a) + self.slot_b_pa = Path(ab_slots.slot_b) + + # + # ------ init otaclient configs ------ # + # + # create otaclient config with different active rootfs + self.mocked_otaclient_cfg_slot_a = otaclient_Config( + ACTIVE_ROOTFS=str(self.slot_a_pa) + ) + self.mocked_otaclient_cfg_slot_b = otaclient_Config( + ACTIVE_ROOTFS=str(self.slot_b_pa) + ) + + # + # ------ setup shared system-boot partition ------ # + # + # NOTE: rpi_boot uses shared system_boot partition, here we only use slot_a_boot_dev + # to simlulate such condition + self.system_boot_mp = Path(ab_slots.slot_a_boot_dev) + self.system_boot_mp.mkdir(parents=True, exist_ok=True) + + # + # ------ setup system-boot mount point ------ # + # + # simulate rpi device mounts system-boot partition to /boot/firmware mountpoint + slot_a_system_boot_mp = self.slot_a_pa / "boot" / "firmware" + slot_a_system_boot_mp.symlink_to(self.system_boot_mp, target_is_directory=True) + + (self.slot_b_pa / "boot").mkdir(parents=True, exist_ok=True) + slot_b_system_boot_mp = self.slot_b_pa / "boot" / "firmware" + slot_b_system_boot_mp.symlink_to(self.system_boot_mp, target_is_directory=True) + + # setup etc folder for slot_b + (self.slot_b_pa / "etc").mkdir(parents=True, exist_ok=True) + + @pytest.fixture(autouse=True) + def mock_setup(self, mocker: pytest_mock.MockerFixture, rpi_boot_ab_slot): + from otaclient.app.boot_control._rpi_boot import _RPIBootControl + from otaclient.app.boot_control._common import CMDHelperFuncs + + # + # ------ start the test FSM ------ # + # + self._fsm = RPIBootABPartitionFSM() + + # + # ------ rpi_boot configs patch applying ------ # + # + mocker.patch( + f"{test_cfg.BOOT_CONTROL_CONFIG_MODULE_PATH}.cfg", + self.mocked_otaclient_cfg_slot_a, + ) + mocker.patch( + f"{test_cfg.RPI_BOOT_MODULE_PATH}.cfg", self.mocked_otaclient_cfg_slot_a + ) + # NOTE: remember to also patch otaclient cfg in boot.common module + mocker.patch( + f"{test_cfg.BOOT_CONTROL_COMMON_MODULE_PATH}.cfg", + self.mocked_otaclient_cfg_slot_a, + ) + # after the mocked cfg is applied, we can init rpi_boot cfg instance + self.mocked_boot_cfg_slot_a = RPIBootControlConfig() + + # + # ------ prepare mocked _RPIBootControl ------ # + # + _mocked__rpi_boot_ctrl = typing.cast( + "type[_RPIBootControl]", + type("_mocked_RPIBootControl", (_RPIBootControl,), {}), + ) + + # bind slots related methods to test FSM + _mocked__rpi_boot_ctrl.standby_slot = mocker.PropertyMock( # type: ignore + wraps=self._fsm.get_standby_slot + ) + _mocked__rpi_boot_ctrl.active_slot = mocker.PropertyMock( # type: ignore + wraps=self._fsm.get_active_slot + ) + _mocked__rpi_boot_ctrl.active_slot_dev = mocker.PropertyMock( # type: ignore + wraps=self._fsm.get_active_slot_dev + ) + _mocked__rpi_boot_ctrl.standby_slot_dev = mocker.PropertyMock( # type: ignore + wraps=self._fsm.get_standby_slot_dev + ) + _mocked__rpi_boot_ctrl.reboot_tryboot = mocker.Mock( + side_effect=self._fsm.reboot_tryboot + ) + _mocked__rpi_boot_ctrl.system_boot_path = mocker.PropertyMock( # type: ignore + return_value=self.system_boot_mp + ) + + # hide away this method as slots detection is handled by test FSM + _mocked__rpi_boot_ctrl._init_slots_info = mocker.Mock() + + # mock firmware update method, only check if it is called + _mocked__rpi_boot_ctrl._update_firmware = mocker.Mock() + self.mocked__rpi_boot_ctrl_type = _mocked__rpi_boot_ctrl + + # + # ------ patch CMDHelperFuncs ------ # + # + self._mocked_cmdhelper = typing.cast( + CMDHelperFuncs, mocker.MagicMock(spec=CMDHelperFuncs) + ) + # NOTE: especially patch this method, as _RPIBootControl's __init__ uses this method + # to check if system-boot folder is a mount point. + self._mocked_cmdhelper.is_target_mounted = mocker.Mock(return_value=True) + + # NOTE: rpi_boot only call CMDHelperFuncs.reboot once in finalize_switch_boot method + self._mocked_cmdhelper.reboot = mocker.Mock(side_effect=_RebootEXP("reboot")) + mocker.patch( + _RPIBootTestCfg.rpi_boot_RPIBoot_CMDHelperFuncs_MODULE, + self._mocked_cmdhelper, + ) + # NOTE: also remember to patch CMDHelperFuncs in boot_control.common module + mocker.patch( + _RPIBootTestCfg.boot_control_common_CMDHelperFuncs_MODULE, + self._mocked_cmdhelper, + ) + + @pytest.fixture(autouse=True) + def setup_test(self, mocker: pytest_mock.MockerFixture, mock_setup): + _otaclient_cfg = self.mocked_otaclient_cfg_slot_a + _rpi_boot_cfg = self.mocked_boot_cfg_slot_a + + # + # ------ setup mount space ------ # + # + # NOTE: as we mock CMDHelpers, mount is not executed, so we prepare the mount points + # by ourselves.(In the future we can use FSM to do it.) + Path(_otaclient_cfg.OTACLIENT_MOUNT_SPACE_DPATH).mkdir( + parents=True, exist_ok=True + ) + Path(_otaclient_cfg.ACTIVE_SLOT_MP).symlink_to(self.slot_a_pa) + Path(_otaclient_cfg.STANDBY_SLOT_MP).symlink_to(self.slot_b_pa) + + # + # ------ setup OTA status files ------ # + # + + # setup slot a + self.slot_a_ota_status_dir = Path(_rpi_boot_cfg.ACTIVE_BOOT_OTA_STATUS_DPATH) self.slot_a_ota_status_dir.mkdir(parents=True, exist_ok=True) - slot_a_ota_status = self.slot_a_ota_status_dir / "status" - slot_a_ota_status.write_text("SUCCESS") - slot_a_version = self.slot_a_ota_status_dir / "version" - slot_a_version.write_text(cfg.CURRENT_VERSION) - slot_a_slot_in_use = self.slot_a_ota_status_dir / "slot_in_use" - slot_a_slot_in_use.write_text(rpi_boot_cfg.SLOT_A_FSLABEL) - # setup ota dir for slot_a - slot_a_ota_dir = self.slot_a_mp / "boot" / "ota" + + slot_a_ota_dir = self.slot_a_pa / "boot" / "ota" slot_a_ota_dir.mkdir(parents=True, exist_ok=True) - # setup /etc dir for slot_b - (self.slot_b_mp / "etc").mkdir(parents=True, exist_ok=True) + slot_a_ota_status = self.slot_a_ota_status_dir / _otaclient_cfg.OTA_STATUS_FNAME + slot_a_ota_status.write_text(wrapper.StatusOta.SUCCESS.name) - # setup ota_status dir for slot_b - self.slot_b_ota_status_dir = self.slot_b_mp / Path( - rpi_boot_cfg.OTA_STATUS_DIR - ).relative_to("/") + slot_a_version = self.slot_a_ota_status_dir / _otaclient_cfg.OTA_VERSION_FNAME + slot_a_version.write_text(test_cfg.CURRENT_VERSION) + slot_a_slot_in_use = ( + self.slot_a_ota_status_dir / _otaclient_cfg.SLOT_IN_USE_FNAME + ) + slot_a_slot_in_use.write_text(_rpi_boot_cfg.SLOT_A_FSLABEL) - # setup shared system-boot - self.system_boot = tmp_path / "system-boot" - self.system_boot.mkdir(parents=True, exist_ok=True) + # setup slot_b + self.slot_b_ota_status_dir = Path(_rpi_boot_cfg.STANDBY_BOOT_OTA_STATUS_DPATH) + self.slot_b_ota_status_dir.mkdir(parents=True, exist_ok=True) + + # + # ------ setup shared system-boot partition ------ # + # # NOTE: primary config.txt is for slot_a at the beginning - (self.system_boot / f"{rpi_boot_cfg.CONFIG_TXT}").write_text( + (self.system_boot_mp / f"{_rpi_boot_cfg.CONFIG_TXT_FNAME}").write_text( _RPIBootTestCfg.CONFIG_TXT_SLOT_A ) - # NOTE: rpi_boot controller now doesn't check the content of boot files, but only ensure the existence + # NOTE: rpi_boot controller currently doesn't check the content of boot files, but only ensure the existence ( - self.system_boot - / f"{rpi_boot_cfg.CONFIG_TXT}{_RPIBootTestCfg.SEP_CHAR}{_RPIBootTestCfg.SLOT_A}" + self.system_boot_mp + / f"{_rpi_boot_cfg.CONFIG_TXT_FNAME}{_RPIBootTestCfg.SEP_CHAR}{_RPIBootTestCfg.SLOT_A}" ).write_text(_RPIBootTestCfg.CONFIG_TXT_SLOT_A) ( - self.system_boot - / f"{rpi_boot_cfg.CONFIG_TXT}{_RPIBootTestCfg.SEP_CHAR}{_RPIBootTestCfg.SLOT_B}" + self.system_boot_mp + / f"{_rpi_boot_cfg.CONFIG_TXT_FNAME}{_RPIBootTestCfg.SEP_CHAR}{_RPIBootTestCfg.SLOT_B}" ).write_text(_RPIBootTestCfg.CONFIG_TXT_SLOT_B) ( - self.system_boot - / f"{rpi_boot_cfg.CMDLINE_TXT}{_RPIBootTestCfg.SEP_CHAR}{_RPIBootTestCfg.SLOT_A}" + self.system_boot_mp + / f"{_rpi_boot_cfg.CMDLINE_TXT_FNAME}{_RPIBootTestCfg.SEP_CHAR}{_RPIBootTestCfg.SLOT_A}" ).write_text(_RPIBootTestCfg.CMDLINE_TXT_SLOT_A) ( - self.system_boot - / f"{rpi_boot_cfg.CMDLINE_TXT}{_RPIBootTestCfg.SEP_CHAR}{_RPIBootTestCfg.SLOT_B}" + self.system_boot_mp + / f"{_rpi_boot_cfg.CMDLINE_TXT_FNAME}{_RPIBootTestCfg.SEP_CHAR}{_RPIBootTestCfg.SLOT_B}" ).write_text(_RPIBootTestCfg.CMDLINE_TXT_SLOT_B) ( - self.system_boot - / f"{rpi_boot_cfg.VMLINUZ}{_RPIBootTestCfg.SEP_CHAR}{_RPIBootTestCfg.SLOT_A}" + self.system_boot_mp + / f"{_rpi_boot_cfg.VMLINUZ_FNAME}{_RPIBootTestCfg.SEP_CHAR}{_RPIBootTestCfg.SLOT_A}" ).write_text("slot_a_vmlinux") ( - self.system_boot - / f"{rpi_boot_cfg.INITRD_IMG}{_RPIBootTestCfg.SEP_CHAR}{_RPIBootTestCfg.SLOT_A}" + self.system_boot_mp + / f"{_rpi_boot_cfg.INITRD_IMG_FNAME}{_RPIBootTestCfg.SEP_CHAR}{_RPIBootTestCfg.SLOT_A}" ).write_text("slot_a_initrdimg") self.vmlinuz_slot_b = ( - self.system_boot - / f"{rpi_boot_cfg.VMLINUZ}{_RPIBootTestCfg.SEP_CHAR}{_RPIBootTestCfg.SLOT_B}" + self.system_boot_mp + / f"{_rpi_boot_cfg.VMLINUZ_FNAME}{_RPIBootTestCfg.SEP_CHAR}{_RPIBootTestCfg.SLOT_B}" ) self.initrd_img_slot_b = ( - self.system_boot - / f"{rpi_boot_cfg.INITRD_IMG}{_RPIBootTestCfg.SEP_CHAR}{_RPIBootTestCfg.SLOT_B}" - ) - - @pytest.fixture(autouse=True) - def mock_setup(self, mocker: pytest_mock.MockerFixture, rpi_boot_ab_slot): - from otaclient.app.boot_control._rpi_boot import _RPIBootControl - from otaclient.app.boot_control._common import CMDHelperFuncs - - # start the test FSM - self._fsm = RPIBootABPartitionFSM() - - # mocking _RPIBootControl - _RPIBootControl.standby_slot = mocker.PropertyMock(wraps=self._fsm.get_standby_slot) # type: ignore - _RPIBootControl.active_slot = mocker.PropertyMock(wraps=self._fsm.get_active_slot) # type: ignore - _RPIBootControl.active_slot_dev = mocker.PropertyMock(wraps=self._fsm.get_active_slot_dev) # type: ignore - _RPIBootControl.standby_slot_dev = mocker.PropertyMock(wraps=self._fsm.get_standby_slot_dev) # type: ignore - _RPIBootControl._init_slots_info = mocker.Mock() - _RPIBootControl.reboot_tryboot = mocker.Mock( - side_effect=self._fsm.reboot_tryboot + self.system_boot_mp + / f"{_rpi_boot_cfg.INITRD_IMG_FNAME}{_RPIBootTestCfg.SEP_CHAR}{_RPIBootTestCfg.SLOT_B}" ) - _RPIBootControl._update_firmware = mocker.Mock() - - # patch boot_control module - self._mocked__rpiboot_control = _RPIBootControl - mocker.patch(_RPIBootTestCfg.rpi_boot__RPIBootControl_MODULE, _RPIBootControl) - # patch CMDHelperFuncs - # NOTE: also remember to patch CMDHelperFuncs in common - self._CMDHelper_mock = typing.cast( - CMDHelperFuncs, mocker.MagicMock(spec=CMDHelperFuncs) - ) - self._CMDHelper_mock.is_target_mounted = mocker.Mock(return_value=True) - # NOTE: rpi_boot only call CMDHelperFuncs.reboot once in finalize_switch_boot method - self._CMDHelper_mock.reboot = mocker.Mock(side_effect=_RebootEXP("reboot")) - mocker.patch( - _RPIBootTestCfg.rpi_boot_RPIBoot_CMDHelperFuncs_MODULE, self._CMDHelper_mock - ) + # + # ------ patch rpi_boot module for slot_a ------ # + # mocker.patch( - _RPIBootTestCfg.boot_control_common_CMDHelperFuncs_MODULE, - self._CMDHelper_mock, + _RPIBootTestCfg.rpiboot_control_module_path, self.mocked__rpi_boot_ctrl_type ) + mocker.patch(f"{test_cfg.RPI_BOOT_MODULE_PATH}.boot_cfg", _rpi_boot_cfg) def test_rpi_boot_normal_update(self, mocker: pytest_mock.MockerFixture): from otaclient.app.boot_control._rpi_boot import RPIBootController - # ------ patch rpi_boot_cfg for boot_controller_inst1.stage 1~3 ------# - _rpi_boot_cfg_path = f"{cfg.BOOT_CONTROL_CONFIG_MODULE_PATH}.rpi_boot_cfg" - mocker.patch( - f"{_rpi_boot_cfg_path}.SYSTEM_BOOT_MOUNT_POINT", str(self.system_boot) - ) - mocker.patch(f"{_rpi_boot_cfg_path}.ACTIVE_ROOTFS_PATH", str(self.slot_a_mp)) - mocker.patch(f"{_rpi_boot_cfg_path}.MOUNT_POINT", str(self.slot_b_mp)) - mocker.patch( - f"{_rpi_boot_cfg_path}.ACTIVE_ROOT_MOUNT_POINT", str(self.slot_a_mp) - ) + _otaclient_cfg = self.mocked_otaclient_cfg_slot_a + _rpi_boot_cfg = self.mocked_boot_cfg_slot_a # ------ boot_controller_inst1.stage1: init ------ # rpi_boot_controller1 = RPIBootController() @@ -222,33 +321,39 @@ def test_rpi_boot_normal_update(self, mocker: pytest_mock.MockerFixture): # 1. make sure the ota-status is updated properly # 2. make sure the mount points are prepared assert ( - self.slot_a_ota_status_dir / "status" + self.slot_a_ota_status_dir / _otaclient_cfg.OTA_STATUS_FNAME ).read_text() == wrapper.StatusOta.FAILURE.name assert ( - self.slot_b_ota_status_dir / "status" + self.slot_b_ota_status_dir / _otaclient_cfg.OTA_STATUS_FNAME ).read_text() == wrapper.StatusOta.UPDATING.name assert ( - (self.slot_a_ota_status_dir / "slot_in_use").read_text() - == (self.slot_b_ota_status_dir / "slot_in_use").read_text() + (self.slot_a_ota_status_dir / _otaclient_cfg.SLOT_IN_USE_FNAME).read_text() + == ( + self.slot_b_ota_status_dir / _otaclient_cfg.SLOT_IN_USE_FNAME + ).read_text() == _RPIBootTestCfg.SLOT_B ) - self._CMDHelper_mock.mount_rw.assert_called_once_with( - target=self._fsm._standby_slot_dev, mount_point=self.slot_b_mp + + self._mocked_cmdhelper.mount_rw.assert_called_once_with( + target=self._fsm._standby_slot_dev, + mount_point=Path(_otaclient_cfg.STANDBY_SLOT_MP), ) - self._CMDHelper_mock.mount_ro.assert_called_once_with( - target=self._fsm._active_slot_dev, mount_point=self.slot_a_mp + self._mocked_cmdhelper.mount_ro.assert_called_once_with( + target=self._fsm._active_slot_dev, + mount_point=Path(_otaclient_cfg.ACTIVE_SLOT_MP), ) # ------ mocked in_update ------ # # this should be done by create_standby module, so we do it manually here instead - self.slot_b_boot_dir = self.slot_b_mp / "boot" - self.slot_a_boot_dir = self.slot_a_mp / "boot" + self.slot_b_boot_dir = self.slot_b_pa / "boot" + self.slot_a_boot_dir = self.slot_a_pa / "boot" + # NOTE: copy slot_a's kernel and initrd.img to slot_b, # because we skip the create_standby step # NOTE 2: not copy the symlinks - _vmlinuz = self.slot_a_boot_dir / "vmlinuz" + _vmlinuz = self.slot_a_boot_dir / _rpi_boot_cfg.VMLINUZ_FNAME shutil.copy(os.path.realpath(_vmlinuz), self.slot_b_boot_dir) - _initrd_img = self.slot_a_boot_dir / "initrd.img" + _initrd_img = self.slot_a_boot_dir / _rpi_boot_cfg.INITRD_IMG_FNAME shutil.copy(os.path.realpath(_initrd_img), self.slot_b_boot_dir) # ------ boot_controller_inst1.stage3: post_update, reboot switch boot ------ # @@ -262,27 +367,43 @@ def test_rpi_boot_normal_update(self, mocker: pytest_mock.MockerFixture): # 3. assert kernel and initrd.img are copied to system-boot # 4. make sure tryboot.txt is presented and correct # 5. make sure config.txt is untouched - self._mocked__rpiboot_control.reboot_tryboot.assert_called_once() + self.mocked__rpi_boot_ctrl_type.reboot_tryboot.assert_called_once() assert self._fsm.is_switched_boot - assert ( - self.slot_b_mp / Path(rpi_boot_cfg.FSTAB_FPATH).relative_to("/") + + assert Path( + replace_root( + _otaclient_cfg.FSTAB_FPATH, + _otaclient_cfg.ACTIVE_ROOTFS, + self.slot_b_pa, + ) ).read_text() == Template(_FSTAB_TEMPLATE_STR).substitute( rootfs_fslabel=_RPIBootTestCfg.SLOT_B ) + assert self.initrd_img_slot_b.is_file() assert self.vmlinuz_slot_b.is_file() + assert ( - self.system_boot / "tryboot.txt" + self.system_boot_mp / _rpi_boot_cfg.TRYBOOT_TXT_FNAME ).read_text() == _RPIBootTestCfg.CONFIG_TXT_SLOT_B assert ( - self.system_boot / "config.txt" + self.system_boot_mp / _rpi_boot_cfg.CONFIG_TXT_FNAME ).read_text() == _RPIBootTestCfg.CONFIG_TXT_SLOT_A + # # ------ boot_controller_inst2: first reboot ------ # + # + # Now active rootfs is slot_b. + # patch rpi_boot_cfg for boot_controller_inst2 - _rpi_boot_cfg_path = f"{cfg.BOOT_CONTROL_CONFIG_MODULE_PATH}.rpi_boot_cfg" - mocker.patch(f"{_rpi_boot_cfg_path}.ACTIVE_ROOTFS_PATH", str(self.slot_b_mp)) - mocker.patch(f"{_rpi_boot_cfg_path}.MOUNT_POINT", str(self.slot_a_mp)) + _otaclient_cfg = self.mocked_otaclient_cfg_slot_b + mocker.patch(f"{test_cfg.BOOT_CONTROL_CONFIG_MODULE_PATH}.cfg", _otaclient_cfg) + mocker.patch(f"{test_cfg.RPI_BOOT_MODULE_PATH}.cfg", _otaclient_cfg) + mocker.patch(f"{test_cfg.BOOT_CONTROL_COMMON_MODULE_PATH}.cfg", _otaclient_cfg) + + # after the mocked cfg is applied, we can init rpi_boot cfg instance + _rpi_boot_cfg = RPIBootControlConfig() + mocker.patch(f"{test_cfg.RPI_BOOT_MODULE_PATH}.boot_cfg", _rpi_boot_cfg) # ------ boot_controller_inst2.stage1: first reboot finalizing switch boot and update firmware ------ # logger.info("1st reboot: finalize switch boot and update firmware....") @@ -290,6 +411,7 @@ def test_rpi_boot_normal_update(self, mocker: pytest_mock.MockerFixture): # NOTE: raise a _RebootEXP to simulate reboot and interrupt the otaclient with pytest.raises(_RebootEXP): RPIBootController() # NOTE: init only + # --- assertions: --- # # 1. assert that otaclient reboots the device # 2. assert firmware update is called @@ -298,17 +420,17 @@ def test_rpi_boot_normal_update(self, mocker: pytest_mock.MockerFixture): # 5. assert slot_in_use is slot_b # 6. make sure the SWITCH_BOOT_FLAG_FILE file is created # 7. make sure ota_status is still UPDATING - self._mocked__rpiboot_control._update_firmware.assert_called_once() - self._CMDHelper_mock.reboot.assert_called_once() + self.mocked__rpi_boot_ctrl_type._update_firmware.assert_called_once() + self._mocked_cmdhelper.reboot.assert_called_once() assert ( - self.system_boot / "config.txt" + self.system_boot_mp / _rpi_boot_cfg.CONFIG_TXT_FNAME ).read_text() == _RPIBootTestCfg.CONFIG_TXT_SLOT_B assert ( - self.slot_b_ota_status_dir / rpi_boot_cfg.SLOT_IN_USE_FNAME + self.slot_b_ota_status_dir / _otaclient_cfg.SLOT_IN_USE_FNAME ).read_text() == "slot_b" - assert (self.system_boot / rpi_boot_cfg.SWITCH_BOOT_FLAG_FILE).is_file() + assert (self.system_boot_mp / _rpi_boot_cfg.SWITCH_BOOT_FLAG_FNAME).is_file() assert ( - self.slot_b_ota_status_dir / rpi_boot_cfg.OTA_STATUS_FNAME + self.slot_b_ota_status_dir / _otaclient_cfg.OTA_STATUS_FNAME ).read_text() == wrapper.StatusOta.UPDATING.name # ------ boot_controller_inst3.stage1: second reboot, apply updated firmware and finish up ota update ------ # @@ -323,9 +445,11 @@ def test_rpi_boot_normal_update(self, mocker: pytest_mock.MockerFixture): rpi_boot_controller4_2.get_booted_ota_status() == wrapper.StatusOta.SUCCESS ) assert ( - self.slot_b_ota_status_dir / rpi_boot_cfg.OTA_STATUS_FNAME + self.slot_b_ota_status_dir / _otaclient_cfg.OTA_STATUS_FNAME ).read_text() == wrapper.StatusOta.SUCCESS.name - assert not (self.system_boot / rpi_boot_cfg.SWITCH_BOOT_FLAG_FILE).is_file() + assert not ( + self.system_boot_mp / _rpi_boot_cfg.SWITCH_BOOT_FLAG_FNAME + ).is_file() assert ( rpi_boot_controller4_2._ota_status_control._load_current_slot_in_use() == "slot_b" diff --git a/tests/test_common.py b/tests/test_common.py index 534e311da..cb77c256a 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -42,7 +42,7 @@ write_str_to_file_sync, ) from tests.utils import compare_dir -from tests.conftest import cfg, run_http_server +from tests.conftest import run_http_server logger = logging.getLogger(__name__) diff --git a/tests/test_create_standby.py b/tests/test_create_standby.py index c95f9c316..67a267b33 100644 --- a/tests/test_create_standby.py +++ b/tests/test_create_standby.py @@ -21,10 +21,9 @@ from pytest_mock import MockerFixture from otaclient.app.boot_control import BootControllerProtocol -from otaclient.app.configs import config as otaclient_cfg -from otaclient.app.proto import wrapper +from otaclient.configs.app_cfg import Config as otaclient_Config -from tests.conftest import TestConfiguration as cfg +from tests.conftest import TestConfiguration as test_cfg from tests.utils import SlotMeta, compare_dir import logging @@ -35,39 +34,57 @@ class Test_OTAupdate_with_create_standby_RebuildMode: """ NOTE: the boot_control is mocked, only testing - create_standby and the logics directly implemented by OTAUpdater + create_standby and the logics directly implemented by OTAUpdater. + + NOTE: testing the system using separated boot dev for each slots(like cboot). """ @pytest.fixture - def prepare_ab_slots(self, tmp_path: Path, ab_slots: SlotMeta): + def setup_test(self, tmp_path: Path, ab_slots: SlotMeta): + # + # ------ prepare ab slots ------ # + # self.slot_a = Path(ab_slots.slot_a) self.slot_b = Path(ab_slots.slot_b) self.slot_a_boot_dir = Path(ab_slots.slot_a_boot_dev) / "boot" self.slot_b_boot_dir = Path(ab_slots.slot_b_boot_dev) / "boot" - self.ota_image_dir = Path(cfg.OTA_IMAGE_DIR) + self.ota_image_dir = Path(test_cfg.OTA_IMAGE_DIR) self.otaclient_run_dir = tmp_path / "otaclient_run_dir" self.otaclient_run_dir.mkdir(parents=True, exist_ok=True) - # ------ cleanup and prepare slot_b ------ # - shutil.rmtree(self.slot_b, ignore_errors=True) - self.slot_b.mkdir(exist_ok=True) - # some important paths - self.ota_metafiles_tmp_dir = self.slot_b / Path( - otaclient_cfg.OTA_TMP_META_STORE - ).relative_to("/") - self.ota_tmp_dir = self.slot_b / Path(otaclient_cfg.OTA_TMP_STORE).relative_to( - "/" + + self.slot_a_boot_dir.mkdir(exist_ok=True, parents=True) + self.slot_b_boot_dir.mkdir(exist_ok=True, parents=True) + + # + # ------ prepare config ------ # + # + _otaclient_cfg = otaclient_Config(ACTIVE_ROOTFS=str(self.slot_a)) + self.otaclient_cfg = _otaclient_cfg + + # ------ prepare otaclient run dir ------ # + Path(_otaclient_cfg.RUN_DPATH).mkdir(exist_ok=True, parents=True) + + # + # ------ prepare mount space ------ # + # + Path(_otaclient_cfg.OTACLIENT_MOUNT_SPACE_DPATH).mkdir( + exist_ok=True, parents=True ) + # directly point standby slot mp to self.slot_b + _standby_slot_mp = Path(_otaclient_cfg.STANDBY_SLOT_MP) + _standby_slot_mp.symlink_to(self.slot_b) + + # some important paths + self.ota_metafiles_tmp_dir = Path(_otaclient_cfg.STANDBY_IMAGE_META_DPATH) + self.ota_tmp_dir = Path(_otaclient_cfg.STANDBY_OTA_TMP_DPATH) yield # cleanup slot_b after test shutil.rmtree(self.slot_b, ignore_errors=True) @pytest.fixture(autouse=True) - def mock_setup(self, mocker: MockerFixture, prepare_ab_slots): - from otaclient.app.proxy_info import ProxyInfo - from otaclient.app.configs import BaseConfig - + def mock_setup(self, mocker: MockerFixture, setup_test): # ------ mock boot_controller ------ # self._boot_control = typing.cast( BootControllerProtocol, mocker.MagicMock(spec=BootControllerProtocol) @@ -75,13 +92,12 @@ def mock_setup(self, mocker: MockerFixture, prepare_ab_slots): self._boot_control.get_standby_boot_dir.return_value = self.slot_b_boot_dir # ------ mock otaclient cfg ------ # - _cfg = BaseConfig() - _cfg.MOUNT_POINT = str(self.slot_b) # type: ignore - _cfg.ACTIVE_ROOT_MOUNT_POINT = str(self.slot_a) # type: ignore - _cfg.RUN_DIR = str(self.otaclient_run_dir) # type: ignore - mocker.patch(f"{cfg.OTACLIENT_MODULE_PATH}.cfg", _cfg) - mocker.patch(f"{cfg.CREATE_STANDBY_MODULE_PATH}.rebuild_mode.cfg", _cfg) - mocker.patch(f"{cfg.OTAMETA_MODULE_PATH}.cfg", _cfg) + mocker.patch(f"{test_cfg.OTACLIENT_MODULE_PATH}.cfg", self.otaclient_cfg) + mocker.patch( + f"{test_cfg.CREATE_STANDBY_MODULE_PATH}.rebuild_mode.cfg", + self.otaclient_cfg, + ) + mocker.patch(f"{test_cfg.OTAMETA_MODULE_PATH}.cfg", self.otaclient_cfg) def test_update_with_create_standby_RebuildMode(self, mocker: MockerFixture): from otaclient.app.ota_client import _OTAUpdater, OTAClientControlFlags @@ -108,8 +124,8 @@ def test_update_with_create_standby_RebuildMode(self, mocker: MockerFixture): _updater.shutdown = mocker.MagicMock() _updater.execute( - version=cfg.UPDATE_VERSION, - raw_url_base=cfg.OTA_IMAGE_URL, + version=test_cfg.UPDATE_VERSION, + raw_url_base=test_cfg.OTA_IMAGE_URL, cookies_json=r'{"test": "my-cookie"}', ) time.sleep(2) # wait for downloader to record stats @@ -134,18 +150,14 @@ def test_update_with_create_standby_RebuildMode(self, mocker: MockerFixture): # NOTE: for some reason tmp dir is created under OTA_IMAGE_DIR/data, but not listed # in the regulars.txt, so we create one here to make the test passed (self.slot_b / "tmp").mkdir(exist_ok=True) + # NOTE: remove the ota-meta dir and ota-tmp dir to resolve the difference with OTA image - shutil.rmtree( - self.slot_b / Path(otaclient_cfg.OTA_TMP_META_STORE).relative_to("/"), - ignore_errors=True, - ) - shutil.rmtree( - self.slot_b / Path(otaclient_cfg.OTA_TMP_STORE).relative_to("/"), - ignore_errors=True, - ) + shutil.rmtree(self.ota_metafiles_tmp_dir, ignore_errors=True) + shutil.rmtree(self.ota_tmp_dir, ignore_errors=True) shutil.rmtree(self.slot_b / "opt/ota", ignore_errors=True) + # --- check standby slot, ensure it is correctly populated - compare_dir(Path(cfg.OTA_IMAGE_DIR) / "data", self.slot_b) + compare_dir(Path(test_cfg.OTA_IMAGE_DIR) / "data", self.slot_b) # ------ finally close the updater ------ # _updater_shutdown() diff --git a/tests/test_downloader.py b/tests/test_downloader.py index 2c5f34062..26fe1b9d6 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -13,6 +13,7 @@ # limitations under the License. +from __future__ import annotations import asyncio import logging import threading @@ -23,7 +24,6 @@ from pathlib import Path from urllib.parse import urlsplit, urljoin - from otaclient.app.common import file_sha256, urljoin_ensure_base from otaclient.app.downloader import ( DownloadError, @@ -35,15 +35,20 @@ UnhandledHTTPError, ) -from tests.conftest import TestConfiguration as cfg +from tests.conftest import TestConfiguration as test_cfg from tests.utils import zstd_compress_file logger = logging.getLogger(__name__) +HTTP_ERROR_ECHO_SERVER_PORT = 9999 +HTTP_ERROR_ECHO_SERVER_ADDR = "127.0.0.1" + + +class _HTTPErrorCodeEchoApp: + """A simple server that responses with HTTP error code specified via URL.""" -class _SimpleDummyApp: - async def __call__(self, scope, receive, send) -> None: + async def __call__(self, scope: dict[str, str], receive, send) -> None: if scope["type"] != "http" or scope["method"] != "GET": return @@ -63,24 +68,29 @@ async def __call__(self, scope, receive, send) -> None: @pytest.fixture(scope="module") -def launch_dummy_server(host: str = "127.0.0.1", port: int = 9999): +def launch_dummy_server( + host: str = HTTP_ERROR_ECHO_SERVER_ADDR, port: int = HTTP_ERROR_ECHO_SERVER_PORT +): _should_exit = threading.Event() async def _launcher(): import uvicorn _config = uvicorn.Config( - _SimpleDummyApp(), + _HTTPErrorCodeEchoApp(), host=host, port=port, ) server = uvicorn.Server(_config) + config = server.config if not config.loaded: config.load() + server.lifespan = config.lifespan_class(config) await server.startup() - logger.info("dummy server started") + logger.info(f"dummy server started at {host}:{port}") + while True: if _should_exit.is_set(): await server.shutdown() @@ -98,20 +108,24 @@ async def _launcher(): class TestDownloader: + # NOTE: here we download the metadata.jwt file to test the downloader. # NOTE: full URL is http:///metadata.jwt # full path is /metadata.jwt - TEST_FILE = "metadata.jwt" - TEST_FILE_PATH = Path(cfg.OTA_IMAGE_DIR) / TEST_FILE - TEST_FILE_SHA256 = file_sha256(TEST_FILE_PATH) - TEST_FILE_SIZE = len(TEST_FILE_PATH.read_bytes()) + TEST_FILE_FNAME = test_cfg.METADATA_JWT_FNAME + TEST_FILE_FPATH = Path(test_cfg.OTA_IMAGE_DIR) / test_cfg.METADATA_JWT_FNAME + TEST_FILE_SHA256 = file_sha256(TEST_FILE_FPATH) + TEST_FILE_SIZE = len(TEST_FILE_FPATH.read_bytes()) @pytest.fixture def prepare_zstd_compressed_files(self): # prepare a compressed file under OTA image dir, # and then remove it after test finished try: - self.zstd_compressed = Path(cfg.OTA_IMAGE_DIR) / f"{self.TEST_FILE}.zst" - zstd_compress_file(self.TEST_FILE_PATH, self.zstd_compressed) + self.zstd_compressed = ( + Path(test_cfg.OTA_IMAGE_DIR) / f"{self.TEST_FILE_FNAME}.zst" + ) + zstd_compress_file(self.TEST_FILE_FPATH, self.zstd_compressed) + yield finally: self.zstd_compressed.unlink(missing_ok=True) @@ -129,9 +143,10 @@ def launch_downloader(self, mocker: pytest_mock.MockerFixture): self.downloader.shutdown() def test_normal_download(self, tmp_path: Path): - _target_path = tmp_path / self.TEST_FILE + """Download the test file using downloader.""" + _target_path = tmp_path / self.TEST_FILE_FNAME - url = urljoin_ensure_base(cfg.OTA_IMAGE_URL, self.TEST_FILE) + url = urljoin_ensure_base(test_cfg.OTA_IMAGE_URL, self.TEST_FILE_FNAME) _error, _read_download_size, _ = self.downloader.download( url, _target_path, @@ -140,15 +155,15 @@ def test_normal_download(self, tmp_path: Path): ) assert _error == 0 - assert _read_download_size == self.TEST_FILE_PATH.stat().st_size + assert _read_download_size == self.TEST_FILE_FPATH.stat().st_size assert file_sha256(_target_path) == self.TEST_FILE_SHA256 def test_download_zstd_compressed_file( self, tmp_path: Path, prepare_zstd_compressed_files ): - _target_path = tmp_path / self.TEST_FILE + _target_path = tmp_path / self.TEST_FILE_FNAME - url = urljoin_ensure_base(cfg.OTA_IMAGE_URL, f"{self.TEST_FILE}.zst") + url = urljoin_ensure_base(test_cfg.OTA_IMAGE_URL, f"{self.TEST_FILE_FNAME}.zst") # first test directly download without decompression _error, _read_download_bytes_a, _ = self.downloader.download(url, _target_path) assert _error == 0 @@ -172,9 +187,9 @@ def test_download_zstd_compressed_file( assert file_sha256(_target_path) == self.TEST_FILE_SHA256 def test_download_mismatch_sha256(self, tmp_path: Path): - _target_path = tmp_path / self.TEST_FILE + _target_path = tmp_path / self.TEST_FILE_FNAME - url = urljoin_ensure_base(cfg.OTA_IMAGE_URL, self.TEST_FILE) + url = urljoin_ensure_base(test_cfg.OTA_IMAGE_URL, self.TEST_FILE_FNAME) with pytest.raises(HashVerificaitonError): self.downloader.download( url, @@ -208,10 +223,10 @@ def test_download_errors_handling( ) # load the mocker adapter to the Downloader session - self.session.mount(cfg.OTA_IMAGE_URL, _mock_adapter) + self.session.mount(test_cfg.OTA_IMAGE_URL, _mock_adapter) - _target_path = tmp_path / self.TEST_FILE - url = urljoin_ensure_base(cfg.OTA_IMAGE_URL, self.TEST_FILE) + _target_path = tmp_path / self.TEST_FILE_FNAME + url = urljoin_ensure_base(test_cfg.OTA_IMAGE_URL, self.TEST_FILE_FNAME) with pytest.raises(expected_ota_download_err): self.downloader.download( url, @@ -242,8 +257,11 @@ def test_download_server_with_http_error( expected_ota_download_err, launch_dummy_server, ): - url = urljoin("http://127.0.0.1:9999/", str(status_code)) - _target_path = tmp_path / self.TEST_FILE + url = urljoin( + f"http://{HTTP_ERROR_ECHO_SERVER_ADDR}:{HTTP_ERROR_ECHO_SERVER_PORT}", + str(status_code), + ) + _target_path = tmp_path / self.TEST_FILE_FNAME with pytest.raises(expected_ota_download_err): self.downloader.download( url, @@ -258,8 +276,8 @@ def test_retry_headers_injection( _mock_get = mocker.MagicMock(wraps=self.session.get) self.session.get = _mock_get - _target_path = tmp_path / self.TEST_FILE - url = urljoin_ensure_base(cfg.OTA_IMAGE_URL, self.TEST_FILE) + _target_path = tmp_path / self.TEST_FILE_FNAME + url = urljoin_ensure_base(test_cfg.OTA_IMAGE_URL, self.TEST_FILE_FNAME) with pytest.raises(HashVerificaitonError): self.downloader.download(url, _target_path, digest="wrong_digest") diff --git a/tests/test_main.py b/tests/test_main.py index 55794d933..81b8b3ba6 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -22,7 +22,7 @@ from pytest import LogCaptureFixture from otaclient.app.configs import config as otaclient_cfg -from tests.conftest import TestConfiguration as cfg +from tests.conftest import TestConfiguration as test_cfg FIRST_LINE_LOG = "d3b6bdb | 2021-10-27 09:36:48 +0900 | Initial commit" @@ -30,14 +30,14 @@ class TestMain: @pytest.fixture(autouse=True) def patch_main(self, mocker: MockerFixture, tmp_path: Path): - mocker.patch(f"{cfg.MAIN_MODULE_PATH}.launch_otaclient_grpc_server") + mocker.patch(f"{test_cfg.MAIN_MODULE_PATH}.launch_otaclient_grpc_server") self._sys_exit_mocker = mocker.MagicMock(side_effect=ValueError) - mocker.patch(f"{cfg.MAIN_MODULE_PATH}.sys.exit", self._sys_exit_mocker) + mocker.patch(f"{test_cfg.MAIN_MODULE_PATH}.sys.exit", self._sys_exit_mocker) version_file = tmp_path / "version.txt" version_file.write_text(FIRST_LINE_LOG) - mocker.patch(f"{cfg.MAIN_MODULE_PATH}.EXTRA_VERSION_FILE", version_file) + mocker.patch(f"{test_cfg.MAIN_MODULE_PATH}.EXTRA_VERSION_FILE", version_file) @pytest.fixture def background_process(self): @@ -47,7 +47,7 @@ def _waiting(): _p = Process(target=_waiting) try: _p.start() - Path(otaclient_cfg.OTACLIENT_PID_FILE).write_text(f"{_p.pid}") + Path(otaclient_cfg.OTACLIENT_PID_FPATH).write_text(f"{_p.pid}") yield _p.pid finally: _p.kill() @@ -58,7 +58,7 @@ def test_main_with_version(self, caplog: LogCaptureFixture): main() assert caplog.records[0].msg == "started" assert caplog.records[1].msg == FIRST_LINE_LOG - assert Path(otaclient_cfg.OTACLIENT_PID_FILE).read_text() == f"{os.getpid()}" + assert Path(otaclient_cfg.OTACLIENT_PID_FPATH).read_text() == f"{os.getpid()}" def test_with_other_otaclient_started(self, background_process): from otaclient.app.main import main @@ -67,4 +67,4 @@ def test_with_other_otaclient_started(self, background_process): with pytest.raises(ValueError): main() self._sys_exit_mocker.assert_called_once() - assert Path(otaclient_cfg.OTACLIENT_PID_FILE).read_text() == _other_pid + assert Path(otaclient_cfg.OTACLIENT_PID_FPATH).read_text() == _other_pid diff --git a/tests/test_ota_client.py b/tests/test_ota_client.py index 5990e5295..75b72f721 100644 --- a/tests/test_ota_client.py +++ b/tests/test_ota_client.py @@ -13,6 +13,7 @@ # limitations under the License. +from __future__ import annotations import asyncio import threading import typing @@ -28,9 +29,8 @@ from otaclient.app.boot_control.configs import BootloaderType from otaclient.app.create_standby import StandbySlotCreatorProtocol from otaclient.app.create_standby.common import DeltaBundle, RegularDelta -from otaclient.app.configs import config as otaclient_cfg from otaclient.app.ecu_info import ECUInfo -from otaclient.app.errors import OTAError, OTAErrorRecoverable +from otaclient.app.errors import OTAErrorRecoverable from otaclient.app.ota_client import ( OTAClient, _OTAUpdater, @@ -40,8 +40,9 @@ from otaclient.app.ota_metadata import parse_regulars_from_txt, parse_dirs_from_txt from otaclient.app.proto.wrapper import RegularInf, DirectoryInf from otaclient.app.proto import wrapper +from otaclient.configs.app_cfg import Config as otaclient_Config -from tests.conftest import TestConfiguration as cfg +from tests.conftest import TestConfiguration as test_cfg from tests.utils import SlotMeta @@ -52,34 +53,48 @@ class Test_OTAUpdater: """ @pytest.fixture - def prepare_ab_slots(self, tmp_path: Path, ab_slots: SlotMeta): + def setup_test(self, tmp_path: Path, ab_slots: SlotMeta): + # + # ------ prepare ab slots ------ # + # self.slot_a = Path(ab_slots.slot_a) self.slot_b = Path(ab_slots.slot_b) self.slot_a_boot_dir = Path(ab_slots.slot_a_boot_dev) / "boot" self.slot_b_boot_dir = Path(ab_slots.slot_b_boot_dev) / "boot" - self.ota_image_dir = Path(cfg.OTA_IMAGE_DIR) - - self.otaclient_run_dir = tmp_path / "otaclient_run_dir" - self.otaclient_run_dir.mkdir(parents=True, exist_ok=True) + self.ota_image_dir = Path(test_cfg.OTA_IMAGE_DIR) # ------ cleanup and prepare slot_b ------ # shutil.rmtree(self.slot_b, ignore_errors=True) self.slot_b.mkdir(exist_ok=True) - # some important paths - self.ota_metafiles_tmp_dir = self.slot_b / Path( - otaclient_cfg.OTA_TMP_META_STORE - ).relative_to("/") - self.ota_tmp_dir = self.slot_b / Path(otaclient_cfg.OTA_TMP_STORE).relative_to( - "/" + + # + # ------ init otaclient config ------ # + # + self.otaclient_cfg = _otaclient_cfg = otaclient_Config( + ACTIVE_ROOTFS=ab_slots.slot_a + ) + + # prepare dummy ab slots mount points + Path(_otaclient_cfg.OTACLIENT_MOUNT_SPACE_DPATH).mkdir( + parents=True, exist_ok=True ) + Path(_otaclient_cfg.ACTIVE_SLOT_MP).symlink_to(self.slot_a) + Path(_otaclient_cfg.STANDBY_SLOT_MP).symlink_to(self.slot_b) + + # some important paths + self.ota_metafiles_tmp_dir = Path(self.otaclient_cfg.STANDBY_IMAGE_META_DPATH) + self.ota_tmp_dir = Path(self.otaclient_cfg.STANDBY_OTA_TMP_DPATH) + + self.otaclient_run_dir = Path(_otaclient_cfg.RUN_DPATH) + self.otaclient_run_dir.mkdir(parents=True, exist_ok=True) yield # cleanup slot_b after test shutil.rmtree(self.slot_b, ignore_errors=True) @pytest.fixture - def _delta_generate(self, prepare_ab_slots): - _ota_image_dir = Path(cfg.OTA_IMAGE_DIR) + def _delta_generate(self, setup_test): + _ota_image_dir = Path(test_cfg.OTA_IMAGE_DIR) _standby_ota_tmp = self.slot_b / ".ota-tmp" # ------ manually create delta bundle ------ # @@ -122,9 +137,6 @@ def _delta_generate(self, prepare_ab_slots): @pytest.fixture(autouse=True) def mock_setup(self, mocker: pytest_mock.MockerFixture, _delta_generate): - from otaclient.app.proxy_info import ProxyInfo - from otaclient.app.configs import BaseConfig - # ------ mock boot_controller ------ # self._boot_control = typing.cast( BootControllerProtocol, mocker.MagicMock(spec=BootControllerProtocol) @@ -143,16 +155,13 @@ def mock_setup(self, mocker: pytest_mock.MockerFixture, _delta_generate): self._create_standby.should_erase_standby_slot.return_value = False # ------ mock otaclient cfg ------ # - _cfg = BaseConfig() - _cfg.MOUNT_POINT = str(self.slot_b) # type: ignore - _cfg.ACTIVE_ROOTFS_PATH = str(self.slot_a) # type: ignore - _cfg.RUN_DIR = str(self.otaclient_run_dir) # type: ignore - mocker.patch(f"{cfg.OTACLIENT_MODULE_PATH}.cfg", _cfg) - mocker.patch(f"{cfg.OTAMETA_MODULE_PATH}.cfg", _cfg) + mocker.patch(f"{test_cfg.OTACLIENT_MODULE_PATH}.cfg", self.otaclient_cfg) + mocker.patch(f"{test_cfg.OTAMETA_MODULE_PATH}.cfg", self.otaclient_cfg) # ------ mock stats collector ------ # mocker.patch( - f"{cfg.OTACLIENT_MODULE_PATH}.OTAUpdateStatsCollector", mocker.MagicMock() + f"{test_cfg.OTACLIENT_MODULE_PATH}.OTAUpdateStatsCollector", + mocker.MagicMock(), ) def test_OTAUpdater(self, mocker: pytest_mock.MockerFixture): @@ -170,8 +179,8 @@ def test_OTAUpdater(self, mocker: pytest_mock.MockerFixture): ) _updater.execute( - version=cfg.UPDATE_VERSION, - raw_url_base=cfg.OTA_IMAGE_URL, + version=test_cfg.UPDATE_VERSION, + raw_url_base=test_cfg.OTA_IMAGE_URL, cookies_json=r'{"test": "my-cookie"}', ) @@ -183,7 +192,7 @@ def test_OTAUpdater(self, mocker: pytest_mock.MockerFixture): assert _downloaded_files_size == self._delta_bundle.total_download_files_size # assert the control_flags has been waited otaclient_control_flags.wait_can_reboot_flag.assert_called_once() - assert _updater.updating_version == cfg.UPDATE_VERSION + assert _updater.updating_version == test_cfg.UPDATE_VERSION # assert boot controller is used self._boot_control.pre_update.assert_called_once() self._boot_control.post_update.assert_called_once() @@ -243,7 +252,8 @@ def mock_setup(self, mocker: pytest_mock.MockerFixture): # patch inject mocked updater mocker.patch( - f"{cfg.OTACLIENT_MODULE_PATH}._OTAUpdater", return_value=self.ota_updater + f"{test_cfg.OTACLIENT_MODULE_PATH}._OTAUpdater", + return_value=self.ota_updater, ) # inject lock into otaclient self.ota_client._lock = self.ota_lock @@ -375,20 +385,20 @@ async def mock_setup(self, mocker: pytest_mock.MockerFixture): self.control_flags = mocker.MagicMock(spec=OTAClientControlFlags) # - # ------ patching ------ + # ------ patching ------ # # - mocker.patch(f"{cfg.OTACLIENT_MODULE_PATH}.OTAClient", self.otaclient_cls) + mocker.patch(f"{test_cfg.OTACLIENT_MODULE_PATH}.OTAClient", self.otaclient_cls) mocker.patch( - f"{cfg.OTACLIENT_MODULE_PATH}.get_boot_controller", + f"{test_cfg.OTACLIENT_MODULE_PATH}.get_boot_controller", return_value=mocker.MagicMock(return_value=self.boot_controller), ) mocker.patch( - f"{cfg.OTACLIENT_MODULE_PATH}.get_standby_slot_creator", + f"{test_cfg.OTACLIENT_MODULE_PATH}.get_standby_slot_creator", return_value=self.standby_slot_creator_cls, ) # - # ------ start OTAServicer instance ------ + # ------ start OTAServicer instance ------ # # self.local_use_proxy = "" self.otaclient_stub = OTAServicer( @@ -404,10 +414,6 @@ async def mock_setup(self, mocker: pytest_mock.MockerFixture): self._executor.shutdown(wait=False) def test_stub_initializing(self): - # - # ------ assertion ------ - # - # ensure the OTAServicer properly compose otaclient core self.otaclient_cls.assert_called_once_with( boot_controller=self.boot_controller, diff --git a/tests/test_ota_client_service.py b/tests/test_ota_client_service.py index 9a2482a0e..e041e505a 100644 --- a/tests/test_ota_client_service.py +++ b/tests/test_ota_client_service.py @@ -17,12 +17,12 @@ import pytest import pytest_mock -from otaclient.app.configs import server_cfg from otaclient.app.ecu_info import ECUInfo from otaclient.app.ota_client_service import create_otaclient_grpc_server from otaclient.app.ota_client_call import OtaClientCall from otaclient.app.proto import wrapper -from tests.conftest import cfg +from otaclient.configs.ota_service_cfg import OTAServiceConfig +from tests.conftest import test_cfg from tests.utils import compare_message @@ -62,13 +62,14 @@ async def status(self, *args, **kwargs): class Test_ota_client_service: MY_ECU_ID = _MockedOTAClientServiceStub.MY_ECU_ID LISTEN_ADDR = "127.0.0.1" - LISTEN_PORT = server_cfg.SERVER_PORT @pytest.fixture(autouse=True) def setup_test(self, mocker: pytest_mock.MockerFixture): + self.otaclient_cfg = OTAServiceConfig() + self.otaclient_service_stub = _MockedOTAClientServiceStub() mocker.patch( - f"{cfg.OTACLIENT_SERVICE_MODULE_PATH}.OTAClientServiceStub", + f"{test_cfg.OTACLIENT_SERVICE_MODULE_PATH}.OTAClientServiceStub", return_value=self.otaclient_service_stub, ) @@ -78,7 +79,7 @@ def setup_test(self, mocker: pytest_mock.MockerFixture): ecu_id=self.otaclient_service_stub.MY_ECU_ID, ip_addr=self.LISTEN_ADDR, ) - mocker.patch(f"{cfg.OTACLIENT_SERVICE_MODULE_PATH}.ECUInfo", ecu_info_mock) + mocker.patch(f"{test_cfg.OTACLIENT_SERVICE_MODULE_PATH}.ECUInfo", ecu_info_mock) @pytest.fixture(autouse=True) async def launch_otaclient_server(self, setup_test): @@ -95,7 +96,7 @@ async def test_otaclient_service(self): update_resp = await OtaClientCall.update_call( ecu_id=self.MY_ECU_ID, ecu_ipaddr=self.LISTEN_ADDR, - ecu_port=self.LISTEN_PORT, + ecu_port=self.otaclient_cfg.SERVER_PORT, request=wrapper.UpdateRequest(), ) compare_message(update_resp, self.otaclient_service_stub.UPDATE_RESP) @@ -104,7 +105,7 @@ async def test_otaclient_service(self): rollback_resp = await OtaClientCall.rollback_call( ecu_id=self.MY_ECU_ID, ecu_ipaddr=self.LISTEN_ADDR, - ecu_port=self.LISTEN_PORT, + ecu_port=self.otaclient_cfg.SERVER_PORT, request=wrapper.RollbackRequest(), ) compare_message(rollback_resp, self.otaclient_service_stub.ROLLBACK_RESP) @@ -113,7 +114,7 @@ async def test_otaclient_service(self): status_resp = await OtaClientCall.status_call( ecu_id=self.MY_ECU_ID, ecu_ipaddr=self.LISTEN_ADDR, - ecu_port=self.LISTEN_PORT, + ecu_port=self.otaclient_cfg.SERVER_PORT, request=wrapper.StatusRequest(), ) compare_message(status_resp, self.otaclient_service_stub.STATUS_RESP) diff --git a/tests/test_ota_client_stub.py b/tests/test_ota_client_stub.py index a741aeb43..f763e8242 100644 --- a/tests/test_ota_client_stub.py +++ b/tests/test_ota_client_stub.py @@ -35,7 +35,7 @@ from otaclient.ota_proxy.config import Config as otaproxyConfig from tests.utils import compare_message -from tests.conftest import cfg +from tests.conftest import test_cfg import logging @@ -731,23 +731,23 @@ async def setup_test(self, tmp_path: Path, mocker: pytest_mock.MockerFixture): # --- patching and mocking --- # mocker.patch( - f"{cfg.OTACLIENT_STUB_MODULE_PATH}.ECUStatusStorage", + f"{test_cfg.OTACLIENT_STUB_MODULE_PATH}.ECUStatusStorage", mocker.MagicMock(return_value=self.ecu_storage), ) mocker.patch( - f"{cfg.OTACLIENT_STUB_MODULE_PATH}.OTAServicer", + f"{test_cfg.OTACLIENT_STUB_MODULE_PATH}.OTAServicer", mocker.MagicMock(return_value=self.otaclient_wrapper), ) mocker.patch( - f"{cfg.OTACLIENT_STUB_MODULE_PATH}._ECUTracker", + f"{test_cfg.OTACLIENT_STUB_MODULE_PATH}._ECUTracker", mocker.MagicMock(return_value=self.ecu_status_tracker), ) mocker.patch( - f"{cfg.OTACLIENT_STUB_MODULE_PATH}.OTAProxyLauncher", + f"{test_cfg.OTACLIENT_STUB_MODULE_PATH}.OTAProxyLauncher", mocker.MagicMock(return_value=self.otaproxy_launcher), ) mocker.patch( - f"{cfg.OTACLIENT_STUB_MODULE_PATH}.OtaClientCall", self.otaclient_call + f"{test_cfg.OTACLIENT_STUB_MODULE_PATH}.OtaClientCall", self.otaclient_call ) # --- start the OTAClientServiceStub --- # diff --git a/tests/test_ota_proxy/test_ota_proxy_server.py b/tests/test_ota_proxy/test_ota_proxy_server.py index 18b297616..a05d65a25 100644 --- a/tests/test_ota_proxy/test_ota_proxy_server.py +++ b/tests/test_ota_proxy/test_ota_proxy_server.py @@ -29,7 +29,7 @@ from otaclient.app.proto.wrapper import RegularInf from otaclient.app.ota_metadata import parse_regulars_from_txt from otaclient.ota_proxy.utils import url_based_hash -from tests.conftest import ThreadpoolExecutorFixtureMixin, cfg +from tests.conftest import ThreadpoolExecutorFixtureMixin, test_cfg logger = logging.getLogger(__name__) @@ -37,8 +37,8 @@ SPECIAL_FILE_NAME = r"path;adf.ae?qu.er\y=str#fragファイルement" SPECIAL_FILE_CONTENT = SPECIAL_FILE_NAME SPECIAL_FILE_PATH = f"/data/{SPECIAL_FILE_NAME}" -SPECIAL_FILE_URL = f"{cfg.OTA_IMAGE_URL}{quote(SPECIAL_FILE_PATH)}" -SPECIAL_FILE_FPATH = f"{cfg.OTA_IMAGE_DIR}/data/{SPECIAL_FILE_NAME}" +SPECIAL_FILE_URL = f"{test_cfg.OTA_IMAGE_URL}{quote(SPECIAL_FILE_PATH)}" +SPECIAL_FILE_FPATH = f"{test_cfg.OTA_IMAGE_DIR}/data/{SPECIAL_FILE_NAME}" SPECIAL_FILE_SHA256HASH = sha256(SPECIAL_FILE_CONTENT.encode()).hexdigest() @@ -54,10 +54,14 @@ async def _start_uvicorn_server(server: uvicorn.Server): class TestOTAProxyServer(ThreadpoolExecutorFixtureMixin): - THTREADPOOL_EXECUTOR_PATCH_PATH = f"{cfg.OTAPROXY_MODULE_PATH}.otacache" - OTA_IMAGE_URL = f"http://{cfg.OTA_IMAGE_SERVER_ADDR}:{cfg.OTA_IMAGE_SERVER_PORT}" - OTA_PROXY_URL = f"http://{cfg.OTA_PROXY_SERVER_ADDR}:{cfg.OTA_PROXY_SERVER_PORT}" - REGULARS_TXT_PATH = f"{cfg.OTA_IMAGE_DIR}/regulars.txt" + THTREADPOOL_EXECUTOR_PATCH_PATH = f"{test_cfg.OTAPROXY_MODULE_PATH}.otacache" + OTA_IMAGE_URL = ( + f"http://{test_cfg.OTA_IMAGE_SERVER_ADDR}:{test_cfg.OTA_IMAGE_SERVER_PORT}" + ) + OTA_PROXY_URL = ( + f"http://{test_cfg.OTA_PROXY_SERVER_ADDR}:{test_cfg.OTA_PROXY_SERVER_PORT}" + ) + REGULARS_TXT_PATH = f"{test_cfg.OTA_IMAGE_DIR}/regulars.txt" CLIENTS_NUM = 6 @pytest.fixture( @@ -128,8 +132,8 @@ def _mocked_background_check_freespace(self): ) _config = uvicorn.Config( App(_ota_cache), - host=cfg.OTA_PROXY_SERVER_ADDR, - port=cfg.OTA_PROXY_SERVER_PORT, + host=test_cfg.OTA_PROXY_SERVER_ADDR, + port=test_cfg.OTA_PROXY_SERVER_PORT, log_level="error", lifespan="on", loop="asyncio", @@ -225,7 +229,7 @@ async def ota_image_downloader(self, regular_entries, sync_event: asyncio.Event) await asyncio.sleep(random.randrange(100, 200) // 100) for entry in regular_entries: url = urljoin( - cfg.OTA_IMAGE_URL, quote(f'/data/{entry.relative_to("/")}') + test_cfg.OTA_IMAGE_URL, quote(f'/data/{entry.relative_to("/")}') ) _retry_count_for_exceed_hard_limit = 0 @@ -282,10 +286,14 @@ async def test_multiple_clients_download_ota_image(self, parse_regulars): class TestOTAProxyServerWithoutCache(ThreadpoolExecutorFixtureMixin): - THTREADPOOL_EXECUTOR_PATCH_PATH = f"{cfg.OTAPROXY_MODULE_PATH}.otacache" - OTA_IMAGE_URL = f"http://{cfg.OTA_IMAGE_SERVER_ADDR}:{cfg.OTA_IMAGE_SERVER_PORT}" - OTA_PROXY_URL = f"http://{cfg.OTA_PROXY_SERVER_ADDR}:{cfg.OTA_PROXY_SERVER_PORT}" - REGULARS_TXT_PATH = f"{cfg.OTA_IMAGE_DIR}/regulars.txt" + THTREADPOOL_EXECUTOR_PATCH_PATH = f"{test_cfg.OTAPROXY_MODULE_PATH}.otacache" + OTA_IMAGE_URL = ( + f"http://{test_cfg.OTA_IMAGE_SERVER_ADDR}:{test_cfg.OTA_IMAGE_SERVER_PORT}" + ) + OTA_PROXY_URL = ( + f"http://{test_cfg.OTA_PROXY_SERVER_ADDR}:{test_cfg.OTA_PROXY_SERVER_PORT}" + ) + REGULARS_TXT_PATH = f"{test_cfg.OTA_IMAGE_DIR}/regulars.txt" CLIENTS_NUM = 3 @pytest.fixture(autouse=True) @@ -310,8 +318,8 @@ async def setup_ota_proxy_server(self, tmp_path: Path): ) _config = uvicorn.Config( App(_ota_cache), - host=cfg.OTA_PROXY_SERVER_ADDR, - port=cfg.OTA_PROXY_SERVER_PORT, + host=test_cfg.OTA_PROXY_SERVER_ADDR, + port=test_cfg.OTA_PROXY_SERVER_PORT, log_level="error", lifespan="on", loop="asyncio", @@ -348,7 +356,7 @@ async def ota_image_downloader(self, regular_entries, sync_event: asyncio.Event) await asyncio.sleep(random.randrange(100, 200) // 100) for entry in regular_entries: url = urljoin( - cfg.OTA_IMAGE_URL, quote(f'/data/{entry.relative_to("/")}') + test_cfg.OTA_IMAGE_URL, quote(f'/data/{entry.relative_to("/")}') ) async with session.get( url, diff --git a/tests/utils.py b/tests/utils.py index 9d06850f5..3086e7f64 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -84,7 +84,7 @@ def _dummy_logger(*args, **kwargs): def compare_dir(left: Path, right: Path): _a_glob = set(map(lambda x: x.relative_to(left), left.glob("**/*"))) _b_glob = set(map(lambda x: x.relative_to(right), right.glob("**/*"))) - if not _a_glob == _b_glob: # first check paths are identical + if _a_glob != _b_glob: # first check paths are identical raise ValueError( f"left and right mismatch, diff: {_a_glob.symmetric_difference(_b_glob)}\n" f"{_a_glob=}\n"