From f2700163a428bccf8d6964891cb085e03d22236f Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Thu, 18 Apr 2024 02:46:07 +0000 Subject: [PATCH 001/193] implement jetson-common --- otaclient/app/boot_control/_jetson_common.py | 240 +++++++++++++++++++ 1 file changed, 240 insertions(+) create mode 100644 otaclient/app/boot_control/_jetson_common.py diff --git a/otaclient/app/boot_control/_jetson_common.py b/otaclient/app/boot_control/_jetson_common.py new file mode 100644 index 000000000..cc968b68e --- /dev/null +++ b/otaclient/app/boot_control/_jetson_common.py @@ -0,0 +1,240 @@ +# Copyright 2022 TIER IV, INC. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Jetson device boot control implementation common.""" + + +from __future__ import annotations +import logging +import re +import subprocess +from pathlib import Path +from typing import Any, NamedTuple, Optional + +from pydantic import BaseModel, BeforeValidator, PlainSerializer +from typing_extensions import Annotated, Literal, Self + +from otaclient.app.common import write_str_to_file_sync + +logger = logging.getLogger(__name__) + + +class SlotID(str): + """slot_id for A/B slots. + + On NVIDIA Jetson device, slot_a has slot_id=0, slot_b has slot_id=1. + For slot_a, the slot partition name suffix is "" or "_a". + For slot_b, the slot partition name suffix is "_b". + """ + + VALID_SLOTS = ["0", "1"] + + def __new__(cls, _in: str | Self) -> Self: + if isinstance(_in, cls): + return _in + if _in in cls.VALID_SLOTS: + return str.__new__(cls, _in) + raise ValueError(f"{_in=} is not valid slot num, should be '0' or '1'.") + + +class BSPVersion(NamedTuple): + """BSP version in NamedTuple representation. + + Example: R32.6.1 -> (32, 6, 1) + """ + + major_ver: int + major_rev: int + minor_rev: int + + @classmethod + def parse(cls, _in: str | BSPVersion | Any) -> Self: + """Parse "Rxx.yy.z string into BSPVersion.""" + if isinstance(_in, cls): + return _in + if isinstance(_in, str): + major_ver, major_rev, minor_rev = _in[1:].split(".") + return cls(int(major_ver), int(major_rev), int(minor_rev)) + raise ValueError(f"expect str or BSPVersion instance, get {type(_in)}") + + @staticmethod + def dump(to_export: BSPVersion) -> str: + """Dump BSPVersion to string as "Rxx.yy.z".""" + return f"R{to_export.major_ver}.{to_export.major_rev}.{to_export.minor_rev}" + + +class FirmwareBSPVersion(BaseModel): + """ + BSP version string schema: Rxx.yy.z + """ + + BSPVersionStr = Annotated[ + BSPVersion, + BeforeValidator(BSPVersion.parse), + PlainSerializer(BSPVersion.dump, return_type=str), + ] + """BSPVersion in string representation, used by FirmwareBSPVersion model.""" + + slot_a: Optional[BSPVersionStr] = None + slot_b: Optional[BSPVersionStr] = None + + +class NVBootctrlCommon: + """Helper for calling nvbootctrl commands. + + Without -t option, the target will be bootloader by default. + + The NVBootctrlCommon class only contains methods that both exist on + jetson-cboot and jetson-uefi, which are: + + 1. get-current-slot + 2. set-active-boot-slot + 3. dump-slots-info + + Also, get-standby-slot is not a nvbootctrl and it is implemented using + nvbootctrl get-current-slot command. + """ + + NVBOOTCTRL = "nvbootctrl" + NVBootctrlTarget = Literal["bootloader", "rootfs"] + + @classmethod + def _nvbootctrl( + cls, + _cmd: str, + _slot_id: Optional[SlotID] = None, + *, + check_output, + target: Optional[NVBootctrlTarget] = None, + ) -> Any: + cmd = [cls.NVBOOTCTRL] + if target: + cmd.extend(["-t", target]) + cmd.append(_cmd) + if _slot_id: + cmd.append(str(_slot_id)) + + res = subprocess.run( + cmd, + check=True, + capture_output=True, + ) + if check_output: + return res.stdout.decode() + return + + @classmethod + def get_current_slot(cls, *, target: Optional[NVBootctrlTarget] = None) -> SlotID: + """Prints currently running SLOT.""" + cmd = "get-current-slot" + res = cls._nvbootctrl(cmd, check_output=True, target=target) + assert isinstance(res, str), f"invalid output from get-current-slot: {res}" + return SlotID(res.strip()) + + @classmethod + def get_standby_slot(cls, *, target: Optional[NVBootctrlTarget] = None) -> SlotID: + """Prints standby SLOT. + + NOTE: this method is implemented with nvbootctrl get-current-slot. + """ + current_slot = cls.get_current_slot(target=target) + return SlotID("0") if current_slot == "1" else SlotID("1") + + @classmethod + def set_active_boot_slot( + cls, slot_id: SlotID, *, target: Optional[NVBootctrlTarget] = None + ) -> None: + """On next boot, load and execute SLOT.""" + cmd = "set-active-boot-slot" + return cls._nvbootctrl(cmd, SlotID(slot_id), check_output=False, target=target) + + @classmethod + def dump_slots_info(cls, *, target: Optional[NVBootctrlTarget] = None) -> str: + """Prints info for slots.""" + cmd = "dump-slots-info" + return cls._nvbootctrl(cmd, target=target, check_output=True) + + +class FirmwareBSPVersionControl: + """firmware_bsp_version ota-status file for tracking firmware version. + + The firmware BSP version is stored in /boot/ota-status/firmware_bsp_version json file, + tracking the firmware BSP version for each slot. + + Each slot should keep the same firmware_bsp_version file, this file is passed to standby slot + during OTA update. + """ + + def __init__( + self, current_firmware_bsp_vf: Path, standby_firmware_bsp_vf: Path + ) -> None: + self._current_fw_bsp_vf = current_firmware_bsp_vf + self._standby_fw_bsp_vf = standby_firmware_bsp_vf + + self._version = FirmwareBSPVersion() + try: + self._version = _version = FirmwareBSPVersion.model_validate_json( + self._current_fw_bsp_vf.read_text() + ) + logger.info(f"firmware_version: {_version}") + except Exception as e: + logger.warning( + f"invalid or missing firmware_bsp_verion file, removed: {e!r}" + ) + self._current_fw_bsp_vf.unlink(missing_ok=True) + + def write_current_firmware_bsp_version(self) -> None: + """Write firmware_bsp_version from memory to firmware_bsp_version file.""" + write_str_to_file_sync(self._current_fw_bsp_vf, self._version.model_dump_json()) + + def write_standby_firmware_bsp_version(self) -> None: + """Write firmware_bsp_version from memory to firmware_bsp_version file.""" + write_str_to_file_sync(self._standby_fw_bsp_vf, self._version.model_dump_json()) + + def get_version_by_slot(self, slot_id: SlotID) -> Optional[BSPVersion]: + """Get slot's firmware version from memory.""" + if slot_id == "0": + return self._version.slot_a + return self._version.slot_b + + def set_version_by_slot( + self, slot_id: SlotID, version: Optional[BSPVersion] + ) -> None: + """Set slot's firmware version into memory.""" + if slot_id == "0": + self._version.slot_a = version + else: + self._version.slot_b = version + + +BSP_VER_PA = re.compile( + ( + r"# R(?P\d+) \(\w+\), REVISION: (?P\d+)\.(?P\d+), " + r"GCID: (?P\d+), BOARD: (?P\w+), EABI: (?P\w+)" + ) +) +"""Example: # R32 (release), REVISION: 6.1, GCID: 27863751, BOARD: t186ref, EABI: aarch64, DATE: Mon Jul 26 19:36:31 UTC 2021 """ + + +def parse_bsp_version(nv_tegra_release: str) -> BSPVersion: + """Parse BSP version from contents of /etc/nv_tegra_release. + + see https://developer.nvidia.com/embedded/jetson-linux-archive for BSP version history. + """ + ma = BSP_VER_PA.match(nv_tegra_release) + assert ma, f"invalid nv_tegra_release content: {nv_tegra_release}" + return BSPVersion( + int(ma.group("major_ver")), + int(ma.group("major_rev")), + int(ma.group("minor_rev")), + ) From 063416e2b4dceeb2f56f4bbb56c42bc35ff15329 Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Thu, 18 Apr 2024 02:50:43 +0000 Subject: [PATCH 002/193] strip jetson-common from jetson-cboot --- otaclient/app/boot_control/_jetson_cboot.py | 190 ++----------------- otaclient/app/boot_control/_jetson_common.py | 5 +- 2 files changed, 16 insertions(+), 179 deletions(-) diff --git a/otaclient/app/boot_control/_jetson_cboot.py b/otaclient/app/boot_control/_jetson_cboot.py index dbbe8dc42..5b4f5d8d5 100644 --- a/otaclient/app/boot_control/_jetson_cboot.py +++ b/otaclient/app/boot_control/_jetson_cboot.py @@ -22,10 +22,8 @@ import subprocess from functools import partial from pathlib import Path -from typing import Any, Generator, Literal, NamedTuple, Optional - -from pydantic import BaseModel, BeforeValidator, PlainSerializer -from typing_extensions import Annotated, Self +from subprocess import CompletedProcess, run, CalledProcessError +from typing import Generator, Literal, Optional from otaclient.app import errors as ota_errors from otaclient.app.common import ( @@ -37,68 +35,22 @@ from ._common import CMDHelperFuncs, OTAStatusFilesControl, SlotMountHelper from .configs import cboot_cfg as cfg +from ._jetson_common import ( + FirmwareBSPVersionControl, + NVBootctrlCommon, + SlotID, + parse_bsp_version, +) from .protocol import BootControllerProtocol logger = logging.getLogger(__name__) -class SlotID(str): - VALID_SLOTS = ["0", "1"] - - def __new__(cls, _in: str | Self) -> Self: - if isinstance(_in, cls): - return _in - if _in in cls.VALID_SLOTS: - return str.__new__(cls, _in) - raise ValueError(f"{_in=} is not valid slot num, should be '0' or '1'.") - - -class BSPVersion(NamedTuple): - """ - example version string: R32.6.1 - """ - - major_ver: int - major_rev: int - minor_rev: int - - @classmethod - def parse(cls, _in: str | BSPVersion | Any) -> Self: - """Parse "Rxx.yy.z string into BSPVersion.""" - if isinstance(_in, cls): - return _in - if isinstance(_in, str): - major_ver, major_rev, minor_rev = _in[1:].split(".") - return cls(int(major_ver), int(major_rev), int(minor_rev)) - raise ValueError(f"expect str or BSPVersion instance, get {type(_in)}") - - @staticmethod - def dump(to_export: BSPVersion) -> str: - """Dump BSPVersion to string as "Rxx.yy.z".""" - return f"R{to_export.major_ver}.{to_export.major_rev}.{to_export.minor_rev}" - - -BSPVersionStr = Annotated[ - BSPVersion, - BeforeValidator(BSPVersion.parse), - PlainSerializer(BSPVersion.dump, return_type=str), -] - - -class FirmwareBSPVersion(BaseModel): - """ - BSP version string schema: Rxx.yy.z - """ - - slot_a: Optional[BSPVersionStr] = None - slot_b: Optional[BSPVersionStr] = None - - class JetsonCBootContrlError(Exception): """Exception types for covering jetson-cboot related errors.""" -class _NVBootctrl: +class _NVBootctrl(NVBootctrlCommon): """Helper for calling nvbootctrl commands. Without -t option, the target will be bootloader by default. @@ -107,62 +59,13 @@ class _NVBootctrl: NVBOOTCTRL = "nvbootctrl" NVBootctrlTarget = Literal["bootloader", "rootfs"] - @classmethod - def _nvbootctrl( - cls, - _cmd: str, - _slot_id: Optional[SlotID] = None, - *, - check_output=False, - target: Optional[NVBootctrlTarget] = None, - ) -> Any: - cmd = [cls.NVBOOTCTRL] - if target: - cmd.extend(["-t", target]) - cmd.append(_cmd) - if _slot_id: - cmd.append(str(_slot_id)) - - res = subprocess_run_wrapper( - cmd, - check=True, - check_output=True, - ) - if check_output: - return res.stdout.decode().strip() - - @classmethod - def get_current_slot(cls, *, target: Optional[NVBootctrlTarget] = None) -> SlotID: - """Prints currently running SLOT.""" - cmd = "get-current-slot" - res = cls._nvbootctrl(cmd, check_output=True, target=target) - assert isinstance(res, str), f"invalid output from get-current-slot: {res}" - return SlotID(res.strip()) - @classmethod def mark_boot_successful( cls, slot_id: SlotID, *, target: Optional[NVBootctrlTarget] = None ) -> None: """Mark current slot as GOOD.""" cmd = "mark-boot-successful" - cls._nvbootctrl(cmd, slot_id, target=target) - - @classmethod - def get_standby_slot(cls, *, target: Optional[NVBootctrlTarget] = None) -> SlotID: - """Prints standby SLOT. - - NOTE: this method is implemented with nvbootctrl get-current-slot. - """ - current_slot = cls.get_current_slot(target=target) - return SlotID("0") if current_slot == "1" else SlotID("1") - - @classmethod - def set_active_boot_slot( - cls, slot_id: SlotID, *, target: Optional[NVBootctrlTarget] = None - ) -> None: - """On next boot, load and execute SLOT.""" - cmd = "set-active-boot-slot" - return cls._nvbootctrl(cmd, SlotID(slot_id), target=target) + cls._nvbootctrl(cmd, slot_id, check_output=False, target=target) @classmethod def set_slot_as_unbootable( @@ -170,13 +73,7 @@ def set_slot_as_unbootable( ) -> None: """Mark SLOT as invalid.""" cmd = "set-slot-as-unbootable" - return cls._nvbootctrl(cmd, SlotID(slot_id), target=target) - - @classmethod - def dump_slots_info(cls, *, target: Optional[NVBootctrlTarget] = None) -> str: - """Prints info for slots.""" - cmd = "dump-slots-info" - return cls._nvbootctrl(cmd, target=target, check_output=True) + return cls._nvbootctrl(cmd, SlotID(slot_id), check_output=False, target=target) @classmethod def is_unified_enabled(cls) -> bool | None: @@ -196,7 +93,7 @@ def is_unified_enabled(cls) -> bool | None: """ cmd = "is-unified-enabled" try: - cls._nvbootctrl(cmd) + cls._nvbootctrl(cmd, check_output=False) return True except subprocess.CalledProcessError as e: if e.returncode == 70: @@ -270,69 +167,6 @@ def verify_update(cls) -> subprocess.CompletedProcess[bytes]: return subprocess_run_wrapper(cmd, check=False, check_output=True) -class FirmwareBSPVersionControl: - """firmware_bsp_version ota-status file for tracking firmware version.""" - - def __init__( - self, current_firmware_bsp_vf: Path, standby_firmware_bsp_vf: Path - ) -> None: - self._current_fw_bsp_vf = current_firmware_bsp_vf - self._standby_fw_bsp_vf = standby_firmware_bsp_vf - - self._version = FirmwareBSPVersion() - try: - self._version = FirmwareBSPVersion.model_validate_json( - self._current_fw_bsp_vf.read_text() - ) - except Exception as e: - logger.warning( - f"invalid or missing firmware_bsp_verion file, removed: {e!r}" - ) - self._current_fw_bsp_vf.unlink(missing_ok=True) - - def write_current_firmware_bsp_version(self) -> None: - """Write instance firmware_bsp_version to firmware_bsp_version file.""" - write_str_to_file_sync(self._current_fw_bsp_vf, self._version.model_dump_json()) - - def write_standby_firmware_bsp_version(self) -> None: - """Write instance firmware_bsp_version to firmware_bsp_version file.""" - write_str_to_file_sync(self._standby_fw_bsp_vf, self._version.model_dump_json()) - - def get_version_by_slot(self, slot_id: SlotID) -> Optional[BSPVersion]: - if slot_id == "0": - return self._version.slot_a - return self._version.slot_b - - def set_version_by_slot(self, slot_id: SlotID, version: Optional[BSPVersion]): - if slot_id == "0": - self._version.slot_a = version - else: - self._version.slot_b = version - - -BSP_VER_PA = re.compile( - ( - r"# R(?P\d+) \(\w+\), REVISION: (?P\d+)\.(?P\d+), " - r"GCID: (?P\d+), BOARD: (?P\w+), EABI: (?P\w+)" - ) -) -"""Example: # R32 (release), REVISION: 6.1, GCID: 27863751, BOARD: t186ref, EABI: aarch64, DATE: Mon Jul 26 19:36:31 UTC 2021 """ - - -def parse_bsp_version(nv_tegra_release: str) -> BSPVersion: - """Get current BSP version from contents of /etc/nv_tegra_release. - - see https://developer.nvidia.com/embedded/jetson-linux-archive for BSP version history. - """ - ma = BSP_VER_PA.match(nv_tegra_release) - assert ma, f"invalid nv_tegra_release content: {nv_tegra_release}" - return BSPVersion( - int(ma.group("major_ver")), - int(ma.group("major_rev")), - int(ma.group("minor_rev")), - ) - - class _CBootControl: MMCBLK_DEV_PREFIX = "mmcblk" # internal emmc NVMESSD_DEV_PREFIX = "nvme" # external nvme ssd diff --git a/otaclient/app/boot_control/_jetson_common.py b/otaclient/app/boot_control/_jetson_common.py index cc968b68e..8bcd302c9 100644 --- a/otaclient/app/boot_control/_jetson_common.py +++ b/otaclient/app/boot_control/_jetson_common.py @@ -11,7 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Jetson device boot control implementation common.""" +"""Jetson device boot control implementation common. + +This module is shared by jetson-cboot and jetson-uefi bootloader type. +""" from __future__ import annotations From 2327ae8569b08d8f6a4c98e52a3fda393f1f9804 Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Thu, 18 Apr 2024 08:32:36 +0000 Subject: [PATCH 003/193] implement jetson-uefi boot --- otaclient/app/boot_control/_jetson_cboot.py | 6 +- otaclient/app/boot_control/_jetson_uefi.py | 572 ++++++++++++++++++++ 2 files changed, 577 insertions(+), 1 deletion(-) create mode 100644 otaclient/app/boot_control/_jetson_uefi.py diff --git a/otaclient/app/boot_control/_jetson_cboot.py b/otaclient/app/boot_control/_jetson_cboot.py index 5b4f5d8d5..f2b01cfee 100644 --- a/otaclient/app/boot_control/_jetson_cboot.py +++ b/otaclient/app/boot_control/_jetson_cboot.py @@ -11,7 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Boot control implementation for NVIDIA Jetson device boot with cboot.""" +"""Boot control implementation for NVIDIA Jetson device boots with cboot. + +Supports BSP version < R34. +""" from __future__ import annotations @@ -53,6 +56,7 @@ class JetsonCBootContrlError(Exception): class _NVBootctrl(NVBootctrlCommon): """Helper for calling nvbootctrl commands. + For BSP version < R34. Without -t option, the target will be bootloader by default. """ diff --git a/otaclient/app/boot_control/_jetson_uefi.py b/otaclient/app/boot_control/_jetson_uefi.py new file mode 100644 index 000000000..bf094eed2 --- /dev/null +++ b/otaclient/app/boot_control/_jetson_uefi.py @@ -0,0 +1,572 @@ +# Copyright 2022 TIER IV, INC. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Boot control implementation for NVIDIA Jetson device boots with UEFI. + +jetson-uefi module currently support BSP version >= R35.2. +NOTE: R34~R35.1 is not supported as Capsule update is only available after R35.2. +""" + + +from __future__ import annotations +import logging +import os +import re +import shutil +from functools import partial +from pathlib import Path +from typing import Generator, Literal + +from otaclient.app import errors as ota_errors +from otaclient.app.common import copytree_identical, write_str_to_file_sync +from otaclient.app.proto import wrapper +from ._common import ( + OTAStatusFilesControl, + SlotMountHelper, + CMDHelperFuncs, +) +from .configs import cboot_cfg as cfg +from ._jetson_common import ( + FirmwareBSPVersionControl, + NVBootctrlCommon, + SlotID, + parse_bsp_version, +) +from .protocol import BootControllerProtocol + +logger = logging.getLogger(__name__) + + +class JetsonUEFIBootControlError(Exception): + """Exception type for covering jetson-uefi related errors.""" + + +class _NVBootctrl(NVBootctrlCommon): + """Helper for calling nvbootctrl commands. + + For BSP version >= R34 with UEFI boot. + Without -t option, the target will be bootloader by default. + """ + + NVBOOTCTRL = "nvbootctrl" + NVBootctrlTarget = Literal["bootloader", "rootfs"] + + @classmethod + def get_capsule_update_result(cls) -> str: + """Check the Capsule update status. + + NOTE: this is NOT a nvbootctrl command, but implemented by parsing + the result of calling nvbootctrl dump-slots-info. + + The output value of Capsule update status can be following: + 0 - No Capsule update + 1 - Capsule update successfully + 2 - Capsule install successfully but boot new firmware failed + 3 - Capsule install failed + + Returns: + The Capsulte update result status. + """ + slots_info = cls.dump_slots_info() + logger.info(f"checking Capsule update result: \n{slots_info}") + + pattern = re.compile(r"Capsule update status: (?P\d+)") + ma = pattern.search(slots_info) + assert ma, "failed to get Capsule update result" + + update_result = ma.group("status") + return update_result + + +class CapsuleUpdate: + """Firmware update implementation using Capsule update.""" + + CAPSULE_PAYLOAD_LOCATION = "EFI/UpdateCapsule" + MAGIC_BYTES = b"\x07\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00" + UPDATE_TRIGGER_EFIVAR = "OsIndications-8be4df61-93ca-11d2-aa0d-00e098032b8c" + EFIVARS_DPATH = "/sys/firmware/efi/efivars/" + EFIVARS_FSTYPE = "efivarfs" + ESP_PARTLABEL = "esp" + ESP_MOUNTPOINT = "/mnt/esp" + # TODO: move these options into uefi boot configs + CAPSULE_PAYLOAD_FNAME = "bl_only_payload.Cap" + + def __init__(self, boot_devpath: Path | str) -> None: + # NOTE: use the esp partition at the current booted device + # i.e., if we boot from nvme0n1, then bootdev_path is /dev/nvme0n1 and + # we use the esp at nvme0n1. + self.boot_devpath = Path(boot_devpath) + self.boot_devname = Path(boot_devpath).name + + self.esp_mp = self.ESP_MOUNTPOINT + + # if boots from external, expects to have multiple esp parts, we need to get the one at our booted dev + esp_parts = CMDHelperFuncs.get_dev_by_token( + token="PARTLABEL", value=self.ESP_PARTLABEL + ) + for _esp_part in esp_parts: + if _esp_part.find(self.boot_devname) != -1: + logger.info(f"find esp partition at {_esp_part}") + self.esp_part = _esp_part + break + else: + _err_msg = f"failed to find esp partition on {self.boot_devpath}" + logger.error(_err_msg) + raise JetsonUEFIBootControlError(_err_msg) + + @classmethod + def _ensure_efivarfs_mounted(cls) -> None: + if CMDHelperFuncs.is_target_mounted(cls.EFIVARS_DPATH): + return + + logger.warning( + f"efivars is not mounted! try to mount it at {cls.EFIVARS_DPATH}" + ) + try: + CMDHelperFuncs._mount( + cls.EFIVARS_FSTYPE, + cls.EFIVARS_DPATH, + fstype=cls.EFIVARS_FSTYPE, + options=["rw", "nosuid", "nodev", "noexec", "relatime"], + ) + except Exception as e: + raise JetsonUEFIBootControlError( + f"failed to mount {cls.EFIVARS_FSTYPE} on {cls.EFIVARS_DPATH}: {e!r}" + ) from e + + def _prepare_payload(self) -> None: + """Copy the Capsule update payload to the specified location.""" + try: + CMDHelperFuncs.mount_rw(self.esp_part, self.esp_mp) + except Exception as e: + raise JetsonUEFIBootControlError( + f"failed to mount {self.esp_part} onto {self.esp_mp}: {e!r}" + ) + + capsule_payload_fpath = Path(cfg.FIRMWARE_DPATH) / self.CAPSULE_PAYLOAD_FNAME + capsule_payload_location = Path(self.esp_mp) / self.CAPSULE_PAYLOAD_LOCATION + try: + shutil.copy(capsule_payload_fpath, capsule_payload_location) + except Exception as e: + _err_msg = f"failed to copy update Capsule from {capsule_payload_fpath} to {capsule_payload_location}: {e!r}" + logger.error(_err_msg) + raise JetsonUEFIBootControlError(_err_msg) from e + finally: + CMDHelperFuncs.umount(self.esp_mp, ignore_error=True) + + def _write_efivar(self) -> None: + """Write magic efivar to trigger firmware Capsule update in next boot.""" + magic_efivar_fpath = Path(self.EFIVARS_DPATH) / self.UPDATE_TRIGGER_EFIVAR + try: + with open(magic_efivar_fpath, "rb") as f: + f.write(self.MAGIC_BYTES) + os.fsync(f.fileno()) + except Exception as e: + _err_msg = f"failed to write magic bytes into {magic_efivar_fpath}: {e!r}" + logger.error(_err_msg) + raise JetsonUEFIBootControlError(_err_msg) from e + + def firmware_update(self) -> None: + """Trigger firmware update in next boot.""" + logger.info("prepare for firmware Capsule update ...") + self._ensure_efivarfs_mounted() + self._prepare_payload() + self._write_efivar() + logger.info( + "firmware Capsule update is configured and will be triggerred in next boot" + ) + + +class _UEFIBoot: + """Low-level boot control implementation for jetson-uefi.""" + + MMCBLK_DEV_PREFIX = "mmcblk" # internal emmc + NVMESSD_DEV_PREFIX = "nvme" # external nvme ssd + INTERNAL_EMMC_DEVNAME = "mmcblk0" + _slot_id_partid = {SlotID("0"): "1", SlotID("1"): "2"} + + def __init__(self): + # ------ sanity check, confirm we are at jetson device ------ # + if not os.path.exists(cfg.TEGRA_CHIP_ID_PATH): + _err_msg = f"not a jetson device, {cfg.TEGRA_CHIP_ID_PATH} doesn't exist" + logger.error(_err_msg) + raise JetsonUEFIBootControlError(_err_msg) + + # ------ check BSP version ------ # + try: + self.bsp_version = bsp_version = parse_bsp_version( + Path(cfg.NV_TEGRA_RELEASE_FPATH).read_text() + ) + except Exception as e: + _err_msg = f"failed to detect BSP version: {e!r}" + logger.error(_err_msg) + raise JetsonUEFIBootControlError(_err_msg) + logger.info(f"{bsp_version=}") + + # ------ sanity check, currently jetson-uefi only supports >= R35.2 ----- # + if not bsp_version >= (35, 2, 0): + _err_msg = f"jetson-uefi only supports BSP version >= R35.2, but get {bsp_version=}. " + logger.error(_err_msg) + raise JetsonUEFIBootControlError(_err_msg) + + # NOTE: unified A/B is enabled by default and cannot be disabled after BSP R34. + logger.info("unified A/B is enabled") + + # ------ check A/B slots ------ # + self.current_slot = current_slot = _NVBootctrl.get_current_slot() + self.standby_slot = standby_slot = _NVBootctrl.get_standby_slot() + logger.info(f"{current_slot=}, {standby_slot=}") + + # ------ detect rootfs_dev and parent_dev ------ # + self.curent_rootfs_devpath = current_rootfs_devpath = ( + CMDHelperFuncs.get_current_rootfs_dev().strip() + ) + self.parent_devpath = parent_devpath = Path( + CMDHelperFuncs.get_parent_dev(current_rootfs_devpath).strip() + ) + + self._external_rootfs = False + parent_devname = parent_devpath.name + if parent_devname.startswith(self.MMCBLK_DEV_PREFIX): + logger.info(f"device boots from internal emmc: {parent_devpath}") + elif parent_devname.startswith(self.NVMESSD_DEV_PREFIX): + logger.info(f"device boots from external nvme ssd: {parent_devpath}") + self._external_rootfs = True + else: + _err_msg = f"we don't support boot from {parent_devpath=} currently" + logger.error(_err_msg) + raise JetsonUEFIBootControlError(_err_msg) from NotImplementedError( + f"unsupported bootdev {parent_devpath}" + ) + + # rootfs partition + self.standby_rootfs_devpath = ( + f"/dev/{parent_devname}p{self._slot_id_partid[standby_slot]}" + ) + self.standby_rootfs_dev_partuuid = CMDHelperFuncs.get_partuuid_by_dev( + f"{self.standby_rootfs_devpath}" + ).strip() + current_rootfs_dev_partuuid = CMDHelperFuncs.get_partuuid_by_dev( + current_rootfs_devpath + ).strip() + + logger.info( + "rootfs device: \n" + f"active_rootfs(slot {current_slot}): {self.curent_rootfs_devpath=}, {current_rootfs_dev_partuuid=}\n" + f"standby_rootfs(slot {standby_slot}): {self.standby_rootfs_devpath=}, {self.standby_rootfs_dev_partuuid=}" + ) + + # internal emmc partition + self.standby_internal_emmc_devpath = ( + f"/dev/{self.INTERNAL_EMMC_DEVNAME}p{self._slot_id_partid[standby_slot]}" + ) + + logger.info("finished jetson-uefi boot control startup") + logger.info(f"nvbootctrl dump-slots-info: \n{_NVBootctrl.dump_slots_info()}") + + # API + + @property + def external_rootfs_enabled(self) -> bool: + """Indicate whether rootfs on external storage is enabled. + + NOTE: distiguish from boot from external storage, as R32.5 and below doesn't + support native NVMe boot. + """ + return self._external_rootfs + + def switch_boot_to_standby(self) -> None: + target_slot = self.standby_slot + + logger.info(f"switch boot to standby slot({target_slot})") + # when unified_ab enabled, switching bootloader slot will also switch + # the rootfs slot. + _NVBootctrl.set_active_boot_slot(target_slot) + + def prepare_standby_dev(self, *, erase_standby: bool): + CMDHelperFuncs.umount(self.standby_rootfs_devpath, ignore_error=True) + + if erase_standby: + try: + CMDHelperFuncs.mkfs_ext4(self.standby_rootfs_devpath) + except Exception as e: + _err_msg = f"failed to mkfs.ext4 on standby dev: {e!r}" + logger.error(_err_msg) + raise JetsonUEFIBootControlError(_err_msg) from e + # TODO: in the future if in-place update mode is implemented, do a + # fschck over the standby slot file system. + + @staticmethod + def update_extlinux_cfg(_input: str, partuuid: str) -> str: + """Update input exlinux text with input rootfs .""" + + partuuid_str = f"PARTUUID={partuuid}" + + def _replace(ma: re.Match, repl: str): + append_l: str = ma.group(0) + if append_l.startswith("#"): + return append_l + res, n = re.compile(r"root=[\w\-=]*").subn(repl, append_l) + if not n: # this APPEND line doesn't contain root= placeholder + res = f"{append_l} {repl}" + + return res + + _repl_func = partial(_replace, repl=f"root={partuuid_str}") + return re.compile(r"\n\s*APPEND.*").sub(_repl_func, _input) + + +class JetsonUEFIBootControl(BootControllerProtocol): + """BootControllerProtocol implementation for jetson-uefi.""" + + def __init__(self) -> None: + try: + # startup boot controller + self._uefi_control = _UEFIBoot() + + # mount point prepare + self._mp_control = SlotMountHelper( + standby_slot_dev=self._uefi_control.standby_rootfs_devpath, + standby_slot_mount_point=cfg.MOUNT_POINT, + active_slot_dev=self._uefi_control.curent_rootfs_devpath, + active_slot_mount_point=cfg.ACTIVE_ROOT_MOUNT_POINT, + ) + current_ota_status_dir = Path(cfg.OTA_STATUS_DIR) + standby_ota_status_dir = self._mp_control.standby_slot_mount_point / Path( + cfg.OTA_STATUS_DIR + ).relative_to("/") + + # load firmware BSP version from current rootfs slot + self._firmware_ver_control = FirmwareBSPVersionControl( + current_firmware_bsp_vf=current_ota_status_dir + / cfg.FIRMWARE_BSP_VERSION_FNAME, + standby_firmware_bsp_vf=standby_ota_status_dir + / cfg.FIRMWARE_BSP_VERSION_FNAME, + ) + + # init ota-status files + self._ota_status_control = OTAStatusFilesControl( + active_slot=str(self._uefi_control.current_slot), + standby_slot=str(self._uefi_control.standby_slot), + current_ota_status_dir=current_ota_status_dir, + # NOTE: might not yet be populated before OTA update applied! + standby_ota_status_dir=standby_ota_status_dir, + finalize_switching_boot=self._finalize_switching_boot, + ) + except Exception as e: + _err_msg = f"failed to start jetson-uefi controller: {e!r}" + raise ota_errors.BootControlStartupFailed(_err_msg, module=__name__) from e + + def _finalize_switching_boot(self) -> bool: + """Verify firmware update result and write firmware BSP version file. + + NOTE: due to unified A/B is enabled, actually it is impossible to boot to + a firmware updated failed slot. + Since finalize_switching_boot is only called when first reboot succeeds, + we can only observe result status 0 or 1 here. + """ + current_slot = self._uefi_control.current_slot + current_slot_bsp_ver = self._uefi_control.bsp_version + + try: + update_result_status = _NVBootctrl.get_capsule_update_result() + except Exception as e: + _err_msg = ( + f"failed to get the Capsule update result status, assume failed: {e!r}" + ) + logger.error(_err_msg) + return False + + if update_result_status == "0": + logger.info("no firmware update occurs") + return True + + if update_result_status == "1": + logger.info("firmware successfully updated") + self._firmware_ver_control.set_version_by_slot( + current_slot, current_slot_bsp_ver + ) + self._firmware_ver_control.write_current_firmware_bsp_version() + return True + + return False + + def _copy_standby_slot_boot_to_internal_emmc(self): + """Copy the standby slot's /boot to internal emmc dev. + + This method is involved when external rootfs is enabled, aligning with + the behavior of the NVIDIA flashing script. + + WARNING: DO NOT call this method if we are not booted from external rootfs! + NOTE: at the time this method is called, the /boot folder at + standby slot rootfs MUST be fully setup! + """ + # mount corresponding internal emmc device + internal_emmc_mp = Path(cfg.SEPARATE_BOOT_MOUNT_POINT) + internal_emmc_mp.mkdir(exist_ok=True, parents=True) + internal_emmc_devpath = self._uefi_control.standby_internal_emmc_devpath + + try: + CMDHelperFuncs.umount(internal_emmc_devpath, ignore_error=True) + CMDHelperFuncs.mount_rw(internal_emmc_devpath, internal_emmc_mp) + except Exception as e: + _msg = f"failed to mount standby internal emmc dev: {e!r}" + logger.error(_msg) + raise JetsonUEFIBootControlError(_msg) from e + + try: + dst = internal_emmc_mp / "boot" + src = self._mp_control.standby_slot_mount_point / "boot" + # copy the standby slot's boot folder to emmc boot dev + copytree_identical(src, dst) + except Exception as e: + _msg = f"failed to populate standby slot's /boot folder to standby internal emmc dev: {e!r}" + logger.error(_msg) + raise JetsonUEFIBootControlError(_msg) from e + finally: + CMDHelperFuncs.umount(internal_emmc_mp, ignore_error=True) + + def _preserve_ota_config_files_to_standby(self): + """Preserve /boot/ota to standby /boot folder.""" + src = self._mp_control.active_slot_mount_point / "boot" / "ota" + if not src.is_dir(): # basically this should not happen + logger.warning(f"{src} doesn't exist, skip preserve /boot/ota folder.") + return + + dst = self._mp_control.standby_slot_mount_point / "boot" / "ota" + # TODO: (20240411) reconsidering should we preserve /boot/ota? + copytree_identical(src, dst) + + def _update_standby_slot_extlinux_cfg(self): + """update standby slot's /boot/extlinux/extlinux.conf to update root indicator.""" + src = standby_slot_extlinux = self._mp_control.standby_slot_mount_point / Path( + cfg.EXTLINUX_FILE + ).relative_to("/") + # if standby slot doesn't have extlinux.conf installed, use current booted + # extlinux.conf as template source. + if not standby_slot_extlinux.is_file(): + src = Path(cfg.EXTLINUX_FILE) + + # update the extlinux.conf with standby slot rootfs' partuuid + write_str_to_file_sync( + standby_slot_extlinux, + self._uefi_control.update_extlinux_cfg( + src.read_text(), + self._uefi_control.standby_rootfs_dev_partuuid, + ), + ) + + # APIs + + def get_standby_slot_path(self) -> Path: + return self._mp_control.standby_slot_mount_point + + def get_standby_boot_dir(self) -> Path: + return self._mp_control.standby_boot_dir + + def pre_update(self, version: str, *, standby_as_ref: bool, erase_standby: bool): + try: + logger.info("jetson-uefi: pre-update ...") + # udpate active slot's ota_status + self._ota_status_control.pre_update_current() + + # prepare standby slot dev + self._uefi_control.prepare_standby_dev(erase_standby=erase_standby) + # mount slots + self._mp_control.mount_standby() + self._mp_control.mount_active() + + # update standby slot's ota_status files + self._ota_status_control.pre_update_standby(version=version) + except Exception as e: + _err_msg = f"failed on pre_update: {e!r}" + logger.error(_err_msg) + raise ota_errors.BootControlPreUpdateFailed( + _err_msg, module=__name__ + ) from e + + def post_update(self) -> Generator[None, None, None]: + try: + logger.info("jetson-uefi: post-update ...") + # ------ update extlinux.conf ------ # + self._update_standby_slot_extlinux_cfg() + + # ------ firmware update ------ # + firmware_updater = CapsuleUpdate(self._uefi_control.parent_devpath) + firmware_updater.firmware_update() + + # ------ preserve /boot/ota folder to standby rootfs ------ # + self._preserve_ota_config_files_to_standby() + + # ------ for external rootfs, preserve /boot folder to internal ------ # + if self._uefi_control._external_rootfs: + logger.info( + "rootfs on external storage enabled: " + "copy standby slot rootfs' /boot folder " + "to corresponding internal emmc dev ..." + ) + self._copy_standby_slot_boot_to_internal_emmc() + + # ------ switch boot to standby ------ # + self._uefi_control.switch_boot_to_standby() + + # ------ prepare to reboot ------ # + self._mp_control.umount_all(ignore_error=True) + logger.info(f"[post-update]: \n{_NVBootctrl.dump_slots_info()}") + logger.info("post update finished, wait for reboot ...") + yield # hand over control back to otaclient + CMDHelperFuncs.reboot() + except Exception as e: + _err_msg = f"jetson-uefi: failed on post_update: {e!r}" + logger.error(_err_msg) + raise ota_errors.BootControlPostUpdateFailed( + _err_msg, module=__name__ + ) from e + + def pre_rollback(self): + try: + logger.info("jetson-uefi: pre-rollback setup ...") + self._ota_status_control.pre_rollback_current() + self._mp_control.mount_standby() + self._ota_status_control.pre_rollback_standby() + except Exception as e: + _err_msg = f"jetson-uefi: failed on pre_rollback: {e!r}" + logger.error(_err_msg) + raise ota_errors.BootControlPreRollbackFailed( + _err_msg, module=__name__ + ) from e + + def post_rollback(self): + try: + logger.info("jetson-uefi: post-rollback setup...") + self._mp_control.umount_all(ignore_error=True) + self._uefi_control.switch_boot_to_standby() + CMDHelperFuncs.reboot() + except Exception as e: + _err_msg = f"jetson-uefi: failed on post_rollback: {e!r}" + logger.error(_err_msg) + raise ota_errors.BootControlPostRollbackFailed( + _err_msg, module=__name__ + ) from e + + def on_operation_failure(self): + """Failure registering and cleanup at failure.""" + logger.warning("on failure try to unmounting standby slot...") + self._ota_status_control.on_failure() + self._mp_control.umount_all(ignore_error=True) + + def load_version(self) -> str: + return self._ota_status_control.load_active_slot_version() + + def get_booted_ota_status(self) -> wrapper.StatusOta: + return self._ota_status_control.booted_ota_status From 4a3d19c47ffe65a0a6226741e3ee9f031c9ed36b Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Thu, 18 Apr 2024 09:18:16 +0000 Subject: [PATCH 004/193] jetson-boot: refine configs --- otaclient/app/boot_control/_jetson_uefi.py | 73 +++++++++++----------- otaclient/app/boot_control/configs.py | 42 ++++++++----- 2 files changed, 64 insertions(+), 51 deletions(-) diff --git a/otaclient/app/boot_control/_jetson_uefi.py b/otaclient/app/boot_control/_jetson_uefi.py index bf094eed2..9781ecaf2 100644 --- a/otaclient/app/boot_control/_jetson_uefi.py +++ b/otaclient/app/boot_control/_jetson_uefi.py @@ -25,9 +25,10 @@ import shutil from functools import partial from pathlib import Path -from typing import Generator, Literal +from typing import Generator from otaclient.app import errors as ota_errors +from otaclient.app.configs import config as cfg from otaclient.app.common import copytree_identical, write_str_to_file_sync from otaclient.app.proto import wrapper from ._common import ( @@ -35,7 +36,7 @@ SlotMountHelper, CMDHelperFuncs, ) -from .configs import cboot_cfg as cfg +from .configs import jetson_uefi_cfg as boot_cfg from ._jetson_common import ( FirmwareBSPVersionControl, NVBootctrlCommon, @@ -58,9 +59,6 @@ class _NVBootctrl(NVBootctrlCommon): Without -t option, the target will be bootloader by default. """ - NVBOOTCTRL = "nvbootctrl" - NVBootctrlTarget = Literal["bootloader", "rootfs"] - @classmethod def get_capsule_update_result(cls) -> str: """Check the Capsule update status. @@ -91,28 +89,23 @@ def get_capsule_update_result(cls) -> str: class CapsuleUpdate: """Firmware update implementation using Capsule update.""" - CAPSULE_PAYLOAD_LOCATION = "EFI/UpdateCapsule" - MAGIC_BYTES = b"\x07\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00" - UPDATE_TRIGGER_EFIVAR = "OsIndications-8be4df61-93ca-11d2-aa0d-00e098032b8c" - EFIVARS_DPATH = "/sys/firmware/efi/efivars/" EFIVARS_FSTYPE = "efivarfs" - ESP_PARTLABEL = "esp" - ESP_MOUNTPOINT = "/mnt/esp" - # TODO: move these options into uefi boot configs - CAPSULE_PAYLOAD_FNAME = "bl_only_payload.Cap" - def __init__(self, boot_devpath: Path | str) -> None: + def __init__(self, boot_devpath: Path | str, standby_slot_mp: Path | str) -> None: # NOTE: use the esp partition at the current booted device # i.e., if we boot from nvme0n1, then bootdev_path is /dev/nvme0n1 and # we use the esp at nvme0n1. self.boot_devpath = Path(boot_devpath) self.boot_devname = Path(boot_devpath).name - self.esp_mp = self.ESP_MOUNTPOINT + self.esp_mp = boot_cfg.ESP_MOUNTPOINT + + # NOTE: we get the update capsule from the standby slot + self.standby_slot_mp = Path(standby_slot_mp) # if boots from external, expects to have multiple esp parts, we need to get the one at our booted dev esp_parts = CMDHelperFuncs.get_dev_by_token( - token="PARTLABEL", value=self.ESP_PARTLABEL + token="PARTLABEL", value=boot_cfg.ESP_PARTLABEL ) for _esp_part in esp_parts: if _esp_part.find(self.boot_devname) != -1: @@ -126,22 +119,22 @@ def __init__(self, boot_devpath: Path | str) -> None: @classmethod def _ensure_efivarfs_mounted(cls) -> None: - if CMDHelperFuncs.is_target_mounted(cls.EFIVARS_DPATH): + if CMDHelperFuncs.is_target_mounted(boot_cfg.EFIVARS_DPATH): return logger.warning( - f"efivars is not mounted! try to mount it at {cls.EFIVARS_DPATH}" + f"efivars is not mounted! try to mount it at {boot_cfg.EFIVARS_DPATH}" ) try: CMDHelperFuncs._mount( cls.EFIVARS_FSTYPE, - cls.EFIVARS_DPATH, + boot_cfg.EFIVARS_DPATH, fstype=cls.EFIVARS_FSTYPE, options=["rw", "nosuid", "nodev", "noexec", "relatime"], ) except Exception as e: raise JetsonUEFIBootControlError( - f"failed to mount {cls.EFIVARS_FSTYPE} on {cls.EFIVARS_DPATH}: {e!r}" + f"failed to mount {cls.EFIVARS_FSTYPE} on {boot_cfg.EFIVARS_DPATH}: {e!r}" ) from e def _prepare_payload(self) -> None: @@ -153,10 +146,11 @@ def _prepare_payload(self) -> None: f"failed to mount {self.esp_part} onto {self.esp_mp}: {e!r}" ) - capsule_payload_fpath = Path(cfg.FIRMWARE_DPATH) / self.CAPSULE_PAYLOAD_FNAME - capsule_payload_location = Path(self.esp_mp) / self.CAPSULE_PAYLOAD_LOCATION + capsule_payload_location = Path(self.esp_mp) / boot_cfg.CAPSULE_PAYLOAD_LOCATION try: - shutil.copy(capsule_payload_fpath, capsule_payload_location) + for capsule_fname in boot_cfg.FIRMWARE_LIST: + capsule_payload_fpath = self.standby_slot_mp / capsule_fname + shutil.copy(capsule_payload_fpath, capsule_payload_location) except Exception as e: _err_msg = f"failed to copy update Capsule from {capsule_payload_fpath} to {capsule_payload_location}: {e!r}" logger.error(_err_msg) @@ -166,10 +160,12 @@ def _prepare_payload(self) -> None: def _write_efivar(self) -> None: """Write magic efivar to trigger firmware Capsule update in next boot.""" - magic_efivar_fpath = Path(self.EFIVARS_DPATH) / self.UPDATE_TRIGGER_EFIVAR + magic_efivar_fpath = ( + Path(boot_cfg.EFIVARS_DPATH) / boot_cfg.UPDATE_TRIGGER_EFIVAR + ) try: with open(magic_efivar_fpath, "rb") as f: - f.write(self.MAGIC_BYTES) + f.write(boot_cfg.MAGIC_BYTES) os.fsync(f.fileno()) except Exception as e: _err_msg = f"failed to write magic bytes into {magic_efivar_fpath}: {e!r}" @@ -197,15 +193,17 @@ class _UEFIBoot: def __init__(self): # ------ sanity check, confirm we are at jetson device ------ # - if not os.path.exists(cfg.TEGRA_CHIP_ID_PATH): - _err_msg = f"not a jetson device, {cfg.TEGRA_CHIP_ID_PATH} doesn't exist" + if not os.path.exists(boot_cfg.TEGRA_CHIP_ID_PATH): + _err_msg = ( + f"not a jetson device, {boot_cfg.TEGRA_CHIP_ID_PATH} doesn't exist" + ) logger.error(_err_msg) raise JetsonUEFIBootControlError(_err_msg) # ------ check BSP version ------ # try: self.bsp_version = bsp_version = parse_bsp_version( - Path(cfg.NV_TEGRA_RELEASE_FPATH).read_text() + Path(boot_cfg.NV_TEGRA_RELEASE_FPATH).read_text() ) except Exception as e: _err_msg = f"failed to detect BSP version: {e!r}" @@ -341,17 +339,17 @@ def __init__(self) -> None: active_slot_dev=self._uefi_control.curent_rootfs_devpath, active_slot_mount_point=cfg.ACTIVE_ROOT_MOUNT_POINT, ) - current_ota_status_dir = Path(cfg.OTA_STATUS_DIR) + current_ota_status_dir = Path(boot_cfg.OTA_STATUS_DIR) standby_ota_status_dir = self._mp_control.standby_slot_mount_point / Path( - cfg.OTA_STATUS_DIR + boot_cfg.OTA_STATUS_DIR ).relative_to("/") # load firmware BSP version from current rootfs slot self._firmware_ver_control = FirmwareBSPVersionControl( current_firmware_bsp_vf=current_ota_status_dir - / cfg.FIRMWARE_BSP_VERSION_FNAME, + / boot_cfg.FIRMWARE_BSP_VERSION_FNAME, standby_firmware_bsp_vf=standby_ota_status_dir - / cfg.FIRMWARE_BSP_VERSION_FNAME, + / boot_cfg.FIRMWARE_BSP_VERSION_FNAME, ) # init ota-status files @@ -412,7 +410,7 @@ def _copy_standby_slot_boot_to_internal_emmc(self): standby slot rootfs MUST be fully setup! """ # mount corresponding internal emmc device - internal_emmc_mp = Path(cfg.SEPARATE_BOOT_MOUNT_POINT) + internal_emmc_mp = Path(boot_cfg.SEPARATE_BOOT_MOUNT_POINT) internal_emmc_mp.mkdir(exist_ok=True, parents=True) internal_emmc_devpath = self._uefi_control.standby_internal_emmc_devpath @@ -450,12 +448,12 @@ def _preserve_ota_config_files_to_standby(self): def _update_standby_slot_extlinux_cfg(self): """update standby slot's /boot/extlinux/extlinux.conf to update root indicator.""" src = standby_slot_extlinux = self._mp_control.standby_slot_mount_point / Path( - cfg.EXTLINUX_FILE + boot_cfg.EXTLINUX_FILE ).relative_to("/") # if standby slot doesn't have extlinux.conf installed, use current booted # extlinux.conf as template source. if not standby_slot_extlinux.is_file(): - src = Path(cfg.EXTLINUX_FILE) + src = Path(boot_cfg.EXTLINUX_FILE) # update the extlinux.conf with standby slot rootfs' partuuid write_str_to_file_sync( @@ -502,7 +500,10 @@ def post_update(self) -> Generator[None, None, None]: self._update_standby_slot_extlinux_cfg() # ------ firmware update ------ # - firmware_updater = CapsuleUpdate(self._uefi_control.parent_devpath) + firmware_updater = CapsuleUpdate( + boot_devpath=self._uefi_control.parent_devpath, + standby_slot_mp=self._mp_control.standby_slot_mount_point, + ) firmware_updater.firmware_update() # ------ preserve /boot/ota folder to standby rootfs ------ # diff --git a/otaclient/app/boot_control/configs.py b/otaclient/app/boot_control/configs.py index d44862d56..b6d1ca3c5 100644 --- a/otaclient/app/boot_control/configs.py +++ b/otaclient/app/boot_control/configs.py @@ -36,26 +36,37 @@ class GrubControlConfig(BaseConfig): BOOT_OTA_PARTITION_FILE: str = "ota-partition" -@dataclass -class JetsonCBootControlConfig(BaseConfig): +class JetsonBootCommon: + TEGRA_CHIP_ID_PATH = "/sys/module/tegra_fuse/parameters/tegra_chip_id" + OTA_STATUS_DIR = "/boot/ota-status" + FIRMWARE_BSP_VERSION_FNAME = "firmware_bsp_version" + EXTLINUX_FILE = "/boot/extlinux/extlinux.conf" + FIRMWARE_DPATH = "/opt/ota_package" + """Refer to standby slot rootfs.""" + + NV_TEGRA_RELEASE_FPATH = "/etc/nv_tegra_release" + SEPARATE_BOOT_MOUNT_POINT = "/mnt/standby_boot" + + +class JetsonCBootControlConfig(JetsonBootCommon): """Jetson device booted with cboot. Suuports BSP version < R34. """ - BOOTLOADER: BootloaderType = BootloaderType.CBOOT - TEGRA_CHIP_ID_PATH: str = "/sys/module/tegra_fuse/parameters/tegra_chip_id" - CHIP_ID_MODEL_MAP: Dict[int, str] = field(default_factory=lambda: {0x19: "rqx_580"}) - OTA_STATUS_DIR: str = "/boot/ota-status" - FIRMWARE_BSP_VERSION_FNAME: str = "firmware_bsp_version" - EXTLINUX_FILE: str = "/boot/extlinux/extlinux.conf" - SEPARATE_BOOT_MOUNT_POINT: str = "/mnt/standby_boot" - # refer to the standby slot - FIRMWARE_DPATH: str = "/opt/ota_package" - FIRMWARE_LIST: List[str] = field( - default_factory=lambda: ["bl_only_payload", "xusb_only_payload"] - ) - NV_TEGRA_RELEASE_FPATH: str = "/etc/nv_tegra_release" + BOOTLOADER = BootloaderType.JETSON_CBOOT + FIRMWARE_LIST = ["bl_only_payload", "xusb_only_payload"] + + +class JetsonUEFIBootControlConfig(JetsonBootCommon): + BOOTLOADER = BootloaderType.JETSON_UEFI + FIRMWARE_LIST = ["bl_only_payload.Cap"] + ESP_MOUNTPOINT = "/mnt/esp" + ESP_PARTLABEL = "esp" + EFIVARS_DPATH = "/sys/firmware/efi/efivars/" + UPDATE_TRIGGER_EFIVAR = "OsIndications-8be4df61-93ca-11d2-aa0d-00e098032b8c" + MAGIC_BYTES = b"\x07\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00" + CAPSULE_PAYLOAD_LOCATION = "EFI/UpdateCapsule" @dataclass @@ -84,4 +95,5 @@ class RPIBootControlConfig(BaseConfig): grub_cfg = GrubControlConfig() cboot_cfg = JetsonCBootControlConfig() +jetson_uefi_cfg = JetsonUEFIBootControlConfig() rpi_boot_cfg = RPIBootControlConfig() From df9ff8926c1ca062c6a80b645e960447fa2ee1f3 Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Thu, 18 Apr 2024 09:18:37 +0000 Subject: [PATCH 005/193] ecu_info: add jetson-uefi bootloader type --- otaclient/configs/ecu_info.py | 1 + 1 file changed, 1 insertion(+) diff --git a/otaclient/configs/ecu_info.py b/otaclient/configs/ecu_info.py index 50b4c8815..f6fa332bc 100644 --- a/otaclient/configs/ecu_info.py +++ b/otaclient/configs/ecu_info.py @@ -46,6 +46,7 @@ class BootloaderType(str, Enum): GRUB = "grub" CBOOT = "cboot" # deprecated, use jetson-cboot instead JETSON_CBOOT = "jetson-cboot" + JETSON_UEFI = "jetson-uefi" RPI_BOOT = "rpi_boot" @staticmethod From 6a45ac458c442469613f82016585b7d36cc1fce2 Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Thu, 18 Apr 2024 09:24:04 +0000 Subject: [PATCH 006/193] move more things and configs into common jetson-boot configs and common --- otaclient/app/boot_control/_jetson_cboot.py | 38 ++++++++++---------- otaclient/app/boot_control/_jetson_common.py | 20 +++++++++++ otaclient/app/boot_control/configs.py | 4 +++ 3 files changed, 44 insertions(+), 18 deletions(-) diff --git a/otaclient/app/boot_control/_jetson_cboot.py b/otaclient/app/boot_control/_jetson_cboot.py index f2b01cfee..e6d199c8d 100644 --- a/otaclient/app/boot_control/_jetson_cboot.py +++ b/otaclient/app/boot_control/_jetson_cboot.py @@ -43,6 +43,7 @@ NVBootctrlCommon, SlotID, parse_bsp_version, + update_extlinux_cfg, ) from .protocol import BootControllerProtocol @@ -60,7 +61,6 @@ class _NVBootctrl(NVBootctrlCommon): Without -t option, the target will be bootloader by default. """ - NVBOOTCTRL = "nvbootctrl" NVBootctrlTarget = Literal["bootloader", "rootfs"] @classmethod @@ -179,15 +179,17 @@ class _CBootControl: def __init__(self): # ------ sanity check, confirm we are at jetson device ------ # - if not os.path.exists(cfg.TEGRA_CHIP_ID_PATH): - _err_msg = f"not a jetson device, {cfg.TEGRA_CHIP_ID_PATH} doesn't exist" + if not os.path.exists(boot_cfg.TEGRA_CHIP_ID_PATH): + _err_msg = ( + f"not a jetson device, {boot_cfg.TEGRA_CHIP_ID_PATH} doesn't exist" + ) logger.error(_err_msg) raise JetsonCBootContrlError(_err_msg) # ------ check BSP version ------ # try: self.bsp_version = bsp_version = parse_bsp_version( - Path(cfg.NV_TEGRA_RELEASE_FPATH).read_text() + Path(boot_cfg.NV_TEGRA_RELEASE_FPATH).read_text() ) except Exception as e: _err_msg = f"failed to detect BSP version: {e!r}" @@ -266,9 +268,9 @@ def __init__(self): self._external_rootfs = False parent_devname = parent_devpath.name - if parent_devname.startswith(self.MMCBLK_DEV_PREFIX): + if parent_devname.startswith(boot_cfg.MMCBLK_DEV_PREFIX): logger.info(f"device boots from internal emmc: {parent_devpath}") - elif parent_devname.startswith(self.NVMESSD_DEV_PREFIX): + elif parent_devname.startswith(boot_cfg.NVMESSD_DEV_PREFIX): logger.info(f"device boots from external nvme ssd: {parent_devpath}") self._external_rootfs = True else: @@ -296,7 +298,7 @@ def __init__(self): ) # internal emmc partition - self.standby_internal_emmc_devpath = f"/dev/{self.INTERNAL_EMMC_DEVNAME}p{self._slot_id_partid[standby_rootfs_slot]}" + self.standby_internal_emmc_devpath = f"/dev/{boot_cfg.INTERNAL_EMMC_DEVNAME}p{self._slot_id_partid[standby_rootfs_slot]}" logger.info(f"finished cboot control init: {current_rootfs_slot=}") logger.info(f"nvbootctrl dump-slots-info: \n{_NVBootctrl.dump_slots_info()}") @@ -366,17 +368,17 @@ def __init__(self) -> None: active_slot_dev=self._cboot_control.curent_rootfs_devpath, active_slot_mount_point=cfg.ACTIVE_ROOT_MOUNT_POINT, ) - current_ota_status_dir = Path(cfg.OTA_STATUS_DIR) + current_ota_status_dir = Path(boot_cfg.OTA_STATUS_DIR) standby_ota_status_dir = self._mp_control.standby_slot_mount_point / Path( - cfg.OTA_STATUS_DIR + boot_cfg.OTA_STATUS_DIR ).relative_to("/") # load firmware BSP version from current rootfs slot self._firmware_ver_control = FirmwareBSPVersionControl( current_firmware_bsp_vf=current_ota_status_dir - / cfg.FIRMWARE_BSP_VERSION_FNAME, + / boot_cfg.FIRMWARE_BSP_VERSION_FNAME, standby_firmware_bsp_vf=standby_ota_status_dir - / cfg.FIRMWARE_BSP_VERSION_FNAME, + / boot_cfg.FIRMWARE_BSP_VERSION_FNAME, ) # init ota-status files @@ -438,7 +440,7 @@ def _copy_standby_slot_boot_to_internal_emmc(self): standby slot rootfs MUST be fully setup! """ # mount corresponding internal emmc device - internal_emmc_mp = Path(cfg.SEPARATE_BOOT_MOUNT_POINT) + internal_emmc_mp = Path(boot_cfg.SEPARATE_BOOT_MOUNT_POINT) internal_emmc_mp.mkdir(exist_ok=True, parents=True) internal_emmc_devpath = self._cboot_control.standby_internal_emmc_devpath @@ -480,17 +482,17 @@ def _preserve_ota_config_files_to_standby(self): def _update_standby_slot_extlinux_cfg(self): """update standby slot's /boot/extlinux/extlinux.conf to update root indicator.""" src = standby_slot_extlinux = self._mp_control.standby_slot_mount_point / Path( - cfg.EXTLINUX_FILE + boot_cfg.EXTLINUX_FILE ).relative_to("/") # if standby slot doesn't have extlinux.conf installed, use current booted # extlinux.conf as template source. if not standby_slot_extlinux.is_file(): - src = Path(cfg.EXTLINUX_FILE) + src = Path(boot_cfg.EXTLINUX_FILE) # update the extlinux.conf with standby slot rootfs' partuuid write_str_to_file_sync( standby_slot_extlinux, - self._cboot_control.update_extlinux_cfg( + update_extlinux_cfg( src.read_text(), self._cboot_control.standby_rootfs_dev_partuuid, ), @@ -512,7 +514,7 @@ def _nv_firmware_update(self) -> Optional[bool]: # ------ check if we need to do firmware update ------ # _new_bsp_v_fpath = self._mp_control.standby_slot_mount_point / Path( - cfg.NV_TEGRA_RELEASE_FPATH + boot_cfg.NV_TEGRA_RELEASE_FPATH ).relative_to("/") try: new_bsp_v = parse_bsp_version(_new_bsp_v_fpath.read_text()) @@ -530,11 +532,11 @@ def _nv_firmware_update(self) -> Optional[bool]: # ------ preform firmware update ------ # firmware_dpath = self._mp_control.standby_slot_mount_point / Path( - cfg.FIRMWARE_DPATH + boot_cfg.FIRMWARE_DPATH ).relative_to("/") _firmware_applied = False - for firmware_fname in cfg.FIRMWARE_LIST: + for firmware_fname in boot_cfg.FIRMWARE_LIST: if (firmware_fpath := firmware_dpath / firmware_fname).is_file(): logger.info(f"nv_firmware: apply {firmware_fpath} ...") try: diff --git a/otaclient/app/boot_control/_jetson_common.py b/otaclient/app/boot_control/_jetson_common.py index 8bcd302c9..c62c1d1a3 100644 --- a/otaclient/app/boot_control/_jetson_common.py +++ b/otaclient/app/boot_control/_jetson_common.py @@ -21,6 +21,7 @@ import logging import re import subprocess +from functools import partial from pathlib import Path from typing import Any, NamedTuple, Optional @@ -241,3 +242,22 @@ def parse_bsp_version(nv_tegra_release: str) -> BSPVersion: int(ma.group("major_rev")), int(ma.group("minor_rev")), ) + + +def update_extlinux_cfg(_input: str, partuuid: str) -> str: + """Update input exlinux text with input rootfs .""" + + partuuid_str = f"PARTUUID={partuuid}" + + def _replace(ma: re.Match, repl: str): + append_l: str = ma.group(0) + if append_l.startswith("#"): + return append_l + res, n = re.compile(r"root=[\w\-=]*").subn(repl, append_l) + if not n: # this APPEND line doesn't contain root= placeholder + res = f"{append_l} {repl}" + + return res + + _repl_func = partial(_replace, repl=f"root={partuuid_str}") + return re.compile(r"\n\s*APPEND.*").sub(_repl_func, _input) diff --git a/otaclient/app/boot_control/configs.py b/otaclient/app/boot_control/configs.py index b6d1ca3c5..696381fe1 100644 --- a/otaclient/app/boot_control/configs.py +++ b/otaclient/app/boot_control/configs.py @@ -47,6 +47,10 @@ class JetsonBootCommon: NV_TEGRA_RELEASE_FPATH = "/etc/nv_tegra_release" SEPARATE_BOOT_MOUNT_POINT = "/mnt/standby_boot" + MMCBLK_DEV_PREFIX = "mmcblk" # internal emmc + NVMESSD_DEV_PREFIX = "nvme" # external nvme ssd + INTERNAL_EMMC_DEVNAME = "mmcblk0" + class JetsonCBootControlConfig(JetsonBootCommon): """Jetson device booted with cboot. From 7ae63a5a5232b034b6ee5c70f9cf06e62defe86d Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Thu, 18 Apr 2024 09:31:39 +0000 Subject: [PATCH 007/193] jetson-uefi: cleanup --- otaclient/app/boot_control/_jetson_uefi.py | 63 +++++----------------- 1 file changed, 14 insertions(+), 49 deletions(-) diff --git a/otaclient/app/boot_control/_jetson_uefi.py b/otaclient/app/boot_control/_jetson_uefi.py index 9781ecaf2..3b00a710d 100644 --- a/otaclient/app/boot_control/_jetson_uefi.py +++ b/otaclient/app/boot_control/_jetson_uefi.py @@ -23,7 +23,6 @@ import os import re import shutil -from functools import partial from pathlib import Path from typing import Generator @@ -42,6 +41,7 @@ NVBootctrlCommon, SlotID, parse_bsp_version, + update_extlinux_cfg, ) from .protocol import BootControllerProtocol @@ -95,9 +95,7 @@ def __init__(self, boot_devpath: Path | str, standby_slot_mp: Path | str) -> Non # NOTE: use the esp partition at the current booted device # i.e., if we boot from nvme0n1, then bootdev_path is /dev/nvme0n1 and # we use the esp at nvme0n1. - self.boot_devpath = Path(boot_devpath) - self.boot_devname = Path(boot_devpath).name - + boot_devpath = str(boot_devpath) self.esp_mp = boot_cfg.ESP_MOUNTPOINT # NOTE: we get the update capsule from the standby slot @@ -108,12 +106,12 @@ def __init__(self, boot_devpath: Path | str, standby_slot_mp: Path | str) -> Non token="PARTLABEL", value=boot_cfg.ESP_PARTLABEL ) for _esp_part in esp_parts: - if _esp_part.find(self.boot_devname) != -1: + if _esp_part.find(boot_devpath) != -1: logger.info(f"find esp partition at {_esp_part}") self.esp_part = _esp_part break else: - _err_msg = f"failed to find esp partition on {self.boot_devpath}" + _err_msg = f"failed to find esp partition on {boot_devpath}" logger.error(_err_msg) raise JetsonUEFIBootControlError(_err_msg) @@ -159,7 +157,11 @@ def _prepare_payload(self) -> None: CMDHelperFuncs.umount(self.esp_mp, ignore_error=True) def _write_efivar(self) -> None: - """Write magic efivar to trigger firmware Capsule update in next boot.""" + """Write magic efivar to trigger firmware Capsule update in next boot. + + Raises: + JetsonUEFIBootControlError on failed Capsule update preparing. + """ magic_efivar_fpath = ( Path(boot_cfg.EFIVARS_DPATH) / boot_cfg.UPDATE_TRIGGER_EFIVAR ) @@ -186,9 +188,6 @@ def firmware_update(self) -> None: class _UEFIBoot: """Low-level boot control implementation for jetson-uefi.""" - MMCBLK_DEV_PREFIX = "mmcblk" # internal emmc - NVMESSD_DEV_PREFIX = "nvme" # external nvme ssd - INTERNAL_EMMC_DEVNAME = "mmcblk0" _slot_id_partid = {SlotID("0"): "1", SlotID("1"): "2"} def __init__(self): @@ -235,9 +234,9 @@ def __init__(self): self._external_rootfs = False parent_devname = parent_devpath.name - if parent_devname.startswith(self.MMCBLK_DEV_PREFIX): + if parent_devname.startswith(boot_cfg.MMCBLK_DEV_PREFIX): logger.info(f"device boots from internal emmc: {parent_devpath}") - elif parent_devname.startswith(self.NVMESSD_DEV_PREFIX): + elif parent_devname.startswith(boot_cfg.NVMESSD_DEV_PREFIX): logger.info(f"device boots from external nvme ssd: {parent_devpath}") self._external_rootfs = True else: @@ -265,9 +264,7 @@ def __init__(self): ) # internal emmc partition - self.standby_internal_emmc_devpath = ( - f"/dev/{self.INTERNAL_EMMC_DEVNAME}p{self._slot_id_partid[standby_slot]}" - ) + self.standby_internal_emmc_devpath = f"/dev/{boot_cfg.INTERNAL_EMMC_DEVNAME}p{self._slot_id_partid[standby_slot]}" logger.info("finished jetson-uefi boot control startup") logger.info(f"nvbootctrl dump-slots-info: \n{_NVBootctrl.dump_slots_info()}") @@ -291,38 +288,6 @@ def switch_boot_to_standby(self) -> None: # the rootfs slot. _NVBootctrl.set_active_boot_slot(target_slot) - def prepare_standby_dev(self, *, erase_standby: bool): - CMDHelperFuncs.umount(self.standby_rootfs_devpath, ignore_error=True) - - if erase_standby: - try: - CMDHelperFuncs.mkfs_ext4(self.standby_rootfs_devpath) - except Exception as e: - _err_msg = f"failed to mkfs.ext4 on standby dev: {e!r}" - logger.error(_err_msg) - raise JetsonUEFIBootControlError(_err_msg) from e - # TODO: in the future if in-place update mode is implemented, do a - # fschck over the standby slot file system. - - @staticmethod - def update_extlinux_cfg(_input: str, partuuid: str) -> str: - """Update input exlinux text with input rootfs .""" - - partuuid_str = f"PARTUUID={partuuid}" - - def _replace(ma: re.Match, repl: str): - append_l: str = ma.group(0) - if append_l.startswith("#"): - return append_l - res, n = re.compile(r"root=[\w\-=]*").subn(repl, append_l) - if not n: # this APPEND line doesn't contain root= placeholder - res = f"{append_l} {repl}" - - return res - - _repl_func = partial(_replace, repl=f"root={partuuid_str}") - return re.compile(r"\n\s*APPEND.*").sub(_repl_func, _input) - class JetsonUEFIBootControl(BootControllerProtocol): """BootControllerProtocol implementation for jetson-uefi.""" @@ -458,7 +423,7 @@ def _update_standby_slot_extlinux_cfg(self): # update the extlinux.conf with standby slot rootfs' partuuid write_str_to_file_sync( standby_slot_extlinux, - self._uefi_control.update_extlinux_cfg( + update_extlinux_cfg( src.read_text(), self._uefi_control.standby_rootfs_dev_partuuid, ), @@ -479,7 +444,7 @@ def pre_update(self, version: str, *, standby_as_ref: bool, erase_standby: bool) self._ota_status_control.pre_update_current() # prepare standby slot dev - self._uefi_control.prepare_standby_dev(erase_standby=erase_standby) + self._mp_control.prepare_standby_dev(erase_standby=erase_standby) # mount slots self._mp_control.mount_standby() self._mp_control.mount_active() From 1776d0f6217d8d43f14ba1a2efdc39230a532399 Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Thu, 18 Apr 2024 10:11:51 +0000 Subject: [PATCH 008/193] jetson-uefi: minor update --- otaclient/app/boot_control/_jetson_uefi.py | 38 ++++++++++++---------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/otaclient/app/boot_control/_jetson_uefi.py b/otaclient/app/boot_control/_jetson_uefi.py index 3b00a710d..7d9ec9d54 100644 --- a/otaclient/app/boot_control/_jetson_uefi.py +++ b/otaclient/app/boot_control/_jetson_uefi.py @@ -59,6 +59,8 @@ class _NVBootctrl(NVBootctrlCommon): Without -t option, the target will be bootloader by default. """ + CAPSULE_UPDATE_PATTERN = re.compile(r"Capsule update status: (?P\d+)") + @classmethod def get_capsule_update_result(cls) -> str: """Check the Capsule update status. @@ -78,8 +80,7 @@ def get_capsule_update_result(cls) -> str: slots_info = cls.dump_slots_info() logger.info(f"checking Capsule update result: \n{slots_info}") - pattern = re.compile(r"Capsule update status: (?P\d+)") - ma = pattern.search(slots_info) + ma = cls.CAPSULE_UPDATE_PATTERN.search(slots_info) assert ma, "failed to get Capsule update result" update_result = ma.group("status") @@ -91,29 +92,33 @@ class CapsuleUpdate: EFIVARS_FSTYPE = "efivarfs" - def __init__(self, boot_devpath: Path | str, standby_slot_mp: Path | str) -> None: + def __init__( + self, boot_parent_devpath: Path | str, standby_slot_mp: Path | str + ) -> None: # NOTE: use the esp partition at the current booted device # i.e., if we boot from nvme0n1, then bootdev_path is /dev/nvme0n1 and # we use the esp at nvme0n1. - boot_devpath = str(boot_devpath) + boot_parent_devpath = str(boot_parent_devpath) self.esp_mp = boot_cfg.ESP_MOUNTPOINT # NOTE: we get the update capsule from the standby slot self.standby_slot_mp = Path(standby_slot_mp) - # if boots from external, expects to have multiple esp parts, we need to get the one at our booted dev + # NOTE: if boots from external, expects to have multiple esp parts, + # we need to get the one at our booted parent dev. esp_parts = CMDHelperFuncs.get_dev_by_token( token="PARTLABEL", value=boot_cfg.ESP_PARTLABEL ) for _esp_part in esp_parts: - if _esp_part.find(boot_devpath) != -1: + if _esp_part.find(boot_parent_devpath) != -1: logger.info(f"find esp partition at {_esp_part}") - self.esp_part = _esp_part + esp_part = _esp_part break else: - _err_msg = f"failed to find esp partition on {boot_devpath}" + _err_msg = f"failed to find esp partition on {boot_parent_devpath}" logger.error(_err_msg) raise JetsonUEFIBootControlError(_err_msg) + self.esp_part = esp_part @classmethod def _ensure_efivarfs_mounted(cls) -> None: @@ -140,9 +145,9 @@ def _prepare_payload(self) -> None: try: CMDHelperFuncs.mount_rw(self.esp_part, self.esp_mp) except Exception as e: - raise JetsonUEFIBootControlError( - f"failed to mount {self.esp_part} onto {self.esp_mp}: {e!r}" - ) + _err_msg = f"failed to mount {self.esp_part=} to {self.esp_mp}: {e!r}" + logger.error(_err_msg) + raise JetsonUEFIBootControlError(_err_msg) from e capsule_payload_location = Path(self.esp_mp) / boot_cfg.CAPSULE_PAYLOAD_LOCATION try: @@ -246,15 +251,14 @@ def __init__(self): f"unsupported bootdev {parent_devpath}" ) - # rootfs partition self.standby_rootfs_devpath = ( f"/dev/{parent_devname}p{self._slot_id_partid[standby_slot]}" ) - self.standby_rootfs_dev_partuuid = CMDHelperFuncs.get_partuuid_by_dev( - f"{self.standby_rootfs_devpath}" + self.standby_rootfs_dev_partuuid = CMDHelperFuncs.get_attrs_by_dev( + "PARTUUID", f"{self.standby_rootfs_devpath}" ).strip() - current_rootfs_dev_partuuid = CMDHelperFuncs.get_partuuid_by_dev( - current_rootfs_devpath + current_rootfs_dev_partuuid = CMDHelperFuncs.get_attrs_by_dev( + "PARTUUID", current_rootfs_devpath ).strip() logger.info( @@ -466,7 +470,7 @@ def post_update(self) -> Generator[None, None, None]: # ------ firmware update ------ # firmware_updater = CapsuleUpdate( - boot_devpath=self._uefi_control.parent_devpath, + boot_parent_devpath=self._uefi_control.parent_devpath, standby_slot_mp=self._mp_control.standby_slot_mount_point, ) firmware_updater.firmware_update() From 4c7fa47f3ccb6a2c50d65aa347333ae9797b3444 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 10 May 2024 11:44:50 +0000 Subject: [PATCH 009/193] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- otaclient/app/boot_control/_jetson_cboot.py | 4 ++-- otaclient/app/boot_control/_jetson_common.py | 1 + otaclient/app/boot_control/_jetson_uefi.py | 12 +++++------- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/otaclient/app/boot_control/_jetson_cboot.py b/otaclient/app/boot_control/_jetson_cboot.py index e6d199c8d..e46bc0490 100644 --- a/otaclient/app/boot_control/_jetson_cboot.py +++ b/otaclient/app/boot_control/_jetson_cboot.py @@ -25,7 +25,7 @@ import subprocess from functools import partial from pathlib import Path -from subprocess import CompletedProcess, run, CalledProcessError +from subprocess import CalledProcessError, CompletedProcess, run from typing import Generator, Literal, Optional from otaclient.app import errors as ota_errors @@ -37,7 +37,6 @@ from otaclient.app.proto import wrapper from ._common import CMDHelperFuncs, OTAStatusFilesControl, SlotMountHelper -from .configs import cboot_cfg as cfg from ._jetson_common import ( FirmwareBSPVersionControl, NVBootctrlCommon, @@ -45,6 +44,7 @@ parse_bsp_version, update_extlinux_cfg, ) +from .configs import cboot_cfg as cfg from .protocol import BootControllerProtocol logger = logging.getLogger(__name__) diff --git a/otaclient/app/boot_control/_jetson_common.py b/otaclient/app/boot_control/_jetson_common.py index c62c1d1a3..da3c0194d 100644 --- a/otaclient/app/boot_control/_jetson_common.py +++ b/otaclient/app/boot_control/_jetson_common.py @@ -18,6 +18,7 @@ from __future__ import annotations + import logging import re import subprocess diff --git a/otaclient/app/boot_control/_jetson_uefi.py b/otaclient/app/boot_control/_jetson_uefi.py index 7d9ec9d54..0ae7e22cf 100644 --- a/otaclient/app/boot_control/_jetson_uefi.py +++ b/otaclient/app/boot_control/_jetson_uefi.py @@ -19,6 +19,7 @@ from __future__ import annotations + import logging import os import re @@ -27,15 +28,11 @@ from typing import Generator from otaclient.app import errors as ota_errors -from otaclient.app.configs import config as cfg from otaclient.app.common import copytree_identical, write_str_to_file_sync +from otaclient.app.configs import config as cfg from otaclient.app.proto import wrapper -from ._common import ( - OTAStatusFilesControl, - SlotMountHelper, - CMDHelperFuncs, -) -from .configs import jetson_uefi_cfg as boot_cfg + +from ._common import CMDHelperFuncs, OTAStatusFilesControl, SlotMountHelper from ._jetson_common import ( FirmwareBSPVersionControl, NVBootctrlCommon, @@ -43,6 +40,7 @@ parse_bsp_version, update_extlinux_cfg, ) +from .configs import jetson_uefi_cfg as boot_cfg from .protocol import BootControllerProtocol logger = logging.getLogger(__name__) From d003534578fd80f0b483eec0429656d597a05655 Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Fri, 10 May 2024 11:50:22 +0000 Subject: [PATCH 010/193] fix test_jetson_cboot --- tests/test_boot_control/test_jetson_cboot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_boot_control/test_jetson_cboot.py b/tests/test_boot_control/test_jetson_cboot.py index 001f7517d..a292d4a86 100644 --- a/tests/test_boot_control/test_jetson_cboot.py +++ b/tests/test_boot_control/test_jetson_cboot.py @@ -25,13 +25,13 @@ import pytest from otaclient.app.boot_control import _jetson_cboot -from otaclient.app.boot_control._jetson_cboot import ( +from otaclient.app.boot_control._jetson_common import ( BSPVersion, FirmwareBSPVersion, SlotID, - _CBootControl, parse_bsp_version, ) +from otaclient.app.boot_control._jetson_cboot import _CBootControl logger = logging.getLogger(__name__) From 0e7ddc8768725b4f9fd11da394a6473eaff73a58 Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Fri, 10 May 2024 11:57:56 +0000 Subject: [PATCH 011/193] minor fix jetson_cboot related --- otaclient/app/boot_control/_jetson_cboot.py | 9 ++++----- otaclient/app/boot_control/_jetson_common.py | 19 +++++++++++-------- tests/test_boot_control/test_jetson_cboot.py | 2 +- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/otaclient/app/boot_control/_jetson_cboot.py b/otaclient/app/boot_control/_jetson_cboot.py index e46bc0490..63c4ddab1 100644 --- a/otaclient/app/boot_control/_jetson_cboot.py +++ b/otaclient/app/boot_control/_jetson_cboot.py @@ -25,8 +25,7 @@ import subprocess from functools import partial from pathlib import Path -from subprocess import CalledProcessError, CompletedProcess, run -from typing import Generator, Literal, Optional +from typing import Generator, Optional from otaclient.app import errors as ota_errors from otaclient.app.common import ( @@ -36,15 +35,17 @@ ) from otaclient.app.proto import wrapper +from ..configs import config as cfg from ._common import CMDHelperFuncs, OTAStatusFilesControl, SlotMountHelper from ._jetson_common import ( + NVBootctrlTarget, FirmwareBSPVersionControl, NVBootctrlCommon, SlotID, parse_bsp_version, update_extlinux_cfg, ) -from .configs import cboot_cfg as cfg +from .configs import cboot_cfg as boot_cfg from .protocol import BootControllerProtocol logger = logging.getLogger(__name__) @@ -61,8 +62,6 @@ class _NVBootctrl(NVBootctrlCommon): Without -t option, the target will be bootloader by default. """ - NVBootctrlTarget = Literal["bootloader", "rootfs"] - @classmethod def mark_boot_successful( cls, slot_id: SlotID, *, target: Optional[NVBootctrlTarget] = None diff --git a/otaclient/app/boot_control/_jetson_common.py b/otaclient/app/boot_control/_jetson_common.py index da3c0194d..8ddb1821c 100644 --- a/otaclient/app/boot_control/_jetson_common.py +++ b/otaclient/app/boot_control/_jetson_common.py @@ -78,22 +78,26 @@ def dump(to_export: BSPVersion) -> str: return f"R{to_export.major_ver}.{to_export.major_rev}.{to_export.minor_rev}" +BSPVersionStr = Annotated[ + BSPVersion, + BeforeValidator(BSPVersion.parse), + PlainSerializer(BSPVersion.dump, return_type=str), +] +"""BSPVersion in string representation, used by FirmwareBSPVersion model.""" + + class FirmwareBSPVersion(BaseModel): """ BSP version string schema: Rxx.yy.z """ - BSPVersionStr = Annotated[ - BSPVersion, - BeforeValidator(BSPVersion.parse), - PlainSerializer(BSPVersion.dump, return_type=str), - ] - """BSPVersion in string representation, used by FirmwareBSPVersion model.""" - slot_a: Optional[BSPVersionStr] = None slot_b: Optional[BSPVersionStr] = None +NVBootctrlTarget = Literal["bootloader", "rootfs"] + + class NVBootctrlCommon: """Helper for calling nvbootctrl commands. @@ -111,7 +115,6 @@ class NVBootctrlCommon: """ NVBOOTCTRL = "nvbootctrl" - NVBootctrlTarget = Literal["bootloader", "rootfs"] @classmethod def _nvbootctrl( diff --git a/tests/test_boot_control/test_jetson_cboot.py b/tests/test_boot_control/test_jetson_cboot.py index a292d4a86..ce12fdf7a 100644 --- a/tests/test_boot_control/test_jetson_cboot.py +++ b/tests/test_boot_control/test_jetson_cboot.py @@ -25,13 +25,13 @@ import pytest from otaclient.app.boot_control import _jetson_cboot +from otaclient.app.boot_control._jetson_cboot import _CBootControl from otaclient.app.boot_control._jetson_common import ( BSPVersion, FirmwareBSPVersion, SlotID, parse_bsp_version, ) -from otaclient.app.boot_control._jetson_cboot import _CBootControl logger = logging.getLogger(__name__) From a26216f2f0fea86fd8e31a4647b0f07e3f329925 Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Fri, 10 May 2024 12:08:17 +0000 Subject: [PATCH 012/193] fix up jetson_uefi --- otaclient/app/boot_control/_jetson_uefi.py | 33 +++++++++++++++------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/otaclient/app/boot_control/_jetson_uefi.py b/otaclient/app/boot_control/_jetson_uefi.py index 0ae7e22cf..7d1de2b70 100644 --- a/otaclient/app/boot_control/_jetson_uefi.py +++ b/otaclient/app/boot_control/_jetson_uefi.py @@ -28,7 +28,11 @@ from typing import Generator from otaclient.app import errors as ota_errors -from otaclient.app.common import copytree_identical, write_str_to_file_sync +from otaclient.app.common import ( + copytree_identical, + subprocess_call, + write_str_to_file_sync, +) from otaclient.app.configs import config as cfg from otaclient.app.proto import wrapper @@ -107,6 +111,9 @@ def __init__( esp_parts = CMDHelperFuncs.get_dev_by_token( token="PARTLABEL", value=boot_cfg.ESP_PARTLABEL ) + if not esp_parts: + raise JetsonUEFIBootControlError("no ESP partition presented") + for _esp_part in esp_parts: if _esp_part.find(boot_parent_devpath) != -1: logger.info(f"find esp partition at {_esp_part}") @@ -120,19 +127,25 @@ def __init__( @classmethod def _ensure_efivarfs_mounted(cls) -> None: + # TODO: remount if efivarfs is mounted as read-only if CMDHelperFuncs.is_target_mounted(boot_cfg.EFIVARS_DPATH): return logger.warning( f"efivars is not mounted! try to mount it at {boot_cfg.EFIVARS_DPATH}" ) + # fmt: off + cmd = [ + "mount", + "-t", cls.EFIVARS_FSTYPE, + "-o", "rw,nosuid,nodev,noexec,relatime", + cls.EFIVARS_FSTYPE, + boot_cfg.EFIVARS_DPATH + ] + # fmt: on + try: - CMDHelperFuncs._mount( - cls.EFIVARS_FSTYPE, - boot_cfg.EFIVARS_DPATH, - fstype=cls.EFIVARS_FSTYPE, - options=["rw", "nosuid", "nodev", "noexec", "relatime"], - ) + subprocess_call(cmd, raise_exception=True) except Exception as e: raise JetsonUEFIBootControlError( f"failed to mount {cls.EFIVARS_FSTYPE} on {boot_cfg.EFIVARS_DPATH}: {e!r}" @@ -157,7 +170,7 @@ def _prepare_payload(self) -> None: logger.error(_err_msg) raise JetsonUEFIBootControlError(_err_msg) from e finally: - CMDHelperFuncs.umount(self.esp_mp, ignore_error=True) + CMDHelperFuncs.umount(self.esp_mp, raise_exception=False) def _write_efivar(self) -> None: """Write magic efivar to trigger firmware Capsule update in next boot. @@ -382,7 +395,7 @@ def _copy_standby_slot_boot_to_internal_emmc(self): internal_emmc_devpath = self._uefi_control.standby_internal_emmc_devpath try: - CMDHelperFuncs.umount(internal_emmc_devpath, ignore_error=True) + CMDHelperFuncs.umount(internal_emmc_devpath, raise_exception=False) CMDHelperFuncs.mount_rw(internal_emmc_devpath, internal_emmc_mp) except Exception as e: _msg = f"failed to mount standby internal emmc dev: {e!r}" @@ -399,7 +412,7 @@ def _copy_standby_slot_boot_to_internal_emmc(self): logger.error(_msg) raise JetsonUEFIBootControlError(_msg) from e finally: - CMDHelperFuncs.umount(internal_emmc_mp, ignore_error=True) + CMDHelperFuncs.umount(internal_emmc_mp, raise_exception=False) def _preserve_ota_config_files_to_standby(self): """Preserve /boot/ota to standby /boot folder.""" From 8971b46f2adf33d3a25436436fc51c953961570a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 10 May 2024 12:08:41 +0000 Subject: [PATCH 013/193] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- otaclient/app/boot_control/_jetson_cboot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/otaclient/app/boot_control/_jetson_cboot.py b/otaclient/app/boot_control/_jetson_cboot.py index 63c4ddab1..223244bcd 100644 --- a/otaclient/app/boot_control/_jetson_cboot.py +++ b/otaclient/app/boot_control/_jetson_cboot.py @@ -38,9 +38,9 @@ from ..configs import config as cfg from ._common import CMDHelperFuncs, OTAStatusFilesControl, SlotMountHelper from ._jetson_common import ( - NVBootctrlTarget, FirmwareBSPVersionControl, NVBootctrlCommon, + NVBootctrlTarget, SlotID, parse_bsp_version, update_extlinux_cfg, From 91e92e4eb9e5092c4c80e8c9db9d428b6602c072 Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Fri, 10 May 2024 12:16:10 +0000 Subject: [PATCH 014/193] fix by flake8 --- otaclient/app/boot_control/configs.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/otaclient/app/boot_control/configs.py b/otaclient/app/boot_control/configs.py index 696381fe1..373afe45c 100644 --- a/otaclient/app/boot_control/configs.py +++ b/otaclient/app/boot_control/configs.py @@ -15,8 +15,7 @@ from __future__ import annotations -from dataclasses import dataclass, field -from typing import Dict, List +from dataclasses import dataclass from otaclient.configs.ecu_info import BootloaderType From d6908ebd7b3ff2b609691b8db66aa02cf782310c Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Mon, 13 May 2024 07:12:10 +0000 Subject: [PATCH 015/193] jetson-common: move copy_standby_slot_boot_to_internal_emmc, preserve_ota_config_files_to_standby and update_standby_slot_extlinux_cfg here --- otaclient/app/boot_control/_jetson_common.py | 82 +++++++++++++++++++- 1 file changed, 81 insertions(+), 1 deletion(-) diff --git a/otaclient/app/boot_control/_jetson_common.py b/otaclient/app/boot_control/_jetson_common.py index 8ddb1821c..d1ae79682 100644 --- a/otaclient/app/boot_control/_jetson_common.py +++ b/otaclient/app/boot_control/_jetson_common.py @@ -30,6 +30,8 @@ from typing_extensions import Annotated, Literal, Self from otaclient.app.common import write_str_to_file_sync +from ..common import copytree_identical +from ._common import CMDHelperFuncs logger = logging.getLogger(__name__) @@ -250,7 +252,6 @@ def parse_bsp_version(nv_tegra_release: str) -> BSPVersion: def update_extlinux_cfg(_input: str, partuuid: str) -> str: """Update input exlinux text with input rootfs .""" - partuuid_str = f"PARTUUID={partuuid}" def _replace(ma: re.Match, repl: str): @@ -265,3 +266,82 @@ def _replace(ma: re.Match, repl: str): _repl_func = partial(_replace, repl=f"root={partuuid_str}") return re.compile(r"\n\s*APPEND.*").sub(_repl_func, _input) + + +def copy_standby_slot_boot_to_internal_emmc( + *, + internal_emmc_mp: Path | str, + internal_emmc_devpath: Path | str, + standby_slot_boot_dirpath: Path | str, +) -> None: + """Copy the standby slot's /boot to internal emmc dev. + + This method is involved when external rootfs is enabled, aligning with + the behavior of the NVIDIA flashing script. + + WARNING: DO NOT call this method if we are not booted from external rootfs! + NOTE: at the time this method is called, the /boot folder at + standby slot rootfs MUST be fully setup! + """ + internal_emmc_mp = Path(internal_emmc_mp) + internal_emmc_mp.mkdir(exist_ok=True, parents=True) + + try: + CMDHelperFuncs.umount(internal_emmc_devpath, raise_exception=False) + CMDHelperFuncs.mount_rw( + target=str(internal_emmc_devpath), + mount_point=internal_emmc_mp, + ) + except Exception as e: + _msg = f"failed to mount standby internal emmc dev: {e!r}" + logger.error(_msg) + raise ValueError(_msg) from e + + try: + dst = internal_emmc_mp / "boot" + # copy the standby slot's boot folder to emmc boot dev + copytree_identical(Path(standby_slot_boot_dirpath), dst) + except Exception as e: + _msg = f"failed to populate standby slot's /boot folder to standby internal emmc dev: {e!r}" + logger.error(_msg) + raise ValueError(_msg) from e + finally: + CMDHelperFuncs.umount(internal_emmc_mp, raise_exception=False) + + +def preserve_ota_config_files_to_standby( + *, active_slot_ota_dirpath: Path, standby_slot_ota_dirpath: Path +) -> None: + """Preserve /boot/ota to standby /boot folder.""" + if not active_slot_ota_dirpath.is_dir(): # basically this should not happen + logger.warning( + f"{active_slot_ota_dirpath} doesn't exist, skip preserve /boot/ota folder." + ) + return + # TODO: (20240411) reconsidering should we preserve /boot/ota? + copytree_identical(active_slot_ota_dirpath, standby_slot_ota_dirpath) + + +def update_standby_slot_extlinux_cfg( + *, + active_slot_extlinux_fpath: Path, + standby_slot_extlinux_fpath: Path, + standby_slot_partuuid: str, +): + """update standby slot's /boot/extlinux/extlinux.conf to update root indicator.""" + src = standby_slot_extlinux_fpath + # if standby slot doesn't have extlinux.conf installed, use current booted + # extlinux.conf as template source. + if not standby_slot_extlinux_fpath.is_file(): + logger.warning( + f"{standby_slot_extlinux_fpath} doesn't exist, use active slot's extlinux file as template" + ) + src = active_slot_extlinux_fpath + + write_str_to_file_sync( + standby_slot_extlinux_fpath, + update_extlinux_cfg( + src.read_text(), + standby_slot_partuuid, + ), + ) From a9bc9014ae3cc677d7dab5e7793adb4fbd40cac2 Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Mon, 13 May 2024 07:14:42 +0000 Subject: [PATCH 016/193] jetson-cboot: use common methods from jetson-common --- otaclient/app/boot_control/_jetson_cboot.py | 125 ++++---------------- 1 file changed, 26 insertions(+), 99 deletions(-) diff --git a/otaclient/app/boot_control/_jetson_cboot.py b/otaclient/app/boot_control/_jetson_cboot.py index 223244bcd..306f0e018 100644 --- a/otaclient/app/boot_control/_jetson_cboot.py +++ b/otaclient/app/boot_control/_jetson_cboot.py @@ -21,18 +21,12 @@ import logging import os -import re import subprocess -from functools import partial from pathlib import Path from typing import Generator, Optional from otaclient.app import errors as ota_errors -from otaclient.app.common import ( - copytree_identical, - subprocess_run_wrapper, - write_str_to_file_sync, -) +from otaclient.app.common import subprocess_run_wrapper from otaclient.app.proto import wrapper from ..configs import config as cfg @@ -43,7 +37,9 @@ NVBootctrlTarget, SlotID, parse_bsp_version, - update_extlinux_cfg, + copy_standby_slot_boot_to_internal_emmc, + preserve_ota_config_files_to_standby, + update_standby_slot_extlinux_cfg, ) from .configs import cboot_cfg as boot_cfg from .protocol import BootControllerProtocol @@ -332,25 +328,6 @@ def switch_boot_to_standby(self) -> None: # the rootfs slot. _NVBootctrl.set_active_boot_slot(target_slot) - @staticmethod - def update_extlinux_cfg(_input: str, partuuid: str) -> str: - """Update input exlinux text with input rootfs .""" - - partuuid_str = f"PARTUUID={partuuid}" - - def _replace(ma: re.Match, repl: str): - append_l: str = ma.group(0) - if append_l.startswith("#"): - return append_l - res, n = re.compile(r"root=[\w\-=]*").subn(repl, append_l) - if not n: # this APPEND line doesn't contain root= placeholder - res = f"{append_l} {repl}" - - return res - - _repl_func = partial(_replace, repl=f"root={partuuid_str}") - return re.compile(r"\n\s*APPEND.*").sub(_repl_func, _input) - class JetsonCBootControl(BootControllerProtocol): """BootControllerProtocol implementation for jetson-cboot.""" @@ -428,75 +405,6 @@ def _finalize_switching_boot(self) -> bool: ) return True - def _copy_standby_slot_boot_to_internal_emmc(self): - """Copy the standby slot's /boot to internal emmc dev. - - This method is involved when external rootfs is enabled, aligning with - the behavior of the NVIDIA flashing script. - - WARNING: DO NOT call this method if we are not booted from external rootfs! - NOTE: at the time this method is called, the /boot folder at - standby slot rootfs MUST be fully setup! - """ - # mount corresponding internal emmc device - internal_emmc_mp = Path(boot_cfg.SEPARATE_BOOT_MOUNT_POINT) - internal_emmc_mp.mkdir(exist_ok=True, parents=True) - internal_emmc_devpath = self._cboot_control.standby_internal_emmc_devpath - - try: - if CMDHelperFuncs.is_target_mounted( - internal_emmc_devpath, raise_exception=False - ): - logger.debug("internal emmc device is mounted, try to unmount ...") - CMDHelperFuncs.umount(internal_emmc_devpath, raise_exception=False) - CMDHelperFuncs.mount_rw(internal_emmc_devpath, internal_emmc_mp) - except Exception as e: - _msg = f"failed to mount standby internal emmc dev: {e!r}" - logger.error(_msg) - raise JetsonCBootContrlError(_msg) from e - - try: - dst = internal_emmc_mp / "boot" - src = self._mp_control.standby_slot_mount_point / "boot" - # copy the standby slot's boot folder to emmc boot dev - copytree_identical(src, dst) - except Exception as e: - _msg = f"failed to populate standby slot's /boot folder to standby internal emmc dev: {e!r}" - logger.error(_msg) - raise JetsonCBootContrlError(_msg) from e - finally: - CMDHelperFuncs.umount(internal_emmc_mp, raise_exception=False) - - def _preserve_ota_config_files_to_standby(self): - """Preserve /boot/ota to standby /boot folder.""" - src = self._mp_control.active_slot_mount_point / "boot" / "ota" - if not src.is_dir(): # basically this should not happen - logger.warning(f"{src} doesn't exist, skip preserve /boot/ota folder.") - return - - dst = self._mp_control.standby_slot_mount_point / "boot" / "ota" - # TODO: (20240411) reconsidering should we preserve /boot/ota? - copytree_identical(src, dst) - - def _update_standby_slot_extlinux_cfg(self): - """update standby slot's /boot/extlinux/extlinux.conf to update root indicator.""" - src = standby_slot_extlinux = self._mp_control.standby_slot_mount_point / Path( - boot_cfg.EXTLINUX_FILE - ).relative_to("/") - # if standby slot doesn't have extlinux.conf installed, use current booted - # extlinux.conf as template source. - if not standby_slot_extlinux.is_file(): - src = Path(boot_cfg.EXTLINUX_FILE) - - # update the extlinux.conf with standby slot rootfs' partuuid - write_str_to_file_sync( - standby_slot_extlinux, - update_extlinux_cfg( - src.read_text(), - self._cboot_control.standby_rootfs_dev_partuuid, - ), - ) - def _nv_firmware_update(self) -> Optional[bool]: """Perform firmware update with nv_update_engine. @@ -613,7 +521,12 @@ def post_update(self) -> Generator[None, None, None]: try: logger.info("jetson-cboot: post-update ...") # ------ update extlinux.conf ------ # - self._update_standby_slot_extlinux_cfg() + update_standby_slot_extlinux_cfg( + active_slot_extlinux_fpath=Path(boot_cfg.EXTLINUX_FILE), + standby_slot_extlinux_fpath=self._mp_control.standby_slot_mount_point + / Path(boot_cfg.EXTLINUX_FILE).relative_to("/"), + standby_slot_partuuid=self._cboot_control.standby_rootfs_dev_partuuid, + ) # ------ firmware update ------ # _fw_update_result = self._nv_firmware_update() @@ -625,7 +538,14 @@ def post_update(self) -> Generator[None, None, None]: raise JetsonCBootContrlError("firmware update failed") # ------ preserve /boot/ota folder to standby rootfs ------ # - self._preserve_ota_config_files_to_standby() + preserve_ota_config_files_to_standby( + active_slot_ota_dirpath=self._mp_control.active_slot_mount_point + / "boot" + / "ota", + standby_slot_ota_dirpath=self._mp_control.standby_slot_mount_point + / "boot" + / "ota", + ) # ------ for external rootfs, preserve /boot folder to internal ------ # if self._cboot_control._external_rootfs: @@ -634,7 +554,14 @@ def post_update(self) -> Generator[None, None, None]: "copy standby slot rootfs' /boot folder " "to corresponding internal emmc dev ..." ) - self._copy_standby_slot_boot_to_internal_emmc() + copy_standby_slot_boot_to_internal_emmc( + internal_emmc_mp=Path(boot_cfg.SEPARATE_BOOT_MOUNT_POINT), + internal_emmc_devpath=Path( + self._cboot_control.standby_internal_emmc_devpath + ), + standby_slot_boot_dirpath=self._mp_control.standby_slot_mount_point + / "boot", + ) # ------ switch boot to standby ------ # self._cboot_control.switch_boot_to_standby() From 3622480b6b507ad85776b92f4d682d982e5cba07 Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Mon, 13 May 2024 07:20:50 +0000 Subject: [PATCH 017/193] jetson-uefi: use common methods from jetson-common --- otaclient/app/boot_control/_jetson_uefi.py | 98 ++++++---------------- 1 file changed, 24 insertions(+), 74 deletions(-) diff --git a/otaclient/app/boot_control/_jetson_uefi.py b/otaclient/app/boot_control/_jetson_uefi.py index 7d1de2b70..8134c8f4f 100644 --- a/otaclient/app/boot_control/_jetson_uefi.py +++ b/otaclient/app/boot_control/_jetson_uefi.py @@ -28,11 +28,7 @@ from typing import Generator from otaclient.app import errors as ota_errors -from otaclient.app.common import ( - copytree_identical, - subprocess_call, - write_str_to_file_sync, -) +from otaclient.app.common import subprocess_call from otaclient.app.configs import config as cfg from otaclient.app.proto import wrapper @@ -42,7 +38,9 @@ NVBootctrlCommon, SlotID, parse_bsp_version, - update_extlinux_cfg, + copy_standby_slot_boot_to_internal_emmc, + preserve_ota_config_files_to_standby, + update_standby_slot_extlinux_cfg, ) from .configs import jetson_uefi_cfg as boot_cfg from .protocol import BootControllerProtocol @@ -379,71 +377,6 @@ def _finalize_switching_boot(self) -> bool: return False - def _copy_standby_slot_boot_to_internal_emmc(self): - """Copy the standby slot's /boot to internal emmc dev. - - This method is involved when external rootfs is enabled, aligning with - the behavior of the NVIDIA flashing script. - - WARNING: DO NOT call this method if we are not booted from external rootfs! - NOTE: at the time this method is called, the /boot folder at - standby slot rootfs MUST be fully setup! - """ - # mount corresponding internal emmc device - internal_emmc_mp = Path(boot_cfg.SEPARATE_BOOT_MOUNT_POINT) - internal_emmc_mp.mkdir(exist_ok=True, parents=True) - internal_emmc_devpath = self._uefi_control.standby_internal_emmc_devpath - - try: - CMDHelperFuncs.umount(internal_emmc_devpath, raise_exception=False) - CMDHelperFuncs.mount_rw(internal_emmc_devpath, internal_emmc_mp) - except Exception as e: - _msg = f"failed to mount standby internal emmc dev: {e!r}" - logger.error(_msg) - raise JetsonUEFIBootControlError(_msg) from e - - try: - dst = internal_emmc_mp / "boot" - src = self._mp_control.standby_slot_mount_point / "boot" - # copy the standby slot's boot folder to emmc boot dev - copytree_identical(src, dst) - except Exception as e: - _msg = f"failed to populate standby slot's /boot folder to standby internal emmc dev: {e!r}" - logger.error(_msg) - raise JetsonUEFIBootControlError(_msg) from e - finally: - CMDHelperFuncs.umount(internal_emmc_mp, raise_exception=False) - - def _preserve_ota_config_files_to_standby(self): - """Preserve /boot/ota to standby /boot folder.""" - src = self._mp_control.active_slot_mount_point / "boot" / "ota" - if not src.is_dir(): # basically this should not happen - logger.warning(f"{src} doesn't exist, skip preserve /boot/ota folder.") - return - - dst = self._mp_control.standby_slot_mount_point / "boot" / "ota" - # TODO: (20240411) reconsidering should we preserve /boot/ota? - copytree_identical(src, dst) - - def _update_standby_slot_extlinux_cfg(self): - """update standby slot's /boot/extlinux/extlinux.conf to update root indicator.""" - src = standby_slot_extlinux = self._mp_control.standby_slot_mount_point / Path( - boot_cfg.EXTLINUX_FILE - ).relative_to("/") - # if standby slot doesn't have extlinux.conf installed, use current booted - # extlinux.conf as template source. - if not standby_slot_extlinux.is_file(): - src = Path(boot_cfg.EXTLINUX_FILE) - - # update the extlinux.conf with standby slot rootfs' partuuid - write_str_to_file_sync( - standby_slot_extlinux, - update_extlinux_cfg( - src.read_text(), - self._uefi_control.standby_rootfs_dev_partuuid, - ), - ) - # APIs def get_standby_slot_path(self) -> Path: @@ -477,7 +410,12 @@ def post_update(self) -> Generator[None, None, None]: try: logger.info("jetson-uefi: post-update ...") # ------ update extlinux.conf ------ # - self._update_standby_slot_extlinux_cfg() + update_standby_slot_extlinux_cfg( + active_slot_extlinux_fpath=Path(boot_cfg.EXTLINUX_FILE), + standby_slot_extlinux_fpath=self._mp_control.standby_slot_mount_point + / Path(boot_cfg.EXTLINUX_FILE).relative_to("/"), + standby_slot_partuuid=self._uefi_control.standby_rootfs_dev_partuuid, + ) # ------ firmware update ------ # firmware_updater = CapsuleUpdate( @@ -487,7 +425,14 @@ def post_update(self) -> Generator[None, None, None]: firmware_updater.firmware_update() # ------ preserve /boot/ota folder to standby rootfs ------ # - self._preserve_ota_config_files_to_standby() + preserve_ota_config_files_to_standby( + active_slot_ota_dirpath=self._mp_control.active_slot_mount_point + / "boot" + / "ota", + standby_slot_ota_dirpath=self._mp_control.standby_slot_mount_point + / "boot" + / "ota", + ) # ------ for external rootfs, preserve /boot folder to internal ------ # if self._uefi_control._external_rootfs: @@ -496,7 +441,12 @@ def post_update(self) -> Generator[None, None, None]: "copy standby slot rootfs' /boot folder " "to corresponding internal emmc dev ..." ) - self._copy_standby_slot_boot_to_internal_emmc() + copy_standby_slot_boot_to_internal_emmc( + internal_emmc_mp=Path(boot_cfg.SEPARATE_BOOT_MOUNT_POINT), + internal_emmc_devpath=self._uefi_control.standby_internal_emmc_devpath, + standby_slot_boot_dirpath=self._mp_control.standby_slot_mount_point + / "boot", + ) # ------ switch boot to standby ------ # self._uefi_control.switch_boot_to_standby() From 223497432e390d9d8557f4c07a855251518d046b Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Mon, 13 May 2024 08:04:02 +0000 Subject: [PATCH 018/193] jetson-uefi: refine ensure_efivarfs_mounted --- otaclient/app/boot_control/_jetson_uefi.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/otaclient/app/boot_control/_jetson_uefi.py b/otaclient/app/boot_control/_jetson_uefi.py index 8134c8f4f..f16c685ee 100644 --- a/otaclient/app/boot_control/_jetson_uefi.py +++ b/otaclient/app/boot_control/_jetson_uefi.py @@ -125,23 +125,24 @@ def __init__( @classmethod def _ensure_efivarfs_mounted(cls) -> None: - # TODO: remount if efivarfs is mounted as read-only + """Ensure the efivarfs is mounted as rw.""" if CMDHelperFuncs.is_target_mounted(boot_cfg.EFIVARS_DPATH): - return + options = "remount,rw,nosuid,nodev,noexec,relatime" + else: + logger.warning( + f"efivars is not mounted! try to mount it at {boot_cfg.EFIVARS_DPATH}" + ) + options = "rw,nosuid,nodev,noexec,relatime" - logger.warning( - f"efivars is not mounted! try to mount it at {boot_cfg.EFIVARS_DPATH}" - ) # fmt: off cmd = [ "mount", "-t", cls.EFIVARS_FSTYPE, - "-o", "rw,nosuid,nodev,noexec,relatime", + "-o", options, cls.EFIVARS_FSTYPE, boot_cfg.EFIVARS_DPATH ] # fmt: on - try: subprocess_call(cmd, raise_exception=True) except Exception as e: From 86d36ae94a31abffba2c11f0d90c01c0311d8371 Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Mon, 13 May 2024 08:05:37 +0000 Subject: [PATCH 019/193] jetson-uefi: fix prepare_payload --- otaclient/app/boot_control/_jetson_uefi.py | 29 +++++++++++++--------- otaclient/app/boot_control/configs.py | 3 ++- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/otaclient/app/boot_control/_jetson_uefi.py b/otaclient/app/boot_control/_jetson_uefi.py index f16c685ee..628e1d21a 100644 --- a/otaclient/app/boot_control/_jetson_uefi.py +++ b/otaclient/app/boot_control/_jetson_uefi.py @@ -151,7 +151,7 @@ def _ensure_efivarfs_mounted(cls) -> None: ) from e def _prepare_payload(self) -> None: - """Copy the Capsule update payload to the specified location.""" + """Copy the Capsule update payloads to specific location at esp partition.""" try: CMDHelperFuncs.mount_rw(self.esp_part, self.esp_mp) except Exception as e: @@ -159,17 +159,22 @@ def _prepare_payload(self) -> None: logger.error(_err_msg) raise JetsonUEFIBootControlError(_err_msg) from e - capsule_payload_location = Path(self.esp_mp) / boot_cfg.CAPSULE_PAYLOAD_LOCATION - try: - for capsule_fname in boot_cfg.FIRMWARE_LIST: - capsule_payload_fpath = self.standby_slot_mp / capsule_fname - shutil.copy(capsule_payload_fpath, capsule_payload_location) - except Exception as e: - _err_msg = f"failed to copy update Capsule from {capsule_payload_fpath} to {capsule_payload_location}: {e!r}" - logger.error(_err_msg) - raise JetsonUEFIBootControlError(_err_msg) from e - finally: - CMDHelperFuncs.umount(self.esp_mp, raise_exception=False) + capsule_at_esp = Path(self.esp_mp) / boot_cfg.CAPSULE_PAYLOAD_AT_ESP + capsule_at_standby_slot = self.standby_slot_mp / Path( + boot_cfg.CAPSULE_PAYLOAD_AT_ROOTFS + ).relative_to("/") + for capsule_fname in boot_cfg.FIRMWARE_LIST: + try: + shutil.copy( + src=capsule_at_standby_slot / capsule_fname, + dst=capsule_at_esp / capsule_fname, + ) + except Exception as e: + logger.warning( + f"failed to copy {capsule_fname} from {capsule_at_standby_slot} to {capsule_at_esp}: {e!r}" + ) + logger.warning(f"skip {capsule_fname}") + CMDHelperFuncs.umount(self.esp_mp, raise_exception=False) def _write_efivar(self) -> None: """Write magic efivar to trigger firmware Capsule update in next boot. diff --git a/otaclient/app/boot_control/configs.py b/otaclient/app/boot_control/configs.py index 373afe45c..1b20f4eda 100644 --- a/otaclient/app/boot_control/configs.py +++ b/otaclient/app/boot_control/configs.py @@ -69,7 +69,8 @@ class JetsonUEFIBootControlConfig(JetsonBootCommon): EFIVARS_DPATH = "/sys/firmware/efi/efivars/" UPDATE_TRIGGER_EFIVAR = "OsIndications-8be4df61-93ca-11d2-aa0d-00e098032b8c" MAGIC_BYTES = b"\x07\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00" - CAPSULE_PAYLOAD_LOCATION = "EFI/UpdateCapsule" + CAPSULE_PAYLOAD_AT_ESP = "EFI/UpdateCapsule" + CAPSULE_PAYLOAD_AT_ROOTFS = "/opt/ota_package/" @dataclass From 276bfdcee2f8ebd6f4745db4ad09b30f74b055df Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Mon, 13 May 2024 08:08:22 +0000 Subject: [PATCH 020/193] fix test_jetson_cboot --- otaclient/app/boot_control/_jetson_uefi.py | 10 +++++++--- tests/test_boot_control/test_jetson_cboot.py | 3 ++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/otaclient/app/boot_control/_jetson_uefi.py b/otaclient/app/boot_control/_jetson_uefi.py index 628e1d21a..701bb5bd3 100644 --- a/otaclient/app/boot_control/_jetson_uefi.py +++ b/otaclient/app/boot_control/_jetson_uefi.py @@ -231,12 +231,13 @@ def __init__(self): logger.info(f"{bsp_version=}") # ------ sanity check, currently jetson-uefi only supports >= R35.2 ----- # + # only after R35.2, the Capsule Firmware update is available. if not bsp_version >= (35, 2, 0): _err_msg = f"jetson-uefi only supports BSP version >= R35.2, but get {bsp_version=}. " logger.error(_err_msg) raise JetsonUEFIBootControlError(_err_msg) - # NOTE: unified A/B is enabled by default and cannot be disabled after BSP R34. + # unified A/B is enabled by default and cannot be disabled after BSP R34. logger.info("unified A/B is enabled") # ------ check A/B slots ------ # @@ -252,6 +253,7 @@ def __init__(self): CMDHelperFuncs.get_parent_dev(current_rootfs_devpath).strip() ) + # --- detect boot device --- # self._external_rootfs = False parent_devname = parent_devpath.name if parent_devname.startswith(boot_cfg.MMCBLK_DEV_PREFIX): @@ -270,7 +272,7 @@ def __init__(self): f"/dev/{parent_devname}p{self._slot_id_partid[standby_slot]}" ) self.standby_rootfs_dev_partuuid = CMDHelperFuncs.get_attrs_by_dev( - "PARTUUID", f"{self.standby_rootfs_devpath}" + "PARTUUID", self.standby_rootfs_devpath ).strip() current_rootfs_dev_partuuid = CMDHelperFuncs.get_attrs_by_dev( "PARTUUID", current_rootfs_devpath @@ -282,7 +284,6 @@ def __init__(self): f"standby_rootfs(slot {standby_slot}): {self.standby_rootfs_devpath=}, {self.standby_rootfs_dev_partuuid=}" ) - # internal emmc partition self.standby_internal_emmc_devpath = f"/dev/{boot_cfg.INTERNAL_EMMC_DEVNAME}p{self._slot_id_partid[standby_slot]}" logger.info("finished jetson-uefi boot control startup") @@ -323,6 +324,7 @@ def __init__(self) -> None: active_slot_dev=self._uefi_control.curent_rootfs_devpath, active_slot_mount_point=cfg.ACTIVE_ROOT_MOUNT_POINT, ) + current_ota_status_dir = Path(boot_cfg.OTA_STATUS_DIR) standby_ota_status_dir = self._mp_control.standby_slot_mount_point / Path( boot_cfg.OTA_STATUS_DIR @@ -332,6 +334,8 @@ def __init__(self) -> None: self._firmware_ver_control = FirmwareBSPVersionControl( current_firmware_bsp_vf=current_ota_status_dir / boot_cfg.FIRMWARE_BSP_VERSION_FNAME, + # NOTE: standby slot's bsp version file might be not yet + # available before an OTA. standby_firmware_bsp_vf=standby_ota_status_dir / boot_cfg.FIRMWARE_BSP_VERSION_FNAME, ) diff --git a/tests/test_boot_control/test_jetson_cboot.py b/tests/test_boot_control/test_jetson_cboot.py index ce12fdf7a..8183eb57f 100644 --- a/tests/test_boot_control/test_jetson_cboot.py +++ b/tests/test_boot_control/test_jetson_cboot.py @@ -31,6 +31,7 @@ FirmwareBSPVersion, SlotID, parse_bsp_version, + update_extlinux_cfg, ) logger = logging.getLogger(__name__) @@ -140,4 +141,4 @@ def test_parse_bsp_version(_in: str, expected: BSPVersion): def test_update_extlinux_conf(_template_f: Path, _updated_f: Path, partuuid: str): _in = (TEST_DIR / _template_f).read_text() _expected = (TEST_DIR / _updated_f).read_text() - assert _CBootControl.update_extlinux_cfg(_in, partuuid) == _expected + assert update_extlinux_cfg(_in, partuuid) == _expected From c30087404eaf8f444fc7854b9ffaa70d321be535 Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Wed, 15 May 2024 03:56:48 +0000 Subject: [PATCH 021/193] boot_ctrl.selecter: add jetson-uefi --- otaclient/app/boot_control/selecter.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/otaclient/app/boot_control/selecter.py b/otaclient/app/boot_control/selecter.py index 6a771bae7..7e08982fa 100644 --- a/otaclient/app/boot_control/selecter.py +++ b/otaclient/app/boot_control/selecter.py @@ -89,6 +89,12 @@ def get_boot_controller( from ._jetson_cboot import JetsonCBootControl return JetsonCBootControl + + if bootloader_type == BootloaderType.JETSON_UEFI: + from ._jetson_uefi import JetsonUEFIBootControl + + return JetsonUEFIBootControl + if bootloader_type == BootloaderType.RPI_BOOT: from ._rpi_boot import RPIBootController From a4466db08947dad483f144ca51de6cd8536cb6ee Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Wed, 15 May 2024 04:04:03 +0000 Subject: [PATCH 022/193] jetson_boot: the /sys/module/tegra_fuse/parameters/tegra_chip_id only exists on xavier --- otaclient/app/boot_control/configs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/otaclient/app/boot_control/configs.py b/otaclient/app/boot_control/configs.py index 1b20f4eda..b2c1f44ee 100644 --- a/otaclient/app/boot_control/configs.py +++ b/otaclient/app/boot_control/configs.py @@ -36,7 +36,6 @@ class GrubControlConfig(BaseConfig): class JetsonBootCommon: - TEGRA_CHIP_ID_PATH = "/sys/module/tegra_fuse/parameters/tegra_chip_id" OTA_STATUS_DIR = "/boot/ota-status" FIRMWARE_BSP_VERSION_FNAME = "firmware_bsp_version" EXTLINUX_FILE = "/boot/extlinux/extlinux.conf" @@ -58,6 +57,8 @@ class JetsonCBootControlConfig(JetsonBootCommon): """ BOOTLOADER = BootloaderType.JETSON_CBOOT + # this path only exists on xavier + TEGRA_CHIP_ID_PATH = "/sys/module/tegra_fuse/parameters/tegra_chip_id" FIRMWARE_LIST = ["bl_only_payload", "xusb_only_payload"] From 2636bfeb807ed5ee5f95688fd4f96201f5131291 Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Wed, 15 May 2024 04:12:34 +0000 Subject: [PATCH 023/193] jetson-uefi: use a new method to check if it is jetson device --- otaclient/app/boot_control/_jetson_uefi.py | 16 ++++++++++++---- otaclient/app/boot_control/configs.py | 1 + 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/otaclient/app/boot_control/_jetson_uefi.py b/otaclient/app/boot_control/_jetson_uefi.py index 701bb5bd3..b2057471f 100644 --- a/otaclient/app/boot_control/_jetson_uefi.py +++ b/otaclient/app/boot_control/_jetson_uefi.py @@ -212,12 +212,20 @@ class _UEFIBoot: def __init__(self): # ------ sanity check, confirm we are at jetson device ------ # - if not os.path.exists(boot_cfg.TEGRA_CHIP_ID_PATH): - _err_msg = ( - f"not a jetson device, {boot_cfg.TEGRA_CHIP_ID_PATH} doesn't exist" - ) + tegra_compat_info_fpath = Path(boot_cfg.TEGRA_COMPAT_PATH) + if not tegra_compat_info_fpath.is_file(): + _err_msg = f"not a jetson device, {tegra_compat_info_fpath} doesn't exist" + logger.error(_err_msg) + raise JetsonUEFIBootControlError(_err_msg) + + compat_info = tegra_compat_info_fpath.read_text() + # example compatible string: + # nvidia,p3737-0000+p3701-0000nvidia,tegra234nvidia,tegra23x + if compat_info.find("tegra") == -1: + _err_msg = f"uncompatible device: {compat_info=}" logger.error(_err_msg) raise JetsonUEFIBootControlError(_err_msg) + logger.info(f"dev compatibility: {compat_info}") # ------ check BSP version ------ # try: diff --git a/otaclient/app/boot_control/configs.py b/otaclient/app/boot_control/configs.py index b2c1f44ee..0e042b538 100644 --- a/otaclient/app/boot_control/configs.py +++ b/otaclient/app/boot_control/configs.py @@ -64,6 +64,7 @@ class JetsonCBootControlConfig(JetsonBootCommon): class JetsonUEFIBootControlConfig(JetsonBootCommon): BOOTLOADER = BootloaderType.JETSON_UEFI + TEGRA_COMPAT_PATH = "/sys/firmware/devicetree/base/compatible" FIRMWARE_LIST = ["bl_only_payload.Cap"] ESP_MOUNTPOINT = "/mnt/esp" ESP_PARTLABEL = "esp" From 3b766a1cda19a8f1afe53f50c0f7c52dcaeb253e Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Wed, 15 May 2024 04:17:30 +0000 Subject: [PATCH 024/193] boot_control.common: minor fix --- otaclient/app/boot_control/_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/otaclient/app/boot_control/_common.py b/otaclient/app/boot_control/_common.py index 189287405..f34689139 100644 --- a/otaclient/app/boot_control/_common.py +++ b/otaclient/app/boot_control/_common.py @@ -448,7 +448,7 @@ def _load_status_file(self): _loaded_ota_status = None # initialize ota_status files if not presented/incompleted/invalid - if not _loaded_ota_status: + if _loaded_ota_status is None: logger.info( "ota_status files incompleted/not presented, " f"initializing and set/store status to {wrapper.StatusOta.INITIALIZED.name}..." From 36ada3695835e7a5c5c410c6f68254578414fca0 Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Wed, 15 May 2024 04:43:04 +0000 Subject: [PATCH 025/193] jetson-uefi: fix esp mounting --- otaclient/app/boot_control/_jetson_uefi.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/otaclient/app/boot_control/_jetson_uefi.py b/otaclient/app/boot_control/_jetson_uefi.py index b2057471f..e71e26938 100644 --- a/otaclient/app/boot_control/_jetson_uefi.py +++ b/otaclient/app/boot_control/_jetson_uefi.py @@ -152,14 +152,17 @@ def _ensure_efivarfs_mounted(cls) -> None: def _prepare_payload(self) -> None: """Copy the Capsule update payloads to specific location at esp partition.""" + esp_mp = Path(self.esp_mp) + esp_mp.mkdir(exist_ok=True, parents=True) + try: - CMDHelperFuncs.mount_rw(self.esp_part, self.esp_mp) + CMDHelperFuncs.mount_rw(self.esp_part, esp_mp) except Exception as e: - _err_msg = f"failed to mount {self.esp_part=} to {self.esp_mp}: {e!r}" + _err_msg = f"failed to mount {self.esp_part=} to {esp_mp}: {e!r}" logger.error(_err_msg) raise JetsonUEFIBootControlError(_err_msg) from e - capsule_at_esp = Path(self.esp_mp) / boot_cfg.CAPSULE_PAYLOAD_AT_ESP + capsule_at_esp = esp_mp / boot_cfg.CAPSULE_PAYLOAD_AT_ESP capsule_at_standby_slot = self.standby_slot_mp / Path( boot_cfg.CAPSULE_PAYLOAD_AT_ROOTFS ).relative_to("/") @@ -174,7 +177,7 @@ def _prepare_payload(self) -> None: f"failed to copy {capsule_fname} from {capsule_at_standby_slot} to {capsule_at_esp}: {e!r}" ) logger.warning(f"skip {capsule_fname}") - CMDHelperFuncs.umount(self.esp_mp, raise_exception=False) + CMDHelperFuncs.umount(esp_mp, raise_exception=False) def _write_efivar(self) -> None: """Write magic efivar to trigger firmware Capsule update in next boot. From 112c0598e3c801549bbb1f3dbf0da294ce01d2be Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Wed, 15 May 2024 04:49:03 +0000 Subject: [PATCH 026/193] jetson-uefi: minor fix --- otaclient/app/boot_control/_jetson_uefi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/otaclient/app/boot_control/_jetson_uefi.py b/otaclient/app/boot_control/_jetson_uefi.py index e71e26938..3c2b2d30d 100644 --- a/otaclient/app/boot_control/_jetson_uefi.py +++ b/otaclient/app/boot_control/_jetson_uefi.py @@ -189,7 +189,7 @@ def _write_efivar(self) -> None: Path(boot_cfg.EFIVARS_DPATH) / boot_cfg.UPDATE_TRIGGER_EFIVAR ) try: - with open(magic_efivar_fpath, "rb") as f: + with open(magic_efivar_fpath, "wb") as f: f.write(boot_cfg.MAGIC_BYTES) os.fsync(f.fileno()) except Exception as e: From 9fd6681803fa0df2f64bb317d38f326c1a725a8b Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Wed, 15 May 2024 04:51:05 +0000 Subject: [PATCH 027/193] jetson-uefi: efivarfs doesn't need flush --- otaclient/app/boot_control/_jetson_uefi.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/otaclient/app/boot_control/_jetson_uefi.py b/otaclient/app/boot_control/_jetson_uefi.py index 3c2b2d30d..c2e57b940 100644 --- a/otaclient/app/boot_control/_jetson_uefi.py +++ b/otaclient/app/boot_control/_jetson_uefi.py @@ -21,7 +21,6 @@ from __future__ import annotations import logging -import os import re import shutil from pathlib import Path @@ -189,9 +188,7 @@ def _write_efivar(self) -> None: Path(boot_cfg.EFIVARS_DPATH) / boot_cfg.UPDATE_TRIGGER_EFIVAR ) try: - with open(magic_efivar_fpath, "wb") as f: - f.write(boot_cfg.MAGIC_BYTES) - os.fsync(f.fileno()) + magic_efivar_fpath.write_bytes(boot_cfg.MAGIC_BYTES) except Exception as e: _err_msg = f"failed to write magic bytes into {magic_efivar_fpath}: {e!r}" logger.error(_err_msg) From 91e3ac6ba847e6fa58bc1dd995c8e6534aff9d34 Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Wed, 15 May 2024 05:37:21 +0000 Subject: [PATCH 028/193] jetson-uefi: refactor firmware update logic --- otaclient/app/boot_control/_jetson_uefi.py | 88 +++++++++++++++++----- 1 file changed, 70 insertions(+), 18 deletions(-) diff --git a/otaclient/app/boot_control/_jetson_uefi.py b/otaclient/app/boot_control/_jetson_uefi.py index c2e57b940..19a5c6949 100644 --- a/otaclient/app/boot_control/_jetson_uefi.py +++ b/otaclient/app/boot_control/_jetson_uefi.py @@ -20,11 +20,13 @@ from __future__ import annotations +import contextlib import logging +import os import re import shutil from pathlib import Path -from typing import Generator +from typing import Any, Generator from otaclient.app import errors as ota_errors from otaclient.app.common import subprocess_call @@ -123,7 +125,8 @@ def __init__( self.esp_part = esp_part @classmethod - def _ensure_efivarfs_mounted(cls) -> None: + @contextlib.contextmanager + def _ensure_efivarfs_mounted(cls) -> Generator[None, Any, None]: """Ensure the efivarfs is mounted as rw.""" if CMDHelperFuncs.is_target_mounted(boot_cfg.EFIVARS_DPATH): options = "remount,rw,nosuid,nodev,noexec,relatime" @@ -144,13 +147,19 @@ def _ensure_efivarfs_mounted(cls) -> None: # fmt: on try: subprocess_call(cmd, raise_exception=True) + yield except Exception as e: raise JetsonUEFIBootControlError( f"failed to mount {cls.EFIVARS_FSTYPE} on {boot_cfg.EFIVARS_DPATH}: {e!r}" ) from e - def _prepare_payload(self) -> None: - """Copy the Capsule update payloads to specific location at esp partition.""" + def _prepare_payload(self) -> bool: + """Copy the Capsule update payloads to specific location at esp partition. + + Returns: + True if at least one of the update capsule is prepared, False if no update + capsule is available and configured. + """ esp_mp = Path(self.esp_mp) esp_mp.mkdir(exist_ok=True, parents=True) @@ -165,12 +174,15 @@ def _prepare_payload(self) -> None: capsule_at_standby_slot = self.standby_slot_mp / Path( boot_cfg.CAPSULE_PAYLOAD_AT_ROOTFS ).relative_to("/") + + firmware_package_configured = False for capsule_fname in boot_cfg.FIRMWARE_LIST: try: shutil.copy( src=capsule_at_standby_slot / capsule_fname, dst=capsule_at_esp / capsule_fname, ) + firmware_package_configured = True except Exception as e: logger.warning( f"failed to copy {capsule_fname} from {capsule_at_standby_slot} to {capsule_at_esp}: {e!r}" @@ -178,6 +190,8 @@ def _prepare_payload(self) -> None: logger.warning(f"skip {capsule_fname}") CMDHelperFuncs.umount(esp_mp, raise_exception=False) + return firmware_package_configured + def _write_efivar(self) -> None: """Write magic efivar to trigger firmware Capsule update in next boot. @@ -189,20 +203,26 @@ def _write_efivar(self) -> None: ) try: magic_efivar_fpath.write_bytes(boot_cfg.MAGIC_BYTES) + os.sync() except Exception as e: _err_msg = f"failed to write magic bytes into {magic_efivar_fpath}: {e!r}" logger.error(_err_msg) raise JetsonUEFIBootControlError(_err_msg) from e - def firmware_update(self) -> None: - """Trigger firmware update in next boot.""" - logger.info("prepare for firmware Capsule update ...") - self._ensure_efivarfs_mounted() - self._prepare_payload() - self._write_efivar() - logger.info( - "firmware Capsule update is configured and will be triggerred in next boot" - ) + def firmware_update(self) -> bool: + """Trigger firmware update in next boot. + + Returns: + True if firmware update is configured, False if there is no firmware update. + """ + if not self._prepare_payload(): + logger.info("no firmware file is prepared, skip firmware update") + return False + + with self._ensure_efivarfs_mounted(): + self._write_efivar() + logger.info("firmware update package prepare finished") + return True class _UEFIBoot: @@ -395,6 +415,42 @@ def _finalize_switching_boot(self) -> bool: return False + def _capsule_firmware_update(self) -> None: + """Perform firmware update with UEFI Capsule update.""" + logger.info("jetson-uefi: checking if we need to do firmware update ...") + standby_bootloader_slot = self._uefi_control.standby_slot + standby_firmware_bsp_ver = self._firmware_ver_control.get_version_by_slot( + standby_bootloader_slot + ) + logger.info(f"{standby_bootloader_slot=} BSP ver: {standby_firmware_bsp_ver}") + + # ------ check if we need to do firmware update ------ # + _new_bsp_v_fpath = self._mp_control.standby_slot_mount_point / Path( + boot_cfg.NV_TEGRA_RELEASE_FPATH + ).relative_to("/") + try: + new_bsp_v = parse_bsp_version(_new_bsp_v_fpath.read_text()) + except Exception as e: + logger.warning(f"failed to detect new image's BSP version: {e!r}") + logger.info("skip firmware update due to new image BSP version unknown") + return + + if standby_firmware_bsp_ver and standby_firmware_bsp_ver >= new_bsp_v: + logger.info( + f"{standby_bootloader_slot=} has newer or equal ver of firmware, skip firmware update" + ) + return + + # ------ prepare firmware update ------ # + firmware_updater = CapsuleUpdate( + boot_parent_devpath=self._uefi_control.parent_devpath, + standby_slot_mp=self._mp_control.standby_slot_mount_point, + ) + if firmware_updater.firmware_update(): + logger.info( + f"will update to new firmware version in next reboot: {new_bsp_v=}" + ) + # APIs def get_standby_slot_path(self) -> Path: @@ -436,11 +492,7 @@ def post_update(self) -> Generator[None, None, None]: ) # ------ firmware update ------ # - firmware_updater = CapsuleUpdate( - boot_parent_devpath=self._uefi_control.parent_devpath, - standby_slot_mp=self._mp_control.standby_slot_mount_point, - ) - firmware_updater.firmware_update() + self._capsule_firmware_update() # ------ preserve /boot/ota folder to standby rootfs ------ # preserve_ota_config_files_to_standby( From cf0422d4de0f709f635beff2c632e111d7b3755b Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Wed, 15 May 2024 05:55:07 +0000 Subject: [PATCH 029/193] jetson-uefi: fix firmware update is not triggered --- otaclient/app/boot_control/_jetson_uefi.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/otaclient/app/boot_control/_jetson_uefi.py b/otaclient/app/boot_control/_jetson_uefi.py index 19a5c6949..2c00480eb 100644 --- a/otaclient/app/boot_control/_jetson_uefi.py +++ b/otaclient/app/boot_control/_jetson_uefi.py @@ -415,7 +415,7 @@ def _finalize_switching_boot(self) -> bool: return False - def _capsule_firmware_update(self) -> None: + def _capsule_firmware_update(self) -> bool: """Perform firmware update with UEFI Capsule update.""" logger.info("jetson-uefi: checking if we need to do firmware update ...") standby_bootloader_slot = self._uefi_control.standby_slot @@ -433,13 +433,13 @@ def _capsule_firmware_update(self) -> None: except Exception as e: logger.warning(f"failed to detect new image's BSP version: {e!r}") logger.info("skip firmware update due to new image BSP version unknown") - return + return False if standby_firmware_bsp_ver and standby_firmware_bsp_ver >= new_bsp_v: logger.info( f"{standby_bootloader_slot=} has newer or equal ver of firmware, skip firmware update" ) - return + return False # ------ prepare firmware update ------ # firmware_updater = CapsuleUpdate( @@ -450,6 +450,8 @@ def _capsule_firmware_update(self) -> None: logger.info( f"will update to new firmware version in next reboot: {new_bsp_v=}" ) + return True + return False # APIs @@ -491,9 +493,6 @@ def post_update(self) -> Generator[None, None, None]: standby_slot_partuuid=self._uefi_control.standby_rootfs_dev_partuuid, ) - # ------ firmware update ------ # - self._capsule_firmware_update() - # ------ preserve /boot/ota folder to standby rootfs ------ # preserve_ota_config_files_to_standby( active_slot_ota_dirpath=self._mp_control.active_slot_mount_point @@ -519,7 +518,11 @@ def post_update(self) -> Generator[None, None, None]: ) # ------ switch boot to standby ------ # - self._uefi_control.switch_boot_to_standby() + firmware_update_triggered = self._capsule_firmware_update() + # NOTE: manual switch boot will cancel the firmware update and cancel the switch boot itself! + if not firmware_update_triggered: + logger.info("no firmware update configured, manually switch slot") + self._uefi_control.switch_boot_to_standby() # ------ prepare to reboot ------ # self._mp_control.umount_all(ignore_error=True) From e41e4c495178eb5dd27475bbd3fdd52fa1898647 Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Wed, 15 May 2024 06:19:30 +0000 Subject: [PATCH 030/193] jetson-uefi: fix firmware version is not preserved to standby slot --- otaclient/app/boot_control/_jetson_uefi.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/otaclient/app/boot_control/_jetson_uefi.py b/otaclient/app/boot_control/_jetson_uefi.py index 2c00480eb..3da905143 100644 --- a/otaclient/app/boot_control/_jetson_uefi.py +++ b/otaclient/app/boot_control/_jetson_uefi.py @@ -422,7 +422,9 @@ def _capsule_firmware_update(self) -> bool: standby_firmware_bsp_ver = self._firmware_ver_control.get_version_by_slot( standby_bootloader_slot ) - logger.info(f"{standby_bootloader_slot=} BSP ver: {standby_firmware_bsp_ver}") + logger.info( + f"standby slot current firmware ver: {standby_bootloader_slot=} BSP ver: {standby_firmware_bsp_ver}" + ) # ------ check if we need to do firmware update ------ # _new_bsp_v_fpath = self._mp_control.standby_slot_mount_point / Path( @@ -450,6 +452,9 @@ def _capsule_firmware_update(self) -> bool: logger.info( f"will update to new firmware version in next reboot: {new_bsp_v=}" ) + logger.info( + f"will switch to Slot({standby_bootloader_slot}) on successful firmware update" + ) return True return False @@ -493,6 +498,9 @@ def post_update(self) -> Generator[None, None, None]: standby_slot_partuuid=self._uefi_control.standby_rootfs_dev_partuuid, ) + # ------ write BSP version file ------ # + self._firmware_ver_control.write_standby_firmware_bsp_version() + # ------ preserve /boot/ota folder to standby rootfs ------ # preserve_ota_config_files_to_standby( active_slot_ota_dirpath=self._mp_control.active_slot_mount_point @@ -504,6 +512,7 @@ def post_update(self) -> Generator[None, None, None]: ) # ------ for external rootfs, preserve /boot folder to internal ------ # + # NOTE: the copy should happen AFTER the changes to /boot folder at active slot. if self._uefi_control._external_rootfs: logger.info( "rootfs on external storage enabled: " @@ -521,12 +530,13 @@ def post_update(self) -> Generator[None, None, None]: firmware_update_triggered = self._capsule_firmware_update() # NOTE: manual switch boot will cancel the firmware update and cancel the switch boot itself! if not firmware_update_triggered: - logger.info("no firmware update configured, manually switch slot") self._uefi_control.switch_boot_to_standby() + logger.info( + f"no firmware update configured, manually switch slot: \n{_NVBootctrl.dump_slots_info()}" + ) # ------ prepare to reboot ------ # self._mp_control.umount_all(ignore_error=True) - logger.info(f"[post-update]: \n{_NVBootctrl.dump_slots_info()}") logger.info("post update finished, wait for reboot ...") yield # hand over control back to otaclient CMDHelperFuncs.reboot() From 3bec9639fad88718a06e7de6e73c0b2b262cf053 Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Wed, 15 May 2024 06:35:33 +0000 Subject: [PATCH 031/193] jetson-uefi: copy to internal emmc should happen after firmware update --- otaclient/app/boot_control/_jetson_uefi.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/otaclient/app/boot_control/_jetson_uefi.py b/otaclient/app/boot_control/_jetson_uefi.py index 3da905143..17b743a93 100644 --- a/otaclient/app/boot_control/_jetson_uefi.py +++ b/otaclient/app/boot_control/_jetson_uefi.py @@ -498,7 +498,7 @@ def post_update(self) -> Generator[None, None, None]: standby_slot_partuuid=self._uefi_control.standby_rootfs_dev_partuuid, ) - # ------ write BSP version file ------ # + # ------ preserve BSP version file to standby slot ------ # self._firmware_ver_control.write_standby_firmware_bsp_version() # ------ preserve /boot/ota folder to standby rootfs ------ # @@ -511,6 +511,15 @@ def post_update(self) -> Generator[None, None, None]: / "ota", ) + # ------ switch boot to standby ------ # + firmware_update_triggered = self._capsule_firmware_update() + # NOTE: manual switch boot will cancel the firmware update and cancel the switch boot itself! + if not firmware_update_triggered: + self._uefi_control.switch_boot_to_standby() + logger.info( + f"no firmware update configured, manually switch slot: \n{_NVBootctrl.dump_slots_info()}" + ) + # ------ for external rootfs, preserve /boot folder to internal ------ # # NOTE: the copy should happen AFTER the changes to /boot folder at active slot. if self._uefi_control._external_rootfs: @@ -526,15 +535,6 @@ def post_update(self) -> Generator[None, None, None]: / "boot", ) - # ------ switch boot to standby ------ # - firmware_update_triggered = self._capsule_firmware_update() - # NOTE: manual switch boot will cancel the firmware update and cancel the switch boot itself! - if not firmware_update_triggered: - self._uefi_control.switch_boot_to_standby() - logger.info( - f"no firmware update configured, manually switch slot: \n{_NVBootctrl.dump_slots_info()}" - ) - # ------ prepare to reboot ------ # self._mp_control.umount_all(ignore_error=True) logger.info("post update finished, wait for reboot ...") From 9c24fe2857d5f0b29590ac4db3e33c73a6ac5171 Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Wed, 15 May 2024 06:36:05 +0000 Subject: [PATCH 032/193] -----------------changes above are tested on real ECU--------------- From 481dc91c3d369516e64950185d00a6d8776c0c95 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 28 May 2024 08:39:27 +0000 Subject: [PATCH 033/193] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- otaclient/app/boot_control/_jetson_cboot.py | 2 +- otaclient/app/boot_control/_jetson_common.py | 1 + otaclient/app/boot_control/_jetson_uefi.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/otaclient/app/boot_control/_jetson_cboot.py b/otaclient/app/boot_control/_jetson_cboot.py index 373f23d1c..4d9eb98b9 100644 --- a/otaclient/app/boot_control/_jetson_cboot.py +++ b/otaclient/app/boot_control/_jetson_cboot.py @@ -36,8 +36,8 @@ NVBootctrlCommon, NVBootctrlTarget, SlotID, - parse_bsp_version, copy_standby_slot_boot_to_internal_emmc, + parse_bsp_version, preserve_ota_config_files_to_standby, update_standby_slot_extlinux_cfg, ) diff --git a/otaclient/app/boot_control/_jetson_common.py b/otaclient/app/boot_control/_jetson_common.py index d1ae79682..6a5b26bf1 100644 --- a/otaclient/app/boot_control/_jetson_common.py +++ b/otaclient/app/boot_control/_jetson_common.py @@ -30,6 +30,7 @@ from typing_extensions import Annotated, Literal, Self from otaclient.app.common import write_str_to_file_sync + from ..common import copytree_identical from ._common import CMDHelperFuncs diff --git a/otaclient/app/boot_control/_jetson_uefi.py b/otaclient/app/boot_control/_jetson_uefi.py index 17b743a93..d593c95c2 100644 --- a/otaclient/app/boot_control/_jetson_uefi.py +++ b/otaclient/app/boot_control/_jetson_uefi.py @@ -38,8 +38,8 @@ FirmwareBSPVersionControl, NVBootctrlCommon, SlotID, - parse_bsp_version, copy_standby_slot_boot_to_internal_emmc, + parse_bsp_version, preserve_ota_config_files_to_standby, update_standby_slot_extlinux_cfg, ) From 9927b800ec6622fc2c785fd7169723167e5d9be7 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Tue, 28 May 2024 08:51:02 +0000 Subject: [PATCH 034/193] jetson_uefi: do not write bsp version file if current slot bsp ver is None --- otaclient/app/boot_control/_jetson_uefi.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/otaclient/app/boot_control/_jetson_uefi.py b/otaclient/app/boot_control/_jetson_uefi.py index d593c95c2..5e824fb26 100644 --- a/otaclient/app/boot_control/_jetson_uefi.py +++ b/otaclient/app/boot_control/_jetson_uefi.py @@ -405,8 +405,12 @@ def _finalize_switching_boot(self) -> bool: logger.info("no firmware update occurs") return True - if update_result_status == "1": - logger.info("firmware successfully updated") + # NOTE(20240528): seems like if there is a firmware update ever occurs, + # the Capsule update status will always be 1. So by just looking at + # the Capsule update status we cannot tell if previous OTA contains + # firmware update. + if update_result_status == "1" and current_slot_bsp_ver is not None: + logger.info("the previous firmware update is successful") self._firmware_ver_control.set_version_by_slot( current_slot, current_slot_bsp_ver ) From 9ed43484b7190d9450224d728c404f659bff351c Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 29 May 2024 06:15:37 +0000 Subject: [PATCH 035/193] jetson-common: minor fix --- otaclient/app/boot_control/_jetson_common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/otaclient/app/boot_control/_jetson_common.py b/otaclient/app/boot_control/_jetson_common.py index 6a5b26bf1..7b332c239 100644 --- a/otaclient/app/boot_control/_jetson_common.py +++ b/otaclient/app/boot_control/_jetson_common.py @@ -70,8 +70,8 @@ def parse(cls, _in: str | BSPVersion | Any) -> Self: """Parse "Rxx.yy.z string into BSPVersion.""" if isinstance(_in, cls): return _in - if isinstance(_in, str): - major_ver, major_rev, minor_rev = _in[1:].split(".") + if isinstance(_in, str) and len(_split := _in[1:].split(".")) == 3: + major_ver, major_rev, minor_rev = _split return cls(int(major_ver), int(major_rev), int(minor_rev)) raise ValueError(f"expect str or BSPVersion instance, get {type(_in)}") From dc9e5eb4a6623e81431b69ada7ad0edc988fe6ee Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 29 May 2024 06:23:04 +0000 Subject: [PATCH 036/193] boot_control.configs: jetson-uefi add FIRMWARE_UPDATE_HINT_FNAME --- otaclient/app/boot_control/configs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/otaclient/app/boot_control/configs.py b/otaclient/app/boot_control/configs.py index 0e042b538..ae69a4edc 100644 --- a/otaclient/app/boot_control/configs.py +++ b/otaclient/app/boot_control/configs.py @@ -73,6 +73,7 @@ class JetsonUEFIBootControlConfig(JetsonBootCommon): MAGIC_BYTES = b"\x07\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00" CAPSULE_PAYLOAD_AT_ESP = "EFI/UpdateCapsule" CAPSULE_PAYLOAD_AT_ROOTFS = "/opt/ota_package/" + FIRMWARE_UPDATE_HINT_FNAME = ".firmware_update" @dataclass From ce8982411eb48d20f9503561ddb2d7dddfcff273 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 29 May 2024 06:32:00 +0000 Subject: [PATCH 037/193] jetson-uefi: use a hint file to track firmware update --- otaclient/app/boot_control/_jetson_uefi.py | 93 +++++++++++++++++----- 1 file changed, 72 insertions(+), 21 deletions(-) diff --git a/otaclient/app/boot_control/_jetson_uefi.py b/otaclient/app/boot_control/_jetson_uefi.py index 5e824fb26..c9655e461 100644 --- a/otaclient/app/boot_control/_jetson_uefi.py +++ b/otaclient/app/boot_control/_jetson_uefi.py @@ -28,13 +28,15 @@ from pathlib import Path from typing import Any, Generator +from otaclient._utils.typing import StrOrPath from otaclient.app import errors as ota_errors -from otaclient.app.common import subprocess_call +from otaclient.app.common import subprocess_call, write_str_to_file_sync from otaclient.app.configs import config as cfg from otaclient.app.proto import wrapper from ._common import CMDHelperFuncs, OTAStatusFilesControl, SlotMountHelper from ._jetson_common import ( + BSPVersion, FirmwareBSPVersionControl, NVBootctrlCommon, SlotID, @@ -94,7 +96,7 @@ class CapsuleUpdate: EFIVARS_FSTYPE = "efivarfs" def __init__( - self, boot_parent_devpath: Path | str, standby_slot_mp: Path | str + self, boot_parent_devpath: StrOrPath, standby_slot_mp: StrOrPath ) -> None: # NOTE: use the esp partition at the current booted device # i.e., if we boot from nvme0n1, then bootdev_path is /dev/nvme0n1 and @@ -224,6 +226,30 @@ def firmware_update(self) -> bool: logger.info("firmware update package prepare finished") return True + @staticmethod + def write_firmware_update_hint_file( + hint_fpath: StrOrPath, slot_id: SlotID, bsp_version: BSPVersion + ) -> None: + """When capsule firmware update is scheduled, write this file to + hint the otaclient in the new slot. + + Schema: , + """ + write_str_to_file_sync(hint_fpath, f"{slot_id},{BSPVersion.dump(bsp_version)}") + + @staticmethod + def parse_firmware_update_hint_file( + hint_fpath: StrOrPath, + ) -> tuple[SlotID, BSPVersion]: + """Parse the slot_id and firmware bsp_version from firmware update hint file.""" + _raw = Path(hint_fpath).read_text() + + try: + _slot_id, _bsp_v = _raw.split(",") + return SlotID(_slot_id), BSPVersion.parse(_bsp_v) + except Exception as e: + raise ValueError(f"invalid hint file content: {_raw}: {e!r}") from e + class _UEFIBoot: """Low-level boot control implementation for jetson-uefi.""" @@ -358,6 +384,11 @@ def __init__(self) -> None: boot_cfg.OTA_STATUS_DIR ).relative_to("/") + # NOTE: this hint file is referred by finalize_switching_boot + self.firmware_update_hint_fpath = ( + standby_ota_status_dir / boot_cfg.FIRMWARE_UPDATE_HINT_FNAME + ) + # load firmware BSP version from current rootfs slot self._firmware_ver_control = FirmwareBSPVersionControl( current_firmware_bsp_vf=current_ota_status_dir @@ -377,6 +408,11 @@ def __init__(self) -> None: standby_ota_status_dir=standby_ota_status_dir, finalize_switching_boot=self._finalize_switching_boot, ) + + # NOTE: the hint file is checked during OTAStatusFilesControl __init__, + # by finalize_switching_boot if we are in first reboot after OTA. + # once we have done parsing the hint file, we must remove it immediately. + self.firmware_update_hint_fpath.unlink(missing_ok=True) except Exception as e: _err_msg = f"failed to start jetson-uefi controller: {e!r}" raise ota_errors.BootControlStartupFailed(_err_msg, module=__name__) from e @@ -393,35 +429,44 @@ def _finalize_switching_boot(self) -> bool: current_slot_bsp_ver = self._uefi_control.bsp_version try: - update_result_status = _NVBootctrl.get_capsule_update_result() + slot_id, bsp_v = CapsuleUpdate.parse_firmware_update_hint_file( + self.firmware_update_hint_fpath + ) + except FileNotFoundError: + logger.info("no firmware update occurs in previous OTA") + return True except Exception as e: - _err_msg = ( - f"failed to get the Capsule update result status, assume failed: {e!r}" + logger.error( + ( + f"firmware update hint file presented but invalid: {e!r}" + "assuming firmware update failed" + ) ) - logger.error(_err_msg) return False - if update_result_status == "0": - logger.info("no firmware update occurs") - return True - - # NOTE(20240528): seems like if there is a firmware update ever occurs, - # the Capsule update status will always be 1. So by just looking at - # the Capsule update status we cannot tell if previous OTA contains - # firmware update. - if update_result_status == "1" and current_slot_bsp_ver is not None: - logger.info("the previous firmware update is successful") - self._firmware_ver_control.set_version_by_slot( - current_slot, current_slot_bsp_ver + if slot_id != current_slot: + logger.error( + ( + "firmware update hint file indicates firmware update occurs on " + f"slot {slot_id}, but expects slot {current_slot}" + ) ) - self._firmware_ver_control.write_current_firmware_bsp_version() - return True + return False - return False + if bsp_v != current_slot_bsp_ver: + logger.error( + ( + f"firmware update hint file indicates the firmware on slot {slot_id} is " + f"updated to {bsp_v}, but current slot's BSP version is {current_slot_bsp_ver}" + ) + ) + return False + return True def _capsule_firmware_update(self) -> bool: """Perform firmware update with UEFI Capsule update.""" logger.info("jetson-uefi: checking if we need to do firmware update ...") + standby_bootloader_slot = self._uefi_control.standby_slot standby_firmware_bsp_ver = self._firmware_ver_control.get_version_by_slot( standby_bootloader_slot @@ -453,6 +498,12 @@ def _capsule_firmware_update(self) -> bool: standby_slot_mp=self._mp_control.standby_slot_mount_point, ) if firmware_updater.firmware_update(): + CapsuleUpdate.write_firmware_update_hint_file( + self.firmware_update_hint_fpath, + slot_id=self._uefi_control.standby_slot, + bsp_version=new_bsp_v, + ) + logger.info( f"will update to new firmware version in next reboot: {new_bsp_v=}" ) From e85ac79d32e7f39df68019436d9e33f997caeb6a Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 30 May 2024 06:26:42 +0000 Subject: [PATCH 038/193] jetson-uefi: fix finalize_switching_boot doesn't store firmware version --- otaclient/app/boot_control/_jetson_uefi.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/otaclient/app/boot_control/_jetson_uefi.py b/otaclient/app/boot_control/_jetson_uefi.py index c9655e461..967ac8e43 100644 --- a/otaclient/app/boot_control/_jetson_uefi.py +++ b/otaclient/app/boot_control/_jetson_uefi.py @@ -428,6 +428,10 @@ def _finalize_switching_boot(self) -> bool: current_slot = self._uefi_control.current_slot current_slot_bsp_ver = self._uefi_control.bsp_version + if current_slot_bsp_ver is None: + logger.warning("current slot BSP version is unknown, skip") + return True + try: slot_id, bsp_v = CapsuleUpdate.parse_firmware_update_hint_file( self.firmware_update_hint_fpath @@ -444,6 +448,8 @@ def _finalize_switching_boot(self) -> bool: ) return False + # ------ firmware update occurs, check hints ------ # + if slot_id != current_slot: logger.error( ( @@ -461,6 +467,12 @@ def _finalize_switching_boot(self) -> bool: ) ) return False + + # ------ firmware update succeeded, write firmware version file ------ # + self._firmware_ver_control.set_version_by_slot( + current_slot, current_slot_bsp_ver + ) + self._firmware_ver_control.write_current_firmware_bsp_version() return True def _capsule_firmware_update(self) -> bool: From 2d34dba780215f5aa43f136c3c2391fa25bcc89a Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 30 May 2024 06:28:51 +0000 Subject: [PATCH 039/193] jetson-uefi: remove out-of-date docs --- otaclient/app/boot_control/_jetson_uefi.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/otaclient/app/boot_control/_jetson_uefi.py b/otaclient/app/boot_control/_jetson_uefi.py index 967ac8e43..ffd94f6e5 100644 --- a/otaclient/app/boot_control/_jetson_uefi.py +++ b/otaclient/app/boot_control/_jetson_uefi.py @@ -418,13 +418,7 @@ def __init__(self) -> None: raise ota_errors.BootControlStartupFailed(_err_msg, module=__name__) from e def _finalize_switching_boot(self) -> bool: - """Verify firmware update result and write firmware BSP version file. - - NOTE: due to unified A/B is enabled, actually it is impossible to boot to - a firmware updated failed slot. - Since finalize_switching_boot is only called when first reboot succeeds, - we can only observe result status 0 or 1 here. - """ + """Verify firmware update result and write firmware BSP version file.""" current_slot = self._uefi_control.current_slot current_slot_bsp_ver = self._uefi_control.bsp_version From ee15377e9a232dac1bc10a73e7d40aad92123bd3 Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Fri, 31 May 2024 01:47:48 +0000 Subject: [PATCH 040/193] partially apply src layout, ready for merge the main branch --- {otaclient => src}/ota_proxy/README.md | 0 {otaclient => src}/ota_proxy/__init__.py | 0 {otaclient => src}/ota_proxy/__main__.py | 0 {otaclient => src}/ota_proxy/_consts.py | 0 {otaclient => src}/ota_proxy/cache_control.py | 0 {otaclient => src}/ota_proxy/config.py | 0 {otaclient => src}/ota_proxy/db.py | 0 {otaclient => src}/ota_proxy/errors.py | 0 {otaclient => src}/ota_proxy/orm.py | 0 {otaclient => src}/ota_proxy/ota_cache.py | 0 {otaclient => src}/ota_proxy/server_app.py | 0 {otaclient => src}/ota_proxy/utils.py | 0 {otaclient => src/otaclient}/.gitignore | 0 {otaclient => src/otaclient}/__init__.py | 0 {otaclient => src/otaclient}/__main__.py | 0 {otaclient => src/otaclient}/_utils/__init__.py | 0 {otaclient => src/otaclient}/_utils/linux.py | 0 {otaclient => src/otaclient}/_utils/logging.py | 0 {otaclient => src/otaclient}/_utils/typing.py | 0 {otaclient => src/otaclient}/app/__init__.py | 0 {otaclient => src/otaclient}/app/__main__.py | 0 {otaclient => src/otaclient}/app/boot_control/__init__.py | 0 {otaclient => src/otaclient}/app/boot_control/_common.py | 0 {otaclient => src/otaclient}/app/boot_control/_grub.py | 0 {otaclient => src/otaclient}/app/boot_control/_jetson_cboot.py | 0 {otaclient => src/otaclient}/app/boot_control/_jetson_common.py | 0 {otaclient => src/otaclient}/app/boot_control/_jetson_uefi.py | 0 {otaclient => src/otaclient}/app/boot_control/_rpi_boot.py | 0 {otaclient => src/otaclient}/app/boot_control/configs.py | 0 {otaclient => src/otaclient}/app/boot_control/protocol.py | 0 {otaclient => src/otaclient}/app/boot_control/selecter.py | 0 {otaclient => src/otaclient}/app/common.py | 0 {otaclient => src/otaclient}/app/configs.py | 0 {otaclient => src/otaclient}/app/create_standby/__init__.py | 0 {otaclient => src/otaclient}/app/create_standby/common.py | 0 {otaclient => src/otaclient}/app/create_standby/interface.py | 0 {otaclient => src/otaclient}/app/create_standby/rebuild_mode.py | 0 {otaclient => src/otaclient}/app/downloader.py | 0 {otaclient => src/otaclient}/app/errors.py | 0 {otaclient => src/otaclient}/app/interface.py | 0 {otaclient => src/otaclient}/app/log_setting.py | 0 {otaclient => src/otaclient}/app/main.py | 0 {otaclient => src/otaclient}/app/ota_client.py | 0 {otaclient => src/otaclient}/app/ota_client_call.py | 0 {otaclient => src/otaclient}/app/ota_client_service.py | 0 {otaclient => src/otaclient}/app/ota_client_stub.py | 0 {otaclient => src/otaclient}/app/ota_metadata.py | 0 {otaclient => src/otaclient}/app/ota_status.py | 0 {otaclient => src/otaclient}/app/proto/README.md | 0 {otaclient => src/otaclient}/app/proto/__init__.py | 0 {otaclient => src/otaclient}/app/proto/_common.py | 0 {otaclient => src/otaclient}/app/proto/_ota_metafiles_wrapper.py | 0 .../otaclient}/app/proto/_otaclient_v2_pb2_wrapper.py | 0 {otaclient => src/otaclient}/app/proto/ota_metafiles_pb2.py | 0 {otaclient => src/otaclient}/app/proto/ota_metafiles_pb2.pyi | 0 {otaclient => src/otaclient}/app/proto/otaclient_v2_pb2.py | 0 {otaclient => src/otaclient}/app/proto/otaclient_v2_pb2.pyi | 0 {otaclient => src/otaclient}/app/proto/otaclient_v2_pb2_grpc.py | 0 {otaclient => src/otaclient}/app/proto/streamer.py | 0 {otaclient => src/otaclient}/app/proto/wrapper.py | 0 {otaclient => src/otaclient}/app/update_stats.py | 0 {otaclient => src/otaclient}/configs/__init__.py | 0 {otaclient => src/otaclient}/configs/_common.py | 0 {otaclient => src/otaclient}/configs/ecu_info.py | 0 {otaclient => src/otaclient}/configs/proxy_info.py | 0 {otaclient => src/otaclient}/requirements.txt | 0 66 files changed, 0 insertions(+), 0 deletions(-) rename {otaclient => src}/ota_proxy/README.md (100%) rename {otaclient => src}/ota_proxy/__init__.py (100%) rename {otaclient => src}/ota_proxy/__main__.py (100%) rename {otaclient => src}/ota_proxy/_consts.py (100%) rename {otaclient => src}/ota_proxy/cache_control.py (100%) rename {otaclient => src}/ota_proxy/config.py (100%) rename {otaclient => src}/ota_proxy/db.py (100%) rename {otaclient => src}/ota_proxy/errors.py (100%) rename {otaclient => src}/ota_proxy/orm.py (100%) rename {otaclient => src}/ota_proxy/ota_cache.py (100%) rename {otaclient => src}/ota_proxy/server_app.py (100%) rename {otaclient => src}/ota_proxy/utils.py (100%) rename {otaclient => src/otaclient}/.gitignore (100%) rename {otaclient => src/otaclient}/__init__.py (100%) rename {otaclient => src/otaclient}/__main__.py (100%) rename {otaclient => src/otaclient}/_utils/__init__.py (100%) rename {otaclient => src/otaclient}/_utils/linux.py (100%) rename {otaclient => src/otaclient}/_utils/logging.py (100%) rename {otaclient => src/otaclient}/_utils/typing.py (100%) rename {otaclient => src/otaclient}/app/__init__.py (100%) rename {otaclient => src/otaclient}/app/__main__.py (100%) rename {otaclient => src/otaclient}/app/boot_control/__init__.py (100%) rename {otaclient => src/otaclient}/app/boot_control/_common.py (100%) rename {otaclient => src/otaclient}/app/boot_control/_grub.py (100%) rename {otaclient => src/otaclient}/app/boot_control/_jetson_cboot.py (100%) rename {otaclient => src/otaclient}/app/boot_control/_jetson_common.py (100%) rename {otaclient => src/otaclient}/app/boot_control/_jetson_uefi.py (100%) rename {otaclient => src/otaclient}/app/boot_control/_rpi_boot.py (100%) rename {otaclient => src/otaclient}/app/boot_control/configs.py (100%) rename {otaclient => src/otaclient}/app/boot_control/protocol.py (100%) rename {otaclient => src/otaclient}/app/boot_control/selecter.py (100%) rename {otaclient => src/otaclient}/app/common.py (100%) rename {otaclient => src/otaclient}/app/configs.py (100%) rename {otaclient => src/otaclient}/app/create_standby/__init__.py (100%) rename {otaclient => src/otaclient}/app/create_standby/common.py (100%) rename {otaclient => src/otaclient}/app/create_standby/interface.py (100%) rename {otaclient => src/otaclient}/app/create_standby/rebuild_mode.py (100%) rename {otaclient => src/otaclient}/app/downloader.py (100%) rename {otaclient => src/otaclient}/app/errors.py (100%) rename {otaclient => src/otaclient}/app/interface.py (100%) rename {otaclient => src/otaclient}/app/log_setting.py (100%) rename {otaclient => src/otaclient}/app/main.py (100%) rename {otaclient => src/otaclient}/app/ota_client.py (100%) rename {otaclient => src/otaclient}/app/ota_client_call.py (100%) rename {otaclient => src/otaclient}/app/ota_client_service.py (100%) rename {otaclient => src/otaclient}/app/ota_client_stub.py (100%) rename {otaclient => src/otaclient}/app/ota_metadata.py (100%) rename {otaclient => src/otaclient}/app/ota_status.py (100%) rename {otaclient => src/otaclient}/app/proto/README.md (100%) rename {otaclient => src/otaclient}/app/proto/__init__.py (100%) rename {otaclient => src/otaclient}/app/proto/_common.py (100%) rename {otaclient => src/otaclient}/app/proto/_ota_metafiles_wrapper.py (100%) rename {otaclient => src/otaclient}/app/proto/_otaclient_v2_pb2_wrapper.py (100%) rename {otaclient => src/otaclient}/app/proto/ota_metafiles_pb2.py (100%) rename {otaclient => src/otaclient}/app/proto/ota_metafiles_pb2.pyi (100%) rename {otaclient => src/otaclient}/app/proto/otaclient_v2_pb2.py (100%) rename {otaclient => src/otaclient}/app/proto/otaclient_v2_pb2.pyi (100%) rename {otaclient => src/otaclient}/app/proto/otaclient_v2_pb2_grpc.py (100%) rename {otaclient => src/otaclient}/app/proto/streamer.py (100%) rename {otaclient => src/otaclient}/app/proto/wrapper.py (100%) rename {otaclient => src/otaclient}/app/update_stats.py (100%) rename {otaclient => src/otaclient}/configs/__init__.py (100%) rename {otaclient => src/otaclient}/configs/_common.py (100%) rename {otaclient => src/otaclient}/configs/ecu_info.py (100%) rename {otaclient => src/otaclient}/configs/proxy_info.py (100%) rename {otaclient => src/otaclient}/requirements.txt (100%) diff --git a/otaclient/ota_proxy/README.md b/src/ota_proxy/README.md similarity index 100% rename from otaclient/ota_proxy/README.md rename to src/ota_proxy/README.md diff --git a/otaclient/ota_proxy/__init__.py b/src/ota_proxy/__init__.py similarity index 100% rename from otaclient/ota_proxy/__init__.py rename to src/ota_proxy/__init__.py diff --git a/otaclient/ota_proxy/__main__.py b/src/ota_proxy/__main__.py similarity index 100% rename from otaclient/ota_proxy/__main__.py rename to src/ota_proxy/__main__.py diff --git a/otaclient/ota_proxy/_consts.py b/src/ota_proxy/_consts.py similarity index 100% rename from otaclient/ota_proxy/_consts.py rename to src/ota_proxy/_consts.py diff --git a/otaclient/ota_proxy/cache_control.py b/src/ota_proxy/cache_control.py similarity index 100% rename from otaclient/ota_proxy/cache_control.py rename to src/ota_proxy/cache_control.py diff --git a/otaclient/ota_proxy/config.py b/src/ota_proxy/config.py similarity index 100% rename from otaclient/ota_proxy/config.py rename to src/ota_proxy/config.py diff --git a/otaclient/ota_proxy/db.py b/src/ota_proxy/db.py similarity index 100% rename from otaclient/ota_proxy/db.py rename to src/ota_proxy/db.py diff --git a/otaclient/ota_proxy/errors.py b/src/ota_proxy/errors.py similarity index 100% rename from otaclient/ota_proxy/errors.py rename to src/ota_proxy/errors.py diff --git a/otaclient/ota_proxy/orm.py b/src/ota_proxy/orm.py similarity index 100% rename from otaclient/ota_proxy/orm.py rename to src/ota_proxy/orm.py diff --git a/otaclient/ota_proxy/ota_cache.py b/src/ota_proxy/ota_cache.py similarity index 100% rename from otaclient/ota_proxy/ota_cache.py rename to src/ota_proxy/ota_cache.py diff --git a/otaclient/ota_proxy/server_app.py b/src/ota_proxy/server_app.py similarity index 100% rename from otaclient/ota_proxy/server_app.py rename to src/ota_proxy/server_app.py diff --git a/otaclient/ota_proxy/utils.py b/src/ota_proxy/utils.py similarity index 100% rename from otaclient/ota_proxy/utils.py rename to src/ota_proxy/utils.py diff --git a/otaclient/.gitignore b/src/otaclient/.gitignore similarity index 100% rename from otaclient/.gitignore rename to src/otaclient/.gitignore diff --git a/otaclient/__init__.py b/src/otaclient/__init__.py similarity index 100% rename from otaclient/__init__.py rename to src/otaclient/__init__.py diff --git a/otaclient/__main__.py b/src/otaclient/__main__.py similarity index 100% rename from otaclient/__main__.py rename to src/otaclient/__main__.py diff --git a/otaclient/_utils/__init__.py b/src/otaclient/_utils/__init__.py similarity index 100% rename from otaclient/_utils/__init__.py rename to src/otaclient/_utils/__init__.py diff --git a/otaclient/_utils/linux.py b/src/otaclient/_utils/linux.py similarity index 100% rename from otaclient/_utils/linux.py rename to src/otaclient/_utils/linux.py diff --git a/otaclient/_utils/logging.py b/src/otaclient/_utils/logging.py similarity index 100% rename from otaclient/_utils/logging.py rename to src/otaclient/_utils/logging.py diff --git a/otaclient/_utils/typing.py b/src/otaclient/_utils/typing.py similarity index 100% rename from otaclient/_utils/typing.py rename to src/otaclient/_utils/typing.py diff --git a/otaclient/app/__init__.py b/src/otaclient/app/__init__.py similarity index 100% rename from otaclient/app/__init__.py rename to src/otaclient/app/__init__.py diff --git a/otaclient/app/__main__.py b/src/otaclient/app/__main__.py similarity index 100% rename from otaclient/app/__main__.py rename to src/otaclient/app/__main__.py diff --git a/otaclient/app/boot_control/__init__.py b/src/otaclient/app/boot_control/__init__.py similarity index 100% rename from otaclient/app/boot_control/__init__.py rename to src/otaclient/app/boot_control/__init__.py diff --git a/otaclient/app/boot_control/_common.py b/src/otaclient/app/boot_control/_common.py similarity index 100% rename from otaclient/app/boot_control/_common.py rename to src/otaclient/app/boot_control/_common.py diff --git a/otaclient/app/boot_control/_grub.py b/src/otaclient/app/boot_control/_grub.py similarity index 100% rename from otaclient/app/boot_control/_grub.py rename to src/otaclient/app/boot_control/_grub.py diff --git a/otaclient/app/boot_control/_jetson_cboot.py b/src/otaclient/app/boot_control/_jetson_cboot.py similarity index 100% rename from otaclient/app/boot_control/_jetson_cboot.py rename to src/otaclient/app/boot_control/_jetson_cboot.py diff --git a/otaclient/app/boot_control/_jetson_common.py b/src/otaclient/app/boot_control/_jetson_common.py similarity index 100% rename from otaclient/app/boot_control/_jetson_common.py rename to src/otaclient/app/boot_control/_jetson_common.py diff --git a/otaclient/app/boot_control/_jetson_uefi.py b/src/otaclient/app/boot_control/_jetson_uefi.py similarity index 100% rename from otaclient/app/boot_control/_jetson_uefi.py rename to src/otaclient/app/boot_control/_jetson_uefi.py diff --git a/otaclient/app/boot_control/_rpi_boot.py b/src/otaclient/app/boot_control/_rpi_boot.py similarity index 100% rename from otaclient/app/boot_control/_rpi_boot.py rename to src/otaclient/app/boot_control/_rpi_boot.py diff --git a/otaclient/app/boot_control/configs.py b/src/otaclient/app/boot_control/configs.py similarity index 100% rename from otaclient/app/boot_control/configs.py rename to src/otaclient/app/boot_control/configs.py diff --git a/otaclient/app/boot_control/protocol.py b/src/otaclient/app/boot_control/protocol.py similarity index 100% rename from otaclient/app/boot_control/protocol.py rename to src/otaclient/app/boot_control/protocol.py diff --git a/otaclient/app/boot_control/selecter.py b/src/otaclient/app/boot_control/selecter.py similarity index 100% rename from otaclient/app/boot_control/selecter.py rename to src/otaclient/app/boot_control/selecter.py diff --git a/otaclient/app/common.py b/src/otaclient/app/common.py similarity index 100% rename from otaclient/app/common.py rename to src/otaclient/app/common.py diff --git a/otaclient/app/configs.py b/src/otaclient/app/configs.py similarity index 100% rename from otaclient/app/configs.py rename to src/otaclient/app/configs.py diff --git a/otaclient/app/create_standby/__init__.py b/src/otaclient/app/create_standby/__init__.py similarity index 100% rename from otaclient/app/create_standby/__init__.py rename to src/otaclient/app/create_standby/__init__.py diff --git a/otaclient/app/create_standby/common.py b/src/otaclient/app/create_standby/common.py similarity index 100% rename from otaclient/app/create_standby/common.py rename to src/otaclient/app/create_standby/common.py diff --git a/otaclient/app/create_standby/interface.py b/src/otaclient/app/create_standby/interface.py similarity index 100% rename from otaclient/app/create_standby/interface.py rename to src/otaclient/app/create_standby/interface.py diff --git a/otaclient/app/create_standby/rebuild_mode.py b/src/otaclient/app/create_standby/rebuild_mode.py similarity index 100% rename from otaclient/app/create_standby/rebuild_mode.py rename to src/otaclient/app/create_standby/rebuild_mode.py diff --git a/otaclient/app/downloader.py b/src/otaclient/app/downloader.py similarity index 100% rename from otaclient/app/downloader.py rename to src/otaclient/app/downloader.py diff --git a/otaclient/app/errors.py b/src/otaclient/app/errors.py similarity index 100% rename from otaclient/app/errors.py rename to src/otaclient/app/errors.py diff --git a/otaclient/app/interface.py b/src/otaclient/app/interface.py similarity index 100% rename from otaclient/app/interface.py rename to src/otaclient/app/interface.py diff --git a/otaclient/app/log_setting.py b/src/otaclient/app/log_setting.py similarity index 100% rename from otaclient/app/log_setting.py rename to src/otaclient/app/log_setting.py diff --git a/otaclient/app/main.py b/src/otaclient/app/main.py similarity index 100% rename from otaclient/app/main.py rename to src/otaclient/app/main.py diff --git a/otaclient/app/ota_client.py b/src/otaclient/app/ota_client.py similarity index 100% rename from otaclient/app/ota_client.py rename to src/otaclient/app/ota_client.py diff --git a/otaclient/app/ota_client_call.py b/src/otaclient/app/ota_client_call.py similarity index 100% rename from otaclient/app/ota_client_call.py rename to src/otaclient/app/ota_client_call.py diff --git a/otaclient/app/ota_client_service.py b/src/otaclient/app/ota_client_service.py similarity index 100% rename from otaclient/app/ota_client_service.py rename to src/otaclient/app/ota_client_service.py diff --git a/otaclient/app/ota_client_stub.py b/src/otaclient/app/ota_client_stub.py similarity index 100% rename from otaclient/app/ota_client_stub.py rename to src/otaclient/app/ota_client_stub.py diff --git a/otaclient/app/ota_metadata.py b/src/otaclient/app/ota_metadata.py similarity index 100% rename from otaclient/app/ota_metadata.py rename to src/otaclient/app/ota_metadata.py diff --git a/otaclient/app/ota_status.py b/src/otaclient/app/ota_status.py similarity index 100% rename from otaclient/app/ota_status.py rename to src/otaclient/app/ota_status.py diff --git a/otaclient/app/proto/README.md b/src/otaclient/app/proto/README.md similarity index 100% rename from otaclient/app/proto/README.md rename to src/otaclient/app/proto/README.md diff --git a/otaclient/app/proto/__init__.py b/src/otaclient/app/proto/__init__.py similarity index 100% rename from otaclient/app/proto/__init__.py rename to src/otaclient/app/proto/__init__.py diff --git a/otaclient/app/proto/_common.py b/src/otaclient/app/proto/_common.py similarity index 100% rename from otaclient/app/proto/_common.py rename to src/otaclient/app/proto/_common.py diff --git a/otaclient/app/proto/_ota_metafiles_wrapper.py b/src/otaclient/app/proto/_ota_metafiles_wrapper.py similarity index 100% rename from otaclient/app/proto/_ota_metafiles_wrapper.py rename to src/otaclient/app/proto/_ota_metafiles_wrapper.py diff --git a/otaclient/app/proto/_otaclient_v2_pb2_wrapper.py b/src/otaclient/app/proto/_otaclient_v2_pb2_wrapper.py similarity index 100% rename from otaclient/app/proto/_otaclient_v2_pb2_wrapper.py rename to src/otaclient/app/proto/_otaclient_v2_pb2_wrapper.py diff --git a/otaclient/app/proto/ota_metafiles_pb2.py b/src/otaclient/app/proto/ota_metafiles_pb2.py similarity index 100% rename from otaclient/app/proto/ota_metafiles_pb2.py rename to src/otaclient/app/proto/ota_metafiles_pb2.py diff --git a/otaclient/app/proto/ota_metafiles_pb2.pyi b/src/otaclient/app/proto/ota_metafiles_pb2.pyi similarity index 100% rename from otaclient/app/proto/ota_metafiles_pb2.pyi rename to src/otaclient/app/proto/ota_metafiles_pb2.pyi diff --git a/otaclient/app/proto/otaclient_v2_pb2.py b/src/otaclient/app/proto/otaclient_v2_pb2.py similarity index 100% rename from otaclient/app/proto/otaclient_v2_pb2.py rename to src/otaclient/app/proto/otaclient_v2_pb2.py diff --git a/otaclient/app/proto/otaclient_v2_pb2.pyi b/src/otaclient/app/proto/otaclient_v2_pb2.pyi similarity index 100% rename from otaclient/app/proto/otaclient_v2_pb2.pyi rename to src/otaclient/app/proto/otaclient_v2_pb2.pyi diff --git a/otaclient/app/proto/otaclient_v2_pb2_grpc.py b/src/otaclient/app/proto/otaclient_v2_pb2_grpc.py similarity index 100% rename from otaclient/app/proto/otaclient_v2_pb2_grpc.py rename to src/otaclient/app/proto/otaclient_v2_pb2_grpc.py diff --git a/otaclient/app/proto/streamer.py b/src/otaclient/app/proto/streamer.py similarity index 100% rename from otaclient/app/proto/streamer.py rename to src/otaclient/app/proto/streamer.py diff --git a/otaclient/app/proto/wrapper.py b/src/otaclient/app/proto/wrapper.py similarity index 100% rename from otaclient/app/proto/wrapper.py rename to src/otaclient/app/proto/wrapper.py diff --git a/otaclient/app/update_stats.py b/src/otaclient/app/update_stats.py similarity index 100% rename from otaclient/app/update_stats.py rename to src/otaclient/app/update_stats.py diff --git a/otaclient/configs/__init__.py b/src/otaclient/configs/__init__.py similarity index 100% rename from otaclient/configs/__init__.py rename to src/otaclient/configs/__init__.py diff --git a/otaclient/configs/_common.py b/src/otaclient/configs/_common.py similarity index 100% rename from otaclient/configs/_common.py rename to src/otaclient/configs/_common.py diff --git a/otaclient/configs/ecu_info.py b/src/otaclient/configs/ecu_info.py similarity index 100% rename from otaclient/configs/ecu_info.py rename to src/otaclient/configs/ecu_info.py diff --git a/otaclient/configs/proxy_info.py b/src/otaclient/configs/proxy_info.py similarity index 100% rename from otaclient/configs/proxy_info.py rename to src/otaclient/configs/proxy_info.py diff --git a/otaclient/requirements.txt b/src/otaclient/requirements.txt similarity index 100% rename from otaclient/requirements.txt rename to src/otaclient/requirements.txt From 10d687a0317df01d82ae6c725a20590bf75e54ed Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Fri, 31 May 2024 01:56:11 +0000 Subject: [PATCH 041/193] remove unneeded --- src/otaclient/.gitignore | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 src/otaclient/.gitignore diff --git a/src/otaclient/.gitignore b/src/otaclient/.gitignore deleted file mode 100644 index 7fd4c95dc..000000000 --- a/src/otaclient/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# generated version file by build -_version.py From 3d134f5150f8800b073ccccc7aec55509c11f1fa Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Fri, 31 May 2024 03:00:32 +0000 Subject: [PATCH 042/193] jetson-uefi: fix capsule location is not created at EFI partition --- src/otaclient/app/boot_control/_jetson_uefi.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/otaclient/app/boot_control/_jetson_uefi.py b/src/otaclient/app/boot_control/_jetson_uefi.py index ffd94f6e5..36f68ee93 100644 --- a/src/otaclient/app/boot_control/_jetson_uefi.py +++ b/src/otaclient/app/boot_control/_jetson_uefi.py @@ -173,6 +173,8 @@ def _prepare_payload(self) -> bool: raise JetsonUEFIBootControlError(_err_msg) from e capsule_at_esp = esp_mp / boot_cfg.CAPSULE_PAYLOAD_AT_ESP + capsule_at_esp.mkdir(parents=True, exist_ok=True) + capsule_at_standby_slot = self.standby_slot_mp / Path( boot_cfg.CAPSULE_PAYLOAD_AT_ROOTFS ).relative_to("/") From 7f04350efdcdf7df241b6e7f2587183a3439b996 Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Fri, 31 May 2024 03:16:17 +0000 Subject: [PATCH 043/193] jetson-uefi: fix init --- .../app/boot_control/_jetson_uefi.py | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/otaclient/app/boot_control/_jetson_uefi.py b/src/otaclient/app/boot_control/_jetson_uefi.py index 36f68ee93..552019360 100644 --- a/src/otaclient/app/boot_control/_jetson_uefi.py +++ b/src/otaclient/app/boot_control/_jetson_uefi.py @@ -369,6 +369,16 @@ class JetsonUEFIBootControl(BootControllerProtocol): """BootControllerProtocol implementation for jetson-uefi.""" def __init__(self) -> None: + current_ota_status_dir = Path(boot_cfg.OTA_STATUS_DIR) + standby_ota_status_dir = Path(cfg.MOUNT_POINT) / Path( + boot_cfg.OTA_STATUS_DIR + ).relative_to("/") + + # NOTE: this hint file is referred by finalize_switching_boot + self.firmware_update_hint_fpath = ( + standby_ota_status_dir / boot_cfg.FIRMWARE_UPDATE_HINT_FNAME + ) + try: # startup boot controller self._uefi_control = _UEFIBoot() @@ -381,16 +391,6 @@ def __init__(self) -> None: active_slot_mount_point=cfg.ACTIVE_ROOT_MOUNT_POINT, ) - current_ota_status_dir = Path(boot_cfg.OTA_STATUS_DIR) - standby_ota_status_dir = self._mp_control.standby_slot_mount_point / Path( - boot_cfg.OTA_STATUS_DIR - ).relative_to("/") - - # NOTE: this hint file is referred by finalize_switching_boot - self.firmware_update_hint_fpath = ( - standby_ota_status_dir / boot_cfg.FIRMWARE_UPDATE_HINT_FNAME - ) - # load firmware BSP version from current rootfs slot self._firmware_ver_control = FirmwareBSPVersionControl( current_firmware_bsp_vf=current_ota_status_dir @@ -410,14 +410,14 @@ def __init__(self) -> None: standby_ota_status_dir=standby_ota_status_dir, finalize_switching_boot=self._finalize_switching_boot, ) - + except Exception as e: + _err_msg = f"failed to start jetson-uefi controller: {e!r}" + raise ota_errors.BootControlStartupFailed(_err_msg, module=__name__) from e + finally: # NOTE: the hint file is checked during OTAStatusFilesControl __init__, # by finalize_switching_boot if we are in first reboot after OTA. # once we have done parsing the hint file, we must remove it immediately. self.firmware_update_hint_fpath.unlink(missing_ok=True) - except Exception as e: - _err_msg = f"failed to start jetson-uefi controller: {e!r}" - raise ota_errors.BootControlStartupFailed(_err_msg, module=__name__) from e def _finalize_switching_boot(self) -> bool: """Verify firmware update result and write firmware BSP version file.""" From a8296f76da624ebd980f38a358f3f6969290d565 Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Fri, 31 May 2024 03:18:51 +0000 Subject: [PATCH 044/193] jetson-uefi: fix fwupdate hint file --- src/otaclient/app/boot_control/_jetson_uefi.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/otaclient/app/boot_control/_jetson_uefi.py b/src/otaclient/app/boot_control/_jetson_uefi.py index 552019360..d8737d41b 100644 --- a/src/otaclient/app/boot_control/_jetson_uefi.py +++ b/src/otaclient/app/boot_control/_jetson_uefi.py @@ -375,7 +375,10 @@ def __init__(self) -> None: ).relative_to("/") # NOTE: this hint file is referred by finalize_switching_boot - self.firmware_update_hint_fpath = ( + self.current_fwupdate_hint_fpath = ( + current_ota_status_dir / boot_cfg.FIRMWARE_UPDATE_HINT_FNAME + ) + self.standby_fwupdate_hint_fpath = ( standby_ota_status_dir / boot_cfg.FIRMWARE_UPDATE_HINT_FNAME ) @@ -417,7 +420,7 @@ def __init__(self) -> None: # NOTE: the hint file is checked during OTAStatusFilesControl __init__, # by finalize_switching_boot if we are in first reboot after OTA. # once we have done parsing the hint file, we must remove it immediately. - self.firmware_update_hint_fpath.unlink(missing_ok=True) + self.current_fwupdate_hint_fpath.unlink(missing_ok=True) def _finalize_switching_boot(self) -> bool: """Verify firmware update result and write firmware BSP version file.""" @@ -430,7 +433,7 @@ def _finalize_switching_boot(self) -> bool: try: slot_id, bsp_v = CapsuleUpdate.parse_firmware_update_hint_file( - self.firmware_update_hint_fpath + self.current_fwupdate_hint_fpath ) except FileNotFoundError: logger.info("no firmware update occurs in previous OTA") @@ -507,7 +510,7 @@ def _capsule_firmware_update(self) -> bool: ) if firmware_updater.firmware_update(): CapsuleUpdate.write_firmware_update_hint_file( - self.firmware_update_hint_fpath, + self.standby_fwupdate_hint_fpath, slot_id=self._uefi_control.standby_slot, bsp_version=new_bsp_v, ) From 5ce0db27ddc48cbbc90ce97996289e988fd3a78d Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Fri, 31 May 2024 07:23:41 +0000 Subject: [PATCH 045/193] add some logs --- src/otaclient/app/boot_control/_jetson_common.py | 2 +- src/otaclient/app/boot_control/_jetson_uefi.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/otaclient/app/boot_control/_jetson_common.py b/src/otaclient/app/boot_control/_jetson_common.py index 7b332c239..b01fab29a 100644 --- a/src/otaclient/app/boot_control/_jetson_common.py +++ b/src/otaclient/app/boot_control/_jetson_common.py @@ -197,7 +197,7 @@ def __init__( self._version = _version = FirmwareBSPVersion.model_validate_json( self._current_fw_bsp_vf.read_text() ) - logger.info(f"firmware_version: {_version}") + logger.info(f"loaded firmware_version from version file: {_version}") except Exception as e: logger.warning( f"invalid or missing firmware_bsp_verion file, removed: {e!r}" diff --git a/src/otaclient/app/boot_control/_jetson_uefi.py b/src/otaclient/app/boot_control/_jetson_uefi.py index d8737d41b..aca4774d5 100644 --- a/src/otaclient/app/boot_control/_jetson_uefi.py +++ b/src/otaclient/app/boot_control/_jetson_uefi.py @@ -468,6 +468,9 @@ def _finalize_switching_boot(self) -> bool: return False # ------ firmware update succeeded, write firmware version file ------ # + logger.info( + f"successfully update {current_slot=} firmware version to {current_slot_bsp_ver}" + ) self._firmware_ver_control.set_version_by_slot( current_slot, current_slot_bsp_ver ) From b4d068ae53b6000d7b7ef41e693a2444d12396b0 Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Fri, 31 May 2024 07:24:04 +0000 Subject: [PATCH 046/193] ----------------above commits are tested on real machine-------------- From 56572fc7f2ff92f360da6d129b9a5d45189a4129 Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Wed, 5 Jun 2024 13:04:50 +0000 Subject: [PATCH 047/193] Squashed commit of the following: MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit commit bfa3bb5821d5e85c2330cfd31afbe36e9df699ca Author: Bodong Yang <86948717+Bodong-Yang@users.noreply.github.com> Date: Wed Jun 5 19:16:32 2024 +0900 refactor: project restruct phase2 (#311) Project restruct phase2, in this phase most of the changes are restructring the packages to make the project more modularized and decoupling each components as much as possible. 1. ota_metadata becomes a standalone package, current implementation is treated as the implementation for OTA image format legacy version. 2. create new otaclient_api, current version is v2 and all the protobuf pb2 generated code and proto wrappers are grouped under v2. 3. create new otaclient_common, all common shared libs, helper funcs and types are grouped under this package, categorized by functionality. 4. fix according to import paths changes. 5. restruct tests according to the new project layout. 1. cleanup bootstrap, refactor into samples, which provide sample ecu_info.yaml, proxy_info.yaml and otaclient.service for single ECU setup. 2. fix up tools/offline_ota_image_builder and status_monitor package. 3. not use relative imports anymore. commit 9f975adcc7eeec9faef33f4467e3028a7724c185 Author: Bodong Yang <86948717+Bodong-Yang@users.noreply.github.com> Date: Tue Jun 4 17:12:05 2024 +0900 chore(tools): remove out-of-date unmaintained tools (#313) This PR removes the unmaintained old tools, including tools.emulator, tools.test_utils and tools/build_image.sh. commit 0ce3cc9687a038b8643d3984c503efb37ef3d288 Author: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue Jun 4 16:32:45 2024 +0900 [pre-commit.ci] pre-commit autoupdate (#312) * [pre-commit.ci] pre-commit autoupdate updates: - [github.com/tox-dev/pyproject-fmt: 1.8.0 → 2.1.3](https://github.com/tox-dev/pyproject-fmt/compare/1.8.0...2.1.3) - [github.com/igorshubovych/markdownlint-cli: v0.40.0 → v0.41.0](https://github.com/igorshubovych/markdownlint-cli/compare/v0.40.0...v0.41.0) * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .dockerignore | 1 - .gitignore | 4 - .pre-commit-config.yaml | 4 +- README.md | 31 +- bootstrap/root/boot/ota/ecu_info.yaml | 7 - proto/README.md | 3 + pyproject.toml | 81 +- samples/README.md | 3 + samples/ecu_info.yaml | 7 + .../system => samples}/otaclient.service | 6 +- samples/proxy_info.yaml | 9 + src/ota_metadata/README.md | 3 + .../ota_metadata/legacy/__init__.py | 23 +- .../legacy}/ota_metafiles_pb2.py | 0 .../legacy}/ota_metafiles_pb2.pyi | 0 .../legacy/parser.py} | 61 +- .../legacy/types.py} | 5 +- src/ota_proxy/cache_control.py | 2 +- src/ota_proxy/server_app.py | 2 +- src/otaclient/app/boot_control/_common.py | 58 +- src/otaclient/app/boot_control/_grub.py | 24 +- .../app/boot_control/_jetson_cboot.py | 24 +- .../app/boot_control/_jetson_common.py | 6 +- src/otaclient/app/boot_control/_rpi_boot.py | 22 +- src/otaclient/app/boot_control/protocol.py | 4 +- src/otaclient/app/boot_control/selecter.py | 6 +- src/otaclient/app/common.py | 805 ------------------ src/otaclient/app/configs.py | 12 +- src/otaclient/app/create_standby/common.py | 7 +- src/otaclient/app/create_standby/interface.py | 3 +- .../app/create_standby/rebuild_mode.py | 8 +- src/otaclient/app/errors.py | 10 +- src/otaclient/app/interface.py | 5 +- src/otaclient/app/main.py | 39 +- src/otaclient/app/ota_client.py | 169 ++-- src/otaclient/app/ota_client_service.py | 57 -- src/otaclient/app/ota_client_stub.py | 56 +- src/otaclient/app/ota_status.py | 45 - src/otaclient/app/proto/__init__.py | 70 -- src/otaclient/app/proto/wrapper.py | 19 - src/otaclient/app/update_stats.py | 11 +- src/otaclient/configs/ecu_info.py | 2 +- src/otaclient/configs/proxy_info.py | 2 +- src/otaclient_api/v2/README.md | 3 + src/otaclient_api/v2/__init__.py | 48 ++ .../v2/api_caller.py} | 39 +- src/otaclient_api/v2/api_stub.py | 41 + .../v2}/otaclient_v2_pb2.py | 0 .../v2}/otaclient_v2_pb2.pyi | 0 .../v2}/otaclient_v2_pb2_grpc.py | 0 .../v2/types.py} | 126 +-- .../_utils => otaclient_common}/__init__.py | 50 +- src/otaclient_common/common.py | 404 +++++++++ .../app => otaclient_common}/downloader.py | 10 +- .../_utils => otaclient_common}/linux.py | 0 .../_utils => otaclient_common}/logging.py | 0 src/otaclient_common/persist_file_handling.py | 219 +++++ .../proto_streamer.py} | 2 +- .../proto_wrapper.py} | 0 .../proto_wrapper_README.md} | 0 src/otaclient_common/retry_task_map.py | 225 +++++ .../_utils => otaclient_common}/typing.py | 35 +- tests/conftest.py | 5 +- tests/{test__utils => }/test_logging.py | 2 +- tests/test_ota_metadata/__init__.py | 0 .../test_legacy.py} | 9 +- tests/test_ota_proxy/test_ota_proxy_server.py | 4 +- tests/test_otaclient/__init__.py | 0 .../test_boot_control/__init__.py | 0 .../test_boot_control/default_grub | 0 .../test_boot_control/extlinux.conf_slot_a | 0 .../test_boot_control/extlinux.conf_slot_b | 0 .../test_boot_control/fstab_origin | 0 .../test_boot_control/fstab_updated | 0 .../test_boot_control/grub.cfg_slot_a | 0 .../grub.cfg_slot_a_non_otapartition | 0 .../test_boot_control/grub.cfg_slot_a_updated | 0 .../test_boot_control/grub.cfg_slot_b | 0 .../test_boot_control/grub.cfg_slot_b_updated | 0 .../test_boot_control/test_grub.py | 12 +- .../test_boot_control/test_jetson_cboot.py | 7 +- .../test_ota_status_control.py | 42 +- .../test_boot_control/test_rpi_boot.py | 13 +- .../test_configs}/test_ecu_info.py | 0 .../test_configs}/test_proxy_info.py | 0 .../test_create_standby.py | 1 - .../{ => test_otaclient}/test_log_setting.py | 0 tests/{ => test_otaclient}/test_main.py | 0 tests/{ => test_otaclient}/test_ota_client.py | 65 +- .../test_ota_client_service.py | 44 +- .../test_ota_client_stub.py | 340 ++++---- .../{ => test_otaclient}/test_update_stats.py | 0 tests/test_otaclient_api/__init__.py | 0 tests/test_otaclient_api/test_v2/__init__.py | 0 .../test_v2/test_apli_caller.py} | 22 +- .../test_v2/test_types.py} | 73 +- tests/test_otaclient_common/__init__.py | 0 .../test_common.py | 125 +-- .../test_downloader.py | 6 +- .../test_persist_file_handling.py | 4 +- .../test_proto_wrapper}/__init__.py | 0 .../test_proto_wrapper}/example.proto | 0 .../test_proto_wrapper}/example_pb2.py | 0 .../test_proto_wrapper}/example_pb2.pyi | 0 .../example_pb2_wrapper.py | 4 +- .../test_proto_wrapper}/test_proto_wrapper.py | 8 +- .../test_retry_task_map.py | 144 ++++ tests/utils.py | 31 +- tools/build_image.sh | 23 - tools/emulator/README.md | 10 - tools/emulator/config.yml | 31 - tools/emulator/ecu.py | 180 ---- tools/emulator/main.py | 102 --- tools/emulator/ota_client_stub.py | 105 --- tools/emulator/requirements.txt | 3 - tools/offline_ota_image_builder/builder.py | 12 +- tools/status_monitor/ecu_status_box.py | 14 +- tools/status_monitor/ecu_status_tracker.py | 15 +- tools/test_utils/README.md | 43 - tools/test_utils/__init__.py | 13 - tools/test_utils/_logutil.py | 27 - tools/test_utils/_update_call.py | 58 -- tools/test_utils/api_caller.py | 109 --- tools/test_utils/ecu_info.yaml | 9 - tools/test_utils/setup_ecu.py | 197 ----- tools/test_utils/update_request.yaml | 13 - 126 files changed, 2026 insertions(+), 2817 deletions(-) delete mode 100644 .dockerignore delete mode 100644 bootstrap/root/boot/ota/ecu_info.yaml create mode 100644 proto/README.md create mode 100644 samples/README.md create mode 100644 samples/ecu_info.yaml rename {bootstrap/root/etc/systemd/system => samples}/otaclient.service (56%) create mode 100644 samples/proxy_info.yaml create mode 100644 src/ota_metadata/README.md rename tools/emulator/path_loader.py => src/ota_metadata/legacy/__init__.py (52%) rename src/{otaclient/app/proto => ota_metadata/legacy}/ota_metafiles_pb2.py (100%) rename src/{otaclient/app/proto => ota_metadata/legacy}/ota_metafiles_pb2.pyi (100%) rename src/{otaclient/app/ota_metadata.py => ota_metadata/legacy/parser.py} (94%) rename src/{otaclient/app/proto/_ota_metafiles_wrapper.py => ota_metadata/legacy/types.py} (97%) delete mode 100644 src/otaclient/app/common.py delete mode 100644 src/otaclient/app/ota_client_service.py delete mode 100644 src/otaclient/app/ota_status.py delete mode 100644 src/otaclient/app/proto/__init__.py delete mode 100644 src/otaclient/app/proto/wrapper.py create mode 100644 src/otaclient_api/v2/README.md create mode 100644 src/otaclient_api/v2/__init__.py rename src/{otaclient/app/ota_client_call.py => otaclient_api/v2/api_caller.py} (72%) create mode 100644 src/otaclient_api/v2/api_stub.py rename src/{otaclient/app/proto => otaclient_api/v2}/otaclient_v2_pb2.py (100%) rename src/{otaclient/app/proto => otaclient_api/v2}/otaclient_v2_pb2.pyi (100%) rename src/{otaclient/app/proto => otaclient_api/v2}/otaclient_v2_pb2_grpc.py (100%) rename src/{otaclient/app/proto/_otaclient_v2_pb2_wrapper.py => otaclient_api/v2/types.py} (84%) rename src/{otaclient/_utils => otaclient_common}/__init__.py (67%) create mode 100644 src/otaclient_common/common.py rename src/{otaclient/app => otaclient_common}/downloader.py (98%) rename src/{otaclient/_utils => otaclient_common}/linux.py (100%) rename src/{otaclient/_utils => otaclient_common}/logging.py (100%) create mode 100644 src/otaclient_common/persist_file_handling.py rename src/{otaclient/app/proto/streamer.py => otaclient_common/proto_streamer.py} (97%) rename src/{otaclient/app/proto/_common.py => otaclient_common/proto_wrapper.py} (100%) rename src/{otaclient/app/proto/README.md => otaclient_common/proto_wrapper_README.md} (100%) create mode 100644 src/otaclient_common/retry_task_map.py rename src/{otaclient/_utils => otaclient_common}/typing.py (55%) rename tests/{test__utils => }/test_logging.py (97%) create mode 100644 tests/test_ota_metadata/__init__.py rename tests/{test_ota_metadata.py => test_ota_metadata/test_legacy.py} (99%) create mode 100644 tests/test_otaclient/__init__.py rename tests/{ => test_otaclient}/test_boot_control/__init__.py (100%) rename tests/{ => test_otaclient}/test_boot_control/default_grub (100%) rename tests/{ => test_otaclient}/test_boot_control/extlinux.conf_slot_a (100%) rename tests/{ => test_otaclient}/test_boot_control/extlinux.conf_slot_b (100%) rename tests/{ => test_otaclient}/test_boot_control/fstab_origin (100%) rename tests/{ => test_otaclient}/test_boot_control/fstab_updated (100%) rename tests/{ => test_otaclient}/test_boot_control/grub.cfg_slot_a (100%) rename tests/{ => test_otaclient}/test_boot_control/grub.cfg_slot_a_non_otapartition (100%) rename tests/{ => test_otaclient}/test_boot_control/grub.cfg_slot_a_updated (100%) rename tests/{ => test_otaclient}/test_boot_control/grub.cfg_slot_b (100%) rename tests/{ => test_otaclient}/test_boot_control/grub.cfg_slot_b_updated (100%) rename tests/{ => test_otaclient}/test_boot_control/test_grub.py (98%) rename tests/{ => test_otaclient}/test_boot_control/test_jetson_cboot.py (95%) rename tests/{ => test_otaclient}/test_boot_control/test_ota_status_control.py (86%) rename tests/{ => test_otaclient}/test_boot_control/test_rpi_boot.py (97%) rename tests/{ => test_otaclient/test_configs}/test_ecu_info.py (100%) rename tests/{ => test_otaclient/test_configs}/test_proxy_info.py (100%) rename tests/{ => test_otaclient}/test_create_standby.py (99%) rename tests/{ => test_otaclient}/test_log_setting.py (100%) rename tests/{ => test_otaclient}/test_main.py (100%) rename tests/{ => test_otaclient}/test_ota_client.py (90%) rename tests/{ => test_otaclient}/test_ota_client_service.py (72%) rename tests/{ => test_otaclient}/test_ota_client_stub.py (73%) rename tests/{ => test_otaclient}/test_update_stats.py (100%) create mode 100644 tests/test_otaclient_api/__init__.py create mode 100644 tests/test_otaclient_api/test_v2/__init__.py rename tests/{test_ota_client_call.py => test_otaclient_api/test_v2/test_apli_caller.py} (90%) rename tests/{test_proto/test_otaclient_pb2_wrapper.py => test_otaclient_api/test_v2/test_types.py} (70%) create mode 100644 tests/test_otaclient_common/__init__.py rename tests/{ => test_otaclient_common}/test_common.py (72%) rename tests/{ => test_otaclient_common}/test_downloader.py (98%) rename tests/{ => test_otaclient_common}/test_persist_file_handling.py (99%) rename tests/{test_proto => test_otaclient_common/test_proto_wrapper}/__init__.py (100%) rename tests/{test_proto => test_otaclient_common/test_proto_wrapper}/example.proto (100%) rename tests/{test_proto => test_otaclient_common/test_proto_wrapper}/example_pb2.py (100%) rename tests/{test_proto => test_otaclient_common/test_proto_wrapper}/example_pb2.pyi (100%) rename tests/{test_proto => test_otaclient_common/test_proto_wrapper}/example_pb2_wrapper.py (96%) rename tests/{test_proto => test_otaclient_common/test_proto_wrapper}/test_proto_wrapper.py (97%) create mode 100644 tests/test_otaclient_common/test_retry_task_map.py delete mode 100644 tools/build_image.sh delete mode 100644 tools/emulator/README.md delete mode 100644 tools/emulator/config.yml delete mode 100644 tools/emulator/ecu.py delete mode 100644 tools/emulator/main.py delete mode 100644 tools/emulator/ota_client_stub.py delete mode 100644 tools/emulator/requirements.txt delete mode 100644 tools/test_utils/README.md delete mode 100644 tools/test_utils/__init__.py delete mode 100644 tools/test_utils/_logutil.py delete mode 100644 tools/test_utils/_update_call.py delete mode 100644 tools/test_utils/api_caller.py delete mode 100644 tools/test_utils/ecu_info.yaml delete mode 100644 tools/test_utils/setup_ecu.py delete mode 100644 tools/test_utils/update_request.yaml diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 85196eea0..000000000 --- a/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -ota-image.* diff --git a/.gitignore b/.gitignore index 12e4e8092..5787b2cc8 100644 --- a/.gitignore +++ b/.gitignore @@ -159,10 +159,6 @@ cython_debug/ # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ -# build related -build -*.egg-info - # local vscode configs .devcontainer .vscode diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4348be26d..610b2bb88 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,7 +27,7 @@ repos: - flake8-comprehensions - flake8-simplify - repo: https://github.com/tox-dev/pyproject-fmt - rev: "1.8.0" + rev: "2.1.3" hooks: - id: pyproject-fmt # https://pyproject-fmt.readthedocs.io/en/latest/#calculating-max-supported-python-version @@ -39,7 +39,7 @@ repos: # additional_dependencies: # - tomli - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.40.0 + rev: v0.41.0 hooks: - id: markdownlint args: ["-c", ".markdownlint.yaml", "--fix"] diff --git a/README.md b/README.md index c5a57f6aa..0e6eace46 100644 --- a/README.md +++ b/README.md @@ -1,26 +1,23 @@ -# OTA client +# OTAClient ## Overview -This OTA client is a client software to perform over-the-air software updates for linux devices. -To enable updating of software at any layer (kernel, kernel module, user library, user application), the OTA client targets the entire rootfs for updating. -When the OTA client receives an update request, it downloads a list from the OTA server that contains the file paths and the hash values of the files, etc., to be updated, and compares them with the files in its own storage and if there is a match, that file is used to update the rootfs. By this delta mechanism, it is possible to reduce the download size even if the entire rootfs is targeted and this mechanism does not require any specific server implementation, nor does it require the server to keep a delta for each version of the rootfs. +OTAClient is software to perform over-the-air software updates for linux devices. +It provides a set of APIs for user to start the OTA and monitor the progress and status. + +It is designed to work with web.auto FMS OTA component. ## Feature -- Rootfs updating -- Delta updating -- Redundant configuration with A/B partition update -- Arbitrary files can be copied from A to B partition. This can be used to take over individual files. -- No specific server implementation is required. The server that supports HTTP GET is only required. - - TLS connection is also required. -- Delta management is not required for server side. -- To restrict access to the server, cookie can be used. -- All files to be updated are verified by the hash included in the metadata, and the metadata is also verified by X.509 certificate locally installed. -- Transfer data is encrypted by TLS -- Multiple ECU(Electronic Control Unit) support -- By the internal proxy cache mechanism, the cache can be used for the download requests to the same file from multiple ECU. +- A/B partition update with support for generic x86_64 device, NVIDIA Jetson series based devices and Raspberry Pi device. +- Full Rootfs update, with delta update support. +- Local delta calculation, allowing update to any version of OTA image without the need of a pre-generated delta OTA package. +- Support persist files from active slot to newly updated slot. +- Verification over OTA image by digital signature and PKI. +- Support for protected OTA server with cookie. +- Optional OTA proxy support and OTA cache support. +- Multiple ECU OTA supports. ## License -OTA client is licensed under the Apache License, Version 2.0. +OTAClient is licensed under the Apache License, Version 2.0. diff --git a/bootstrap/root/boot/ota/ecu_info.yaml b/bootstrap/root/boot/ota/ecu_info.yaml deleted file mode 100644 index 85f520012..000000000 --- a/bootstrap/root/boot/ota/ecu_info.yaml +++ /dev/null @@ -1,7 +0,0 @@ -format_version: 1 -ecu_id: "autoware" -#secondaries: -# - ecu_id: "perception1" -# ip_addr: "192.168.0.11" -# - ecu_id: "perception2" -# ip_addr: "192.168.0.12" diff --git a/proto/README.md b/proto/README.md new file mode 100644 index 000000000..fd819de29 --- /dev/null +++ b/proto/README.md @@ -0,0 +1,3 @@ +# OTA Service API proto + +This folder includes the OTA service API proto file, and a set of tools to generate the python lib from the proto files. diff --git a/pyproject.toml b/pyproject.toml index 41e11a020..40d2e2bea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,22 +25,21 @@ dynamic = [ ] dependencies = [ "aiofiles==22.1", - "aiohttp<3.10.0,>=3.9.5", - "cryptography<43.0.0,>=42.0.4", - "grpcio<1.54.0,>=1.53.2", - "protobuf<4.22.0,>=4.21.12", + "aiohttp<3.10,>=3.9.5", + "cryptography<43,>=42.0.4", + "grpcio<1.54,>=1.53.2", + "protobuf<4.22,>=4.21.12", "pydantic==2.7", "pydantic-settings==2.2.1", - "pyOpenSSL==24.1", - "PyYAML>=3.12", - "requests<2.32.0,>=2.31", - 'typing_extensions>=4.6.3; python_version < "3.11"', - "urllib3<2.0.0,>=1.26.8", + "pyopenssl==24.1", + "pyyaml>=3.12", + "requests<2.32,>=2.31", + "typing-extensions>=4.6.3", + "urllib3<2,>=1.26.8", "uvicorn[standard]==0.20", "zstandard==0.18", ] -[project.optional-dependencies] -dev = [ +optional-dependencies.dev = [ "black", "coverage", "flake8", @@ -50,8 +49,7 @@ dev = [ "pytest-mock==3.8.2", "requests-mock", ] -[project.urls] -Source = "https://github.com/tier4/ota-client" +urls.Source = "https://github.com/tier4/ota-client" [tool.hatch.version] source = "vcs" @@ -60,26 +58,44 @@ source = "vcs" version-file = "src/_otaclient_version.py" [tool.hatch.build.targets.sdist] -exclude = ["/tools"] +exclude = [ + "/tools", + ".github", +] [tool.hatch.build.targets.wheel] -only-include = ["src"] -sources = ["src"] +exclude = [ + "**/.gitignore", + "**/*README.md", +] +only-include = [ + "src", +] +sources = [ + "src", +] [tool.hatch.envs.dev] type = "virtual" -features = ["dev"] +features = [ + "dev", +] [tool.black] line-length = 88 -target-version = ['py38'] +target-version = [ + 'py38', +] extend-exclude = '''( ^.*(_pb2.pyi?|_pb2_grpc.pyi?)$ )''' [tool.isort] profile = "black" -extend_skip_glob = ["*_pb2.py*", "_pb2_grpc.py*"] +extend_skip_glob = [ + "*_pb2.py*", + "_pb2_grpc.py*", +] [tool.pytest.ini_options] asyncio_mode = "auto" @@ -87,16 +103,26 @@ log_auto_indent = true log_format = "%(asctime)s %(levelname)s %(filename)s %(funcName)s,%(lineno)d %(message)s" log_cli = true log_cli_level = "INFO" -pythonpath = ["otaclient"] -testpaths = ["./tests"] +testpaths = [ + "./tests", +] [tool.coverage.run] branch = false relative_files = true -source = ["otaclient"] +source = [ + "otaclient", + "otaclient_api", + "otaclient_common", + "ota_metadata", + "ota_proxy", +] [tool.coverage.report] -omit = ["**/*_pb2.py*", "**/*_pb2_grpc.py*"] +omit = [ + "**/*_pb2.py*", + "**/*_pb2_grpc.py*", +] exclude_also = [ "def __repr__", "if __name__ == .__main__.:", @@ -108,6 +134,11 @@ skip_empty = true skip_covered = true [tool.pyright] -exclude = ["**/__pycache__"] -ignore = ["**/*_pb2.py*", "**/*_pb2_grpc.py*"] +exclude = [ + "**/__pycache__", +] +ignore = [ + "**/*_pb2.py*", + "**/*_pb2_grpc.py*", +] pythonVersion = "3.8" diff --git a/samples/README.md b/samples/README.md new file mode 100644 index 000000000..859d9a6ea --- /dev/null +++ b/samples/README.md @@ -0,0 +1,3 @@ +# OTAClient configuration files samples + +This folder contains the sample otaclient configuration files **ecu_info.yaml**, **proxy_info.yaml** and systemd service unit file **otaclient.service** for a single ECU OTA setup. diff --git a/samples/ecu_info.yaml b/samples/ecu_info.yaml new file mode 100644 index 000000000..46b3700fb --- /dev/null +++ b/samples/ecu_info.yaml @@ -0,0 +1,7 @@ +# This is the sample ecu_info.yaml for a single x86_64 ECU setup. +# Please check ecu_info.yaml spec for more details: https://tier4.atlassian.net/l/cp/AGmpqFFc. +format_version: 1 +ecu_id: autoware +bootloader: grub +available_ecu_ids: + - autoware diff --git a/bootstrap/root/etc/systemd/system/otaclient.service b/samples/otaclient.service similarity index 56% rename from bootstrap/root/etc/systemd/system/otaclient.service rename to samples/otaclient.service index 39e1552dd..7bb9d28bb 100644 --- a/bootstrap/root/etc/systemd/system/otaclient.service +++ b/samples/otaclient.service @@ -1,5 +1,3 @@ -# otaclient.service - [Unit] Description=OTA Client After=network-online.target nss-lookup.target @@ -7,9 +5,9 @@ Wants=network-online.target [Service] Type=simple -ExecStart=/bin/bash -c 'source /opt/ota/.venv/bin/activate && PYTHONPATH=/opt/ota python3 -m otaclient' +ExecStart=/opt/ota/client/venv/bin/python3 -m otaclient Restart=always -RestartSec=10 +RestartSec=16 [Install] WantedBy=multi-user.target diff --git a/samples/proxy_info.yaml b/samples/proxy_info.yaml new file mode 100644 index 000000000..5c893ca05 --- /dev/null +++ b/samples/proxy_info.yaml @@ -0,0 +1,9 @@ +# This is the sample proxy_info.yaml for a single ECU setup. +# Please check proxy_info.yaml spec for more details: https://tier4.atlassian.net/l/cp/qT4N4K0X. +format_version: 1 +enable_local_ota_proxy: true +enable_local_ota_proxy_cache: true +local_ota_proxy_listen_addr: 127.0.0.1 +local_ota_proxy_listen_port: 8082 +# if otaclient-logger is installed locally +logging_server: "http://127.0.0.1:8083" diff --git a/src/ota_metadata/README.md b/src/ota_metadata/README.md new file mode 100644 index 000000000..84dcc9084 --- /dev/null +++ b/src/ota_metadata/README.md @@ -0,0 +1,3 @@ +# OTA image metadata + +Libs for parsing OTA image. diff --git a/tools/emulator/path_loader.py b/src/ota_metadata/legacy/__init__.py similarity index 52% rename from tools/emulator/path_loader.py rename to src/ota_metadata/legacy/__init__.py index dd93760fb..6e383fff3 100644 --- a/tools/emulator/path_loader.py +++ b/src/ota_metadata/legacy/__init__.py @@ -11,19 +11,24 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +"""OTA image metadata, legacy version.""" -# NOTE: this file should only be loaded once by the program entry! +from __future__ import annotations +import sys +from pathlib import Path -###### load path ###### -def _path_load(): - import sys - from pathlib import Path +from otaclient_common import import_from_file - project_base = Path(__file__).absolute().parent.parent - sys.path.extend([str(project_base), str(project_base / "app")]) +SUPORTED_COMPRESSION_TYPES = ("zst", "zstd") +# ------ dynamically import pb2 generated code ------ # -_path_load() -###### +_PROTO_DIR = Path(__file__).parent +_PB2_FPATH = _PROTO_DIR / "ota_metafiles_pb2.py" +_PACKAGE_PREFIX = ".".join(__name__.split(".")[:-1]) + +_module_name, _module = import_from_file(_PB2_FPATH) +sys.modules[_module_name] = _module +sys.modules[f"{_PACKAGE_PREFIX}.{_module_name}"] = _module diff --git a/src/otaclient/app/proto/ota_metafiles_pb2.py b/src/ota_metadata/legacy/ota_metafiles_pb2.py similarity index 100% rename from src/otaclient/app/proto/ota_metafiles_pb2.py rename to src/ota_metadata/legacy/ota_metafiles_pb2.py diff --git a/src/otaclient/app/proto/ota_metafiles_pb2.pyi b/src/ota_metadata/legacy/ota_metafiles_pb2.pyi similarity index 100% rename from src/otaclient/app/proto/ota_metafiles_pb2.pyi rename to src/ota_metadata/legacy/ota_metafiles_pb2.pyi diff --git a/src/otaclient/app/ota_metadata.py b/src/ota_metadata/legacy/parser.py similarity index 94% rename from src/otaclient/app/ota_metadata.py rename to src/ota_metadata/legacy/parser.py index b69a99241..25283004c 100644 --- a/src/otaclient/app/ota_metadata.py +++ b/src/ota_metadata/legacy/parser.py @@ -72,18 +72,17 @@ from typing_extensions import Self from ota_proxy import OTAFileCacheControl - -from .common import RetryTaskMap, get_backoff, urljoin_ensure_base -from .configs import config as cfg -from .downloader import Downloader -from .proto.streamer import Uint32LenDelimitedMsgReader, Uint32LenDelimitedMsgWriter -from .proto.wrapper import ( - DirectoryInf, - MessageWrapper, - PersistentInf, - RegularInf, - SymbolicLinkInf, +from otaclient_common.common import get_backoff, urljoin_ensure_base +from otaclient_common.downloader import Downloader +from otaclient_common.proto_streamer import ( + Uint32LenDelimitedMsgReader, + Uint32LenDelimitedMsgWriter, ) +from otaclient_common.proto_wrapper import MessageWrapper +from otaclient_common.retry_task_map import RetryTaskMap + +from . import SUPORTED_COMPRESSION_TYPES +from .types import DirectoryInf, PersistentInf, RegularInf, SymbolicLinkInf logger = logging.getLogger(__name__) @@ -592,10 +591,25 @@ class OTAMetadata: ), } - def __init__(self, *, url_base: str, downloader: Downloader) -> None: + MAX_COCURRENT = 2 + BACKOFF_FACTOR = 1 + BACKOFF_MAX = 6 + + def __init__( + self, + *, + url_base: str, + downloader: Downloader, + run_dir: Path, + certs_dir: Path, + download_max_idle_time: int, + ) -> None: self.url_base = url_base self._downloader = downloader - self._tmp_dir = TemporaryDirectory(prefix="ota_metadata", dir=cfg.RUN_DIR) + self.run_dir = run_dir + self.certs_dir = certs_dir + self.download_max_idle_time = download_max_idle_time + self._tmp_dir = TemporaryDirectory(prefix="ota_metadata", dir=run_dir) self._tmp_dir_path = Path(self._tmp_dir.name) # download and parse the metadata.jwt @@ -622,7 +636,7 @@ def _process_metadata_jwt(self) -> _MetadataJWTClaimsLayout: """Download, loading and parsing metadata.jwt.""" logger.debug("process metadata.jwt...") # download and parse metadata.jwt - with NamedTemporaryFile(prefix="metadata_jwt", dir=cfg.RUN_DIR) as meta_f: + with NamedTemporaryFile(prefix="metadata_jwt", dir=self.run_dir) as meta_f: _downloaded_meta_f = Path(meta_f.name) self._downloader.download_retry_inf( urljoin_ensure_base(self.url_base, self.METADATA_JWT), @@ -636,13 +650,13 @@ def _process_metadata_jwt(self) -> _MetadataJWTClaimsLayout: ) _parser = _MetadataJWTParser( - _downloaded_meta_f.read_text(), certs_dir=cfg.CERTS_DIR + _downloaded_meta_f.read_text(), certs_dir=self.certs_dir ) # get not yet verified parsed ota_metadata _ota_metadata = _parser.get_otametadata() # download certificate and verify metadata against this certificate - with NamedTemporaryFile(prefix="metadata_cert", dir=cfg.RUN_DIR) as cert_f: + with NamedTemporaryFile(prefix="metadata_cert", dir=self.run_dir) as cert_f: cert_info = _ota_metadata.certificate cert_fname, cert_hash = cert_info.file, cert_info.hash cert_file = Path(cert_f.name) @@ -696,11 +710,11 @@ def _process_text_base_otameta_file(_metafile: MetaFile): last_active_timestamp = int(time.time()) _mapper = RetryTaskMap( - max_concurrent=cfg.MAX_CONCURRENT_DOWNLOAD_TASKS, + max_concurrent=self.MAX_COCURRENT, backoff_func=partial( get_backoff, - factor=cfg.DOWNLOAD_GROUP_BACKOFF_FACTOR, - _max=cfg.DOWNLOAD_GROUP_BACKOFF_MAX, + factor=self.BACKOFF_FACTOR, + _max=self.BACKOFF_MAX, ), max_retry=0, # NOTE: we use another strategy below ) @@ -718,12 +732,9 @@ def _process_text_base_otameta_file(_metafile: MetaFile): last_active_timestamp = max( last_active_timestamp, self._downloader.last_active_timestamp ) - if ( - int(time.time()) - last_active_timestamp - > cfg.DOWNLOAD_GROUP_INACTIVE_TIMEOUT - ): + if int(time.time()) - last_active_timestamp > self.download_max_idle_time: logger.error( - f"downloader becomes stuck for {cfg.DOWNLOAD_GROUP_INACTIVE_TIMEOUT=} seconds, abort" + f"downloader becomes stuck for {self.download_max_idle_time=} seconds, abort" ) _mapper.shutdown(raise_last_exc=True) @@ -753,7 +764,7 @@ def get_download_url(self, reg_inf: RegularInf) -> Tuple[str, Optional[str]]: if ( self.image_compressed_rootfs_url and reg_inf.compressed_alg - and reg_inf.compressed_alg in cfg.SUPPORTED_COMPRESS_ALG + and reg_inf.compressed_alg in SUPORTED_COMPRESSION_TYPES ): return ( urljoin_ensure_base( diff --git a/src/otaclient/app/proto/_ota_metafiles_wrapper.py b/src/ota_metadata/legacy/types.py similarity index 97% rename from src/otaclient/app/proto/_ota_metafiles_wrapper.py rename to src/ota_metadata/legacy/types.py index bca6a24f8..c14807470 100644 --- a/src/otaclient/app/proto/_ota_metafiles_wrapper.py +++ b/src/ota_metadata/legacy/types.py @@ -20,9 +20,8 @@ from pathlib import Path from typing import Union -import ota_metafiles_pb2 as ota_metafiles - -from ._common import MessageWrapper, calculate_slots +from ota_metadata.legacy import ota_metafiles_pb2 as ota_metafiles +from otaclient_common.proto_wrapper import MessageWrapper, calculate_slots # helper mixin diff --git a/src/ota_proxy/cache_control.py b/src/ota_proxy/cache_control.py index 2ec4ceeca..3f5548009 100644 --- a/src/ota_proxy/cache_control.py +++ b/src/ota_proxy/cache_control.py @@ -18,7 +18,7 @@ from typing_extensions import Self -from otaclient._utils import copy_callable_typehint_to_method +from otaclient_common.typing import copy_callable_typehint_to_method _FIELDS = "_fields" diff --git a/src/ota_proxy/server_app.py b/src/ota_proxy/server_app.py index 2349ff284..0ab4e467c 100644 --- a/src/ota_proxy/server_app.py +++ b/src/ota_proxy/server_app.py @@ -22,7 +22,7 @@ import aiohttp -from otaclient._utils.logging import BurstSuppressFilter +from otaclient_common.logging import BurstSuppressFilter from ._consts import ( BHEADER_AUTHORIZATION, diff --git a/src/otaclient/app/boot_control/_common.py b/src/otaclient/app/boot_control/_common.py index 7064a31e6..9f146f773 100644 --- a/src/otaclient/app/boot_control/_common.py +++ b/src/otaclient/app/boot_control/_common.py @@ -24,14 +24,14 @@ from subprocess import CalledProcessError from typing import Callable, Literal, NoReturn, Optional, Union -from ..common import ( +from otaclient.app.configs import config as cfg +from otaclient_api.v2 import types as api_types +from otaclient_common.common import ( read_str_from_file, subprocess_call, subprocess_check_output, write_str_to_file_sync, ) -from ..configs import config as cfg -from ..proto import wrapper logger = logging.getLogger(__name__) @@ -451,18 +451,18 @@ def _load_status_file(self): if _loaded_ota_status is None: logger.info( "ota_status files incompleted/not presented, " - f"initializing and set/store status to {wrapper.StatusOta.INITIALIZED.name}..." + f"initializing and set/store status to {api_types.StatusOta.INITIALIZED.name}..." ) - self._store_current_status(wrapper.StatusOta.INITIALIZED) - self._ota_status = wrapper.StatusOta.INITIALIZED + self._store_current_status(api_types.StatusOta.INITIALIZED) + self._ota_status = api_types.StatusOta.INITIALIZED return logger.info(f"status loaded from file: {_loaded_ota_status.name}") # status except UPDATING and ROLLBACKING(like SUCCESS/FAILURE/ROLLBACK_FAILURE) # are remained as it if _loaded_ota_status not in [ - wrapper.StatusOta.UPDATING, - wrapper.StatusOta.ROLLBACKING, + api_types.StatusOta.UPDATING, + api_types.StatusOta.ROLLBACKING, ]: self._ota_status = _loaded_ota_status return @@ -478,13 +478,13 @@ def _load_status_file(self): # in such case, otaclient will terminate and ota_status will not be updated. if self._is_switching_boot(self.active_slot): if self.finalize_switching_boot(): - self._ota_status = wrapper.StatusOta.SUCCESS - self._store_current_status(wrapper.StatusOta.SUCCESS) + self._ota_status = api_types.StatusOta.SUCCESS + self._store_current_status(api_types.StatusOta.SUCCESS) else: self._ota_status = ( - wrapper.StatusOta.ROLLBACK_FAILURE - if _loaded_ota_status == wrapper.StatusOta.ROLLBACKING - else wrapper.StatusOta.FAILURE + api_types.StatusOta.ROLLBACK_FAILURE + if _loaded_ota_status == api_types.StatusOta.ROLLBACKING + else api_types.StatusOta.FAILURE ) self._store_current_status(self._ota_status) logger.error( @@ -498,9 +498,9 @@ def _load_status_file(self): "this indicates a failed first reboot" ) self._ota_status = ( - wrapper.StatusOta.ROLLBACK_FAILURE - if _loaded_ota_status == wrapper.StatusOta.ROLLBACKING - else wrapper.StatusOta.FAILURE + api_types.StatusOta.ROLLBACK_FAILURE + if _loaded_ota_status == api_types.StatusOta.ROLLBACKING + else api_types.StatusOta.FAILURE ) self._store_current_status(self._ota_status) @@ -545,23 +545,23 @@ def _load_current_slot_in_use(self) -> Optional[str]: # status control - def _store_current_status(self, _status: wrapper.StatusOta): + def _store_current_status(self, _status: api_types.StatusOta): write_str_to_file_sync( self.current_ota_status_dir / cfg.OTA_STATUS_FNAME, _status.name ) - def _store_standby_status(self, _status: wrapper.StatusOta): + def _store_standby_status(self, _status: api_types.StatusOta): write_str_to_file_sync( self.standby_ota_status_dir / cfg.OTA_STATUS_FNAME, _status.name ) - def _load_current_status(self) -> Optional[wrapper.StatusOta]: + def _load_current_status(self) -> Optional[api_types.StatusOta]: if _status_str := read_str_from_file( self.current_ota_status_dir / cfg.OTA_STATUS_FNAME ).upper(): with contextlib.suppress(KeyError): # invalid status string - return wrapper.StatusOta[_status_str] + return api_types.StatusOta[_status_str] # version control @@ -577,8 +577,8 @@ def _is_switching_boot(self, active_slot: str) -> bool: """Detect whether we should switch boot or not with ota_status files.""" # evidence: ota_status _is_updating_or_rollbacking = self._load_current_status() in [ - wrapper.StatusOta.UPDATING, - wrapper.StatusOta.ROLLBACKING, + api_types.StatusOta.UPDATING, + api_types.StatusOta.ROLLBACKING, ] # evidence: slot_in_use @@ -598,7 +598,7 @@ def _is_switching_boot(self, active_slot: str) -> bool: def pre_update_current(self): """On pre_update stage, set current slot's status to FAILURE and set slot_in_use to standby slot.""" - self._store_current_status(wrapper.StatusOta.FAILURE) + self._store_current_status(api_types.StatusOta.FAILURE) self._store_current_slot_in_use(self.standby_slot) def pre_update_standby(self, *, version: str): @@ -610,17 +610,17 @@ def pre_update_standby(self, *, version: str): # create the ota-status folder unconditionally self.standby_ota_status_dir.mkdir(exist_ok=True, parents=True) # store status to standby slot - self._store_standby_status(wrapper.StatusOta.UPDATING) + self._store_standby_status(api_types.StatusOta.UPDATING) self._store_standby_version(version) self._store_standby_slot_in_use(self.standby_slot) def pre_rollback_current(self): - self._store_current_status(wrapper.StatusOta.FAILURE) + self._store_current_status(api_types.StatusOta.FAILURE) def pre_rollback_standby(self): # store ROLLBACKING status to standby self.standby_ota_status_dir.mkdir(exist_ok=True, parents=True) - self._store_standby_status(wrapper.StatusOta.ROLLBACKING) + self._store_standby_status(api_types.StatusOta.ROLLBACKING) def load_active_slot_version(self) -> str: return read_str_from_file( @@ -631,13 +631,13 @@ def load_active_slot_version(self) -> str: def on_failure(self): """Store FAILURE to status file on failure.""" - self._store_current_status(wrapper.StatusOta.FAILURE) + self._store_current_status(api_types.StatusOta.FAILURE) # when standby slot is not created, otastatus is not needed to be set if self.standby_ota_status_dir.is_dir(): - self._store_standby_status(wrapper.StatusOta.FAILURE) + self._store_standby_status(api_types.StatusOta.FAILURE) @property - def booted_ota_status(self) -> wrapper.StatusOta: + def booted_ota_status(self) -> api_types.StatusOta: """Loaded current slot's ota_status during boot control starts. NOTE: distinguish between the live ota_status maintained by otaclient. diff --git a/src/otaclient/app/boot_control/_grub.py b/src/otaclient/app/boot_control/_grub.py index 90766691c..dc01370b8 100644 --- a/src/otaclient/app/boot_control/_grub.py +++ b/src/otaclient/app/boot_control/_grub.py @@ -42,23 +42,23 @@ from subprocess import CalledProcessError from typing import ClassVar, Dict, Generator, List, Optional, Tuple -from .. import errors as ota_errors -from ..common import ( +from otaclient.app import errors as ota_errors +from otaclient.app.boot_control._common import ( + CMDHelperFuncs, + OTAStatusFilesControl, + SlotMountHelper, + cat_proc_cmdline, +) +from otaclient.app.boot_control.configs import grub_cfg as cfg +from otaclient.app.boot_control.protocol import BootControllerProtocol +from otaclient_api.v2 import types as api_types +from otaclient_common.common import ( re_symlink_atomic, read_str_from_file, subprocess_call, subprocess_check_output, write_str_to_file_sync, ) -from ..proto import wrapper -from ._common import ( - CMDHelperFuncs, - OTAStatusFilesControl, - SlotMountHelper, - cat_proc_cmdline, -) -from .configs import grub_cfg as cfg -from .protocol import BootControllerProtocol logger = logging.getLogger(__name__) @@ -865,7 +865,7 @@ def get_standby_boot_dir(self) -> Path: def load_version(self) -> str: return self._ota_status_control.load_active_slot_version() - def get_booted_ota_status(self) -> wrapper.StatusOta: + def get_booted_ota_status(self) -> api_types.StatusOta: return self._ota_status_control.booted_ota_status def on_operation_failure(self): diff --git a/src/otaclient/app/boot_control/_jetson_cboot.py b/src/otaclient/app/boot_control/_jetson_cboot.py index 4d9eb98b9..40b751ab7 100644 --- a/src/otaclient/app/boot_control/_jetson_cboot.py +++ b/src/otaclient/app/boot_control/_jetson_cboot.py @@ -26,12 +26,12 @@ from typing import Generator, Optional from otaclient.app import errors as ota_errors -from otaclient.app.common import subprocess_run_wrapper -from otaclient.app.proto import wrapper - -from ..configs import config as cfg -from ._common import CMDHelperFuncs, OTAStatusFilesControl, SlotMountHelper -from ._jetson_common import ( +from otaclient.app.boot_control._common import ( + CMDHelperFuncs, + OTAStatusFilesControl, + SlotMountHelper, +) +from otaclient.app.boot_control._jetson_common import ( FirmwareBSPVersionControl, NVBootctrlCommon, NVBootctrlTarget, @@ -41,8 +41,14 @@ preserve_ota_config_files_to_standby, update_standby_slot_extlinux_cfg, ) -from .configs import cboot_cfg as boot_cfg -from .protocol import BootControllerProtocol +from otaclient.app.boot_control.configs import cboot_cfg as boot_cfg +from otaclient.app.boot_control.protocol import BootControllerProtocol +from otaclient.app.configs import config as cfg +from otaclient_api.v2 import types as api_types +from otaclient_common.common import subprocess_run_wrapper + +logger = logging.getLogger(__name__) + logger = logging.getLogger(__name__) @@ -617,5 +623,5 @@ def on_operation_failure(self): def load_version(self) -> str: return self._ota_status_control.load_active_slot_version() - def get_booted_ota_status(self) -> wrapper.StatusOta: + def get_booted_ota_status(self) -> api_types.StatusOta: return self._ota_status_control.booted_ota_status diff --git a/src/otaclient/app/boot_control/_jetson_common.py b/src/otaclient/app/boot_control/_jetson_common.py index b01fab29a..b4d3b79b6 100644 --- a/src/otaclient/app/boot_control/_jetson_common.py +++ b/src/otaclient/app/boot_control/_jetson_common.py @@ -29,10 +29,8 @@ from pydantic import BaseModel, BeforeValidator, PlainSerializer from typing_extensions import Annotated, Literal, Self -from otaclient.app.common import write_str_to_file_sync - -from ..common import copytree_identical -from ._common import CMDHelperFuncs +from otaclient.app.boot_control._common import CMDHelperFuncs +from otaclient_common.common import copytree_identical, write_str_to_file_sync logger = logging.getLogger(__name__) diff --git a/src/otaclient/app/boot_control/_rpi_boot.py b/src/otaclient/app/boot_control/_rpi_boot.py index 3c78ae94f..330d7022b 100644 --- a/src/otaclient/app/boot_control/_rpi_boot.py +++ b/src/otaclient/app/boot_control/_rpi_boot.py @@ -23,17 +23,21 @@ from string import Template from typing import Generator -from .. import errors as ota_errors -from ..common import replace_atomic, subprocess_call, subprocess_check_output -from ..proto import wrapper -from ._common import ( +import otaclient.app.errors as ota_errors +from otaclient.app.boot_control._common import ( CMDHelperFuncs, OTAStatusFilesControl, SlotMountHelper, write_str_to_file_sync, ) -from .configs import rpi_boot_cfg as cfg -from .protocol import BootControllerProtocol +from otaclient.app.boot_control.configs import rpi_boot_cfg as cfg +from otaclient.app.boot_control.protocol import BootControllerProtocol +from otaclient_api.v2 import types as api_types +from otaclient_common.common import ( + replace_atomic, + subprocess_call, + subprocess_check_output, +) logger = logging.getLogger(__name__) @@ -375,8 +379,8 @@ def __init__(self) -> None: # 20230613: remove any leftover flag file if ota_status is not UPDATING/ROLLBACKING if self._ota_status_control.booted_ota_status not in ( - wrapper.StatusOta.UPDATING, - wrapper.StatusOta.ROLLBACKING, + api_types.StatusOta.UPDATING, + api_types.StatusOta.ROLLBACKING, ): _flag_file = ( self._rpiboot_control.system_boot_path / cfg.SWITCH_BOOT_FLAG_FILE @@ -546,5 +550,5 @@ def on_operation_failure(self): def load_version(self) -> str: return self._ota_status_control.load_active_slot_version() - def get_booted_ota_status(self) -> wrapper.StatusOta: + def get_booted_ota_status(self) -> api_types.StatusOta: return self._ota_status_control.booted_ota_status diff --git a/src/otaclient/app/boot_control/protocol.py b/src/otaclient/app/boot_control/protocol.py index 9883aee47..d9f579685 100644 --- a/src/otaclient/app/boot_control/protocol.py +++ b/src/otaclient/app/boot_control/protocol.py @@ -17,14 +17,14 @@ from pathlib import Path from typing import Generator, Protocol -from ..proto import wrapper +from otaclient_api.v2 import types as api_types class BootControllerProtocol(Protocol): """Boot controller protocol for otaclient.""" @abstractmethod - def get_booted_ota_status(self) -> wrapper.StatusOta: + def get_booted_ota_status(self) -> api_types.StatusOta: """Get the ota_status loaded from status file during otaclient starts up. This value is meant to be used only once during otaclient starts up, diff --git a/src/otaclient/app/boot_control/selecter.py b/src/otaclient/app/boot_control/selecter.py index 7e08982fa..778ff4f3d 100644 --- a/src/otaclient/app/boot_control/selecter.py +++ b/src/otaclient/app/boot_control/selecter.py @@ -21,9 +21,9 @@ from typing_extensions import deprecated -from ..common import read_str_from_file -from .configs import BootloaderType -from .protocol import BootControllerProtocol +from otaclient.app.boot_control.configs import BootloaderType +from otaclient.app.boot_control.protocol import BootControllerProtocol +from otaclient_common.common import read_str_from_file logger = logging.getLogger(__name__) diff --git a/src/otaclient/app/common.py b/src/otaclient/app/common.py deleted file mode 100644 index 08e874917..000000000 --- a/src/otaclient/app/common.py +++ /dev/null @@ -1,805 +0,0 @@ -# Copyright 2022 TIER IV, INC. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -r"""Utils that shared between modules are listed here.""" - - -from __future__ import annotations - -import itertools -import logging -import os -import shlex -import shutil -import subprocess -import threading -import time -from concurrent.futures import Future, ThreadPoolExecutor -from functools import lru_cache, partial -from hashlib import sha256 -from pathlib import Path -from queue import Queue -from typing import ( - Any, - Callable, - Generator, - Generic, - Iterable, - NamedTuple, - Optional, - Set, - TypeVar, - Union, -) -from urllib.parse import urljoin - -import requests - -from otaclient._utils.linux import ( - ParsedGroup, - ParsedPasswd, - map_gid_by_grpnam, - map_uid_by_pwnam, -) - -from .configs import config as cfg - -logger = logging.getLogger(__name__) - - -def get_backoff(n: int, factor: float, _max: float) -> float: - return min(_max, factor * (2 ** (n - 1))) - - -def wait_with_backoff(_retry_cnt: int, *, _backoff_factor: float, _backoff_max: float): - time.sleep( - get_backoff( - _retry_cnt, - _backoff_factor, - _backoff_max, - ) - ) - - -# file verification -def file_sha256(filename: Union[Path, str]) -> str: - with open(filename, "rb") as f: - m = sha256() - while True: - d = f.read(cfg.LOCAL_CHUNK_SIZE) - if len(d) == 0: - break - m.update(d) - return m.hexdigest() - - -def verify_file(fpath: Path, fhash: str, fsize: Optional[int]) -> bool: - if ( - fpath.is_symlink() - or (not fpath.is_file()) - or (fsize is not None and fpath.stat().st_size != fsize) - ): - return False - return file_sha256(fpath) == fhash - - -# handled file read/write -def read_str_from_file(path: Union[Path, str], *, missing_ok=True, default="") -> str: - """ - Params: - missing_ok: if set to False, FileNotFoundError will be raised to upper - default: the default value to return when missing_ok=True and file not found - """ - try: - return Path(path).read_text().strip() - except FileNotFoundError: - if missing_ok: - return default - - raise - - -def write_str_to_file(path: Path, input: str): - path.write_text(input) - - -def write_str_to_file_sync(path: Union[Path, str], input: str): - with open(path, "w") as f: - f.write(input) - f.flush() - os.fsync(f.fileno()) - - -def subprocess_run_wrapper( - cmd: str | list[str], - *, - check: bool, - check_output: bool, - timeout: Optional[float] = None, -) -> subprocess.CompletedProcess[bytes]: - """A wrapper for subprocess.run method. - - NOTE: this is for the requirement of customized subprocess call - in the future, like chroot or nsenter before execution. - - Args: - cmd (str | list[str]): command to be executed. - check (bool): if True, raise CalledProcessError on non 0 return code. - check_output (bool): if True, the UTF-8 decoded stdout will be returned. - timeout (Optional[float], optional): timeout for execution. Defaults to None. - - Returns: - subprocess.CompletedProcess[bytes]: the result of the execution. - """ - if isinstance(cmd, str): - cmd = shlex.split(cmd) - - return subprocess.run( - cmd, - check=check, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE if check_output else None, - timeout=timeout, - ) - - -def subprocess_check_output( - cmd: str | list[str], - *, - raise_exception: bool = False, - default: str = "", - timeout: Optional[float] = None, -) -> str: - """Run the and return UTF-8 decoded stripped stdout. - - Args: - cmd (str | list[str]): command to be executed. - raise_exception (bool, optional): raise the underlying CalledProcessError. Defaults to False. - default (str, optional): if is False, return on underlying - subprocess call failed. Defaults to "". - timeout (Optional[float], optional): timeout for execution. Defaults to None. - - Returns: - str: UTF-8 decoded stripped stdout. - """ - try: - res = subprocess_run_wrapper( - cmd, check=True, check_output=True, timeout=timeout - ) - return res.stdout.decode().strip() - except subprocess.CalledProcessError as e: - _err_msg = ( - f"command({cmd=}) failed(retcode={e.returncode}: \n" - f"stderr={e.stderr.decode()}" - ) - logger.debug(_err_msg) - - if raise_exception: - raise - return default - - -def subprocess_call( - cmd: str | list[str], - *, - raise_exception: bool = False, - timeout: Optional[float] = None, -) -> None: - """Run the . - - Args: - cmd (str | list[str]): command to be executed. - raise_exception (bool, optional): raise the underlying CalledProcessError. Defaults to False. - timeout (Optional[float], optional): timeout for execution. Defaults to None. - """ - try: - subprocess_run_wrapper(cmd, check=True, check_output=False, timeout=timeout) - except subprocess.CalledProcessError as e: - _err_msg = ( - f"command({cmd=}) failed(retcode={e.returncode}: \n" - f"stderr={e.stderr.decode()}" - ) - logger.debug(_err_msg) - - if raise_exception: - raise - - -def copy_stat(src: Union[Path, str], dst: Union[Path, str]): - """Copy file/dir permission bits and owner info from src to dst.""" - _stat = Path(src).stat() - os.chown(dst, _stat.st_uid, _stat.st_gid) - os.chmod(dst, _stat.st_mode) - - -def copytree_identical(src: Path, dst: Path): - """Recursively copy from the src folder to dst folder. - - Source folder MUST be a dir. - - This function populate files/dirs from the src to the dst, - and make sure the dst is identical to the src. - - By updating the dst folder in-place, we can prevent the case - that the copy is interrupted and the dst is not yet fully populated. - - This function is different from shutil.copytree as follow: - 1. it covers the case that the same path points to different - file type, in this case, the dst path will be clean and - new file/dir will be populated as the src. - 2. it deals with the same symlinks by checking the link target, - re-generate the symlink if the dst symlink is not the same - as the src. - 3. it will remove files that not presented in the src, and - unconditionally override files with same path, ensuring - that the dst will be identical with the src. - - NOTE: is_file/is_dir also returns True if it is a symlink and - the link target is_file/is_dir - """ - if src.is_symlink() or not src.is_dir(): - raise ValueError(f"{src} is not a dir") - - if dst.is_symlink() or not dst.is_dir(): - logger.info(f"{dst=} doesn't exist or not a dir, cleanup and mkdir") - dst.unlink(missing_ok=True) # unlink doesn't follow the symlink - dst.mkdir(mode=src.stat().st_mode, parents=True) - - # phase1: populate files to the dst - for cur_dir, dirs, files in os.walk(src, topdown=True, followlinks=False): - _cur_dir = Path(cur_dir) - _cur_dir_on_dst = dst / _cur_dir.relative_to(src) - - # NOTE(20220803): os.walk now lists symlinks pointed to dir - # in the tuple, we have to handle this behavior - for _dir in dirs: - _src_dir = _cur_dir / _dir - _dst_dir = _cur_dir_on_dst / _dir - if _src_dir.is_symlink(): # this "dir" is a symlink to a dir - if (not _dst_dir.is_symlink()) and _dst_dir.is_dir(): - # if dst is a dir, remove it - shutil.rmtree(_dst_dir, ignore_errors=True) - else: # dst is symlink or file - _dst_dir.unlink() - _dst_dir.symlink_to(os.readlink(_src_dir)) - - # cover the edge case that dst is not a dir. - if _cur_dir_on_dst.is_symlink() or not _cur_dir_on_dst.is_dir(): - _cur_dir_on_dst.unlink(missing_ok=True) - _cur_dir_on_dst.mkdir(parents=True) - copy_stat(_cur_dir, _cur_dir_on_dst) - - # populate files - for fname in files: - _src_f = _cur_dir / fname - _dst_f = _cur_dir_on_dst / fname - - # prepare dst - # src is file but dst is a folder - # delete the dst in advance - if (not _dst_f.is_symlink()) and _dst_f.is_dir(): - # if dst is a dir, remove it - shutil.rmtree(_dst_f, ignore_errors=True) - else: - # dst is symlink or file - _dst_f.unlink(missing_ok=True) - - # copy/symlink dst as src - # if src is symlink, check symlink, re-link if needed - if _src_f.is_symlink(): - _dst_f.symlink_to(os.readlink(_src_f)) - else: - # copy/override src to dst - shutil.copy(_src_f, _dst_f, follow_symlinks=False) - copy_stat(_src_f, _dst_f) - - # phase2: remove unused files in the dst - for cur_dir, dirs, files in os.walk(dst, topdown=True, followlinks=False): - _cur_dir_on_dst = Path(cur_dir) - _cur_dir_on_src = src / _cur_dir_on_dst.relative_to(dst) - - # remove unused dir - if not _cur_dir_on_src.is_dir(): - shutil.rmtree(_cur_dir_on_dst, ignore_errors=True) - dirs.clear() # stop iterate the subfolders of this dir - continue - - # NOTE(20220803): os.walk now lists symlinks pointed to dir - # in the tuple, we have to handle this behavior - for _dir in dirs: - _src_dir = _cur_dir_on_src / _dir - _dst_dir = _cur_dir_on_dst / _dir - if (not _src_dir.is_symlink()) and _dst_dir.is_symlink(): - _dst_dir.unlink() - - for fname in files: - _src_f = _cur_dir_on_src / fname - if not (_src_f.is_symlink() or _src_f.is_file()): - (_cur_dir_on_dst / fname).unlink(missing_ok=True) - - -def re_symlink_atomic(src: Path, target: Union[Path, str]): - """Make the a symlink to atomically. - - If the src is already existed as a file/symlink, - the src will be replaced by the newly created link unconditionally. - - NOTE: os.rename is atomic when src and dst are on - the same filesystem under linux. - NOTE 2: src should not exist or exist as file/symlink. - """ - if not (src.is_symlink() and str(os.readlink(src)) == str(target)): - tmp_link = Path(src).parent / f"tmp_link_{os.urandom(6).hex()}" - try: - tmp_link.symlink_to(target) - os.rename(tmp_link, src) # unconditionally override - except Exception: - tmp_link.unlink(missing_ok=True) - raise - - -def replace_atomic(src: Union[str, Path], dst: Union[str, Path]): - """Atomically replace dst file with src file. - - NOTE: atomic is ensured by os.rename/os.replace under the same filesystem. - """ - src, dst = Path(src), Path(dst) - if not src.is_file(): - raise ValueError(f"{src=} is not a regular file or not exist") - - _tmp_file = dst.parent / f".tmp_{os.urandom(6).hex()}" - try: - # prepare a copy of src file under dst's parent folder - shutil.copy(src, _tmp_file, follow_symlinks=True) - # atomically rename/replace the dst file with the copy - os.replace(_tmp_file, dst) - os.sync() - except Exception: - _tmp_file.unlink(missing_ok=True) - raise - - -def urljoin_ensure_base(base: str, url: str): - """ - NOTE: this method ensure the base_url will be preserved. - for example: - base="http://example.com/data", url="path/to/file" - with urljoin, joined url will be "http://example.com/path/to/file", - with this func, joined url will be "http://example.com/data/path/to/file" - """ - return urljoin(f"{base.rstrip('/')}/", url) - - -# ------ RetryTaskMap ------ # - -T = TypeVar("T") - - -class DoneTask(NamedTuple): - fut: Future - entry: Any - - -class RetryTaskMapInterrupted(Exception): - pass - - -class _TaskMap(Generic[T]): - def __init__( - self, - executor: ThreadPoolExecutor, - max_concurrent: int, - backoff_func: Callable[[int], float], - ) -> None: - # task dispatch interval for continues failling - self.started = False # can only be started once - self._backoff_func = backoff_func - self._executor = executor - self._shutdown_event = threading.Event() - self._se = threading.Semaphore(max_concurrent) - - self._total_tasks_count = 0 - self._dispatched_tasks: Set[Future] = set() - self._failed_tasks: Set[T] = set() - self._last_failed_fut: Optional[Future] = None - - # NOTE: itertools.count is only thread-safe in CPython with GIL, - # as itertools.count is pure C implemented, calling next over - # it is atomic in Python level. - self._done_task_counter = itertools.count(start=1) - self._all_done = threading.Event() - self._dispatch_done = False - - self._done_que: Queue[DoneTask] = Queue() - - def _done_task_cb(self, item: T, fut: Future): - """ - Tracking done counting, set all_done event. - add failed to failed list. - """ - self._se.release() # always release se first - # NOTE: don't change dispatched_tasks if shutdown_event is set - if self._shutdown_event.is_set(): - return - - self._dispatched_tasks.discard(fut) - # check if we finish all tasks - _done_task_num = next(self._done_task_counter) - if self._dispatch_done and _done_task_num == self._total_tasks_count: - logger.debug("all done!") - self._all_done.set() - - if fut.exception(): - self._failed_tasks.add(item) - self._last_failed_fut = fut - self._done_que.put_nowait(DoneTask(fut, item)) - - def _task_dispatcher(self, func: Callable[[T], Any], _iter: Iterable[T]): - """A dispatcher in a dedicated thread that dispatches - tasks to threadpool.""" - for item in _iter: - if self._shutdown_event.is_set(): - return - self._se.acquire() - self._total_tasks_count += 1 - - fut = self._executor.submit(func, item) - fut.add_done_callback(partial(self._done_task_cb, item)) - self._dispatched_tasks.add(fut) - logger.debug(f"dispatcher done: {self._total_tasks_count=}") - self._dispatch_done = True - - def _done_task_collector(self) -> Generator[DoneTask, None, None]: - """A generator for caller to yield done task from.""" - _count = 0 - while not self._shutdown_event.is_set(): - if self._all_done.is_set() and _count == self._total_tasks_count: - logger.debug("collector done!") - return - - yield self._done_que.get() - _count += 1 - - def map(self, func: Callable[[T], Any], _iter: Iterable[T]): - if self.started: - raise ValueError(f"{self.__class__} inst can only be started once") - self.started = True - - self._task_dispatcher_fut = self._executor.submit( - self._task_dispatcher, func, _iter - ) - self._task_collector_gen = self._done_task_collector() - return self._task_collector_gen - - def shutdown(self, *, raise_last_exc=False) -> Optional[Set[T]]: - """Set the shutdown event, and cancal/cleanup ongoing tasks.""" - if not self.started or self._shutdown_event.is_set(): - return - - self._shutdown_event.set() - self._task_collector_gen.close() - # wait for dispatch to stop - self._task_dispatcher_fut.result() - - # cancel all the dispatched tasks - for fut in self._dispatched_tasks: - fut.cancel() - self._dispatched_tasks.clear() - - if not self._failed_tasks: - return - try: - if self._last_failed_fut: - _exc = self._last_failed_fut.exception() - _err_msg = f"{len(self._failed_tasks)=}, last failed: {_exc!r}" - if raise_last_exc: - raise RetryTaskMapInterrupted(_err_msg) from _exc - else: - logger.warning(_err_msg) - return self._failed_tasks.copy() - finally: - # be careful not to create ref cycle here - self._failed_tasks.clear() - _exc, self = None, None - - -class RetryTaskMap(Generic[T]): - def __init__( - self, - *, - backoff_func: Callable[[int], float], - max_retry: int, - max_concurrent: int, - max_workers: Optional[int] = None, - ) -> None: - self._running_inst: Optional[_TaskMap] = None - self._map_gen: Optional[Generator] = None - - self._backoff_func = backoff_func - self._retry_counter = range(max_retry) if max_retry else itertools.count() - self._max_concurrent = max_concurrent - self._max_workers = max_workers - self._executor = ThreadPoolExecutor(max_workers=self._max_workers) - - def map( - self, _func: Callable[[T], Any], _iter: Iterable[T] - ) -> Generator[DoneTask, None, None]: - retry_round = 0 - for retry_round in self._retry_counter: - self._running_inst = _inst = _TaskMap( - self._executor, self._max_concurrent, self._backoff_func - ) - logger.debug(f"{retry_round=} started") - - yield from _inst.map(_func, _iter) - - # this retry round ends, check overall result - if _failed_list := _inst.shutdown(raise_last_exc=False): - _iter = _failed_list # feed failed to next round - # deref before entering sleep - self._running_inst, _inst = None, None - - logger.warning(f"retry#{retry_round+1}: retry on {len(_failed_list)=}") - time.sleep(self._backoff_func(retry_round)) - else: # all tasks finished successfully - self._running_inst, _inst = None, None - return - try: - raise RetryTaskMapInterrupted(f"exceed try limit: {retry_round}") - finally: - # cleanup the defs - _func, _iter = None, None # type: ignore - - def shutdown(self, *, raise_last_exc: bool): - try: - logger.debug("shutdown retry task map") - if self._running_inst: - self._running_inst.shutdown(raise_last_exc=raise_last_exc) - # NOTE: passthrough the exception from underlying running_inst - finally: - self._running_inst = None - self._executor.shutdown(wait=True) - - -def create_tmp_fname(prefix="tmp", length=6, sep="_") -> str: - return f"{prefix}{sep}{os.urandom(length).hex()}" - - -def ensure_otaproxy_start( - otaproxy_url: str, - *, - interval: float = 1, - connection_timeout: float = 5, - probing_timeout: Optional[float] = None, - warning_interval: int = 3 * 60, # seconds -): - """Loop probing until online or exceed . - - This function will issue a logging.warning every seconds. - - Raises: - A ConnectionError if exceeds . - """ - start_time = int(time.time()) - next_warning = start_time + warning_interval - probing_timeout = ( - probing_timeout if probing_timeout and probing_timeout >= 0 else float("inf") - ) - with requests.Session() as session: - while start_time + probing_timeout > (cur_time := int(time.time())): - try: - resp = session.get(otaproxy_url, timeout=connection_timeout) - resp.close() - return - except Exception as e: # server is not up yet - if cur_time >= next_warning: - logger.warning( - f"otaproxy@{otaproxy_url} is not up after {cur_time - start_time} seconds" - f"it might be something wrong with this otaproxy: {e!r}" - ) - next_warning = next_warning + warning_interval - time.sleep(interval) - raise ConnectionError( - f"failed to ensure connection to {otaproxy_url} in {probing_timeout=}seconds" - ) - - -# -# ------ persist files handling ------ # -# -class PersistFilesHandler: - """Preserving files in persist list from to . - - Files being copied will have mode bits preserved, - and uid/gid preserved with mapping as follow: - - src_uid -> src_name -> dst_name -> dst_uid - src_gid -> src_name -> dst_name -> dst_gid - """ - - def __init__( - self, - src_passwd_file: str | Path, - src_group_file: str | Path, - dst_passwd_file: str | Path, - dst_group_file: str | Path, - *, - src_root: str | Path, - dst_root: str | Path, - ): - self._uid_mapper = lru_cache()( - partial( - self.map_uid_by_pwnam, - src_db=ParsedPasswd(src_passwd_file), - dst_db=ParsedPasswd(dst_passwd_file), - ) - ) - self._gid_mapper = lru_cache()( - partial( - self.map_gid_by_grpnam, - src_db=ParsedGroup(src_group_file), - dst_db=ParsedGroup(dst_group_file), - ) - ) - self._src_root = Path(src_root) - self._dst_root = Path(dst_root) - - @staticmethod - def map_uid_by_pwnam( - *, src_db: ParsedPasswd, dst_db: ParsedPasswd, uid: int - ) -> int: - _mapped_uid = map_uid_by_pwnam(src_db=src_db, dst_db=dst_db, uid=uid) - _usern = src_db._by_uid[uid] - - logger.info(f"{_usern=}: mapping src_{uid=} to {_mapped_uid=}") - return _mapped_uid - - @staticmethod - def map_gid_by_grpnam(*, src_db: ParsedGroup, dst_db: ParsedGroup, gid: int) -> int: - _mapped_gid = map_gid_by_grpnam(src_db=src_db, dst_db=dst_db, gid=gid) - _groupn = src_db._by_gid[gid] - - logger.info(f"{_groupn=}: mapping src_{gid=} to {_mapped_gid=}") - return _mapped_gid - - def _chown_with_mapping( - self, _src_stat: os.stat_result, _dst_path: str | Path - ) -> None: - _src_uid, _src_gid = _src_stat.st_uid, _src_stat.st_gid - try: - _dst_uid = self._uid_mapper(uid=_src_uid) - except ValueError: - logger.warning(f"failed to find mapping for {_src_uid=}, keep unchanged") - _dst_uid = _src_uid - - try: - _dst_gid = self._gid_mapper(gid=_src_gid) - except ValueError: - logger.warning(f"failed to find mapping for {_src_gid=}, keep unchanged") - _dst_gid = _src_gid - os.chown(_dst_path, uid=_dst_uid, gid=_dst_gid, follow_symlinks=False) - - @staticmethod - def _rm_target(_target: Path) -> None: - """Remove target with proper methods.""" - if _target.is_symlink() or _target.is_file(): - return _target.unlink(missing_ok=True) - elif _target.is_dir(): - return shutil.rmtree(_target, ignore_errors=True) - elif _target.exists(): - raise ValueError( - f"{_target} is not normal file/symlink/dir, failed to remove" - ) - - def _prepare_symlink(self, _src_path: Path, _dst_path: Path) -> None: - _dst_path.symlink_to(os.readlink(_src_path)) - # NOTE: to get stat from symlink, using os.stat with follow_symlinks=False - self._chown_with_mapping(os.stat(_src_path, follow_symlinks=False), _dst_path) - - def _prepare_dir(self, _src_path: Path, _dst_path: Path) -> None: - _dst_path.mkdir(exist_ok=True) - - _src_stat = os.stat(_src_path, follow_symlinks=False) - os.chmod(_dst_path, _src_stat.st_mode) - self._chown_with_mapping(_src_stat, _dst_path) - - def _prepare_file(self, _src_path: Path, _dst_path: Path) -> None: - shutil.copy(_src_path, _dst_path, follow_symlinks=False) - - _src_stat = os.stat(_src_path, follow_symlinks=False) - os.chmod(_dst_path, _src_stat.st_mode) - self._chown_with_mapping(_src_stat, _dst_path) - - def _prepare_parent(self, _origin_entry: Path) -> None: - for _parent in reversed(_origin_entry.parents): - _src_parent, _dst_parent = ( - self._src_root / _parent, - self._dst_root / _parent, - ) - if _dst_parent.is_dir(): # keep the origin parent on dst as it - continue - if _dst_parent.is_symlink() or _dst_parent.is_file(): - _dst_parent.unlink(missing_ok=True) - self._prepare_dir(_src_parent, _dst_parent) - continue - if _dst_parent.exists(): - raise ValueError( - f"{_dst_parent=} is not a normal file/symlink/dir, cannot cleanup" - ) - self._prepare_dir(_src_parent, _dst_parent) - - # API - - def preserve_persist_entry( - self, _persist_entry: str | Path, *, src_missing_ok: bool = True - ): - logger.info(f"preserving {_persist_entry}") - # persist_entry in persists.txt must be rooted at / - origin_entry = Path(_persist_entry).relative_to("/") - src_path = self._src_root / origin_entry - dst_path = self._dst_root / origin_entry - - # ------ src is symlink ------ # - # NOTE: always check if symlink first as is_file/is_dir/exists all follow_symlinks - if src_path.is_symlink(): - self._rm_target(dst_path) - self._prepare_parent(origin_entry) - self._prepare_symlink(src_path, dst_path) - return - - # ------ src is file ------ # - if src_path.is_file(): - self._rm_target(dst_path) - self._prepare_parent(origin_entry) - self._prepare_file(src_path, dst_path) - return - - # ------ src is not regular file/symlink/dir ------ # - # we only process normal file/symlink/dir - if src_path.exists() and not src_path.is_dir(): - raise ValueError(f"{src_path=} must be either a file/symlink/dir") - - # ------ src doesn't exist ------ # - if not src_path.exists(): - _err_msg = f"{src_path=} not found" - logger.warning(_err_msg) - if not src_missing_ok: - raise ValueError(_err_msg) - return - - # ------ src is dir ------ # - # dive into src_dir and preserve everything under the src dir - self._prepare_parent(origin_entry) - for src_curdir, dnames, fnames in os.walk(src_path, followlinks=False): - src_cur_dpath = Path(src_curdir) - dst_cur_dpath = self._dst_root / src_cur_dpath.relative_to(self._src_root) - - # ------ prepare current dir itself ------ # - self._rm_target(dst_cur_dpath) - self._prepare_dir(src_cur_dpath, dst_cur_dpath) - - # ------ prepare entries in current dir ------ # - for _fname in fnames: - _src_fpath, _dst_fpath = src_cur_dpath / _fname, dst_cur_dpath / _fname - self._rm_target(_dst_fpath) - if _src_fpath.is_symlink(): - self._prepare_symlink(_src_fpath, _dst_fpath) - continue - self._prepare_file(_src_fpath, _dst_fpath) - - # symlinks to dirs also included in dnames, we must handle it - for _dname in dnames: - _src_dpath, _dst_dpath = src_cur_dpath / _dname, dst_cur_dpath / _dname - if _src_dpath.is_symlink(): - self._rm_target(_dst_dpath) - self._prepare_symlink(_src_dpath, _dst_dpath) diff --git a/src/otaclient/app/configs.py b/src/otaclient/app/configs.py index f7a0ed725..9aa5da3f2 100644 --- a/src/otaclient/app/configs.py +++ b/src/otaclient/app/configs.py @@ -93,14 +93,10 @@ class BaseConfig(_InternalSettings): # ------ otaclient logging setting ------ # DEFAULT_LOG_LEVEL = INFO LOG_LEVEL_TABLE: Dict[str, int] = { - "otaclient.app.boot_control.cboot": INFO, - "otaclient.app.boot_control.grub": INFO, - "otaclient.app.ota_client": INFO, - "otaclient.app.ota_client_service": INFO, - "otaclient.app.ota_client_stub": INFO, - "otaclient.app.ota_metadata": INFO, - "otaclient.app.downloader": INFO, - "otaclient.app.main": INFO, + "ota_metadata": INFO, + "otaclient": INFO, + "otaclient_api": INFO, + "otaclient_common": INFO, } LOG_FORMAT = ( "[%(asctime)s][%(levelname)s]-%(name)s:%(funcName)s:%(lineno)d,%(message)s" diff --git a/src/otaclient/app/create_standby/common.py b/src/otaclient/app/create_standby/common.py index 6adb20c6e..eb3df40a2 100644 --- a/src/otaclient/app/create_standby/common.py +++ b/src/otaclient/app/create_standby/common.py @@ -30,10 +30,11 @@ from typing import Any, Dict, Iterator, List, Optional, OrderedDict, Set, Tuple, Union from weakref import WeakKeyDictionary, WeakValueDictionary -from ..common import create_tmp_fname +from ota_metadata.legacy.parser import MetafilesV1, OTAMetadata +from ota_metadata.legacy.types import DirectoryInf, RegularInf +from otaclient_common.common import create_tmp_fname + from ..configs import config as cfg -from ..ota_metadata import MetafilesV1, OTAMetadata -from ..proto.wrapper import DirectoryInf, RegularInf from ..update_stats import ( OTAUpdateStatsCollector, RegInfProcessedStats, diff --git a/src/otaclient/app/create_standby/interface.py b/src/otaclient/app/create_standby/interface.py index 41f5e5709..c66a6123e 100644 --- a/src/otaclient/app/create_standby/interface.py +++ b/src/otaclient/app/create_standby/interface.py @@ -33,7 +33,8 @@ from abc import abstractmethod from typing import Protocol -from ..ota_metadata import OTAMetadata +from ota_metadata.legacy.parser import OTAMetadata + from ..update_stats import OTAUpdateStatsCollector from .common import DeltaBundle diff --git a/src/otaclient/app/create_standby/rebuild_mode.py b/src/otaclient/app/create_standby/rebuild_mode.py index b8f7e0a24..26d7b6a2d 100644 --- a/src/otaclient/app/create_standby/rebuild_mode.py +++ b/src/otaclient/app/create_standby/rebuild_mode.py @@ -21,10 +21,12 @@ from pathlib import Path from typing import List, Set, Tuple -from ..common import RetryTaskMap, get_backoff +from ota_metadata.legacy.parser import MetafilesV1, OTAMetadata +from ota_metadata.legacy.types import RegularInf +from otaclient_common.common import get_backoff +from otaclient_common.retry_task_map import RetryTaskMap + from ..configs import config as cfg -from ..ota_metadata import MetafilesV1, OTAMetadata -from ..proto.wrapper import RegularInf from ..update_stats import ( OTAUpdateStatsCollector, RegInfProcessedStats, diff --git a/src/otaclient/app/errors.py b/src/otaclient/app/errors.py index 83408009f..6a673672e 100644 --- a/src/otaclient/app/errors.py +++ b/src/otaclient/app/errors.py @@ -18,7 +18,7 @@ from enum import Enum, unique from typing import ClassVar -from .proto import wrapper +from otaclient_api.v2 import types as api_types @unique @@ -70,7 +70,7 @@ class OTAError(Exception): ERROR_PREFIX: ClassVar[str] = "E" - failure_type: wrapper.FailureType = wrapper.FailureType.RECOVERABLE + failure_type: api_types.FailureType = api_types.FailureType.RECOVERABLE failure_errcode: OTAErrorCode = OTAErrorCode.E_UNSPECIFIC failure_description: str = "no description available for this error" @@ -118,7 +118,7 @@ def get_error_report(self, title: str = "") -> str: class NetworkError(OTAError): """Generic network error""" - failure_type: wrapper.FailureType = wrapper.FailureType.RECOVERABLE + failure_type: api_types.FailureType = api_types.FailureType.RECOVERABLE failure_errcode: OTAErrorCode = OTAErrorCode.E_NETWORK failure_description: str = _NETWORK_ERR_DEFAULT_DESC @@ -141,7 +141,7 @@ class OTAMetaDownloadFailed(NetworkError): class OTAErrorRecoverable(OTAError): - failure_type: wrapper.FailureType = wrapper.FailureType.RECOVERABLE + failure_type: api_types.FailureType = api_types.FailureType.RECOVERABLE failure_errcode: OTAErrorCode = OTAErrorCode.E_OTA_ERR_RECOVERABLE failure_description: str = _RECOVERABLE_DEFAULT_DESC @@ -168,7 +168,7 @@ class InvalidStatusForOTARollback(OTAErrorRecoverable): class OTAErrorUnrecoverable(OTAError): - failure_type: wrapper.FailureType = wrapper.FailureType.RECOVERABLE + failure_type: api_types.FailureType = api_types.FailureType.RECOVERABLE failure_errcode: OTAErrorCode = OTAErrorCode.E_OTA_ERR_UNRECOVERABLE failure_description: str = _UNRECOVERABLE_DEFAULT_DESC diff --git a/src/otaclient/app/interface.py b/src/otaclient/app/interface.py index cda4e0cfe..fe8bfc794 100644 --- a/src/otaclient/app/interface.py +++ b/src/otaclient/app/interface.py @@ -16,9 +16,10 @@ from abc import abstractmethod from typing import Protocol, Type +from otaclient_api.v2 import otaclient_v2_pb2 as pb2 + from .boot_control.protocol import BootControllerProtocol from .create_standby.interface import StandbySlotCreatorProtocol -from .proto import v2 class OTAClientProtocol(Protocol): @@ -44,4 +45,4 @@ def update( def rollback(self) -> None: ... @abstractmethod - def status(self) -> v2.StatusResponseEcu: ... + def status(self) -> pb2.StatusResponseEcu: ... diff --git a/src/otaclient/app/main.py b/src/otaclient/app/main.py index 63855d9c4..3d99d502e 100644 --- a/src/otaclient/app/main.py +++ b/src/otaclient/app/main.py @@ -13,20 +13,27 @@ # limitations under the License. +from __future__ import annotations + import asyncio import logging import os import sys from pathlib import Path -from otaclient import __version__ # type: ignore +import grpc.aio -from .common import read_str_from_file, write_str_to_file_sync -from .configs import config as cfg -from .configs import ecu_info -from .log_setting import configure_logging -from .ota_client_service import launch_otaclient_grpc_server -from .proto import ota_metafiles, v2, v2_grpc, wrapper # noqa: F401 +# NOTE: as otaclient_api and ota_metadata are using dynamic module import, +# we need to import them before any other otaclient modules. +import ota_metadata.legacy # noqa: F401 +from otaclient import __version__ +from otaclient.app.configs import config as cfg +from otaclient.app.configs import ecu_info, server_cfg +from otaclient.app.log_setting import configure_logging +from otaclient.app.ota_client_stub import OTAClientServiceStub +from otaclient_api.v2 import otaclient_v2_pb2_grpc as v2_grpc +from otaclient_api.v2.api_stub import OtaClientServiceV2 +from otaclient_common.common import read_str_from_file, write_str_to_file_sync # configure logging before any code being executed configure_logging() @@ -52,6 +59,24 @@ def _check_other_otaclient(): write_str_to_file_sync(cfg.OTACLIENT_PID_FILE, f"{os.getpid()}") +def create_otaclient_grpc_server(): + service_stub = OTAClientServiceStub() + ota_client_service_v2 = OtaClientServiceV2(service_stub) + + server = grpc.aio.server() + v2_grpc.add_OtaClientServiceServicer_to_server( + server=server, servicer=ota_client_service_v2 + ) + server.add_insecure_port(f"{ecu_info.ip_addr}:{server_cfg.SERVER_PORT}") + return server + + +async def launch_otaclient_grpc_server(): + server = create_otaclient_grpc_server() + await server.start() + await server.wait_for_termination() + + def main(): logger.info("started") logger.info(f"otaclient version: {__version__}") diff --git a/src/otaclient/app/ota_client.py b/src/otaclient/app/ota_client.py index fa7eed016..8ef78728b 100644 --- a/src/otaclient/app/ota_client.py +++ b/src/otaclient/app/ota_client.py @@ -29,37 +29,55 @@ from typing import Iterator, Optional, Type from urllib.parse import urlparse -from . import downloader +from ota_metadata.legacy import parser as ota_metadata_parser +from ota_metadata.legacy import types as ota_metadata_types +from otaclient import __version__ +from otaclient_api.v2 import types as api_types +from otaclient_common import downloader +from otaclient_common.common import ensure_otaproxy_start, get_backoff +from otaclient_common.persist_file_handling import PersistFilesHandler +from otaclient_common.retry_task_map import RetryTaskMap, RetryTaskMapInterrupted + from . import errors as ota_errors -from . import ota_metadata from .boot_control import BootControllerProtocol, get_boot_controller -from .common import ( - PersistFilesHandler, - RetryTaskMap, - RetryTaskMapInterrupted, - ensure_otaproxy_start, - get_backoff, -) from .configs import config as cfg from .configs import ecu_info from .create_standby import StandbySlotCreatorProtocol, get_standby_slot_creator from .interface import OTAClientProtocol -from .ota_status import LiveOTAStatus -from .proto import wrapper from .update_stats import ( OTAUpdateStatsCollector, RegInfProcessedStats, RegProcessOperation, ) -try: - from otaclient import __version__ # type: ignore -except ImportError: - __version__ = "unknown" - logger = logging.getLogger(__name__) +class LiveOTAStatus: + def __init__(self, ota_status: api_types.StatusOta) -> None: + self.live_ota_status = ota_status + + def get_ota_status(self) -> api_types.StatusOta: + return self.live_ota_status + + def set_ota_status(self, _status: api_types.StatusOta): + self.live_ota_status = _status + + def request_update(self) -> bool: + return self.live_ota_status in [ + api_types.StatusOta.INITIALIZED, + api_types.StatusOta.SUCCESS, + api_types.StatusOta.FAILURE, + api_types.StatusOta.ROLLBACK_FAILURE, + ] + + def request_rollback(self) -> bool: + return self.live_ota_status in [ + api_types.StatusOta.SUCCESS, + api_types.StatusOta.ROLLBACK_FAILURE, + ] + + class OTAClientControlFlags: """ When self ECU's otaproxy is enabled, all the child ECUs of this ECU @@ -100,12 +118,12 @@ def __init__( self._create_standby_cls = create_standby_cls # init update status - self.update_phase = wrapper.UpdatePhase.INITIALIZING + self.update_phase = api_types.UpdatePhase.INITIALIZING self.update_start_time = 0 self.updating_version: str = "" self.failure_reason = "" # init variables needed for update - self._otameta: ota_metadata.OTAMetadata = None # type: ignore + self._otameta: ota_metadata_parser.OTAMetadata = None # type: ignore self._url_base: str = None # type: ignore # dynamic update status @@ -132,10 +150,12 @@ def __init__( # helper methods - def _download_files(self, download_list: Iterator[wrapper.RegularInf]): + def _download_files(self, download_list: Iterator[ota_metadata_types.RegularInf]): """Download all needed OTA image files indicated by calculated bundle.""" - def _download_file(entry: wrapper.RegularInf) -> RegInfProcessedStats: + def _download_file( + entry: ota_metadata_types.RegularInf, + ) -> RegInfProcessedStats: """Download single OTA image file.""" cur_stat = RegInfProcessedStats(op=RegProcessOperation.DOWNLOAD_REMOTE_COPY) @@ -228,7 +248,7 @@ def _update_standby_slot(self): # --- init standby_slot creator, calculate delta --- # logger.info("start to calculate and prepare delta...") - self.update_phase = wrapper.UpdatePhase.CALCULATING_DELTA + self.update_phase = api_types.UpdatePhase.CALCULATING_DELTA self._standby_slot_creator = self._create_standby_cls( ota_metadata=self._otameta, boot_dir=str(self._boot_controller.get_standby_boot_dir()), @@ -254,7 +274,7 @@ def _update_standby_slot(self): "start to download needed files..." f"total_download_files_size={_delta_bundle.total_download_files_size:,}bytes" ) - self.update_phase = wrapper.UpdatePhase.DOWNLOADING_OTA_FILES + self.update_phase = api_types.UpdatePhase.DOWNLOADING_OTA_FILES try: self._download_files(_delta_bundle.get_download_list()) except downloader.DownloadFailedSpaceNotEnough: @@ -280,7 +300,7 @@ def _update_standby_slot(self): # ------ in_update ------ # logger.info("start to apply changes to standby slot...") - self.update_phase = wrapper.UpdatePhase.APPLYING_UPDATE + self.update_phase = api_types.UpdatePhase.APPLYING_UPDATE try: self._standby_slot_creator.create_standby_slot() except Exception as e: @@ -304,7 +324,7 @@ def _process_persistents(self): ) for _perinf in self._otameta.iter_metafile( - ota_metadata.MetafilesV1.PERSISTENT_FNAME + ota_metadata_parser.MetafilesV1.PERSISTENT_FNAME ): _per_fpath = Path(_perinf.path) @@ -353,7 +373,7 @@ def _execute_update( self._update_stats_collector.start() # ------ init, processing metadata ------ # - self.update_phase = wrapper.UpdatePhase.PROCESSING_METADATA + self.update_phase = api_types.UpdatePhase.PROCESSING_METADATA # parse url_base # unconditionally regulate the url_base _url_base = urlparse(raw_url_base) @@ -388,9 +408,12 @@ def _execute_update( # process metadata.jwt and ota metafiles logger.debug("process metadata.jwt...") try: - self._otameta = ota_metadata.OTAMetadata( + self._otameta = ota_metadata_parser.OTAMetadata( url_base=self._url_base, downloader=self._downloader, + run_dir=Path(cfg.RUN_DIR), + certs_dir=Path(cfg.CERTS_DIR), + download_max_idle_time=cfg.DOWNLOAD_GROUP_INACTIVE_TIMEOUT, ) self.total_files_num = self._otameta.total_files_num self.total_files_size_uncompressed = ( @@ -406,13 +429,13 @@ def _execute_update( _err_msg = f"downloader: failed to save ota metafiles: {e!r}" logger.error(_err_msg) raise ota_errors.OTAErrorUnrecoverable(_err_msg, module=__name__) from e - except ota_metadata.MetadataJWTVerificationFailed as e: + except ota_metadata_parser.MetadataJWTVerificationFailed as e: _err_msg = f"failed to verify metadata.jwt: {e!r}" logger.error(_err_msg) raise ota_errors.MetadataJWTVerficationFailed( _err_msg, module=__name__ ) from e - except ota_metadata.MetadataJWTPayloadInvalid as e: + except ota_metadata_parser.MetadataJWTPayloadInvalid as e: _err_msg = f"metadata.jwt is invalid: {e!r}" logger.error(_err_msg) raise ota_errors.MetadataJWTInvalid(_err_msg, module=__name__) from e @@ -434,7 +457,7 @@ def _execute_update( # ------ post update ------ # logger.info("enter post update phase...") - self.update_phase = wrapper.UpdatePhase.PROCESSING_POSTUPDATE + self.update_phase = api_types.UpdatePhase.PROCESSING_POSTUPDATE # NOTE(20240219): move persist file handling here self._process_persistents() @@ -448,11 +471,11 @@ def _execute_update( # API def shutdown(self): - self.update_phase = wrapper.UpdatePhase.INITIALIZING + self.update_phase = api_types.UpdatePhase.INITIALIZING self._downloader.shutdown() self._update_stats_collector.stop() - def get_update_status(self) -> wrapper.UpdateStatus: + def get_update_status(self) -> api_types.UpdateStatus: """ Returns: A tuple contains the version and the update_progress. @@ -472,13 +495,13 @@ def get_update_status(self) -> wrapper.UpdateStatus: update_progress.total_remove_files_num = self.total_remove_files_num # downloading stats update_progress.downloaded_bytes = self._downloader.downloaded_bytes - update_progress.downloading_elapsed_time = wrapper.Duration( + update_progress.downloading_elapsed_time = api_types.Duration( seconds=self._downloader.downloader_active_seconds ) # update other information update_progress.phase = self.update_phase - update_progress.total_elapsed_time = wrapper.Duration.from_nanoseconds( + update_progress.total_elapsed_time = api_types.Duration.from_nanoseconds( time.time_ns() - self.update_start_time ) return update_progress @@ -562,7 +585,7 @@ def __init__( self._rollback_executor: _OTARollbacker = None # type: ignore # err record - self.last_failure_type = wrapper.FailureType.NO_FAILURE + self.last_failure_type = api_types.FailureType.NO_FAILURE self.last_failure_reason = "" self.last_failure_traceback = "" except Exception as e: @@ -570,7 +593,7 @@ def __init__( logger.error(_err_msg) raise ota_errors.OTAClientStartupFailed(_err_msg, module=__name__) from e - def _on_failure(self, exc: ota_errors.OTAError, ota_status: wrapper.StatusOta): + def _on_failure(self, exc: ota_errors.OTAError, ota_status: api_types.StatusOta): self.live_ota_status.set_ota_status(ota_status) try: self.last_failure_type = exc.failure_type @@ -598,15 +621,15 @@ def update(self, version: str, url_base: str, cookies_json: str): ) # reset failure information on handling new update request - self.last_failure_type = wrapper.FailureType.NO_FAILURE + self.last_failure_type = api_types.FailureType.NO_FAILURE self.last_failure_reason = "" self.last_failure_traceback = "" # enter update - self.live_ota_status.set_ota_status(wrapper.StatusOta.UPDATING) + self.live_ota_status.set_ota_status(api_types.StatusOta.UPDATING) self._update_executor.execute(version, url_base, cookies_json) except ota_errors.OTAError as e: - self._on_failure(e, wrapper.StatusOta.FAILURE) + self._on_failure(e, api_types.StatusOta.FAILURE) finally: self._update_executor = None # type: ignore gc.collect() # trigger a forced gc @@ -625,16 +648,16 @@ def rollback(self): ) # clear failure information on handling new rollback request - self.last_failure_type = wrapper.FailureType.NO_FAILURE + self.last_failure_type = api_types.FailureType.NO_FAILURE self.last_failure_reason = "" self.last_failure_traceback = "" # entering rollback - self.live_ota_status.set_ota_status(wrapper.StatusOta.ROLLBACKING) + self.live_ota_status.set_ota_status(api_types.StatusOta.ROLLBACKING) self._rollback_executor.execute() # silently ignore overlapping request except ota_errors.OTAError as e: - self._on_failure(e, wrapper.StatusOta.ROLLBACK_FAILURE) + self._on_failure(e, api_types.StatusOta.ROLLBACK_FAILURE) finally: self._rollback_executor = None # type: ignore self._lock.release() @@ -643,9 +666,9 @@ def rollback(self): "ignore incoming rollback request as local update/rollback is ongoing" ) - def status(self) -> wrapper.StatusResponseEcuV2: + def status(self) -> api_types.StatusResponseEcuV2: live_ota_status = self.live_ota_status.get_ota_status() - status_report = wrapper.StatusResponseEcuV2( + status_report = api_types.StatusResponseEcuV2( ecu_id=self.my_ecu_id, firmware_version=self.current_version, otaclient_version=self.OTACLIENT_VERSION, @@ -654,7 +677,7 @@ def status(self) -> wrapper.StatusResponseEcuV2: failure_reason=self.last_failure_reason, failure_traceback=self.last_failure_traceback, ) - if live_ota_status == wrapper.StatusOta.UPDATING and self._update_executor: + if live_ota_status == api_types.StatusOta.UPDATING and self._update_executor: status_report.update_status = self._update_executor.get_update_status() return status_report @@ -671,15 +694,15 @@ def __init__( self.ecu_id = ecu_info.ecu_id self.otaclient_version = otaclient_version self.local_used_proxy_url = proxy - self.last_operation: Optional[wrapper.StatusOta] = None + self.last_operation: Optional[api_types.StatusOta] = None # default boot startup failure if boot_controller/otaclient_core crashed without # raising specific error - self._otaclient_startup_failed_status = wrapper.StatusResponseEcuV2( + self._otaclient_startup_failed_status = api_types.StatusResponseEcuV2( ecu_id=ecu_info.ecu_id, otaclient_version=otaclient_version, - ota_status=wrapper.StatusOta.FAILURE, - failure_type=wrapper.FailureType.UNRECOVERABLE, + ota_status=api_types.StatusOta.FAILURE, + failure_type=api_types.FailureType.UNRECOVERABLE, failure_reason="unspecific error", ) self._update_rollback_lock = asyncio.Lock() @@ -703,11 +726,11 @@ def __init__( logger.error( e.get_error_report(title=f"boot controller startup failed: {e!r}") ) - self._otaclient_startup_failed_status = wrapper.StatusResponseEcuV2( + self._otaclient_startup_failed_status = api_types.StatusResponseEcuV2( ecu_id=ecu_info.ecu_id, otaclient_version=otaclient_version, - ota_status=wrapper.StatusOta.FAILURE, - failure_type=wrapper.FailureType.UNRECOVERABLE, + ota_status=api_types.StatusOta.FAILURE, + failure_type=api_types.FailureType.UNRECOVERABLE, failure_reason=e.get_failure_reason(), ) @@ -730,11 +753,11 @@ def __init__( logger.error( e.get_error_report(title=f"otaclient core startup failed: {e!r}") ) - self._otaclient_startup_failed_status = wrapper.StatusResponseEcuV2( + self._otaclient_startup_failed_status = api_types.StatusResponseEcuV2( ecu_id=ecu_info.ecu_id, otaclient_version=otaclient_version, - ota_status=wrapper.StatusOta.FAILURE, - failure_type=wrapper.FailureType.UNRECOVERABLE, + ota_status=api_types.StatusOta.FAILURE, + failure_type=api_types.FailureType.UNRECOVERABLE, failure_reason=e.get_failure_reason(), ) @@ -749,12 +772,12 @@ def is_busy(self) -> bool: return self._update_rollback_lock.locked() async def dispatch_update( - self, request: wrapper.UpdateRequestEcu - ) -> wrapper.UpdateResponseEcu: + self, request: api_types.UpdateRequestEcu + ) -> api_types.UpdateResponseEcu: # prevent update operation if otaclient is not started if self._otaclient_inst is None: - return wrapper.UpdateResponseEcu( - ecu_id=self.ecu_id, result=wrapper.FailureType.UNRECOVERABLE + return api_types.UpdateResponseEcu( + ecu_id=self.ecu_id, result=api_types.FailureType.UNRECOVERABLE ) # check and acquire lock @@ -762,13 +785,13 @@ async def dispatch_update( logger.warning( f"ongoing operation: {self.last_operation=}, ignore incoming {request=}" ) - return wrapper.UpdateResponseEcu( - ecu_id=self.ecu_id, result=wrapper.FailureType.RECOVERABLE + return api_types.UpdateResponseEcu( + ecu_id=self.ecu_id, result=api_types.FailureType.RECOVERABLE ) # immediately take the lock if not locked await self._update_rollback_lock.acquire() - self.last_operation = wrapper.StatusOta.UPDATING + self.last_operation = api_types.StatusOta.UPDATING async def _update_task(): if self._otaclient_inst is None: @@ -790,17 +813,17 @@ async def _update_task(): # dispatch update to background asyncio.create_task(_update_task()) - return wrapper.UpdateResponseEcu( - ecu_id=self.ecu_id, result=wrapper.FailureType.NO_FAILURE + return api_types.UpdateResponseEcu( + ecu_id=self.ecu_id, result=api_types.FailureType.NO_FAILURE ) async def dispatch_rollback( - self, request: wrapper.RollbackRequestEcu - ) -> wrapper.RollbackResponseEcu: + self, request: api_types.RollbackRequestEcu + ) -> api_types.RollbackResponseEcu: # prevent rollback operation if otaclient is not started if self._otaclient_inst is None: - return wrapper.RollbackResponseEcu( - ecu_id=self.ecu_id, result=wrapper.FailureType.UNRECOVERABLE + return api_types.RollbackResponseEcu( + ecu_id=self.ecu_id, result=api_types.FailureType.UNRECOVERABLE ) # check and acquire lock @@ -808,13 +831,13 @@ async def dispatch_rollback( logger.warning( f"ongoing operation: {self.last_operation=}, ignore incoming {request=}" ) - return wrapper.RollbackResponseEcu( - ecu_id=self.ecu_id, result=wrapper.FailureType.RECOVERABLE + return api_types.RollbackResponseEcu( + ecu_id=self.ecu_id, result=api_types.FailureType.RECOVERABLE ) # immediately take the lock if not locked await self._update_rollback_lock.acquire() - self.last_operation = wrapper.StatusOta.ROLLBACKING + self.last_operation = api_types.StatusOta.ROLLBACKING async def _rollback_task(): if self._otaclient_inst is None: @@ -829,11 +852,11 @@ async def _rollback_task(): # dispatch to background asyncio.create_task(_rollback_task()) - return wrapper.RollbackResponseEcu( - ecu_id=self.ecu_id, result=wrapper.FailureType.NO_FAILURE + return api_types.RollbackResponseEcu( + ecu_id=self.ecu_id, result=api_types.FailureType.NO_FAILURE ) - async def get_status(self) -> wrapper.StatusResponseEcuV2: + async def get_status(self) -> api_types.StatusResponseEcuV2: # otaclient is not started due to boot control startup failed if self._otaclient_inst is None: return self._otaclient_startup_failed_status diff --git a/src/otaclient/app/ota_client_service.py b/src/otaclient/app/ota_client_service.py deleted file mode 100644 index d9681817c..000000000 --- a/src/otaclient/app/ota_client_service.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2022 TIER IV, INC. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import grpc.aio - -from .configs import ecu_info, server_cfg -from .ota_client_stub import OTAClientServiceStub -from .proto import v2, v2_grpc, wrapper - - -class OtaClientServiceV2(v2_grpc.OtaClientServiceServicer): - def __init__(self, ota_client_stub: OTAClientServiceStub): - self._stub = ota_client_stub - - async def Update(self, request: v2.UpdateRequest, context) -> v2.UpdateResponse: - response = await self._stub.update(wrapper.UpdateRequest.convert(request)) - return response.export_pb() - - async def Rollback( - self, request: v2.RollbackRequest, context - ) -> v2.RollbackResponse: - response = await self._stub.rollback(wrapper.RollbackRequest.convert(request)) - return response.export_pb() - - async def Status(self, request: v2.StatusRequest, context) -> v2.StatusResponse: - response = await self._stub.status(wrapper.StatusRequest.convert(request)) - return response.export_pb() - - -def create_otaclient_grpc_server(): - service_stub = OTAClientServiceStub() - ota_client_service_v2 = OtaClientServiceV2(service_stub) - - server = grpc.aio.server() - v2_grpc.add_OtaClientServiceServicer_to_server( - server=server, servicer=ota_client_service_v2 - ) - server.add_insecure_port(f"{ecu_info.ip_addr}:{server_cfg.SERVER_PORT}") - return server - - -async def launch_otaclient_grpc_server(): - server = create_otaclient_grpc_server() - await server.start() - await server.wait_for_termination() diff --git a/src/otaclient/app/ota_client_stub.py b/src/otaclient/app/ota_client_stub.py index e2cf6c3ef..08bcd0ca9 100644 --- a/src/otaclient/app/ota_client_stub.py +++ b/src/otaclient/app/ota_client_stub.py @@ -31,16 +31,16 @@ from ota_proxy import OTAProxyContextProto from ota_proxy import config as local_otaproxy_cfg from ota_proxy import subprocess_otaproxy_launcher +from otaclient.app import log_setting from otaclient.configs.ecu_info import ECUContact +from otaclient_api.v2 import types as api_types +from otaclient_api.v2.api_caller import ECUNoResponse, OTAClientCall +from otaclient_common.common import ensure_otaproxy_start -from . import log_setting from .boot_control._common import CMDHelperFuncs -from .common import ensure_otaproxy_start from .configs import config as cfg from .configs import ecu_info, proxy_info, server_cfg from .ota_client import OTAClientControlFlags, OTAServicer -from .ota_client_call import ECUNoResponse, OtaClientCall -from .proto import wrapper logger = logging.getLogger(__name__) @@ -348,8 +348,8 @@ def __init__(self) -> None: ecu_info.get_available_ecu_ids() ) - self._all_ecus_status_v2: Dict[str, wrapper.StatusResponseEcuV2] = {} - self._all_ecus_status_v1: Dict[str, wrapper.StatusResponseEcu] = {} + self._all_ecus_status_v2: Dict[str, api_types.StatusResponseEcuV2] = {} + self._all_ecus_status_v1: Dict[str, api_types.StatusResponseEcu] = {} self._all_ecus_last_contact_timestamp: Dict[str, int] = {} # overall ECU status report @@ -511,7 +511,7 @@ async def _loop_updating_properties(self): # API - async def update_from_child_ecu(self, status_resp: wrapper.StatusResponse): + async def update_from_child_ecu(self, status_resp: api_types.StatusResponse): """Update the ECU status storage with child ECU's status report(StatusResponse).""" async with self._writer_lock: self.storage_last_updated_timestamp = cur_timestamp = int(time.time()) @@ -537,7 +537,7 @@ async def update_from_child_ecu(self, status_resp: wrapper.StatusResponse): self._all_ecus_last_contact_timestamp[ecu_id] = cur_timestamp self._all_ecus_status_v2.pop(ecu_id, None) - async def update_from_local_ecu(self, ecu_status: wrapper.StatusResponseEcuV2): + async def update_from_local_ecu(self, ecu_status: api_types.StatusResponseEcuV2): """Update ECU status storage with local ECU's status report(StatusResponseEcuV2).""" async with self._writer_lock: self.storage_last_updated_timestamp = cur_timestamp = int(time.time()) @@ -615,7 +615,7 @@ async def _waiter(): return _waiter - async def export(self) -> wrapper.StatusResponse: + async def export(self) -> api_types.StatusResponse: """Export the contents of this storage to an instance of StatusResponse. NOTE: wrapper.StatusResponse's add_ecu method already takes care of @@ -625,7 +625,7 @@ async def export(self) -> wrapper.StatusResponse: entry in status API response, simulate this behavior by skipping disconnected ECU's status report entry. """ - res = wrapper.StatusResponse() + res = api_types.StatusResponse() async with self._writer_lock: res.available_ecu_ids.extend(self._available_ecu_ids) @@ -678,12 +678,12 @@ async def _polling_direct_subecu_status(self, ecu_contact: ECUContact): """Task entry for loop polling one subECU's status.""" while not self._debug_ecu_status_polling_shutdown_event.is_set(): try: - _ecu_resp = await OtaClientCall.status_call( + _ecu_resp = await OTAClientCall.status_call( ecu_contact.ecu_id, str(ecu_contact.ip_addr), ecu_contact.port, timeout=server_cfg.QUERYING_SUBECU_STATUS_TIMEOUT, - request=wrapper.StatusRequest(), + request=api_types.StatusRequest(), ) await self._ecu_status_storage.update_from_child_ecu(_ecu_resp) except ECUNoResponse as e: @@ -810,10 +810,12 @@ async def _otaclient_control_flags_managing(self): # API stub - async def update(self, request: wrapper.UpdateRequest) -> wrapper.UpdateResponse: + async def update( + self, request: api_types.UpdateRequest + ) -> api_types.UpdateResponse: logger.info(f"receive update request: {request}") update_acked_ecus = set() - response = wrapper.UpdateResponse() + response = api_types.UpdateResponse() # first: dispatch update request to all directly connected subECUs tasks: Dict[asyncio.Task, ECUContact] = {} @@ -821,7 +823,7 @@ async def update(self, request: wrapper.UpdateRequest) -> wrapper.UpdateResponse if not request.if_contains_ecu(ecu_contact.ecu_id): continue _task = asyncio.create_task( - OtaClientCall.update_call( + OTAClientCall.update_call( ecu_contact.ecu_id, str(ecu_contact.ip_addr), ecu_contact.port, @@ -834,7 +836,7 @@ async def update(self, request: wrapper.UpdateRequest) -> wrapper.UpdateResponse done, _ = await asyncio.wait(tasks) for _task in done: try: - _ecu_resp: wrapper.UpdateResponse = _task.result() + _ecu_resp: api_types.UpdateResponse = _task.result() update_acked_ecus.update(_ecu_resp.ecus_acked_update) response.merge_from(_ecu_resp) except ECUNoResponse as e: @@ -847,9 +849,9 @@ async def update(self, request: wrapper.UpdateRequest) -> wrapper.UpdateResponse # response with RECOVERABLE OTA error for unresponsive # ECU. response.add_ecu( - wrapper.UpdateResponseEcu( + api_types.UpdateResponseEcu( ecu_id=_ecu_contact.ecu_id, - result=wrapper.FailureType.RECOVERABLE, + result=api_types.FailureType.RECOVERABLE, ) ) tasks.clear() @@ -858,7 +860,7 @@ async def update(self, request: wrapper.UpdateRequest) -> wrapper.UpdateResponse if update_req_ecu := request.find_ecu(self.my_ecu_id): _resp_ecu = await self._otaclient_wrapper.dispatch_update(update_req_ecu) # local otaclient accepts the update request - if _resp_ecu.result == wrapper.FailureType.NO_FAILURE: + if _resp_ecu.result == api_types.FailureType.NO_FAILURE: update_acked_ecus.add(self.my_ecu_id) response.add_ecu(_resp_ecu) @@ -874,10 +876,10 @@ async def update(self, request: wrapper.UpdateRequest) -> wrapper.UpdateResponse return response async def rollback( - self, request: wrapper.RollbackRequest - ) -> wrapper.RollbackResponse: + self, request: api_types.RollbackRequest + ) -> api_types.RollbackResponse: logger.info(f"receive rollback request: {request}") - response = wrapper.RollbackResponse() + response = api_types.RollbackResponse() # first: dispatch rollback request to all directly connected subECUs tasks: Dict[asyncio.Task, ECUContact] = {} @@ -885,7 +887,7 @@ async def rollback( if not request.if_contains_ecu(ecu_contact.ecu_id): continue _task = asyncio.create_task( - OtaClientCall.rollback_call( + OTAClientCall.rollback_call( ecu_contact.ecu_id, str(ecu_contact.ip_addr), ecu_contact.port, @@ -898,7 +900,7 @@ async def rollback( done, _ = await asyncio.wait(tasks) for _task in done: try: - _ecu_resp: wrapper.RollbackResponse = _task.result() + _ecu_resp: api_types.RollbackResponse = _task.result() response.merge_from(_ecu_resp) except ECUNoResponse as e: _ecu_contact = tasks[_task] @@ -910,9 +912,9 @@ async def rollback( # response with RECOVERABLE OTA error for unresponsive # ECU. response.add_ecu( - wrapper.RollbackResponseEcu( + api_types.RollbackResponseEcu( ecu_id=_ecu_contact.ecu_id, - result=wrapper.FailureType.RECOVERABLE, + result=api_types.FailureType.RECOVERABLE, ) ) tasks.clear() @@ -925,5 +927,5 @@ async def rollback( return response - async def status(self, _=None) -> wrapper.StatusResponse: + async def status(self, _=None) -> api_types.StatusResponse: return await self._ecu_status_storage.export() diff --git a/src/otaclient/app/ota_status.py b/src/otaclient/app/ota_status.py deleted file mode 100644 index 53ff85c8f..000000000 --- a/src/otaclient/app/ota_status.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2022 TIER IV, INC. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import logging - -from .proto import wrapper - -logger = logging.getLogger(__name__) - - -class LiveOTAStatus: - def __init__(self, ota_status: wrapper.StatusOta) -> None: - self.live_ota_status = ota_status - - def get_ota_status(self) -> wrapper.StatusOta: - return self.live_ota_status - - def set_ota_status(self, _status: wrapper.StatusOta): - self.live_ota_status = _status - - def request_update(self) -> bool: - return self.live_ota_status in [ - wrapper.StatusOta.INITIALIZED, - wrapper.StatusOta.SUCCESS, - wrapper.StatusOta.FAILURE, - wrapper.StatusOta.ROLLBACK_FAILURE, - ] - - def request_rollback(self) -> bool: - return self.live_ota_status in [ - wrapper.StatusOta.SUCCESS, - wrapper.StatusOta.ROLLBACK_FAILURE, - ] diff --git a/src/otaclient/app/proto/__init__.py b/src/otaclient/app/proto/__init__.py deleted file mode 100644 index 82b5169a5..000000000 --- a/src/otaclient/app/proto/__init__.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2022 TIER IV, INC. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -"""Packed compiled protobuf files for otaclient.""" -import importlib.util -import sys -from pathlib import Path -from types import ModuleType -from typing import Tuple - -_PROTO_DIR = Path(__file__).parent -# NOTE: order matters here! v2_pb2_grpc depends on v2_pb2 -_FILES_TO_LOAD = [ - _PROTO_DIR / _fname - for _fname in [ - "otaclient_v2_pb2.py", - "otaclient_v2_pb2_grpc.py", - "ota_metafiles_pb2.py", - ] -] - - -def _import_from_file(path: Path) -> Tuple[str, ModuleType]: - if not path.is_file(): - raise ValueError(f"{path} is not a valid module file") - try: - _module_name = path.stem - _spec = importlib.util.spec_from_file_location(_module_name, path) - _module = importlib.util.module_from_spec(_spec) # type: ignore - _spec.loader.exec_module(_module) # type: ignore - return _module_name, _module - except Exception: - raise ImportError(f"failed to import module from {path=}.") - - -def _import_proto(*module_fpaths: Path): - """Import the protobuf modules to path under this folder. - - NOTE: compiled protobuf files under proto folder will be - imported as modules to the global namespace. - """ - for _fpath in module_fpaths: - _module_name, _module = _import_from_file(_fpath) # noqa: F821 - # add the module to the global module namespace - sys.modules[_module_name] = _module - - -_import_proto(*_FILES_TO_LOAD) -del _import_proto, _import_from_file - -import ota_metafiles_pb2 as ota_metafiles # noqa: E402 -import otaclient_v2_pb2 as v2 # noqa: E402 -import otaclient_v2_pb2_grpc as v2_grpc # noqa: E402 - -from . import streamer # noqa: E402 -from . import wrapper # noqa: E402 - -__all__ = ["v2", "v2_grpc", "ota_metafiles", "wrapper", "streamer"] diff --git a/src/otaclient/app/proto/wrapper.py b/src/otaclient/app/proto/wrapper.py deleted file mode 100644 index d7e0d8eb7..000000000 --- a/src/otaclient/app/proto/wrapper.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2022 TIER IV, INC. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Modules for registering wrapped compiled protobuf types.""" - - -from ._common import * # noqa: F403, F401 -from ._ota_metafiles_wrapper import * # noqa: F403, F401 -from ._otaclient_v2_pb2_wrapper import * # noqa: F403, F401 diff --git a/src/otaclient/app/update_stats.py b/src/otaclient/app/update_stats.py index c01ee6cda..c4b68fecb 100644 --- a/src/otaclient/app/update_stats.py +++ b/src/otaclient/app/update_stats.py @@ -13,6 +13,8 @@ # limitations under the License. +from __future__ import annotations + import logging import time from contextlib import contextmanager @@ -20,10 +22,11 @@ from enum import Enum from queue import Empty, Queue from threading import Event, Lock, Thread -from typing import Generator, List +from typing import Generator + +from otaclient_api.v2.types import UpdateStatus from .configs import config as cfg -from .proto.wrapper import UpdateStatus logger = logging.getLogger(__name__) @@ -60,7 +63,7 @@ def __init__(self) -> None: self.collect_interval = cfg.STATS_COLLECT_INTERVAL self.terminated = Event() self._que: Queue[RegInfProcessedStats] = Queue() - self._staging: List[RegInfProcessedStats] = [] + self._staging: list[RegInfProcessedStats] = [] self._collector_thread = None @contextmanager @@ -111,7 +114,7 @@ def get_snapshot(self) -> UpdateStatus: report_download_ota_files = _report report_prepare_local_copy = _report - def report_apply_delta(self, stats_list: List[RegInfProcessedStats]): + def report_apply_delta(self, stats_list: list[RegInfProcessedStats]): """Stats report for APPLY_DELTA operation. Params: diff --git a/src/otaclient/configs/ecu_info.py b/src/otaclient/configs/ecu_info.py index f6fa332bc..08b850bb7 100644 --- a/src/otaclient/configs/ecu_info.py +++ b/src/otaclient/configs/ecu_info.py @@ -26,8 +26,8 @@ from pydantic import AfterValidator, BeforeValidator, Field, IPvAnyAddress from typing_extensions import Annotated -from otaclient._utils.typing import NetworkPort, StrOrPath, gen_strenum_validator from otaclient.configs._common import BaseFixedConfig +from otaclient_common.typing import NetworkPort, StrOrPath, gen_strenum_validator logger = logging.getLogger(__name__) diff --git a/src/otaclient/configs/proxy_info.py b/src/otaclient/configs/proxy_info.py index 3ed4d4c3e..db16a1844 100644 --- a/src/otaclient/configs/proxy_info.py +++ b/src/otaclient/configs/proxy_info.py @@ -26,8 +26,8 @@ from pydantic import AliasChoices, AnyHttpUrl, Field, IPvAnyAddress from pydantic_core import Url -from otaclient._utils.typing import NetworkPort, StrOrPath from otaclient.configs._common import BaseFixedConfig +from otaclient_common.typing import NetworkPort, StrOrPath logger = logging.getLogger(__name__) diff --git a/src/otaclient_api/v2/README.md b/src/otaclient_api/v2/README.md new file mode 100644 index 000000000..7fb5b1cfd --- /dev/null +++ b/src/otaclient_api/v2/README.md @@ -0,0 +1,3 @@ +# OTAClient API version 2 + +Package for holding the protobuf python pb2 generated files and related wrappers and libs for OTAClient API version 2. diff --git a/src/otaclient_api/v2/__init__.py b/src/otaclient_api/v2/__init__.py new file mode 100644 index 000000000..ff3cd5252 --- /dev/null +++ b/src/otaclient_api/v2/__init__.py @@ -0,0 +1,48 @@ +# Copyright 2022 TIER IV, INC. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""OTAClient API, version 2.""" + + +from __future__ import annotations + +import sys +from pathlib import Path + +from otaclient_common import import_from_file + +# ------ dynamically import pb2 generated code ------ # + +_PROTO_DIR = Path(__file__).parent +# NOTE: order matters here! pb2_grpc depends on pb2 +_FILES_TO_LOAD = [ + _PROTO_DIR / "otaclient_v2_pb2.py", + _PROTO_DIR / "otaclient_v2_pb2_grpc.py", +] +PACKAGE_PREFIX = ".".join(__name__.split(".")[:-1]) + + +def _import_pb2_proto(*module_fpaths: Path): + """Import the protobuf modules to path under this folder. + + NOTE: compiled protobuf files under proto folder will be + imported as modules to the global namespace. + """ + for _fpath in module_fpaths: + _module_name, _module = import_from_file(_fpath) + sys.modules[f"{PACKAGE_PREFIX}.{_module_name}"] = _module + sys.modules[_module_name] = _module + + +_import_pb2_proto(*_FILES_TO_LOAD) +del _import_pb2_proto diff --git a/src/otaclient/app/ota_client_call.py b/src/otaclient_api/v2/api_caller.py similarity index 72% rename from src/otaclient/app/ota_client_call.py rename to src/otaclient_api/v2/api_caller.py index a655232e4..72bc6d38f 100644 --- a/src/otaclient/app/ota_client_call.py +++ b/src/otaclient_api/v2/api_caller.py @@ -11,34 +11,37 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +"""OTAClient API caller implementation.""" +from __future__ import annotations + import grpc.aio -from .configs import server_cfg -from .proto import v2_grpc, wrapper +from otaclient_api.v2 import otaclient_v2_pb2_grpc as pb2_grpc +from otaclient_api.v2 import types class ECUNoResponse(Exception): """Raised when ECU cannot response to request on-time.""" -class OtaClientCall: +class OTAClientCall: @staticmethod async def status_call( ecu_id: str, ecu_ipaddr: str, - ecu_port: int = server_cfg.SERVER_PORT, + ecu_port: int, *, - request: wrapper.StatusRequest, + request: types.StatusRequest, timeout=None, - ) -> wrapper.StatusResponse: + ) -> types.StatusResponse: try: ecu_addr = f"{ecu_ipaddr}:{ecu_port}" async with grpc.aio.insecure_channel(ecu_addr) as channel: - stub = v2_grpc.OtaClientServiceStub(channel) + stub = pb2_grpc.OtaClientServiceStub(channel) resp = await stub.Status(request.export_pb(), timeout=timeout) - return wrapper.StatusResponse.convert(resp) + return types.StatusResponse.convert(resp) except Exception as e: _msg = f"{ecu_id=} failed to respond to status request on-time: {e!r}" raise ECUNoResponse(_msg) @@ -47,17 +50,17 @@ async def status_call( async def update_call( ecu_id: str, ecu_ipaddr: str, - ecu_port: int = server_cfg.SERVER_PORT, + ecu_port: int, *, - request: wrapper.UpdateRequest, + request: types.UpdateRequest, timeout=None, - ) -> wrapper.UpdateResponse: + ) -> types.UpdateResponse: try: ecu_addr = f"{ecu_ipaddr}:{ecu_port}" async with grpc.aio.insecure_channel(ecu_addr) as channel: - stub = v2_grpc.OtaClientServiceStub(channel) + stub = pb2_grpc.OtaClientServiceStub(channel) resp = await stub.Update(request.export_pb(), timeout=timeout) - return wrapper.UpdateResponse.convert(resp) + return types.UpdateResponse.convert(resp) except Exception as e: _msg = f"{ecu_id=} failed to respond to update request on-time: {e!r}" raise ECUNoResponse(_msg) @@ -66,17 +69,17 @@ async def update_call( async def rollback_call( ecu_id: str, ecu_ipaddr: str, - ecu_port: int = server_cfg.SERVER_PORT, + ecu_port: int, *, - request: wrapper.RollbackRequest, + request: types.RollbackRequest, timeout=None, - ) -> wrapper.RollbackResponse: + ) -> types.RollbackResponse: try: ecu_addr = f"{ecu_ipaddr}:{ecu_port}" async with grpc.aio.insecure_channel(ecu_addr) as channel: - stub = v2_grpc.OtaClientServiceStub(channel) + stub = pb2_grpc.OtaClientServiceStub(channel) resp = await stub.Rollback(request.export_pb(), timeout=timeout) - return wrapper.RollbackResponse.convert(resp) + return types.RollbackResponse.convert(resp) except Exception as e: _msg = f"{ecu_id=} failed to respond to rollback request on-time: {e!r}" raise ECUNoResponse(_msg) diff --git a/src/otaclient_api/v2/api_stub.py b/src/otaclient_api/v2/api_stub.py new file mode 100644 index 000000000..167eb8f49 --- /dev/null +++ b/src/otaclient_api/v2/api_stub.py @@ -0,0 +1,41 @@ +# Copyright 2022 TIER IV, INC. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import annotations + +from typing import Any + +from otaclient_api.v2 import otaclient_v2_pb2 as pb2 +from otaclient_api.v2 import otaclient_v2_pb2_grpc as pb2_grpc +from otaclient_api.v2 import types + + +class OtaClientServiceV2(pb2_grpc.OtaClientServiceServicer): + def __init__(self, ota_client_stub: Any): + self._stub = ota_client_stub + + async def Update(self, request: pb2.UpdateRequest, context) -> pb2.UpdateResponse: + response = await self._stub.update(types.UpdateRequest.convert(request)) + return response.export_pb() + + async def Rollback( + self, request: pb2.RollbackRequest, context + ) -> pb2.RollbackResponse: + response = await self._stub.rollback(types.RollbackRequest.convert(request)) + return response.export_pb() + + async def Status(self, request: pb2.StatusRequest, context) -> pb2.StatusResponse: + response = await self._stub.status(types.StatusRequest.convert(request)) + return response.export_pb() diff --git a/src/otaclient/app/proto/otaclient_v2_pb2.py b/src/otaclient_api/v2/otaclient_v2_pb2.py similarity index 100% rename from src/otaclient/app/proto/otaclient_v2_pb2.py rename to src/otaclient_api/v2/otaclient_v2_pb2.py diff --git a/src/otaclient/app/proto/otaclient_v2_pb2.pyi b/src/otaclient_api/v2/otaclient_v2_pb2.pyi similarity index 100% rename from src/otaclient/app/proto/otaclient_v2_pb2.pyi rename to src/otaclient_api/v2/otaclient_v2_pb2.pyi diff --git a/src/otaclient/app/proto/otaclient_v2_pb2_grpc.py b/src/otaclient_api/v2/otaclient_v2_pb2_grpc.py similarity index 100% rename from src/otaclient/app/proto/otaclient_v2_pb2_grpc.py rename to src/otaclient_api/v2/otaclient_v2_pb2_grpc.py diff --git a/src/otaclient/app/proto/_otaclient_v2_pb2_wrapper.py b/src/otaclient_api/v2/types.py similarity index 84% rename from src/otaclient/app/proto/_otaclient_v2_pb2_wrapper.py rename to src/otaclient_api/v2/types.py index cc6409d08..f784a568c 100644 --- a/src/otaclient/app/proto/_otaclient_v2_pb2_wrapper.py +++ b/src/otaclient_api/v2/types.py @@ -30,10 +30,10 @@ from typing import TypeVar as _TypeVar from typing import Union as _Union -import otaclient_v2_pb2 as _v2 from typing_extensions import Self -from ._common import ( +from otaclient_api.v2 import otaclient_v2_pb2 as pb2 +from otaclient_common.proto_wrapper import ( Duration, EnumWrapper, MessageWrapper, @@ -123,41 +123,41 @@ def requires_network(self) -> bool: class FailureType(EnumWrapper): - NO_FAILURE = _v2.NO_FAILURE - RECOVERABLE = _v2.RECOVERABLE - UNRECOVERABLE = _v2.UNRECOVERABLE + NO_FAILURE = pb2.NO_FAILURE + RECOVERABLE = pb2.RECOVERABLE + UNRECOVERABLE = pb2.UNRECOVERABLE def to_str(self) -> str: return f"{self.value:0>1}" class StatusOta(EnumWrapper): - INITIALIZED = _v2.INITIALIZED - SUCCESS = _v2.SUCCESS - FAILURE = _v2.FAILURE - UPDATING = _v2.UPDATING - ROLLBACKING = _v2.ROLLBACKING - ROLLBACK_FAILURE = _v2.ROLLBACK_FAILURE + INITIALIZED = pb2.INITIALIZED + SUCCESS = pb2.SUCCESS + FAILURE = pb2.FAILURE + UPDATING = pb2.UPDATING + ROLLBACKING = pb2.ROLLBACKING + ROLLBACK_FAILURE = pb2.ROLLBACK_FAILURE class StatusProgressPhase(EnumWrapper): - INITIAL = _v2.INITIAL - METADATA = _v2.METADATA - DIRECTORY = _v2.DIRECTORY - SYMLINK = _v2.SYMLINK - REGULAR = _v2.REGULAR - PERSISTENT = _v2.PERSISTENT - POST_PROCESSING = _v2.POST_PROCESSING + INITIAL = pb2.INITIAL + METADATA = pb2.METADATA + DIRECTORY = pb2.DIRECTORY + SYMLINK = pb2.SYMLINK + REGULAR = pb2.REGULAR + PERSISTENT = pb2.PERSISTENT + POST_PROCESSING = pb2.POST_PROCESSING class UpdatePhase(EnumWrapper): - INITIALIZING = _v2.INITIALIZING - PROCESSING_METADATA = _v2.PROCESSING_METADATA - CALCULATING_DELTA = _v2.CALCULATING_DELTA - DOWNLOADING_OTA_FILES = _v2.DOWNLOADING_OTA_FILES - APPLYING_UPDATE = _v2.APPLYING_UPDATE - PROCESSING_POSTUPDATE = _v2.PROCESSING_POSTUPDATE - FINALIZING_UPDATE = _v2.FINALIZING_UPDATE + INITIALIZING = pb2.INITIALIZING + PROCESSING_METADATA = pb2.PROCESSING_METADATA + CALCULATING_DELTA = pb2.CALCULATING_DELTA + DOWNLOADING_OTA_FILES = pb2.DOWNLOADING_OTA_FILES + APPLYING_UPDATE = pb2.APPLYING_UPDATE + PROCESSING_POSTUPDATE = pb2.PROCESSING_POSTUPDATE + FINALIZING_UPDATE = pb2.FINALIZING_UPDATE # message wrapper definitions @@ -166,15 +166,15 @@ class UpdatePhase(EnumWrapper): # rollback API -class RollbackRequestEcu(MessageWrapper[_v2.RollbackRequestEcu]): - __slots__ = calculate_slots(_v2.RollbackRequestEcu) +class RollbackRequestEcu(MessageWrapper[pb2.RollbackRequestEcu]): + __slots__ = calculate_slots(pb2.RollbackRequestEcu) ecu_id: str def __init__(self, *, ecu_id: _Optional[str] = ...) -> None: ... -class RollbackRequest(ECUList[RollbackRequestEcu], MessageWrapper[_v2.RollbackRequest]): - __slots__ = calculate_slots(_v2.RollbackRequest) +class RollbackRequest(ECUList[RollbackRequestEcu], MessageWrapper[pb2.RollbackRequest]): + __slots__ = calculate_slots(pb2.RollbackRequest) ecu: RepeatedCompositeContainer[RollbackRequestEcu] def __init__( @@ -182,8 +182,8 @@ def __init__( ) -> None: ... -class RollbackResponseEcu(MessageWrapper[_v2.RollbackResponseEcu]): - __slots__ = calculate_slots(_v2.RollbackRequestEcu) +class RollbackResponseEcu(MessageWrapper[pb2.RollbackResponseEcu]): + __slots__ = calculate_slots(pb2.RollbackRequestEcu) ecu_id: str result: FailureType @@ -196,17 +196,17 @@ def __init__( class RollbackResponse( - ECUList[RollbackResponseEcu], MessageWrapper[_v2.RollbackResponse] + ECUList[RollbackResponseEcu], MessageWrapper[pb2.RollbackResponse] ): - __slots__ = calculate_slots(_v2.RollbackResponse) + __slots__ = calculate_slots(pb2.RollbackResponse) ecu: RepeatedCompositeContainer[RollbackResponseEcu] def __init__( self, *, ecu: _Optional[_Iterable[RollbackResponseEcu]] = ... ) -> None: ... - def merge_from(self, rollback_response: _Union[Self, _v2.RollbackResponse]): - if isinstance(rollback_response, _v2.RollbackResponse): + def merge_from(self, rollback_response: _Union[Self, pb2.RollbackResponse]): + if isinstance(rollback_response, pb2.RollbackResponse): rollback_response = self.__class__.convert(rollback_response) # NOTE, TODO: duplication check is not done self.ecu.extend(rollback_response.ecu) @@ -215,8 +215,8 @@ def merge_from(self, rollback_response: _Union[Self, _v2.RollbackResponse]): # status API -class StatusProgress(MessageWrapper[_v2.StatusProgress]): - __slots__ = calculate_slots(_v2.StatusProgress) +class StatusProgress(MessageWrapper[pb2.StatusProgress]): + __slots__ = calculate_slots(pb2.StatusProgress) download_bytes: int elapsed_time_copy: Duration elapsed_time_download: Duration @@ -263,8 +263,8 @@ def add_elapsed_time(self, _field_name: str, _value: int): _field.add_nanoseconds(_value) -class Status(MessageWrapper[_v2.Status]): - __slots__ = calculate_slots(_v2.Status) +class Status(MessageWrapper[pb2.Status]): + __slots__ = calculate_slots(pb2.Status) failure: FailureType failure_reason: str progress: StatusProgress @@ -282,12 +282,12 @@ def __init__( ) -> None: ... -class StatusRequest(MessageWrapper[_v2.StatusRequest]): - __slots__ = calculate_slots(_v2.StatusRequest) +class StatusRequest(MessageWrapper[pb2.StatusRequest]): + __slots__ = calculate_slots(pb2.StatusRequest) -class StatusResponseEcu(ECUStatusSummary, MessageWrapper[_v2.StatusResponseEcu]): - __slots__ = calculate_slots(_v2.StatusResponseEcu) +class StatusResponseEcu(ECUStatusSummary, MessageWrapper[pb2.StatusResponseEcu]): + __slots__ = calculate_slots(pb2.StatusResponseEcu) ecu_id: str result: FailureType status: Status @@ -334,8 +334,8 @@ def requires_network(self) -> bool: } -class UpdateStatus(MessageWrapper[_v2.UpdateStatus]): - __slots__ = calculate_slots(_v2.UpdateStatus) +class UpdateStatus(MessageWrapper[pb2.UpdateStatus]): + __slots__ = calculate_slots(pb2.UpdateStatus) delta_generating_elapsed_time: Duration downloaded_bytes: int downloaded_files_num: int @@ -422,8 +422,8 @@ def convert_to_v1_StatusProgress(self) -> StatusProgress: return _res -class StatusResponseEcuV2(ECUStatusSummary, MessageWrapper[_v2.StatusResponseEcuV2]): - __slots__ = calculate_slots(_v2.StatusResponseEcuV2) +class StatusResponseEcuV2(ECUStatusSummary, MessageWrapper[pb2.StatusResponseEcuV2]): + __slots__ = calculate_slots(pb2.StatusResponseEcuV2) ecu_id: str failure_reason: str failure_traceback: str @@ -483,9 +483,9 @@ def requires_network(self) -> bool: class StatusResponse( ECUV2List[StatusResponseEcuV2], ECUList[StatusResponseEcu], - MessageWrapper[_v2.StatusResponse], + MessageWrapper[pb2.StatusResponse], ): - __slots__ = calculate_slots(_v2.StatusResponse) + __slots__ = calculate_slots(pb2.StatusResponse) available_ecu_ids: RepeatedScalarContainer[str] ecu: RepeatedCompositeContainer[StatusResponseEcu] ecu_v2: RepeatedCompositeContainer[StatusResponseEcuV2] @@ -502,20 +502,20 @@ def add_ecu(self, _response_ecu: Any): if isinstance(_response_ecu, StatusResponseEcuV2): self.ecu_v2.append(_response_ecu) self.ecu.append(_response_ecu.convert_to_v1()) # v1 compat - elif isinstance(_response_ecu, _v2.StatusResponseEcuV2): + elif isinstance(_response_ecu, pb2.StatusResponseEcuV2): _converted = StatusResponseEcuV2.convert(_response_ecu) self.ecu_v2.append(_response_ecu) self.ecu.append(_converted.convert_to_v1()) # v1 compat # v1 elif isinstance(_response_ecu, StatusResponseEcu): self.ecu.append(_response_ecu) - elif isinstance(_response_ecu, _v2.StatusResponseEcu): + elif isinstance(_response_ecu, pb2.StatusResponseEcu): self.ecu.append(StatusResponseEcu.convert(_response_ecu)) else: raise TypeError - def merge_from(self, status_resp: _Union[Self, _v2.StatusResponse]): - if isinstance(status_resp, _v2.StatusResponse): + def merge_from(self, status_resp: _Union[Self, pb2.StatusResponse]): + if isinstance(status_resp, pb2.StatusResponse): status_resp = self.__class__.convert(status_resp) # merge ecu only, don't merge available_ecu_ids! # NOTE, TODO: duplication check is not done @@ -526,8 +526,8 @@ def merge_from(self, status_resp: _Union[Self, _v2.StatusResponse]): # update API -class UpdateRequestEcu(MessageWrapper[_v2.UpdateRequestEcu]): - __slots__ = calculate_slots(_v2.UpdateRequestEcu) +class UpdateRequestEcu(MessageWrapper[pb2.UpdateRequestEcu]): + __slots__ = calculate_slots(pb2.UpdateRequestEcu) cookies: str ecu_id: str url: str @@ -543,8 +543,8 @@ def __init__( ) -> None: ... -class UpdateRequest(ECUList[UpdateRequestEcu], MessageWrapper[_v2.UpdateRequest]): - __slots__ = calculate_slots(_v2.UpdateRequest) +class UpdateRequest(ECUList[UpdateRequestEcu], MessageWrapper[pb2.UpdateRequest]): + __slots__ = calculate_slots(pb2.UpdateRequest) ecu: RepeatedCompositeContainer[UpdateRequestEcu] def __init__( @@ -552,8 +552,8 @@ def __init__( ) -> None: ... -class UpdateResponseEcu(MessageWrapper[_v2.UpdateResponseEcu]): - __slots__ = calculate_slots(_v2.UpdateResponseEcu) +class UpdateResponseEcu(MessageWrapper[pb2.UpdateResponseEcu]): + __slots__ = calculate_slots(pb2.UpdateResponseEcu) ecu_id: str result: FailureType @@ -565,8 +565,8 @@ def __init__( ) -> None: ... -class UpdateResponse(ECUList[UpdateResponseEcu], MessageWrapper[_v2.UpdateResponse]): - __slots__ = calculate_slots(_v2.UpdateResponse) +class UpdateResponse(ECUList[UpdateResponseEcu], MessageWrapper[pb2.UpdateResponse]): + __slots__ = calculate_slots(pb2.UpdateResponse) ecu: RepeatedCompositeContainer[UpdateResponseEcu] def __init__( @@ -581,8 +581,8 @@ def ecus_acked_update(self) -> _Set[str]: if ecu_resp.result is FailureType.NO_FAILURE } - def merge_from(self, update_response: _Union[Self, _v2.UpdateResponse]): - if isinstance(update_response, _v2.UpdateResponse): + def merge_from(self, update_response: _Union[Self, pb2.UpdateResponse]): + if isinstance(update_response, pb2.UpdateResponse): update_response = self.__class__.convert(update_response) # NOTE, TODO: duplication check is not done self.ecu.extend(update_response.ecu) diff --git a/src/otaclient/_utils/__init__.py b/src/otaclient_common/__init__.py similarity index 67% rename from src/otaclient/_utils/__init__.py rename to src/otaclient_common/__init__.py index dd75b77f2..a73c58270 100644 --- a/src/otaclient/_utils/__init__.py +++ b/src/otaclient_common/__init__.py @@ -11,46 +11,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +"""Common shared libs for otaclient.""" from __future__ import annotations +import importlib.util import os from math import ceil from pathlib import Path -from typing import Any, Callable, Optional, TypeVar - -from typing_extensions import Concatenate, Literal, ParamSpec - -P = ParamSpec("P") - - -def copy_callable_typehint(_source: Callable[P, Any]): - """This helper function return a decorator that can type hint the target - function as the _source function. - - At runtime, this decorator actually does nothing, but just return the input function as it. - But the returned function will have the same type hint as the source function in ide. - It will not impact the runtime behavior of the decorated function. - """ - - def _decorator(target) -> Callable[P, Any]: - return target - - return _decorator - - -RT = TypeVar("RT") - - -def copy_callable_typehint_to_method(_source: Callable[P, Any]): - """Works the same as copy_callable_typehint, but omit the first arg.""" - - def _decorator(target: Callable[..., RT]) -> Callable[Concatenate[Any, P], RT]: - return target # type: ignore - - return _decorator +from types import ModuleType +from typing import Optional +from typing_extensions import Literal _MultiUnits = Literal["GiB", "MiB", "KiB", "Bytes", "KB", "MB", "GB"] # fmt: off @@ -87,3 +60,16 @@ def replace_root(path: str | Path, old_root: str | Path, new_root: str | Path) - if os.path.commonpath([path, old_root]) != old_root: raise ValueError(f"{old_root=} is not the root of {path=}") return os.path.join(new_root, os.path.relpath(path, old_root)) + + +def import_from_file(path: Path) -> tuple[str, ModuleType]: + if not path.is_file(): + raise ValueError(f"{path} is not a valid module file") + try: + _module_name = path.stem + _spec = importlib.util.spec_from_file_location(_module_name, path) + _module = importlib.util.module_from_spec(_spec) # type: ignore + _spec.loader.exec_module(_module) # type: ignore + return _module_name, _module + except Exception: + raise ImportError(f"failed to import module from {path=}.") diff --git a/src/otaclient_common/common.py b/src/otaclient_common/common.py new file mode 100644 index 000000000..10433bc2f --- /dev/null +++ b/src/otaclient_common/common.py @@ -0,0 +1,404 @@ +# Copyright 2022 TIER IV, INC. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Utils that shared between modules are listed here. + +TODO(20240603): the old otaclient.app.common, split it by functionalities in + the future. +""" + + +from __future__ import annotations + +import logging +import os +import shlex +import shutil +import subprocess +import time +from hashlib import sha256 +from pathlib import Path +from typing import Optional, Union +from urllib.parse import urljoin + +import requests + +logger = logging.getLogger(__name__) + + +def get_backoff(n: int, factor: float, _max: float) -> float: + return min(_max, factor * (2 ** (n - 1))) + + +def wait_with_backoff(_retry_cnt: int, *, _backoff_factor: float, _backoff_max: float): + time.sleep( + get_backoff( + _retry_cnt, + _backoff_factor, + _backoff_max, + ) + ) + + +# file verification +def file_sha256( + filename: Union[Path, str], *, chunk_size: int = 1 * 1024 * 1024 +) -> str: + with open(filename, "rb") as f: + m = sha256() + while True: + d = f.read(chunk_size) + if len(d) == 0: + break + m.update(d) + return m.hexdigest() + + +def verify_file(fpath: Path, fhash: str, fsize: Optional[int]) -> bool: + if ( + fpath.is_symlink() + or (not fpath.is_file()) + or (fsize is not None and fpath.stat().st_size != fsize) + ): + return False + return file_sha256(fpath) == fhash + + +# handled file read/write +def read_str_from_file(path: Union[Path, str], *, missing_ok=True, default="") -> str: + """ + Params: + missing_ok: if set to False, FileNotFoundError will be raised to upper + default: the default value to return when missing_ok=True and file not found + """ + try: + return Path(path).read_text().strip() + except FileNotFoundError: + if missing_ok: + return default + + raise + + +def write_str_to_file(path: Path, input: str): + path.write_text(input) + + +def write_str_to_file_sync(path: Union[Path, str], input: str): + with open(path, "w") as f: + f.write(input) + f.flush() + os.fsync(f.fileno()) + + +def subprocess_run_wrapper( + cmd: str | list[str], + *, + check: bool, + check_output: bool, + timeout: Optional[float] = None, +) -> subprocess.CompletedProcess[bytes]: + """A wrapper for subprocess.run method. + + NOTE: this is for the requirement of customized subprocess call + in the future, like chroot or nsenter before execution. + + Args: + cmd (str | list[str]): command to be executed. + check (bool): if True, raise CalledProcessError on non 0 return code. + check_output (bool): if True, the UTF-8 decoded stdout will be returned. + timeout (Optional[float], optional): timeout for execution. Defaults to None. + + Returns: + subprocess.CompletedProcess[bytes]: the result of the execution. + """ + if isinstance(cmd, str): + cmd = shlex.split(cmd) + + return subprocess.run( + cmd, + check=check, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE if check_output else None, + timeout=timeout, + ) + + +def subprocess_check_output( + cmd: str | list[str], + *, + raise_exception: bool = False, + default: str = "", + timeout: Optional[float] = None, +) -> str: + """Run the and return UTF-8 decoded stripped stdout. + + Args: + cmd (str | list[str]): command to be executed. + raise_exception (bool, optional): raise the underlying CalledProcessError. Defaults to False. + default (str, optional): if is False, return on underlying + subprocess call failed. Defaults to "". + timeout (Optional[float], optional): timeout for execution. Defaults to None. + + Returns: + str: UTF-8 decoded stripped stdout. + """ + try: + res = subprocess_run_wrapper( + cmd, check=True, check_output=True, timeout=timeout + ) + return res.stdout.decode().strip() + except subprocess.CalledProcessError as e: + _err_msg = ( + f"command({cmd=}) failed(retcode={e.returncode}: \n" + f"stderr={e.stderr.decode()}" + ) + logger.debug(_err_msg) + + if raise_exception: + raise + return default + + +def subprocess_call( + cmd: str | list[str], + *, + raise_exception: bool = False, + timeout: Optional[float] = None, +) -> None: + """Run the . + + Args: + cmd (str | list[str]): command to be executed. + raise_exception (bool, optional): raise the underlying CalledProcessError. Defaults to False. + timeout (Optional[float], optional): timeout for execution. Defaults to None. + """ + try: + subprocess_run_wrapper(cmd, check=True, check_output=False, timeout=timeout) + except subprocess.CalledProcessError as e: + _err_msg = ( + f"command({cmd=}) failed(retcode={e.returncode}: \n" + f"stderr={e.stderr.decode()}" + ) + logger.debug(_err_msg) + + if raise_exception: + raise + + +def copy_stat(src: Union[Path, str], dst: Union[Path, str]): + """Copy file/dir permission bits and owner info from src to dst.""" + _stat = Path(src).stat() + os.chown(dst, _stat.st_uid, _stat.st_gid) + os.chmod(dst, _stat.st_mode) + + +def copytree_identical(src: Path, dst: Path): + """Recursively copy from the src folder to dst folder. + + Source folder MUST be a dir. + + This function populate files/dirs from the src to the dst, + and make sure the dst is identical to the src. + + By updating the dst folder in-place, we can prevent the case + that the copy is interrupted and the dst is not yet fully populated. + + This function is different from shutil.copytree as follow: + 1. it covers the case that the same path points to different + file type, in this case, the dst path will be clean and + new file/dir will be populated as the src. + 2. it deals with the same symlinks by checking the link target, + re-generate the symlink if the dst symlink is not the same + as the src. + 3. it will remove files that not presented in the src, and + unconditionally override files with same path, ensuring + that the dst will be identical with the src. + + NOTE: is_file/is_dir also returns True if it is a symlink and + the link target is_file/is_dir + """ + if src.is_symlink() or not src.is_dir(): + raise ValueError(f"{src} is not a dir") + + if dst.is_symlink() or not dst.is_dir(): + logger.info(f"{dst=} doesn't exist or not a dir, cleanup and mkdir") + dst.unlink(missing_ok=True) # unlink doesn't follow the symlink + dst.mkdir(mode=src.stat().st_mode, parents=True) + + # phase1: populate files to the dst + for cur_dir, dirs, files in os.walk(src, topdown=True, followlinks=False): + _cur_dir = Path(cur_dir) + _cur_dir_on_dst = dst / _cur_dir.relative_to(src) + + # NOTE(20220803): os.walk now lists symlinks pointed to dir + # in the tuple, we have to handle this behavior + for _dir in dirs: + _src_dir = _cur_dir / _dir + _dst_dir = _cur_dir_on_dst / _dir + if _src_dir.is_symlink(): # this "dir" is a symlink to a dir + if (not _dst_dir.is_symlink()) and _dst_dir.is_dir(): + # if dst is a dir, remove it + shutil.rmtree(_dst_dir, ignore_errors=True) + else: # dst is symlink or file + _dst_dir.unlink() + _dst_dir.symlink_to(os.readlink(_src_dir)) + + # cover the edge case that dst is not a dir. + if _cur_dir_on_dst.is_symlink() or not _cur_dir_on_dst.is_dir(): + _cur_dir_on_dst.unlink(missing_ok=True) + _cur_dir_on_dst.mkdir(parents=True) + copy_stat(_cur_dir, _cur_dir_on_dst) + + # populate files + for fname in files: + _src_f = _cur_dir / fname + _dst_f = _cur_dir_on_dst / fname + + # prepare dst + # src is file but dst is a folder + # delete the dst in advance + if (not _dst_f.is_symlink()) and _dst_f.is_dir(): + # if dst is a dir, remove it + shutil.rmtree(_dst_f, ignore_errors=True) + else: + # dst is symlink or file + _dst_f.unlink(missing_ok=True) + + # copy/symlink dst as src + # if src is symlink, check symlink, re-link if needed + if _src_f.is_symlink(): + _dst_f.symlink_to(os.readlink(_src_f)) + else: + # copy/override src to dst + shutil.copy(_src_f, _dst_f, follow_symlinks=False) + copy_stat(_src_f, _dst_f) + + # phase2: remove unused files in the dst + for cur_dir, dirs, files in os.walk(dst, topdown=True, followlinks=False): + _cur_dir_on_dst = Path(cur_dir) + _cur_dir_on_src = src / _cur_dir_on_dst.relative_to(dst) + + # remove unused dir + if not _cur_dir_on_src.is_dir(): + shutil.rmtree(_cur_dir_on_dst, ignore_errors=True) + dirs.clear() # stop iterate the subfolders of this dir + continue + + # NOTE(20220803): os.walk now lists symlinks pointed to dir + # in the tuple, we have to handle this behavior + for _dir in dirs: + _src_dir = _cur_dir_on_src / _dir + _dst_dir = _cur_dir_on_dst / _dir + if (not _src_dir.is_symlink()) and _dst_dir.is_symlink(): + _dst_dir.unlink() + + for fname in files: + _src_f = _cur_dir_on_src / fname + if not (_src_f.is_symlink() or _src_f.is_file()): + (_cur_dir_on_dst / fname).unlink(missing_ok=True) + + +def re_symlink_atomic(src: Path, target: Union[Path, str]): + """Make the a symlink to atomically. + + If the src is already existed as a file/symlink, + the src will be replaced by the newly created link unconditionally. + + NOTE: os.rename is atomic when src and dst are on + the same filesystem under linux. + NOTE 2: src should not exist or exist as file/symlink. + """ + if not (src.is_symlink() and str(os.readlink(src)) == str(target)): + tmp_link = Path(src).parent / f"tmp_link_{os.urandom(6).hex()}" + try: + tmp_link.symlink_to(target) + os.rename(tmp_link, src) # unconditionally override + except Exception: + tmp_link.unlink(missing_ok=True) + raise + + +def replace_atomic(src: Union[str, Path], dst: Union[str, Path]): + """Atomically replace dst file with src file. + + NOTE: atomic is ensured by os.rename/os.replace under the same filesystem. + """ + src, dst = Path(src), Path(dst) + if not src.is_file(): + raise ValueError(f"{src=} is not a regular file or not exist") + + _tmp_file = dst.parent / f".tmp_{os.urandom(6).hex()}" + try: + # prepare a copy of src file under dst's parent folder + shutil.copy(src, _tmp_file, follow_symlinks=True) + # atomically rename/replace the dst file with the copy + os.replace(_tmp_file, dst) + os.sync() + except Exception: + _tmp_file.unlink(missing_ok=True) + raise + + +def urljoin_ensure_base(base: str, url: str): + """ + NOTE: this method ensure the base_url will be preserved. + for example: + base="http://example.com/data", url="path/to/file" + with urljoin, joined url will be "http://example.com/path/to/file", + with this func, joined url will be "http://example.com/data/path/to/file" + """ + return urljoin(f"{base.rstrip('/')}/", url) + + +def create_tmp_fname(prefix="tmp", length=6, sep="_") -> str: + return f"{prefix}{sep}{os.urandom(length).hex()}" + + +def ensure_otaproxy_start( + otaproxy_url: str, + *, + interval: float = 1, + connection_timeout: float = 5, + probing_timeout: Optional[float] = None, + warning_interval: int = 3 * 60, # seconds +): + """Loop probing until online or exceed . + + This function will issue a logging.warning every seconds. + + Raises: + A ConnectionError if exceeds . + """ + start_time = int(time.time()) + next_warning = start_time + warning_interval + probing_timeout = ( + probing_timeout if probing_timeout and probing_timeout >= 0 else float("inf") + ) + with requests.Session() as session: + while start_time + probing_timeout > (cur_time := int(time.time())): + try: + resp = session.get(otaproxy_url, timeout=connection_timeout) + resp.close() + return + except Exception as e: # server is not up yet + if cur_time >= next_warning: + logger.warning( + f"otaproxy@{otaproxy_url} is not up after {cur_time - start_time} seconds" + f"it might be something wrong with this otaproxy: {e!r}" + ) + next_warning = next_warning + warning_interval + time.sleep(interval) + raise ConnectionError( + f"failed to ensure connection to {otaproxy_url} in {probing_timeout=}seconds" + ) diff --git a/src/otaclient/app/downloader.py b/src/otaclient_common/downloader.py similarity index 98% rename from src/otaclient/app/downloader.py rename to src/otaclient_common/downloader.py index 173d9b1a3..b67a3371c 100644 --- a/src/otaclient/app/downloader.py +++ b/src/otaclient_common/downloader.py @@ -11,8 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +"""A common used downloader implementation for otaclient.""" +from __future__ import annotations + import errno import logging import os @@ -49,10 +52,9 @@ from urllib3.util.retry import Retry from ota_proxy import OTAFileCacheControl -from otaclient._utils import copy_callable_typehint - -from .common import wait_with_backoff -from .configs import config as cfg +from otaclient.app.configs import config as cfg +from otaclient_common.common import wait_with_backoff +from otaclient_common.typing import copy_callable_typehint logger = logging.getLogger(__name__) diff --git a/src/otaclient/_utils/linux.py b/src/otaclient_common/linux.py similarity index 100% rename from src/otaclient/_utils/linux.py rename to src/otaclient_common/linux.py diff --git a/src/otaclient/_utils/logging.py b/src/otaclient_common/logging.py similarity index 100% rename from src/otaclient/_utils/logging.py rename to src/otaclient_common/logging.py diff --git a/src/otaclient_common/persist_file_handling.py b/src/otaclient_common/persist_file_handling.py new file mode 100644 index 000000000..6b0178017 --- /dev/null +++ b/src/otaclient_common/persist_file_handling.py @@ -0,0 +1,219 @@ +# Copyright 2022 TIER IV, INC. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import annotations + +import logging +import os +import shutil +from functools import lru_cache, partial +from pathlib import Path + +from otaclient_common.linux import ( + ParsedGroup, + ParsedPasswd, + map_gid_by_grpnam, + map_uid_by_pwnam, +) + +logger = logging.getLogger(__name__) + + +class PersistFilesHandler: + """Preserving files in persist list from to . + + Files being copied will have mode bits preserved, + and uid/gid preserved with mapping as follow: + + src_uid -> src_name -> dst_name -> dst_uid + src_gid -> src_name -> dst_name -> dst_gid + """ + + def __init__( + self, + src_passwd_file: str | Path, + src_group_file: str | Path, + dst_passwd_file: str | Path, + dst_group_file: str | Path, + *, + src_root: str | Path, + dst_root: str | Path, + ): + self._uid_mapper = lru_cache()( + partial( + self.map_uid_by_pwnam, + src_db=ParsedPasswd(src_passwd_file), + dst_db=ParsedPasswd(dst_passwd_file), + ) + ) + self._gid_mapper = lru_cache()( + partial( + self.map_gid_by_grpnam, + src_db=ParsedGroup(src_group_file), + dst_db=ParsedGroup(dst_group_file), + ) + ) + self._src_root = Path(src_root) + self._dst_root = Path(dst_root) + + @staticmethod + def map_uid_by_pwnam( + *, src_db: ParsedPasswd, dst_db: ParsedPasswd, uid: int + ) -> int: + _mapped_uid = map_uid_by_pwnam(src_db=src_db, dst_db=dst_db, uid=uid) + _usern = src_db._by_uid[uid] + + logger.info(f"{_usern=}: mapping src_{uid=} to {_mapped_uid=}") + return _mapped_uid + + @staticmethod + def map_gid_by_grpnam(*, src_db: ParsedGroup, dst_db: ParsedGroup, gid: int) -> int: + _mapped_gid = map_gid_by_grpnam(src_db=src_db, dst_db=dst_db, gid=gid) + _groupn = src_db._by_gid[gid] + + logger.info(f"{_groupn=}: mapping src_{gid=} to {_mapped_gid=}") + return _mapped_gid + + def _chown_with_mapping( + self, _src_stat: os.stat_result, _dst_path: str | Path + ) -> None: + _src_uid, _src_gid = _src_stat.st_uid, _src_stat.st_gid + try: + _dst_uid = self._uid_mapper(uid=_src_uid) + except ValueError: + logger.warning(f"failed to find mapping for {_src_uid=}, keep unchanged") + _dst_uid = _src_uid + + try: + _dst_gid = self._gid_mapper(gid=_src_gid) + except ValueError: + logger.warning(f"failed to find mapping for {_src_gid=}, keep unchanged") + _dst_gid = _src_gid + os.chown(_dst_path, uid=_dst_uid, gid=_dst_gid, follow_symlinks=False) + + @staticmethod + def _rm_target(_target: Path) -> None: + """Remove target with proper methods.""" + if _target.is_symlink() or _target.is_file(): + return _target.unlink(missing_ok=True) + elif _target.is_dir(): + return shutil.rmtree(_target, ignore_errors=True) + elif _target.exists(): + raise ValueError( + f"{_target} is not normal file/symlink/dir, failed to remove" + ) + + def _prepare_symlink(self, _src_path: Path, _dst_path: Path) -> None: + _dst_path.symlink_to(os.readlink(_src_path)) + # NOTE: to get stat from symlink, using os.stat with follow_symlinks=False + self._chown_with_mapping(os.stat(_src_path, follow_symlinks=False), _dst_path) + + def _prepare_dir(self, _src_path: Path, _dst_path: Path) -> None: + _dst_path.mkdir(exist_ok=True) + + _src_stat = os.stat(_src_path, follow_symlinks=False) + os.chmod(_dst_path, _src_stat.st_mode) + self._chown_with_mapping(_src_stat, _dst_path) + + def _prepare_file(self, _src_path: Path, _dst_path: Path) -> None: + shutil.copy(_src_path, _dst_path, follow_symlinks=False) + + _src_stat = os.stat(_src_path, follow_symlinks=False) + os.chmod(_dst_path, _src_stat.st_mode) + self._chown_with_mapping(_src_stat, _dst_path) + + def _prepare_parent(self, _origin_entry: Path) -> None: + for _parent in reversed(_origin_entry.parents): + _src_parent, _dst_parent = ( + self._src_root / _parent, + self._dst_root / _parent, + ) + if _dst_parent.is_dir(): # keep the origin parent on dst as it + continue + if _dst_parent.is_symlink() or _dst_parent.is_file(): + _dst_parent.unlink(missing_ok=True) + self._prepare_dir(_src_parent, _dst_parent) + continue + if _dst_parent.exists(): + raise ValueError( + f"{_dst_parent=} is not a normal file/symlink/dir, cannot cleanup" + ) + self._prepare_dir(_src_parent, _dst_parent) + + # API + + def preserve_persist_entry( + self, _persist_entry: str | Path, *, src_missing_ok: bool = True + ): + logger.info(f"preserving {_persist_entry}") + # persist_entry in persists.txt must be rooted at / + origin_entry = Path(_persist_entry).relative_to("/") + src_path = self._src_root / origin_entry + dst_path = self._dst_root / origin_entry + + # ------ src is symlink ------ # + # NOTE: always check if symlink first as is_file/is_dir/exists all follow_symlinks + if src_path.is_symlink(): + self._rm_target(dst_path) + self._prepare_parent(origin_entry) + self._prepare_symlink(src_path, dst_path) + return + + # ------ src is file ------ # + if src_path.is_file(): + self._rm_target(dst_path) + self._prepare_parent(origin_entry) + self._prepare_file(src_path, dst_path) + return + + # ------ src is not regular file/symlink/dir ------ # + # we only process normal file/symlink/dir + if src_path.exists() and not src_path.is_dir(): + raise ValueError(f"{src_path=} must be either a file/symlink/dir") + + # ------ src doesn't exist ------ # + if not src_path.exists(): + _err_msg = f"{src_path=} not found" + logger.warning(_err_msg) + if not src_missing_ok: + raise ValueError(_err_msg) + return + + # ------ src is dir ------ # + # dive into src_dir and preserve everything under the src dir + self._prepare_parent(origin_entry) + for src_curdir, dnames, fnames in os.walk(src_path, followlinks=False): + src_cur_dpath = Path(src_curdir) + dst_cur_dpath = self._dst_root / src_cur_dpath.relative_to(self._src_root) + + # ------ prepare current dir itself ------ # + self._rm_target(dst_cur_dpath) + self._prepare_dir(src_cur_dpath, dst_cur_dpath) + + # ------ prepare entries in current dir ------ # + for _fname in fnames: + _src_fpath, _dst_fpath = src_cur_dpath / _fname, dst_cur_dpath / _fname + self._rm_target(_dst_fpath) + if _src_fpath.is_symlink(): + self._prepare_symlink(_src_fpath, _dst_fpath) + continue + self._prepare_file(_src_fpath, _dst_fpath) + + # symlinks to dirs also included in dnames, we must handle it + for _dname in dnames: + _src_dpath, _dst_dpath = src_cur_dpath / _dname, dst_cur_dpath / _dname + if _src_dpath.is_symlink(): + self._rm_target(_dst_dpath) + self._prepare_symlink(_src_dpath, _dst_dpath) diff --git a/src/otaclient/app/proto/streamer.py b/src/otaclient_common/proto_streamer.py similarity index 97% rename from src/otaclient/app/proto/streamer.py rename to src/otaclient_common/proto_streamer.py index fb887dc01..e52934730 100644 --- a/src/otaclient/app/proto/streamer.py +++ b/src/otaclient_common/proto_streamer.py @@ -23,7 +23,7 @@ from typing import BinaryIO, Generic, Iterable, Optional, Type -from ._common import MessageType, MessageWrapperType +from otaclient_common.proto_wrapper import MessageType, MessageWrapperType UINT32_LEN = 4 # bytes diff --git a/src/otaclient/app/proto/_common.py b/src/otaclient_common/proto_wrapper.py similarity index 100% rename from src/otaclient/app/proto/_common.py rename to src/otaclient_common/proto_wrapper.py diff --git a/src/otaclient/app/proto/README.md b/src/otaclient_common/proto_wrapper_README.md similarity index 100% rename from src/otaclient/app/proto/README.md rename to src/otaclient_common/proto_wrapper_README.md diff --git a/src/otaclient_common/retry_task_map.py b/src/otaclient_common/retry_task_map.py new file mode 100644 index 000000000..80c10e061 --- /dev/null +++ b/src/otaclient_common/retry_task_map.py @@ -0,0 +1,225 @@ +# Copyright 2022 TIER IV, INC. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import annotations + +import itertools +import logging +import threading +import time +from concurrent.futures import Future, ThreadPoolExecutor +from functools import partial +from queue import Queue +from typing import ( + Any, + Callable, + Generator, + Generic, + Iterable, + NamedTuple, + Optional, + Set, + TypeVar, +) + +logger = logging.getLogger(__name__) + +T = TypeVar("T") + + +class DoneTask(NamedTuple): + fut: Future + entry: Any + + +class RetryTaskMapInterrupted(Exception): + pass + + +class _TaskMap(Generic[T]): + def __init__( + self, + executor: ThreadPoolExecutor, + max_concurrent: int, + backoff_func: Callable[[int], float], + ) -> None: + # task dispatch interval for continues failling + self.started = False # can only be started once + self._backoff_func = backoff_func + self._executor = executor + self._shutdown_event = threading.Event() + self._se = threading.Semaphore(max_concurrent) + + self._total_tasks_count = 0 + self._dispatched_tasks: Set[Future] = set() + self._failed_tasks: Set[T] = set() + self._last_failed_fut: Optional[Future] = None + + # NOTE: itertools.count is only thread-safe in CPython with GIL, + # as itertools.count is pure C implemented, calling next over + # it is atomic in Python level. + self._done_task_counter = itertools.count(start=1) + self._all_done = threading.Event() + self._dispatch_done = False + + self._done_que: Queue[DoneTask] = Queue() + + def _done_task_cb(self, item: T, fut: Future): + """ + Tracking done counting, set all_done event. + add failed to failed list. + """ + self._se.release() # always release se first + # NOTE: don't change dispatched_tasks if shutdown_event is set + if self._shutdown_event.is_set(): + return + + self._dispatched_tasks.discard(fut) + # check if we finish all tasks + _done_task_num = next(self._done_task_counter) + if self._dispatch_done and _done_task_num == self._total_tasks_count: + logger.debug("all done!") + self._all_done.set() + + if fut.exception(): + self._failed_tasks.add(item) + self._last_failed_fut = fut + self._done_que.put_nowait(DoneTask(fut, item)) + + def _task_dispatcher(self, func: Callable[[T], Any], _iter: Iterable[T]): + """A dispatcher in a dedicated thread that dispatches + tasks to threadpool.""" + for item in _iter: + if self._shutdown_event.is_set(): + return + self._se.acquire() + self._total_tasks_count += 1 + + fut = self._executor.submit(func, item) + fut.add_done_callback(partial(self._done_task_cb, item)) + self._dispatched_tasks.add(fut) + logger.debug(f"dispatcher done: {self._total_tasks_count=}") + self._dispatch_done = True + + def _done_task_collector(self) -> Generator[DoneTask, None, None]: + """A generator for caller to yield done task from.""" + _count = 0 + while not self._shutdown_event.is_set(): + if self._all_done.is_set() and _count == self._total_tasks_count: + logger.debug("collector done!") + return + + yield self._done_que.get() + _count += 1 + + def map(self, func: Callable[[T], Any], _iter: Iterable[T]): + if self.started: + raise ValueError(f"{self.__class__} inst can only be started once") + self.started = True + + self._task_dispatcher_fut = self._executor.submit( + self._task_dispatcher, func, _iter + ) + self._task_collector_gen = self._done_task_collector() + return self._task_collector_gen + + def shutdown(self, *, raise_last_exc=False) -> Optional[Set[T]]: + """Set the shutdown event, and cancal/cleanup ongoing tasks.""" + if not self.started or self._shutdown_event.is_set(): + return + + self._shutdown_event.set() + self._task_collector_gen.close() + # wait for dispatch to stop + self._task_dispatcher_fut.result() + + # cancel all the dispatched tasks + for fut in self._dispatched_tasks: + fut.cancel() + self._dispatched_tasks.clear() + + if not self._failed_tasks: + return + try: + if self._last_failed_fut: + _exc = self._last_failed_fut.exception() + _err_msg = f"{len(self._failed_tasks)=}, last failed: {_exc!r}" + if raise_last_exc: + raise RetryTaskMapInterrupted(_err_msg) from _exc + else: + logger.warning(_err_msg) + return self._failed_tasks.copy() + finally: + # be careful not to create ref cycle here + self._failed_tasks.clear() + _exc, self = None, None + + +class RetryTaskMap(Generic[T]): + def __init__( + self, + *, + backoff_func: Callable[[int], float], + max_retry: int, + max_concurrent: int, + max_workers: Optional[int] = None, + ) -> None: + self._running_inst: Optional[_TaskMap] = None + self._map_gen: Optional[Generator] = None + + self._backoff_func = backoff_func + self._retry_counter = range(max_retry) if max_retry else itertools.count() + self._max_concurrent = max_concurrent + self._max_workers = max_workers + self._executor = ThreadPoolExecutor(max_workers=self._max_workers) + + def map( + self, _func: Callable[[T], Any], _iter: Iterable[T] + ) -> Generator[DoneTask, None, None]: + retry_round = 0 + for retry_round in self._retry_counter: + self._running_inst = _inst = _TaskMap( + self._executor, self._max_concurrent, self._backoff_func + ) + logger.debug(f"{retry_round=} started") + + yield from _inst.map(_func, _iter) + + # this retry round ends, check overall result + if _failed_list := _inst.shutdown(raise_last_exc=False): + _iter = _failed_list # feed failed to next round + # deref before entering sleep + self._running_inst, _inst = None, None + + logger.warning(f"retry#{retry_round+1}: retry on {len(_failed_list)=}") + time.sleep(self._backoff_func(retry_round)) + else: # all tasks finished successfully + self._running_inst, _inst = None, None + return + try: + raise RetryTaskMapInterrupted(f"exceed try limit: {retry_round}") + finally: + # cleanup the defs + _func, _iter = None, None # type: ignore + + def shutdown(self, *, raise_last_exc: bool): + try: + logger.debug("shutdown retry task map") + if self._running_inst: + self._running_inst.shutdown(raise_last_exc=raise_last_exc) + # NOTE: passthrough the exception from underlying running_inst + finally: + self._running_inst = None + self._executor.shutdown(wait=True) diff --git a/src/otaclient/_utils/typing.py b/src/otaclient_common/typing.py similarity index 55% rename from src/otaclient/_utils/typing.py rename to src/otaclient_common/typing.py index 941935f93..b487a818a 100644 --- a/src/otaclient/_utils/typing.py +++ b/src/otaclient_common/typing.py @@ -20,12 +20,13 @@ from typing import Any, Callable, TypeVar, Union from pydantic import Field -from typing_extensions import Annotated, ParamSpec +from typing_extensions import Annotated, Concatenate, ParamSpec P = ParamSpec("P") -T = TypeVar("T", bound=Enum) RT = TypeVar("RT") +T = TypeVar("T") +EnumT = TypeVar("EnumT", bound=Enum) StrOrPath = Union[str, Path] # pydantic helpers @@ -33,7 +34,9 @@ NetworkPort = Annotated[int, Field(ge=1, le=65535)] -def gen_strenum_validator(enum_type: type[T]) -> Callable[[T | str | Any], T]: +def gen_strenum_validator( + enum_type: type[EnumT], +) -> Callable[[EnumT | str | Any], EnumT]: """A before validator generator that converts input value into enum before passing it to pydantic validator. @@ -41,10 +44,34 @@ def gen_strenum_validator(enum_type: type[T]) -> Callable[[T | str | Any], T]: pass strict validation if input is str. """ - def _inner(value: T | str | Any) -> T: + def _inner(value: EnumT | str | Any) -> EnumT: assert isinstance( value, (enum_type, str) ), f"{value=} should be {enum_type} or str type" return enum_type(value) return _inner + + +def copy_callable_typehint(_source: Callable[P, Any]): + """This helper function return a decorator that can type hint the target + function as the _source function. + + At runtime, this decorator actually does nothing, but just return the input function as it. + But the returned function will have the same type hint as the source function in ide. + It will not impact the runtime behavior of the decorated function. + """ + + def _decorator(target) -> Callable[P, Any]: + return target + + return _decorator + + +def copy_callable_typehint_to_method(_source: Callable[P, Any]): + """Works the same as copy_callable_typehint, but omit the first arg.""" + + def _decorator(target: Callable[..., RT]) -> Callable[Concatenate[Any, P], RT]: + return target # type: ignore + + return _decorator diff --git a/tests/conftest.py b/tests/conftest.py index c923caf1a..2e060f7a4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,6 +28,8 @@ logger = logging.getLogger(__name__) +TEST_DIR = Path(__file__).parent + @dataclass class TestConfiguration: @@ -40,8 +42,7 @@ class TestConfiguration: RPI_BOOT_MODULE_PATH = "otaclient.app.boot_control._rpi_boot" OTACLIENT_MODULE_PATH = "otaclient.app.ota_client" OTACLIENT_STUB_MODULE_PATH = "otaclient.app.ota_client_stub" - OTACLIENT_SERVICE_MODULE_PATH = "otaclient.app.ota_client_service" - OTAMETA_MODULE_PATH = "otaclient.app.ota_metadata" + OTAMETA_MODULE_PATH = "ota_metadata.legacy.parser" OTAPROXY_MODULE_PATH = "ota_proxy" CREATE_STANDBY_MODULE_PATH = "otaclient.app.create_standby" MAIN_MODULE_PATH = "otaclient.app.main" diff --git a/tests/test__utils/test_logging.py b/tests/test_logging.py similarity index 97% rename from tests/test__utils/test_logging.py rename to tests/test_logging.py index cdfd627ce..ef3c15e18 100644 --- a/tests/test__utils/test_logging.py +++ b/tests/test_logging.py @@ -18,7 +18,7 @@ from pytest import LogCaptureFixture -from otaclient._utils import logging as _logging +from otaclient_common import logging as _logging def test_BurstSuppressFilter(caplog: LogCaptureFixture): diff --git a/tests/test_ota_metadata/__init__.py b/tests/test_ota_metadata/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_ota_metadata.py b/tests/test_ota_metadata/test_legacy.py similarity index 99% rename from tests/test_ota_metadata.py rename to tests/test_ota_metadata/test_legacy.py index eb5bb502d..74e550a8f 100644 --- a/tests/test_ota_metadata.py +++ b/tests/test_ota_metadata/test_legacy.py @@ -27,7 +27,7 @@ from cryptography.hazmat.primitives import hashes, serialization from cryptography.hazmat.primitives.asymmetric import ec -from otaclient.app.ota_metadata import ( +from ota_metadata.legacy.parser import ( MetadataJWTPayloadInvalid, MetadataJWTVerificationFailed, _MetadataJWTParser, @@ -36,6 +36,9 @@ parse_regulars_from_txt, parse_symlinks_from_txt, ) +from tests.conftest import TEST_DIR + +GEN_CERTS_SCRIPT = TEST_DIR / "keys" / "gen_certs.sh" HEADER = """\ {"alg": "ES256"}\ @@ -162,10 +165,6 @@ def generate_jwt(payload_str: str, sign_key_file: Path): return f"{header}.{payload}.{signature}" -TEST_DIR = Path(__file__).parent -GEN_CERTS_SCRIPT = TEST_DIR / "keys" / "gen_certs.sh" - - class CertsDirs(TypedDict): multi_chain: Path chain_a: Path diff --git a/tests/test_ota_proxy/test_ota_proxy_server.py b/tests/test_ota_proxy/test_ota_proxy_server.py index c38015cda..361ef9868 100644 --- a/tests/test_ota_proxy/test_ota_proxy_server.py +++ b/tests/test_ota_proxy/test_ota_proxy_server.py @@ -27,9 +27,9 @@ import pytest import uvicorn +from ota_metadata.legacy.parser import parse_regulars_from_txt +from ota_metadata.legacy.types import RegularInf from ota_proxy.utils import url_based_hash -from otaclient.app.ota_metadata import parse_regulars_from_txt -from otaclient.app.proto.wrapper import RegularInf from tests.conftest import ThreadpoolExecutorFixtureMixin, cfg logger = logging.getLogger(__name__) diff --git a/tests/test_otaclient/__init__.py b/tests/test_otaclient/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_boot_control/__init__.py b/tests/test_otaclient/test_boot_control/__init__.py similarity index 100% rename from tests/test_boot_control/__init__.py rename to tests/test_otaclient/test_boot_control/__init__.py diff --git a/tests/test_boot_control/default_grub b/tests/test_otaclient/test_boot_control/default_grub similarity index 100% rename from tests/test_boot_control/default_grub rename to tests/test_otaclient/test_boot_control/default_grub diff --git a/tests/test_boot_control/extlinux.conf_slot_a b/tests/test_otaclient/test_boot_control/extlinux.conf_slot_a similarity index 100% rename from tests/test_boot_control/extlinux.conf_slot_a rename to tests/test_otaclient/test_boot_control/extlinux.conf_slot_a diff --git a/tests/test_boot_control/extlinux.conf_slot_b b/tests/test_otaclient/test_boot_control/extlinux.conf_slot_b similarity index 100% rename from tests/test_boot_control/extlinux.conf_slot_b rename to tests/test_otaclient/test_boot_control/extlinux.conf_slot_b diff --git a/tests/test_boot_control/fstab_origin b/tests/test_otaclient/test_boot_control/fstab_origin similarity index 100% rename from tests/test_boot_control/fstab_origin rename to tests/test_otaclient/test_boot_control/fstab_origin diff --git a/tests/test_boot_control/fstab_updated b/tests/test_otaclient/test_boot_control/fstab_updated similarity index 100% rename from tests/test_boot_control/fstab_updated rename to tests/test_otaclient/test_boot_control/fstab_updated diff --git a/tests/test_boot_control/grub.cfg_slot_a b/tests/test_otaclient/test_boot_control/grub.cfg_slot_a similarity index 100% rename from tests/test_boot_control/grub.cfg_slot_a rename to tests/test_otaclient/test_boot_control/grub.cfg_slot_a diff --git a/tests/test_boot_control/grub.cfg_slot_a_non_otapartition b/tests/test_otaclient/test_boot_control/grub.cfg_slot_a_non_otapartition similarity index 100% rename from tests/test_boot_control/grub.cfg_slot_a_non_otapartition rename to tests/test_otaclient/test_boot_control/grub.cfg_slot_a_non_otapartition diff --git a/tests/test_boot_control/grub.cfg_slot_a_updated b/tests/test_otaclient/test_boot_control/grub.cfg_slot_a_updated similarity index 100% rename from tests/test_boot_control/grub.cfg_slot_a_updated rename to tests/test_otaclient/test_boot_control/grub.cfg_slot_a_updated diff --git a/tests/test_boot_control/grub.cfg_slot_b b/tests/test_otaclient/test_boot_control/grub.cfg_slot_b similarity index 100% rename from tests/test_boot_control/grub.cfg_slot_b rename to tests/test_otaclient/test_boot_control/grub.cfg_slot_b diff --git a/tests/test_boot_control/grub.cfg_slot_b_updated b/tests/test_otaclient/test_boot_control/grub.cfg_slot_b_updated similarity index 100% rename from tests/test_boot_control/grub.cfg_slot_b_updated rename to tests/test_otaclient/test_boot_control/grub.cfg_slot_b_updated diff --git a/tests/test_boot_control/test_grub.py b/tests/test_otaclient/test_boot_control/test_grub.py similarity index 98% rename from tests/test_boot_control/test_grub.py rename to tests/test_otaclient/test_boot_control/test_grub.py index b8560e837..8a3cd1401 100644 --- a/tests/test_boot_control/test_grub.py +++ b/tests/test_otaclient/test_boot_control/test_grub.py @@ -22,7 +22,7 @@ import pytest import pytest_mock -from otaclient.app.proto import wrapper +from otaclient_api.v2 import types as api_types from tests.conftest import TestConfiguration as cfg from tests.utils import SlotMeta @@ -323,7 +323,7 @@ def test_grub_normal_update(self, mocker: pytest_mock.MockerFixture): grub_controller = GrubController() assert ( self.slot_a_ota_partition_dir / "status" - ).read_text() == wrapper.StatusOta.INITIALIZED.name + ).read_text() == api_types.StatusOta.INITIALIZED.name # assert ota-partition file points to slot_a ota-partition folder assert ( os.readlink(self.boot_dir / cfg.OTA_PARTITION_DIRNAME) @@ -342,10 +342,10 @@ def test_grub_normal_update(self, mocker: pytest_mock.MockerFixture): # update slot_b, slot_a_ota_status->FAILURE, slot_b_ota_status->UPDATING assert ( self.slot_a_ota_partition_dir / "status" - ).read_text() == wrapper.StatusOta.FAILURE.name + ).read_text() == api_types.StatusOta.FAILURE.name assert ( self.slot_b_ota_partition_dir / "status" - ).read_text() == wrapper.StatusOta.UPDATING.name + ).read_text() == api_types.StatusOta.UPDATING.name # NOTE: we have to copy the new kernel files to the slot_b's boot dir # this is done by the create_standby module _kernel = f"{cfg.KERNEL_PREFIX}-{cfg.KERNEL_VERSION}" @@ -385,7 +385,7 @@ def test_grub_normal_update(self, mocker: pytest_mock.MockerFixture): assert self._fsm.is_boot_switched assert ( self.slot_b_ota_partition_dir / "status" - ).read_text() == wrapper.StatusOta.UPDATING.name + ).read_text() == api_types.StatusOta.UPDATING.name # assert ota-partition file is not yet switched before first reboot init assert ( os.readlink(self.boot_dir / cfg.OTA_PARTITION_DIRNAME) @@ -401,7 +401,7 @@ def test_grub_normal_update(self, mocker: pytest_mock.MockerFixture): ) assert ( self.slot_b_ota_partition_dir / "status" - ).read_text() == wrapper.StatusOta.SUCCESS.name + ).read_text() == api_types.StatusOta.SUCCESS.name assert ( self.slot_b_ota_partition_dir / "version" ).read_text() == cfg.UPDATE_VERSION diff --git a/tests/test_boot_control/test_jetson_cboot.py b/tests/test_otaclient/test_boot_control/test_jetson_cboot.py similarity index 95% rename from tests/test_boot_control/test_jetson_cboot.py rename to tests/test_otaclient/test_boot_control/test_jetson_cboot.py index 8183eb57f..937c6b677 100644 --- a/tests/test_boot_control/test_jetson_cboot.py +++ b/tests/test_otaclient/test_boot_control/test_jetson_cboot.py @@ -33,11 +33,12 @@ parse_bsp_version, update_extlinux_cfg, ) +from tests.conftest import TEST_DIR logger = logging.getLogger(__name__) MODULE_NAME = _jetson_cboot.__name__ -TEST_DIR = Path(__file__).parent.parent / "data" +TEST_DATA_DIR = TEST_DIR / "data" def test_SlotID(): @@ -139,6 +140,6 @@ def test_parse_bsp_version(_in: str, expected: BSPVersion): ), ) def test_update_extlinux_conf(_template_f: Path, _updated_f: Path, partuuid: str): - _in = (TEST_DIR / _template_f).read_text() - _expected = (TEST_DIR / _updated_f).read_text() + _in = (TEST_DATA_DIR / _template_f).read_text() + _expected = (TEST_DATA_DIR / _updated_f).read_text() assert update_extlinux_cfg(_in, partuuid) == _expected diff --git a/tests/test_boot_control/test_ota_status_control.py b/tests/test_otaclient/test_boot_control/test_ota_status_control.py similarity index 86% rename from tests/test_boot_control/test_ota_status_control.py rename to tests/test_otaclient/test_boot_control/test_ota_status_control.py index 053ad88d5..6a3d2b6a6 100644 --- a/tests/test_boot_control/test_ota_status_control.py +++ b/tests/test_otaclient/test_boot_control/test_ota_status_control.py @@ -23,8 +23,8 @@ from otaclient.app.boot_control._common import OTAStatusFilesControl from otaclient.app.boot_control.configs import BaseConfig as cfg -from otaclient.app.common import read_str_from_file, write_str_to_file -from otaclient.app.proto import wrapper +from otaclient_api.v2 import types as api_types +from otaclient_common.common import read_str_from_file, write_str_to_file logger = logging.getLogger(__name__) @@ -74,27 +74,27 @@ def setup(self, tmp_path: Path): "", False, # output - wrapper.StatusOta.INITIALIZED, + api_types.StatusOta.INITIALIZED, SLOT_A_ID, ), ( "test_force_initialize", # input - wrapper.StatusOta.SUCCESS, + api_types.StatusOta.SUCCESS, SLOT_A_ID, True, # output - wrapper.StatusOta.INITIALIZED, + api_types.StatusOta.INITIALIZED, SLOT_A_ID, ), ( "test_normal_boot", # input - wrapper.StatusOta.SUCCESS, + api_types.StatusOta.SUCCESS, SLOT_A_ID, False, # output - wrapper.StatusOta.SUCCESS, + api_types.StatusOta.SUCCESS, SLOT_A_ID, ), ), @@ -102,10 +102,10 @@ def setup(self, tmp_path: Path): def test_ota_status_files_loading( self, test_case: str, - input_slot_a_status: Optional[wrapper.StatusOta], + input_slot_a_status: Optional[api_types.StatusOta], input_slot_a_slot_in_use: str, force_initialize: bool, - output_slot_a_status: wrapper.StatusOta, + output_slot_a_status: api_types.StatusOta, output_slot_a_slot_in_use: str, ): logger.info(f"{test_case=}") @@ -158,7 +158,7 @@ def test_pre_update(self): # slot_a: current slot assert ( read_str_from_file(self.slot_a_status_file) - == wrapper.StatusOta.FAILURE.name + == api_types.StatusOta.FAILURE.name ) assert ( read_str_from_file(self.slot_a_slot_in_use_file) @@ -168,7 +168,7 @@ def test_pre_update(self): # slot_b: standby slot assert ( read_str_from_file(self.slot_b_status_file) - == wrapper.StatusOta.UPDATING.name + == api_types.StatusOta.UPDATING.name ) assert read_str_from_file(self.slot_b_slot_in_use_file) == self.slot_b @@ -193,9 +193,9 @@ def test_switching_boot( """First reboot after OTA from slot_a to slot_b.""" logger.info(f"{test_case=}") # ------ setup ------ # - write_str_to_file(self.slot_a_status_file, wrapper.StatusOta.FAILURE.name) + write_str_to_file(self.slot_a_status_file, api_types.StatusOta.FAILURE.name) write_str_to_file(self.slot_a_slot_in_use_file, self.slot_b) - write_str_to_file(self.slot_b_status_file, wrapper.StatusOta.UPDATING.name) + write_str_to_file(self.slot_b_status_file, api_types.StatusOta.UPDATING.name) write_str_to_file(self.slot_b_slot_in_use_file, self.slot_b) # ------ execution ------ # @@ -218,7 +218,7 @@ def test_switching_boot( # check slot a assert ( read_str_from_file(self.slot_a_status_file) - == wrapper.StatusOta.FAILURE.name + == api_types.StatusOta.FAILURE.name ) assert ( read_str_from_file(self.slot_a_slot_in_use_file) @@ -233,25 +233,25 @@ def test_switching_boot( # finalizing succeeded if finalizing_result: - assert status_control.booted_ota_status == wrapper.StatusOta.SUCCESS + assert status_control.booted_ota_status == api_types.StatusOta.SUCCESS assert ( read_str_from_file(self.slot_b_status_file) - == wrapper.StatusOta.SUCCESS.name + == api_types.StatusOta.SUCCESS.name ) else: - assert status_control.booted_ota_status == wrapper.StatusOta.FAILURE + assert status_control.booted_ota_status == api_types.StatusOta.FAILURE assert ( read_str_from_file(self.slot_b_status_file) - == wrapper.StatusOta.FAILURE.name + == api_types.StatusOta.FAILURE.name ) def test_accidentally_boots_back_to_standby(self): """slot_a should be active slot but boots back to slot_b.""" # ------ setup ------ # - write_str_to_file(self.slot_a_status_file, wrapper.StatusOta.SUCCESS.name) + write_str_to_file(self.slot_a_status_file, api_types.StatusOta.SUCCESS.name) write_str_to_file(self.slot_a_slot_in_use_file, self.slot_a) - write_str_to_file(self.slot_b_status_file, wrapper.StatusOta.FAILURE.name) + write_str_to_file(self.slot_b_status_file, api_types.StatusOta.FAILURE.name) write_str_to_file(self.slot_b_slot_in_use_file, self.slot_a) # ------ execution ------ # @@ -268,4 +268,4 @@ def test_accidentally_boots_back_to_standby(self): # ------ assertion ------ # assert not self.finalize_switch_boot_flag.is_set() # slot_b's status is read - assert status_control.booted_ota_status == wrapper.StatusOta.FAILURE + assert status_control.booted_ota_status == api_types.StatusOta.FAILURE diff --git a/tests/test_boot_control/test_rpi_boot.py b/tests/test_otaclient/test_boot_control/test_rpi_boot.py similarity index 97% rename from tests/test_boot_control/test_rpi_boot.py rename to tests/test_otaclient/test_boot_control/test_rpi_boot.py index 59164681d..f4b5b943e 100644 --- a/tests/test_boot_control/test_rpi_boot.py +++ b/tests/test_otaclient/test_boot_control/test_rpi_boot.py @@ -10,7 +10,7 @@ from otaclient.app.boot_control._rpi_boot import _FSTAB_TEMPLATE_STR from otaclient.app.boot_control.configs import rpi_boot_cfg -from otaclient.app.proto import wrapper +from otaclient_api.v2 import types as api_types from tests.conftest import TestConfiguration as cfg from tests.utils import SlotMeta @@ -223,10 +223,10 @@ def test_rpi_boot_normal_update(self, mocker: pytest_mock.MockerFixture): # 2. make sure the mount points are prepared assert ( self.slot_a_ota_status_dir / "status" - ).read_text() == wrapper.StatusOta.FAILURE.name + ).read_text() == api_types.StatusOta.FAILURE.name assert ( self.slot_b_ota_status_dir / "status" - ).read_text() == wrapper.StatusOta.UPDATING.name + ).read_text() == api_types.StatusOta.UPDATING.name assert ( (self.slot_a_ota_status_dir / "slot_in_use").read_text() == (self.slot_b_ota_status_dir / "slot_in_use").read_text() @@ -309,7 +309,7 @@ def test_rpi_boot_normal_update(self, mocker: pytest_mock.MockerFixture): assert (self.system_boot / rpi_boot_cfg.SWITCH_BOOT_FLAG_FILE).is_file() assert ( self.slot_b_ota_status_dir / rpi_boot_cfg.OTA_STATUS_FNAME - ).read_text() == wrapper.StatusOta.UPDATING.name + ).read_text() == api_types.StatusOta.UPDATING.name # ------ boot_controller_inst3.stage1: second reboot, apply updated firmware and finish up ota update ------ # logger.info("2nd reboot: finish up ota update....") @@ -320,11 +320,12 @@ def test_rpi_boot_normal_update(self, mocker: pytest_mock.MockerFixture): # 2. make sure the flag file is cleared # 3. make sure the config.txt is still for slot_b assert ( - rpi_boot_controller4_2.get_booted_ota_status() == wrapper.StatusOta.SUCCESS + rpi_boot_controller4_2.get_booted_ota_status() + == api_types.StatusOta.SUCCESS ) assert ( self.slot_b_ota_status_dir / rpi_boot_cfg.OTA_STATUS_FNAME - ).read_text() == wrapper.StatusOta.SUCCESS.name + ).read_text() == api_types.StatusOta.SUCCESS.name assert not (self.system_boot / rpi_boot_cfg.SWITCH_BOOT_FLAG_FILE).is_file() assert ( rpi_boot_controller4_2._ota_status_control._load_current_slot_in_use() diff --git a/tests/test_ecu_info.py b/tests/test_otaclient/test_configs/test_ecu_info.py similarity index 100% rename from tests/test_ecu_info.py rename to tests/test_otaclient/test_configs/test_ecu_info.py diff --git a/tests/test_proxy_info.py b/tests/test_otaclient/test_configs/test_proxy_info.py similarity index 100% rename from tests/test_proxy_info.py rename to tests/test_otaclient/test_configs/test_proxy_info.py diff --git a/tests/test_create_standby.py b/tests/test_otaclient/test_create_standby.py similarity index 99% rename from tests/test_create_standby.py rename to tests/test_otaclient/test_create_standby.py index f4845129c..5b3156e1b 100644 --- a/tests/test_create_standby.py +++ b/tests/test_otaclient/test_create_standby.py @@ -78,7 +78,6 @@ def mock_setup(self, mocker: MockerFixture, prepare_ab_slots): _cfg.RUN_DIR = str(self.otaclient_run_dir) # type: ignore mocker.patch(f"{cfg.OTACLIENT_MODULE_PATH}.cfg", _cfg) mocker.patch(f"{cfg.CREATE_STANDBY_MODULE_PATH}.rebuild_mode.cfg", _cfg) - mocker.patch(f"{cfg.OTAMETA_MODULE_PATH}.cfg", _cfg) def test_update_with_create_standby_RebuildMode(self, mocker: MockerFixture): from otaclient.app.create_standby.rebuild_mode import RebuildMode diff --git a/tests/test_log_setting.py b/tests/test_otaclient/test_log_setting.py similarity index 100% rename from tests/test_log_setting.py rename to tests/test_otaclient/test_log_setting.py diff --git a/tests/test_main.py b/tests/test_otaclient/test_main.py similarity index 100% rename from tests/test_main.py rename to tests/test_otaclient/test_main.py diff --git a/tests/test_ota_client.py b/tests/test_otaclient/test_ota_client.py similarity index 90% rename from tests/test_ota_client.py rename to tests/test_otaclient/test_ota_client.py index 5496d647a..0fa260243 100644 --- a/tests/test_ota_client.py +++ b/tests/test_otaclient/test_ota_client.py @@ -13,6 +13,8 @@ # limitations under the License. +from __future__ import annotations + import asyncio import shutil import threading @@ -25,6 +27,8 @@ import pytest import pytest_mock +from ota_metadata.legacy.parser import parse_dirs_from_txt, parse_regulars_from_txt +from ota_metadata.legacy.types import DirectoryInf, RegularInf from otaclient.app.boot_control import BootControllerProtocol from otaclient.app.boot_control.configs import BootloaderType from otaclient.app.configs import config as otaclient_cfg @@ -37,10 +41,8 @@ OTAServicer, _OTAUpdater, ) -from otaclient.app.ota_metadata import parse_dirs_from_txt, parse_regulars_from_txt -from otaclient.app.proto import wrapper -from otaclient.app.proto.wrapper import DirectoryInf, RegularInf from otaclient.configs.ecu_info import ECUInfo +from otaclient_api.v2 import types as api_types from tests.conftest import TestConfiguration as cfg from tests.utils import SlotMeta @@ -148,7 +150,6 @@ def mock_setup(self, mocker: pytest_mock.MockerFixture, _delta_generate): _cfg.ACTIVE_ROOTFS_PATH = str(self.slot_a) # type: ignore _cfg.RUN_DIR = str(self.otaclient_run_dir) # type: ignore mocker.patch(f"{cfg.OTACLIENT_MODULE_PATH}.cfg", _cfg) - mocker.patch(f"{cfg.OTAMETA_MODULE_PATH}.cfg", _cfg) # ------ mock stats collector ------ # mocker.patch( @@ -201,7 +202,7 @@ class Test_OTAClient: CURRENT_FIRMWARE_VERSION = "firmware_version" UPDATE_FIRMWARE_VERSION = "update_firmware_version" - MOCKED_STATUS_PROGRESS = wrapper.UpdateStatus( + MOCKED_STATUS_PROGRESS = api_types.UpdateStatus( update_firmware_version=UPDATE_FIRMWARE_VERSION, downloaded_bytes=456789, downloaded_files_num=567, @@ -233,7 +234,7 @@ def mock_setup(self, mocker: pytest_mock.MockerFixture): # patch boot_controller for otaclient initializing self.boot_controller.load_version.return_value = self.CURRENT_FIRMWARE_VERSION self.boot_controller.get_booted_ota_status.return_value = ( - wrapper.StatusOta.SUCCESS + api_types.StatusOta.SUCCESS ) self.ota_client = OTAClient( @@ -270,7 +271,7 @@ def test_update_normal_finished(self): self.ota_lock.release.assert_called_once() assert ( self.ota_client.live_ota_status.get_ota_status() - == wrapper.StatusOta.UPDATING + == api_types.StatusOta.UPDATING ) def test_update_interrupted(self): @@ -296,9 +297,9 @@ def test_update_interrupted(self): assert ( self.ota_client.live_ota_status.get_ota_status() - == wrapper.StatusOta.FAILURE + == api_types.StatusOta.FAILURE ) - assert self.ota_client.last_failure_type == wrapper.FailureType.RECOVERABLE + assert self.ota_client.last_failure_type == api_types.FailureType.RECOVERABLE def test_rollback(self): # TODO @@ -309,49 +310,49 @@ def test_status_not_in_update(self): _status = self.ota_client.status() # assert v2 to v1 conversion - assert _status.convert_to_v1() == wrapper.StatusResponseEcu( + assert _status.convert_to_v1() == api_types.StatusResponseEcu( ecu_id=self.MY_ECU_ID, - result=wrapper.FailureType.NO_FAILURE, - status=wrapper.Status( + result=api_types.FailureType.NO_FAILURE, + status=api_types.Status( version=self.CURRENT_FIRMWARE_VERSION, - status=wrapper.StatusOta.SUCCESS, + status=api_types.StatusOta.SUCCESS, ), ) # assert to v2 - assert _status == wrapper.StatusResponseEcuV2( + assert _status == api_types.StatusResponseEcuV2( ecu_id=self.MY_ECU_ID, otaclient_version=self.OTACLIENT_VERSION, firmware_version=self.CURRENT_FIRMWARE_VERSION, - failure_type=wrapper.FailureType.NO_FAILURE, - ota_status=wrapper.StatusOta.SUCCESS, + failure_type=api_types.FailureType.NO_FAILURE, + ota_status=api_types.StatusOta.SUCCESS, ) def test_status_in_update(self): # --- mock setup --- # # inject ota_updater and set ota_status to UPDATING to simulate ota updating self.ota_client._update_executor = self.ota_updater - self.ota_client.live_ota_status.set_ota_status(wrapper.StatusOta.UPDATING) + self.ota_client.live_ota_status.set_ota_status(api_types.StatusOta.UPDATING) # let mocked updater return mocked_status_progress self.ota_updater.get_update_status.return_value = self.MOCKED_STATUS_PROGRESS # --- assertion --- # _status = self.ota_client.status() # test v2 to v1 conversion - assert _status.convert_to_v1() == wrapper.StatusResponseEcu( + assert _status.convert_to_v1() == api_types.StatusResponseEcu( ecu_id=self.MY_ECU_ID, - result=wrapper.FailureType.NO_FAILURE, - status=wrapper.Status( + result=api_types.FailureType.NO_FAILURE, + status=api_types.Status( version=self.CURRENT_FIRMWARE_VERSION, - status=wrapper.StatusOta.UPDATING, + status=api_types.StatusOta.UPDATING, progress=self.MOCKED_STATUS_PROGRESS_V1, ), ) # assert to v2 - assert _status == wrapper.StatusResponseEcuV2( + assert _status == api_types.StatusResponseEcuV2( ecu_id=self.MY_ECU_ID, otaclient_version=self.OTACLIENT_VERSION, - failure_type=wrapper.FailureType.NO_FAILURE, - ota_status=wrapper.StatusOta.UPDATING, + failure_type=api_types.FailureType.NO_FAILURE, + ota_status=api_types.StatusOta.UPDATING, firmware_version=self.CURRENT_FIRMWARE_VERSION, update_status=self.MOCKED_STATUS_PROGRESS, ) @@ -423,7 +424,7 @@ def test_stub_initializing(self): assert self.otaclient_stub.local_used_proxy_url is self.local_use_proxy async def test_dispatch_update(self): - update_request_ecu = wrapper.UpdateRequestEcu( + update_request_ecu = api_types.UpdateRequestEcu( ecu_id=self.ECU_INFO.ecu_id, version="version", url="url", @@ -442,11 +443,11 @@ def _updating(*args, **kwargs): await self.otaclient_stub.dispatch_update(update_request_ecu) await asyncio.sleep(0.1) # wait for inner async closure to run - assert self.otaclient_stub.last_operation is wrapper.StatusOta.UPDATING + assert self.otaclient_stub.last_operation is api_types.StatusOta.UPDATING assert self.otaclient_stub.is_busy # test ota update/rollback exclusive lock, resp = await self.otaclient_stub.dispatch_update(update_request_ecu) - assert resp.result == wrapper.FailureType.RECOVERABLE + assert resp.result == api_types.FailureType.RECOVERABLE # finish up update _updating_event.set() @@ -469,14 +470,16 @@ def _rollbacking(*args, **kwargs): self.otaclient.rollback.side_effect = _rollbacking # dispatch rollback - await self.otaclient_stub.dispatch_rollback(wrapper.RollbackRequestEcu()) + await self.otaclient_stub.dispatch_rollback(api_types.RollbackRequestEcu()) await asyncio.sleep(0.1) # wait for inner async closure to run - assert self.otaclient_stub.last_operation is wrapper.StatusOta.ROLLBACKING + assert self.otaclient_stub.last_operation is api_types.StatusOta.ROLLBACKING assert self.otaclient_stub.is_busy # test ota update/rollback exclusive lock, - resp = await self.otaclient_stub.dispatch_rollback(wrapper.RollbackRequestEcu()) - assert resp.result == wrapper.FailureType.RECOVERABLE + resp = await self.otaclient_stub.dispatch_rollback( + api_types.RollbackRequestEcu() + ) + assert resp.result == api_types.FailureType.RECOVERABLE # finish up rollback _rollbacking_event.set() diff --git a/tests/test_ota_client_service.py b/tests/test_otaclient/test_ota_client_service.py similarity index 72% rename from tests/test_ota_client_service.py rename to tests/test_otaclient/test_ota_client_service.py index 6b7c7776a..ee1ca15cc 100644 --- a/tests/test_ota_client_service.py +++ b/tests/test_otaclient/test_ota_client_service.py @@ -21,34 +21,36 @@ import pytest_mock from otaclient.app.configs import server_cfg -from otaclient.app.ota_client_call import OtaClientCall -from otaclient.app.ota_client_service import create_otaclient_grpc_server -from otaclient.app.proto import wrapper +from otaclient.app.main import create_otaclient_grpc_server from otaclient.configs.ecu_info import ECUInfo +from otaclient_api.v2 import types as api_types +from otaclient_api.v2.api_caller import OTAClientCall from tests.conftest import cfg from tests.utils import compare_message +OTACLIENT_APP_MAIN = "otaclient.app.main" + class _MockedOTAClientServiceStub: MY_ECU_ID = "autoware" - UPDATE_RESP_ECU = wrapper.UpdateResponseEcu( + UPDATE_RESP_ECU = api_types.UpdateResponseEcu( ecu_id=MY_ECU_ID, - result=wrapper.FailureType.NO_FAILURE, + result=api_types.FailureType.NO_FAILURE, ) - UPDATE_RESP = wrapper.UpdateResponse(ecu=[UPDATE_RESP_ECU]) - ROLLBACK_RESP_ECU = wrapper.RollbackResponseEcu( + UPDATE_RESP = api_types.UpdateResponse(ecu=[UPDATE_RESP_ECU]) + ROLLBACK_RESP_ECU = api_types.RollbackResponseEcu( ecu_id=MY_ECU_ID, - result=wrapper.FailureType.NO_FAILURE, + result=api_types.FailureType.NO_FAILURE, ) - ROLLBACK_RESP = wrapper.RollbackResponse(ecu=[ROLLBACK_RESP_ECU]) - STATUS_RESP_ECU = wrapper.StatusResponseEcuV2( + ROLLBACK_RESP = api_types.RollbackResponse(ecu=[ROLLBACK_RESP_ECU]) + STATUS_RESP_ECU = api_types.StatusResponseEcuV2( ecu_id=MY_ECU_ID, otaclient_version="mocked_otaclient", firmware_version="firmware", - ota_status=wrapper.StatusOta.SUCCESS, - failure_type=wrapper.FailureType.NO_FAILURE, + ota_status=api_types.StatusOta.SUCCESS, + failure_type=api_types.FailureType.NO_FAILURE, ) - STATUS_RESP = wrapper.StatusResponse( + STATUS_RESP = api_types.StatusResponse( available_ecu_ids=[MY_ECU_ID], ecu_v2=[STATUS_RESP_ECU] ) @@ -71,7 +73,7 @@ class Test_ota_client_service: def setup_test(self, mocker: pytest_mock.MockerFixture): self.otaclient_service_stub = _MockedOTAClientServiceStub() mocker.patch( - f"{cfg.OTACLIENT_SERVICE_MODULE_PATH}.OTAClientServiceStub", + f"{OTACLIENT_APP_MAIN}.OTAClientServiceStub", return_value=self.otaclient_service_stub, ) @@ -79,7 +81,7 @@ def setup_test(self, mocker: pytest_mock.MockerFixture): ecu_id=self.otaclient_service_stub.MY_ECU_ID, ip_addr=self.LISTEN_ADDR, # type: ignore ) - mocker.patch(f"{cfg.OTACLIENT_SERVICE_MODULE_PATH}.ecu_info", ecu_info_mock) + mocker.patch(f"{OTACLIENT_APP_MAIN}.ecu_info", ecu_info_mock) @pytest.fixture(autouse=True) async def launch_otaclient_server(self, setup_test): @@ -93,28 +95,28 @@ async def launch_otaclient_server(self, setup_test): async def test_otaclient_service(self): # --- test update call --- # - update_resp = await OtaClientCall.update_call( + update_resp = await OTAClientCall.update_call( ecu_id=self.MY_ECU_ID, ecu_ipaddr=self.LISTEN_ADDR, ecu_port=self.LISTEN_PORT, - request=wrapper.UpdateRequest(), + request=api_types.UpdateRequest(), ) compare_message(update_resp, self.otaclient_service_stub.UPDATE_RESP) # --- test rollback call --- # - rollback_resp = await OtaClientCall.rollback_call( + rollback_resp = await OTAClientCall.rollback_call( ecu_id=self.MY_ECU_ID, ecu_ipaddr=self.LISTEN_ADDR, ecu_port=self.LISTEN_PORT, - request=wrapper.RollbackRequest(), + request=api_types.RollbackRequest(), ) compare_message(rollback_resp, self.otaclient_service_stub.ROLLBACK_RESP) # --- test status call --- # - status_resp = await OtaClientCall.status_call( + status_resp = await OTAClientCall.status_call( ecu_id=self.MY_ECU_ID, ecu_ipaddr=self.LISTEN_ADDR, ecu_port=self.LISTEN_PORT, - request=wrapper.StatusRequest(), + request=api_types.StatusRequest(), ) compare_message(status_resp, self.otaclient_service_stub.STATUS_RESP) diff --git a/tests/test_ota_client_stub.py b/tests/test_otaclient/test_ota_client_stub.py similarity index 73% rename from tests/test_ota_client_stub.py rename to tests/test_otaclient/test_ota_client_stub.py index 77734df46..3c57cc7bb 100644 --- a/tests/test_ota_client_stub.py +++ b/tests/test_otaclient/test_ota_client_stub.py @@ -27,15 +27,15 @@ from ota_proxy import OTAProxyContextProto from ota_proxy.config import Config as otaproxyConfig from otaclient.app.ota_client import OTAServicer -from otaclient.app.ota_client_call import OtaClientCall from otaclient.app.ota_client_stub import ( ECUStatusStorage, OTAClientServiceStub, OTAProxyLauncher, ) -from otaclient.app.proto import wrapper from otaclient.configs.ecu_info import ECUInfo, parse_ecu_info from otaclient.configs.proxy_info import ProxyInfo, parse_proxy_info +from otaclient_api.v2 import types as api_types +from otaclient_api.v2.api_caller import OTAClientCall from tests.conftest import cfg from tests.utils import compare_message @@ -184,33 +184,33 @@ async def setup_test(self, mocker: MockerFixture, ecu_info_fixture): # case 1 ( # local ECU's status report - wrapper.StatusResponseEcuV2( + api_types.StatusResponseEcuV2( ecu_id="autoware", - ota_status=wrapper.StatusOta.SUCCESS, + ota_status=api_types.StatusOta.SUCCESS, firmware_version="123.x", - failure_type=wrapper.FailureType.NO_FAILURE, + failure_type=api_types.FailureType.NO_FAILURE, ), # sub ECU's status report [ - wrapper.StatusResponse( + api_types.StatusResponse( available_ecu_ids=["p1"], ecu_v2=[ - wrapper.StatusResponseEcuV2( + api_types.StatusResponseEcuV2( ecu_id="p1", - ota_status=wrapper.StatusOta.SUCCESS, + ota_status=api_types.StatusOta.SUCCESS, firmware_version="123.x", - failure_type=wrapper.FailureType.NO_FAILURE, + failure_type=api_types.FailureType.NO_FAILURE, ) ], ), - wrapper.StatusResponse( + api_types.StatusResponse( available_ecu_ids=["p2"], ecu=[ - wrapper.StatusResponseEcu( + api_types.StatusResponseEcu( ecu_id="p2", - result=wrapper.FailureType.NO_FAILURE, - status=wrapper.Status( - status=wrapper.StatusOta.SUCCESS, + result=api_types.FailureType.NO_FAILURE, + status=api_types.Status( + status=api_types.StatusOta.SUCCESS, version="123.x", ), ), @@ -218,46 +218,46 @@ async def setup_test(self, mocker: MockerFixture, ecu_info_fixture): ), ], # expected export - wrapper.StatusResponse( + api_types.StatusResponse( available_ecu_ids=["autoware", "p1", "p2"], # explicitly v1 format compatibility ecu=[ - wrapper.StatusResponseEcu( + api_types.StatusResponseEcu( ecu_id="autoware", - result=wrapper.FailureType.NO_FAILURE, - status=wrapper.Status( - status=wrapper.StatusOta.SUCCESS, + result=api_types.FailureType.NO_FAILURE, + status=api_types.Status( + status=api_types.StatusOta.SUCCESS, version="123.x", ), ), - wrapper.StatusResponseEcu( + api_types.StatusResponseEcu( ecu_id="p1", - result=wrapper.FailureType.NO_FAILURE, - status=wrapper.Status( - status=wrapper.StatusOta.SUCCESS, + result=api_types.FailureType.NO_FAILURE, + status=api_types.Status( + status=api_types.StatusOta.SUCCESS, version="123.x", ), ), - wrapper.StatusResponseEcu( + api_types.StatusResponseEcu( ecu_id="p2", - result=wrapper.FailureType.NO_FAILURE, - status=wrapper.Status( - status=wrapper.StatusOta.SUCCESS, + result=api_types.FailureType.NO_FAILURE, + status=api_types.Status( + status=api_types.StatusOta.SUCCESS, version="123.x", ), ), ], ecu_v2=[ - wrapper.StatusResponseEcuV2( + api_types.StatusResponseEcuV2( ecu_id="autoware", - ota_status=wrapper.StatusOta.SUCCESS, - failure_type=wrapper.FailureType.NO_FAILURE, + ota_status=api_types.StatusOta.SUCCESS, + failure_type=api_types.FailureType.NO_FAILURE, firmware_version="123.x", ), - wrapper.StatusResponseEcuV2( + api_types.StatusResponseEcuV2( ecu_id="p1", - ota_status=wrapper.StatusOta.SUCCESS, - failure_type=wrapper.FailureType.NO_FAILURE, + ota_status=api_types.StatusOta.SUCCESS, + failure_type=api_types.FailureType.NO_FAILURE, firmware_version="123.x", ), ], @@ -266,15 +266,15 @@ async def setup_test(self, mocker: MockerFixture, ecu_info_fixture): # case 2 ( # local ecu status report - wrapper.StatusResponseEcuV2( + api_types.StatusResponseEcuV2( ecu_id="autoware", - ota_status=wrapper.StatusOta.UPDATING, + ota_status=api_types.StatusOta.UPDATING, firmware_version="123.x", - failure_type=wrapper.FailureType.NO_FAILURE, - update_status=wrapper.UpdateStatus( + failure_type=api_types.FailureType.NO_FAILURE, + update_status=api_types.UpdateStatus( update_firmware_version="789.x", - phase=wrapper.UpdatePhase.DOWNLOADING_OTA_FILES, - total_elapsed_time=wrapper.Duration(seconds=123), + phase=api_types.UpdatePhase.DOWNLOADING_OTA_FILES, + total_elapsed_time=api_types.Duration(seconds=123), total_files_num=123456, processed_files_num=123, processed_files_size=456, @@ -285,18 +285,18 @@ async def setup_test(self, mocker: MockerFixture, ecu_info_fixture): ), # sub ECUs' status report [ - wrapper.StatusResponse( + api_types.StatusResponse( available_ecu_ids=["p1"], ecu_v2=[ - wrapper.StatusResponseEcuV2( + api_types.StatusResponseEcuV2( ecu_id="p1", - ota_status=wrapper.StatusOta.UPDATING, + ota_status=api_types.StatusOta.UPDATING, firmware_version="123.x", - failure_type=wrapper.FailureType.NO_FAILURE, - update_status=wrapper.UpdateStatus( + failure_type=api_types.FailureType.NO_FAILURE, + update_status=api_types.UpdateStatus( update_firmware_version="789.x", - phase=wrapper.UpdatePhase.DOWNLOADING_OTA_FILES, - total_elapsed_time=wrapper.Duration(seconds=123), + phase=api_types.UpdatePhase.DOWNLOADING_OTA_FILES, + total_elapsed_time=api_types.Duration(seconds=123), total_files_num=123456, processed_files_num=123, processed_files_size=456, @@ -307,14 +307,14 @@ async def setup_test(self, mocker: MockerFixture, ecu_info_fixture): ) ], ), - wrapper.StatusResponse( + api_types.StatusResponse( available_ecu_ids=["p2"], ecu=[ - wrapper.StatusResponseEcu( + api_types.StatusResponseEcu( ecu_id="p2", - result=wrapper.FailureType.NO_FAILURE, - status=wrapper.Status( - status=wrapper.StatusOta.SUCCESS, + result=api_types.FailureType.NO_FAILURE, + status=api_types.Status( + status=api_types.StatusOta.SUCCESS, version="123.x", ), ), @@ -322,20 +322,20 @@ async def setup_test(self, mocker: MockerFixture, ecu_info_fixture): ), ], # expected export result - wrapper.StatusResponse( + api_types.StatusResponse( available_ecu_ids=["autoware", "p1", "p2"], # explicitly v1 format compatibility # NOTE: processed_files_num(v2) = files_processed_download(v1) + files_processed_copy(v1) - # check wrapper.UpdateStatus.convert_to_v1_StatusProgress for more details. + # check api_types.UpdateStatus.convert_to_v1_StatusProgress for more details. ecu=[ - wrapper.StatusResponseEcu( + api_types.StatusResponseEcu( ecu_id="autoware", - result=wrapper.FailureType.NO_FAILURE, - status=wrapper.Status( - status=wrapper.StatusOta.UPDATING, + result=api_types.FailureType.NO_FAILURE, + status=api_types.Status( + status=api_types.StatusOta.UPDATING, version="123.x", - progress=wrapper.StatusProgress( - phase=wrapper.StatusProgressPhase.REGULAR, + progress=api_types.StatusProgress( + phase=api_types.StatusProgressPhase.REGULAR, total_regular_files=123456, files_processed_download=100, file_size_processed_download=400, @@ -343,18 +343,18 @@ async def setup_test(self, mocker: MockerFixture, ecu_info_fixture): file_size_processed_copy=56, download_bytes=789, regular_files_processed=123, - total_elapsed_time=wrapper.Duration(seconds=123), + total_elapsed_time=api_types.Duration(seconds=123), ), ), ), - wrapper.StatusResponseEcu( + api_types.StatusResponseEcu( ecu_id="p1", - result=wrapper.FailureType.NO_FAILURE, - status=wrapper.Status( - status=wrapper.StatusOta.UPDATING, + result=api_types.FailureType.NO_FAILURE, + status=api_types.Status( + status=api_types.StatusOta.UPDATING, version="123.x", - progress=wrapper.StatusProgress( - phase=wrapper.StatusProgressPhase.REGULAR, + progress=api_types.StatusProgress( + phase=api_types.StatusProgressPhase.REGULAR, total_regular_files=123456, files_processed_download=100, file_size_processed_download=400, @@ -362,29 +362,29 @@ async def setup_test(self, mocker: MockerFixture, ecu_info_fixture): file_size_processed_copy=56, download_bytes=789, regular_files_processed=123, - total_elapsed_time=wrapper.Duration(seconds=123), + total_elapsed_time=api_types.Duration(seconds=123), ), ), ), - wrapper.StatusResponseEcu( + api_types.StatusResponseEcu( ecu_id="p2", - result=wrapper.FailureType.NO_FAILURE, - status=wrapper.Status( + result=api_types.FailureType.NO_FAILURE, + status=api_types.Status( version="123.x", - status=wrapper.StatusOta.SUCCESS, + status=api_types.StatusOta.SUCCESS, ), ), ], ecu_v2=[ - wrapper.StatusResponseEcuV2( + api_types.StatusResponseEcuV2( ecu_id="autoware", - ota_status=wrapper.StatusOta.UPDATING, - failure_type=wrapper.FailureType.NO_FAILURE, + ota_status=api_types.StatusOta.UPDATING, + failure_type=api_types.FailureType.NO_FAILURE, firmware_version="123.x", - update_status=wrapper.UpdateStatus( + update_status=api_types.UpdateStatus( update_firmware_version="789.x", - phase=wrapper.UpdatePhase.DOWNLOADING_OTA_FILES, - total_elapsed_time=wrapper.Duration(seconds=123), + phase=api_types.UpdatePhase.DOWNLOADING_OTA_FILES, + total_elapsed_time=api_types.Duration(seconds=123), total_files_num=123456, processed_files_num=123, processed_files_size=456, @@ -393,15 +393,15 @@ async def setup_test(self, mocker: MockerFixture, ecu_info_fixture): downloaded_files_size=400, ), ), - wrapper.StatusResponseEcuV2( + api_types.StatusResponseEcuV2( ecu_id="p1", - ota_status=wrapper.StatusOta.UPDATING, - failure_type=wrapper.FailureType.NO_FAILURE, + ota_status=api_types.StatusOta.UPDATING, + failure_type=api_types.FailureType.NO_FAILURE, firmware_version="123.x", - update_status=wrapper.UpdateStatus( + update_status=api_types.UpdateStatus( update_firmware_version="789.x", - phase=wrapper.UpdatePhase.DOWNLOADING_OTA_FILES, - total_elapsed_time=wrapper.Duration(seconds=123), + phase=api_types.UpdatePhase.DOWNLOADING_OTA_FILES, + total_elapsed_time=api_types.Duration(seconds=123), total_files_num=123456, processed_files_num=123, processed_files_size=456, @@ -417,9 +417,9 @@ async def setup_test(self, mocker: MockerFixture, ecu_info_fixture): ) async def test_export( self, - local_ecu_status: wrapper.StatusResponseEcuV2, - sub_ecus_status: List[wrapper.StatusResponse], - expected: wrapper.StatusResponse, + local_ecu_status: api_types.StatusResponseEcuV2, + sub_ecus_status: List[api_types.StatusResponse], + expected: api_types.StatusResponse, ): # --- prepare --- # await self.ecu_storage.update_from_local_ecu(local_ecu_status) @@ -438,34 +438,34 @@ async def test_export( # case 1: ( # local ECU status: UPDATING, requires network - wrapper.StatusResponseEcuV2( + api_types.StatusResponseEcuV2( ecu_id="autoware", - ota_status=wrapper.StatusOta.UPDATING, - update_status=wrapper.UpdateStatus( - phase=wrapper.UpdatePhase.DOWNLOADING_OTA_FILES + ota_status=api_types.StatusOta.UPDATING, + update_status=api_types.UpdateStatus( + phase=api_types.UpdatePhase.DOWNLOADING_OTA_FILES ), ), # sub ECUs status [ - wrapper.StatusResponse( + api_types.StatusResponse( available_ecu_ids=["p1"], ecu_v2=[ - wrapper.StatusResponseEcuV2( + api_types.StatusResponseEcuV2( ecu_id="p1", - ota_status=wrapper.StatusOta.FAILURE, + ota_status=api_types.StatusOta.FAILURE, ), ], ), # p2: updating, doesn't require network - wrapper.StatusResponse( + api_types.StatusResponse( available_ecu_ids=["p2"], ecu=[ - wrapper.StatusResponseEcu( + api_types.StatusResponseEcu( ecu_id="p2", - status=wrapper.Status( - status=wrapper.StatusOta.UPDATING, - progress=wrapper.StatusProgress( - phase=wrapper.StatusProgressPhase.POST_PROCESSING, + status=api_types.Status( + status=api_types.StatusOta.UPDATING, + progress=api_types.StatusProgress( + phase=api_types.StatusProgressPhase.POST_PROCESSING, ), ), ) @@ -486,32 +486,32 @@ async def test_export( # case 2: ( # local ECU status: SUCCESS - wrapper.StatusResponseEcuV2( + api_types.StatusResponseEcuV2( ecu_id="autoware", - ota_status=wrapper.StatusOta.SUCCESS, + ota_status=api_types.StatusOta.SUCCESS, ), # sub ECUs status [ # p1: FAILURE - wrapper.StatusResponse( + api_types.StatusResponse( available_ecu_ids=["p1"], ecu_v2=[ - wrapper.StatusResponseEcuV2( + api_types.StatusResponseEcuV2( ecu_id="p1", - ota_status=wrapper.StatusOta.FAILURE, + ota_status=api_types.StatusOta.FAILURE, ), ], ), # p2: updating, requires network - wrapper.StatusResponse( + api_types.StatusResponse( available_ecu_ids=["p2"], ecu=[ - wrapper.StatusResponseEcu( + api_types.StatusResponseEcu( ecu_id="p2", - status=wrapper.Status( - status=wrapper.StatusOta.UPDATING, - progress=wrapper.StatusProgress( - phase=wrapper.StatusProgressPhase.REGULAR, + status=api_types.Status( + status=api_types.StatusOta.UPDATING, + progress=api_types.StatusProgress( + phase=api_types.StatusProgressPhase.REGULAR, ), ), ) @@ -533,8 +533,8 @@ async def test_export( ) async def test_overall_ecu_status_report_generation( self, - local_ecu_status: wrapper.StatusResponseEcuV2, - sub_ecus_status: List[wrapper.StatusResponse], + local_ecu_status: api_types.StatusResponseEcuV2, + sub_ecus_status: List[api_types.StatusResponse], properties_dict: Dict[str, Any], ): # --- prepare --- # @@ -559,32 +559,32 @@ async def test_overall_ecu_status_report_generation( # based on the status change of ECUs that accept update request. ( # local ECU status: FAILED - wrapper.StatusResponseEcuV2( + api_types.StatusResponseEcuV2( ecu_id="autoware", - ota_status=wrapper.StatusOta.FAILURE, + ota_status=api_types.StatusOta.FAILURE, ), # sub ECUs status [ # p1: FAILED - wrapper.StatusResponse( + api_types.StatusResponse( available_ecu_ids=["p1"], ecu_v2=[ - wrapper.StatusResponseEcuV2( + api_types.StatusResponseEcuV2( ecu_id="p1", - ota_status=wrapper.StatusOta.FAILURE, + ota_status=api_types.StatusOta.FAILURE, ), ], ), # p2: UPDATING - wrapper.StatusResponse( + api_types.StatusResponse( available_ecu_ids=["p2"], ecu=[ - wrapper.StatusResponseEcu( + api_types.StatusResponseEcu( ecu_id="p2", - status=wrapper.Status( - status=wrapper.StatusOta.UPDATING, - progress=wrapper.StatusProgress( - phase=wrapper.StatusProgressPhase.REGULAR, + status=api_types.Status( + status=api_types.StatusOta.UPDATING, + progress=api_types.StatusProgress( + phase=api_types.StatusProgressPhase.REGULAR, ), ), ) @@ -610,33 +610,33 @@ async def test_overall_ecu_status_report_generation( # based on the status change of ECUs that accept update request. ( # local ECU status: UPDATING - wrapper.StatusResponseEcuV2( + api_types.StatusResponseEcuV2( ecu_id="autoware", - ota_status=wrapper.StatusOta.UPDATING, - update_status=wrapper.UpdateStatus( - phase=wrapper.UpdatePhase.DOWNLOADING_OTA_FILES, + ota_status=api_types.StatusOta.UPDATING, + update_status=api_types.UpdateStatus( + phase=api_types.UpdatePhase.DOWNLOADING_OTA_FILES, ), ), # sub ECUs status [ # p1: FAILED - wrapper.StatusResponse( + api_types.StatusResponse( available_ecu_ids=["p1"], ecu_v2=[ - wrapper.StatusResponseEcuV2( + api_types.StatusResponseEcuV2( ecu_id="p1", - ota_status=wrapper.StatusOta.FAILURE, + ota_status=api_types.StatusOta.FAILURE, ), ], ), # p2: SUCCESS - wrapper.StatusResponse( + api_types.StatusResponse( available_ecu_ids=["p2"], ecu=[ - wrapper.StatusResponseEcu( + api_types.StatusResponseEcu( ecu_id="p2", - status=wrapper.Status( - status=wrapper.StatusOta.SUCCESS, + status=api_types.Status( + status=api_types.StatusOta.SUCCESS, ), ) ], @@ -658,8 +658,8 @@ async def test_overall_ecu_status_report_generation( ) async def test_on_receive_update_request( self, - local_ecu_status: wrapper.StatusResponseEcuV2, - sub_ecus_status: List[wrapper.StatusResponse], + local_ecu_status: api_types.StatusResponseEcuV2, + sub_ecus_status: List[api_types.StatusResponse], ecus_accept_update_request: List[str], properties_dict: Dict[str, Any], ): @@ -710,10 +710,10 @@ class TestOTAClientServiceStub: @staticmethod async def _subecu_accept_update_request(ecu_id, *args, **kwargs): - return wrapper.UpdateResponse( + return api_types.UpdateResponse( ecu=[ - wrapper.UpdateResponseEcu( - ecu_id=ecu_id, result=wrapper.FailureType.NO_FAILURE + api_types.UpdateResponseEcu( + ecu_id=ecu_id, result=api_types.FailureType.NO_FAILURE ) ] ) @@ -741,11 +741,11 @@ async def setup_test( await asyncio.sleep(self.ENSURE_NEXT_CHECKING_ROUND) # ensure the task stopping # --- mocker --- # - self.otaclient_wrapper = mocker.MagicMock(spec=OTAServicer) + self.otaclient_api_types = mocker.MagicMock(spec=OTAServicer) self.ecu_status_tracker = mocker.MagicMock() self.otaproxy_launcher = mocker.MagicMock(spec=OTAProxyLauncher) # mock OTAClientCall, make update_call return success on any update dispatches to subECUs - self.otaclient_call = mocker.AsyncMock(spec=OtaClientCall) + self.otaclient_call = mocker.AsyncMock(spec=OTAClientCall) self.otaclient_call.update_call = mocker.AsyncMock( wraps=self._subecu_accept_update_request ) @@ -761,7 +761,7 @@ async def setup_test( ) mocker.patch( f"{cfg.OTACLIENT_STUB_MODULE_PATH}.OTAServicer", - mocker.MagicMock(return_value=self.otaclient_wrapper), + mocker.MagicMock(return_value=self.otaclient_api_types), ) mocker.patch( f"{cfg.OTACLIENT_STUB_MODULE_PATH}._ECUTracker", @@ -772,7 +772,7 @@ async def setup_test( mocker.MagicMock(return_value=self.otaproxy_launcher), ) mocker.patch( - f"{cfg.OTACLIENT_STUB_MODULE_PATH}.OtaClientCall", self.otaclient_call + f"{cfg.OTACLIENT_STUB_MODULE_PATH}.OTAClientCall", self.otaclient_call ) # --- start the OTAClientServiceStub --- # @@ -845,15 +845,15 @@ async def test__otaclient_control_flags_managing(self): ( # update request for autoware, p1 ecus ( - wrapper.UpdateRequest( + api_types.UpdateRequest( ecu=[ - wrapper.UpdateRequestEcu( + api_types.UpdateRequestEcu( ecu_id="autoware", version="789.x", url="url", cookies="cookies", ), - wrapper.UpdateRequestEcu( + api_types.UpdateRequestEcu( ecu_id="p1", version="789.x", url="url", @@ -865,24 +865,24 @@ async def test__otaclient_control_flags_managing(self): # NOTE: order matters! # update request dispatching to subECUs happens first, # and then to the local ECU. - wrapper.UpdateResponse( + api_types.UpdateResponse( ecu=[ - wrapper.UpdateResponseEcu( + api_types.UpdateResponseEcu( ecu_id="p1", - result=wrapper.FailureType.NO_FAILURE, + result=api_types.FailureType.NO_FAILURE, ), - wrapper.UpdateResponseEcu( + api_types.UpdateResponseEcu( ecu_id="autoware", - result=wrapper.FailureType.NO_FAILURE, + result=api_types.FailureType.NO_FAILURE, ), ] ), ), # update only p2 ( - wrapper.UpdateRequest( + api_types.UpdateRequest( ecu=[ - wrapper.UpdateRequestEcu( + api_types.UpdateRequestEcu( ecu_id="p2", version="789.x", url="url", @@ -891,11 +891,11 @@ async def test__otaclient_control_flags_managing(self): ] ), {"p2"}, - wrapper.UpdateResponse( + api_types.UpdateResponse( ecu=[ - wrapper.UpdateResponseEcu( + api_types.UpdateResponseEcu( ecu_id="p2", - result=wrapper.FailureType.NO_FAILURE, + result=api_types.FailureType.NO_FAILURE, ), ] ), @@ -904,13 +904,15 @@ async def test__otaclient_control_flags_managing(self): ) async def test_update_normal( self, - update_request: wrapper.UpdateRequest, + update_request: api_types.UpdateRequest, update_target_ids: Set[str], - expected: wrapper.UpdateResponse, + expected: api_types.UpdateResponse, ): # --- setup --- # - self.otaclient_wrapper.dispatch_update.return_value = wrapper.UpdateResponseEcu( - ecu_id=self.ecu_info.ecu_id, result=wrapper.FailureType.NO_FAILURE + self.otaclient_api_types.dispatch_update.return_value = ( + api_types.UpdateResponseEcu( + ecu_id=self.ecu_info.ecu_id, result=api_types.FailureType.NO_FAILURE + ) ) # --- execution --- # @@ -925,27 +927,29 @@ async def test_update_normal( async def test_update_local_ecu_busy(self): # --- preparation --- # - self.otaclient_wrapper.dispatch_update.return_value = wrapper.UpdateResponseEcu( - ecu_id="autoware", result=wrapper.FailureType.RECOVERABLE + self.otaclient_api_types.dispatch_update.return_value = ( + api_types.UpdateResponseEcu( + ecu_id="autoware", result=api_types.FailureType.RECOVERABLE + ) ) - update_request_ecu = wrapper.UpdateRequestEcu( + update_request_ecu = api_types.UpdateRequestEcu( ecu_id="autoware", version="version", url="url", cookies="cookies" ) # --- execution --- # resp = await self.otaclient_service_stub.update( - wrapper.UpdateRequest(ecu=[update_request_ecu]) + api_types.UpdateRequest(ecu=[update_request_ecu]) ) # --- assertion --- # - assert resp == wrapper.UpdateResponse( + assert resp == api_types.UpdateResponse( ecu=[ - wrapper.UpdateResponseEcu( + api_types.UpdateResponseEcu( ecu_id="autoware", - result=wrapper.FailureType.RECOVERABLE, + result=api_types.FailureType.RECOVERABLE, ) ] ) - self.otaclient_wrapper.dispatch_update.assert_called_once_with( + self.otaclient_api_types.dispatch_update.assert_called_once_with( update_request_ecu ) diff --git a/tests/test_update_stats.py b/tests/test_otaclient/test_update_stats.py similarity index 100% rename from tests/test_update_stats.py rename to tests/test_otaclient/test_update_stats.py diff --git a/tests/test_otaclient_api/__init__.py b/tests/test_otaclient_api/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_otaclient_api/test_v2/__init__.py b/tests/test_otaclient_api/test_v2/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_ota_client_call.py b/tests/test_otaclient_api/test_v2/test_apli_caller.py similarity index 90% rename from tests/test_ota_client_call.py rename to tests/test_otaclient_api/test_v2/test_apli_caller.py index f883851e8..96298acfe 100644 --- a/tests/test_ota_client_call.py +++ b/tests/test_otaclient_api/test_v2/test_apli_caller.py @@ -17,8 +17,10 @@ import pytest import pytest_asyncio -from otaclient.app.ota_client_call import ECUNoResponse, OtaClientCall -from otaclient.app.proto import v2, v2_grpc, wrapper +from otaclient_api.v2 import otaclient_v2_pb2 as v2 +from otaclient_api.v2 import otaclient_v2_pb2_grpc as v2_grpc +from otaclient_api.v2 import types as api_types +from otaclient_api.v2.api_caller import ECUNoResponse, OTAClientCall from tests.utils import compare_message @@ -133,10 +135,10 @@ async def dummy_ota_client_service(self): await server.stop(None) async def test_update_call(self, dummy_ota_client_service): - _req = wrapper.UpdateRequest.convert( + _req = api_types.UpdateRequest.convert( _DummyOTAClientService.DUMMY_UPDATE_REQUEST ) - _response = await OtaClientCall.update_call( + _response = await OTAClientCall.update_call( ecu_id=self.DUMMY_ECU_ID, ecu_ipaddr=self.OTA_CLIENT_SERVICE_IP, ecu_port=self.OTA_CLIENT_SERVICE_PORT, @@ -147,10 +149,10 @@ async def test_update_call(self, dummy_ota_client_service): ) async def test_rollback_call(self, dummy_ota_client_service): - _req = wrapper.RollbackRequest.convert( + _req = api_types.RollbackRequest.convert( _DummyOTAClientService.DUMMY_ROLLBACK_REQUEST ) - _response = await OtaClientCall.rollback_call( + _response = await OTAClientCall.rollback_call( ecu_id=self.DUMMY_ECU_ID, ecu_ipaddr=self.OTA_CLIENT_SERVICE_IP, ecu_port=self.OTA_CLIENT_SERVICE_PORT, @@ -161,22 +163,22 @@ async def test_rollback_call(self, dummy_ota_client_service): ) async def test_status_call(self, dummy_ota_client_service): - _response = await OtaClientCall.status_call( + _response = await OTAClientCall.status_call( ecu_id=self.DUMMY_ECU_ID, ecu_ipaddr=self.OTA_CLIENT_SERVICE_IP, ecu_port=self.OTA_CLIENT_SERVICE_PORT, - request=wrapper.StatusRequest(), + request=api_types.StatusRequest(), ) assert _response is not None compare_message(_response.export_pb(), _DummyOTAClientService.DUMMY_STATUS) async def test_update_call_no_response(self): - _req = wrapper.UpdateRequest.convert( + _req = api_types.UpdateRequest.convert( _DummyOTAClientService.DUMMY_UPDATE_REQUEST ) with pytest.raises(ECUNoResponse): - await OtaClientCall.update_call( + await OTAClientCall.update_call( ecu_id=self.DUMMY_ECU_ID, ecu_ipaddr=self.OTA_CLIENT_SERVICE_IP, ecu_port=self.OTA_CLIENT_SERVICE_PORT, diff --git a/tests/test_proto/test_otaclient_pb2_wrapper.py b/tests/test_otaclient_api/test_v2/test_types.py similarity index 70% rename from tests/test_proto/test_otaclient_pb2_wrapper.py rename to tests/test_otaclient_api/test_v2/test_types.py index 87c63fd0f..b3d74f8dc 100644 --- a/tests/test_proto/test_otaclient_pb2_wrapper.py +++ b/tests/test_otaclient_api/test_v2/test_types.py @@ -18,7 +18,8 @@ import pytest from google.protobuf.duration_pb2 import Duration as _Duration -from otaclient.app.proto import v2, wrapper +from otaclient_api.v2 import otaclient_v2_pb2 as v2 +from otaclient_api.v2 import types as api_types from tests.utils import compare_message @@ -32,12 +33,14 @@ total_regular_files=123456, elapsed_time_download=_Duration(seconds=1, nanos=5678), ), - wrapper.StatusProgress( - phase=wrapper.StatusProgressPhase.REGULAR, + api_types.StatusProgress( + phase=api_types.StatusProgressPhase.REGULAR, total_regular_files=123456, - elapsed_time_download=wrapper.Duration.from_nanoseconds(1_000_005_678), + elapsed_time_download=api_types.Duration.from_nanoseconds( + 1_000_005_678 + ), ), - wrapper.StatusProgress, + api_types.StatusProgress, ), # UpdateRequest: with protobuf repeated composite field ( @@ -47,13 +50,13 @@ v2.UpdateRequestEcu(ecu_id="ecu_2"), ] ), - wrapper.UpdateRequest( + api_types.UpdateRequest( ecu=[ - wrapper.UpdateRequestEcu(ecu_id="ecu_1"), - wrapper.UpdateRequestEcu(ecu_id="ecu_2"), + api_types.UpdateRequestEcu(ecu_id="ecu_1"), + api_types.UpdateRequestEcu(ecu_id="ecu_2"), ] ), - wrapper.UpdateRequest, + api_types.UpdateRequest, ), # UpdateRequest: with protobuf repeated composite field, ( @@ -63,13 +66,13 @@ v2.UpdateRequestEcu(ecu_id="ecu_2"), ] ), - wrapper.UpdateRequest( + api_types.UpdateRequest( ecu=[ - wrapper.UpdateRequestEcu(ecu_id="ecu_1"), - wrapper.UpdateRequestEcu(ecu_id="ecu_2"), + api_types.UpdateRequestEcu(ecu_id="ecu_1"), + api_types.UpdateRequestEcu(ecu_id="ecu_2"), ] ), - wrapper.UpdateRequest, + api_types.UpdateRequest, ), # StatusResponse: multiple layer nested message, multiple protobuf message types ( @@ -100,29 +103,29 @@ ], available_ecu_ids=["ecu_1", "ecu_2"], ), - wrapper.StatusResponse( + api_types.StatusResponse( ecu=[ - wrapper.StatusResponseEcu( + api_types.StatusResponseEcu( ecu_id="ecu_1", - status=wrapper.Status( - status=wrapper.StatusOta.UPDATING, - progress=wrapper.StatusProgress( - phase=wrapper.StatusProgressPhase.REGULAR, + status=api_types.Status( + status=api_types.StatusOta.UPDATING, + progress=api_types.StatusProgress( + phase=api_types.StatusProgressPhase.REGULAR, total_regular_files=123456, - elapsed_time_copy=wrapper.Duration.from_nanoseconds( + elapsed_time_copy=api_types.Duration.from_nanoseconds( 1_000_056_789 ), ), ), ), - wrapper.StatusResponseEcu( + api_types.StatusResponseEcu( ecu_id="ecu_2", - status=wrapper.Status( - status=wrapper.StatusOta.UPDATING, - progress=wrapper.StatusProgress( - phase=wrapper.StatusProgressPhase.REGULAR, + status=api_types.Status( + status=api_types.StatusOta.UPDATING, + progress=api_types.StatusProgress( + phase=api_types.StatusProgressPhase.REGULAR, total_regular_files=456789, - elapsed_time_copy=wrapper.Duration.from_nanoseconds( + elapsed_time_copy=api_types.Duration.from_nanoseconds( 1_000_012_345 ), ), @@ -131,14 +134,14 @@ ], available_ecu_ids=["ecu_1", "ecu_2"], ), - wrapper.StatusResponse, + api_types.StatusResponse, ), ), ) def test_convert_message( origin_msg, - converted_msg: wrapper.MessageWrapper, - wrapper_type: Type[wrapper.MessageWrapper], + converted_msg: api_types.MessageWrapper, + wrapper_type: Type[api_types.MessageWrapper], ): # ------ converting message ------ # _converted = wrapper_type.convert(origin_msg) @@ -155,13 +158,13 @@ class Test_enum_wrapper_cooperate: def test_direct_compare(self): """protobuf enum and wrapper enum can compare directly.""" _protobuf_enum = v2.UPDATING - _wrapped = wrapper.StatusOta.UPDATING + _wrapped = api_types.StatusOta.UPDATING assert _protobuf_enum == _wrapped def test_assign_to_protobuf_message(self): """wrapper enum can be directly assigned in protobuf message.""" l, r = v2.StatusProgress(phase=v2.REGULAR), v2.StatusProgress( - phase=wrapper.StatusProgressPhase.REGULAR.value, + phase=api_types.StatusProgressPhase.REGULAR.value, # type: ignore ) compare_message(l, r) @@ -169,8 +172,8 @@ def test_used_in_message_wrapper(self): """wrapper enum can be exported.""" l, r = ( v2.StatusProgress(phase=v2.REGULAR), - wrapper.StatusProgress( - phase=wrapper.StatusProgressPhase.REGULAR + api_types.StatusProgress( + phase=api_types.StatusProgressPhase.REGULAR ).export_pb(), ) compare_message(l, r) @@ -178,6 +181,6 @@ def test_used_in_message_wrapper(self): def test_converted_from_protobuf_enum(self): """wrapper enum can be converted from and to protobuf enum.""" _protobuf_enum = v2.REGULAR - _converted = wrapper.StatusProgressPhase(_protobuf_enum) + _converted = api_types.StatusProgressPhase(_protobuf_enum) assert _protobuf_enum == _converted - assert _converted == wrapper.StatusProgressPhase.REGULAR + assert _converted == api_types.StatusProgressPhase.REGULAR diff --git a/tests/test_otaclient_common/__init__.py b/tests/test_otaclient_common/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_common.py b/tests/test_otaclient_common/test_common.py similarity index 72% rename from tests/test_common.py rename to tests/test_otaclient_common/test_common.py index d745faf76..28589fe22 100644 --- a/tests/test_common.py +++ b/tests/test_otaclient_common/test_common.py @@ -13,12 +13,12 @@ # limitations under the License. +from __future__ import annotations + import logging import os -import random import subprocess import time -from functools import partial from hashlib import sha256 from multiprocessing import Process from pathlib import Path @@ -26,13 +26,10 @@ import pytest -from otaclient.app.common import ( - RetryTaskMap, - RetryTaskMapInterrupted, +from otaclient_common.common import ( copytree_identical, ensure_otaproxy_start, file_sha256, - get_backoff, re_symlink_atomic, read_str_from_file, subprocess_call, @@ -231,122 +228,6 @@ def test_symlink_to_directory(self, tmp_path: Path): assert _symlink.is_symlink() and os.readlink(_symlink) == str(_target) -class _RetryTaskMapTestErr(Exception): - """""" - - -class TestRetryTaskMap: - WAIT_CONST = 100_000_000 - TASKS_COUNT = 2000 - MAX_CONCURRENT = 128 - DOWNLOAD_GROUP_NO_SUCCESS_RETRY_TIMEOUT = 6 # seconds - MAX_WAIT_BEFORE_SUCCESS = 10 - - @pytest.fixture(autouse=True) - def setup(self): - self._start_time = time.time() - self._success_wait_dict = { - idx: random.randint(0, self.MAX_WAIT_BEFORE_SUCCESS) - for idx in range(self.TASKS_COUNT) - } - self._succeeded_tasks = [False for _ in range(self.TASKS_COUNT)] - - def workload_aways_failed(self, idx: int) -> int: - time.sleep((self.TASKS_COUNT - random.randint(0, idx)) / self.WAIT_CONST) - raise _RetryTaskMapTestErr - - def workload_failed_and_then_succeed(self, idx: int) -> int: - time.sleep((self.TASKS_COUNT - random.randint(0, idx)) / self.WAIT_CONST) - if time.time() > self._start_time + self._success_wait_dict[idx]: - self._succeeded_tasks[idx] = True - return idx - raise _RetryTaskMapTestErr - - def workload_succeed(self, idx: int) -> int: - time.sleep((self.TASKS_COUNT - random.randint(0, idx)) / self.WAIT_CONST) - self._succeeded_tasks[idx] = True - return idx - - def test_retry_keep_failing_timeout(self): - _keep_failing_timer = time.time() - with pytest.raises(RetryTaskMapInterrupted): - _mapper = RetryTaskMap( - backoff_func=partial(get_backoff, factor=0.1, _max=1), - max_concurrent=self.MAX_CONCURRENT, - max_retry=0, # we are testing keep failing timeout here - ) - for done_task in _mapper.map( - self.workload_aways_failed, range(self.TASKS_COUNT) - ): - if not done_task.fut.exception(): - # reset the failing timer on one succeeded task - _keep_failing_timer = time.time() - continue - if ( - time.time() - _keep_failing_timer - > self.DOWNLOAD_GROUP_NO_SUCCESS_RETRY_TIMEOUT - ): - logger.error( - f"RetryTaskMap successfully failed after keep failing in {self.DOWNLOAD_GROUP_NO_SUCCESS_RETRY_TIMEOUT}s" - ) - _mapper.shutdown(raise_last_exc=True) - - def test_retry_exceed_retry_limit(self): - with pytest.raises(RetryTaskMapInterrupted): - _mapper = RetryTaskMap( - backoff_func=partial(get_backoff, factor=0.1, _max=1), - max_concurrent=self.MAX_CONCURRENT, - max_retry=3, - ) - for _ in _mapper.map(self.workload_aways_failed, range(self.TASKS_COUNT)): - pass - - def test_retry_finally_succeeded(self): - _keep_failing_timer = time.time() - - _mapper = RetryTaskMap( - backoff_func=partial(get_backoff, factor=0.1, _max=1), - max_concurrent=self.MAX_CONCURRENT, - max_retry=0, # we are testing keep failing timeout here - ) - for done_task in _mapper.map( - self.workload_failed_and_then_succeed, range(self.TASKS_COUNT) - ): - # task successfully finished - if not done_task.fut.exception(): - # reset the failing timer on one succeeded task - _keep_failing_timer = time.time() - continue - - if ( - time.time() - _keep_failing_timer - > self.DOWNLOAD_GROUP_NO_SUCCESS_RETRY_TIMEOUT - ): - _mapper.shutdown(raise_last_exc=True) - assert all(self._succeeded_tasks) - - def test_succeeded_in_one_try(self): - _keep_failing_timer = time.time() - _mapper = RetryTaskMap( - backoff_func=partial(get_backoff, factor=0.1, _max=1), - max_concurrent=self.MAX_CONCURRENT, - max_retry=0, # we are testing keep failing timeout here - ) - for done_task in _mapper.map(self.workload_succeed, range(self.TASKS_COUNT)): - # task successfully finished - if not done_task.fut.exception(): - # reset the failing timer on one succeeded task - _keep_failing_timer = time.time() - continue - - if ( - time.time() - _keep_failing_timer - > self.DOWNLOAD_GROUP_NO_SUCCESS_RETRY_TIMEOUT - ): - _mapper.shutdown(raise_last_exc=True) - assert all(self._succeeded_tasks) - - class Test_ensure_otaproxy_start: DUMMY_SERVER_ADDR, DUMMY_SERVER_PORT = "127.0.0.1", 18888 DUMMY_SERVER_URL = f"http://{DUMMY_SERVER_ADDR}:{DUMMY_SERVER_PORT}" diff --git a/tests/test_downloader.py b/tests/test_otaclient_common/test_downloader.py similarity index 98% rename from tests/test_downloader.py rename to tests/test_otaclient_common/test_downloader.py index b11324d53..4af3d23c1 100644 --- a/tests/test_downloader.py +++ b/tests/test_otaclient_common/test_downloader.py @@ -13,6 +13,8 @@ # limitations under the License. +from __future__ import annotations + import asyncio import logging import threading @@ -24,8 +26,8 @@ import requests import requests_mock -from otaclient.app.common import file_sha256, urljoin_ensure_base -from otaclient.app.downloader import ( +from otaclient_common.common import file_sha256, urljoin_ensure_base +from otaclient_common.downloader import ( ChunkStreamingError, DestinationNotAvailableError, Downloader, diff --git a/tests/test_persist_file_handling.py b/tests/test_otaclient_common/test_persist_file_handling.py similarity index 99% rename from tests/test_persist_file_handling.py rename to tests/test_otaclient_common/test_persist_file_handling.py index cc00a0c0d..1b9823811 100644 --- a/tests/test_persist_file_handling.py +++ b/tests/test_otaclient_common/test_persist_file_handling.py @@ -19,8 +19,8 @@ import stat from pathlib import Path -from otaclient._utils import replace_root -from otaclient.app.common import PersistFilesHandler +from otaclient_common import replace_root +from otaclient_common.persist_file_handling import PersistFilesHandler def create_files(tmp_path: Path): diff --git a/tests/test_proto/__init__.py b/tests/test_otaclient_common/test_proto_wrapper/__init__.py similarity index 100% rename from tests/test_proto/__init__.py rename to tests/test_otaclient_common/test_proto_wrapper/__init__.py diff --git a/tests/test_proto/example.proto b/tests/test_otaclient_common/test_proto_wrapper/example.proto similarity index 100% rename from tests/test_proto/example.proto rename to tests/test_otaclient_common/test_proto_wrapper/example.proto diff --git a/tests/test_proto/example_pb2.py b/tests/test_otaclient_common/test_proto_wrapper/example_pb2.py similarity index 100% rename from tests/test_proto/example_pb2.py rename to tests/test_otaclient_common/test_proto_wrapper/example_pb2.py diff --git a/tests/test_proto/example_pb2.pyi b/tests/test_otaclient_common/test_proto_wrapper/example_pb2.pyi similarity index 100% rename from tests/test_proto/example_pb2.pyi rename to tests/test_otaclient_common/test_proto_wrapper/example_pb2.pyi diff --git a/tests/test_proto/example_pb2_wrapper.py b/tests/test_otaclient_common/test_proto_wrapper/example_pb2_wrapper.py similarity index 96% rename from tests/test_proto/example_pb2_wrapper.py rename to tests/test_otaclient_common/test_proto_wrapper/example_pb2_wrapper.py index 45ae7cd04..60f14e297 100644 --- a/tests/test_proto/example_pb2_wrapper.py +++ b/tests/test_otaclient_common/test_proto_wrapper/example_pb2_wrapper.py @@ -13,12 +13,14 @@ # limitations under the License. +from __future__ import annotations + from typing import Iterable as _Iterable from typing import Mapping as _Mapping from typing import Optional as _Optional from typing import Union as _Union -from otaclient.app.proto.wrapper import ( +from otaclient_common.proto_wrapper import ( Duration, EnumWrapper, MessageMapContainer, diff --git a/tests/test_proto/test_proto_wrapper.py b/tests/test_otaclient_common/test_proto_wrapper/test_proto_wrapper.py similarity index 97% rename from tests/test_proto/test_proto_wrapper.py rename to tests/test_otaclient_common/test_proto_wrapper/test_proto_wrapper.py index 269ae8245..876b81a5d 100644 --- a/tests/test_proto/test_proto_wrapper.py +++ b/tests/test_otaclient_common/test_proto_wrapper/test_proto_wrapper.py @@ -13,12 +13,14 @@ # limitations under the License. -from typing import Any, Dict +from __future__ import annotations + +from typing import Any import pytest from google.protobuf.duration_pb2 import Duration as _pb2_Duration -from otaclient.app.proto import wrapper as proto_wrapper +from otaclient_common import proto_wrapper from tests.utils import compare_message from . import example_pb2 as pb2 @@ -61,7 +63,7 @@ ), ) def test_default_value_behavior( - input_wrapper_inst: proto_wrapper.MessageWrapper, expected_dict: Dict[str, Any] + input_wrapper_inst: proto_wrapper.MessageWrapper, expected_dict: dict[str, Any] ): for _field_name in input_wrapper_inst._fields: _value = getattr(input_wrapper_inst, _field_name) diff --git a/tests/test_otaclient_common/test_retry_task_map.py b/tests/test_otaclient_common/test_retry_task_map.py new file mode 100644 index 000000000..575d2450d --- /dev/null +++ b/tests/test_otaclient_common/test_retry_task_map.py @@ -0,0 +1,144 @@ +# Copyright 2022 TIER IV, INC. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import annotations + +import logging +import random +import time +from functools import partial + +import pytest + +from otaclient_common.common import get_backoff +from otaclient_common.retry_task_map import RetryTaskMap, RetryTaskMapInterrupted + +logger = logging.getLogger(__name__) + + +class _RetryTaskMapTestErr(Exception): + """""" + + +class TestRetryTaskMap: + WAIT_CONST = 100_000_000 + TASKS_COUNT = 2000 + MAX_CONCURRENT = 128 + DOWNLOAD_GROUP_NO_SUCCESS_RETRY_TIMEOUT = 6 # seconds + MAX_WAIT_BEFORE_SUCCESS = 10 + + @pytest.fixture(autouse=True) + def setup(self): + self._start_time = time.time() + self._success_wait_dict = { + idx: random.randint(0, self.MAX_WAIT_BEFORE_SUCCESS) + for idx in range(self.TASKS_COUNT) + } + self._succeeded_tasks = [False for _ in range(self.TASKS_COUNT)] + + def workload_aways_failed(self, idx: int) -> int: + time.sleep((self.TASKS_COUNT - random.randint(0, idx)) / self.WAIT_CONST) + raise _RetryTaskMapTestErr + + def workload_failed_and_then_succeed(self, idx: int) -> int: + time.sleep((self.TASKS_COUNT - random.randint(0, idx)) / self.WAIT_CONST) + if time.time() > self._start_time + self._success_wait_dict[idx]: + self._succeeded_tasks[idx] = True + return idx + raise _RetryTaskMapTestErr + + def workload_succeed(self, idx: int) -> int: + time.sleep((self.TASKS_COUNT - random.randint(0, idx)) / self.WAIT_CONST) + self._succeeded_tasks[idx] = True + return idx + + def test_retry_keep_failing_timeout(self): + _keep_failing_timer = time.time() + with pytest.raises(RetryTaskMapInterrupted): + _mapper = RetryTaskMap( + backoff_func=partial(get_backoff, factor=0.1, _max=1), + max_concurrent=self.MAX_CONCURRENT, + max_retry=0, # we are testing keep failing timeout here + ) + for done_task in _mapper.map( + self.workload_aways_failed, range(self.TASKS_COUNT) + ): + if not done_task.fut.exception(): + # reset the failing timer on one succeeded task + _keep_failing_timer = time.time() + continue + if ( + time.time() - _keep_failing_timer + > self.DOWNLOAD_GROUP_NO_SUCCESS_RETRY_TIMEOUT + ): + logger.error( + f"RetryTaskMap successfully failed after keep failing in {self.DOWNLOAD_GROUP_NO_SUCCESS_RETRY_TIMEOUT}s" + ) + _mapper.shutdown(raise_last_exc=True) + + def test_retry_exceed_retry_limit(self): + with pytest.raises(RetryTaskMapInterrupted): + _mapper = RetryTaskMap( + backoff_func=partial(get_backoff, factor=0.1, _max=1), + max_concurrent=self.MAX_CONCURRENT, + max_retry=3, + ) + for _ in _mapper.map(self.workload_aways_failed, range(self.TASKS_COUNT)): + pass + + def test_retry_finally_succeeded(self): + _keep_failing_timer = time.time() + + _mapper = RetryTaskMap( + backoff_func=partial(get_backoff, factor=0.1, _max=1), + max_concurrent=self.MAX_CONCURRENT, + max_retry=0, # we are testing keep failing timeout here + ) + for done_task in _mapper.map( + self.workload_failed_and_then_succeed, range(self.TASKS_COUNT) + ): + # task successfully finished + if not done_task.fut.exception(): + # reset the failing timer on one succeeded task + _keep_failing_timer = time.time() + continue + + if ( + time.time() - _keep_failing_timer + > self.DOWNLOAD_GROUP_NO_SUCCESS_RETRY_TIMEOUT + ): + _mapper.shutdown(raise_last_exc=True) + assert all(self._succeeded_tasks) + + def test_succeeded_in_one_try(self): + _keep_failing_timer = time.time() + _mapper = RetryTaskMap( + backoff_func=partial(get_backoff, factor=0.1, _max=1), + max_concurrent=self.MAX_CONCURRENT, + max_retry=0, # we are testing keep failing timeout here + ) + for done_task in _mapper.map(self.workload_succeed, range(self.TASKS_COUNT)): + # task successfully finished + if not done_task.fut.exception(): + # reset the failing timer on one succeeded task + _keep_failing_timer = time.time() + continue + + if ( + time.time() - _keep_failing_timer + > self.DOWNLOAD_GROUP_NO_SUCCESS_RETRY_TIMEOUT + ): + _mapper.shutdown(raise_last_exc=True) + assert all(self._succeeded_tasks) diff --git a/tests/utils.py b/tests/utils.py index f3674a763..8a4e5bd2a 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -13,6 +13,8 @@ # limitations under the License. +from __future__ import annotations + import asyncio import logging import os @@ -27,8 +29,9 @@ import zstandard from google.protobuf.message import Message as _Message -from otaclient.app.common import file_sha256 -from otaclient.app.proto import v2_grpc, wrapper +from otaclient_api.v2 import otaclient_v2_pb2_grpc as v2_grpc +from otaclient_api.v2 import types as api_types +from otaclient_common.common import file_sha256 logger = logging.getLogger(__name__) @@ -113,13 +116,13 @@ def compare_dir(left: Path, right: Path): class DummySubECU: - SUCCESS_RESPONSE = wrapper.Status( - status=wrapper.StatusOta.SUCCESS, - failure=wrapper.FailureType.NO_FAILURE, + SUCCESS_RESPONSE = api_types.Status( + status=api_types.StatusOta.SUCCESS, + failure=api_types.FailureType.NO_FAILURE, ) - UPDATING_RESPONSE = wrapper.Status( - status=wrapper.StatusOta.UPDATING, - failure=wrapper.FailureType.NO_FAILURE, + UPDATING_RESPONSE = api_types.Status( + status=api_types.StatusOta.UPDATING, + failure=api_types.FailureType.NO_FAILURE, ) UPDATE_TIME_COST = 6 REBOOT_TIME_COST = 1 @@ -138,9 +141,9 @@ def status(self): # update not yet started if self._receive_update_time is None: logger.debug(f"{self.ecu_id=}, update not yet started") - res = wrapper.StatusResponse( + res = api_types.StatusResponse( ecu=[ - wrapper.StatusResponseEcu( + api_types.StatusResponseEcu( ecu_id=self.ecu_id, status=self.SUCCESS_RESPONSE, ) @@ -155,9 +158,9 @@ def status(self): logger.debug( f"update finished for {self.ecu_id=}, {self._receive_update_time=}, {time.time()=}" ) - res = wrapper.StatusResponse( + res = api_types.StatusResponse( ecu=[ - wrapper.StatusResponseEcu( + api_types.StatusResponseEcu( ecu_id=self.ecu_id, status=self.SUCCESS_RESPONSE, ) @@ -172,9 +175,9 @@ def status(self): return None # updating logger.debug(f"{self.ecu_id=}, updating") - res = wrapper.StatusResponse( + res = api_types.StatusResponse( ecu=[ - wrapper.StatusResponseEcu( + api_types.StatusResponseEcu( ecu_id=self.ecu_id, status=self.UPDATING_RESPONSE, ) diff --git a/tools/build_image.sh b/tools/build_image.sh deleted file mode 100644 index 9cad89ec4..000000000 --- a/tools/build_image.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -set -eux - -docker build -t ota-image -f ./docker/build_image/Dockerfile . -id=$(docker create -it ota-image) -ota_image_dir="ota-image.$(date +%Y%m%d%H%M%S)" -mkdir ${ota_image_dir} - -cd ${ota_image_dir} -docker export ${id} > ota-image.tar -docker rm ${id} -mkdir data -sudo tar xf ota-image.tar -C data -git clone https://github.com/tier4/ota-metadata - -cp ../tests/keys/sign.pem . -cp ota-metadata/metadata/persistents.txt . - -sudo python3 ota-metadata/metadata/ota_metadata/metadata_gen.py --target-dir data --ignore-file ota-metadata/metadata/ignore.txt -sudo python3 ota-metadata/metadata/ota_metadata/metadata_sign.py --sign-key ../tests/keys/sign.key --cert-file sign.pem --persistent-file persistents.txt --rootfs-directory data - -sudo chown -R $(whoami) data diff --git a/tools/emulator/README.md b/tools/emulator/README.md deleted file mode 100644 index a536010de..000000000 --- a/tools/emulator/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# ota-client emulator - -This tool mimics ota-client behavior. -This tool can be used to develop software which requests ota-client. - -## Usage - -```bash -python main.py --config config.yml -``` diff --git a/tools/emulator/config.yml b/tools/emulator/config.yml deleted file mode 100644 index 8532f3185..000000000 --- a/tools/emulator/config.yml +++ /dev/null @@ -1,31 +0,0 @@ -ecus: -- main: true # main ecu or not. only one ecu should be main. - name: autoware # should be unique - status: INITIALIZED # INITIALIZED | SUCCESS | FAILURE | UPDATING | NO_CONNECTION - version: 123.456 # current version - time_to_update: 40 # in second - time_to_restart: 10 # in second - -- name: perception1 # should be unique - status: INITIALIZED # INITIALIZED | SUCCESS | FAILURE | UPDATING | NO_CONNECTION - version: abc.def # current version - time_to_update: 60 # in second - time_to_restart: 10 # in second - -- name: perception2 - status: INITIALIZED # INITIALIZED | SUCCESS | FAILURE | UPDATING | NO_CONNECTION - version: abc.def - time_to_update: 60 # in second - time_to_restart: 10 # in second - -- name: perception3 - status: INITIALIZED # INITIALIZED | SUCCESS | FAILURE | UPDATING | NO_CONNECTION - version: abc.def - time_to_update: 60 # in second - time_to_restart: 10 # in second - -#- name: perception4 -# status: INITIALIZED # INITIALIZED | SUCCESS | FAILURE | UPDATING | NO_CONNECTION -# version: abc.def -# time_to_update: 60 # in second -# time_to_restart: 10 # in second diff --git a/tools/emulator/ecu.py b/tools/emulator/ecu.py deleted file mode 100644 index afd8e2795..000000000 --- a/tools/emulator/ecu.py +++ /dev/null @@ -1,180 +0,0 @@ -# Copyright 2022 TIER IV, INC. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import time - -import log_setting -import otaclient_v2_pb2 as v2 -from configs import config as cfg - -logger = log_setting.get_logger( - __name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL) -) - - -class Ecu: - TOTAL_REGULAR_FILES = 123456789 - TOTAL_REGULAR_FILE_SIZE = 987654321 - TIME_TO_UPDATE = 60 * 5 - TIME_TO_RESTART = 10 - - def __init__( - self, - is_main, - name, - status, - version, - time_to_update=TIME_TO_UPDATE, - time_to_restart=TIME_TO_RESTART, - ): - self._is_main = is_main - self._name = name - self._version = version - self._version_to_update = None - self._status = v2.Status() - self._status.status = v2.StatusOta.Value(status) - self._update_time = None - self._time_to_update = time_to_update - self._time_to_restart = time_to_restart - - def create(self): - return Ecu( - is_main=self._is_main, - name=self._name, - status=v2.StatusOta.Name(self._status.status), - version=self._version, - time_to_update=self._time_to_update, - time_to_restart=self._time_to_restart, - ) - - def change_to_success(self): - if self._status.status != v2.StatusOta.UPDATING: - logger.warning(f"current status: {v2.StatusOta.Name(self._status.status)}") - return - logger.info(f"change_to_success: {self._name=}") - self._status.status = v2.StatusOta.SUCCESS - self._version = self._version_to_update - - def update(self, response_ecu, version): - ecu = response_ecu - ecu.ecu_id = self._name - ecu.result = v2.FailureType.NO_FAILURE - - # update status - self._status = v2.Status() # reset - self._status.status = v2.StatusOta.UPDATING - self._version_to_update = version - self._update_time = time.time() - - def status(self, response_ecu): - ecu = response_ecu - ecu.ecu_id = self._name - ecu.result = v2.FailureType.NO_FAILURE - - try: - elapsed = time.time() - self._update_time - progress_rate = elapsed / self._time_to_update - should_restart = elapsed > (self._time_to_update + self._time_to_restart) - except TypeError: # when self._update_time is None - elapsed = 0 - progress_rate = 0 - should_restart = False - - # Main ecu waits for all sub ecu sucesss, while sub ecu transitions to - # success by itself. This code is intended to mimic that. - # The actual ecu updates, restarts and then transitions to success. - # In this code, after starting update and after time_to_update + - # time_to_restart elapsed, it transitions to success. - if not self._is_main and should_restart: - self.change_to_success() - else: - ecu.status.progress.CopyFrom(self._progress_rate_to_progress(progress_rate)) - - ecu.status.status = self._status.status - ecu.status.failure = v2.FailureType.NO_FAILURE - ecu.status.failure_reason = "" - ecu.status.version = self._version - - def _progress_rate_to_progress(self, rate): - progress = v2.StatusProgress() - if rate == 0: - progress.phase = v2.StatusProgressPhase.INITIAL - elif rate <= 0.01: - progress.phase = v2.StatusProgressPhase.METADATA - elif rate <= 0.02: - progress.phase = v2.StatusProgressPhase.DIRECTORY - elif rate <= 0.03: - progress.phase = v2.StatusProgressPhase.SYMLINK - elif rate <= 0.95: - progress.phase = v2.StatusProgressPhase.REGULAR - progress.total_regular_files = self.TOTAL_REGULAR_FILES - progress.regular_files_processed = int(self.TOTAL_REGULAR_FILES * rate) - - progress.files_processed_copy = int(progress.regular_files_processed * 0.4) - progress.files_processed_link = int(progress.regular_files_processed * 0.01) - progress.files_processed_download = ( - progress.regular_files_processed - - progress.files_processed_copy - - progress.files_processed_link - ) - size_processed = int(self.TOTAL_REGULAR_FILE_SIZE * rate) - progress.file_size_processed_copy = int(size_processed * 0.4) - progress.file_size_processed_link = int(size_processed * 0.01) - progress.file_size_processed_download = ( - size_processed - - progress.file_size_processed_copy - - progress.file_size_processed_link - ) - - progress.elapsed_time_copy.FromSeconds( - int(self._time_to_update * rate * 0.4) - ) - progress.elapsed_time_link.FromSeconds( - int(self._time_to_update * rate * 0.01) - ) - progress.elapsed_time_download.FromSeconds( - int(self._time_to_update * rate * 0.6) - ) - progress.errors_download = int(rate * 0.1) - progress.total_regular_file_size = self.TOTAL_REGULAR_FILE_SIZE - progress.total_elapsed_time.FromSeconds(int(self._time_to_update * rate)) - else: - progress.phase = v2.StatusProgressPhase.PERSISTENT - progress.total_regular_files = self.TOTAL_REGULAR_FILES - progress.regular_files_processed = self.TOTAL_REGULAR_FILES - - progress.files_processed_copy = int(progress.regular_files_processed * 0.4) - progress.files_processed_link = int(progress.regular_files_processed * 0.01) - progress.files_processed_download = ( - progress.regular_files_processed - - progress.files_processed_copy - - progress.files_processed_link - ) - size_processed = self.TOTAL_REGULAR_FILE_SIZE - progress.file_size_processed_copy = int(size_processed * 0.4) - progress.file_size_processed_link = int(size_processed * 0.01) - progress.file_size_processed_download = ( - size_processed - - progress.file_size_processed_copy - - progress.file_size_processed_link - ) - - progress.elapsed_time_copy.FromSeconds(int(self._time_to_update * 0.4)) - progress.elapsed_time_link.FromSeconds(int(self._time_to_update * 0.01)) - progress.elapsed_time_download.FromSeconds(int(self._time_to_update * 0.6)) - progress.errors_download = int(rate * 0.1) - progress.total_regular_file_size = self.TOTAL_REGULAR_FILE_SIZE - progress.total_elapsed_time.FromSeconds(self._time_to_update) - return progress diff --git a/tools/emulator/main.py b/tools/emulator/main.py deleted file mode 100644 index 8c859d54e..000000000 --- a/tools/emulator/main.py +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright 2022 TIER IV, INC. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import time -from pathlib import Path - -import log_setting -import otaclient_v2_pb2 as v2 -import otaclient_v2_pb2_grpc as v2_grpc -import path_loader # noqa -import yaml -from configs import config as cfg -from configs import server_cfg -from ecu import Ecu -from ota_client_service import ( - OtaClientServiceV2, - service_start, - service_stop, - service_wait_for_termination, -) -from ota_client_stub import OtaClientStub - -logger = log_setting.get_logger( - __name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL) -) - -DEFAULT_ECUS = [ - {"main": True, "id": "autoware", "status": "INITIALIZED", "version": "123.456"} -] - - -def main(config_file): - logger.info("started") - - server = None - - try: - config = yaml.safe_load(config_file.read_text()) - ecu_config = config["ecus"] - except Exception as e: - logger.warning(e) - logger.warning( - f"{config_file} couldn't be parsed. Default config is used instead." - ) - ecu_config = DEFAULT_ECUS - ecus = [] - logger.info(ecu_config) - for ecu in ecu_config: - e = Ecu( - is_main=ecu.get("main", False), - name=ecu.get("name", "autoware"), - status=ecu.get("status", "INITIALIZED"), - version=str(ecu.get("version", "")), - time_to_update=ecu.get("time_to_update"), - time_to_restart=ecu.get("time_to_restart"), - ) - ecus.append(e) - logger.info(ecus) - - def terminate(restart_time): - logger.info(f"{server=}") - service_stop(server) - logger.info(f"restarting. wait {restart_time}s.") - time.sleep(restart_time) - - while True: - ota_client_stub = OtaClientStub(ecus, terminate) - ota_client_service_v2 = OtaClientServiceV2(ota_client_stub) - - logger.info("starting grpc server.") - server = service_start( - f"localhost:{server_cfg.SERVER_PORT}", - [ - {"grpc": v2_grpc, "instance": ota_client_service_v2}, - ], - ) - - service_wait_for_termination(server) - - -if __name__ == "__main__": - import argparse - - parser = argparse.ArgumentParser() - parser.add_argument("--config", help="config.yml", default="config.yml") - args = parser.parse_args() - - logger.info(args) - - main(Path(args.config)) diff --git a/tools/emulator/ota_client_stub.py b/tools/emulator/ota_client_stub.py deleted file mode 100644 index c20d7c2ac..000000000 --- a/tools/emulator/ota_client_stub.py +++ /dev/null @@ -1,105 +0,0 @@ -# Copyright 2022 TIER IV, INC. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from pathlib import Path -from threading import Thread, Timer - -import log_setting -import otaclient_v2_pb2 as v2 -from configs import config as cfg -from ecu import Ecu - -logger = log_setting.get_logger( - __name__, cfg.LOG_LEVEL_TABLE.get(__name__, cfg.DEFAULT_LOG_LEVEL) -) - - -class OtaClientStub: - def __init__(self, ecus: list, terminate=None): - # check if all the names are unique - names = [ecu._name for ecu in ecus] - assert len(names) == len(set(names)) - # check if only one ecu is main - mains = [ecu for ecu in ecus if ecu._is_main] - assert len(mains) == 1 - - self._ecus = ecus - self._main_ecu = mains[0] - self._terminate = terminate - - async def update(self, request: v2.UpdateRequest) -> v2.UpdateResponse: - logger.info(f"{request=}") - response = v2.UpdateResponse() - - for ecu in self._ecus: - entry = OtaClientStub._find_request(request.ecu, ecu._name) - if entry: - logger.info(f"{ecu=}, {entry.version=}") - response_ecu = response.ecu.add() - ecu.update(response_ecu, entry.version) - - logger.info(f"{response=}") - return response - - def rollback(self, request): - logger.info(f"{request=}") - response = v2.RollbackResponse() - - return response - - async def status(self, request: v2.StatusRequest) -> v2.StatusResponse: - logger.info(f"{request=}") - response = v2.StatusResponse() - - for ecu in self._ecus: - response_ecu = response.ecu.add() - ecu.status(response_ecu) - response.available_ecu_ids.extend([ecu._name]) - - logger.debug(f"{response=}") - - if self._sub_ecus_success_and_main_ecu_phase_persistent(response.ecu): - self._main_ecu.change_to_success() - for index, ecu in enumerate(self._ecus): - self._ecus[index] = ecu.create() # create new ecu instances - self._terminate(self._main_ecu._time_to_restart) - - return response - - @staticmethod - def _find_request(update_request, ecu_id): - for request in update_request: - if request.ecu_id == ecu_id: - return request - return None - - def _update(self, ecu, response): - ecu.update(response) - - def _status(self, ecu, response): - ecu.status(response) - - def _sub_ecus_success_and_main_ecu_phase_persistent(self, response_ecu): - for ecu in response_ecu: - if ecu.ecu_id == self._main_ecu._name: - if ( - ecu.status.status != v2.StatusOta.UPDATING - or ecu.status.progress.phase != v2.StatusProgressPhase.PERSISTENT - ): - return False - else: - if ecu.status.status != v2.StatusOta.SUCCESS: - return False - return True diff --git a/tools/emulator/requirements.txt b/tools/emulator/requirements.txt deleted file mode 100644 index 3eadadb31..000000000 --- a/tools/emulator/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -grpcio==1.53.0 -protobuf==3.18.3 -PyYAML>=3.12 diff --git a/tools/offline_ota_image_builder/builder.py b/tools/offline_ota_image_builder/builder.py index bbf31e522..2a8b05b20 100644 --- a/tools/offline_ota_image_builder/builder.py +++ b/tools/offline_ota_image_builder/builder.py @@ -23,8 +23,8 @@ from pathlib import Path from typing import Mapping, Optional, Sequence -from otaclient.app import ota_metadata -from otaclient.app.common import subprocess_call +from ota_metadata.legacy import parser as ota_metadata_parser +from otaclient_common.common import subprocess_call from .configs import cfg from .manifest import ImageMetadata, Manifest @@ -70,9 +70,9 @@ def _process_ota_image(ota_image_dir: StrPath, *, data_dir: StrPath, meta_dir: S ota_image_dir = Path(ota_image_dir) # ------ process OTA image metadata ------ # - metadata_jwt_fpath = ota_image_dir / ota_metadata.OTAMetadata.METADATA_JWT + metadata_jwt_fpath = ota_image_dir / ota_metadata_parser.OTAMetadata.METADATA_JWT # NOTE: we don't need to do certificate verification here, so set certs_dir to empty - metadata_jwt = ota_metadata._MetadataJWTParser( + metadata_jwt = ota_metadata_parser._MetadataJWTParser( metadata_jwt_fpath.read_text(), certs_dir="" ).get_otametadata() @@ -82,7 +82,7 @@ def _process_ota_image(ota_image_dir: StrPath, *, data_dir: StrPath, meta_dir: S # ------ update data_dir with the contents of this OTA image ------ # with open(ota_image_dir / metadata_jwt.regular.file, "r") as f: for line in f: - reg_inf = ota_metadata.parse_regulars_from_txt(line) + reg_inf = ota_metadata_parser.parse_regulars_from_txt(line) ota_file_sha256 = reg_inf.sha256hash.hex() if reg_inf.compressed_alg == cfg.OTA_IMAGE_COMPRESSION_ALG: @@ -108,7 +108,7 @@ def _process_ota_image(ota_image_dir: StrPath, *, data_dir: StrPath, meta_dir: S # ------ update meta_dir with the OTA meta files in this image ------ # # copy OTA metafiles to image specific meta folder - for _metaf in ota_metadata.MetafilesV1: + for _metaf in ota_metadata_parser.MetafilesV1: shutil.move(str(ota_image_dir / _metaf.value), meta_dir) # copy certificate and metadata.jwt shutil.move(str(ota_image_dir / metadata_jwt.certificate.file), meta_dir) diff --git a/tools/status_monitor/ecu_status_box.py b/tools/status_monitor/ecu_status_box.py index 61cc38c1d..4b3a57cb5 100644 --- a/tools/status_monitor/ecu_status_box.py +++ b/tools/status_monitor/ecu_status_box.py @@ -13,13 +13,15 @@ # limitations under the License. +from __future__ import annotations + import curses import datetime import threading import time from typing import Sequence, Tuple -from otaclient.app.proto import wrapper as proto_wrapper +from otaclient_api.v2 import types as api_types from .configs import config from .utils import FormatValue, ScreenHandler @@ -47,7 +49,7 @@ def __init__(self, ecu_id: str, index: int) -> None: # contents for raw ecu status info sub window self.raw_ecu_status_contents = [] - self._last_status = proto_wrapper.StatusResponseEcuV2() + self._last_status = api_types.StatusResponseEcuV2() # prevent conflicts between status update and pad update self._lock = threading.Lock() self.last_updated = 0 @@ -60,9 +62,7 @@ def get_raw_info_contents(self) -> Tuple[Sequence[str], int]: """Getter for raw_ecu_status_contents.""" return self.raw_ecu_status_contents, self.last_updated - def update_ecu_status( - self, ecu_status: proto_wrapper.StatusResponseEcuV2, index: int - ): + def update_ecu_status(self, ecu_status: api_types.StatusResponseEcuV2, index: int): """Update internal contents storage with input . This method is called by tracker module to update the contents within @@ -80,7 +80,7 @@ def update_ecu_status( "-" * (self.DISPLAY_BOX_HCOLS - 2), ] - if ecu_status.ota_status is proto_wrapper.StatusOta.UPDATING: + if ecu_status.ota_status is api_types.StatusOta.UPDATING: update_status = ecu_status.update_status # TODO: render a progress bar according to ECU status V2's specification self.contents.extend( @@ -114,7 +114,7 @@ def update_ecu_status( "No detailed failure information.", ] - elif ecu_status.ota_status is proto_wrapper.StatusOta.FAILURE: + elif ecu_status.ota_status is api_types.StatusOta.FAILURE: self.contents.extend( [ f"ota_status: {ecu_status.ota_status.name}", diff --git a/tools/status_monitor/ecu_status_tracker.py b/tools/status_monitor/ecu_status_tracker.py index 5cebfc200..f4342f504 100644 --- a/tools/status_monitor/ecu_status_tracker.py +++ b/tools/status_monitor/ecu_status_tracker.py @@ -13,13 +13,15 @@ # limitations under the License. +from __future__ import annotations + import asyncio import threading from queue import Queue from typing import Dict, List, Optional -from otaclient.app.ota_client_call import ECUNoResponse, OtaClientCall -from otaclient.app.proto import wrapper as proto_wrapper +from otaclient_api.v2 import types as api_types +from otaclient_api.v2.api_caller import ECUNoResponse, OTAClientCall from .configs import config as cfg from .ecu_status_box import ECUStatusDisplayBox @@ -36,8 +38,11 @@ async def status_polling_thread( ): while not stop_event.is_set(): try: - resp = await OtaClientCall.status_call( - ecu_id, host, port, request=proto_wrapper.StatusRequest() + resp = await OTAClientCall.status_call( + ecu_id, + host, + port, + request=api_types.StatusRequest(), ) que.put_nowait(resp) except ECUNoResponse: @@ -84,7 +89,7 @@ def _polling_thread(): def _update_thread(): while not self._stop_event.is_set(): - _ecu_status: proto_wrapper.StatusResponse = self._que.get() + _ecu_status: api_types.StatusResponse = self._que.get() if _ecu_status is self._END_SENTINEL: return diff --git a/tools/test_utils/README.md b/tools/test_utils/README.md deleted file mode 100644 index 8384ef764..000000000 --- a/tools/test_utils/README.md +++ /dev/null @@ -1,43 +0,0 @@ -# Test utils for debugging otaclient - -This test utils set provides lib to directly query otaclient `update/status/rollback` API, and a tool to simulate dummy multi-ecu setups. - -## Usage guide for setting up test environemnt - -This test_utils can be used to setup a test environment consists of a real otaclient(either on VM or on actually ECU) as main ECU, -and setup many dummy subECUs that can receive update request and return the expected status report. - -### 1. Install the otaclient's dependencies - -`test_utils` depends on otaclient, so you need to install at least the dependencies of otaclient. -Please refer to [docs/INSTALLATION.md](docs/INSTALLATION.md). - -### 2. Update the `ecu_info.yaml` and `update_request.yaml` accordingly - -Update the `ecu_info.yaml` under the `test_utils` folder as your test environment design, -the example `ecu_info.yaml` consists of one mainECU `autoware`(expecting to be a real otaclient), -and 2 dummy subECUs which will be prepared at step 2. - -Update the `update_request.yaml` under the `test_utils` folder as your test environment setup. -This file contains the update request to be sent. - -### 3. Launch `setup_ecu.py` to setup dummy subECUs, and launch the real otaclient - -Setup subECUs: - -```python -# with venv, under the tools/ folder -python3 -m test_utils.setup_ecu subecus -``` - -And then launch the real otaclient, be sure that the otaclient is reachable to the machine -that running the test_utils. - -### 4. Send an update request to main ECU - -For example, we have `autoware` ECU as main ECU, then - -```python -# with venv, under the tools/ folder -python3 -m test_utils.api_caller update -t autoware -``` diff --git a/tools/test_utils/__init__.py b/tools/test_utils/__init__.py deleted file mode 100644 index bcfd866ad..000000000 --- a/tools/test_utils/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2022 TIER IV, INC. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tools/test_utils/_logutil.py b/tools/test_utils/_logutil.py deleted file mode 100644 index a26aabd64..000000000 --- a/tools/test_utils/_logutil.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2022 TIER IV, INC. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import logging - -_log_format = ( - "[%(asctime)s][%(levelname)s]-%(filename)s:%(funcName)s:%(lineno)d,%(message)s" -) -logging.basicConfig(format=_log_format) - - -def get_logger(name: str, level: int = logging.DEBUG) -> logging.Logger: - logger = logging.getLogger(name) - logger.setLevel(level) - return logger diff --git a/tools/test_utils/_update_call.py b/tools/test_utils/_update_call.py deleted file mode 100644 index b3c45fcbb..000000000 --- a/tools/test_utils/_update_call.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2022 TIER IV, INC. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import yaml - -from otaclient.app.ota_client_call import ECUNoResponse, OtaClientCall -from otaclient.app.proto import wrapper - -from . import _logutil - -logger = _logutil.get_logger(__name__) - - -def load_external_update_request(request_yaml_file: str) -> wrapper.UpdateRequest: - with open(request_yaml_file, "r") as f: - try: - request_yaml = yaml.safe_load(f) - assert isinstance(request_yaml, list), "expect update request to be a list" - except Exception as e: - logger.exception(f"invalid update request yaml: {e!r}") - raise - - logger.info(f"load external request: {request_yaml!r}") - request = wrapper.UpdateRequest() - for request_ecu in request_yaml: - request.ecu.append(wrapper.UpdateRequestEcu(**request_ecu)) - return request - - -async def call_update( - ecu_id: str, - ecu_ip: str, - ecu_port: int, - *, - request_file: str, -): - logger.debug(f"request update on ecu(@{ecu_id}) at {ecu_ip}:{ecu_port}") - update_request = load_external_update_request(request_file) - - try: - update_response = await OtaClientCall.update_call( - ecu_id, ecu_ip, ecu_port, request=update_request - ) - logger.info(f"{update_response.export_pb()=}") - except ECUNoResponse as e: - logger.exception(f"update request failed: {e!r}") diff --git a/tools/test_utils/api_caller.py b/tools/test_utils/api_caller.py deleted file mode 100644 index 529e9cd54..000000000 --- a/tools/test_utils/api_caller.py +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright 2022 TIER IV, INC. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import argparse -import asyncio -import sys -from pathlib import Path - -import yaml - -try: - import otaclient # noqa: F401 -except ImportError: - sys.path.insert(0, str(Path(__file__).parent.parent.parent)) -from . import _logutil, _update_call - -logger = _logutil.get_logger(__name__) - - -async def main(args: argparse.Namespace): - with open(args.ecu_info, "r") as f: - ecu_info = yaml.safe_load(f) - assert isinstance(ecu_info, dict) - - target_ecu_id = args.target - # by default, send request to main ECU - try: - ecu_id = ecu_info["ecu_id"] - ecu_ip = ecu_info["ip_addr"] - ecu_port = 50051 - except KeyError: - raise ValueError(f"invalid ecu_info: {ecu_info=}") - - if target_ecu_id != ecu_info.get("ecu_id"): - found = False - # search for target by ecu_id - for subecu in ecu_info.get("secondaries", []): - try: - if subecu["ecu_id"] == target_ecu_id: - ecu_id = subecu["ecu_id"] - ecu_ip = subecu["ip_addr"] - ecu_port = int(subecu.get("port", 50051)) - found = True - break - except KeyError: - continue - - if not found: - logger.critical(f"target ecu {target_ecu_id} is not found") - sys.exit(-1) - - logger.info(f"send request to target ecu: {ecu_id=}, {ecu_ip=}") - cmd = args.command - if cmd == "update": - await _update_call.call_update( - ecu_id, - ecu_ip, - ecu_port, - request_file=args.request, - ) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="calling ECU's API", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - parser.add_argument( - "-c", - "--ecu_info", - type=str, - default="test_utils/ecu_info.yaml", - help="ecu_info file to configure the caller", - ) - parser.add_argument("command", help="API to call, available API: update") - parser.add_argument( - "-t", - "--target", - default="autoware", - help="indicate the target for the API request", - ) - parser.add_argument( - "-r", - "--request", - default="test_utils/update_request.yaml", - help="(update) yaml file that contains the request to send", - ) - - args = parser.parse_args() - if args.command != "update": - parser.error(f"unknown API: {args.command} (available: update)") - if not Path(args.ecu_info).is_file(): - parser.error(f"ecu_info file {args.ecu_info} not found!") - if args.command == "update" and not Path(args.request).is_file(): - parser.error(f"update request file {args.request} not found!") - - asyncio.run(main(args)) diff --git a/tools/test_utils/ecu_info.yaml b/tools/test_utils/ecu_info.yaml deleted file mode 100644 index 2c8c7c8e4..000000000 --- a/tools/test_utils/ecu_info.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# sample ecu_info.yaml, with 2 perception ECUs -format_version: 1 -ecu_id: "autoware" -ip_addr: "10.0.0.2" -secondaries: - - ecu_id: "p1" - ip_addr: "10.0.0.11" - - ecu_id: "p2" - ip_addr: "10.0.0.12" diff --git a/tools/test_utils/setup_ecu.py b/tools/test_utils/setup_ecu.py deleted file mode 100644 index 4190262a6..000000000 --- a/tools/test_utils/setup_ecu.py +++ /dev/null @@ -1,197 +0,0 @@ -# Copyright 2022 TIER IV, INC. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import argparse -import asyncio -import sys -from pathlib import Path -from typing import List - -import grpc -import yaml - -try: - import otaclient # noqa: F401 -except ImportError: - sys.path.insert(0, str(Path(__file__).parent.parent.parent)) - -from otaclient.app.ota_client_service import service_wait_for_termination -from otaclient.app.proto import v2, v2_grpc - -from . import _logutil - -logger = _logutil.get_logger(__name__) - -_DEFAULT_PORT = 50051 -_MODE = {"standalone", "mainecu", "subecus"} - - -class MiniOtaClientServiceV2(v2_grpc.OtaClientServiceServicer): - UPDATE_TIME_COST = 10 - REBOOT_INTERVAL = 5 - - def __init__(self, ecu_id: str): - self.ecu_id = ecu_id - self._lock = asyncio.Lock() - self._in_update = asyncio.Event() - self._rebooting = asyncio.Event() - - async def _on_update(self): - await asyncio.sleep(self.UPDATE_TIME_COST) - logger.debug(f"{self.ecu_id=} finished update, rebooting...") - self._rebooting.set() - await asyncio.sleep(self.REBOOT_INTERVAL) - self._rebooting.clear() - self._in_update.clear() - - async def Update(self, request: v2.UpdateRequest, context: grpc.ServicerContext): - peer = context.peer() - logger.debug(f"{self.ecu_id}: update request from {peer=}") - logger.debug(f"{request=}") - - # return if not listed as target - found = False - for ecu in request.ecu: - if ecu.ecu_id == self.ecu_id: - found = True - break - if not found: - logger.debug(f"{self.ecu_id}, Update: not listed as update target, abort") - return v2.UpdateResponse() - - results = v2.UpdateResponse() - if self._in_update.is_set(): - resp_ecu = v2.UpdateResponseEcu( - ecu_id=self.ecu_id, - result=v2.RECOVERABLE, - ) - results.ecu.append(resp_ecu) - else: - logger.debug("start update") - self._in_update.set() - asyncio.create_task(self._on_update()) - - return results - - async def Status(self, _, context: grpc.ServicerContext): - peer = context.peer() - logger.debug(f"{self.ecu_id}: status request from {peer=}") - if self._rebooting.is_set(): - return v2.StatusResponse() - - result_ecu = v2.StatusResponseEcu( - ecu_id=self.ecu_id, - result=v2.NO_FAILURE, - ) - - ecu_status = result_ecu.status - if self._in_update.is_set(): - ecu_status.status = v2.UPDATING - else: - ecu_status.status = v2.SUCCESS - - result = v2.StatusResponse() - result.ecu.append(result_ecu) - return result - - -async def launch_otaclient(ecu_id, ecu_ip, ecu_port): - server = grpc.aio.server() - service = MiniOtaClientServiceV2(ecu_id) - v2_grpc.add_OtaClientServiceServicer_to_server(service, server) - - server.add_insecure_port(f"{ecu_ip}:{ecu_port}") - await server.start() - await service_wait_for_termination(server) - - -async def mainecu_mode(ecu_info_file: str): - ecu_info = yaml.safe_load(Path(ecu_info_file).read_text()) - ecu_id = ecu_info["ecu_id"] - ecu_ip = ecu_info["ip_addr"] - ecu_port = int(ecu_info.get("port", _DEFAULT_PORT)) - - logger.info(f"start {ecu_id=} at {ecu_ip}:{ecu_port}") - await launch_otaclient(ecu_id, ecu_ip, ecu_port) - - -async def subecu_mode(ecu_info_file: str): - ecu_info = yaml.safe_load(Path(ecu_info_file).read_text()) - - # schedule the servers to the thread pool - tasks: List[asyncio.Task] = [] - for subecu in ecu_info["secondaries"]: - ecu_id = subecu["ecu_id"] - ecu_ip = subecu["ip_addr"] - ecu_port = int(subecu.get("port", _DEFAULT_PORT)) - logger.info(f"start {ecu_id=} at {ecu_ip}:{ecu_port}") - tasks.append(asyncio.create_task(launch_otaclient(ecu_id, ecu_ip, ecu_port))) - - await asyncio.gather(*tasks) - - -async def standalone_mode(args: argparse.Namespace): - await launch_otaclient("standalone", args.ip, args.port) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="calling main ECU's API", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - parser.add_argument( - "-c", "--ecu_info", default="test_utils/ecu_info.yaml", help="ecu_info" - ) - parser.add_argument( - "mode", - default="standalone", - help=( - "running mode for mini_ota_client(standalone, subecus, mainecu)\n" - "\tstandalone: run a single mini_ota_client\n" - "\tmainecu: run a single mini_ota_client as mainecu according to ecu_info.yaml\n" - "\tsubecus: run subecu(s) according to ecu_info.yaml" - ), - ) - parser.add_argument( - "--ip", - default="127.0.0.1", - help="(standalone) listen at IP", - ) - parser.add_argument( - "--port", - default=_DEFAULT_PORT, - help="(standalone) use port PORT", - ) - - args = parser.parse_args() - - if args.mode not in _MODE: - parser.error(f"invalid mode {args.mode}, should be one of {_MODE}") - if args.mode != "standalone" and not Path(args.ecu_info).is_file(): - parser.error( - f"invalid ecu_info_file {args.ecu_info!r}. ecu_info.yaml is required for non-standalone mode" - ) - - if args.mode == "subecus": - logger.info("subecus mode") - coro = subecu_mode(args.ecu_info) - elif args.mode == "mainecu": - logger.info("mainecu mode") - coro = mainecu_mode(args.ecu_info) - else: - logger.info("standalone mode") - coro = standalone_mode(args) - - asyncio.run(coro) diff --git a/tools/test_utils/update_request.yaml b/tools/test_utils/update_request.yaml deleted file mode 100644 index 02e612a9e..000000000 --- a/tools/test_utils/update_request.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# sample update request -- ecu_id: "autoware" - version: "789.x" - url: "http://10.0.0.1:8080" - cookies: '{"test": "my-cookie"}' -- ecu_id: "p1" - version: "789.x" - url: "http://10.0.0.1:8080" - cookies: '{"test": "my-cookie"}' -- ecu_id: "p2" - version: "789.x" - url: "http://10.0.0.1:8080" - cookies: '{"test": "my-cookie"}' From 8d0990cca5b5c88c439d2a149629235be3185e9c Mon Sep 17 00:00:00 2001 From: Bodong Yang Date: Wed, 5 Jun 2024 13:08:16 +0000 Subject: [PATCH 048/193] minor fix --- src/otaclient/app/boot_control/_jetson_uefi.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/otaclient/app/boot_control/_jetson_uefi.py b/src/otaclient/app/boot_control/_jetson_uefi.py index aca4774d5..83e5ee9f6 100644 --- a/src/otaclient/app/boot_control/_jetson_uefi.py +++ b/src/otaclient/app/boot_control/_jetson_uefi.py @@ -28,11 +28,11 @@ from pathlib import Path from typing import Any, Generator -from otaclient._utils.typing import StrOrPath +from otaclient_common.typing import StrOrPath from otaclient.app import errors as ota_errors -from otaclient.app.common import subprocess_call, write_str_to_file_sync +from otaclient_common.common import subprocess_call, write_str_to_file_sync from otaclient.app.configs import config as cfg -from otaclient.app.proto import wrapper +from otaclient_api.v2 import types as api_types from ._common import CMDHelperFuncs, OTAStatusFilesControl, SlotMountHelper from ._jetson_common import ( @@ -651,5 +651,5 @@ def on_operation_failure(self): def load_version(self) -> str: return self._ota_status_control.load_active_slot_version() - def get_booted_ota_status(self) -> wrapper.StatusOta: + def get_booted_ota_status(self) -> api_types.StatusOta: return self._ota_status_control.booted_ota_status From 7ee8d2005ebd1ad25c0200c395625dff205857a0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 5 Jun 2024 13:08:47 +0000 Subject: [PATCH 049/193] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/otaclient/app/boot_control/_jetson_uefi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/otaclient/app/boot_control/_jetson_uefi.py b/src/otaclient/app/boot_control/_jetson_uefi.py index 83e5ee9f6..e086d165d 100644 --- a/src/otaclient/app/boot_control/_jetson_uefi.py +++ b/src/otaclient/app/boot_control/_jetson_uefi.py @@ -28,11 +28,11 @@ from pathlib import Path from typing import Any, Generator -from otaclient_common.typing import StrOrPath from otaclient.app import errors as ota_errors -from otaclient_common.common import subprocess_call, write_str_to_file_sync from otaclient.app.configs import config as cfg from otaclient_api.v2 import types as api_types +from otaclient_common.common import subprocess_call, write_str_to_file_sync +from otaclient_common.typing import StrOrPath from ._common import CMDHelperFuncs, OTAStatusFilesControl, SlotMountHelper from ._jetson_common import ( From cff9c8278fcdec8f10fff36b69b5979e836aa44a Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Sun, 9 Jun 2024 15:43:49 +0000 Subject: [PATCH 050/193] jetson-uefi: clear current slot firmware bsp on a failed OTA reboot --- .../app/boot_control/_jetson_uefi.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/otaclient/app/boot_control/_jetson_uefi.py b/src/otaclient/app/boot_control/_jetson_uefi.py index e086d165d..6b4066242 100644 --- a/src/otaclient/app/boot_control/_jetson_uefi.py +++ b/src/otaclient/app/boot_control/_jetson_uefi.py @@ -445,26 +445,23 @@ def _finalize_switching_boot(self) -> bool: "assuming firmware update failed" ) ) + self.current_fwupdate_hint_fpath.unlink(missing_ok=True) return False # ------ firmware update occurs, check hints ------ # - if slot_id != current_slot: + if slot_id != current_slot or bsp_v != current_slot_bsp_ver: logger.error( ( "firmware update hint file indicates firmware update occurs on " - f"slot {slot_id}, but expects slot {current_slot}" - ) - ) - return False - - if bsp_v != current_slot_bsp_ver: - logger.error( - ( - f"firmware update hint file indicates the firmware on slot {slot_id} is " - f"updated to {bsp_v}, but current slot's BSP version is {current_slot_bsp_ver}" + f"slot {slot_id} with version {bsp_v}, " + f"but expects slot {current_slot} with version {current_slot_bsp_ver}" ) ) + # NOTE(20240610): on a failed OTA with firmware update, we always assume a failed + # firmware update(even the failure might be caused by rootfs update failed). + self._firmware_ver_control.set_version_by_slot(current_slot, None) + self._firmware_ver_control.write_current_firmware_bsp_version() return False # ------ firmware update succeeded, write firmware version file ------ # From 59aa47ebc4777790ad5c787da3ca0cbb5c3defb8 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Tue, 11 Jun 2024 07:28:50 +0000 Subject: [PATCH 051/193] jetson-uefi: add support for skipping firmware update --- src/otaclient/app/boot_control/configs.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/otaclient/app/boot_control/configs.py b/src/otaclient/app/boot_control/configs.py index ae69a4edc..2555c0b3e 100644 --- a/src/otaclient/app/boot_control/configs.py +++ b/src/otaclient/app/boot_control/configs.py @@ -66,6 +66,7 @@ class JetsonUEFIBootControlConfig(JetsonBootCommon): BOOTLOADER = BootloaderType.JETSON_UEFI TEGRA_COMPAT_PATH = "/sys/firmware/devicetree/base/compatible" FIRMWARE_LIST = ["bl_only_payload.Cap"] + L4TLAUNCHER_FNAME = "BOOTAA64.efi" ESP_MOUNTPOINT = "/mnt/esp" ESP_PARTLABEL = "esp" EFIVARS_DPATH = "/sys/firmware/efi/efivars/" @@ -75,6 +76,9 @@ class JetsonUEFIBootControlConfig(JetsonBootCommon): CAPSULE_PAYLOAD_AT_ROOTFS = "/opt/ota_package/" FIRMWARE_UPDATE_HINT_FNAME = ".firmware_update" + NO_FIRMWARE_UPDATE_HINT_FNAME = ".otaclient_no_firmware_update" + """Skip firmware update if this file is presented.""" + @dataclass class RPIBootControlConfig(BaseConfig): From b529a9be42dd32b916b8bf6fdeec64ebc2837915 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Tue, 11 Jun 2024 07:31:19 +0000 Subject: [PATCH 052/193] jetson-uefi: remove unused --- .../app/boot_control/_jetson_uefi.py | 28 ------------------- 1 file changed, 28 deletions(-) diff --git a/src/otaclient/app/boot_control/_jetson_uefi.py b/src/otaclient/app/boot_control/_jetson_uefi.py index 6b4066242..e366d7f49 100644 --- a/src/otaclient/app/boot_control/_jetson_uefi.py +++ b/src/otaclient/app/boot_control/_jetson_uefi.py @@ -23,7 +23,6 @@ import contextlib import logging import os -import re import shutil from pathlib import Path from typing import Any, Generator @@ -62,33 +61,6 @@ class _NVBootctrl(NVBootctrlCommon): Without -t option, the target will be bootloader by default. """ - CAPSULE_UPDATE_PATTERN = re.compile(r"Capsule update status: (?P\d+)") - - @classmethod - def get_capsule_update_result(cls) -> str: - """Check the Capsule update status. - - NOTE: this is NOT a nvbootctrl command, but implemented by parsing - the result of calling nvbootctrl dump-slots-info. - - The output value of Capsule update status can be following: - 0 - No Capsule update - 1 - Capsule update successfully - 2 - Capsule install successfully but boot new firmware failed - 3 - Capsule install failed - - Returns: - The Capsulte update result status. - """ - slots_info = cls.dump_slots_info() - logger.info(f"checking Capsule update result: \n{slots_info}") - - ma = cls.CAPSULE_UPDATE_PATTERN.search(slots_info) - assert ma, "failed to get Capsule update result" - - update_result = ma.group("status") - return update_result - class CapsuleUpdate: """Firmware update implementation using Capsule update.""" From 234111abf60d38843ed2196054020405e4f2e153 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Tue, 11 Jun 2024 07:57:44 +0000 Subject: [PATCH 053/193] jetson-uefi: add get_current_fw_bsp_version --- .../app/boot_control/_jetson_common.py | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/otaclient/app/boot_control/_jetson_common.py b/src/otaclient/app/boot_control/_jetson_common.py index b4d3b79b6..d28de5ef3 100644 --- a/src/otaclient/app/boot_control/_jetson_common.py +++ b/src/otaclient/app/boot_control/_jetson_common.py @@ -174,6 +174,26 @@ def dump_slots_info(cls, *, target: Optional[NVBootctrlTarget] = None) -> str: return cls._nvbootctrl(cmd, target=target, check_output=True) +def get_current_fw_bsp_version() -> BSPVersion | None: + """Get current boot chain's firmware BSP version with nvbootctrl.""" + _raw = NVBootctrlCommon.dump_slots_info() + pa = re.compile(r"\s*Current version:\s(?P[\.\d]+)\s*") + + if not (ma := pa.search(_raw)): + logger.warning("nvbootctrl failed to report BSP version") + return + + bsp_ver_str = ma.group("bsp_ver") + bsp_ver = BSPVersion.parse(bsp_ver_str) + if bsp_ver.major_rev == 0: + logger.warning( + f"invalid BSP version: {bsp_ver_str}, this might indicate broken firmware" + ) + logger.warning("return empty bsp version") + return + return bsp_ver + + class FirmwareBSPVersionControl: """firmware_bsp_version ota-status file for tracking firmware version. @@ -212,7 +232,7 @@ def write_standby_firmware_bsp_version(self) -> None: def get_version_by_slot(self, slot_id: SlotID) -> Optional[BSPVersion]: """Get slot's firmware version from memory.""" - if slot_id == "0": + if slot_id == SlotID("0"): return self._version.slot_a return self._version.slot_b @@ -220,7 +240,7 @@ def set_version_by_slot( self, slot_id: SlotID, version: Optional[BSPVersion] ) -> None: """Set slot's firmware version into memory.""" - if slot_id == "0": + if slot_id == SlotID("0"): self._version.slot_a = version else: self._version.slot_b = version From 521c8beb71e34fde6759b1d6e339b2e1a500f238 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Tue, 11 Jun 2024 08:48:38 +0000 Subject: [PATCH 054/193] jetson-uefi: FirmwareBSPVersionControl now always check current slot's firmware version by nvbootctrl --- .../app/boot_control/_jetson_common.py | 84 ++++++++++++------- 1 file changed, 55 insertions(+), 29 deletions(-) diff --git a/src/otaclient/app/boot_control/_jetson_common.py b/src/otaclient/app/boot_control/_jetson_common.py index d28de5ef3..9b9d2f4bd 100644 --- a/src/otaclient/app/boot_control/_jetson_common.py +++ b/src/otaclient/app/boot_control/_jetson_common.py @@ -95,6 +95,22 @@ class FirmwareBSPVersion(BaseModel): slot_a: Optional[BSPVersionStr] = None slot_b: Optional[BSPVersionStr] = None + def set_by_slot(self, slot_id: SlotID, ver: BSPVersion | None) -> None: + if slot_id == SlotID("0"): + self.slot_a = ver + elif slot_id == SlotID("1"): + self.slot_b = ver + else: + raise ValueError(f"invalid slot_id: {slot_id}") + + def get_by_slot(self, slot_id: SlotID) -> BSPVersion | None: + if slot_id == SlotID("0"): + return self.slot_a + elif slot_id == SlotID("1"): + return self.slot_b + else: + raise ValueError(f"invalid slot_id: {slot_id}") + NVBootctrlTarget = Literal["bootloader", "rootfs"] @@ -173,25 +189,25 @@ def dump_slots_info(cls, *, target: Optional[NVBootctrlTarget] = None) -> str: cmd = "dump-slots-info" return cls._nvbootctrl(cmd, target=target, check_output=True) - -def get_current_fw_bsp_version() -> BSPVersion | None: - """Get current boot chain's firmware BSP version with nvbootctrl.""" - _raw = NVBootctrlCommon.dump_slots_info() - pa = re.compile(r"\s*Current version:\s(?P[\.\d]+)\s*") - - if not (ma := pa.search(_raw)): - logger.warning("nvbootctrl failed to report BSP version") - return - - bsp_ver_str = ma.group("bsp_ver") - bsp_ver = BSPVersion.parse(bsp_ver_str) - if bsp_ver.major_rev == 0: - logger.warning( - f"invalid BSP version: {bsp_ver_str}, this might indicate broken firmware" - ) - logger.warning("return empty bsp version") - return - return bsp_ver + @classmethod + def get_current_fw_bsp_version(cls) -> BSPVersion | None: + """Get current boot chain's firmware BSP version with nvbootctrl.""" + _raw = cls.dump_slots_info() + pa = re.compile(r"\s*Current version:\s(?P[\.\d]+)\s*") + + if not (ma := pa.search(_raw)): + logger.warning("nvbootctrl failed to report BSP version") + return + + bsp_ver_str = ma.group("bsp_ver") + bsp_ver = BSPVersion.parse(bsp_ver_str) + if bsp_ver.major_rev == 0: + logger.warning( + f"invalid BSP version: {bsp_ver_str}, this might indicate broken firmware" + ) + logger.warning("return empty bsp version") + return + return bsp_ver class FirmwareBSPVersionControl: @@ -205,23 +221,38 @@ class FirmwareBSPVersionControl: """ def __init__( - self, current_firmware_bsp_vf: Path, standby_firmware_bsp_vf: Path + self, + current_firmware_bsp_vf: Path, + standby_firmware_bsp_vf: Path, ) -> None: self._current_fw_bsp_vf = current_firmware_bsp_vf self._standby_fw_bsp_vf = standby_firmware_bsp_vf self._version = FirmwareBSPVersion() try: - self._version = _version = FirmwareBSPVersion.model_validate_json( + self._version = version_from_file = FirmwareBSPVersion.model_validate_json( self._current_fw_bsp_vf.read_text() ) - logger.info(f"loaded firmware_version from version file: {_version}") + logger.info( + f"loaded firmware_version from version file: {version_from_file}" + ) except Exception as e: logger.warning( f"invalid or missing firmware_bsp_verion file, removed: {e!r}" ) self._current_fw_bsp_vf.unlink(missing_ok=True) + current_slot_bsp_ver = NVBootctrlCommon.get_current_fw_bsp_version() + logger.info( + f"query current slot bsp ver from nvbootctrl: {current_slot_bsp_ver}" + ) + self._version.set_by_slot( + NVBootctrlCommon.get_current_slot(), + current_slot_bsp_ver, + ) + self.write_current_firmware_bsp_version() + logger.info("write current slot's firmware_bsp_version file") + def write_current_firmware_bsp_version(self) -> None: """Write firmware_bsp_version from memory to firmware_bsp_version file.""" write_str_to_file_sync(self._current_fw_bsp_vf, self._version.model_dump_json()) @@ -232,18 +263,13 @@ def write_standby_firmware_bsp_version(self) -> None: def get_version_by_slot(self, slot_id: SlotID) -> Optional[BSPVersion]: """Get slot's firmware version from memory.""" - if slot_id == SlotID("0"): - return self._version.slot_a - return self._version.slot_b + return self._version.get_by_slot(slot_id) def set_version_by_slot( self, slot_id: SlotID, version: Optional[BSPVersion] ) -> None: """Set slot's firmware version into memory.""" - if slot_id == SlotID("0"): - self._version.slot_a = version - else: - self._version.slot_b = version + self._version.set_by_slot(slot_id, version) BSP_VER_PA = re.compile( From 46ea74087bd54c83997de4f6221cf00264f299d9 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Tue, 11 Jun 2024 08:52:58 +0000 Subject: [PATCH 055/193] jetson-uefi: detect firmware bsp version by nvbootctrl --- src/otaclient/app/boot_control/_jetson_uefi.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/otaclient/app/boot_control/_jetson_uefi.py b/src/otaclient/app/boot_control/_jetson_uefi.py index e366d7f49..97bc27138 100644 --- a/src/otaclient/app/boot_control/_jetson_uefi.py +++ b/src/otaclient/app/boot_control/_jetson_uefi.py @@ -247,11 +247,12 @@ def __init__(self): raise JetsonUEFIBootControlError(_err_msg) logger.info(f"dev compatibility: {compat_info}") - # ------ check BSP version ------ # + # ------ check firmware BSP version ------ # try: - self.bsp_version = bsp_version = parse_bsp_version( - Path(boot_cfg.NV_TEGRA_RELEASE_FPATH).read_text() + self.bsp_version = bsp_version = ( + NVBootctrlCommon.get_current_fw_bsp_version() ) + assert bsp_version, "bsp version information not available" except Exception as e: _err_msg = f"failed to detect BSP version: {e!r}" logger.error(_err_msg) From d6fef909697eba91c306ea1cb626f52e4b075993 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Tue, 11 Jun 2024 08:59:45 +0000 Subject: [PATCH 056/193] jetson-uefi: distiguish firmware BSP version and rootfs BSP version --- .../app/boot_control/_jetson_uefi.py | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/otaclient/app/boot_control/_jetson_uefi.py b/src/otaclient/app/boot_control/_jetson_uefi.py index 97bc27138..77923d4dd 100644 --- a/src/otaclient/app/boot_control/_jetson_uefi.py +++ b/src/otaclient/app/boot_control/_jetson_uefi.py @@ -247,18 +247,38 @@ def __init__(self): raise JetsonUEFIBootControlError(_err_msg) logger.info(f"dev compatibility: {compat_info}") - # ------ check firmware BSP version ------ # + # ------ check BSP version ------ # + # check firmware BSP version try: self.bsp_version = bsp_version = ( NVBootctrlCommon.get_current_fw_bsp_version() ) assert bsp_version, "bsp version information not available" + logger.info(f"current slot firmware BSP version: {bsp_version}") except Exception as e: _err_msg = f"failed to detect BSP version: {e!r}" logger.error(_err_msg) raise JetsonUEFIBootControlError(_err_msg) logger.info(f"{bsp_version=}") + # check rootfs BSP version + try: + self.rootfs_bsp_verion = rootfs_bsp_version = parse_bsp_version( + Path(boot_cfg.NV_TEGRA_RELEASE_FPATH).read_text() + ) + logger.info(f"current slot rootfs BSP version: {rootfs_bsp_version}") + except Exception as e: + logger.warning(f"failed to detect rootfs BSP version: {e!r}") + self.rootfs_bsp_verion = rootfs_bsp_version = None + + if rootfs_bsp_version and rootfs_bsp_version >= bsp_version: + logger.warning( + ( + "current slot's rootfs bsp version is newer than the firmware bsp version, " + "this indicates the device previously only receive rootfs update, this is unexpected" + ) + ) + # ------ sanity check, currently jetson-uefi only supports >= R35.2 ----- # # only after R35.2, the Capsule Firmware update is available. if not bsp_version >= (35, 2, 0): From a9b0764a8eec1814de27a893ff44f7898f655af2 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Tue, 11 Jun 2024 09:03:46 +0000 Subject: [PATCH 057/193] jetson-uefi: implement otaclient_no_firmware_update hint file --- src/otaclient/app/boot_control/_jetson_uefi.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/otaclient/app/boot_control/_jetson_uefi.py b/src/otaclient/app/boot_control/_jetson_uefi.py index 77923d4dd..214fa9e28 100644 --- a/src/otaclient/app/boot_control/_jetson_uefi.py +++ b/src/otaclient/app/boot_control/_jetson_uefi.py @@ -468,7 +468,11 @@ def _finalize_switching_boot(self) -> bool: return True def _capsule_firmware_update(self) -> bool: - """Perform firmware update with UEFI Capsule update.""" + """Perform firmware update with UEFI Capsule update. + + Returns: + True if there is firmware update configured, False for no firmware update. + """ logger.info("jetson-uefi: checking if we need to do firmware update ...") standby_bootloader_slot = self._uefi_control.standby_slot @@ -480,6 +484,17 @@ def _capsule_firmware_update(self) -> bool: ) # ------ check if we need to do firmware update ------ # + skip_firmware_update_hint_file = ( + self._mp_control.standby_slot_mount_point + / Path(boot_cfg.CAPSULE_PAYLOAD_AT_ROOTFS).relative_to("/") + / Path(boot_cfg.NO_FIRMWARE_UPDATE_HINT_FNAME) + ) + if skip_firmware_update_hint_file.is_file(): + logger.warning( + "target image is configured to not doing firmware update, skip" + ) + return False + _new_bsp_v_fpath = self._mp_control.standby_slot_mount_point / Path( boot_cfg.NV_TEGRA_RELEASE_FPATH ).relative_to("/") From cef189e02ef0fbd43c39e97db97fb23dfa12bdbb Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 12 Jun 2024 07:57:39 +0000 Subject: [PATCH 058/193] jetson-uefi: support l4tlauncher update --- .../app/boot_control/_jetson_uefi.py | 82 +++++++++++++------ 1 file changed, 55 insertions(+), 27 deletions(-) diff --git a/src/otaclient/app/boot_control/_jetson_uefi.py b/src/otaclient/app/boot_control/_jetson_uefi.py index 214fa9e28..ae0bae00c 100644 --- a/src/otaclient/app/boot_control/_jetson_uefi.py +++ b/src/otaclient/app/boot_control/_jetson_uefi.py @@ -74,7 +74,7 @@ def __init__( # i.e., if we boot from nvme0n1, then bootdev_path is /dev/nvme0n1 and # we use the esp at nvme0n1. boot_parent_devpath = str(boot_parent_devpath) - self.esp_mp = boot_cfg.ESP_MOUNTPOINT + self.esp_mp = Path(boot_cfg.ESP_MOUNTPOINT) # NOTE: we get the update capsule from the standby slot self.standby_slot_mp = Path(standby_slot_mp) @@ -127,6 +127,19 @@ def _ensure_efivarfs_mounted(cls) -> Generator[None, Any, None]: f"failed to mount {cls.EFIVARS_FSTYPE} on {boot_cfg.EFIVARS_DPATH}: {e!r}" ) from e + @contextlib.contextmanager + def _ensure_esp_mounted(self) -> Generator[None, Any, None]: + """Mount the esp partition and then umount it.""" + self.esp_mp.mkdir(exist_ok=True, parents=True) + try: + CMDHelperFuncs.mount_rw(self.esp_part, self.esp_mp) + yield + CMDHelperFuncs.umount(self.esp_mp, raise_exception=False) + except Exception as e: + _err_msg = f"failed to mount {self.esp_part=} to {self.esp_mp}: {e!r}" + logger.error(_err_msg) + raise JetsonUEFIBootControlError(_err_msg) from e + def _prepare_payload(self) -> bool: """Copy the Capsule update payloads to specific location at esp partition. @@ -134,38 +147,47 @@ def _prepare_payload(self) -> bool: True if at least one of the update capsule is prepared, False if no update capsule is available and configured. """ - esp_mp = Path(self.esp_mp) - esp_mp.mkdir(exist_ok=True, parents=True) - - try: - CMDHelperFuncs.mount_rw(self.esp_part, esp_mp) - except Exception as e: - _err_msg = f"failed to mount {self.esp_part=} to {esp_mp}: {e!r}" - logger.error(_err_msg) - raise JetsonUEFIBootControlError(_err_msg) from e - - capsule_at_esp = esp_mp / boot_cfg.CAPSULE_PAYLOAD_AT_ESP + capsule_at_esp = self.esp_mp / boot_cfg.CAPSULE_PAYLOAD_AT_ESP capsule_at_esp.mkdir(parents=True, exist_ok=True) - capsule_at_standby_slot = self.standby_slot_mp / Path( + # where the fw update capsule and l4tlauncher bin located + fw_loc_at_standby_slot = self.standby_slot_mp / Path( boot_cfg.CAPSULE_PAYLOAD_AT_ROOTFS ).relative_to("/") + # ------ prepare capsule update payload ------ # firmware_package_configured = False for capsule_fname in boot_cfg.FIRMWARE_LIST: try: shutil.copy( - src=capsule_at_standby_slot / capsule_fname, + src=fw_loc_at_standby_slot / capsule_fname, dst=capsule_at_esp / capsule_fname, ) firmware_package_configured = True + logger.info(f"copy {capsule_fname} to {capsule_at_esp}") except Exception as e: logger.warning( - f"failed to copy {capsule_fname} from {capsule_at_standby_slot} to {capsule_at_esp}: {e!r}" + f"failed to copy {capsule_fname} from {fw_loc_at_standby_slot} to {capsule_at_esp}: {e!r}" ) logger.warning(f"skip {capsule_fname}") - CMDHelperFuncs.umount(esp_mp, raise_exception=False) + # ------ prepare L4TLauncher update ------ # + # NOTE(20240611): Assume that the new L4TLauncher always keeps backward compatibility to + # work with old firmware. This assumption MUST be confirmed on the real ECU. + if firmware_package_configured: + logger.info("capsule update is scheduled, also update L4TLauncher") + esp_boot_dir = self.esp_mp / "EFI" / "BOOT" + + # the canonical boot entry used by UEFI + bootaa64_at_esp = esp_boot_dir / boot_cfg.L4TLAUNCHER_FNAME + + bootaa64_at_esp_bak = esp_boot_dir / f"{boot_cfg.L4TLAUNCHER_FNAME}_bak" + # l4tlauncher is stored as /BOOTAA64.efi + bootaa64_at_standby = fw_loc_at_standby_slot / boot_cfg.L4TLAUNCHER_FNAME + + shutil.copy(bootaa64_at_esp, bootaa64_at_esp_bak) + shutil.copy(bootaa64_at_standby, bootaa64_at_esp) + os.sync() return firmware_package_configured def _write_efivar(self) -> None: @@ -177,13 +199,8 @@ def _write_efivar(self) -> None: magic_efivar_fpath = ( Path(boot_cfg.EFIVARS_DPATH) / boot_cfg.UPDATE_TRIGGER_EFIVAR ) - try: - magic_efivar_fpath.write_bytes(boot_cfg.MAGIC_BYTES) - os.sync() - except Exception as e: - _err_msg = f"failed to write magic bytes into {magic_efivar_fpath}: {e!r}" - logger.error(_err_msg) - raise JetsonUEFIBootControlError(_err_msg) from e + magic_efivar_fpath.write_bytes(boot_cfg.MAGIC_BYTES) + os.sync() def firmware_update(self) -> bool: """Trigger firmware update in next boot. @@ -191,12 +208,23 @@ def firmware_update(self) -> bool: Returns: True if firmware update is configured, False if there is no firmware update. """ - if not self._prepare_payload(): - logger.info("no firmware file is prepared, skip firmware update") - return False + with self._ensure_esp_mounted(): + if not self._prepare_payload(): + logger.info("no firmware file is prepared, skip firmware update") + return False with self._ensure_efivarfs_mounted(): - self._write_efivar() + try: + self._write_efivar() + except Exception as e: + logger.warning( + ( + f"failed to configure capsule update by write magic value: {e!r}\n" + "firmware update might be skipped!" + ) + ) + return False + logger.info("firmware update package prepare finished") return True From e8a1b89e757f8637a539695c0e1ab52aeb5297d4 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 12 Jun 2024 09:17:19 +0000 Subject: [PATCH 059/193] minor fix --- src/otaclient/app/boot_control/_jetson_common.py | 13 ++++++++++++- src/otaclient/app/boot_control/_jetson_uefi.py | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/otaclient/app/boot_control/_jetson_common.py b/src/otaclient/app/boot_control/_jetson_common.py index 9b9d2f4bd..e44299905 100644 --- a/src/otaclient/app/boot_control/_jetson_common.py +++ b/src/otaclient/app/boot_control/_jetson_common.py @@ -193,13 +193,24 @@ def dump_slots_info(cls, *, target: Optional[NVBootctrlTarget] = None) -> str: def get_current_fw_bsp_version(cls) -> BSPVersion | None: """Get current boot chain's firmware BSP version with nvbootctrl.""" _raw = cls.dump_slots_info() + """Example: + Current version: 35.4.1 + Capsule update status: 1 + Current bootloader slot: A + Active bootloader slot: A + num_slots: 2 + slot: 0, status: normal + slot: 1, status: normal + """ pa = re.compile(r"\s*Current version:\s(?P[\.\d]+)\s*") if not (ma := pa.search(_raw)): logger.warning("nvbootctrl failed to report BSP version") return - bsp_ver_str = ma.group("bsp_ver") + bsp_ver_str = ( + f"r{ma.group('bsp_ver')}" # NOTE: need to add 'r' prefix back here + ) bsp_ver = BSPVersion.parse(bsp_ver_str) if bsp_ver.major_rev == 0: logger.warning( diff --git a/src/otaclient/app/boot_control/_jetson_uefi.py b/src/otaclient/app/boot_control/_jetson_uefi.py index ae0bae00c..817e68f27 100644 --- a/src/otaclient/app/boot_control/_jetson_uefi.py +++ b/src/otaclient/app/boot_control/_jetson_uefi.py @@ -299,7 +299,7 @@ def __init__(self): logger.warning(f"failed to detect rootfs BSP version: {e!r}") self.rootfs_bsp_verion = rootfs_bsp_version = None - if rootfs_bsp_version and rootfs_bsp_version >= bsp_version: + if rootfs_bsp_version and rootfs_bsp_version > bsp_version: logger.warning( ( "current slot's rootfs bsp version is newer than the firmware bsp version, " From aa0b5fb702cd004894ef6ad9a2adeede930cd247 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Fri, 14 Jun 2024 06:57:42 +0000 Subject: [PATCH 060/193] jetson-uefi: mkdir the current slot's ota-status dir --- src/otaclient/app/boot_control/_jetson_uefi.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/otaclient/app/boot_control/_jetson_uefi.py b/src/otaclient/app/boot_control/_jetson_uefi.py index 817e68f27..7d41db999 100644 --- a/src/otaclient/app/boot_control/_jetson_uefi.py +++ b/src/otaclient/app/boot_control/_jetson_uefi.py @@ -391,6 +391,7 @@ class JetsonUEFIBootControl(BootControllerProtocol): def __init__(self) -> None: current_ota_status_dir = Path(boot_cfg.OTA_STATUS_DIR) + current_ota_status_dir.mkdir(exist_ok=True, parents=True) standby_ota_status_dir = Path(cfg.MOUNT_POINT) / Path( boot_cfg.OTA_STATUS_DIR ).relative_to("/") From baf6fee98e0c7c6cf6cfd3fcf9e1429f70b58f5b Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Fri, 14 Jun 2024 08:28:47 +0000 Subject: [PATCH 061/193] jetson-common: update to bspversion dump --- src/otaclient/app/boot_control/_jetson_common.py | 5 ++--- src/otaclient/app/boot_control/_jetson_uefi.py | 2 +- tests/test_otaclient/test_boot_control/test_jetson_cboot.py | 3 +-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/otaclient/app/boot_control/_jetson_common.py b/src/otaclient/app/boot_control/_jetson_common.py index e44299905..dddd90def 100644 --- a/src/otaclient/app/boot_control/_jetson_common.py +++ b/src/otaclient/app/boot_control/_jetson_common.py @@ -73,10 +73,9 @@ def parse(cls, _in: str | BSPVersion | Any) -> Self: return cls(int(major_ver), int(major_rev), int(minor_rev)) raise ValueError(f"expect str or BSPVersion instance, get {type(_in)}") - @staticmethod - def dump(to_export: BSPVersion) -> str: + def dump(self) -> str: """Dump BSPVersion to string as "Rxx.yy.z".""" - return f"R{to_export.major_ver}.{to_export.major_rev}.{to_export.minor_rev}" + return f"R{self.major_ver}.{self.major_rev}.{self.minor_rev}" BSPVersionStr = Annotated[ diff --git a/src/otaclient/app/boot_control/_jetson_uefi.py b/src/otaclient/app/boot_control/_jetson_uefi.py index 7d41db999..81900ae2e 100644 --- a/src/otaclient/app/boot_control/_jetson_uefi.py +++ b/src/otaclient/app/boot_control/_jetson_uefi.py @@ -237,7 +237,7 @@ def write_firmware_update_hint_file( Schema: , """ - write_str_to_file_sync(hint_fpath, f"{slot_id},{BSPVersion.dump(bsp_version)}") + write_str_to_file_sync(hint_fpath, f"{slot_id},{bsp_version.dump()}") @staticmethod def parse_firmware_update_hint_file( diff --git a/tests/test_otaclient/test_boot_control/test_jetson_cboot.py b/tests/test_otaclient/test_boot_control/test_jetson_cboot.py index 937c6b677..6cc42894f 100644 --- a/tests/test_otaclient/test_boot_control/test_jetson_cboot.py +++ b/tests/test_otaclient/test_boot_control/test_jetson_cboot.py @@ -25,7 +25,6 @@ import pytest from otaclient.app.boot_control import _jetson_cboot -from otaclient.app.boot_control._jetson_cboot import _CBootControl from otaclient.app.boot_control._jetson_common import ( BSPVersion, FirmwareBSPVersion, @@ -70,7 +69,7 @@ def test_parse(self, _in: str, expected: BSPVersion): ), ) def test_dump(self, _in: BSPVersion, expected: str): - assert BSPVersion.dump(_in) == expected + assert _in.dump() == expected @pytest.mark.parametrize( From 192bd4a17f5562713261c8d8b5c78bb086d756be Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Mon, 24 Jun 2024 11:25:42 +0000 Subject: [PATCH 062/193] nvbootctrl_common: get_current_fw_bsp_version is jetson-uefi only --- .../app/boot_control/_jetson_common.py | 32 ------------------- .../app/boot_control/_jetson_uefi.py | 32 +++++++++++++++++++ 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/src/otaclient/app/boot_control/_jetson_common.py b/src/otaclient/app/boot_control/_jetson_common.py index dddd90def..987d42f4d 100644 --- a/src/otaclient/app/boot_control/_jetson_common.py +++ b/src/otaclient/app/boot_control/_jetson_common.py @@ -155,7 +155,6 @@ def _nvbootctrl( ) if check_output: return res.stdout.decode() - return @classmethod def get_current_slot(cls, *, target: Optional[NVBootctrlTarget] = None) -> SlotID: @@ -188,37 +187,6 @@ def dump_slots_info(cls, *, target: Optional[NVBootctrlTarget] = None) -> str: cmd = "dump-slots-info" return cls._nvbootctrl(cmd, target=target, check_output=True) - @classmethod - def get_current_fw_bsp_version(cls) -> BSPVersion | None: - """Get current boot chain's firmware BSP version with nvbootctrl.""" - _raw = cls.dump_slots_info() - """Example: - Current version: 35.4.1 - Capsule update status: 1 - Current bootloader slot: A - Active bootloader slot: A - num_slots: 2 - slot: 0, status: normal - slot: 1, status: normal - """ - pa = re.compile(r"\s*Current version:\s(?P[\.\d]+)\s*") - - if not (ma := pa.search(_raw)): - logger.warning("nvbootctrl failed to report BSP version") - return - - bsp_ver_str = ( - f"r{ma.group('bsp_ver')}" # NOTE: need to add 'r' prefix back here - ) - bsp_ver = BSPVersion.parse(bsp_ver_str) - if bsp_ver.major_rev == 0: - logger.warning( - f"invalid BSP version: {bsp_ver_str}, this might indicate broken firmware" - ) - logger.warning("return empty bsp version") - return - return bsp_ver - class FirmwareBSPVersionControl: """firmware_bsp_version ota-status file for tracking firmware version. diff --git a/src/otaclient/app/boot_control/_jetson_uefi.py b/src/otaclient/app/boot_control/_jetson_uefi.py index 81900ae2e..04732b1b2 100644 --- a/src/otaclient/app/boot_control/_jetson_uefi.py +++ b/src/otaclient/app/boot_control/_jetson_uefi.py @@ -23,6 +23,7 @@ import contextlib import logging import os +import re import shutil from pathlib import Path from typing import Any, Generator @@ -61,6 +62,37 @@ class _NVBootctrl(NVBootctrlCommon): Without -t option, the target will be bootloader by default. """ + @classmethod + def get_current_fw_bsp_version(cls) -> BSPVersion | None: + """Get current boot chain's firmware BSP version with nvbootctrl.""" + _raw = cls.dump_slots_info() + """Example: + Current version: 35.4.1 + Capsule update status: 1 + Current bootloader slot: A + Active bootloader slot: A + num_slots: 2 + slot: 0, status: normal + slot: 1, status: normal + """ + pa = re.compile(r"\s*Current version:\s(?P[\.\d]+)\s*") + + if not (ma := pa.search(_raw)): + logger.warning("nvbootctrl failed to report BSP version") + return + + bsp_ver_str = ( + f"r{ma.group('bsp_ver')}" # NOTE: need to add 'r' prefix back here + ) + bsp_ver = BSPVersion.parse(bsp_ver_str) + if bsp_ver.major_rev == 0: + logger.warning( + f"invalid BSP version: {bsp_ver_str}, this might indicate broken firmware" + ) + logger.warning("return empty bsp version") + return + return bsp_ver + class CapsuleUpdate: """Firmware update implementation using Capsule update.""" From 65e7c529808dacb5268c5a8ffb355db45ff33e8e Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Mon, 24 Jun 2024 12:08:02 +0000 Subject: [PATCH 063/193] rewrite FirmwareBSPVersionControl --- .../app/boot_control/_jetson_common.py | 30 +++++++------------ 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/src/otaclient/app/boot_control/_jetson_common.py b/src/otaclient/app/boot_control/_jetson_common.py index 987d42f4d..a48f0e49b 100644 --- a/src/otaclient/app/boot_control/_jetson_common.py +++ b/src/otaclient/app/boot_control/_jetson_common.py @@ -200,6 +200,9 @@ class FirmwareBSPVersionControl: def __init__( self, + current_slot: SlotID, + current_slot_firmware_bsp_ver: BSPVersion | None = None, + *, current_firmware_bsp_vf: Path, standby_firmware_bsp_vf: Path, ) -> None: @@ -208,34 +211,23 @@ def __init__( self._version = FirmwareBSPVersion() try: - self._version = version_from_file = FirmwareBSPVersion.model_validate_json( + self._version = FirmwareBSPVersion.model_validate_json( self._current_fw_bsp_vf.read_text() ) - logger.info( - f"loaded firmware_version from version file: {version_from_file}" - ) except Exception as e: - logger.warning( - f"invalid or missing firmware_bsp_verion file, removed: {e!r}" - ) + logger.warning(f"invalid or missing firmware_bsp_verion file: {e!r}") self._current_fw_bsp_vf.unlink(missing_ok=True) - current_slot_bsp_ver = NVBootctrlCommon.get_current_fw_bsp_version() - logger.info( - f"query current slot bsp ver from nvbootctrl: {current_slot_bsp_ver}" - ) - self._version.set_by_slot( - NVBootctrlCommon.get_current_slot(), - current_slot_bsp_ver, - ) - self.write_current_firmware_bsp_version() - logger.info("write current slot's firmware_bsp_version file") + # NOTE: only check the standby slot's firmware BSP version info from file, + # for current slot, always trust the value from nvbootctrl. + self._version.set_by_slot(current_slot, current_slot_firmware_bsp_ver) + logger.info(f"loading firmware bsp version completed: {self._version}") - def write_current_firmware_bsp_version(self) -> None: + def write_to_currnet_slot(self) -> None: """Write firmware_bsp_version from memory to firmware_bsp_version file.""" write_str_to_file_sync(self._current_fw_bsp_vf, self._version.model_dump_json()) - def write_standby_firmware_bsp_version(self) -> None: + def write_to_standby_slot(self) -> None: """Write firmware_bsp_version from memory to firmware_bsp_version file.""" write_str_to_file_sync(self._standby_fw_bsp_vf, self._version.model_dump_json()) From 46141de001aa121a8838585049ba3a957a16c87e Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Mon, 24 Jun 2024 12:13:06 +0000 Subject: [PATCH 064/193] rewrite FirmwareBSPVersionControl --- .../app/boot_control/_jetson_common.py | 24 ++++++------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/src/otaclient/app/boot_control/_jetson_common.py b/src/otaclient/app/boot_control/_jetson_common.py index a48f0e49b..3d8bbb49c 100644 --- a/src/otaclient/app/boot_control/_jetson_common.py +++ b/src/otaclient/app/boot_control/_jetson_common.py @@ -31,6 +31,7 @@ from otaclient.app.boot_control._common import CMDHelperFuncs from otaclient_common.common import copytree_identical, write_str_to_file_sync +from otaclient_common.typing import StrOrPath logger = logging.getLogger(__name__) @@ -204,40 +205,29 @@ def __init__( current_slot_firmware_bsp_ver: BSPVersion | None = None, *, current_firmware_bsp_vf: Path, - standby_firmware_bsp_vf: Path, ) -> None: - self._current_fw_bsp_vf = current_firmware_bsp_vf - self._standby_fw_bsp_vf = standby_firmware_bsp_vf - self._version = FirmwareBSPVersion() try: self._version = FirmwareBSPVersion.model_validate_json( - self._current_fw_bsp_vf.read_text() + current_firmware_bsp_vf.read_text() ) except Exception as e: logger.warning(f"invalid or missing firmware_bsp_verion file: {e!r}") - self._current_fw_bsp_vf.unlink(missing_ok=True) + current_firmware_bsp_vf.unlink(missing_ok=True) # NOTE: only check the standby slot's firmware BSP version info from file, # for current slot, always trust the value from nvbootctrl. self._version.set_by_slot(current_slot, current_slot_firmware_bsp_ver) logger.info(f"loading firmware bsp version completed: {self._version}") - def write_to_currnet_slot(self) -> None: - """Write firmware_bsp_version from memory to firmware_bsp_version file.""" - write_str_to_file_sync(self._current_fw_bsp_vf, self._version.model_dump_json()) - - def write_to_standby_slot(self) -> None: + def write_to_file(self, fw_bsp_fpath: StrOrPath) -> None: """Write firmware_bsp_version from memory to firmware_bsp_version file.""" - write_str_to_file_sync(self._standby_fw_bsp_vf, self._version.model_dump_json()) + write_str_to_file_sync(fw_bsp_fpath, self._version.model_dump_json()) - def get_version_by_slot(self, slot_id: SlotID) -> Optional[BSPVersion]: - """Get slot's firmware version from memory.""" + def __getitem__(self, slot_id: SlotID) -> BSPVersion | None: return self._version.get_by_slot(slot_id) - def set_version_by_slot( - self, slot_id: SlotID, version: Optional[BSPVersion] - ) -> None: + def __setitem__(self, slot_id: SlotID, version: BSPVersion | None) -> None: """Set slot's firmware version into memory.""" self._version.set_by_slot(slot_id, version) From 8f797375f1a2cbb0c44ccbd5ed4a2b5d255c9000 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Mon, 24 Jun 2024 12:16:48 +0000 Subject: [PATCH 065/193] jetson-uefi: integrate new FirmwareBSPVersionControl --- .../app/boot_control/_jetson_uefi.py | 66 ++++++++++--------- 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/src/otaclient/app/boot_control/_jetson_uefi.py b/src/otaclient/app/boot_control/_jetson_uefi.py index 04732b1b2..76230fda9 100644 --- a/src/otaclient/app/boot_control/_jetson_uefi.py +++ b/src/otaclient/app/boot_control/_jetson_uefi.py @@ -310,16 +310,16 @@ def __init__(self): # ------ check BSP version ------ # # check firmware BSP version try: - self.bsp_version = bsp_version = ( - NVBootctrlCommon.get_current_fw_bsp_version() + self.fw_bsp_version = fw_bsp_version = ( + _NVBootctrl.get_current_fw_bsp_version() ) - assert bsp_version, "bsp version information not available" - logger.info(f"current slot firmware BSP version: {bsp_version}") + assert fw_bsp_version, "bsp version information not available" + logger.info(f"current slot firmware BSP version: {fw_bsp_version}") except Exception as e: _err_msg = f"failed to detect BSP version: {e!r}" logger.error(_err_msg) raise JetsonUEFIBootControlError(_err_msg) - logger.info(f"{bsp_version=}") + logger.info(f"{fw_bsp_version=}") # check rootfs BSP version try: @@ -331,7 +331,7 @@ def __init__(self): logger.warning(f"failed to detect rootfs BSP version: {e!r}") self.rootfs_bsp_verion = rootfs_bsp_version = None - if rootfs_bsp_version and rootfs_bsp_version > bsp_version: + if rootfs_bsp_version and rootfs_bsp_version > fw_bsp_version: logger.warning( ( "current slot's rootfs bsp version is newer than the firmware bsp version, " @@ -341,8 +341,8 @@ def __init__(self): # ------ sanity check, currently jetson-uefi only supports >= R35.2 ----- # # only after R35.2, the Capsule Firmware update is available. - if not bsp_version >= (35, 2, 0): - _err_msg = f"jetson-uefi only supports BSP version >= R35.2, but get {bsp_version=}. " + if fw_bsp_version < (35, 2, 0): + _err_msg = f"jetson-uefi only supports BSP version >= R35.2, but get {fw_bsp_version=}. " logger.error(_err_msg) raise JetsonUEFIBootControlError(_err_msg) @@ -438,24 +438,29 @@ def __init__(self) -> None: try: # startup boot controller - self._uefi_control = _UEFIBoot() + self._uefi_control = uefi_control = _UEFIBoot() # mount point prepare self._mp_control = SlotMountHelper( - standby_slot_dev=self._uefi_control.standby_rootfs_devpath, + standby_slot_dev=uefi_control.standby_rootfs_devpath, standby_slot_mount_point=cfg.MOUNT_POINT, active_slot_dev=self._uefi_control.curent_rootfs_devpath, active_slot_mount_point=cfg.ACTIVE_ROOT_MOUNT_POINT, ) - # load firmware BSP version from current rootfs slot - self._firmware_ver_control = FirmwareBSPVersionControl( - current_firmware_bsp_vf=current_ota_status_dir - / boot_cfg.FIRMWARE_BSP_VERSION_FNAME, - # NOTE: standby slot's bsp version file might be not yet - # available before an OTA. - standby_firmware_bsp_vf=standby_ota_status_dir - / boot_cfg.FIRMWARE_BSP_VERSION_FNAME, + # load firmware BSP version + self._current_fw_bsp_ver_fpath = ( + current_ota_status_dir / boot_cfg.FIRMWARE_BSP_VERSION_FNAME + ) + # NOTE: standby slot's bsp version file might be not yet + # available before an OTA. + self._standby_fw_bsp_ver_fpath = ( + standby_ota_status_dir / boot_cfg.FIRMWARE_BSP_VERSION_FNAME + ) + self._firmware_ver_control = fw_ver_control = FirmwareBSPVersionControl( + current_slot=uefi_control.current_slot, + current_slot_firmware_bsp_ver=uefi_control.fw_bsp_version, + current_firmware_bsp_vf=self._current_fw_bsp_ver_fpath, ) # init ota-status files @@ -467,6 +472,15 @@ def __init__(self) -> None: standby_ota_status_dir=standby_ota_status_dir, finalize_switching_boot=self._finalize_switching_boot, ) + + # post starting up, write the firmware bsp version to current slot + # NOTE 1: we always update and refer current slot's firmware bsp version file. + # NOTE 2: if OTA status is failure, always assume the firmware update on standby slot failed, + # and clear the standby slot's fw bsp version record. + if self._ota_status_control._ota_status == api_types.StatusOta.FAILURE: + fw_ver_control[uefi_control.standby_slot] = None + fw_ver_control[uefi_control.current_slot] = uefi_control.fw_bsp_version + fw_ver_control.write_to_file(self._current_fw_bsp_ver_fpath) except Exception as e: _err_msg = f"failed to start jetson-uefi controller: {e!r}" raise ota_errors.BootControlStartupFailed(_err_msg, module=__name__) from e @@ -479,7 +493,7 @@ def __init__(self) -> None: def _finalize_switching_boot(self) -> bool: """Verify firmware update result and write firmware BSP version file.""" current_slot = self._uefi_control.current_slot - current_slot_bsp_ver = self._uefi_control.bsp_version + current_slot_bsp_ver = self._uefi_control.fw_bsp_version if current_slot_bsp_ver is None: logger.warning("current slot BSP version is unknown, skip") @@ -512,20 +526,12 @@ def _finalize_switching_boot(self) -> bool: f"but expects slot {current_slot} with version {current_slot_bsp_ver}" ) ) - # NOTE(20240610): on a failed OTA with firmware update, we always assume a failed - # firmware update(even the failure might be caused by rootfs update failed). - self._firmware_ver_control.set_version_by_slot(current_slot, None) - self._firmware_ver_control.write_current_firmware_bsp_version() return False # ------ firmware update succeeded, write firmware version file ------ # logger.info( f"successfully update {current_slot=} firmware version to {current_slot_bsp_ver}" ) - self._firmware_ver_control.set_version_by_slot( - current_slot, current_slot_bsp_ver - ) - self._firmware_ver_control.write_current_firmware_bsp_version() return True def _capsule_firmware_update(self) -> bool: @@ -537,9 +543,7 @@ def _capsule_firmware_update(self) -> bool: logger.info("jetson-uefi: checking if we need to do firmware update ...") standby_bootloader_slot = self._uefi_control.standby_slot - standby_firmware_bsp_ver = self._firmware_ver_control.get_version_by_slot( - standby_bootloader_slot - ) + standby_firmware_bsp_ver = self._firmware_ver_control[standby_bootloader_slot] logger.info( f"standby slot current firmware ver: {standby_bootloader_slot=} BSP ver: {standby_firmware_bsp_ver}" ) @@ -634,7 +638,7 @@ def post_update(self) -> Generator[None, None, None]: ) # ------ preserve BSP version file to standby slot ------ # - self._firmware_ver_control.write_standby_firmware_bsp_version() + self._firmware_ver_control.write_to_file(self._standby_fw_bsp_ver_fpath) # ------ preserve /boot/ota folder to standby rootfs ------ # preserve_ota_config_files_to_standby( From e24be0bf40847027d8cc68823737f2993dbbf25c Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Mon, 24 Jun 2024 12:44:52 +0000 Subject: [PATCH 066/193] jetson-uefi: get_current_fw_bsp_version now raises valuerror on detection failed --- src/otaclient/app/boot_control/_jetson_uefi.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/otaclient/app/boot_control/_jetson_uefi.py b/src/otaclient/app/boot_control/_jetson_uefi.py index 76230fda9..24c2cc789 100644 --- a/src/otaclient/app/boot_control/_jetson_uefi.py +++ b/src/otaclient/app/boot_control/_jetson_uefi.py @@ -78,19 +78,18 @@ def get_current_fw_bsp_version(cls) -> BSPVersion | None: pa = re.compile(r"\s*Current version:\s(?P[\.\d]+)\s*") if not (ma := pa.search(_raw)): - logger.warning("nvbootctrl failed to report BSP version") - return + _err_msg = "nvbootctrl failed to report BSP version" + logger.error(_err_msg) + raise ValueError(_err_msg) bsp_ver_str = ( f"r{ma.group('bsp_ver')}" # NOTE: need to add 'r' prefix back here ) bsp_ver = BSPVersion.parse(bsp_ver_str) if bsp_ver.major_rev == 0: - logger.warning( - f"invalid BSP version: {bsp_ver_str}, this might indicate broken firmware" - ) - logger.warning("return empty bsp version") - return + _err_msg = f"invalid BSP version: {bsp_ver_str}, this might indicate broken firmware!" + logger.warning(_err_msg) + raise ValueError(_err_msg) return bsp_ver From c54e19d1934d423e0601f0ca1efc8fbf10d2cf51 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Mon, 24 Jun 2024 15:30:01 +0000 Subject: [PATCH 067/193] jetson-common: define SLOT_A and SLOT_B --- src/otaclient/app/boot_control/_jetson_common.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/otaclient/app/boot_control/_jetson_common.py b/src/otaclient/app/boot_control/_jetson_common.py index 3d8bbb49c..8e04022ee 100644 --- a/src/otaclient/app/boot_control/_jetson_common.py +++ b/src/otaclient/app/boot_control/_jetson_common.py @@ -54,6 +54,10 @@ def __new__(cls, _in: str | Self) -> Self: raise ValueError(f"{_in=} is not valid slot num, should be '0' or '1'.") +SLOT_A, SLOT_B = SlotID("0"), SlotID("1") +SLOT_FLIP = {SLOT_A: SLOT_B, SLOT_B: SLOT_A} + + class BSPVersion(NamedTuple): """BSP version in NamedTuple representation. @@ -96,17 +100,17 @@ class FirmwareBSPVersion(BaseModel): slot_b: Optional[BSPVersionStr] = None def set_by_slot(self, slot_id: SlotID, ver: BSPVersion | None) -> None: - if slot_id == SlotID("0"): + if slot_id == SLOT_A: self.slot_a = ver - elif slot_id == SlotID("1"): + elif slot_id == SLOT_B: self.slot_b = ver else: raise ValueError(f"invalid slot_id: {slot_id}") def get_by_slot(self, slot_id: SlotID) -> BSPVersion | None: - if slot_id == SlotID("0"): + if slot_id == SLOT_A: return self.slot_a - elif slot_id == SlotID("1"): + elif slot_id == SLOT_B: return self.slot_b else: raise ValueError(f"invalid slot_id: {slot_id}") @@ -172,7 +176,7 @@ def get_standby_slot(cls, *, target: Optional[NVBootctrlTarget] = None) -> SlotI NOTE: this method is implemented with nvbootctrl get-current-slot. """ current_slot = cls.get_current_slot(target=target) - return SlotID("0") if current_slot == "1" else SlotID("1") + return SLOT_FLIP[current_slot] @classmethod def set_active_boot_slot( From 42a4f353fa282e9d07d582ef6e30e731e163826a Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Mon, 24 Jun 2024 15:30:54 +0000 Subject: [PATCH 068/193] rewrite FirmwareBSPVersionControl again --- .../app/boot_control/_jetson_common.py | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/otaclient/app/boot_control/_jetson_common.py b/src/otaclient/app/boot_control/_jetson_common.py index 8e04022ee..df4c8fdd5 100644 --- a/src/otaclient/app/boot_control/_jetson_common.py +++ b/src/otaclient/app/boot_control/_jetson_common.py @@ -210,6 +210,8 @@ def __init__( *, current_firmware_bsp_vf: Path, ) -> None: + self.current_slot, self.standby_slot = current_slot, SLOT_FLIP[current_slot] + self._version = FirmwareBSPVersion() try: self._version = FirmwareBSPVersion.model_validate_json( @@ -228,12 +230,21 @@ def write_to_file(self, fw_bsp_fpath: StrOrPath) -> None: """Write firmware_bsp_version from memory to firmware_bsp_version file.""" write_str_to_file_sync(fw_bsp_fpath, self._version.model_dump_json()) - def __getitem__(self, slot_id: SlotID) -> BSPVersion | None: - return self._version.get_by_slot(slot_id) + @property + def current_slot_fw_ver(self) -> BSPVersion | None: + return self._version.get_by_slot(self.current_slot) + + @current_slot_fw_ver.setter + def current_slot_fw_ver(self, bsp_ver: BSPVersion | None): + self._version.set_by_slot(self.current_slot, bsp_ver) + + @property + def standby_slot_fw_ver(self) -> BSPVersion | None: + return self._version.get_by_slot(self.standby_slot) - def __setitem__(self, slot_id: SlotID, version: BSPVersion | None) -> None: - """Set slot's firmware version into memory.""" - self._version.set_by_slot(slot_id, version) + @standby_slot_fw_ver.setter + def standby_slot_fw_ver(self, bsp_ver: BSPVersion | None): + self._version.set_by_slot(self.standby_slot, bsp_ver) BSP_VER_PA = re.compile( From d803ebf723cafd23b3befe07284be2412c2afa42 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Mon, 24 Jun 2024 15:41:34 +0000 Subject: [PATCH 069/193] jetson-common: current slot's fw bsp version must not be None --- src/otaclient/app/boot_control/_jetson_common.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/otaclient/app/boot_control/_jetson_common.py b/src/otaclient/app/boot_control/_jetson_common.py index df4c8fdd5..95baa3aa2 100644 --- a/src/otaclient/app/boot_control/_jetson_common.py +++ b/src/otaclient/app/boot_control/_jetson_common.py @@ -206,7 +206,7 @@ class FirmwareBSPVersionControl: def __init__( self, current_slot: SlotID, - current_slot_firmware_bsp_ver: BSPVersion | None = None, + current_slot_firmware_bsp_ver: BSPVersion, *, current_firmware_bsp_vf: Path, ) -> None: @@ -231,8 +231,9 @@ def write_to_file(self, fw_bsp_fpath: StrOrPath) -> None: write_str_to_file_sync(fw_bsp_fpath, self._version.model_dump_json()) @property - def current_slot_fw_ver(self) -> BSPVersion | None: - return self._version.get_by_slot(self.current_slot) + def current_slot_fw_ver(self) -> BSPVersion: + assert (res := self._version.get_by_slot(self.current_slot)) + return res @current_slot_fw_ver.setter def current_slot_fw_ver(self, bsp_ver: BSPVersion | None): From a705806eaea92b1fb6bc7a38845f878a0f82ce45 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Mon, 24 Jun 2024 16:03:23 +0000 Subject: [PATCH 070/193] jetson-uefi: implement l4tlauncher update, remove firmware update hint file implementation --- .../app/boot_control/_jetson_uefi.py | 327 ++++++++++-------- src/otaclient/app/boot_control/configs.py | 3 +- 2 files changed, 188 insertions(+), 142 deletions(-) diff --git a/src/otaclient/app/boot_control/_jetson_uefi.py b/src/otaclient/app/boot_control/_jetson_uefi.py index 24c2cc789..379f25309 100644 --- a/src/otaclient/app/boot_control/_jetson_uefi.py +++ b/src/otaclient/app/boot_control/_jetson_uefi.py @@ -31,7 +31,7 @@ from otaclient.app import errors as ota_errors from otaclient.app.configs import config as cfg from otaclient_api.v2 import types as api_types -from otaclient_common.common import subprocess_call, write_str_to_file_sync +from otaclient_common.common import subprocess_call, write_str_to_file_sync, file_sha256 from otaclient_common.typing import StrOrPath from ._common import CMDHelperFuncs, OTAStatusFilesControl, SlotMountHelper @@ -50,6 +50,9 @@ logger = logging.getLogger(__name__) +# TODO: calculate this table +L4TLAUNCHER_BSP_VER_SHA256_MAP: dict[str, BSPVersion] = {} + class JetsonUEFIBootControlError(Exception): """Exception type for covering jetson-uefi related errors.""" @@ -63,7 +66,7 @@ class _NVBootctrl(NVBootctrlCommon): """ @classmethod - def get_current_fw_bsp_version(cls) -> BSPVersion | None: + def get_current_fw_bsp_version(cls) -> BSPVersion: """Get current boot chain's firmware BSP version with nvbootctrl.""" _raw = cls.dump_slots_info() """Example: @@ -93,61 +96,58 @@ def get_current_fw_bsp_version(cls) -> BSPVersion | None: return bsp_ver +EFIVARS_FSTYPE = "efivarfs" +EFIVARS_DPATH = "/sys/firmware/efi/efivars/" + + class CapsuleUpdate: """Firmware update implementation using Capsule update.""" - EFIVARS_FSTYPE = "efivarfs" - def __init__( - self, boot_parent_devpath: StrOrPath, standby_slot_mp: StrOrPath + self, + boot_parent_devpath: StrOrPath, + standby_slot_mp: StrOrPath, + *, + ota_image_bsp_ver: BSPVersion, + fw_bsp_ver_control: FirmwareBSPVersionControl, ) -> None: + self.fw_bsp_ver_control = fw_bsp_ver_control + self.ota_image_bsp_ver = ota_image_bsp_ver + # NOTE: use the esp partition at the current booted device # i.e., if we boot from nvme0n1, then bootdev_path is /dev/nvme0n1 and # we use the esp at nvme0n1. - boot_parent_devpath = str(boot_parent_devpath) self.esp_mp = Path(boot_cfg.ESP_MOUNTPOINT) + self.esp_boot_dir = self.esp_mp / "EFI" / "BOOT" + self.l4tlauncher_ver_fpath = self.esp_boot_dir / boot_cfg.L4TLAUNCHER_VER_FNAME + self.bootaa64_at_esp = self.esp_boot_dir / boot_cfg.L4TLAUNCHER_FNAME + self.bootaa64_at_esp_bak = ( + self.esp_boot_dir / f"{boot_cfg.L4TLAUNCHER_FNAME}_bak" + ) # NOTE: we get the update capsule from the standby slot self.standby_slot_mp = Path(standby_slot_mp) + self.esp_part = self._detect_esp_dev(boot_parent_devpath) - # NOTE: if boots from external, expects to have multiple esp parts, - # we need to get the one at our booted parent dev. - esp_parts = CMDHelperFuncs.get_dev_by_token( - token="PARTLABEL", value=boot_cfg.ESP_PARTLABEL - ) - if not esp_parts: - raise JetsonUEFIBootControlError("no ESP partition presented") - - for _esp_part in esp_parts: - if _esp_part.find(boot_parent_devpath) != -1: - logger.info(f"find esp partition at {_esp_part}") - esp_part = _esp_part - break - else: - _err_msg = f"failed to find esp partition on {boot_parent_devpath}" - logger.error(_err_msg) - raise JetsonUEFIBootControlError(_err_msg) - self.esp_part = esp_part - - @classmethod + @staticmethod @contextlib.contextmanager - def _ensure_efivarfs_mounted(cls) -> Generator[None, Any, None]: + def _ensure_efivarfs_mounted() -> Generator[None, Any, None]: """Ensure the efivarfs is mounted as rw.""" - if CMDHelperFuncs.is_target_mounted(boot_cfg.EFIVARS_DPATH): + if CMDHelperFuncs.is_target_mounted(EFIVARS_DPATH): options = "remount,rw,nosuid,nodev,noexec,relatime" else: logger.warning( - f"efivars is not mounted! try to mount it at {boot_cfg.EFIVARS_DPATH}" + f"efivars is not mounted! try to mount it at {EFIVARS_DPATH}" ) options = "rw,nosuid,nodev,noexec,relatime" # fmt: off cmd = [ "mount", - "-t", cls.EFIVARS_FSTYPE, + "-t", EFIVARS_FSTYPE, "-o", options, - cls.EFIVARS_FSTYPE, - boot_cfg.EFIVARS_DPATH + EFIVARS_FSTYPE, + EFIVARS_DPATH ] # fmt: on try: @@ -155,23 +155,28 @@ def _ensure_efivarfs_mounted(cls) -> Generator[None, Any, None]: yield except Exception as e: raise JetsonUEFIBootControlError( - f"failed to mount {cls.EFIVARS_FSTYPE} on {boot_cfg.EFIVARS_DPATH}: {e!r}" + f"failed to mount {EFIVARS_FSTYPE} on {EFIVARS_DPATH}: {e!r}" ) from e + @staticmethod @contextlib.contextmanager - def _ensure_esp_mounted(self) -> Generator[None, Any, None]: + def _ensure_esp_mounted( + esp_dev: StrOrPath, mount_point: StrOrPath + ) -> Generator[None, Any, None]: """Mount the esp partition and then umount it.""" - self.esp_mp.mkdir(exist_ok=True, parents=True) + mount_point = Path(mount_point) + mount_point.mkdir(exist_ok=True, parents=True) + try: - CMDHelperFuncs.mount_rw(self.esp_part, self.esp_mp) + CMDHelperFuncs.mount_rw(str(esp_dev), mount_point) yield - CMDHelperFuncs.umount(self.esp_mp, raise_exception=False) + CMDHelperFuncs.umount(mount_point, raise_exception=False) except Exception as e: - _err_msg = f"failed to mount {self.esp_part=} to {self.esp_mp}: {e!r}" + _err_msg = f"failed to mount {esp_dev} to {mount_point}: {e!r}" logger.error(_err_msg) raise JetsonUEFIBootControlError(_err_msg) from e - def _prepare_payload(self) -> bool: + def _prepare_fwupdate_capsule(self) -> bool: """Copy the Capsule update payloads to specific location at esp partition. Returns: @@ -201,52 +206,132 @@ def _prepare_payload(self) -> bool: f"failed to copy {capsule_fname} from {fw_loc_at_standby_slot} to {capsule_at_esp}: {e!r}" ) logger.warning(f"skip {capsule_fname}") + return firmware_package_configured - # ------ prepare L4TLauncher update ------ # - # NOTE(20240611): Assume that the new L4TLauncher always keeps backward compatibility to - # work with old firmware. This assumption MUST be confirmed on the real ECU. - if firmware_package_configured: - logger.info("capsule update is scheduled, also update L4TLauncher") - esp_boot_dir = self.esp_mp / "EFI" / "BOOT" - - # the canonical boot entry used by UEFI - bootaa64_at_esp = esp_boot_dir / boot_cfg.L4TLAUNCHER_FNAME + def _update_l4tlauncher(self) -> bool: + """update L4TLauncher with OTA image's one.""" + ota_image_l4tlauncher_ver = self.ota_image_bsp_ver + logger.warning(f"update the l4tlauncher to version {ota_image_l4tlauncher_ver}") - bootaa64_at_esp_bak = esp_boot_dir / f"{boot_cfg.L4TLAUNCHER_FNAME}_bak" - # l4tlauncher is stored as /BOOTAA64.efi - bootaa64_at_standby = fw_loc_at_standby_slot / boot_cfg.L4TLAUNCHER_FNAME + # new BOOTAA64.efi is located at /opt/ota_package/BOOTAA64.efi + ota_image_bootaa64 = ( + self.standby_slot_mp + / Path(boot_cfg.CAPSULE_PAYLOAD_AT_ROOTFS).relative_to("/") + / boot_cfg.L4TLAUNCHER_FNAME + ) + if not ota_image_bootaa64.is_file(): + logger.warning(f"{ota_image_bootaa64} not found, skip update l4tlauncher") + return False - shutil.copy(bootaa64_at_esp, bootaa64_at_esp_bak) - shutil.copy(bootaa64_at_standby, bootaa64_at_esp) - os.sync() - return firmware_package_configured + shutil.copy(self.bootaa64_at_esp, self.bootaa64_at_esp_bak) + shutil.copy(ota_image_bootaa64, self.bootaa64_at_esp) + write_str_to_file_sync( + self.l4tlauncher_ver_fpath, ota_image_l4tlauncher_ver.dump() + ) + os.sync() + return True - def _write_efivar(self) -> None: + @staticmethod + def _write_magic_efivar() -> None: """Write magic efivar to trigger firmware Capsule update in next boot. Raises: JetsonUEFIBootControlError on failed Capsule update preparing. """ - magic_efivar_fpath = ( - Path(boot_cfg.EFIVARS_DPATH) / boot_cfg.UPDATE_TRIGGER_EFIVAR - ) + magic_efivar_fpath = Path(EFIVARS_DPATH) / boot_cfg.UPDATE_TRIGGER_EFIVAR magic_efivar_fpath.write_bytes(boot_cfg.MAGIC_BYTES) os.sync() + def _detect_l4tlauncher_version(self) -> BSPVersion: + l4tlauncher_bsp_ver = None + try: + l4tlauncher_bsp_ver = BSPVersion.parse( + self.l4tlauncher_ver_fpath.read_text() + ) + except Exception as e: + logger.warning(f"missing or invalid l4tlauncher version file: {e!r}") + self.l4tlauncher_ver_fpath.unlink(missing_ok=True) + + bootaa64_at_esp = self.esp_boot_dir / boot_cfg.L4TLAUNCHER_FNAME + # NOTE(20240624): since the number of l4tlauncher version is limited, + # we can lookup against a pre-calculated sha256 digest map. + if l4tlauncher_bsp_ver is None: + _l4tlauncher_sha256_digest = file_sha256(bootaa64_at_esp) + logger.info( + f"try to determine the l4tlauncher verison by hash: {_l4tlauncher_sha256_digest}" + ) + l4tlauncher_bsp_ver = L4TLAUNCHER_BSP_VER_SHA256_MAP.get( + _l4tlauncher_sha256_digest + ) + + current_slot_fw_bsp_ver = self.fw_bsp_ver_control.current_slot_fw_ver + # NOTE(20240624): if we failed to detect the l4tlauncher's version, + # we assume that the launcher is the same version as current slot's fw. + # This is typically case for a newly OTA inital setup ECU. + if l4tlauncher_bsp_ver is None: + logger.warning( + ( + "failed to determine the l4tlauncher's version, assuming " + f"version is the same as current slot's fw version: {current_slot_fw_bsp_ver}" + ) + ) + l4tlauncher_bsp_ver = current_slot_fw_bsp_ver + write_str_to_file_sync( + self.l4tlauncher_ver_fpath, l4tlauncher_bsp_ver.dump() + ) + logger.info(f"finish detecting l4tlauncher version: {l4tlauncher_bsp_ver}") + return l4tlauncher_bsp_ver + + @staticmethod + def _detect_esp_dev(boot_parent_devpath: StrOrPath) -> str: + # NOTE: if boots from external, expects to have multiple esp parts, + # we need to get the one at our booted parent dev. + esp_parts = CMDHelperFuncs.get_dev_by_token( + token="PARTLABEL", value=boot_cfg.ESP_PARTLABEL + ) + if not esp_parts: + raise JetsonUEFIBootControlError("no ESP partition presented") + + for _esp_part in esp_parts: + if _esp_part.find(str(boot_parent_devpath)) != -1: + logger.info(f"find esp partition at {_esp_part}") + esp_part = _esp_part + break + else: + _err_msg = f"failed to find esp partition on {boot_parent_devpath}" + logger.error(_err_msg) + raise JetsonUEFIBootControlError(_err_msg) + return esp_part + + # APIs + def firmware_update(self) -> bool: """Trigger firmware update in next boot. Returns: True if firmware update is configured, False if there is no firmware update. """ - with self._ensure_esp_mounted(): - if not self._prepare_payload(): + standby_slot_fw_bsp_ver = self.fw_bsp_ver_control.standby_slot_fw_ver + if ( + standby_slot_fw_bsp_ver + and standby_slot_fw_bsp_ver >= self.ota_image_bsp_ver + ): + logger.info( + ( + "standby slot has newer or equal ver of firmware, skip firmware update: " + f"{standby_slot_fw_bsp_ver=}, {self.ota_image_bsp_ver=}" + ) + ) + return False + + with self._ensure_esp_mounted(self.esp_part, self.esp_mp): + if not self._prepare_fwupdate_capsule(): logger.info("no firmware file is prepared, skip firmware update") return False with self._ensure_efivarfs_mounted(): try: - self._write_efivar() + self._write_magic_efivar() except Exception as e: logger.warning( ( @@ -259,29 +344,30 @@ def firmware_update(self) -> bool: logger.info("firmware update package prepare finished") return True - @staticmethod - def write_firmware_update_hint_file( - hint_fpath: StrOrPath, slot_id: SlotID, bsp_version: BSPVersion - ) -> None: - """When capsule firmware update is scheduled, write this file to - hint the otaclient in the new slot. + def l4tlauncher_update(self) -> bool: + """Update l4tlauncher if needed. - Schema: , - """ - write_str_to_file_sync(hint_fpath, f"{slot_id},{bsp_version.dump()}") + NOTE(20240611): Assume that the new L4TLauncher always keeps backward compatibility to + work with old firmware. This assumption MUST be confirmed on the real ECU. + NOTE(20240611): Only update l4tlauncher but never downgrade it. - @staticmethod - def parse_firmware_update_hint_file( - hint_fpath: StrOrPath, - ) -> tuple[SlotID, BSPVersion]: - """Parse the slot_id and firmware bsp_version from firmware update hint file.""" - _raw = Path(hint_fpath).read_text() + Returns: + True if l4tlauncher is updated, else if there is no l4tlauncher update. + """ + l4tlauncher_bsp_ver = self._detect_l4tlauncher_version() + ota_image_l4tlauncher_ver = self.ota_image_bsp_ver + if l4tlauncher_bsp_ver >= ota_image_l4tlauncher_ver: + logger.info( + ( + "installed l4tlauncher has newer or equal version of l4tlauncher to OTA image's one, " + f"{l4tlauncher_bsp_ver=}, {ota_image_l4tlauncher_ver=}, " + "skip l4tlauncher update" + ) + ) + return False - try: - _slot_id, _bsp_v = _raw.split(",") - return SlotID(_slot_id), BSPVersion.parse(_bsp_v) - except Exception as e: - raise ValueError(f"invalid hint file content: {_raw}: {e!r}") from e + with self._ensure_esp_mounted(self.esp_part, self.esp_mp): + return self._update_l4tlauncher() class _UEFIBoot: @@ -456,7 +542,7 @@ def __init__(self) -> None: self._standby_fw_bsp_ver_fpath = ( standby_ota_status_dir / boot_cfg.FIRMWARE_BSP_VERSION_FNAME ) - self._firmware_ver_control = fw_ver_control = FirmwareBSPVersionControl( + self._firmware_ver_control = fw_bsp_ver = FirmwareBSPVersionControl( current_slot=uefi_control.current_slot, current_slot_firmware_bsp_ver=uefi_control.fw_bsp_version, current_firmware_bsp_vf=self._current_fw_bsp_ver_fpath, @@ -477,9 +563,9 @@ def __init__(self) -> None: # NOTE 2: if OTA status is failure, always assume the firmware update on standby slot failed, # and clear the standby slot's fw bsp version record. if self._ota_status_control._ota_status == api_types.StatusOta.FAILURE: - fw_ver_control[uefi_control.standby_slot] = None - fw_ver_control[uefi_control.current_slot] = uefi_control.fw_bsp_version - fw_ver_control.write_to_file(self._current_fw_bsp_ver_fpath) + fw_bsp_ver.standby_slot_fw_ver = None + fw_bsp_ver.current_slot_fw_ver = uefi_control.fw_bsp_version + fw_bsp_ver.write_to_file(self._current_fw_bsp_ver_fpath) except Exception as e: _err_msg = f"failed to start jetson-uefi controller: {e!r}" raise ota_errors.BootControlStartupFailed(_err_msg, module=__name__) from e @@ -490,51 +576,17 @@ def __init__(self) -> None: self.current_fwupdate_hint_fpath.unlink(missing_ok=True) def _finalize_switching_boot(self) -> bool: - """Verify firmware update result and write firmware BSP version file.""" - current_slot = self._uefi_control.current_slot - current_slot_bsp_ver = self._uefi_control.fw_bsp_version + """Verify firmware update result and write firmware BSP version file. - if current_slot_bsp_ver is None: - logger.warning("current slot BSP version is unknown, skip") - return True - - try: - slot_id, bsp_v = CapsuleUpdate.parse_firmware_update_hint_file( - self.current_fwupdate_hint_fpath - ) - except FileNotFoundError: - logger.info("no firmware update occurs in previous OTA") - return True - except Exception as e: - logger.error( - ( - f"firmware update hint file presented but invalid: {e!r}" - "assuming firmware update failed" - ) - ) - self.current_fwupdate_hint_fpath.unlink(missing_ok=True) - return False - - # ------ firmware update occurs, check hints ------ # - - if slot_id != current_slot or bsp_v != current_slot_bsp_ver: - logger.error( - ( - "firmware update hint file indicates firmware update occurs on " - f"slot {slot_id} with version {bsp_v}, " - f"but expects slot {current_slot} with version {current_slot_bsp_ver}" - ) - ) - return False - - # ------ firmware update succeeded, write firmware version file ------ # - logger.info( - f"successfully update {current_slot=} firmware version to {current_slot_bsp_ver}" - ) + NOTE that if capsule firmware update failed, we must be booted back to the + previous slot, so actually we don't need to do any checks here. + """ return True def _capsule_firmware_update(self) -> bool: - """Perform firmware update with UEFI Capsule update. + """Perform firmware update with UEFI Capsule update if needed. + + If standby slot is known to have newer bootable firmware, skip firmware update. Returns: True if there is firmware update configured, False for no firmware update. @@ -542,10 +594,6 @@ def _capsule_firmware_update(self) -> bool: logger.info("jetson-uefi: checking if we need to do firmware update ...") standby_bootloader_slot = self._uefi_control.standby_slot - standby_firmware_bsp_ver = self._firmware_ver_control[standby_bootloader_slot] - logger.info( - f"standby slot current firmware ver: {standby_bootloader_slot=} BSP ver: {standby_firmware_bsp_ver}" - ) # ------ check if we need to do firmware update ------ # skip_firmware_update_hint_file = ( @@ -569,6 +617,7 @@ def _capsule_firmware_update(self) -> bool: logger.info("skip firmware update due to new image BSP version unknown") return False + standby_firmware_bsp_ver = self._firmware_ver_control.standby_slot_fw_ver if standby_firmware_bsp_ver and standby_firmware_bsp_ver >= new_bsp_v: logger.info( f"{standby_bootloader_slot=} has newer or equal ver of firmware, skip firmware update" @@ -579,19 +628,17 @@ def _capsule_firmware_update(self) -> bool: firmware_updater = CapsuleUpdate( boot_parent_devpath=self._uefi_control.parent_devpath, standby_slot_mp=self._mp_control.standby_slot_mount_point, + ota_image_bsp_ver=new_bsp_v, + fw_bsp_ver_control=self._firmware_ver_control, ) if firmware_updater.firmware_update(): - CapsuleUpdate.write_firmware_update_hint_file( - self.standby_fwupdate_hint_fpath, - slot_id=self._uefi_control.standby_slot, - bsp_version=new_bsp_v, - ) + firmware_updater.l4tlauncher_update() logger.info( - f"will update to new firmware version in next reboot: {new_bsp_v=}" - ) - logger.info( - f"will switch to Slot({standby_bootloader_slot}) on successful firmware update" + ( + f"will update to new firmware version in next reboot: {new_bsp_v=}, \n" + f"will switch to Slot({standby_bootloader_slot}) on successful firmware update" + ) ) return True return False diff --git a/src/otaclient/app/boot_control/configs.py b/src/otaclient/app/boot_control/configs.py index e96f61c0b..950812e18 100644 --- a/src/otaclient/app/boot_control/configs.py +++ b/src/otaclient/app/boot_control/configs.py @@ -69,12 +69,11 @@ class JetsonUEFIBootControlConfig(JetsonBootCommon): L4TLAUNCHER_FNAME = "BOOTAA64.efi" ESP_MOUNTPOINT = "/mnt/esp" ESP_PARTLABEL = "esp" - EFIVARS_DPATH = "/sys/firmware/efi/efivars/" UPDATE_TRIGGER_EFIVAR = "OsIndications-8be4df61-93ca-11d2-aa0d-00e098032b8c" MAGIC_BYTES = b"\x07\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00" CAPSULE_PAYLOAD_AT_ESP = "EFI/UpdateCapsule" CAPSULE_PAYLOAD_AT_ROOTFS = "/opt/ota_package/" - FIRMWARE_UPDATE_HINT_FNAME = ".firmware_update" + L4TLAUNCHER_VER_FNAME = "l4tlauncher_version.json" NO_FIRMWARE_UPDATE_HINT_FNAME = ".otaclient_no_firmware_update" """Skip firmware update if this file is presented.""" From d686b99d150b79e90b35b8719eb4ce8634fff744 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Mon, 24 Jun 2024 16:03:56 +0000 Subject: [PATCH 071/193] minor fix --- src/otaclient/app/boot_control/_jetson_uefi.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/otaclient/app/boot_control/_jetson_uefi.py b/src/otaclient/app/boot_control/_jetson_uefi.py index 379f25309..b38f6d96f 100644 --- a/src/otaclient/app/boot_control/_jetson_uefi.py +++ b/src/otaclient/app/boot_control/_jetson_uefi.py @@ -513,14 +513,6 @@ def __init__(self) -> None: boot_cfg.OTA_STATUS_DIR ).relative_to("/") - # NOTE: this hint file is referred by finalize_switching_boot - self.current_fwupdate_hint_fpath = ( - current_ota_status_dir / boot_cfg.FIRMWARE_UPDATE_HINT_FNAME - ) - self.standby_fwupdate_hint_fpath = ( - standby_ota_status_dir / boot_cfg.FIRMWARE_UPDATE_HINT_FNAME - ) - try: # startup boot controller self._uefi_control = uefi_control = _UEFIBoot() @@ -569,11 +561,6 @@ def __init__(self) -> None: except Exception as e: _err_msg = f"failed to start jetson-uefi controller: {e!r}" raise ota_errors.BootControlStartupFailed(_err_msg, module=__name__) from e - finally: - # NOTE: the hint file is checked during OTAStatusFilesControl __init__, - # by finalize_switching_boot if we are in first reboot after OTA. - # once we have done parsing the hint file, we must remove it immediately. - self.current_fwupdate_hint_fpath.unlink(missing_ok=True) def _finalize_switching_boot(self) -> bool: """Verify firmware update result and write firmware BSP version file. From 401b009ff7da0144a49149e5940e90b45a60e33e Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Mon, 24 Jun 2024 16:10:25 +0000 Subject: [PATCH 072/193] jetson-uefi: implement verify --- src/otaclient/app/boot_control/_jetson_uefi.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/otaclient/app/boot_control/_jetson_uefi.py b/src/otaclient/app/boot_control/_jetson_uefi.py index b38f6d96f..dbb764755 100644 --- a/src/otaclient/app/boot_control/_jetson_uefi.py +++ b/src/otaclient/app/boot_control/_jetson_uefi.py @@ -95,6 +95,11 @@ def get_current_fw_bsp_version(cls) -> BSPVersion: raise ValueError(_err_msg) return bsp_ver + @classmethod + def verify(cls) -> str: + """Verify the bootloader and rootfs boot.""" + return cls._nvbootctrl("verify", check_output=True) + EFIVARS_FSTYPE = "efivarfs" EFIVARS_DPATH = "/sys/firmware/efi/efivars/" @@ -568,6 +573,11 @@ def _finalize_switching_boot(self) -> bool: NOTE that if capsule firmware update failed, we must be booted back to the previous slot, so actually we don't need to do any checks here. """ + try: + fw_update_verify = _NVBootctrl.verify() + logger.info(f"nvbootctrl verify: {fw_update_verify}") + except Exception as e: + logger.warning(f"nvbootctrl verify failed: {e!r}") return True def _capsule_firmware_update(self) -> bool: From 304101d042594b3efc56f8e864b6d53be0782a88 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Tue, 25 Jun 2024 01:07:07 +0000 Subject: [PATCH 073/193] jetson-uefi: finish up the L4TLAUNCHER_BSP_VER_SHA256_MAP --- src/otaclient/app/boot_control/_jetson_uefi.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/otaclient/app/boot_control/_jetson_uefi.py b/src/otaclient/app/boot_control/_jetson_uefi.py index dbb764755..fa46bafc2 100644 --- a/src/otaclient/app/boot_control/_jetson_uefi.py +++ b/src/otaclient/app/boot_control/_jetson_uefi.py @@ -50,8 +50,17 @@ logger = logging.getLogger(__name__) -# TODO: calculate this table -L4TLAUNCHER_BSP_VER_SHA256_MAP: dict[str, BSPVersion] = {} +L4TLAUNCHER_BSP_VER_SHA256_MAP: dict[str, BSPVersion] = { + "b14fa3623f4078d05573d9dcf2a0b46ea2ae07d6b75d9843f9da6ff24db13718": BSPVersion( + 36, 3, 0 + ), + "3928e0feb84e37db87dc6705a02eec95a6fca2dccd3467b61fa66ed8e1046b67": BSPVersion( + 35, 5, 0 + ), + "ac17457772666351154a5952e3b87851a6398da2afcf3a38bedfc0925760bb0e": BSPVersion( + 35, 4, 1 + ), +} class JetsonUEFIBootControlError(Exception): From b960a76ec65da2e4ef7d9745fbad42944f53a05b Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Tue, 25 Jun 2024 01:16:23 +0000 Subject: [PATCH 074/193] jetson-common: use re to match bsp version string --- src/otaclient/app/boot_control/_jetson_common.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/otaclient/app/boot_control/_jetson_common.py b/src/otaclient/app/boot_control/_jetson_common.py index 95baa3aa2..7081dc4d5 100644 --- a/src/otaclient/app/boot_control/_jetson_common.py +++ b/src/otaclient/app/boot_control/_jetson_common.py @@ -58,6 +58,11 @@ def __new__(cls, _in: str | Self) -> Self: SLOT_FLIP = {SLOT_A: SLOT_B, SLOT_B: SLOT_A} +BSP_VERSION_STR_PA = re.compile( + r"\w?(?P\d+)\.(?P\d+)\.(?P\d+)" +) + + class BSPVersion(NamedTuple): """BSP version in NamedTuple representation. @@ -73,8 +78,15 @@ def parse(cls, _in: str | BSPVersion | Any) -> Self: """Parse "Rxx.yy.z string into BSPVersion.""" if isinstance(_in, cls): return _in - if isinstance(_in, str) and len(_split := _in[1:].split(".")) == 3: - major_ver, major_rev, minor_rev = _split + if isinstance(_in, str): + ma = BSP_VERSION_STR_PA.match(_in) + assert ma, f"not a valid bsp version string: {_in}" + + major_ver, major_rev, minor_rev = ( + ma.group("major_ver"), + ma.group("major_rev"), + ma.group("minor_rev"), + ) return cls(int(major_ver), int(major_rev), int(minor_rev)) raise ValueError(f"expect str or BSPVersion instance, get {type(_in)}") From 155e86539c9e4974017aa915f4d59e28ef2418eb Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Tue, 25 Jun 2024 01:39:01 +0000 Subject: [PATCH 075/193] jetson-common: implment detect_rootfs_bsp_version --- .../app/boot_control/_jetson_common.py | 29 ++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/src/otaclient/app/boot_control/_jetson_common.py b/src/otaclient/app/boot_control/_jetson_common.py index 7081dc4d5..f93d9d366 100644 --- a/src/otaclient/app/boot_control/_jetson_common.py +++ b/src/otaclient/app/boot_control/_jetson_common.py @@ -30,6 +30,7 @@ from typing_extensions import Annotated, Literal, Self from otaclient.app.boot_control._common import CMDHelperFuncs +from otaclient.app.boot_control.configs import jetson_common_cfg from otaclient_common.common import copytree_identical, write_str_to_file_sync from otaclient_common.typing import StrOrPath @@ -75,7 +76,7 @@ class BSPVersion(NamedTuple): @classmethod def parse(cls, _in: str | BSPVersion | Any) -> Self: - """Parse "Rxx.yy.z string into BSPVersion.""" + """Parse BSP version string into BSPVersion.""" if isinstance(_in, cls): return _in if isinstance(_in, str): @@ -260,7 +261,7 @@ def standby_slot_fw_ver(self, bsp_ver: BSPVersion | None): self._version.set_by_slot(self.standby_slot, bsp_ver) -BSP_VER_PA = re.compile( +NV_TEGRA_RELEASE_PA = re.compile( ( r"# R(?P\d+) \(\w+\), REVISION: (?P\d+)\.(?P\d+), " r"GCID: (?P\d+), BOARD: (?P\w+), EABI: (?P\w+)" @@ -269,12 +270,12 @@ def standby_slot_fw_ver(self, bsp_ver: BSPVersion | None): """Example: # R32 (release), REVISION: 6.1, GCID: 27863751, BOARD: t186ref, EABI: aarch64, DATE: Mon Jul 26 19:36:31 UTC 2021 """ -def parse_bsp_version(nv_tegra_release: str) -> BSPVersion: +def parse_nv_tegra_release(nv_tegra_release: str) -> BSPVersion: """Parse BSP version from contents of /etc/nv_tegra_release. see https://developer.nvidia.com/embedded/jetson-linux-archive for BSP version history. """ - ma = BSP_VER_PA.match(nv_tegra_release) + ma = NV_TEGRA_RELEASE_PA.match(nv_tegra_release) assert ma, f"invalid nv_tegra_release content: {nv_tegra_release}" return BSPVersion( int(ma.group("major_ver")), @@ -283,6 +284,26 @@ def parse_bsp_version(nv_tegra_release: str) -> BSPVersion: ) +def detect_rootfs_bsp_version(rootfs: StrOrPath) -> BSPVersion: + """Detect rootfs BSP version on . + + Raises: + ValueError on failed detection. + + Returns: + BSPversion of the . + """ + nv_tegra_release_fpath = Path(rootfs) / Path( + jetson_common_cfg.NV_TEGRA_RELEASE_FPATH + ).relative_to("/") + try: + return parse_nv_tegra_release(nv_tegra_release_fpath.read_text()) + except Exception as e: + _err_msg = f"failed to detect rootfs BSP version at: {rootfs}: {e!r}" + logger.error(_err_msg) + raise ValueError(_err_msg) from e + + def update_extlinux_cfg(_input: str, partuuid: str) -> str: """Update input exlinux text with input rootfs .""" partuuid_str = f"PARTUUID={partuuid}" From ccb98ba7cc2f428bb959ad265aba41a6568ea0a0 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Tue, 25 Jun 2024 02:15:32 +0000 Subject: [PATCH 076/193] jetson-uefi: minor update to NVBootControlJetsonUEFI --- .../app/boot_control/_jetson_uefi.py | 38 ++++++++++--------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/src/otaclient/app/boot_control/_jetson_uefi.py b/src/otaclient/app/boot_control/_jetson_uefi.py index fa46bafc2..973192dfd 100644 --- a/src/otaclient/app/boot_control/_jetson_uefi.py +++ b/src/otaclient/app/boot_control/_jetson_uefi.py @@ -41,7 +41,8 @@ NVBootctrlCommon, SlotID, copy_standby_slot_boot_to_internal_emmc, - parse_bsp_version, + detect_rootfs_bsp_version, + parse_nv_tegra_release, preserve_ota_config_files_to_standby, update_standby_slot_extlinux_cfg, ) @@ -67,8 +68,20 @@ class JetsonUEFIBootControlError(Exception): """Exception type for covering jetson-uefi related errors.""" -class _NVBootctrl(NVBootctrlCommon): - """Helper for calling nvbootctrl commands. +class NVBootctrlJetsonUEFI(NVBootctrlCommon): + """Helper for calling nvbootctrl commands, for platform using jetson-uefi. + + Typical output of nvbootctrl dump-slots-info: + + ``` + Current version: 35.4.1 + Capsule update status: 1 + Current bootloader slot: A + Active bootloader slot: A + num_slots: 2 + slot: 0, status: normal + slot: 1, status: normal + ``` For BSP version >= R34 with UEFI boot. Without -t option, the target will be bootloader by default. @@ -78,28 +91,17 @@ class _NVBootctrl(NVBootctrlCommon): def get_current_fw_bsp_version(cls) -> BSPVersion: """Get current boot chain's firmware BSP version with nvbootctrl.""" _raw = cls.dump_slots_info() - """Example: - Current version: 35.4.1 - Capsule update status: 1 - Current bootloader slot: A - Active bootloader slot: A - num_slots: 2 - slot: 0, status: normal - slot: 1, status: normal - """ - pa = re.compile(r"\s*Current version:\s(?P[\.\d]+)\s*") + pa = re.compile(r"\s*Current version:\s*(?P[\.\d]+)\s*") if not (ma := pa.search(_raw)): _err_msg = "nvbootctrl failed to report BSP version" logger.error(_err_msg) - raise ValueError(_err_msg) + raise JetsonUEFIBootControlError(_err_msg) - bsp_ver_str = ( - f"r{ma.group('bsp_ver')}" # NOTE: need to add 'r' prefix back here - ) + bsp_ver_str = ma.group("bsp_ver") bsp_ver = BSPVersion.parse(bsp_ver_str) if bsp_ver.major_rev == 0: - _err_msg = f"invalid BSP version: {bsp_ver_str}, this might indicate broken firmware!" + _err_msg = f"invalid BSP version: {bsp_ver_str}, this might indicate an incomplete flash!" logger.warning(_err_msg) raise ValueError(_err_msg) return bsp_ver From 716456285f1e24c2c3588a079e6806a7f36c9339 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Tue, 25 Jun 2024 02:16:18 +0000 Subject: [PATCH 077/193] jetson-uefi: refactor UEFIFirmwareUpdater --- .../app/boot_control/_jetson_uefi.py | 204 ++++++++++-------- 1 file changed, 109 insertions(+), 95 deletions(-) diff --git a/src/otaclient/app/boot_control/_jetson_uefi.py b/src/otaclient/app/boot_control/_jetson_uefi.py index 973192dfd..8328af574 100644 --- a/src/otaclient/app/boot_control/_jetson_uefi.py +++ b/src/otaclient/app/boot_control/_jetson_uefi.py @@ -113,10 +113,78 @@ def verify(cls) -> str: EFIVARS_FSTYPE = "efivarfs" -EFIVARS_DPATH = "/sys/firmware/efi/efivars/" +EFIVARS_SYS_MOUNT_POINT = "/sys/firmware/efi/efivars/" -class CapsuleUpdate: +@contextlib.contextmanager +def _ensure_efivarfs_mounted() -> Generator[None, Any, None]: + """Ensure the efivarfs is mounted as rw, and then umount it.""" + if CMDHelperFuncs.is_target_mounted(EFIVARS_SYS_MOUNT_POINT): + options = "remount,rw,nosuid,nodev,noexec,relatime" + else: + logger.warning( + f"efivars is not mounted! try to mount it at {EFIVARS_SYS_MOUNT_POINT}" + ) + options = "rw,nosuid,nodev,noexec,relatime" + + # fmt: off + cmd = [ + "mount", + "-t", EFIVARS_FSTYPE, + "-o", options, + EFIVARS_FSTYPE, + EFIVARS_SYS_MOUNT_POINT + ] + # fmt: on + try: + subprocess_call(cmd, raise_exception=True) + yield + except Exception as e: + raise JetsonUEFIBootControlError( + f"failed to mount {EFIVARS_FSTYPE} on {EFIVARS_SYS_MOUNT_POINT}: {e!r}" + ) from e + + +@contextlib.contextmanager +def _ensure_esp_mounted( + esp_dev: StrOrPath, mount_point: StrOrPath +) -> Generator[None, Any, None]: + """Mount the esp partition and then umount it.""" + mount_point = Path(mount_point) + mount_point.mkdir(exist_ok=True, parents=True) + + try: + CMDHelperFuncs.mount_rw(str(esp_dev), mount_point) + yield + CMDHelperFuncs.umount(mount_point, raise_exception=False) + except Exception as e: + _err_msg = f"failed to mount {esp_dev} to {mount_point}: {e!r}" + logger.error(_err_msg) + raise JetsonUEFIBootControlError(_err_msg) from e + + +def _detect_esp_dev(boot_parent_devpath: StrOrPath) -> str: + # NOTE: if boots from external, expects to have multiple esp parts, + # we need to get the one at our booted parent dev. + esp_parts = CMDHelperFuncs.get_dev_by_token( + token="PARTLABEL", value=boot_cfg.ESP_PARTLABEL + ) + if not esp_parts: + raise JetsonUEFIBootControlError("no ESP partition presented") + + for _esp_part in esp_parts: + if _esp_part.find(str(boot_parent_devpath)) != -1: + logger.info(f"find esp partition at {_esp_part}") + esp_part = _esp_part + break + else: + _err_msg = f"failed to find esp partition on {boot_parent_devpath}" + logger.error(_err_msg) + raise JetsonUEFIBootControlError(_err_msg) + return esp_part + + +class UEFIFirmwareUpdater: """Firmware update implementation using Capsule update.""" def __init__( @@ -124,11 +192,20 @@ def __init__( boot_parent_devpath: StrOrPath, standby_slot_mp: StrOrPath, *, - ota_image_bsp_ver: BSPVersion, fw_bsp_ver_control: FirmwareBSPVersionControl, ) -> None: + """Init an instance of UEFIFirmwareUpdater. + + Args: + boot_parent_devpath (StrOrPath): The parent dev of current rootfs device. + standby_slot_mp (StrOrPath): The mount point of updated standby slot. The firmware update package + is located at /opt/ota_package folder. + ota_image_bsp_ver (BSPVersion): The BSP version of OTA image used to update the standby slot. + fw_bsp_ver_control (FirmwareBSPVersionControl): The firmware BSP version of each slots. + """ self.fw_bsp_ver_control = fw_bsp_ver_control - self.ota_image_bsp_ver = ota_image_bsp_ver + # NOTE: standby slot is updated with OTA image + self.ota_image_bsp_ver = detect_rootfs_bsp_version(standby_slot_mp) # NOTE: use the esp partition at the current booted device # i.e., if we boot from nvme0n1, then bootdev_path is /dev/nvme0n1 and @@ -136,61 +213,25 @@ def __init__( self.esp_mp = Path(boot_cfg.ESP_MOUNTPOINT) self.esp_boot_dir = self.esp_mp / "EFI" / "BOOT" self.l4tlauncher_ver_fpath = self.esp_boot_dir / boot_cfg.L4TLAUNCHER_VER_FNAME + """A plain text file stores the BSP version string.""" + self.bootaa64_at_esp = self.esp_boot_dir / boot_cfg.L4TLAUNCHER_FNAME + """The canonical location of L4TLauncher, called by UEFI.""" + self.bootaa64_at_esp_bak = ( self.esp_boot_dir / f"{boot_cfg.L4TLAUNCHER_FNAME}_bak" ) - # NOTE: we get the update capsule from the standby slot self.standby_slot_mp = Path(standby_slot_mp) - self.esp_part = self._detect_esp_dev(boot_parent_devpath) + self.fw_loc_at_standby_slot = self.standby_slot_mp / Path( + boot_cfg.CAPSULE_PAYLOAD_AT_ROOTFS + ).relative_to("/") + """where the fw update capsule and l4tlauncher bin located.""" - @staticmethod - @contextlib.contextmanager - def _ensure_efivarfs_mounted() -> Generator[None, Any, None]: - """Ensure the efivarfs is mounted as rw.""" - if CMDHelperFuncs.is_target_mounted(EFIVARS_DPATH): - options = "remount,rw,nosuid,nodev,noexec,relatime" - else: - logger.warning( - f"efivars is not mounted! try to mount it at {EFIVARS_DPATH}" - ) - options = "rw,nosuid,nodev,noexec,relatime" - - # fmt: off - cmd = [ - "mount", - "-t", EFIVARS_FSTYPE, - "-o", options, - EFIVARS_FSTYPE, - EFIVARS_DPATH - ] - # fmt: on - try: - subprocess_call(cmd, raise_exception=True) - yield - except Exception as e: - raise JetsonUEFIBootControlError( - f"failed to mount {EFIVARS_FSTYPE} on {EFIVARS_DPATH}: {e!r}" - ) from e + self.esp_part = _detect_esp_dev(boot_parent_devpath) - @staticmethod - @contextlib.contextmanager - def _ensure_esp_mounted( - esp_dev: StrOrPath, mount_point: StrOrPath - ) -> Generator[None, Any, None]: - """Mount the esp partition and then umount it.""" - mount_point = Path(mount_point) - mount_point.mkdir(exist_ok=True, parents=True) - - try: - CMDHelperFuncs.mount_rw(str(esp_dev), mount_point) - yield - CMDHelperFuncs.umount(mount_point, raise_exception=False) - except Exception as e: - _err_msg = f"failed to mount {esp_dev} to {mount_point}: {e!r}" - logger.error(_err_msg) - raise JetsonUEFIBootControlError(_err_msg) from e + self.capsule_dir_at_esp = self.esp_mp / boot_cfg.CAPSULE_PAYLOAD_AT_ESP + """The location to put UEFI capsule to. The UEFI will use the capsule in this location.""" def _prepare_fwupdate_capsule(self) -> bool: """Copy the Capsule update payloads to specific location at esp partition. @@ -199,27 +240,22 @@ def _prepare_fwupdate_capsule(self) -> bool: True if at least one of the update capsule is prepared, False if no update capsule is available and configured. """ - capsule_at_esp = self.esp_mp / boot_cfg.CAPSULE_PAYLOAD_AT_ESP - capsule_at_esp.mkdir(parents=True, exist_ok=True) - - # where the fw update capsule and l4tlauncher bin located - fw_loc_at_standby_slot = self.standby_slot_mp / Path( - boot_cfg.CAPSULE_PAYLOAD_AT_ROOTFS - ).relative_to("/") + capsule_dir_at_esp = self.capsule_dir_at_esp + capsule_dir_at_esp.mkdir(parents=True, exist_ok=True) # ------ prepare capsule update payload ------ # firmware_package_configured = False for capsule_fname in boot_cfg.FIRMWARE_LIST: try: shutil.copy( - src=fw_loc_at_standby_slot / capsule_fname, - dst=capsule_at_esp / capsule_fname, + src=self.fw_loc_at_standby_slot / capsule_fname, + dst=capsule_dir_at_esp / capsule_fname, ) firmware_package_configured = True - logger.info(f"copy {capsule_fname} to {capsule_at_esp}") + logger.info(f"copy {capsule_fname} to {capsule_dir_at_esp}") except Exception as e: logger.warning( - f"failed to copy {capsule_fname} from {fw_loc_at_standby_slot} to {capsule_at_esp}: {e!r}" + f"failed to copy {capsule_fname} from {self.fw_loc_at_standby_slot} to {capsule_dir_at_esp}: {e!r}" ) logger.warning(f"skip {capsule_fname}") return firmware_package_configured @@ -230,11 +266,7 @@ def _update_l4tlauncher(self) -> bool: logger.warning(f"update the l4tlauncher to version {ota_image_l4tlauncher_ver}") # new BOOTAA64.efi is located at /opt/ota_package/BOOTAA64.efi - ota_image_bootaa64 = ( - self.standby_slot_mp - / Path(boot_cfg.CAPSULE_PAYLOAD_AT_ROOTFS).relative_to("/") - / boot_cfg.L4TLAUNCHER_FNAME - ) + ota_image_bootaa64 = self.fw_loc_at_standby_slot / boot_cfg.L4TLAUNCHER_FNAME if not ota_image_bootaa64.is_file(): logger.warning(f"{ota_image_bootaa64} not found, skip update l4tlauncher") return False @@ -254,11 +286,14 @@ def _write_magic_efivar() -> None: Raises: JetsonUEFIBootControlError on failed Capsule update preparing. """ - magic_efivar_fpath = Path(EFIVARS_DPATH) / boot_cfg.UPDATE_TRIGGER_EFIVAR + magic_efivar_fpath = ( + Path(EFIVARS_SYS_MOUNT_POINT) / boot_cfg.UPDATE_TRIGGER_EFIVAR + ) magic_efivar_fpath.write_bytes(boot_cfg.MAGIC_BYTES) os.sync() def _detect_l4tlauncher_version(self) -> BSPVersion: + """Try to determine the current in use l4tlauncher version.""" l4tlauncher_bsp_ver = None try: l4tlauncher_bsp_ver = BSPVersion.parse( @@ -268,11 +303,10 @@ def _detect_l4tlauncher_version(self) -> BSPVersion: logger.warning(f"missing or invalid l4tlauncher version file: {e!r}") self.l4tlauncher_ver_fpath.unlink(missing_ok=True) - bootaa64_at_esp = self.esp_boot_dir / boot_cfg.L4TLAUNCHER_FNAME # NOTE(20240624): since the number of l4tlauncher version is limited, # we can lookup against a pre-calculated sha256 digest map. if l4tlauncher_bsp_ver is None: - _l4tlauncher_sha256_digest = file_sha256(bootaa64_at_esp) + _l4tlauncher_sha256_digest = file_sha256(self.bootaa64_at_esp) logger.info( f"try to determine the l4tlauncher verison by hash: {_l4tlauncher_sha256_digest}" ) @@ -283,7 +317,7 @@ def _detect_l4tlauncher_version(self) -> BSPVersion: current_slot_fw_bsp_ver = self.fw_bsp_ver_control.current_slot_fw_ver # NOTE(20240624): if we failed to detect the l4tlauncher's version, # we assume that the launcher is the same version as current slot's fw. - # This is typically case for a newly OTA inital setup ECU. + # This is typically the case of a newly flashed ECU. if l4tlauncher_bsp_ver is None: logger.warning( ( @@ -295,30 +329,10 @@ def _detect_l4tlauncher_version(self) -> BSPVersion: write_str_to_file_sync( self.l4tlauncher_ver_fpath, l4tlauncher_bsp_ver.dump() ) + logger.info(f"finish detecting l4tlauncher version: {l4tlauncher_bsp_ver}") return l4tlauncher_bsp_ver - @staticmethod - def _detect_esp_dev(boot_parent_devpath: StrOrPath) -> str: - # NOTE: if boots from external, expects to have multiple esp parts, - # we need to get the one at our booted parent dev. - esp_parts = CMDHelperFuncs.get_dev_by_token( - token="PARTLABEL", value=boot_cfg.ESP_PARTLABEL - ) - if not esp_parts: - raise JetsonUEFIBootControlError("no ESP partition presented") - - for _esp_part in esp_parts: - if _esp_part.find(str(boot_parent_devpath)) != -1: - logger.info(f"find esp partition at {_esp_part}") - esp_part = _esp_part - break - else: - _err_msg = f"failed to find esp partition on {boot_parent_devpath}" - logger.error(_err_msg) - raise JetsonUEFIBootControlError(_err_msg) - return esp_part - # APIs def firmware_update(self) -> bool: @@ -340,12 +354,12 @@ def firmware_update(self) -> bool: ) return False - with self._ensure_esp_mounted(self.esp_part, self.esp_mp): + with _ensure_esp_mounted(self.esp_part, self.esp_mp): if not self._prepare_fwupdate_capsule(): logger.info("no firmware file is prepared, skip firmware update") return False - with self._ensure_efivarfs_mounted(): + with _ensure_efivarfs_mounted(): try: self._write_magic_efivar() except Exception as e: @@ -382,7 +396,7 @@ def l4tlauncher_update(self) -> bool: ) return False - with self._ensure_esp_mounted(self.esp_part, self.esp_mp): + with _ensure_esp_mounted(self.esp_part, self.esp_mp): return self._update_l4tlauncher() From bb942bf1721cf45e04c257fba5570e2ee4d124df Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Tue, 25 Jun 2024 02:25:35 +0000 Subject: [PATCH 078/193] jetson-common: add SLOT_PAR_MAP --- src/otaclient/app/boot_control/_jetson_common.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/otaclient/app/boot_control/_jetson_common.py b/src/otaclient/app/boot_control/_jetson_common.py index f93d9d366..d42069adf 100644 --- a/src/otaclient/app/boot_control/_jetson_common.py +++ b/src/otaclient/app/boot_control/_jetson_common.py @@ -57,7 +57,8 @@ def __new__(cls, _in: str | Self) -> Self: SLOT_A, SLOT_B = SlotID("0"), SlotID("1") SLOT_FLIP = {SLOT_A: SLOT_B, SLOT_B: SLOT_A} - +SLOT_PAR_MAP = {SLOT_A: 1, SLOT_B: 2} +"""SLOT_A: 1, SLOT_B: 2""" BSP_VERSION_STR_PA = re.compile( r"\w?(?P\d+)\.(?P\d+)\.(?P\d+)" From e1d8654a63b2b5f9fc85d61ff83bc977b7927a93 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Tue, 25 Jun 2024 02:30:08 +0000 Subject: [PATCH 079/193] jetson-uefi: refine UEFIBoot implementation --- .../app/boot_control/_jetson_uefi.py | 62 +++++++++---------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/src/otaclient/app/boot_control/_jetson_uefi.py b/src/otaclient/app/boot_control/_jetson_uefi.py index 8328af574..16d83ee39 100644 --- a/src/otaclient/app/boot_control/_jetson_uefi.py +++ b/src/otaclient/app/boot_control/_jetson_uefi.py @@ -39,7 +39,7 @@ BSPVersion, FirmwareBSPVersionControl, NVBootctrlCommon, - SlotID, + SLOT_PAR_MAP, copy_standby_slot_boot_to_internal_emmc, detect_rootfs_bsp_version, parse_nv_tegra_release, @@ -400,11 +400,13 @@ def l4tlauncher_update(self) -> bool: return self._update_l4tlauncher() +MINIMUM_SUPPORTED_BSP_VERSION = BSPVersion(35, 2, 0) +"""Only after R35.2, the Capsule Firmware update is available.""" + + class _UEFIBoot: """Low-level boot control implementation for jetson-uefi.""" - _slot_id_partid = {SlotID("0"): "1", SlotID("1"): "2"} - def __init__(self): # ------ sanity check, confirm we are at jetson device ------ # tegra_compat_info_fpath = Path(boot_cfg.TEGRA_COMPAT_PATH) @@ -413,6 +415,9 @@ def __init__(self): logger.error(_err_msg) raise JetsonUEFIBootControlError(_err_msg) + # print hardware model + logger.info(f"hardware model: {Path(boot_cfg.MODEL_FPATH)}") + compat_info = tegra_compat_info_fpath.read_text() # example compatible string: # nvidia,p3737-0000+p3701-0000nvidia,tegra234nvidia,tegra23x @@ -422,11 +427,11 @@ def __init__(self): raise JetsonUEFIBootControlError(_err_msg) logger.info(f"dev compatibility: {compat_info}") - # ------ check BSP version ------ # - # check firmware BSP version + # ------ check current slot BSP version ------ # + # check current slot firmware BSP version try: self.fw_bsp_version = fw_bsp_version = ( - _NVBootctrl.get_current_fw_bsp_version() + NVBootctrlJetsonUEFI.get_current_fw_bsp_version() ) assert fw_bsp_version, "bsp version information not available" logger.info(f"current slot firmware BSP version: {fw_bsp_version}") @@ -436,10 +441,10 @@ def __init__(self): raise JetsonUEFIBootControlError(_err_msg) logger.info(f"{fw_bsp_version=}") - # check rootfs BSP version + # check current slot rootfs BSP version try: - self.rootfs_bsp_verion = rootfs_bsp_version = parse_bsp_version( - Path(boot_cfg.NV_TEGRA_RELEASE_FPATH).read_text() + self.rootfs_bsp_verion = rootfs_bsp_version = detect_rootfs_bsp_version( + rootfs=cfg.ACTIVE_ROOTFS_PATH ) logger.info(f"current slot rootfs BSP version: {rootfs_bsp_version}") except Exception as e: @@ -455,8 +460,7 @@ def __init__(self): ) # ------ sanity check, currently jetson-uefi only supports >= R35.2 ----- # - # only after R35.2, the Capsule Firmware update is available. - if fw_bsp_version < (35, 2, 0): + if fw_bsp_version < MINIMUM_SUPPORTED_BSP_VERSION: _err_msg = f"jetson-uefi only supports BSP version >= R35.2, but get {fw_bsp_version=}. " logger.error(_err_msg) raise JetsonUEFIBootControlError(_err_msg) @@ -465,35 +469,36 @@ def __init__(self): logger.info("unified A/B is enabled") # ------ check A/B slots ------ # - self.current_slot = current_slot = _NVBootctrl.get_current_slot() - self.standby_slot = standby_slot = _NVBootctrl.get_standby_slot() + self.current_slot = current_slot = NVBootctrlJetsonUEFI.get_current_slot() + self.standby_slot = standby_slot = NVBootctrlJetsonUEFI.get_standby_slot() logger.info(f"{current_slot=}, {standby_slot=}") # ------ detect rootfs_dev and parent_dev ------ # self.curent_rootfs_devpath = current_rootfs_devpath = ( - CMDHelperFuncs.get_current_rootfs_dev().strip() + CMDHelperFuncs.get_current_rootfs_dev() ) self.parent_devpath = parent_devpath = Path( - CMDHelperFuncs.get_parent_dev(current_rootfs_devpath).strip() + CMDHelperFuncs.get_parent_dev(current_rootfs_devpath) ) # --- detect boot device --- # - self._external_rootfs = False + self.external_rootfs = False + parent_devname = parent_devpath.name if parent_devname.startswith(boot_cfg.MMCBLK_DEV_PREFIX): logger.info(f"device boots from internal emmc: {parent_devpath}") elif parent_devname.startswith(boot_cfg.NVMESSD_DEV_PREFIX): logger.info(f"device boots from external nvme ssd: {parent_devpath}") - self._external_rootfs = True + self.external_rootfs = True else: - _err_msg = f"we don't support boot from {parent_devpath=} currently" + _err_msg = f"currently we don't support booting from {parent_devpath=}" logger.error(_err_msg) raise JetsonUEFIBootControlError(_err_msg) from NotImplementedError( f"unsupported bootdev {parent_devpath}" ) self.standby_rootfs_devpath = ( - f"/dev/{parent_devname}p{self._slot_id_partid[standby_slot]}" + f"/dev/{parent_devname}p{SLOT_PAR_MAP[standby_slot]}" ) self.standby_rootfs_dev_partuuid = CMDHelperFuncs.get_attrs_by_dev( "PARTUUID", self.standby_rootfs_devpath @@ -508,29 +513,24 @@ def __init__(self): f"standby_rootfs(slot {standby_slot}): {self.standby_rootfs_devpath=}, {self.standby_rootfs_dev_partuuid=}" ) - self.standby_internal_emmc_devpath = f"/dev/{boot_cfg.INTERNAL_EMMC_DEVNAME}p{self._slot_id_partid[standby_slot]}" + self.standby_internal_emmc_devpath = ( + f"/dev/{boot_cfg.INTERNAL_EMMC_DEVNAME}p{SLOT_PAR_MAP[standby_slot]}" + ) logger.info("finished jetson-uefi boot control startup") - logger.info(f"nvbootctrl dump-slots-info: \n{_NVBootctrl.dump_slots_info()}") + logger.info( + f"nvbootctrl dump-slots-info: \n{NVBootctrlJetsonUEFI.dump_slots_info()}" + ) # API - @property - def external_rootfs_enabled(self) -> bool: - """Indicate whether rootfs on external storage is enabled. - - NOTE: distiguish from boot from external storage, as R32.5 and below doesn't - support native NVMe boot. - """ - return self._external_rootfs - def switch_boot_to_standby(self) -> None: target_slot = self.standby_slot logger.info(f"switch boot to standby slot({target_slot})") # when unified_ab enabled, switching bootloader slot will also switch # the rootfs slot. - _NVBootctrl.set_active_boot_slot(target_slot) + NVBootctrlJetsonUEFI.set_active_boot_slot(target_slot) class JetsonUEFIBootControl(BootControllerProtocol): From 7e30ce99a3d094c715a9b34ea476115c417fecc4 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Tue, 25 Jun 2024 02:41:31 +0000 Subject: [PATCH 080/193] finish refining of jetson-uefi --- .../app/boot_control/_jetson_uefi.py | 51 ++++++++----------- src/otaclient/app/boot_control/configs.py | 4 ++ 2 files changed, 25 insertions(+), 30 deletions(-) diff --git a/src/otaclient/app/boot_control/_jetson_uefi.py b/src/otaclient/app/boot_control/_jetson_uefi.py index 16d83ee39..f428cd3ba 100644 --- a/src/otaclient/app/boot_control/_jetson_uefi.py +++ b/src/otaclient/app/boot_control/_jetson_uefi.py @@ -42,7 +42,6 @@ SLOT_PAR_MAP, copy_standby_slot_boot_to_internal_emmc, detect_rootfs_bsp_version, - parse_nv_tegra_release, preserve_ota_config_files_to_standby, update_standby_slot_extlinux_cfg, ) @@ -544,7 +543,6 @@ def __init__(self) -> None: ).relative_to("/") try: - # startup boot controller self._uefi_control = uefi_control = _UEFIBoot() # mount point prepare @@ -556,18 +554,14 @@ def __init__(self) -> None: ) # load firmware BSP version - self._current_fw_bsp_ver_fpath = ( + current_fw_bsp_ver_fpath = ( current_ota_status_dir / boot_cfg.FIRMWARE_BSP_VERSION_FNAME ) - # NOTE: standby slot's bsp version file might be not yet - # available before an OTA. - self._standby_fw_bsp_ver_fpath = ( - standby_ota_status_dir / boot_cfg.FIRMWARE_BSP_VERSION_FNAME - ) + self._firmware_ver_control = fw_bsp_ver = FirmwareBSPVersionControl( current_slot=uefi_control.current_slot, current_slot_firmware_bsp_ver=uefi_control.fw_bsp_version, - current_firmware_bsp_vf=self._current_fw_bsp_ver_fpath, + current_firmware_bsp_vf=current_fw_bsp_ver_fpath, ) # init ota-status files @@ -581,13 +575,13 @@ def __init__(self) -> None: ) # post starting up, write the firmware bsp version to current slot - # NOTE 1: we always update and refer current slot's firmware bsp version file. + # NOTE 1: we always update and refer to ONLY current slot's firmware bsp version file. # NOTE 2: if OTA status is failure, always assume the firmware update on standby slot failed, # and clear the standby slot's fw bsp version record. if self._ota_status_control._ota_status == api_types.StatusOta.FAILURE: fw_bsp_ver.standby_slot_fw_ver = None fw_bsp_ver.current_slot_fw_ver = uefi_control.fw_bsp_version - fw_bsp_ver.write_to_file(self._current_fw_bsp_ver_fpath) + fw_bsp_ver.write_to_file(current_fw_bsp_ver_fpath) except Exception as e: _err_msg = f"failed to start jetson-uefi controller: {e!r}" raise ota_errors.BootControlStartupFailed(_err_msg, module=__name__) from e @@ -599,7 +593,7 @@ def _finalize_switching_boot(self) -> bool: previous slot, so actually we don't need to do any checks here. """ try: - fw_update_verify = _NVBootctrl.verify() + fw_update_verify = NVBootctrlJetsonUEFI.verify() logger.info(f"nvbootctrl verify: {fw_update_verify}") except Exception as e: logger.warning(f"nvbootctrl verify failed: {e!r}") @@ -614,10 +608,9 @@ def _capsule_firmware_update(self) -> bool: True if there is firmware update configured, False for no firmware update. """ logger.info("jetson-uefi: checking if we need to do firmware update ...") - standby_bootloader_slot = self._uefi_control.standby_slot - # ------ check if we need to do firmware update ------ # + # ------ check if we need to skip firmware update ------ # skip_firmware_update_hint_file = ( self._mp_control.standby_slot_mount_point / Path(boot_cfg.CAPSULE_PAYLOAD_AT_ROOTFS).relative_to("/") @@ -629,28 +622,26 @@ def _capsule_firmware_update(self) -> bool: ) return False - _new_bsp_v_fpath = self._mp_control.standby_slot_mount_point / Path( - boot_cfg.NV_TEGRA_RELEASE_FPATH - ).relative_to("/") try: - new_bsp_v = parse_bsp_version(_new_bsp_v_fpath.read_text()) + ota_image_bsp_ver = detect_rootfs_bsp_version( + self._mp_control.standby_slot_mount_point + ) except Exception as e: logger.warning(f"failed to detect new image's BSP version: {e!r}") logger.info("skip firmware update due to new image BSP version unknown") return False standby_firmware_bsp_ver = self._firmware_ver_control.standby_slot_fw_ver - if standby_firmware_bsp_ver and standby_firmware_bsp_ver >= new_bsp_v: + if standby_firmware_bsp_ver and standby_firmware_bsp_ver >= ota_image_bsp_ver: logger.info( f"{standby_bootloader_slot=} has newer or equal ver of firmware, skip firmware update" ) return False # ------ prepare firmware update ------ # - firmware_updater = CapsuleUpdate( + firmware_updater = UEFIFirmwareUpdater( boot_parent_devpath=self._uefi_control.parent_devpath, standby_slot_mp=self._mp_control.standby_slot_mount_point, - ota_image_bsp_ver=new_bsp_v, fw_bsp_ver_control=self._firmware_ver_control, ) if firmware_updater.firmware_update(): @@ -658,7 +649,7 @@ def _capsule_firmware_update(self) -> bool: logger.info( ( - f"will update to new firmware version in next reboot: {new_bsp_v=}, \n" + f"will update to new firmware version in next reboot: {ota_image_bsp_ver=}, \n" f"will switch to Slot({standby_bootloader_slot}) on successful firmware update" ) ) @@ -676,16 +667,12 @@ def get_standby_boot_dir(self) -> Path: def pre_update(self, version: str, *, standby_as_ref: bool, erase_standby: bool): try: logger.info("jetson-uefi: pre-update ...") - # udpate active slot's ota_status self._ota_status_control.pre_update_current() - # prepare standby slot dev self._mp_control.prepare_standby_dev(erase_standby=erase_standby) - # mount slots self._mp_control.mount_standby() self._mp_control.mount_active() - # update standby slot's ota_status files self._ota_status_control.pre_update_standby(version=version) except Exception as e: _err_msg = f"failed on pre_update: {e!r}" @@ -706,7 +693,11 @@ def post_update(self) -> Generator[None, None, None]: ) # ------ preserve BSP version file to standby slot ------ # - self._firmware_ver_control.write_to_file(self._standby_fw_bsp_ver_fpath) + standby_fw_bsp_ver_fpath = ( + self._ota_status_control.standby_ota_status_dir + / boot_cfg.FIRMWARE_BSP_VERSION_FNAME + ) + self._firmware_ver_control.write_to_file(standby_fw_bsp_ver_fpath) # ------ preserve /boot/ota folder to standby rootfs ------ # preserve_ota_config_files_to_standby( @@ -720,16 +711,16 @@ def post_update(self) -> Generator[None, None, None]: # ------ switch boot to standby ------ # firmware_update_triggered = self._capsule_firmware_update() - # NOTE: manual switch boot will cancel the firmware update and cancel the switch boot itself! + # NOTE: manual switch boot will cancel the scheduled firmware update! if not firmware_update_triggered: self._uefi_control.switch_boot_to_standby() logger.info( - f"no firmware update configured, manually switch slot: \n{_NVBootctrl.dump_slots_info()}" + f"no firmware update configured, manually switch slot: \n{NVBootctrlJetsonUEFI.dump_slots_info()}" ) # ------ for external rootfs, preserve /boot folder to internal ------ # # NOTE: the copy should happen AFTER the changes to /boot folder at active slot. - if self._uefi_control._external_rootfs: + if self._uefi_control.external_rootfs: logger.info( "rootfs on external storage enabled: " "copy standby slot rootfs' /boot folder " diff --git a/src/otaclient/app/boot_control/configs.py b/src/otaclient/app/boot_control/configs.py index 950812e18..b10429d74 100644 --- a/src/otaclient/app/boot_control/configs.py +++ b/src/otaclient/app/boot_control/configs.py @@ -41,6 +41,7 @@ class JetsonBootCommon: EXTLINUX_FILE = "/boot/extlinux/extlinux.conf" FIRMWARE_DPATH = "/opt/ota_package" """Refer to standby slot rootfs.""" + MODEL_FPATH = "/proc/device-tree/model" NV_TEGRA_RELEASE_FPATH = "/etc/nv_tegra_release" SEPARATE_BOOT_MOUNT_POINT = "/mnt/standby_boot" @@ -92,6 +93,9 @@ class RPIBootControlConfig(BaseConfig): grub_cfg = GrubControlConfig() + +jetson_common_cfg = JetsonBootCommon() cboot_cfg = JetsonCBootControlConfig() jetson_uefi_cfg = JetsonUEFIBootControlConfig() + rpi_boot_cfg = RPIBootControlConfig() From 0584fa45497e2b5e1c048e77f92f83beebe8b5be Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 25 Jun 2024 02:43:41 +0000 Subject: [PATCH 081/193] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/otaclient/app/boot_control/_jetson_uefi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/otaclient/app/boot_control/_jetson_uefi.py b/src/otaclient/app/boot_control/_jetson_uefi.py index f428cd3ba..7263e5906 100644 --- a/src/otaclient/app/boot_control/_jetson_uefi.py +++ b/src/otaclient/app/boot_control/_jetson_uefi.py @@ -31,15 +31,15 @@ from otaclient.app import errors as ota_errors from otaclient.app.configs import config as cfg from otaclient_api.v2 import types as api_types -from otaclient_common.common import subprocess_call, write_str_to_file_sync, file_sha256 +from otaclient_common.common import file_sha256, subprocess_call, write_str_to_file_sync from otaclient_common.typing import StrOrPath from ._common import CMDHelperFuncs, OTAStatusFilesControl, SlotMountHelper from ._jetson_common import ( + SLOT_PAR_MAP, BSPVersion, FirmwareBSPVersionControl, NVBootctrlCommon, - SLOT_PAR_MAP, copy_standby_slot_boot_to_internal_emmc, detect_rootfs_bsp_version, preserve_ota_config_files_to_standby, From db06679eb3f70ff8bbae08fd3120a3aa893b92c7 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 26 Jun 2024 08:51:48 +0000 Subject: [PATCH 082/193] test_jetson_common: layout the test structure --- .../test_boot_control/test_jetson_common.py | 114 ++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 tests/test_otaclient/test_boot_control/test_jetson_common.py diff --git a/tests/test_otaclient/test_boot_control/test_jetson_common.py b/tests/test_otaclient/test_boot_control/test_jetson_common.py new file mode 100644 index 000000000..c4dcc4eca --- /dev/null +++ b/tests/test_otaclient/test_boot_control/test_jetson_common.py @@ -0,0 +1,114 @@ +# Copyright 2022 TIER IV, INC. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for Jetson device boot control implementation common.""" + + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from otaclient.app.boot_control._jetson_common import ( + SlotID, + BSPVersion, + FirmwareBSPVersion, +) + + +def test_SlotID(_in: Any, _expect: SlotID | None, _exc: Exception | None): + pass + + +class TestBSPVersion: + + def test_parse(self, _in: Any, _expect: BSPVersion | None, _exc: Exception | None): + pass + + def test_dump(self, _in: BSPVersion, _expect: str): + pass + + +class TestFirmwareBSPVersion: + + def test_init(self, slot_a_ver: BSPVersion | None, slot_b_ver: BSPVersion | None): + pass + + def test_set_by_slot( + self, + _in: FirmwareBSPVersion, + _slot: SlotID, + _bsp_ver: BSPVersion | None, + _expect: FirmwareBSPVersion, + ): + pass + + def test_get_by_slot( + self, + _in: FirmwareBSPVersion, + _slot: SlotID, + _exp: BSPVersion | None, + ): + pass + + def test_load_and_dump(self, _in: FirmwareBSPVersion): + pass + + +class TestFirmwareBSPVersionControl: + + def test_init(self): + pass + + def test_write_to_file(self): + pass + + def test_get_set_current_slot(self): + pass + + def test_get_set_standby_slot(self): + pass + + +def test_parse_nv_tegra_release(_in: str, _expect: BSPVersion): + pass + + +def test_detect_rootfs_bsp_version(tmp_path: Path): + pass + + +def test_update_extlinux_cfg(_in: str, _partuuid: str, _expect: str): + pass + + +class Test_copy_standby_slot_boot_to_internal_emmc: + + @pytest.fixture(autouse=True) + def setup_test(self): + pass + + def test_copy_standby_slot_boot_to_internal_emmc(self): + pass + + +class Test_preserve_ota_config_files_to_standby: + + @pytest.fixture(autouse=True) + def setup_test(self): + pass + + def test_preserve_ota_config_files_to_standby(self): + pass From a67c2b706907f9fdd94687298d17062bc6efe996 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 26 Jun 2024 09:03:29 +0000 Subject: [PATCH 083/193] temporarily remove the cboot test file --- .../test_boot_control/test_jetson_cboot.py | 144 ------------------ 1 file changed, 144 deletions(-) delete mode 100644 tests/test_otaclient/test_boot_control/test_jetson_cboot.py diff --git a/tests/test_otaclient/test_boot_control/test_jetson_cboot.py b/tests/test_otaclient/test_boot_control/test_jetson_cboot.py deleted file mode 100644 index 6cc42894f..000000000 --- a/tests/test_otaclient/test_boot_control/test_jetson_cboot.py +++ /dev/null @@ -1,144 +0,0 @@ -# Copyright 2022 TIER IV, INC. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -NOTE: this test file only test utils used in jetson-cboot. -""" - - -from __future__ import annotations - -import logging -from pathlib import Path -from typing import Any - -import pytest - -from otaclient.app.boot_control import _jetson_cboot -from otaclient.app.boot_control._jetson_common import ( - BSPVersion, - FirmwareBSPVersion, - SlotID, - parse_bsp_version, - update_extlinux_cfg, -) -from tests.conftest import TEST_DIR - -logger = logging.getLogger(__name__) - -MODULE_NAME = _jetson_cboot.__name__ -TEST_DATA_DIR = TEST_DIR / "data" - - -def test_SlotID(): - SlotID("0") - SlotID("1") - with pytest.raises(ValueError): - SlotID("abc") - - -class TestBSPVersion: - - @pytest.mark.parametrize( - ["_in", "expected"], - ( - ("R32.5.2", BSPVersion(32, 5, 2)), - ("R32.6.1", BSPVersion(32, 6, 1)), - ("R35.4.1", BSPVersion(35, 4, 1)), - ), - ) - def test_parse(self, _in: str, expected: BSPVersion): - assert BSPVersion.parse(_in) == expected - - @pytest.mark.parametrize( - ["_in", "expected"], - ( - (BSPVersion(32, 5, 2), "R32.5.2"), - (BSPVersion(32, 6, 1), "R32.6.1"), - (BSPVersion(35, 4, 1), "R35.4.1"), - ), - ) - def test_dump(self, _in: BSPVersion, expected: str): - assert _in.dump() == expected - - -@pytest.mark.parametrize( - ["_in", "expected"], - ( - ( - { - "slot_a": None, - "slot_b": None, - }, - FirmwareBSPVersion(), - ), - ( - { - "slot_a": None, - "slot_b": "R32.6.1", - }, - FirmwareBSPVersion(slot_a=None, slot_b=BSPVersion(32, 6, 1)), - ), - ( - { - "slot_a": "R32.5.2", - "slot_b": "R32.6.1", - }, - FirmwareBSPVersion( - slot_a=BSPVersion(32, 5, 2), slot_b=BSPVersion(32, 6, 1) - ), - ), - ), -) -def test_FirmwareBSPVersion(_in: dict[str, Any], expected: FirmwareBSPVersion): - assert FirmwareBSPVersion.model_validate(_in) == expected - - -@pytest.mark.parametrize( - ["_in", "expected"], - ( - ( - ( - "# R32 (release), REVISION: 6.1, GCID: 27863751, BOARD: t186ref, EABI: aarch64, DATE: Mon Jul 26 19:36:31 UTC 2021", - BSPVersion(32, 6, 1), - ), - ( - "# R35 (release), REVISION: 4.1, GCID: 33958178, BOARD: t186ref, EABI: aarch64, DATE: Tue Aug 1 19:57:35 UTC 2023", - BSPVersion(35, 4, 1), - ), - ) - ), -) -def test_parse_bsp_version(_in: str, expected: BSPVersion): - assert parse_bsp_version(_in) == expected - - -@pytest.mark.parametrize( - ["_template_f", "_updated_f", "partuuid"], - ( - ( - "extlinux.conf-r35.4.1-template1", - "extlinux.conf-r35.4.1-updated1", - "11aa-bbcc-22dd", - ), - ( - "extlinux.conf-r35.4.1-template2", - "extlinux.conf-r35.4.1-updated2", - "11aa-bbcc-22dd", - ), - ), -) -def test_update_extlinux_conf(_template_f: Path, _updated_f: Path, partuuid: str): - _in = (TEST_DATA_DIR / _template_f).read_text() - _expected = (TEST_DATA_DIR / _updated_f).read_text() - assert update_extlinux_cfg(_in, partuuid) == _expected From 0bf9316a11c231bdbdd674175a8134230a2cf6e0 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 26 Jun 2024 12:14:41 +0000 Subject: [PATCH 084/193] fix jetson-common BSPVersion --- src/otaclient/app/boot_control/_jetson_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/otaclient/app/boot_control/_jetson_common.py b/src/otaclient/app/boot_control/_jetson_common.py index d42069adf..57bc4fac9 100644 --- a/src/otaclient/app/boot_control/_jetson_common.py +++ b/src/otaclient/app/boot_control/_jetson_common.py @@ -61,7 +61,7 @@ def __new__(cls, _in: str | Self) -> Self: """SLOT_A: 1, SLOT_B: 2""" BSP_VERSION_STR_PA = re.compile( - r"\w?(?P\d+)\.(?P\d+)\.(?P\d+)" + r"[rR]?(?P\d+)\.(?P\d+)\.(?P\d+)" ) From 557afa46d569d5a533f6d98b5d691b7a8b547fc2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 26 Jun 2024 12:15:08 +0000 Subject: [PATCH 085/193] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_otaclient/test_boot_control/test_jetson_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_otaclient/test_boot_control/test_jetson_common.py b/tests/test_otaclient/test_boot_control/test_jetson_common.py index c4dcc4eca..9080eb733 100644 --- a/tests/test_otaclient/test_boot_control/test_jetson_common.py +++ b/tests/test_otaclient/test_boot_control/test_jetson_common.py @@ -22,9 +22,9 @@ import pytest from otaclient.app.boot_control._jetson_common import ( - SlotID, BSPVersion, FirmwareBSPVersion, + SlotID, ) From b3b0774c444c85ec17ec0355366cc00eac07690f Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 26 Jun 2024 12:23:41 +0000 Subject: [PATCH 086/193] WIP: implement jetson-common test --- .../test_boot_control/test_jetson_common.py | 213 ++++++++++++++---- 1 file changed, 169 insertions(+), 44 deletions(-) diff --git a/tests/test_otaclient/test_boot_control/test_jetson_common.py b/tests/test_otaclient/test_boot_control/test_jetson_common.py index 9080eb733..943b8ad7a 100644 --- a/tests/test_otaclient/test_boot_control/test_jetson_common.py +++ b/tests/test_otaclient/test_boot_control/test_jetson_common.py @@ -25,27 +25,105 @@ BSPVersion, FirmwareBSPVersion, SlotID, + SLOT_A, + SLOT_B, + parse_nv_tegra_release, + update_extlinux_cfg, ) +from tests.conftest import TEST_DIR -def test_SlotID(_in: Any, _expect: SlotID | None, _exc: Exception | None): - pass +TEST_DATA_DIR = TEST_DIR / "data" -class TestBSPVersion: +@pytest.mark.parametrize( + "_in, _expect, _exc", + ( + (SLOT_A, SLOT_A, None), + (SLOT_B, SLOT_B, None), + ("0", SLOT_A, None), + ("1", SLOT_B, None), + ("not_a_valid_slot_id", None, ValueError), + ), +) +def test_slot_id(_in: Any, _expect: SlotID | None, _exc: type[Exception] | None): + def _test(): + assert SlotID(_in) == _expect - def test_parse(self, _in: Any, _expect: BSPVersion | None, _exc: Exception | None): - pass + if _exc: + with pytest.raises(_exc): + return _test() + else: + _test() + +class TestBSPVersion: + + @pytest.mark.parametrize( + "_in, _expect, _exc", + ( + ("R32.6.1", BSPVersion(32, 6, 1), None), + ("r32.6.1", BSPVersion(32, 6, 1), None), + ("32.6.1", BSPVersion(32, 6, 1), None), + ("R35.4.1", BSPVersion(35, 4, 1), None), + ("1.22.333", BSPVersion(1, 22, 333), None), + ("not_a_valid_bsp_ver", None, AssertionError), + (123, None, ValueError), + ), + ) + def test_parse( + self, _in: Any, _expect: BSPVersion | None, _exc: type[Exception] | None + ): + def _test(): + assert BSPVersion.parse(_in) == _expect + + if _exc: + with pytest.raises(_exc): + _test() + else: + _test() + + @pytest.mark.parametrize( + "_in, _expect", + ( + (BSPVersion(35, 4, 1), "R35.4.1"), + (BSPVersion(32, 6, 1), "R32.6.1"), + (BSPVersion(1, 22, 333), "R1.22.333"), + ), + ) def test_dump(self, _in: BSPVersion, _expect: str): - pass + assert _in.dump() == _expect class TestFirmwareBSPVersion: - def test_init(self, slot_a_ver: BSPVersion | None, slot_b_ver: BSPVersion | None): - pass - + @pytest.mark.parametrize( + "_in, _slot, _bsp_ver, _expect", + ( + ( + FirmwareBSPVersion(), + SLOT_A, + BSPVersion(32, 6, 1), + FirmwareBSPVersion(slot_a=BSPVersion(32, 6, 1)), + ), + ( + FirmwareBSPVersion( + slot_a=BSPVersion(32, 5, 1), slot_b=BSPVersion(32, 6, 1) + ), + SLOT_B, + None, + FirmwareBSPVersion(slot_a=BSPVersion(32, 5, 1), slot_b=None), + ), + ( + FirmwareBSPVersion( + slot_a=BSPVersion(32, 5, 1), slot_b=BSPVersion(32, 6, 1) + ), + SLOT_A, + None, + FirmwareBSPVersion(slot_a=None, slot_b=BSPVersion(32, 6, 1)), + ), + ), + ) def test_set_by_slot( self, _in: FirmwareBSPVersion, @@ -53,18 +131,55 @@ def test_set_by_slot( _bsp_ver: BSPVersion | None, _expect: FirmwareBSPVersion, ): - pass - + _in.set_by_slot(_slot, _bsp_ver) + assert _in == _expect + + @pytest.mark.parametrize( + "_in, _slot, _expect", + ( + ( + FirmwareBSPVersion(), + SLOT_A, + None, + ), + ( + FirmwareBSPVersion( + slot_a=BSPVersion(32, 5, 1), slot_b=BSPVersion(32, 6, 1) + ), + SLOT_B, + BSPVersion(32, 6, 1), + ), + ( + FirmwareBSPVersion( + slot_a=BSPVersion(32, 5, 1), slot_b=BSPVersion(32, 6, 1) + ), + SLOT_A, + BSPVersion(32, 5, 1), + ), + ), + ) def test_get_by_slot( self, _in: FirmwareBSPVersion, _slot: SlotID, - _exp: BSPVersion | None, + _expect: BSPVersion | None, ): - pass - + assert _in.get_by_slot(_slot) == _expect + + @pytest.mark.parametrize( + "_in", + ( + (FirmwareBSPVersion()), + (FirmwareBSPVersion(slot_a=BSPVersion(32, 5, 1))), + ( + FirmwareBSPVersion( + slot_a=BSPVersion(35, 4, 1), slot_b=BSPVersion(35, 5, 0) + ) + ), + ), + ) def test_load_and_dump(self, _in: FirmwareBSPVersion): - pass + assert FirmwareBSPVersion.model_validate_json(_in.model_dump_json()) == _in class TestFirmwareBSPVersionControl: @@ -82,33 +197,43 @@ def test_get_set_standby_slot(self): pass +@pytest.mark.parametrize( + "_in, _expect", + ( + ( + "# R32 (release), REVISION: 6.1, GCID: 27863751, BOARD: t186ref, EABI: aarch64, DATE: Mon Jul 26 19:36:31 UTC 2021", + BSPVersion(32, 6, 1), + ), + ( + "# R35 (release), REVISION: 4.1, GCID: 33958178, BOARD: t186ref, EABI: aarch64, DATE: Tue Aug 1 19:57:35 UTC 2023", + BSPVersion(35, 4, 1), + ), + ( + "# R35 (release), REVISION: 5.0, GCID: 35550185, BOARD: t186ref, EABI: aarch64, DATE: Tue Feb 20 04:46:31 UTC 2024", + BSPVersion(35, 5, 0), + ), + ), +) def test_parse_nv_tegra_release(_in: str, _expect: BSPVersion): - pass - - -def test_detect_rootfs_bsp_version(tmp_path: Path): - pass - - -def test_update_extlinux_cfg(_in: str, _partuuid: str, _expect: str): - pass - - -class Test_copy_standby_slot_boot_to_internal_emmc: - - @pytest.fixture(autouse=True) - def setup_test(self): - pass - - def test_copy_standby_slot_boot_to_internal_emmc(self): - pass - - -class Test_preserve_ota_config_files_to_standby: - - @pytest.fixture(autouse=True) - def setup_test(self): - pass - - def test_preserve_ota_config_files_to_standby(self): - pass + assert parse_nv_tegra_release(_in) == _expect + + +@pytest.mark.parametrize( + ["_template_f", "_updated_f", "partuuid"], + ( + ( + "extlinux.conf-r35.4.1-template1", + "extlinux.conf-r35.4.1-updated1", + "11aa-bbcc-22dd", + ), + ( + "extlinux.conf-r35.4.1-template2", + "extlinux.conf-r35.4.1-updated2", + "11aa-bbcc-22dd", + ), + ), +) +def test_update_extlinux_conf(_template_f: Path, _updated_f: Path, partuuid: str): + _in = (TEST_DATA_DIR / _template_f).read_text() + _expected = (TEST_DATA_DIR / _updated_f).read_text() + assert update_extlinux_cfg(_in, partuuid) == _expected From 225f7650a473b64736439b08386e85fdee50542f Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 26 Jun 2024 13:25:48 +0000 Subject: [PATCH 087/193] finish up test common --- .../test_boot_control/test_jetson_common.py | 42 +++++++++++++++---- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/tests/test_otaclient/test_boot_control/test_jetson_common.py b/tests/test_otaclient/test_boot_control/test_jetson_common.py index 943b8ad7a..d3cdf62c6 100644 --- a/tests/test_otaclient/test_boot_control/test_jetson_common.py +++ b/tests/test_otaclient/test_boot_control/test_jetson_common.py @@ -25,6 +25,7 @@ BSPVersion, FirmwareBSPVersion, SlotID, + FirmwareBSPVersionControl, SLOT_A, SLOT_B, parse_nv_tegra_release, @@ -184,17 +185,44 @@ def test_load_and_dump(self, _in: FirmwareBSPVersion): class TestFirmwareBSPVersionControl: + @pytest.fixture(autouse=True) + def setup_test(self, tmp_path: Path): + self.test_fw_bsp_vf = tmp_path / "firmware_bsp_version" + self.slot_b_ver = BSPVersion(35, 5, 0) + self.slot_a_ver = BSPVersion(35, 4, 1) + def test_init(self): - pass + self.test_fw_bsp_vf.write_text( + FirmwareBSPVersion(slot_b=self.slot_b_ver).model_dump_json() + ) - def test_write_to_file(self): - pass + loaded = FirmwareBSPVersionControl( + SLOT_A, + self.slot_a_ver, + current_firmware_bsp_vf=self.test_fw_bsp_vf, + ) - def test_get_set_current_slot(self): - pass + # NOTE: FirmwareBSPVersionControl will not use the information for current slot. + assert loaded.current_slot_fw_ver == self.slot_a_ver + assert loaded.standby_slot_fw_ver == self.slot_b_ver + + def test_write_to_file(self): + self.test_fw_bsp_vf.write_text( + FirmwareBSPVersion(slot_b=self.slot_b_ver).model_dump_json() + ) + loaded = FirmwareBSPVersionControl( + SLOT_A, + self.slot_a_ver, + current_firmware_bsp_vf=self.test_fw_bsp_vf, + ) + loaded.write_to_file(self.test_fw_bsp_vf) - def test_get_set_standby_slot(self): - pass + assert ( + self.test_fw_bsp_vf.read_text() + == FirmwareBSPVersion( + slot_a=self.slot_a_ver, slot_b=self.slot_b_ver + ).model_dump_json() + ) @pytest.mark.parametrize( From 173bdbf8fe0e4edb7f77fdbc4542318c9d8778a7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 26 Jun 2024 13:27:11 +0000 Subject: [PATCH 088/193] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../test_otaclient/test_boot_control/test_jetson_common.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_otaclient/test_boot_control/test_jetson_common.py b/tests/test_otaclient/test_boot_control/test_jetson_common.py index d3cdf62c6..69578d99e 100644 --- a/tests/test_otaclient/test_boot_control/test_jetson_common.py +++ b/tests/test_otaclient/test_boot_control/test_jetson_common.py @@ -22,16 +22,15 @@ import pytest from otaclient.app.boot_control._jetson_common import ( + SLOT_A, + SLOT_B, BSPVersion, FirmwareBSPVersion, - SlotID, FirmwareBSPVersionControl, - SLOT_A, - SLOT_B, + SlotID, parse_nv_tegra_release, update_extlinux_cfg, ) - from tests.conftest import TEST_DIR TEST_DATA_DIR = TEST_DIR / "data" From 66277f8ed74e58b85abf80a7eb5cd02dcb0ef3cd Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 26 Jun 2024 14:24:11 +0000 Subject: [PATCH 089/193] jetson-uefi: also do sha256 hash check when determine l4tlauncher version with version file --- .../app/boot_control/_jetson_uefi.py | 84 ++++++++++++++----- 1 file changed, 61 insertions(+), 23 deletions(-) diff --git a/src/otaclient/app/boot_control/_jetson_uefi.py b/src/otaclient/app/boot_control/_jetson_uefi.py index 7263e5906..0edb2d827 100644 --- a/src/otaclient/app/boot_control/_jetson_uefi.py +++ b/src/otaclient/app/boot_control/_jetson_uefi.py @@ -26,7 +26,10 @@ import re import shutil from pathlib import Path -from typing import Any, Generator +from typing import Any, ClassVar, Generator, Literal + +from pydantic import BaseModel +from typing_extensions import Self from otaclient.app import errors as ota_errors from otaclient.app.configs import config as cfg @@ -183,6 +186,24 @@ def _detect_esp_dev(boot_parent_devpath: StrOrPath) -> str: return esp_part +class L4TLauncherBSPVersionControl(BaseModel): + """ + Schema: : + """ + + bsp_ver: BSPVersion + sha256_digest: str + SEP: ClassVar[Literal[":"]] = ":" + + @classmethod + def parse(cls, _in: str) -> Self: + bsp_str, digest = _in.split(":") + return cls(bsp_ver=BSPVersion.parse(bsp_str), sha256_digest=digest) + + def dump(self) -> str: + return f"{self.bsp_ver.dump()}{self.SEP}{self.sha256_digest}" + + class UEFIFirmwareUpdater: """Firmware update implementation using Capsule update.""" @@ -293,43 +314,58 @@ def _write_magic_efivar() -> None: def _detect_l4tlauncher_version(self) -> BSPVersion: """Try to determine the current in use l4tlauncher version.""" - l4tlauncher_bsp_ver = None + l4tlauncher_sha256_digest = file_sha256(self.bootaa64_at_esp) + + # try to determine the version with version file try: - l4tlauncher_bsp_ver = BSPVersion.parse( + _ver_control = L4TLauncherBSPVersionControl.parse( self.l4tlauncher_ver_fpath.read_text() ) + if l4tlauncher_sha256_digest == _ver_control.sha256_digest: + return _ver_control.bsp_ver + + logger.warning( + ( + "l4tlauncher sha256 hash mismatched: " + f"{l4tlauncher_sha256_digest=}, {_ver_control.sha256_digest=}, " + "remove version file" + ) + ) + raise ValueError("sha256 hash mismatched") except Exception as e: logger.warning(f"missing or invalid l4tlauncher version file: {e!r}") self.l4tlauncher_ver_fpath.unlink(missing_ok=True) + # try to determine the version by looking up table # NOTE(20240624): since the number of l4tlauncher version is limited, # we can lookup against a pre-calculated sha256 digest map. - if l4tlauncher_bsp_ver is None: - _l4tlauncher_sha256_digest = file_sha256(self.bootaa64_at_esp) - logger.info( - f"try to determine the l4tlauncher verison by hash: {_l4tlauncher_sha256_digest}" - ) - l4tlauncher_bsp_ver = L4TLAUNCHER_BSP_VER_SHA256_MAP.get( - _l4tlauncher_sha256_digest + logger.info( + f"try to determine the l4tlauncher verison by hash: {l4tlauncher_sha256_digest}" + ) + if l4tlauncher_bsp_ver := L4TLAUNCHER_BSP_VER_SHA256_MAP.get( + l4tlauncher_sha256_digest + ): + _ver_control = L4TLauncherBSPVersionControl( + bsp_ver=l4tlauncher_bsp_ver, sha256_digest=l4tlauncher_sha256_digest ) + write_str_to_file_sync(self.l4tlauncher_ver_fpath, _ver_control.dump()) + return l4tlauncher_bsp_ver - current_slot_fw_bsp_ver = self.fw_bsp_ver_control.current_slot_fw_ver # NOTE(20240624): if we failed to detect the l4tlauncher's version, # we assume that the launcher is the same version as current slot's fw. # This is typically the case of a newly flashed ECU. - if l4tlauncher_bsp_ver is None: - logger.warning( - ( - "failed to determine the l4tlauncher's version, assuming " - f"version is the same as current slot's fw version: {current_slot_fw_bsp_ver}" - ) - ) - l4tlauncher_bsp_ver = current_slot_fw_bsp_ver - write_str_to_file_sync( - self.l4tlauncher_ver_fpath, l4tlauncher_bsp_ver.dump() + current_slot_fw_bsp_ver = self.fw_bsp_ver_control.current_slot_fw_ver + logger.error( + ( + "failed to determine the l4tlauncher's version, assuming " + f"version is the same as current slot's fw version: {current_slot_fw_bsp_ver}" ) - - logger.info(f"finish detecting l4tlauncher version: {l4tlauncher_bsp_ver}") + ) + l4tlauncher_bsp_ver = current_slot_fw_bsp_ver + _ver_control = L4TLauncherBSPVersionControl( + bsp_ver=l4tlauncher_bsp_ver, sha256_digest=l4tlauncher_sha256_digest + ) + write_str_to_file_sync(self.l4tlauncher_ver_fpath, _ver_control.dump()) return l4tlauncher_bsp_ver # APIs @@ -384,6 +420,8 @@ def l4tlauncher_update(self) -> bool: True if l4tlauncher is updated, else if there is no l4tlauncher update. """ l4tlauncher_bsp_ver = self._detect_l4tlauncher_version() + logger.info(f"finished detect l4tlauncher version: {l4tlauncher_bsp_ver}") + ota_image_l4tlauncher_ver = self.ota_image_bsp_ver if l4tlauncher_bsp_ver >= ota_image_l4tlauncher_ver: logger.info( From aa7a9175a3d5ab1e9586288102adb9c75627f058 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 27 Jun 2024 05:54:19 +0000 Subject: [PATCH 090/193] jetson-uefi: fix esp is not mounted when checking l4tlauncher version --- .../app/boot_control/_jetson_uefi.py | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/src/otaclient/app/boot_control/_jetson_uefi.py b/src/otaclient/app/boot_control/_jetson_uefi.py index 0edb2d827..8c1e36de2 100644 --- a/src/otaclient/app/boot_control/_jetson_uefi.py +++ b/src/otaclient/app/boot_control/_jetson_uefi.py @@ -231,6 +231,8 @@ def __init__( # i.e., if we boot from nvme0n1, then bootdev_path is /dev/nvme0n1 and # we use the esp at nvme0n1. self.esp_mp = Path(boot_cfg.ESP_MOUNTPOINT) + self.esp_mp.mkdir(exist_ok=True) + self.esp_boot_dir = self.esp_mp / "EFI" / "BOOT" self.l4tlauncher_ver_fpath = self.esp_boot_dir / boot_cfg.L4TLAUNCHER_VER_FNAME """A plain text file stores the BSP version string.""" @@ -419,21 +421,20 @@ def l4tlauncher_update(self) -> bool: Returns: True if l4tlauncher is updated, else if there is no l4tlauncher update. """ - l4tlauncher_bsp_ver = self._detect_l4tlauncher_version() - logger.info(f"finished detect l4tlauncher version: {l4tlauncher_bsp_ver}") + with _ensure_esp_mounted(self.esp_part, self.esp_mp): + l4tlauncher_bsp_ver = self._detect_l4tlauncher_version() + logger.info(f"finished detect l4tlauncher version: {l4tlauncher_bsp_ver}") - ota_image_l4tlauncher_ver = self.ota_image_bsp_ver - if l4tlauncher_bsp_ver >= ota_image_l4tlauncher_ver: - logger.info( - ( - "installed l4tlauncher has newer or equal version of l4tlauncher to OTA image's one, " - f"{l4tlauncher_bsp_ver=}, {ota_image_l4tlauncher_ver=}, " - "skip l4tlauncher update" + ota_image_l4tlauncher_ver = self.ota_image_bsp_ver + if l4tlauncher_bsp_ver >= ota_image_l4tlauncher_ver: + logger.info( + ( + "installed l4tlauncher has newer or equal version of l4tlauncher to OTA image's one, " + f"{l4tlauncher_bsp_ver=}, {ota_image_l4tlauncher_ver=}, " + "skip l4tlauncher update" + ) ) - ) - return False - - with _ensure_esp_mounted(self.esp_part, self.esp_mp): + return False return self._update_l4tlauncher() From bc71dbe6669c068655961ac90a02111bd762f5c6 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 27 Jun 2024 07:39:52 +0000 Subject: [PATCH 091/193] minor fix --- src/otaclient/app/boot_control/_jetson_uefi.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/otaclient/app/boot_control/_jetson_uefi.py b/src/otaclient/app/boot_control/_jetson_uefi.py index 8c1e36de2..68ae726e9 100644 --- a/src/otaclient/app/boot_control/_jetson_uefi.py +++ b/src/otaclient/app/boot_control/_jetson_uefi.py @@ -454,7 +454,8 @@ def __init__(self): raise JetsonUEFIBootControlError(_err_msg) # print hardware model - logger.info(f"hardware model: {Path(boot_cfg.MODEL_FPATH)}") + if (model_fpath := Path(boot_cfg.MODEL_FPATH)).is_file(): + logger.info(f"hardware model: {model_fpath.read_text()}") compat_info = tegra_compat_info_fpath.read_text() # example compatible string: From 72e0609edf47fdb39d889f2f2485da265f53d876 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 27 Jun 2024 08:24:19 +0000 Subject: [PATCH 092/193] minor fix --- src/otaclient/app/boot_control/_jetson_uefi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/otaclient/app/boot_control/_jetson_uefi.py b/src/otaclient/app/boot_control/_jetson_uefi.py index 68ae726e9..aaf01b3f7 100644 --- a/src/otaclient/app/boot_control/_jetson_uefi.py +++ b/src/otaclient/app/boot_control/_jetson_uefi.py @@ -197,7 +197,7 @@ class L4TLauncherBSPVersionControl(BaseModel): @classmethod def parse(cls, _in: str) -> Self: - bsp_str, digest = _in.split(":") + bsp_str, digest = _in.strip().split(":") return cls(bsp_ver=BSPVersion.parse(bsp_str), sha256_digest=digest) def dump(self) -> str: From 86e7530b9abf3a5f4f417fd68d95621c82c70a56 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 27 Jun 2024 08:28:40 +0000 Subject: [PATCH 093/193] jetson-config: L4TLauncher ver file is not a json file --- src/otaclient/app/boot_control/configs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/otaclient/app/boot_control/configs.py b/src/otaclient/app/boot_control/configs.py index b10429d74..fa072be8d 100644 --- a/src/otaclient/app/boot_control/configs.py +++ b/src/otaclient/app/boot_control/configs.py @@ -74,7 +74,7 @@ class JetsonUEFIBootControlConfig(JetsonBootCommon): MAGIC_BYTES = b"\x07\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00" CAPSULE_PAYLOAD_AT_ESP = "EFI/UpdateCapsule" CAPSULE_PAYLOAD_AT_ROOTFS = "/opt/ota_package/" - L4TLAUNCHER_VER_FNAME = "l4tlauncher_version.json" + L4TLAUNCHER_VER_FNAME = "l4tlauncher_version" NO_FIRMWARE_UPDATE_HINT_FNAME = ".otaclient_no_firmware_update" """Skip firmware update if this file is presented.""" From 14493ff616604575d236cdf69049565a63032adf Mon Sep 17 00:00:00 2001 From: Bodong Yang <86948717+Bodong-Yang@users.noreply.github.com> Date: Thu, 11 Jul 2024 20:47:28 +0900 Subject: [PATCH 094/193] refactor: refine persist_file_handling internal implementation (#354) This PR refines the persist_file_handling module's internal implementation, reduces the code cognitive complexity, and refines the workflow of entry processing. Also now otaclient won't break out the OTA on failed entry during persist file handling. Internal behavior change: * OTA will not break on failed entry during persist file handling processing. --- src/otaclient/app/ota_client.py | 7 +- src/otaclient_common/persist_file_handling.py | 152 +++++++++++------- 2 files changed, 94 insertions(+), 65 deletions(-) diff --git a/src/otaclient/app/ota_client.py b/src/otaclient/app/ota_client.py index 8c3ef74ca..ba1cbbf7f 100644 --- a/src/otaclient/app/ota_client.py +++ b/src/otaclient/app/ota_client.py @@ -377,10 +377,11 @@ def _process_persistents(self, ota_metadata: ota_metadata_parser.OTAMetadata): ) continue - if ( - _per_fpath.is_file() or _per_fpath.is_dir() or _per_fpath.is_symlink() - ): # NOTE: not equivalent to perinf.path.exists() + try: _handler.preserve_persist_entry(_per_fpath) + except Exception as e: + _err_msg = f"failed to preserve {_per_fpath}: {e!r}, skip" + logger.warning(_err_msg) def _execute_update(self): """Implementation of OTA updating.""" diff --git a/src/otaclient_common/persist_file_handling.py b/src/otaclient_common/persist_file_handling.py index 6b0178017..babbb8814 100644 --- a/src/otaclient_common/persist_file_handling.py +++ b/src/otaclient_common/persist_file_handling.py @@ -27,6 +27,7 @@ map_gid_by_grpnam, map_uid_by_pwnam, ) +from otaclient_common.typing import StrOrPath logger = logging.getLogger(__name__) @@ -43,13 +44,13 @@ class PersistFilesHandler: def __init__( self, - src_passwd_file: str | Path, - src_group_file: str | Path, - dst_passwd_file: str | Path, - dst_group_file: str | Path, + src_passwd_file: StrOrPath, + src_group_file: StrOrPath, + dst_passwd_file: StrOrPath, + dst_group_file: StrOrPath, *, - src_root: str | Path, - dst_root: str | Path, + src_root: StrOrPath, + dst_root: StrOrPath, ): self._uid_mapper = lru_cache()( partial( @@ -87,7 +88,7 @@ def map_gid_by_grpnam(*, src_db: ParsedGroup, dst_db: ParsedGroup, gid: int) -> return _mapped_gid def _chown_with_mapping( - self, _src_stat: os.stat_result, _dst_path: str | Path + self, _src_stat: os.stat_result, _dst_path: StrOrPath ) -> None: _src_uid, _src_gid = _src_stat.st_uid, _src_stat.st_gid try: @@ -108,12 +109,13 @@ def _rm_target(_target: Path) -> None: """Remove target with proper methods.""" if _target.is_symlink() or _target.is_file(): return _target.unlink(missing_ok=True) - elif _target.is_dir(): + if _target.is_dir(): return shutil.rmtree(_target, ignore_errors=True) - elif _target.exists(): - raise ValueError( - f"{_target} is not normal file/symlink/dir, failed to remove" - ) + # NOTE that exists will follow symlink, so we need to check symlink first + if not _target.exists(): + return + + raise ValueError(f"{_target} is not normal file/symlink/dir, failed to remove") def _prepare_symlink(self, _src_path: Path, _dst_path: Path) -> None: _dst_path.symlink_to(os.readlink(_src_path)) @@ -134,66 +136,36 @@ def _prepare_file(self, _src_path: Path, _dst_path: Path) -> None: os.chmod(_dst_path, _src_stat.st_mode) self._chown_with_mapping(_src_stat, _dst_path) - def _prepare_parent(self, _origin_entry: Path) -> None: - for _parent in reversed(_origin_entry.parents): + def _prepare_parent(self, _path_relative_to_root: Path) -> None: + """ + NOTE that we intensionally keep the already there parents' permission + setting on destination. + """ + for _parent in reversed(_path_relative_to_root.parents): _src_parent, _dst_parent = ( self._src_root / _parent, self._dst_root / _parent, ) - if _dst_parent.is_dir(): # keep the origin parent on dst as it - continue if _dst_parent.is_symlink() or _dst_parent.is_file(): _dst_parent.unlink(missing_ok=True) self._prepare_dir(_src_parent, _dst_parent) continue - if _dst_parent.exists(): - raise ValueError( - f"{_dst_parent=} is not a normal file/symlink/dir, cannot cleanup" - ) - self._prepare_dir(_src_parent, _dst_parent) - - # API - def preserve_persist_entry( - self, _persist_entry: str | Path, *, src_missing_ok: bool = True - ): - logger.info(f"preserving {_persist_entry}") - # persist_entry in persists.txt must be rooted at / - origin_entry = Path(_persist_entry).relative_to("/") - src_path = self._src_root / origin_entry - dst_path = self._dst_root / origin_entry - - # ------ src is symlink ------ # - # NOTE: always check if symlink first as is_file/is_dir/exists all follow_symlinks - if src_path.is_symlink(): - self._rm_target(dst_path) - self._prepare_parent(origin_entry) - self._prepare_symlink(src_path, dst_path) - return + # keep the origin parent on dst as it + # NOTE that is_dir will follow symlink. + if _dst_parent.is_dir(): + continue - # ------ src is file ------ # - if src_path.is_file(): - self._rm_target(dst_path) - self._prepare_parent(origin_entry) - self._prepare_file(src_path, dst_path) - return + if not _dst_parent.exists(): + self._prepare_dir(_src_parent, _dst_parent) + continue - # ------ src is not regular file/symlink/dir ------ # - # we only process normal file/symlink/dir - if src_path.exists() and not src_path.is_dir(): - raise ValueError(f"{src_path=} must be either a file/symlink/dir") - - # ------ src doesn't exist ------ # - if not src_path.exists(): - _err_msg = f"{src_path=} not found" - logger.warning(_err_msg) - if not src_missing_ok: - raise ValueError(_err_msg) - return + raise ValueError( + f"{_dst_parent=} is not a normal file/symlink/dir, cannot cleanup" + ) - # ------ src is dir ------ # - # dive into src_dir and preserve everything under the src dir - self._prepare_parent(origin_entry) + def _recursively_prepare_dir(self, src_path: Path): + """Dive into src_dir and preserve everything under the src dir.""" for src_curdir, dnames, fnames in os.walk(src_path, followlinks=False): src_cur_dpath = Path(src_curdir) dst_cur_dpath = self._dst_root / src_cur_dpath.relative_to(self._src_root) @@ -206,10 +178,12 @@ def preserve_persist_entry( for _fname in fnames: _src_fpath, _dst_fpath = src_cur_dpath / _fname, dst_cur_dpath / _fname self._rm_target(_dst_fpath) + + # NOTE that fnames also contain symlink to normal file if _src_fpath.is_symlink(): self._prepare_symlink(_src_fpath, _dst_fpath) - continue - self._prepare_file(_src_fpath, _dst_fpath) + else: + self._prepare_file(_src_fpath, _dst_fpath) # symlinks to dirs also included in dnames, we must handle it for _dname in dnames: @@ -217,3 +191,57 @@ def preserve_persist_entry( if _src_dpath.is_symlink(): self._rm_target(_dst_dpath) self._prepare_symlink(_src_dpath, _dst_dpath) + # NOTE that we don't need to create dir here, as os.walk will take us + # to this folder later, we will create the folder in the dest when + # we enter the src folder. + + # API + + def preserve_persist_entry(self, _persist_entry: StrOrPath): + """Preserve <_persist_entry> from active slot to standby slot. + + Args: + _persist_entry (StrOrPath): The canonical path of the entry to be preserved. + + Raises: + ValueError: Raised when src <_persist_entry> is not a regular file, symlink or directory, + or failed to prepare destination. + """ + # persist_entry in persists.txt must be rooted at / + path_relative_to_root = Path(_persist_entry).relative_to("/") + src_path = self._src_root / path_relative_to_root + dst_path = self._dst_root / path_relative_to_root + + # ------ src is symlink ------ # + # NOTE: always check if symlink first as is_file/is_dir/exists all follow_symlinks + if src_path.is_symlink(): + logger.info( + f"preserving symlink: {_persist_entry}, points to {os.readlink(src_path)}" + ) + self._rm_target(dst_path) + self._prepare_parent(path_relative_to_root) + self._prepare_symlink(src_path, dst_path) + return + + # ------ src is file ------ # + if src_path.is_file(): + logger.info(f"preserving normal file: {_persist_entry}") + self._rm_target(dst_path) + self._prepare_parent(path_relative_to_root) + self._prepare_file(src_path, dst_path) + return + + # ------ src is dir ------ # + if src_path.is_dir(): + logger.info(f"recursively preserve directory: {_persist_entry}") + self._prepare_parent(path_relative_to_root) + self._recursively_prepare_dir(src_path) + return + + # ------ src is not regular file/symlink/dir or missing ------ # + _err_msg = f"{src_path=} doesn't exist" + if src_path.exists(): + _err_msg = f"src must be either a file/symlink/dir, skip {_persist_entry=}" + + logger.warning(_err_msg) + raise ValueError(_err_msg) From 3a0a7af0fd2e537f4955862909e6b51f667c32ad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 28 Jul 2024 17:56:09 +0900 Subject: [PATCH 095/193] build(deps): Update cryptography requirement (#357) Updates the requirements on [cryptography](https://github.com/pyca/cryptography) to permit the latest version. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/42.0.4...43.0.0) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 16b379412..176ba48e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ dynamic = [ dependencies = [ "aiofiles==24.1", "aiohttp<3.10,>=3.9.5", - "cryptography<43,>=42.0.4", + "cryptography>=42.0.4,<44", "grpcio<1.54,>=1.53.2", "protobuf<4.22,>=4.21.12", "pydantic==2.8", From 9d629014157cbb8fb06bb63c41fb27e3f97ed7f1 Mon Sep 17 00:00:00 2001 From: Bodong Yang <86948717+Bodong-Yang@users.noreply.github.com> Date: Thu, 1 Aug 2024 11:31:41 +0900 Subject: [PATCH 096/193] deps: loosen the deps version specifying (#366) This PR loosens the package version specifying, all the packages now pin with a range with the following strategy: * for most packages, set the upper bound lower than the next API version, and set the lower bound to a recent usable version(or a recent safe version if security fix is available), for example, cryptography>=42.0.4,<44, in which the recent security fixed version if 42.0.4. * for some packages, set the upper bound to the next revision level, for example, requests<2.33,>=2.32. * for packages without a fixed API level(with versioning schame like 0.x.y), set the upper bound to the next revision level. * don't set restriction on patch level, i.e., we allow any patch versions between two revision level. --- pyproject.toml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 176ba48e3..e1b905f95 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,20 +24,20 @@ dynamic = [ "version", ] dependencies = [ - "aiofiles==24.1", + "aiofiles<25,>=24.1", "aiohttp<3.10,>=3.9.5", "cryptography>=42.0.4,<44", "grpcio<1.54,>=1.53.2", "protobuf<4.22,>=4.21.12", - "pydantic==2.8", - "pydantic-settings==2.3.4", - "pyopenssl==24.1", - "pyyaml==6.0.1", + "pydantic<3,>=2.6", + "pydantic-settings<3,>=2.3", + "pyopenssl<25,>=24.1", + "pyyaml<7,>=6.0.1", "requests<2.33,>=2.32", "typing-extensions>=4.6.3", - "urllib3>=2.2.2,<2.3", - "uvicorn[standard]==0.30.1", - "zstandard==0.22", + "urllib3<2.3,>=2.2.2", + "uvicorn[standard]<0.31,>=0.30", + "zstandard<0.23,>=0.22", ] optional-dependencies.dev = [ "black", From 3a96776fad43bf7eefa2781d233886061900ae60 Mon Sep 17 00:00:00 2001 From: Bodong Yang <86948717+Bodong-Yang@users.noreply.github.com> Date: Fri, 2 Aug 2024 14:02:10 +0900 Subject: [PATCH 097/193] refactor: ota_proxy: use simple-sqlite3-orm instead of vendoring an orm inside otaproxy package (#363) This PR introduces to use simple-sqlite3-orm package(which is also implemented by me) instead. This package is feature-rich, well-implemented and well-tested replacement for the vendored sqlite3 ORM inside the otaproxy package. Also this PR fixes a potential race condition, which the database entry being committed before the cache file being finalized, resulting small chance of cache file not found on database lookup hit. Other major changes * split cache_streaming related logic intocache_streaming module. * split lru_cache_helper from ota_cache module into lru_cache_helper module. * in lru_cache_helper, now we limit the max steps to walk down the bucket list when rotating the cache. * in db, now we use RETURNING statement when rotating cache on supported platform. * in ota_cache, now we also check db integrity to determine whether to force init. --- pyproject.toml | 6 +- src/ota_proxy/__init__.py | 2 +- src/ota_proxy/_consts.py | 2 +- ...che_control.py => cache_control_header.py} | 0 src/ota_proxy/cache_streaming.py | 457 +++++++++++++ src/ota_proxy/config.py | 2 + src/ota_proxy/db.py | 455 ++++--------- src/ota_proxy/lru_cache_helper.py | 124 ++++ src/ota_proxy/orm.py | 243 ------- src/ota_proxy/ota_cache.py | 610 ++---------------- tests/test_ota_proxy/test_cachedb.py | 263 -------- tests/test_ota_proxy/test_ota_cache.py | 183 ++++-- tests/test_ota_proxy/test_ota_proxy_server.py | 30 +- 13 files changed, 942 insertions(+), 1435 deletions(-) rename src/ota_proxy/{cache_control.py => cache_control_header.py} (100%) create mode 100644 src/ota_proxy/cache_streaming.py create mode 100644 src/ota_proxy/lru_cache_helper.py delete mode 100644 src/ota_proxy/orm.py delete mode 100644 tests/test_ota_proxy/test_cachedb.py diff --git a/pyproject.toml b/pyproject.toml index e1b905f95..4dd1bc528 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ dependencies = [ "pyopenssl<25,>=24.1", "pyyaml<7,>=6.0.1", "requests<2.33,>=2.32", + "simple-sqlite3-orm @ https://github.com/pga2rn/simple-sqlite3-orm/releases/download/v0.2.0/simple_sqlite3_orm-0.2.0-py3-none-any.whl", "typing-extensions>=4.6.3", "urllib3<2.3,>=2.2.2", "uvicorn[standard]<0.31,>=0.30", @@ -45,7 +46,7 @@ optional-dependencies.dev = [ "flake8", "isort", "pytest==7.1.2", - "pytest-asyncio==0.21", + "pytest-asyncio==0.23.8", "pytest-mock==3.14", "requests-mock", ] @@ -54,6 +55,9 @@ urls.Source = "https://github.com/tier4/ota-client" [tool.hatch.version] source = "vcs" +[tool.hatch.metadata] +allow-direct-references = true + [tool.hatch.build.hooks.vcs] version-file = "src/_otaclient_version.py" diff --git a/src/ota_proxy/__init__.py b/src/ota_proxy/__init__.py index f0eb18cbd..7e87bbf70 100644 --- a/src/ota_proxy/__init__.py +++ b/src/ota_proxy/__init__.py @@ -24,7 +24,7 @@ from typing_extensions import ParamSpec, Self -from .cache_control import OTAFileCacheControl +from .cache_control_header import OTAFileCacheControl from .config import config from .ota_cache import OTACache from .server_app import App diff --git a/src/ota_proxy/_consts.py b/src/ota_proxy/_consts.py index 4fad83696..d07769f03 100644 --- a/src/ota_proxy/_consts.py +++ b/src/ota_proxy/_consts.py @@ -1,6 +1,6 @@ from multidict import istr -from .cache_control import OTAFileCacheControl +from .cache_control_header import OTAFileCacheControl # uvicorn REQ_TYPE_LIFESPAN = "lifespan" diff --git a/src/ota_proxy/cache_control.py b/src/ota_proxy/cache_control_header.py similarity index 100% rename from src/ota_proxy/cache_control.py rename to src/ota_proxy/cache_control_header.py diff --git a/src/ota_proxy/cache_streaming.py b/src/ota_proxy/cache_streaming.py new file mode 100644 index 000000000..7a519326e --- /dev/null +++ b/src/ota_proxy/cache_streaming.py @@ -0,0 +1,457 @@ +# Copyright 2022 TIER IV, INC. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Implementation of cache streaming.""" + + +from __future__ import annotations + +import asyncio +import contextlib +import logging +import os +import threading +import weakref +from concurrent.futures import Executor +from pathlib import Path +from typing import ( + AsyncGenerator, + AsyncIterator, + Callable, + Coroutine, + Generic, + List, + MutableMapping, + Optional, + Tuple, + TypeVar, + Union, +) + +import aiofiles + +from .config import config as cfg +from .db import CacheMeta +from .errors import ( + CacheMultiStreamingFailed, + CacheStreamingFailed, + CacheStreamingInterrupt, + StorageReachHardLimit, +) +from .utils import wait_with_backoff + +logger = logging.getLogger(__name__) + +_WEAKREF = TypeVar("_WEAKREF") + +# cache tracker + + +class CacheTracker(Generic[_WEAKREF]): + """A tracker for an ongoing cache entry. + + This tracker represents an ongoing cache entry under the , + and takes care of the life cycle of this temp cache entry. + It implements the provider/subscriber model for cache writing and cache streaming to + multiple clients. + + This entry will disappear automatically, ensured by gc and weakref + when no strong reference to this tracker(provider finished and no subscribers + attached to this tracker). + When this tracker is garbage collected, the corresponding temp cache entry will + also be removed automatically(ensure by registered finalizer). + + Attributes: + fpath: the path to the temporary cache file. + save_path: the path to finished cache file. + meta: an inst of CacheMeta for the remote OTA file that being cached. + writer_ready: a property indicates whether the provider is + ready to write and streaming data chunks. + writer_finished: a property indicates whether the provider finished the caching. + writer_failed: a property indicates whether provider fails to + finish the caching. + """ + + READER_SUBSCRIBE_WAIT_PROVIDER_TIMEOUT = 2 + FNAME_PART_SEPARATOR = "_" + + @classmethod + def _tmp_file_naming(cls, cache_identifier: str) -> str: + """Create fname for tmp caching entry. + + naming scheme: tmp__ + + NOTE: append 4bytes hex to identify cache entry for the same OTA file between + different trackers. + """ + return ( + f"{cfg.TMP_FILE_PREFIX}{cls.FNAME_PART_SEPARATOR}" + f"{cache_identifier}{cls.FNAME_PART_SEPARATOR}{(os.urandom(4)).hex()}" + ) + + def __init__( + self, + cache_identifier: str, + ref_holder: _WEAKREF, + *, + base_dir: Union[str, Path], + executor: Executor, + callback: _CACHE_ENTRY_REGISTER_CALLBACK, + below_hard_limit_event: threading.Event, + ): + self.fpath = Path(base_dir) / self._tmp_file_naming(cache_identifier) + self.meta: CacheMeta | None = None + self.cache_identifier = cache_identifier + self.save_path = Path(base_dir) / cache_identifier + self._writer_ready = asyncio.Event() + self._writer_finished = asyncio.Event() + self._writer_failed = asyncio.Event() + self._ref = ref_holder + self._subscriber_ref_holder: List[_WEAKREF] = [] + self._executor = executor + self._cache_commit_cb = callback + self._space_availability_event = below_hard_limit_event + + self._bytes_written = 0 + self._cache_write_gen: Optional[AsyncGenerator[int, bytes]] = None + + # self-register the finalizer to this tracker + weakref.finalize( + self, + self.finalizer, + fpath=self.fpath, + ) + + async def _finalize_caching(self): + """Commit cache entry to db and rename tmp cached file with sha256hash + to fialize the caching.""" + # if the file with the same sha256has is already presented, skip the hardlink + # NOTE: no need to clean the tmp file, it will be done by the cache tracker. + assert self.meta is not None + await self._cache_commit_cb(self.meta) + if not self.save_path.is_file(): + self.fpath.link_to(self.save_path) + + @staticmethod + def finalizer(*, fpath: Union[str, Path]): + """Finalizer that cleans up the tmp file when this tracker is gced.""" + Path(fpath).unlink(missing_ok=True) + + @property + def writer_failed(self) -> bool: + return self._writer_failed.is_set() + + @property + def writer_finished(self) -> bool: + return self._writer_finished.is_set() + + @property + def is_cache_valid(self) -> bool: + """Indicates whether the temp cache entry for this tracker is valid.""" + return ( + self.meta is not None + and self._writer_finished.is_set() + and not self._writer_failed.is_set() + ) + + def get_cache_write_gen(self) -> Optional[AsyncGenerator[int, bytes]]: + if self._cache_write_gen: + return self._cache_write_gen + logger.warning(f"tracker for {self.meta} is not ready, skipped") + + async def _provider_write_cache(self) -> AsyncGenerator[int, bytes]: + """Provider writes data chunks from upper caller to tmp cache file. + + If cache writing failed, this method will exit and tracker.writer_failed and + tracker.writer_finished will be set. + """ + logger.debug(f"start to cache for {self.meta=}...") + try: + if not self.meta: + raise ValueError("called before provider tracker is ready, abort") + + async with aiofiles.open(self.fpath, "wb", executor=self._executor) as f: + # let writer become ready when file is open successfully + self._writer_ready.set() + + _written = 0 + while _data := (yield _written): + if not self._space_availability_event.is_set(): + logger.warning( + f"abort writing cache for {self.meta=}: {StorageReachHardLimit.__name__}" + ) + self._writer_failed.set() + return + + _written = await f.write(_data) + self._bytes_written += _written + + logger.debug( + "cache write finished, total bytes written" + f"({self._bytes_written}) for {self.meta=}" + ) + self.meta.cache_size = self._bytes_written + + # NOTE: no need to track the cache commit, + # as writer_finish is meant to set on cache file created. + asyncio.create_task(self._finalize_caching()) + except Exception as e: + logger.exception(f"failed to write cache for {self.meta=}: {e!r}") + self._writer_failed.set() + finally: + self._writer_finished.set() + self._ref = None + + async def _subscribe_cache_streaming(self) -> AsyncIterator[bytes]: + """Subscriber keeps polling chunks from ongoing tmp cache file. + + Subscriber will keep polling until the provider fails or + provider finished and subscriber has read bytes. + + Raises: + CacheMultipleStreamingFailed if provider failed or timeout reading + data chunk from tmp cache file(might be caused by a dead provider). + """ + try: + err_count, _bytes_read = 0, 0 + async with aiofiles.open(self.fpath, "rb", executor=self._executor) as f: + while not (self.writer_finished and _bytes_read == self._bytes_written): + if self.writer_failed: + raise CacheMultiStreamingFailed( + f"provider aborted for {self.meta}" + ) + _bytes_read += len(_chunk := await f.read(cfg.CHUNK_SIZE)) + if _chunk: + err_count = 0 + yield _chunk + continue + + err_count += 1 + if not await wait_with_backoff( + err_count, + _backoff_factor=cfg.STREAMING_BACKOFF_FACTOR, + _backoff_max=cfg.STREAMING_BACKOFF_MAX, + ): + # abort caching due to potential dead streaming coro + _err_msg = f"failed to stream({self.meta=}): timeout getting data, partial read might happen" + logger.error(_err_msg) + # signal streamer to stop streaming + raise CacheMultiStreamingFailed(_err_msg) + finally: + # unsubscribe on finish + self._subscriber_ref_holder.pop() + + async def _read_cache(self) -> AsyncIterator[bytes]: + """Directly open the tmp cache entry and yield data chunks from it. + + Raises: + CacheMultipleStreamingFailed if fails to read from the + cached file, this might indicate a partial written cache file. + """ + _bytes_read, _retry_count = 0, 0 + async with aiofiles.open(self.fpath, "rb", executor=self._executor) as f: + while _bytes_read < self._bytes_written: + if _data := await f.read(cfg.CHUNK_SIZE): + _retry_count = 0 + _bytes_read += len(_data) + yield _data + continue + + # no data is read from the cache entry, + # retry sometimes to ensure all data is acquired + _retry_count += 1 + if not await wait_with_backoff( + _retry_count, + _backoff_factor=cfg.STREAMING_BACKOFF_FACTOR, + _backoff_max=cfg.STREAMING_CACHED_TMP_TIMEOUT, + ): + # abort caching due to potential dead streaming coro + _err_msg = ( + f"open_cached_tmp failed for ({self.meta=}): " + "timeout getting more data, partial written cache file detected" + ) + logger.debug(_err_msg) + # signal streamer to stop streaming + raise CacheMultiStreamingFailed(_err_msg) + + # exposed API + + async def provider_start(self, meta: CacheMeta): + """Register meta to the Tracker, create tmp cache entry and get ready. + + Check _provider_write_cache for more details. + + Args: + meta: inst of CacheMeta for the requested file tracked by this tracker. + This meta is created by open_remote() method. + """ + self.meta = meta + self._cache_write_gen = self._provider_write_cache() + await self._cache_write_gen.asend(None) # type: ignore + + async def provider_on_finished(self): + if not self.writer_finished and self._cache_write_gen: + with contextlib.suppress(StopAsyncIteration): + await self._cache_write_gen.asend(b"") + self._writer_finished.set() + self._ref = None + + async def provider_on_failed(self): + """Manually fail and stop the caching.""" + if not self.writer_finished and self._cache_write_gen: + logger.warning(f"interrupt writer coroutine for {self.meta=}") + with contextlib.suppress(StopAsyncIteration, CacheStreamingInterrupt): + await self._cache_write_gen.athrow(CacheStreamingInterrupt) + + self._writer_failed.set() + self._writer_finished.set() + self._ref = None + + async def subscriber_subscribe_tracker(self) -> Optional[AsyncIterator[bytes]]: + """Reader subscribe this tracker and get a file descriptor to get data chunks.""" + _wait_count = 0 + while not self._writer_ready.is_set(): + _wait_count += 1 + if self.writer_failed or not await wait_with_backoff( + _wait_count, + _backoff_factor=cfg.STREAMING_BACKOFF_FACTOR, + _backoff_max=self.READER_SUBSCRIBE_WAIT_PROVIDER_TIMEOUT, + ): + return # timeout waiting for provider to become ready + + # subscribe on an ongoing cache + if not self.writer_finished and isinstance(self._ref, _Weakref): + self._subscriber_ref_holder.append(self._ref) + return self._subscribe_cache_streaming() + # caching just finished, try to directly read the finished cache entry + elif self.is_cache_valid: + return self._read_cache() + + +# a callback that register the cache entry indicates by input CacheMeta inst to the cache_db +_CACHE_ENTRY_REGISTER_CALLBACK = Callable[[CacheMeta], Coroutine[None, None, None]] + + +class _Weakref: + pass + + +class CachingRegister: + """A tracker register that manages cache trackers. + + For each ongoing caching for unique OTA file, there will be only one unique identifier for it. + + This first caller that requests with the identifier will become the provider and create + a new tracker for this identifier. + The later comes callers will become the subscriber to this tracker. + """ + + def __init__(self, base_dir: Union[str, Path]): + self._base_dir = Path(base_dir) + self._id_ref_dict: MutableMapping[str, _Weakref] = weakref.WeakValueDictionary() + self._ref_tracker_dict: MutableMapping[_Weakref, CacheTracker] = ( + weakref.WeakKeyDictionary() + ) + + async def get_tracker( + self, + cache_identifier: str, + *, + executor: Executor, + callback: _CACHE_ENTRY_REGISTER_CALLBACK, + below_hard_limit_event: threading.Event, + ) -> Tuple[CacheTracker, bool]: + """Get an inst of CacheTracker for the cache_identifier. + + Returns: + An inst of tracker, and a bool indicates the caller is provider(True), + or subscriber(False). + """ + _new_ref = _Weakref() + _ref = self._id_ref_dict.setdefault(cache_identifier, _new_ref) + + # subscriber + if ( + _tracker := self._ref_tracker_dict.get(_ref) + ) and not _tracker.writer_failed: + return _tracker, False + + # provider, or override a failed provider + if _ref is not _new_ref: # override a failed tracker + self._id_ref_dict[cache_identifier] = _new_ref + _ref = _new_ref + + _tracker = CacheTracker( + cache_identifier, + _ref, + base_dir=self._base_dir, + executor=executor, + callback=callback, + below_hard_limit_event=below_hard_limit_event, + ) + self._ref_tracker_dict[_ref] = _tracker + return _tracker, True + + +async def cache_streaming( + fd: AsyncIterator[bytes], + meta: CacheMeta, + tracker: CacheTracker, +) -> AsyncIterator[bytes]: + """A cache streamer that get data chunk from and tees to multiple destination. + + Data chunk yielded from will be teed to: + 1. upper uvicorn otaproxy APP to send back to client, + 2. cache_tracker cache_write_gen for caching. + + Args: + fd: opened connection to a remote file. + meta: meta data of the requested resource. + tracker: an inst of ongoing cache tracker bound to this request. + + Returns: + A bytes async iterator to yield data chunk from, for upper otaproxy uvicorn APP. + + Raises: + CacheStreamingFailed if any exception happens during retrieving. + """ + + async def _inner(): + _cache_write_gen = tracker.get_cache_write_gen() + try: + # tee the incoming chunk to two destinations + async for chunk in fd: + # NOTE: for aiohttp, when HTTP chunk encoding is enabled, + # an empty chunk will be sent to indicate the EOF of stream, + # we MUST handle this empty chunk. + if not chunk: # skip if empty chunk is read from remote + continue + # to caching generator + if _cache_write_gen and not tracker.writer_finished: + try: + await _cache_write_gen.asend(chunk) + except Exception as e: + await tracker.provider_on_failed() # signal tracker + logger.error( + f"cache write coroutine failed for {meta=}, abort caching: {e!r}" + ) + # to uvicorn thread + yield chunk + await tracker.provider_on_finished() + except Exception as e: + logger.exception(f"cache tee failed for {meta=}") + await tracker.provider_on_failed() + raise CacheStreamingFailed from e + + await tracker.provider_start(meta) + return _inner() diff --git a/src/ota_proxy/config.py b/src/ota_proxy/config.py index 6696a77b3..61ab3a2a4 100644 --- a/src/ota_proxy/config.py +++ b/src/ota_proxy/config.py @@ -38,6 +38,8 @@ class Config: 32 * (1024**2): 2, # [32MiB, ~), will not be rotated } DB_FILE = f"{BASE_DIR}/cache_db" + DB_THREADS = 3 + DB_THREAD_WAIT_TIMEOUT = 30 # seconds # DB configuration/setup # ota-cache table diff --git a/src/ota_proxy/db.py b/src/ota_proxy/db.py index 3858bee82..239c9bd51 100644 --- a/src/ota_proxy/db.py +++ b/src/ota_proxy/db.py @@ -15,24 +15,32 @@ from __future__ import annotations -import asyncio import logging import sqlite3 -import threading -import time -from concurrent.futures import ThreadPoolExecutor from pathlib import Path -from typing import Any, Dict, List, Optional, Union +from typing import Optional + +from pydantic import SkipValidation +from simple_sqlite3_orm import ( + ConstrainRepr, + ORMBase, + TableSpec, + TypeAffinityRepr, + utils, +) +from simple_sqlite3_orm._orm import AsyncORMThreadPoolBase +from typing_extensions import Annotated + +from otaclient_common.typing import StrOrPath from ._consts import HEADER_CONTENT_ENCODING, HEADER_OTA_FILE_CACHE_CONTROL -from .cache_control import OTAFileCacheControl +from .cache_control_header import OTAFileCacheControl from .config import config as cfg -from .orm import FV, ColumnDescriptor, ORMBase logger = logging.getLogger(__name__) -class CacheMeta(ORMBase): +class CacheMeta(TableSpec): """revision 4 url: unquoted URL from the request of this cache entry. @@ -47,25 +55,51 @@ class CacheMeta(ORMBase): content_encoding: the content_encoding header string comes with resp from remote server. """ - file_sha256: ColumnDescriptor[str] = ColumnDescriptor( - str, "TEXT", "UNIQUE", "NOT NULL", "PRIMARY KEY" - ) - url: ColumnDescriptor[str] = ColumnDescriptor(str, "TEXT", "NOT NULL") - bucket_idx: ColumnDescriptor[int] = ColumnDescriptor( - int, "INTEGER", "NOT NULL", type_guard=True - ) - last_access: ColumnDescriptor[int] = ColumnDescriptor( - int, "INTEGER", "NOT NULL", type_guard=(int, float) - ) - cache_size: ColumnDescriptor[int] = ColumnDescriptor( - int, "INTEGER", "NOT NULL", type_guard=True - ) - file_compression_alg: ColumnDescriptor[str] = ColumnDescriptor( - str, "TEXT", "NOT NULL" - ) - content_encoding: ColumnDescriptor[str] = ColumnDescriptor(str, "TEXT", "NOT NULL") - - def export_headers_to_client(self) -> Dict[str, str]: + file_sha256: Annotated[ + str, + TypeAffinityRepr(str), + ConstrainRepr("PRIMARY KEY"), + SkipValidation, + ] + url: Annotated[ + str, + TypeAffinityRepr(str), + ConstrainRepr("NOT NULL"), + SkipValidation, + ] + bucket_idx: Annotated[ + int, + TypeAffinityRepr(int), + ConstrainRepr("NOT NULL"), + SkipValidation, + ] = 0 + last_access: Annotated[ + int, + TypeAffinityRepr(int), + ConstrainRepr("NOT NULL"), + SkipValidation, + ] = 0 + cache_size: Annotated[ + int, + TypeAffinityRepr(int), + ConstrainRepr("NOT NULL"), + SkipValidation, + ] = 0 + file_compression_alg: Annotated[ + Optional[str], + TypeAffinityRepr(str), + SkipValidation, + ] = None + content_encoding: Annotated[ + Optional[str], + TypeAffinityRepr(str), + SkipValidation, + ] = None + + def __hash__(self) -> int: + return hash(tuple(getattr(self, attrn) for attrn in self.model_fields)) + + def export_headers_to_client(self) -> dict[str, str]: """Export required headers for client. Currently includes content-type, content-encoding and ota-file-cache-control headers. @@ -81,304 +115,99 @@ def export_headers_to_client(self) -> Dict[str, str]: res[HEADER_OTA_FILE_CACHE_CONTROL] = ( OTAFileCacheControl.export_kwargs_as_header( file_sha256=self.file_sha256, - file_compression_alg=self.file_compression_alg, + file_compression_alg=self.file_compression_alg or "", ) ) return res -class OTACacheDB: - TABLE_NAME: str = cfg.TABLE_NAME - OTA_CACHE_IDX: List[str] = [ - ( - "CREATE INDEX IF NOT EXISTS " - f"bucket_last_access_idx_{TABLE_NAME} " - f"ON {TABLE_NAME}({CacheMeta.bucket_idx.name}, {CacheMeta.last_access.name})" - ), - ] - - @classmethod - def check_db_file(cls, db_file: Union[str, Path]) -> bool: - if not Path(db_file).is_file(): - return False - try: - with sqlite3.connect(db_file) as con: - con.execute("PRAGMA integrity_check;") - # check whether expected table is in it or not - cur = con.execute( - "SELECT name FROM sqlite_master WHERE type='table' AND name=?", - (cls.TABLE_NAME,), - ) - if cur.fetchone() is None: - logger.warning( - f"{cls.TABLE_NAME} not found, this db file should be initialized" - ) - return False - return True - except sqlite3.DatabaseError as e: - logger.warning(f"{db_file} is corrupted: {e!r}") - return False - - @classmethod - def init_db_file(cls, db_file: Union[str, Path]): - """ - Purge the old db file and init the db files, creating table in it. - """ - # remove the db file first - Path(db_file).unlink(missing_ok=True) - try: - with sqlite3.connect(db_file) as con: - # create ota_cache table - con.execute(CacheMeta.get_create_table_stmt(cls.TABLE_NAME), ()) - # create indices - for idx in cls.OTA_CACHE_IDX: - con.execute(idx, ()) - - ### db performance tunning - # enable WAL mode - con.execute("PRAGMA journal_mode = WAL;") - # set temp_store to memory - con.execute("PRAGMA temp_store = memory;") - # enable mmap (size in bytes) - mmap_size = 16 * 1024 * 1024 # 16MiB - con.execute(f"PRAGMA mmap_size = {mmap_size};") - except sqlite3.Error as e: - logger.debug(f"init db failed: {e!r}") - raise e - - def __init__(self, db_file: Union[str, Path]): - """Connects to OTA cache database. - - Args: - db_file: target db file to connect to. +class CacheMetaORM(ORMBase[CacheMeta]): ... - Raises: - ValueError on invalid ota_cache db file, - sqlite3.Error if optimization settings applied failed. - """ - self._con = sqlite3.connect( - db_file, - check_same_thread=True, # one thread per connection in the threadpool - # isolation_level=None, # enable autocommit mode - ) - self._con.row_factory = sqlite3.Row - if not self.check_db_file(db_file): - raise ValueError(f"invalid db file: {db_file}") - - # db performance tunning, enable optimization - with self._con as con: - # enable WAL mode - con.execute("PRAGMA journal_mode = WAL;") - # set temp_store to memory - con.execute("PRAGMA temp_store = memory;") - # enable mmap (size in bytes) - mmap_size = 16 * 1024 * 1024 # 16MiB - con.execute(f"PRAGMA mmap_size = {mmap_size};") - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.close() - return False - def close(self): - self._con.close() +class AsyncCacheMetaORM(AsyncORMThreadPoolBase[CacheMeta]): - def remove_entries(self, fd: ColumnDescriptor[FV], *_inputs: FV) -> int: - """Remove entri(es) indicated by field(s). + async def rotate_cache( + self, bucket_idx: int, num: int + ) -> Optional[list[CacheMeta]]: + bucket_fn, last_access_fn = "bucket_idx", "last_access" - Args: - fd: field descriptor of the column - *_inputs: a list of field value(s) - - Returns: - returns affected rows count. - """ - if not _inputs: - return 0 - if CacheMeta.contains_field(fd): + def _inner(): with self._con as con: - _regulated_input = [(i,) for i in _inputs] - return con.executemany( - f"DELETE FROM {self.TABLE_NAME} WHERE {fd._field_name}=?", - _regulated_input, - ).rowcount - else: - logger.debug(f"invalid inputs detected: {_inputs=}") - return 0 - - def lookup_entry( - self, - fd: ColumnDescriptor[FV], - value: FV, - ) -> Optional[CacheMeta]: - """Lookup entry by field value. - - NOTE: lookup via this method will trigger update to field, - NOTE 2: field is updated by searching key. - - Args: - fd: field descriptor of the column. - value: field value to lookup. - - Returns: - An instance of CacheMeta representing the cache entry, or None if lookup failed. - """ - if not CacheMeta.contains_field(fd): - return - - fd_name = fd.name - with self._con as con: # put the lookup and update into one session - if row := con.execute( - f"SELECT * FROM {self.TABLE_NAME} WHERE {fd_name}=?", - (value,), - ).fetchone(): - # warm up the cache(update last_access timestamp) here - res = CacheMeta.row_to_meta(row) - con.execute( - ( - f"UPDATE {self.TABLE_NAME} SET {CacheMeta.last_access.name}=? " - f"WHERE {fd_name}=?" - ), - (int(time.time()), value), + # check if we have enough entries to rotate + select_stmt = self.orm_table_spec.table_select_stmt( + select_from=self.orm_table_name, + select_cols="*", + function="count", + where_cols=(bucket_fn,), + order_by=(last_access_fn,), + limit=num, ) - return res - - def insert_entry(self, *cache_meta: CacheMeta) -> int: - """Insert an entry into the ota cache table. - - Args: - cache_meta: a list of CacheMeta instances to be inserted - - Returns: - Returns inserted rows count. - """ - if not cache_meta: - return 0 - with self._con as con: - return con.executemany( - f"INSERT OR REPLACE INTO {self.TABLE_NAME} VALUES ({CacheMeta.get_shape()})", - [m.astuple() for m in cache_meta], - ).rowcount - - def lookup_all(self) -> List[CacheMeta]: - """Lookup all entries in the ota cache table. - - Returns: - A list of CacheMeta instances representing each entry. - """ - with self._con as con: - cur = con.execute(f"SELECT * FROM {self.TABLE_NAME}", ()) - return [CacheMeta.row_to_meta(row) for row in cur.fetchall()] - - def rotate_cache(self, bucket_idx: int, num: int) -> Optional[List[str]]: - """Rotate cache entries in LRU flavour. - - Args: - bucket_idx: which bucket for space reserving - num: num of entries needed to be deleted in this bucket - - Return: - A list of OTA file's hashes that needed to be deleted for space reserving, - or None if no enough entries for space reserving. - """ - bucket_fn, last_access_fn = ( - CacheMeta.bucket_idx.name, - CacheMeta.last_access.name, - ) - # first, check whether we have required number of entries in the bucket - with self._con as con: - cur = con.execute( - ( - f"SELECT COUNT(*) FROM {self.TABLE_NAME} WHERE {bucket_fn}=? " - f"ORDER BY {last_access_fn} LIMIT ?" - ), - (bucket_idx, num), - ) - if not (_raw_res := cur.fetchone()): - return - - # NOTE: if we can upgrade to sqlite3 >= 3.35, - # use RETURNING clause instead of using 2 queries as below - - # if we have enough entries for space reserving - if _raw_res[0] >= num: - # first select those entries - _rows = con.execute( - ( - f"SELECT * FROM {self.TABLE_NAME} " - f"WHERE {bucket_fn}=? " - f"ORDER BY {last_access_fn} " - "LIMIT ?" - ), - (bucket_idx, num), - ).fetchall() - # and then delete those entries with same conditions - con.execute( - ( - f"DELETE FROM {self.TABLE_NAME} " - f"WHERE {bucket_fn}=? " - f"ORDER BY {last_access_fn} " - "LIMIT ?" - ), - (bucket_idx, num), - ) - return [row[CacheMeta.file_sha256.name] for row in _rows] - - -class _ProxyBase: - """A proxy class base for OTACacheDB that dispatches all requests into a threadpool.""" - - DB_THREAD_POOL_SIZE = 1 - - def _thread_initializer(self, db_f): - """Init a db connection for each thread worker""" - # NOTE: set init to False always as we only operate db when using proxy - self._thread_local.db = OTACacheDB(db_f) - - def __init__(self, db_f: Union[str, Path]): - """Init the database connecting thread pool.""" - self._thread_local = threading.local() - # set thread_pool_size to 1 to make the db access - # to make it totally concurrent. - self._executor = ThreadPoolExecutor( - max_workers=self.DB_THREAD_POOL_SIZE, - initializer=self._thread_initializer, - initargs=(db_f,), - ) - - def close(self): - self._executor.shutdown(wait=True) - - -class AIO_OTACacheDBProxy(_ProxyBase): - async def insert_entry(self, *cache_meta: CacheMeta) -> int: - def _inner(): - _db: OTACacheDB = self._thread_local.db - return _db.insert_entry(*cache_meta) - - return await asyncio.get_running_loop().run_in_executor(self._executor, _inner) - - async def lookup_entry( - self, fd: ColumnDescriptor, _input: Any - ) -> Optional[CacheMeta]: - def _inner(): - _db: OTACacheDB = self._thread_local.db - return _db.lookup_entry(fd, _input) - - return await asyncio.get_running_loop().run_in_executor(self._executor, _inner) + cur = con.execute(select_stmt, {bucket_fn: bucket_idx}) + # we don't have enough entries to delete + if not (_raw_res := cur.fetchone()) or _raw_res[0] < num: + return + + # RETURNING statement is available only after sqlite3 v3.35.0 + if sqlite3.sqlite_version_info < (3, 35, 0): + # first select entries met the requirements + select_to_delete_stmt = self.orm_table_spec.table_select_stmt( + select_from=self.orm_table_name, + where_cols=(bucket_fn,), + order_by=(last_access_fn,), + limit=num, + ) + cur = con.execute(select_to_delete_stmt, {bucket_fn: bucket_idx}) + rows_to_remove = list(cur) + + # delete the target entries + delete_stmt = self.orm_table_spec.table_delete_stmt( + delete_from=self.orm_table_name, + where_cols=(bucket_fn,), + order_by=(last_access_fn,), + limit=num, + ) + con.execute(delete_stmt, {bucket_fn: bucket_idx}) + + return rows_to_remove + else: + rotate_stmt = self.orm_table_spec.table_delete_stmt( + delete_from=self.orm_table_name, + where_cols=(bucket_fn,), + order_by=(last_access_fn,), + limit=num, + returning_cols="*", + ) + cur = con.execute(rotate_stmt, {bucket_fn: bucket_idx}) + return list(cur) - async def remove_entries(self, fd: ColumnDescriptor, *_inputs: Any) -> int: - def _inner(): - _db: OTACacheDB = self._thread_local.db - return _db.remove_entries(fd, *_inputs) + return await self._run_in_pool(_inner) - return await asyncio.get_running_loop().run_in_executor(self._executor, _inner) - async def rotate_cache(self, bucket_idx: int, num: int) -> Optional[List[str]]: - def _inner(): - _db: OTACacheDB = self._thread_local.db - return _db.rotate_cache(bucket_idx, num) +def check_db(db_f: StrOrPath, table_name: str) -> bool: + """Check whether specific db is normal or not.""" + if not Path(db_f).is_file(): + logger.warning(f"{db_f} not found") + return False - return await asyncio.get_running_loop().run_in_executor(self._executor, _inner) + con = sqlite3.connect(f"file:{db_f}?mode=ro", uri=True) + try: + if not utils.check_db_integrity(con): + logger.warning(f"{db_f} fails integrity check") + return False + if not utils.lookup_table(con, table_name): + logger.warning(f"{table_name} not found in {db_f}") + return False + finally: + con.close() + return True + + +def init_db(db_f: StrOrPath, table_name: str) -> None: + """Init the database.""" + con = sqlite3.connect(db_f) + orm = CacheMetaORM(con, table_name) + try: + orm.orm_create_table(without_rowid=True) + utils.enable_wal_mode(con, relax_sync_mode=True) + finally: + con.close() diff --git a/src/ota_proxy/lru_cache_helper.py b/src/ota_proxy/lru_cache_helper.py new file mode 100644 index 000000000..5c1df1c8c --- /dev/null +++ b/src/ota_proxy/lru_cache_helper.py @@ -0,0 +1,124 @@ +# Copyright 2022 TIER IV, INC. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Implementation of OTA cache control.""" + + +from __future__ import annotations + +import bisect +import logging +import sqlite3 +import time +from pathlib import Path +from typing import Optional, Union + +from simple_sqlite3_orm import utils + +from .db import AsyncCacheMetaORM, CacheMeta + +logger = logging.getLogger(__name__) + + +class LRUCacheHelper: + """A helper class that provides API for accessing/managing cache entries in ota cachedb. + + Serveral buckets are created according to predefined file size threshould. + Each bucket will maintain the cache entries of that bucket's size definition, + LRU is applied on per-bucket scale. + + NOTE: currently entry in first bucket and last bucket will skip LRU rotate. + """ + + def __init__( + self, + db_f: Union[str, Path], + *, + bsize_dict: dict[int, int], + table_name: str, + thread_nums: int, + thread_wait_timeout: int, + ): + self.bsize_list = list(bsize_dict) + self.bsize_dict = bsize_dict.copy() + + def _con_factory(): + con = sqlite3.connect( + db_f, check_same_thread=False, timeout=thread_wait_timeout + ) + + utils.enable_wal_mode(con, relax_sync_mode=True) + utils.enable_mmap(con) + utils.enable_tmp_store_at_memory(con) + return con + + self._async_db = AsyncCacheMetaORM( + table_name=table_name, + con_factory=_con_factory, + number_of_cons=thread_nums, + ) + + self._closed = False + + def close(self): + self._async_db.orm_pool_shutdown(wait=True, close_connections=True) + self._closed = True + + async def commit_entry(self, entry: CacheMeta) -> bool: + """Commit cache entry meta to the database.""" + # populate bucket and last_access column + entry.bucket_idx = bisect.bisect_right(self.bsize_list, entry.cache_size) - 1 + entry.last_access = int(time.time()) + + if (await self._async_db.orm_insert_entry(entry, or_option="replace")) != 1: + logger.error(f"db: failed to add {entry=}") + return False + return True + + async def lookup_entry(self, file_sha256: str) -> Optional[CacheMeta]: + return await self._async_db.orm_select_entry(file_sha256=file_sha256) + + async def remove_entry(self, file_sha256: str) -> bool: + return bool(await self._async_db.orm_delete_entries(file_sha256=file_sha256)) + + async def rotate_cache(self, size: int) -> Optional[list[str]]: + """Wrapper method for calling the database LRU cache rotating method. + + Args: + size int: the size of file that we want to reserve space for + + Returns: + A list of hashes that needed to be cleaned, or empty list if rotation + is not required, or None if cache rotation cannot be executed. + """ + # NOTE: currently item size smaller than 1st bucket and larger than latest bucket + # will be saved without cache rotating. + if size >= self.bsize_list[-1] or size < self.bsize_list[1]: + return [] + + _cur_bucket_idx = bisect.bisect_right(self.bsize_list, size) - 1 + _cur_bucket_size = self.bsize_list[_cur_bucket_idx] + + # first: check the upper bucket, remove 1 item from any of the + # upper bucket is enough. + # NOTE(20240802): limit the max steps we can go to avoid remove too large file + max_steps = min(len(self.bsize_list), _cur_bucket_idx + 3) + for _bucket_idx in range(_cur_bucket_idx + 1, max_steps): + if res := await self._async_db.rotate_cache(_bucket_idx, 1): + return [entry.file_sha256 for entry in res] + + # second: if cannot find one entry at any upper bucket, check current bucket + if res := await self._async_db.rotate_cache( + _cur_bucket_idx, self.bsize_dict[_cur_bucket_size] + ): + return [entry.file_sha256 for entry in res] diff --git a/src/ota_proxy/orm.py b/src/ota_proxy/orm.py deleted file mode 100644 index 7141ccffd..000000000 --- a/src/ota_proxy/orm.py +++ /dev/null @@ -1,243 +0,0 @@ -# Copyright 2022 TIER IV, INC. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from __future__ import annotations - -from dataclasses import asdict, astuple, dataclass, fields -from io import StringIO -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Dict, - Generic, - List, - Optional, - Tuple, - Type, - TypeVar, - Union, - overload, -) - -from typing_extensions import Self, dataclass_transform - -if TYPE_CHECKING: - import sqlite3 - - -class NULL_TYPE: - """Singleton for NULL type.""" - - def __new__(cls, *args, **kwargs) -> None: - return None - - -SQLITE_DATATYPES = Union[ - int, # INTEGER - str, # TEXT - float, # REAL - bytes, # BLOB - bool, # INTEGER 0, 1 - NULL_TYPE, # NULL -] -SQLITE_DATATYPES_SET = {int, str, float, bytes, bool, NULL_TYPE} -FV = TypeVar("FV", bound=SQLITE_DATATYPES) # field value type -TYPE_CHECKER = Callable[[Any], bool] - - -class ColumnDescriptor(Generic[FV]): - """ColumnDescriptor represents a column in a sqlite3 table, - implemented the python descriptor protocol. - - When accessed as attribute of TableCls(subclass of ORMBase) instance, - it will return the value of the column/field. - When accessed as attribute of the TableCls class, - it will return the ColumnDescriptor itself. - """ - - def __init__( - self, - field_type: Type[FV], - *constrains: str, - type_guard: Union[Tuple[Type, ...], TYPE_CHECKER, bool] = False, - default: Optional[FV] = None, - ) -> None: - # whether this field should be included in table def or not - self._skipped = False - self.constrains = " ".join(constrains) # TODO: constrains validation - - self.field_type = field_type - self.default = field_type() if default is None else default - - # init type checker callable - # default to check over the specific field type - self.type_guard_enabled = bool(type_guard) - self.type_checker = lambda x: isinstance(x, field_type) - if isinstance(type_guard, tuple): # check over a list of types - self.type_checker = lambda x: isinstance(x, type_guard) - elif callable(type_guard): # custom type guard function - self.type_checker = type_guard - - @overload - def __get__(self, obj: None, objtype: type) -> Self: - """Descriptor accessed via class.""" - ... - - @overload - def __get__(self, obj, objtype: type) -> FV: - """Descriptor accessed via bound instance.""" - ... - - def __get__(self, obj, objtype=None) -> Union[FV, Self]: - # try accessing bound instance attribute - if not self._skipped and obj is not None: - if isinstance(obj, type): - return self # bound inst is type(class access mode), return descriptor itself - return getattr(obj, self._private_name) # access via instance - return self # return descriptor instance by default - - def __set__(self, obj, value: Any) -> None: - if self._skipped: - return # ignore value setting on skipped field - - # set default value and ignore input value - # 1. field_type is NULL_TYPE - # 2. value is None - # 3. value is descriptor instance itself - # dataclass will retrieve default value with class access form, - # but we return descriptor instance in class access form to expose - # descriptor helper methods(check_type, etc.), so apply special - # treatment here. - if self.field_type is NULL_TYPE or value is None or value is self: - return setattr(obj, self._private_name, self.default) - - # handle normal value setting - if self.type_guard_enabled and not self.type_checker(value): - raise TypeError(f"type_guard: expect {self.field_type}, get {type(value)}") - # if value's type is not field_type but subclass or compatible types that can pass type_checker, - # convert the value to the field_type first before assigning - _input_value = ( - value if type(value) is self.field_type else self.field_type(value) - ) - setattr(obj, self._private_name, _input_value) - - def __set_name__(self, owner: type, name: str) -> None: - self.owner = owner - try: - self._index = list(owner.__annotations__).index(name) - except (AttributeError, ValueError): - self._skipped = True # skipped due to annotation missing - self._field_name = name - self._private_name = f"_{owner.__name__}_{name}" - - @property - def name(self) -> str: - return self._field_name - - @property - def index(self) -> int: - return self._index - - def check_type(self, value: Any) -> bool: - return self.type_checker(value) - - -@dataclass_transform() -class ORMeta(type): - """This metaclass is for generating customized .""" - - def __new__(cls, cls_name: str, bases: Tuple[type, ...], classdict: Dict[str, Any]): - new_cls: type = super().__new__(cls, cls_name, bases, classdict) - if len(new_cls.__mro__) > 2: # , ORMBase, object - # we will define our own eq and hash logics, disable dataclass' - # eq method and hash method generation - return dataclass(eq=False, unsafe_hash=False)(new_cls) - else: # ORMBase, object - return new_cls - - -class ORMBase(metaclass=ORMeta): - """Base class for defining a sqlite3 table programatically. - - Subclass of this base class is also a subclass of dataclass. - """ - - @classmethod - def row_to_meta(cls, row: "Union[sqlite3.Row, Dict[str, Any], Tuple[Any]]") -> Self: - parsed = {} - for field in fields(cls): - try: - col: ColumnDescriptor = getattr(cls, field.name) - if isinstance(row, tuple): - parsed[col.name] = row[col.index] - else: - parsed[col.name] = row[col.name] - except (IndexError, KeyError): - continue # silently ignore unknonw input fields - return cls(**parsed) - - @classmethod - def get_create_table_stmt(cls, table_name: str) -> str: - """Generate the sqlite query statement to create the defined table in database. - - Args: - table_name: the name of table to be created - - Returns: - query statement to create the table defined by this class. - """ - _col_descriptors: List[ColumnDescriptor] = [ - getattr(cls, field.name) for field in fields(cls) - ] - with StringIO() as buffer: - buffer.write(f"CREATE TABLE {table_name}") - buffer.write("(") - buffer.write( - ", ".join([f"{col.name} {col.constrains}" for col in _col_descriptors]) - ) - buffer.write(")") - return buffer.getvalue() - - @classmethod - def contains_field(cls, _input: Union[str, ColumnDescriptor]) -> bool: - """Check if this table contains field indicated by <_input>.""" - if isinstance(_input, ColumnDescriptor): - return _input.owner.__name__ == cls.__name__ - return isinstance(getattr(cls, _input), ColumnDescriptor) - - @classmethod - def get_shape(cls) -> str: - """Used by insert row query.""" - return ",".join(["?"] * len(fields(cls))) - - def __hash__(self) -> int: - """compute the hash with all stored fields' value.""" - return hash(astuple(self)) - - def __eq__(self, __o: object) -> bool: - if not isinstance(__o, self.__class__): - return False - for field in fields(self): - field_name = field.name - if getattr(self, field_name) != getattr(__o, field_name): - return False - return True - - def astuple(self) -> Tuple[SQLITE_DATATYPES, ...]: - return astuple(self) - - def asdict(self) -> Dict[str, SQLITE_DATATYPES]: - return asdict(self) diff --git a/src/ota_proxy/ota_cache.py b/src/ota_proxy/ota_cache.py index 7a4920eb1..ad7fded61 100644 --- a/src/ota_proxy/ota_cache.py +++ b/src/ota_proxy/ota_cache.py @@ -16,56 +16,32 @@ from __future__ import annotations import asyncio -import bisect -import contextlib import logging -import os import shutil import threading import time -import weakref -from concurrent.futures import Executor, ThreadPoolExecutor +from concurrent.futures import ThreadPoolExecutor from pathlib import Path -from typing import ( - AsyncGenerator, - AsyncIterator, - Callable, - Coroutine, - Dict, - Generic, - List, - Mapping, - MutableMapping, - Optional, - Tuple, - TypeVar, - Union, -) +from typing import AsyncIterator, Dict, List, Mapping, Optional, Tuple from urllib.parse import SplitResult, quote, urlsplit -import aiofiles import aiohttp from multidict import CIMultiDictProxy +from otaclient_common.typing import StrOrPath + from ._consts import HEADER_CONTENT_ENCODING, HEADER_OTA_FILE_CACHE_CONTROL -from .cache_control import OTAFileCacheControl +from .cache_control_header import OTAFileCacheControl +from .cache_streaming import CachingRegister, cache_streaming from .config import config as cfg -from .db import AIO_OTACacheDBProxy, CacheMeta, OTACacheDB -from .errors import ( - BaseOTACacheError, - CacheMultiStreamingFailed, - CacheStreamingFailed, - CacheStreamingInterrupt, - StorageReachHardLimit, -) -from .utils import read_file, url_based_hash, wait_with_backoff +from .db import CacheMeta, check_db, init_db +from .errors import BaseOTACacheError +from .lru_cache_helper import LRUCacheHelper +from .utils import read_file, url_based_hash logger = logging.getLogger(__name__) -_WEAKREF = TypeVar("_WEAKREF") - - # helper functions @@ -86,490 +62,22 @@ def create_cachemeta_for_request( compression_alg: pre-collected information from caller resp_headers_from_upper """ - cache_meta = CacheMeta( - url=raw_url, - content_encoding=resp_headers_from_upper.get(HEADER_CONTENT_ENCODING, ""), - ) - _upper_cache_policy = OTAFileCacheControl.parse_header( resp_headers_from_upper.get(HEADER_OTA_FILE_CACHE_CONTROL, "") ) if _upper_cache_policy.file_sha256: - cache_meta.file_sha256 = _upper_cache_policy.file_sha256 - cache_meta.file_compression_alg = _upper_cache_policy.file_compression_alg + file_sha256 = _upper_cache_policy.file_sha256 + file_compression_alg = _upper_cache_policy.file_compression_alg or None else: - cache_meta.file_sha256 = cache_identifier - cache_meta.file_compression_alg = compression_alg - return cache_meta - - -# cache tracker - - -class CacheTracker(Generic[_WEAKREF]): - """A tracker for an ongoing cache entry. - - This tracker represents an ongoing cache entry under the , - and takes care of the life cycle of this temp cache entry. - It implements the provider/subscriber model for cache writing and cache streaming to - multiple clients. - - This entry will disappear automatically, ensured by gc and weakref - when no strong reference to this tracker(provider finished and no subscribers - attached to this tracker). - When this tracker is garbage collected, the corresponding temp cache entry will - also be removed automatically(ensure by registered finalizer). - - Attributes: - fpath: the path to the temporary cache file. - save_path: the path to finished cache file. - meta: an inst of CacheMeta for the remote OTA file that being cached. - writer_ready: a property indicates whether the provider is - ready to write and streaming data chunks. - writer_finished: a property indicates whether the provider finished the caching. - writer_failed: a property indicates whether provider fails to - finish the caching. - """ - - READER_SUBSCRIBE_WAIT_PROVIDER_TIMEOUT = 2 - FNAME_PART_SEPARATOR = "_" - - @classmethod - def _tmp_file_naming(cls, cache_identifier: str) -> str: - """Create fname for tmp caching entry. - - naming scheme: tmp__ - - NOTE: append 4bytes hex to identify cache entry for the same OTA file between - different trackers. - """ - return ( - f"{cfg.TMP_FILE_PREFIX}{cls.FNAME_PART_SEPARATOR}" - f"{cache_identifier}{cls.FNAME_PART_SEPARATOR}{(os.urandom(4)).hex()}" - ) - - def __init__( - self, - cache_identifier: str, - ref_holder: _WEAKREF, - *, - base_dir: Union[str, Path], - executor: Executor, - callback: _CACHE_ENTRY_REGISTER_CALLBACK, - below_hard_limit_event: threading.Event, - ): - self.fpath = Path(base_dir) / self._tmp_file_naming(cache_identifier) - self.meta: CacheMeta = None # type: ignore[assignment] - self.cache_identifier = cache_identifier - self.save_path = Path(base_dir) / cache_identifier - self._writer_ready = asyncio.Event() - self._writer_finished = asyncio.Event() - self._writer_failed = asyncio.Event() - self._ref = ref_holder - self._subscriber_ref_holder: List[_WEAKREF] = [] - self._executor = executor - self._cache_commit_cb = callback - self._space_availability_event = below_hard_limit_event - - self._bytes_written = 0 - self._cache_write_gen: Optional[AsyncGenerator[int, bytes]] = None - - # self-register the finalizer to this tracker - weakref.finalize( - self, - self.finalizer, - fpath=self.fpath, - ) - - async def _finalize_caching(self): - """Commit cache entry to db and rename tmp cached file with sha256hash - to fialize the caching.""" - # if the file with the same sha256has is already presented, skip the hardlink - # NOTE: no need to clean the tmp file, it will be done by the cache tracker. - await self._cache_commit_cb(self.meta) - if not self.save_path.is_file(): - self.fpath.link_to(self.save_path) - - @staticmethod - def finalizer(*, fpath: Union[str, Path]): - """Finalizer that cleans up the tmp file when this tracker is gced.""" - Path(fpath).unlink(missing_ok=True) - - @property - def writer_failed(self) -> bool: - return self._writer_failed.is_set() - - @property - def writer_finished(self) -> bool: - return self._writer_finished.is_set() - - @property - def is_cache_valid(self) -> bool: - """Indicates whether the temp cache entry for this tracker is valid.""" - return ( - self.meta is not None - and self._writer_finished.is_set() - and not self._writer_failed.is_set() - ) - - def get_cache_write_gen(self) -> Optional[AsyncGenerator[int, bytes]]: - if self._cache_write_gen: - return self._cache_write_gen - logger.warning(f"tracker for {self.meta} is not ready, skipped") - - async def _provider_write_cache(self) -> AsyncGenerator[int, bytes]: - """Provider writes data chunks from upper caller to tmp cache file. - - If cache writing failed, this method will exit and tracker.writer_failed and - tracker.writer_finished will be set. - """ - logger.debug(f"start to cache for {self.meta=}...") - try: - if not self.meta: - raise ValueError("called before provider tracker is ready, abort") - - async with aiofiles.open(self.fpath, "wb", executor=self._executor) as f: - # let writer become ready when file is open successfully - self._writer_ready.set() - - _written = 0 - while _data := (yield _written): - if not self._space_availability_event.is_set(): - logger.warning( - f"abort writing cache for {self.meta=}: {StorageReachHardLimit.__name__}" - ) - self._writer_failed.set() - return - - _written = await f.write(_data) - self._bytes_written += _written - - logger.debug( - "cache write finished, total bytes written" - f"({self._bytes_written}) for {self.meta=}" - ) - self.meta.cache_size = self._bytes_written - - # NOTE: no need to track the cache commit, - # as writer_finish is meant to set on cache file created. - asyncio.create_task(self._finalize_caching()) - except Exception as e: - logger.exception(f"failed to write cache for {self.meta=}: {e!r}") - self._writer_failed.set() - finally: - self._writer_finished.set() - self._ref = None - - async def _subscribe_cache_streaming(self) -> AsyncIterator[bytes]: - """Subscriber keeps polling chunks from ongoing tmp cache file. - - Subscriber will keep polling until the provider fails or - provider finished and subscriber has read bytes. - - Raises: - CacheMultipleStreamingFailed if provider failed or timeout reading - data chunk from tmp cache file(might be caused by a dead provider). - """ - try: - err_count, _bytes_read = 0, 0 - async with aiofiles.open(self.fpath, "rb", executor=self._executor) as f: - while not (self.writer_finished and _bytes_read == self._bytes_written): - if self.writer_failed: - raise CacheMultiStreamingFailed( - f"provider aborted for {self.meta}" - ) - _bytes_read += len(_chunk := await f.read(cfg.CHUNK_SIZE)) - if _chunk: - err_count = 0 - yield _chunk - continue - - err_count += 1 - if not await wait_with_backoff( - err_count, - _backoff_factor=cfg.STREAMING_BACKOFF_FACTOR, - _backoff_max=cfg.STREAMING_BACKOFF_MAX, - ): - # abort caching due to potential dead streaming coro - _err_msg = f"failed to stream({self.meta=}): timeout getting data, partial read might happen" - logger.error(_err_msg) - # signal streamer to stop streaming - raise CacheMultiStreamingFailed(_err_msg) - finally: - # unsubscribe on finish - self._subscriber_ref_holder.pop() - - async def _read_cache(self) -> AsyncIterator[bytes]: - """Directly open the tmp cache entry and yield data chunks from it. - - Raises: - CacheMultipleStreamingFailed if fails to read from the - cached file, this might indicate a partial written cache file. - """ - _bytes_read, _retry_count = 0, 0 - async with aiofiles.open(self.fpath, "rb", executor=self._executor) as f: - while _bytes_read < self._bytes_written: - if _data := await f.read(cfg.CHUNK_SIZE): - _retry_count = 0 - _bytes_read += len(_data) - yield _data - continue - - # no data is read from the cache entry, - # retry sometimes to ensure all data is acquired - _retry_count += 1 - if not await wait_with_backoff( - _retry_count, - _backoff_factor=cfg.STREAMING_BACKOFF_FACTOR, - _backoff_max=cfg.STREAMING_CACHED_TMP_TIMEOUT, - ): - # abort caching due to potential dead streaming coro - _err_msg = ( - f"open_cached_tmp failed for ({self.meta=}): " - "timeout getting more data, partial written cache file detected" - ) - logger.debug(_err_msg) - # signal streamer to stop streaming - raise CacheMultiStreamingFailed(_err_msg) - - # exposed API - - async def provider_start(self, meta: CacheMeta): - """Register meta to the Tracker, create tmp cache entry and get ready. - - Check _provider_write_cache for more details. - - Args: - meta: inst of CacheMeta for the requested file tracked by this tracker. - This meta is created by open_remote() method. - """ - self.meta = meta - self._cache_write_gen = self._provider_write_cache() - await self._cache_write_gen.asend(None) # type: ignore - - async def provider_on_finished(self): - if not self.writer_finished and self._cache_write_gen: - with contextlib.suppress(StopAsyncIteration): - await self._cache_write_gen.asend(b"") - self._writer_finished.set() - self._ref = None - - async def provider_on_failed(self): - """Manually fail and stop the caching.""" - if not self.writer_finished and self._cache_write_gen: - logger.warning(f"interrupt writer coroutine for {self.meta=}") - with contextlib.suppress(StopAsyncIteration, CacheStreamingInterrupt): - await self._cache_write_gen.athrow(CacheStreamingInterrupt) - - self._writer_failed.set() - self._writer_finished.set() - self._ref = None - - async def subscriber_subscribe_tracker(self) -> Optional[AsyncIterator[bytes]]: - """Reader subscribe this tracker and get a file descriptor to get data chunks.""" - _wait_count = 0 - while not self._writer_ready.is_set(): - _wait_count += 1 - if self.writer_failed or not await wait_with_backoff( - _wait_count, - _backoff_factor=cfg.STREAMING_BACKOFF_FACTOR, - _backoff_max=self.READER_SUBSCRIBE_WAIT_PROVIDER_TIMEOUT, - ): - return # timeout waiting for provider to become ready - - # subscribe on an ongoing cache - if not self.writer_finished and isinstance(self._ref, _Weakref): - self._subscriber_ref_holder.append(self._ref) - return self._subscribe_cache_streaming() - # caching just finished, try to directly read the finished cache entry - elif self.is_cache_valid: - return self._read_cache() - - -# a callback that register the cache entry indicates by input CacheMeta inst to the cache_db -_CACHE_ENTRY_REGISTER_CALLBACK = Callable[[CacheMeta], Coroutine[None, None, None]] - - -class _Weakref: - pass - - -class CachingRegister: - """A tracker register that manages cache trackers. - - For each ongoing caching for unique OTA file, there will be only one unique identifier for it. - - This first caller that requests with the identifier will become the provider and create - a new tracker for this identifier. - The later comes callers will become the subscriber to this tracker. - """ - - def __init__(self, base_dir: Union[str, Path]): - self._base_dir = Path(base_dir) - self._id_ref_dict: MutableMapping[str, _Weakref] = weakref.WeakValueDictionary() - self._ref_tracker_dict: MutableMapping[_Weakref, CacheTracker] = ( - weakref.WeakKeyDictionary() - ) - - async def get_tracker( - self, - cache_identifier: str, - *, - executor: Executor, - callback: _CACHE_ENTRY_REGISTER_CALLBACK, - below_hard_limit_event: threading.Event, - ) -> Tuple[CacheTracker, bool]: - """Get an inst of CacheTracker for the cache_identifier. - - Returns: - An inst of tracker, and a bool indicates the caller is provider(True), - or subscriber(False). - """ - _new_ref = _Weakref() - _ref = self._id_ref_dict.setdefault(cache_identifier, _new_ref) - - # subscriber - if ( - _tracker := self._ref_tracker_dict.get(_ref) - ) and not _tracker.writer_failed: - return _tracker, False - - # provider, or override a failed provider - if _ref is not _new_ref: # override a failed tracker - self._id_ref_dict[cache_identifier] = _new_ref - _ref = _new_ref - - _tracker = CacheTracker( - cache_identifier, - _ref, - base_dir=self._base_dir, - executor=executor, - callback=callback, - below_hard_limit_event=below_hard_limit_event, - ) - self._ref_tracker_dict[_ref] = _tracker - return _tracker, True - - -class LRUCacheHelper: - """A helper class that provides API for accessing/managing cache entries in ota cachedb. - - Serveral buckets are created according to predefined file size threshould. - Each bucket will maintain the cache entries of that bucket's size definition, - LRU is applied on per-bucket scale. - - NOTE: currently entry in first bucket and last bucket will skip LRU rotate. - """ - - BSIZE_LIST = list(cfg.BUCKET_FILE_SIZE_DICT.keys()) - BSIZE_DICT = cfg.BUCKET_FILE_SIZE_DICT - - def __init__(self, db_f: Union[str, Path]): - self._db = AIO_OTACacheDBProxy(db_f) - self._closed = False - - def close(self): - if not self._closed: - self._db.close() - - async def commit_entry(self, entry: CacheMeta) -> bool: - """Commit cache entry meta to the database.""" - # populate bucket and last_access column - entry.bucket_idx = bisect.bisect_right(self.BSIZE_LIST, entry.cache_size) - 1 - entry.last_access = int(time.time()) - - if (await self._db.insert_entry(entry)) != 1: - logger.error(f"db: failed to add {entry=}") - return False - return True - - async def lookup_entry(self, file_sha256: str) -> Optional[CacheMeta]: - return await self._db.lookup_entry(CacheMeta.file_sha256, file_sha256) - - async def remove_entry(self, file_sha256: str) -> bool: - return (await self._db.remove_entries(CacheMeta.file_sha256, file_sha256)) > 0 - - async def rotate_cache(self, size: int) -> Optional[List[str]]: - """Wrapper method for calling the database LRU cache rotating method. - - Args: - size int: the size of file that we want to reserve space for - - Returns: - A list of hashes that needed to be cleaned, or empty list if rotation - is not required, or None if cache rotation cannot be executed. - """ - # NOTE: currently item size smaller than 1st bucket and larger than latest bucket - # will be saved without cache rotating. - if size >= self.BSIZE_LIST[-1] or size < self.BSIZE_LIST[1]: - return [] - - _cur_bucket_idx = bisect.bisect_right(self.BSIZE_LIST, size) - 1 - _cur_bucket_size = self.BSIZE_LIST[_cur_bucket_idx] - - # first check the upper bucket, remove 1 item from any of the - # upper bucket is enough. - for _bucket_idx in range(_cur_bucket_idx + 1, len(self.BSIZE_LIST)): - if res := await self._db.rotate_cache(_bucket_idx, 1): - return res - # if cannot find one entry at any upper bucket, check current bucket - return await self._db.rotate_cache( - _cur_bucket_idx, self.BSIZE_DICT[_cur_bucket_size] - ) - - -async def cache_streaming( - fd: AsyncIterator[bytes], - meta: CacheMeta, - tracker: CacheTracker, -) -> AsyncIterator[bytes]: - """A cache streamer that get data chunk from and tees to multiple destination. - - Data chunk yielded from will be teed to: - 1. upper uvicorn otaproxy APP to send back to client, - 2. cache_tracker cache_write_gen for caching. - - Args: - fd: opened connection to a remote file. - meta: meta data of the requested resource. - tracker: an inst of ongoing cache tracker bound to this request. - - Returns: - A bytes async iterator to yield data chunk from, for upper otaproxy uvicorn APP. - - Raises: - CacheStreamingFailed if any exception happens during retrieving. - """ + file_sha256 = cache_identifier + file_compression_alg = compression_alg - async def _inner(): - _cache_write_gen = tracker.get_cache_write_gen() - try: - # tee the incoming chunk to two destinations - async for chunk in fd: - # NOTE: for aiohttp, when HTTP chunk encoding is enabled, - # an empty chunk will be sent to indicate the EOF of stream, - # we MUST handle this empty chunk. - if not chunk: # skip if empty chunk is read from remote - continue - # to caching generator - if _cache_write_gen and not tracker.writer_finished: - try: - await _cache_write_gen.asend(chunk) - except Exception as e: - await tracker.provider_on_failed() # signal tracker - logger.error( - f"cache write coroutine failed for {meta=}, abort caching: {e!r}" - ) - # to uvicorn thread - yield chunk - await tracker.provider_on_finished() - except Exception as e: - logger.exception(f"cache tee failed for {meta=}") - await tracker.provider_on_failed() - raise CacheStreamingFailed from e - - await tracker.provider_start(meta) - return _inner() + return CacheMeta( + file_sha256=file_sha256, + file_compression_alg=file_compression_alg, + url=raw_url, + content_encoding=resp_headers_from_upper.get(HEADER_CONTENT_ENCODING), + ) class OTACache: @@ -595,8 +103,8 @@ def __init__( *, cache_enabled: bool, init_cache: bool, - base_dir: Optional[Union[str, Path]] = None, - db_file: Optional[Union[str, Path]] = None, + base_dir: Optional[StrOrPath] = None, + db_file: Optional[StrOrPath] = None, upper_proxy: str = "", enable_https: bool = False, external_cache: Optional[str] = None, @@ -606,14 +114,26 @@ def __init__( f"init ota_cache({cache_enabled=}, {init_cache=}, {upper_proxy=}, {enable_https=})" ) self._closed = True + self._shutdown_lock = asyncio.Lock() + self.table_name = table_name = cfg.TABLE_NAME self._chunk_size = cfg.CHUNK_SIZE - self._base_dir = Path(base_dir) if base_dir else Path(cfg.BASE_DIR) - self._db_file = Path(db_file) if db_file else Path(cfg.DB_FILE) self._cache_enabled = cache_enabled self._init_cache = init_cache self._enable_https = enable_https - self._executor = ThreadPoolExecutor(thread_name_prefix="ota_cache_executor") + + self._base_dir = Path(base_dir) if base_dir else Path(cfg.BASE_DIR) + self._db_file = db_f = Path(db_file) if db_file else Path(cfg.DB_FILE) + + self._base_dir.mkdir(parents=True, exist_ok=True) + if not check_db(self._db_file, table_name): + logger.info(f"db file is broken, force init db file at {db_f}") + db_f.unlink(missing_ok=True) + self._init_cache = True # force init cache on db file cleanup + + self._executor = ThreadPoolExecutor( + thread_name_prefix="ota_cache_fileio_executor" + ) if external_cache and cache_enabled: logger.info(f"external cache source is enabled at: {external_cache}") @@ -623,14 +143,6 @@ def __init__( self._storage_below_soft_limit_event = threading.Event() self._upper_proxy = upper_proxy - def _check_cache_db(self) -> bool: - """Check ota_cache can be reused or not.""" - return ( - self._base_dir.is_dir() - and self._db_file.is_file() - and OTACacheDB.check_db_file(self._db_file) - ) - async def start(self): """Start the ota_cache instance.""" # silently ignore multi launching of ota_cache @@ -657,15 +169,15 @@ async def start(self): if self._cache_enabled: # purge cache dir if requested(init_cache=True) or ota_cache invalid, # and then recreate the cache folder and cache db file. - db_f_valid = self._check_cache_db() - if self._init_cache or not db_f_valid: - logger.warning( - f"purge and init ota_cache: {self._init_cache=}, {db_f_valid}" - ) + if self._init_cache: + logger.warning("purge and init ota_cache") shutil.rmtree(str(self._base_dir), ignore_errors=True) self._base_dir.mkdir(exist_ok=True, parents=True) # init db file with table created - OTACacheDB.init_db_file(self._db_file) + self._db_file.unlink(missing_ok=True) + + init_db(self._db_file, cfg.TABLE_NAME) + # reuse the previously left ota_cache else: # cleanup unfinished tmp files for tmp_f in self._base_dir.glob(f"{cfg.TMP_FILE_PREFIX}*"): @@ -675,7 +187,13 @@ async def start(self): self._executor.submit(self._background_check_free_space) # init cache helper(and connect to ota_cache db) - self._lru_helper = LRUCacheHelper(self._db_file) + self._lru_helper = LRUCacheHelper( + self._db_file, + bsize_dict=cfg.BUCKET_FILE_SIZE_DICT, + table_name=cfg.TABLE_NAME, + thread_nums=cfg.DB_THREADS, + thread_wait_timeout=cfg.DB_THREAD_WAIT_TIMEOUT, + ) self._on_going_caching = CachingRegister(self._base_dir) if self._upper_proxy: @@ -691,11 +209,14 @@ async def close(self): performed by the OTACache. """ logger.debug("shutdown ota-cache...") - if self._cache_enabled and not self._closed: - self._closed = True - await self._session.close() - self._lru_helper.close() - self._executor.shutdown(wait=True) + async with self._shutdown_lock: + if not self._closed: + self._closed = True + await self._session.close() + self._executor.shutdown(wait=True) + + if self._cache_enabled: + self._lru_helper.close() logger.info("shutdown ota-cache completed") @@ -894,6 +415,15 @@ async def _retrieve_file_by_cache( return # check if cache file exists + # NOTE(20240729): there is an edge condition that the finished cached file is not yet renamed, + # but the database entry has already been inserted. Here we wait for 3 rounds for + # cache_commit_callback to rename the tmp file. + _retry_count_max, _factor = 3, 0.01 + for _retry_count in range(_retry_count_max): + if cache_file.is_file(): + break + await asyncio.sleep(_factor * _retry_count) # give away the event loop + if not cache_file.is_file(): logger.warning( f"dangling cache entry found, remove db entry: {meta_db_entry}" @@ -999,9 +529,9 @@ async def retrieve_file( # pre-calculated cache_identifier and corresponding compression_alg cache_identifier = cache_policy.file_sha256 compression_alg = cache_policy.file_compression_alg - if ( - not cache_identifier - ): # fallback to use URL based hash, and clear compression_alg + + # fallback to use URL based hash, and clear compression_alg + if not cache_identifier: cache_identifier = url_based_hash(raw_url) compression_alg = "" diff --git a/tests/test_ota_proxy/test_cachedb.py b/tests/test_ota_proxy/test_cachedb.py deleted file mode 100644 index 4f5992ee4..000000000 --- a/tests/test_ota_proxy/test_cachedb.py +++ /dev/null @@ -1,263 +0,0 @@ -# Copyright 2022 TIER IV, INC. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import logging -import sqlite3 -from dataclasses import dataclass -from os import urandom -from pathlib import Path -from typing import Any, Dict, Tuple - -import pytest - -from ota_proxy import config as cfg -from ota_proxy.orm import NULL_TYPE -from ota_proxy.ota_cache import CacheMeta, OTACacheDB -from ota_proxy.utils import url_based_hash - -logger = logging.getLogger(__name__) - - -class TestORM: - @pytest.fixture(autouse=True) - def create_table_defs(self): - from ota_proxy.orm import NULL_TYPE, ColumnDescriptor, ORMBase - - @dataclass - class TableCls(ORMBase): - str_field: ColumnDescriptor[str] = ColumnDescriptor( - str, - "TEXT", - "UNIQUE", - "NOT NULL", - "PRIMARY KEY", - default="invalid_url", - ) - int_field: ColumnDescriptor[int] = ColumnDescriptor( - int, "INTEGER", "NOT NULL", type_guard=(int, float) - ) - float_field: ColumnDescriptor[float] = ColumnDescriptor( - float, "INTEGER", "NOT NULL", type_guard=(int, float) - ) - op_str_field: ColumnDescriptor[str] = ColumnDescriptor(str, "TEXT") - op_int_field: ColumnDescriptor[int] = ColumnDescriptor( - int, "INTEGER", type_guard=(int, float) - ) - null_field: ColumnDescriptor[NULL_TYPE] = ColumnDescriptor( - NULL_TYPE, "NULL" - ) - - self.table_cls = TableCls - - @pytest.mark.parametrize( - "raw_row, as_dict, as_tuple", - ( - ( - { - "str_field": "unique_str", - "int_field": 123.123, # expect to be converted into int - "float_field": 456.123, - "null_field": "should_not_be_set", - }, - { - "str_field": "unique_str", - "int_field": 123, - "float_field": 456.123, - "op_str_field": "", - "op_int_field": 0, - "null_field": NULL_TYPE(), - }, - ("unique_str", 123, 456.123, "", 0, NULL_TYPE()), - ), - ), - ) - def test_parse_and_export( - self, raw_row, as_dict: Dict[str, Any], as_tuple: Tuple[Any] - ): - table_cls = self.table_cls - parsed = table_cls.row_to_meta(raw_row) - assert parsed.asdict() == as_dict - assert parsed.astuple() == as_tuple - assert table_cls.row_to_meta(parsed.astuple()).asdict() == as_dict - - @pytest.mark.parametrize( - "table_name, expected", - ( - ( - "table_name", - ( - "CREATE TABLE table_name(" - "str_field TEXT UNIQUE NOT NULL PRIMARY KEY, " - "int_field INTEGER NOT NULL, " - "float_field INTEGER NOT NULL, " - "op_str_field TEXT, " - "op_int_field INTEGER, " - "null_field NULL)" - ), - ), - ), - ) - def test_get_create_table_stmt(self, table_name: str, expected: str): - assert self.table_cls.get_create_table_stmt(table_name) == expected - - @pytest.mark.parametrize( - "name", - ( - "str_field", - "int_field", - "float_field", - "op_str_field", - "op_int_field", - "null_field", - ), - ) - def test_contains_field(self, name: str): - table_cls = self.table_cls - assert (col_descriptor := getattr(table_cls, name)) - assert table_cls.contains_field(name) - assert col_descriptor is table_cls.__dict__[name] - assert col_descriptor and table_cls.contains_field(col_descriptor) - - def test_type_check(self): - inst = self.table_cls() - # field float_field is type_checked - inst.float_field = 123.456 - with pytest.raises(TypeError): - inst.float_field = "str_type" - - @pytest.mark.parametrize( - "row_dict", - ( - { - "str_field": "unique_str", - "int_field": 123.9, - "float_field": 456, - }, - ), - ) - def test_with_actual_db(self, row_dict: Dict[str, Any]): - """Setup a new conn to a in-memory otacache_db.""" - table_cls, table_name = self.table_cls, "test_table" - conn = sqlite3.connect(":memory:") - conn.row_factory = sqlite3.Row - with conn: # test create table with table_cls - conn.execute(table_cls.get_create_table_stmt(table_name)) - - row_inst = table_cls.row_to_meta(row_dict) - logger.info(row_inst) - with conn: # insert one entry - assert ( - conn.execute( - f"INSERT INTO {table_name} VALUES ({table_cls.get_shape()})", - row_inst.astuple(), - ).rowcount - == 1 - ) - with conn: # get the entry back - cur = conn.execute(f"SELECT * from {table_name}", ()) - row_parsed = table_cls.row_to_meta(cur.fetchone()) - assert row_parsed == row_inst - assert row_parsed.asdict() == row_inst.asdict() - - conn.close() - - -class TestOTACacheDB: - @pytest.fixture(autouse=True) - def prepare_db(self, tmp_path: Path): - self.db_f = db_f = tmp_path / "db_f" - self.entries = entries = [] - - # prepare data - for target_size, rotate_num in cfg.BUCKET_FILE_SIZE_DICT.items(): - for _i in range(rotate_num): - mocked_url = f"{target_size}#{_i}" - entries.append( - CacheMeta( - file_sha256=url_based_hash(mocked_url), - url=mocked_url, - bucket_idx=target_size, - cache_size=target_size, - ) - ) - # insert entry into db - OTACacheDB.init_db_file(db_f) - with OTACacheDB(db_f) as db_conn: - db_conn.insert_entry(*self.entries) - - # prepare connection - try: - self.conn = OTACacheDB(self.db_f) - yield - finally: - self.conn.close() - - def test_insert(self, tmp_path: Path): - db_f = tmp_path / "db_f2" - # first insert - OTACacheDB.init_db_file(db_f) - with OTACacheDB(db_f) as db_conn: - assert db_conn.insert_entry(*self.entries) == len(self.entries) - # check the insertion with another connection - with OTACacheDB(db_f) as db_conn: - _all = db_conn.lookup_all() - # should have the same order as insertion - assert _all == self.entries - - def test_db_corruption(self, tmp_path: Path): - test_db_f = tmp_path / "corrupted_db_f" - # intensionally create corrupted db file - with open(self.db_f, "rb") as src, open(test_db_f, "wb") as dst: - count = 0 - while data := src.read(32): - count += 1 - dst.write(data) - if count % 2 == 0: - dst.write(urandom(8)) - - assert not OTACacheDB.check_db_file(test_db_f) - - def test_lookup_all(self): - """ - NOTE: the timestamp update is only executed at lookup method - """ - entries_set = set(self.entries) - checked_entries = self.conn.lookup_all() - assert entries_set == set(checked_entries) - - def test_lookup(self): - """ - lookup the one entry in the database, and ensure the timestamp is updated - """ - target = self.entries[-1] - # lookup once to update last_acess - checked_entry = self.conn.lookup_entry(CacheMeta.url, target.url) - assert checked_entry == target - checked_entry = self.conn.lookup_entry(CacheMeta.url, target.url) - assert checked_entry and checked_entry.last_access > target.last_access - - def test_delete(self): - """ - delete the whole 8MiB bucket - """ - bucket_size = 8 * (1024**2) - assert ( - self.conn.remove_entries(CacheMeta.bucket_idx, bucket_size) - == cfg.BUCKET_FILE_SIZE_DICT[bucket_size] - ) - assert ( - len(self.conn.lookup_all()) - == len(self.entries) - cfg.BUCKET_FILE_SIZE_DICT[bucket_size] - ) diff --git a/tests/test_ota_proxy/test_ota_cache.py b/tests/test_ota_proxy/test_ota_cache.py index 26dec048c..734911533 100644 --- a/tests/test_ota_proxy/test_ota_cache.py +++ b/tests/test_ota_proxy/test_ota_cache.py @@ -13,93 +13,141 @@ # limitations under the License. +from __future__ import annotations + import asyncio +import bisect import logging import random +import sqlite3 from pathlib import Path -from typing import Coroutine, Dict, List, Optional, Tuple +from typing import Coroutine, Optional import pytest +import pytest_asyncio +from simple_sqlite3_orm import ORMBase from ota_proxy import config as cfg -from ota_proxy.db import CacheMeta, OTACacheDB +from ota_proxy.db import CacheMeta from ota_proxy.ota_cache import CachingRegister, LRUCacheHelper from ota_proxy.utils import url_based_hash logger = logging.getLogger(__name__) +TEST_DATA_SET_SIZE = 4096 +TEST_LOOKUP_ENTRIES = 1200 +TEST_DELETE_ENTRIES = 512 -class TestLRUCacheHelper: - @pytest.fixture(scope="class") - def prepare_entries(self): - entries: Dict[str, CacheMeta] = {} - for target_size, rotate_num in cfg.BUCKET_FILE_SIZE_DICT.items(): - for _i in range(rotate_num): - mocked_url = f"{target_size}#{_i}" - entries[mocked_url] = CacheMeta( - url=mocked_url, - bucket_idx=target_size, - cache_size=target_size, - file_sha256=url_based_hash(mocked_url), - ) - return entries +class OTACacheDB(ORMBase[CacheMeta]): + pass + + +@pytest.fixture(autouse=True, scope="module") +def setup_testdata() -> dict[str, CacheMeta]: + size_list = list(cfg.BUCKET_FILE_SIZE_DICT) + + entries: dict[str, CacheMeta] = {} + for idx in range(TEST_DATA_SET_SIZE): + target_size = random.choice(size_list) + mocked_url = f"#{idx}/w/targetsize={target_size}" + file_sha256 = url_based_hash(mocked_url) + + entries[file_sha256] = CacheMeta( + url=mocked_url, + # see lru_cache_helper module for more details + bucket_idx=bisect.bisect_right(size_list, target_size), + cache_size=target_size, + file_sha256=file_sha256, + ) + return entries + + +@pytest.fixture(autouse=True, scope="module") +def entries_to_lookup(setup_testdata: dict[str, CacheMeta]) -> list[CacheMeta]: + return random.sample( + list(setup_testdata.values()), + k=TEST_LOOKUP_ENTRIES, + ) + - @pytest.fixture(scope="class") - def launch_lru_helper(self, tmp_path_factory: pytest.TempPathFactory): - # init db +@pytest.fixture(autouse=True, scope="module") +def entries_to_remove(setup_testdata: dict[str, CacheMeta]) -> list[CacheMeta]: + return random.sample( + list(setup_testdata.values()), + k=TEST_DELETE_ENTRIES, + ) + + +@pytest.mark.asyncio(scope="class") +class TestLRUCacheHelper: + + @pytest_asyncio.fixture(autouse=True, scope="class") + async def lru_helper(self, tmp_path_factory: pytest.TempPathFactory): ota_cache_folder = tmp_path_factory.mktemp("ota-cache") - self._db_f = ota_cache_folder / "db_f" - OTACacheDB.init_db_file(self._db_f) + db_f = ota_cache_folder / "db_f" + + # init table + conn = sqlite3.connect(db_f) + orm = OTACacheDB(conn, cfg.TABLE_NAME) + orm.orm_create_table(without_rowid=True) + conn.close() - lru_cache_helper = LRUCacheHelper(self._db_f) + lru_cache_helper = LRUCacheHelper( + db_f, + bsize_dict=cfg.BUCKET_FILE_SIZE_DICT, + table_name=cfg.TABLE_NAME, + thread_nums=cfg.DB_THREADS, + thread_wait_timeout=cfg.DB_THREAD_WAIT_TIMEOUT, + ) try: yield lru_cache_helper finally: lru_cache_helper.close() - @pytest.fixture(autouse=True) - def setup_test(self, launch_lru_helper, prepare_entries): - self.entries: Dict[str, CacheMeta] = prepare_entries - self.cache_helper: LRUCacheHelper = launch_lru_helper + async def test_commit_entry( + self, lru_helper: LRUCacheHelper, setup_testdata: dict[str, CacheMeta] + ): + for _, entry in setup_testdata.items(): + # deliberately clear the bucket_idx, this should be set by commit_entry method + _copy = entry.model_copy() + _copy.bucket_idx = 0 + assert await lru_helper.commit_entry(entry) - async def test_commit_entry(self): - for _, entry in self.entries.items(): - assert await self.cache_helper.commit_entry(entry) + async def test_lookup_entry( + self, + lru_helper: LRUCacheHelper, + entries_to_lookup: list[CacheMeta], + setup_testdata: dict[str, CacheMeta], + ): + for entry in entries_to_lookup: + assert ( + await lru_helper.lookup_entry(entry.file_sha256) + == setup_testdata[entry.file_sha256] + ) - async def test_lookup_entry(self): - target_size, idx = 8 * (1024**2), 6 - target_url = f"{target_size}#{idx}" - file_sha256 = url_based_hash(target_url) + async def test_remove_entry( + self, lru_helper: LRUCacheHelper, entries_to_remove: list[CacheMeta] + ): + for entry in entries_to_remove: + assert await lru_helper.remove_entry(entry.file_sha256) - assert ( - await self.cache_helper.lookup_entry(file_sha256) - == self.entries[target_url] - ) + async def test_rotate_cache(self, lru_helper: LRUCacheHelper): + """Ensure the LRUHelper properly rotates the cache entries. - async def test_remove_entry(self): - target_size, idx = 8 * (1024**2), 6 - target_file_sha256 = url_based_hash(f"{target_size}#{idx}") - assert await self.cache_helper.remove_entry(target_file_sha256) - - async def test_rotate_cache(self): - """Ensure the LRUHelper properly rotates the cache entries.""" - # test 1: reserve space for 32 * (1024**2) bucket - # the 32MB bucket is the last bucket and will not be rotated. - target_bucket = 32 * (1024**2) - entries_to_be_removed = await self.cache_helper.rotate_cache(target_bucket) - assert entries_to_be_removed is not None and len(entries_to_be_removed) == 0 - - # test 2: reserve space for 8 * 1024 bucket - # the next bucket is not empty, so we expecte to remove one entry from the next bucket - target_bucket = 8 * 1024 - assert ( - entries_to_be_removed := await self.cache_helper.rotate_cache(target_bucket) - ) and len(entries_to_be_removed) == 1 + We should file enough entries into the database, so each rotate should be successful. + """ + # NOTE that the first bucket and last bucket will not be rotated, + # see lru_cache_helper module for more details. + for target_bucket in list(cfg.BUCKET_FILE_SIZE_DICT)[1:-1]: + entries_to_be_removed = await lru_helper.rotate_cache(target_bucket) + assert entries_to_be_removed is not None and len(entries_to_be_removed) != 0 class TestOngoingCachingRegister: """ + Testing multiple access to a single resource at the same time with ongoing_cache control. + NOTE; currently this test only testing the weakref implementation part, the file descriptor management part is tested in test_ota_proxy_server """ @@ -108,7 +156,7 @@ class TestOngoingCachingRegister: WORKS_NUM = 128 @pytest.fixture(autouse=True) - def setup_test(self, tmp_path: Path): + async def setup_test(self, tmp_path: Path): self.base_dir = tmp_path / "base_dir" self.base_dir.mkdir(parents=True, exist_ok=True) self.register = CachingRegister(self.base_dir) @@ -128,7 +176,7 @@ async def _wait_for_registeration_finish(self): async def _worker( self, idx: int, - ) -> Tuple[bool, Optional[CacheMeta]]: + ) -> tuple[bool, Optional[CacheMeta]]: """ Returns tuple of bool indicates whether the worker is writter, and CacheMeta from tracker. @@ -149,7 +197,11 @@ async def _worker( logger.info(f"#{idx} is provider") # NOTE: use last_access field to store worker index # NOTE 2: bypass provider_start method, directly set tracker property - _tracker.meta = CacheMeta(last_access=idx) + _tracker.meta = CacheMeta( + last_access=idx, + url="some_url", + file_sha256="some_filesha256_value", + ) _tracker._writer_ready.set() # simulate waiting for writer finished downloading await self.writer_done_event.wait() @@ -167,15 +219,19 @@ async def _worker( _tracker._subscriber_ref_holder.pop() return False, _tracker.meta - async def test_OngoingCachingRegister(self): - coros: List[Coroutine] = [] + async def test_ongoing_cache_register(self): + """ + Test multiple access to single resource with ongoing_cache control mechanism. + """ + coros: list[Coroutine] = [] for idx in range(self.WORKS_NUM): coros.append(self._worker(idx)) + random.shuffle(coros) # shuffle the corotines to simulate unordered access tasks = [asyncio.create_task(c) for c in coros] logger.info(f"{self.WORKS_NUM} workers have been dispatched") - # start all the worker + # start all the worker, all the workers will now access the same resouce. self.sync_event.set() logger.info("all workers start to subscribe to the register") await self._wait_for_registeration_finish() # wait for all workers finish subscribing @@ -183,7 +239,8 @@ async def test_OngoingCachingRegister(self): ###### check the test result ###### meta_set, writer_meta = set(), None - for is_writer, meta in await asyncio.gather(*tasks): + for _fut in asyncio.as_completed(tasks): + is_writer, meta = await _fut if meta is None: logger.warning( "encount edge condition that subscriber subscribes " diff --git a/tests/test_ota_proxy/test_ota_proxy_server.py b/tests/test_ota_proxy/test_ota_proxy_server.py index 361ef9868..cfc65b496 100644 --- a/tests/test_ota_proxy/test_ota_proxy_server.py +++ b/tests/test_ota_proxy/test_ota_proxy_server.py @@ -13,6 +13,8 @@ # limitations under the License. +from __future__ import annotations + import asyncio import logging import random @@ -20,7 +22,6 @@ import time from hashlib import sha256 from pathlib import Path -from typing import List from urllib.parse import quote, unquote, urljoin import aiohttp @@ -158,8 +159,8 @@ async def launch_ota_proxy_server(self, setup_ota_proxy_server): pass # ignore exp on shutting down @pytest.fixture(scope="class") - def parse_regulars(self): - regular_entries: List[RegularInf] = [] + def parse_regulars(self) -> list[RegularInf]: + regular_entries: list[RegularInf] = [] with open(self.REGULARS_TXT_PATH, "r") as f: for _line in f: _entry = parse_regulars_from_txt(_line) @@ -221,7 +222,9 @@ async def test_download_file_with_special_fname(self): elif self.space_availability == "exceed_hard_limit": pass - async def ota_image_downloader(self, regular_entries, sync_event: asyncio.Event): + async def ota_image_downloader( + self, regular_entries: list[RegularInf], sync_event: asyncio.Event + ): """Test single client download the whole ota image.""" async with aiohttp.ClientSession() as session: await sync_event.wait() @@ -258,16 +261,21 @@ async def ota_image_downloader(self, regular_entries, sync_event: asyncio.Event) continue raise - async def test_multiple_clients_download_ota_image(self, parse_regulars): + async def test_multiple_clients_download_ota_image( + self, parse_regulars: list[RegularInf] + ): """Test multiple client download the whole ota image simultaneously.""" # ------ dispatch many clients to download from otaproxy simultaneously ------ # # --- execution --- # sync_event = asyncio.Event() - tasks: List[asyncio.Task] = [] + tasks: list[asyncio.Task] = [] for _ in range(self.CLIENTS_NUM): tasks.append( asyncio.create_task( - self.ota_image_downloader(parse_regulars, sync_event) + self.ota_image_downloader( + parse_regulars, + sync_event, + ) ) ) logger.info( @@ -277,7 +285,9 @@ async def test_multiple_clients_download_ota_image(self, parse_regulars): # --- assertions --- # # 1. ensure all clients finished the downloading successfully - await asyncio.gather(*tasks, return_exceptions=False) + for _fut in asyncio.as_completed(tasks): + await _fut + await self.otaproxy_inst.shutdown() # 2. check there is no tmp files left in the ota_cache dir # ensure that the gc for multi-cache-streaming works @@ -338,7 +348,7 @@ async def setup_ota_proxy_server(self, tmp_path: Path): @pytest.fixture(scope="class") def parse_regulars(self): - regular_entries: List[RegularInf] = [] + regular_entries: list[RegularInf] = [] with open(self.REGULARS_TXT_PATH, "r") as f: for _line in f: _entry = parse_regulars_from_txt(_line) @@ -369,7 +379,7 @@ async def test_multiple_clients_download_ota_image(self, parse_regulars): # ------ dispatch many clients to download from otaproxy simultaneously ------ # # --- execution --- # sync_event = asyncio.Event() - tasks: List[asyncio.Task] = [] + tasks: list[asyncio.Task] = [] for _ in range(self.CLIENTS_NUM): tasks.append( asyncio.create_task( From e41425f55e035307a50d8211c272c08352e220cb Mon Sep 17 00:00:00 2001 From: Bodong Yang <86948717+Bodong-Yang@users.noreply.github.com> Date: Sat, 3 Aug 2024 01:26:04 +0900 Subject: [PATCH 098/193] bump to use simple-sqlite3-orm v0.2.1 (#369) Bumps simple-sqlite3-orm to v0.2.1. Upstream fixes a problem related to using both RETURNING and LIMIT in DELETE stmt, see pga2rn/simple-sqlite3-orm#19 for more details. Bumps the version fixes otaproxy not working on ubuntu 22.04 and newer. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4dd1bc528..0c3486255 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ dependencies = [ "pyopenssl<25,>=24.1", "pyyaml<7,>=6.0.1", "requests<2.33,>=2.32", - "simple-sqlite3-orm @ https://github.com/pga2rn/simple-sqlite3-orm/releases/download/v0.2.0/simple_sqlite3_orm-0.2.0-py3-none-any.whl", + "simple-sqlite3-orm @ https://github.com/pga2rn/simple-sqlite3-orm/releases/download/v0.2.1/simple_sqlite3_orm-0.2.1-py3-none-any.whl", "typing-extensions>=4.6.3", "urllib3<2.3,>=2.2.2", "uvicorn[standard]<0.31,>=0.30", From 800400ed6159f085178c78cabe3dac825e4927e5 Mon Sep 17 00:00:00 2001 From: Bodong Yang <86948717+Bodong-Yang@users.noreply.github.com> Date: Mon, 5 Aug 2024 08:04:57 +0900 Subject: [PATCH 099/193] ci: introduce test for multiple OS version(ubuntu-18.04 and ubuntu-22.04) (#368) This PR introduces the support for running tests on multiple OS version by allowing specifying the base image of test image dockerfile, and resolving any hardcoded logic related to using ubuntu-20.04. --- .github/workflows/test.yaml | 16 ++- docker/build_image/Dockerfile | 27 ---- docker/docker-compose_tests.yml | 38 +++++- docker/test_base/Dockerfile | 74 +++++++---- docker/test_base/Dockerfile_ubuntu-18.04 | 115 ++++++++++++++++++ tests/conftest.py | 39 ++++-- tests/keys/gen_certs.sh | 4 +- tests/test_ota_proxy/test_ota_proxy_server.py | 28 +++-- 8 files changed, 259 insertions(+), 82 deletions(-) delete mode 100644 docker/build_image/Dockerfile create mode 100644 docker/test_base/Dockerfile_ubuntu-18.04 diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index c9cbd9212..7315f953e 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,8 +16,15 @@ on: jobs: pytest_with_coverage: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + test_base: + - ubuntu-18.04 + - ubuntu-20.04 + - ubuntu-22.04 steps: - name: Checkout commit uses: actions/checkout@v4 @@ -26,16 +33,18 @@ jobs: fetch-depth: 0 - name: Build ota-test_base docker image run: | - docker compose -f docker/docker-compose_tests.yml build + docker compose -f docker/docker-compose_tests.yml build tester-${{ matrix.test_base }} - name: Execute pytest with coverage trace under ota-test_base container run: | mkdir -p test_result - docker compose -f docker/docker-compose_tests.yml up --abort-on-container-exit + docker compose -f docker/docker-compose_tests.yml run --rm tester-${{ matrix.test_base }} - name: SonarCloud Scan uses: SonarSource/sonarcloud-github-action@master continue-on-error: true env: SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + if: ${{ matrix.test_base == 'ubuntu-22.04' }} + # export the coverage report to the comment! - name: Add coverage report to PR comment continue-on-error: true @@ -43,3 +52,4 @@ jobs: with: pytest-xml-coverage-path: test_result/coverage.xml junitxml-path: test_result/pytest.xml + if: ${{ matrix.test_base == 'ubuntu-22.04' }} diff --git a/docker/build_image/Dockerfile b/docker/build_image/Dockerfile deleted file mode 100644 index e229047ad..000000000 --- a/docker/build_image/Dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -FROM ubuntu:20.04 -SHELL ["/bin/bash", "-c"] -ENV DEBIAN_FRONTEND=noninteractive -ARG KERNEL_VERSION="5.8.0-53-generic" -# ARG KERNEL_VERSION="5.4.0-74-generic" - -RUN apt-get update && \ - apt-get install -y \ - init systemd linux-image-${KERNEL_VERSION} \ - sudo git python3-pip vim \ - openssh-server netplan.io iputils-ping netbase isc-dhcp-client - -RUN useradd -m autoware -s /bin/bash -RUN echo autoware:autoware | chpasswd -RUN gpasswd -a autoware sudo -RUN echo 'autoware ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers.d/autoware - -COPY app /opt/ota/client/app -COPY systemd/otaclient.service /etc/systemd/system -RUN cd /etc/systemd/system/multi-user.target.wants && ln -s /etc/systemd/system/otaclient.service - -COPY app/requirements.txt /opt/ota/client/app/requirements.txt - -RUN python3 -m pip install -r /opt/ota/client/app/requirements.txt - -WORKDIR /home/autoware -USER autoware diff --git a/docker/docker-compose_tests.yml b/docker/docker-compose_tests.yml index ad574d851..ad6623711 100644 --- a/docker/docker-compose_tests.yml +++ b/docker/docker-compose_tests.yml @@ -1,9 +1,43 @@ services: - tester: + tester-ubuntu-18.04: + build: + context: ../ + dockerfile: ./docker/test_base/Dockerfile_ubuntu-18.04 + args: + - UBUNTU_BASE=ubuntu:18.04 + image: ota-test_base:ubuntu1804 + network_mode: bridge + container_name: ota-test + environment: + OUTPUT_DIR: /test_result + CERTS_DIR: /certs + volumes: + - ..:/otaclient_src:ro + - ../test_result:/test_result:rw + + tester-ubuntu-20.04: + build: + context: ../ + dockerfile: ./docker/test_base/Dockerfile + args: + - UBUNTU_BASE=ubuntu:20.04 + image: ota-test_base:ubuntu2004 + network_mode: bridge + container_name: ota-test + environment: + OUTPUT_DIR: /test_result + CERTS_DIR: /certs + volumes: + - ..:/otaclient_src:ro + - ../test_result:/test_result:rw + + tester-ubuntu-22.04: build: context: ../ dockerfile: ./docker/test_base/Dockerfile - image: ota-test_base + args: + - UBUNTU_BASE=ubuntu:22.04 + image: ota-test_base:ubuntu2204 network_mode: bridge container_name: ota-test environment: diff --git a/docker/test_base/Dockerfile b/docker/test_base/Dockerfile index da57daa1c..bcc63aa6f 100644 --- a/docker/test_base/Dockerfile +++ b/docker/test_base/Dockerfile @@ -1,17 +1,47 @@ -FROM ubuntu:20.04 +ARG UBUNTU_BASE + +# +# ------ stage 1: prepare base image ------ # +# + +FROM ${UBUNTU_BASE} AS builder + +SHELL ["/bin/bash", "-c"] +ENV DEBIAN_FRONTEND=noninteractive +ENV SPECIAL_FILE="path;adf.ae?qu.er\y=str#fragファイルement" + +# special treatment to the ota-image: create file that needs url escaping +# NOTE: include special identifiers #?; into the pathname +RUN echo -n "${SPECIAL_FILE}" > "${OTA_IMAGE_DIR}/${SPECIAL_FILE}" + +# install required packages +RUN set -eux; \ + apt-get update -qq; \ + apt-get install -y linux-image-generic; \ + apt-get clean; \ + rm -rf \ + /tmp/* \ + /var/lib/apt/lists/* \ + /var/tmp/* \ + +# +# ------ stage 2: prepare test environment ------ # +# +ARG UBUNTU_BASE + +FROM ${UBUNTU_BASE} + SHELL ["/bin/bash", "-c"] ENV DEBIAN_FRONTEND=noninteractive -ARG KERNEL_VERSION="5.8.0-53-generic" -ARG BASE_IMG_URL="http://cdimage.ubuntu.com/ubuntu-base/releases/20.04/release/ubuntu-base-20.04.1-base-amd64.tar.gz" -ARG OTA_METADATA_REPO="https://github.com/tier4/ota-metadata" + +ENV OTA_METADATA_REPO="https://github.com/tier4/ota-metadata" ENV OTA_IMAGE_SERVER_ROOT="/ota-image" ENV OTA_IMAGE_DIR="${OTA_IMAGE_SERVER_ROOT}/data" ENV CERTS_DIR="/certs" ENV SPECIAL_FILE="path;adf.ae?qu.er\y=str#fragファイルement" - # install required packages -RUN set -eu; \ +RUN set -eux; \ apt-get update -qq; \ apt-get install -y -qq --no-install-recommends \ python3-minimal \ @@ -23,36 +53,25 @@ RUN set -eu; \ gcc \ wget \ git; \ - apt-get install -y -qq linux-image-${KERNEL_VERSION} - -# prepare ubuntu base -WORKDIR ${OTA_IMAGE_SERVER_ROOT} -RUN set -eu; \ - wget -q -O /tmp/base_image.tar.gz ${BASE_IMG_URL}; \ - mkdir -p ${OTA_IMAGE_DIR}; \ - tar zxf /tmp/base_image.tar.gz -C ${OTA_IMAGE_DIR}; \ - cp -a \ - /boot/vmlinuz-${KERNEL_VERSION} \ - /boot/initrd.img-${KERNEL_VERSION} \ - /boot/config-${KERNEL_VERSION} \ - /boot/System.map-${KERNEL_VERSION} ${OTA_IMAGE_DIR}/boot - -# special treatment to the ota-image: create file that needs url escaping -# NOTE: include special identifiers #?; into the pathname -RUN echo -n "${SPECIAL_FILE}" > "${OTA_IMAGE_DIR}/${SPECIAL_FILE}" + apt-get install -y -qq linux-image-generic; \ + apt-get clean # install hatch -RUN set -eu; \ +RUN set -eux; \ python3 -m pip install --no-cache-dir -q -U pip; \ python3 -m pip install --no-cache-dir -U hatch +WORKDIR ${OTA_IMAGE_SERVER_ROOT} + +COPY --from=builder / /${OTA_IMAGE_DIR} + # generate test certs and sign key COPY --chmod=755 ./tests/keys/gen_certs.sh /tmp/certs/ RUN set -eu; \ mkdir -p "${CERTS_DIR}"; \ pushd /tmp/certs; \ ./gen_certs.sh; \ - cp * "${CERTS_DIR}"; \ + cp ./* "${CERTS_DIR}"; \ popd # build the test OTA image @@ -62,8 +81,11 @@ RUN set -eu; \ git clone ${OTA_METADATA_REPO}; \ python3 -m venv ota-metadata/.venv; \ source ota-metadata/.venv/bin/activate; \ + python3 -m pip install --no-cache-dir -U pip; \ python3 -m pip install --no-cache-dir -q \ -r ota-metadata/metadata/ota_metadata/requirements.txt; \ + # patch the ignore files + echo "" > ota-metadata/metadata/ignore.txt; \ python3 ota-metadata/metadata/ota_metadata/metadata_gen.py \ --target-dir data \ --compressed-dir data.zst \ @@ -77,7 +99,7 @@ RUN set -eu; \ cp ota-metadata/metadata/persistents.txt . # cleanup -RUN set -eu; \ +RUN set -eux; \ apt-get clean; \ rm -rf \ /tmp/* \ diff --git a/docker/test_base/Dockerfile_ubuntu-18.04 b/docker/test_base/Dockerfile_ubuntu-18.04 new file mode 100644 index 000000000..6eb266ecc --- /dev/null +++ b/docker/test_base/Dockerfile_ubuntu-18.04 @@ -0,0 +1,115 @@ +ARG UBUNTU_BASE + +# +# ------ stage 1: prepare base image ------ # +# + +FROM ${UBUNTU_BASE} AS builder + +SHELL ["/bin/bash", "-c"] +ENV DEBIAN_FRONTEND=noninteractive +ENV SPECIAL_FILE="path;adf.ae?qu.er\y=str#fragファイルement" + +# special treatment to the ota-image: create file that needs url escaping +# NOTE: include special identifiers #?; into the pathname +RUN echo -n "${SPECIAL_FILE}" > "/${SPECIAL_FILE}" + +# install required packages +RUN set -eux; \ + apt-get update -qq; \ + apt-get install -y linux-image-generic; \ + apt-get clean; \ + rm -rf \ + /tmp/* \ + /var/lib/apt/lists/* \ + /var/tmp/* \ + +# +# ------ stage 2: prepare test environment ------ # +# +ARG UBUNTU_BASE + +FROM ${UBUNTU_BASE} + +SHELL ["/bin/bash", "-c"] +ENV DEBIAN_FRONTEND=noninteractive + +ENV OTA_METADATA_REPO="https://github.com/tier4/ota-metadata" +ENV OTA_IMAGE_SERVER_ROOT="/ota-image" +ENV OTA_IMAGE_DIR="${OTA_IMAGE_SERVER_ROOT}/data" +ENV CERTS_DIR="/certs" +ENV SPECIAL_FILE="path;adf.ae?qu.er\y=str#fragファイルement" + +# install required packages +RUN set -eux; \ + apt-get update -qq; \ + apt-get install -y -qq --no-install-recommends \ + python3-minimal \ + python3-pip \ + python3-venv \ + python3-dev \ + libcurl4-openssl-dev \ + libssl-dev \ + gcc \ + wget \ + git \ + python3.8-venv; \ + apt-get install -y -qq linux-image-generic; \ + apt-get clean + +# install hatch to the system +RUN set -eux; \ + python3.8 -m pip install --no-cache-dir -q -U pip; \ + python3.8 -m pip install --no-cache-dir -U hatch + +WORKDIR ${OTA_IMAGE_SERVER_ROOT} + +COPY --from=builder / /${OTA_IMAGE_DIR} + +# generate test certs and sign key +COPY --chmod=755 ./tests/keys/gen_certs.sh /tmp/certs/ +RUN set -eux; \ + mkdir -p "${CERTS_DIR}"; \ + pushd /tmp/certs; \ + ./gen_certs.sh; \ + cp ./* "${CERTS_DIR}"; \ + popd + +# build the test OTA image +RUN set -eux; \ + cp "${CERTS_DIR}"/sign.key sign.key; \ + cp "${CERTS_DIR}"/sign.pem sign.pem; \ + git clone ${OTA_METADATA_REPO}; \ + python3.8 -m venv ota-metadata/.venv; \ + source ota-metadata/.venv/bin/activate; \ + python3.8 -m pip install --no-cache-dir -U pip; \ + python3.8 -m pip install --no-cache-dir -q \ + -r ota-metadata/metadata/ota_metadata/requirements.txt; \ + # patch the ignore files + echo "" > ota-metadata/metadata/ignore.txt; \ + python3.8 ota-metadata/metadata/ota_metadata/metadata_gen.py \ + --target-dir data \ + --compressed-dir data.zst \ + --ignore-file ota-metadata/metadata/ignore.txt; \ + python3.8 ota-metadata/metadata/ota_metadata/metadata_sign.py \ + --sign-key sign.key \ + --cert-file sign.pem \ + --persistent-file ota-metadata/metadata/persistents.txt \ + --rootfs-directory data \ + --compressed-rootfs-directory data.zst; \ + cp ota-metadata/metadata/persistents.txt . + +# cleanup +RUN set -eux; \ + apt-get clean; \ + rm -rf \ + /tmp/* \ + /var/lib/apt/lists/* \ + /var/tmp/* \ + ota-metadata + +# copy and setup the entry_point.sh +COPY ./docker/test_base/entry_point.sh /entry_point.sh +RUN chmod +x /entry_point.sh + +ENTRYPOINT [ "/entry_point.sh" ] diff --git a/tests/conftest.py b/tests/conftest.py index 86b75a7a1..6460d80af 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -30,6 +30,21 @@ TEST_DIR = Path(__file__).parent +# see test base Dockerfile for more details. +OTA_IMAGE_DIR = Path("/ota-image") +KERNEL_PREFIX = "vmlinuz" +INITRD_PREFIX = "initrd.img" + + +def _get_kernel_version() -> str: + boot_dir = OTA_IMAGE_DIR / "data/boot" + _kernel_pa = f"{KERNEL_PREFIX}-*" + _kernel = list(boot_dir.glob(_kernel_pa))[0] + return _kernel.name.split("-", maxsplit=1)[1] + + +KERNEL_VERSION = _get_kernel_version() + @dataclass class TestConfiguration: @@ -52,7 +67,7 @@ class TestConfiguration: OTA_IMAGE_SERVER_ADDR = "127.0.0.1" OTA_IMAGE_SERVER_PORT = 8080 OTA_IMAGE_URL = f"http://{OTA_IMAGE_SERVER_ADDR}:{OTA_IMAGE_SERVER_PORT}" - KERNEL_VERSION = "5.8.0-53-generic" + KERNEL_VERSION = str(KERNEL_VERSION) CURRENT_VERSION = "123.x" UPDATE_VERSION = "789.x" @@ -132,6 +147,7 @@ def ab_slots(tmp_path_factory: pytest.TempPathFactory) -> SlotMeta: Return: A tuple includes the path to A/B slots respectly. """ + logger.info("creating ab_slots for testing ...") # prepare slot_a slot_a = tmp_path_factory.mktemp("slot_a") shutil.copytree( @@ -140,17 +156,20 @@ def ab_slots(tmp_path_factory: pytest.TempPathFactory) -> SlotMeta: # simulate the diff between versions shutil.move(str(slot_a / "var"), slot_a / "var_old") shutil.move(str(slot_a / "usr"), slot_a / "usr_old") - # boot dir is a separated folder, so delete the boot folder under slot_a - # shutil.rmtree(slot_a / "boot", ignore_errors=True) + # manually create symlink to kernel and initrd.img vmlinuz_symlink = slot_a / "boot" / TestConfiguration.KERNEL_PREFIX - vmlinuz_symlink.symlink_to( - f"{TestConfiguration.KERNEL_PREFIX}-{TestConfiguration.KERNEL_VERSION}" - ) initrd_symlink = slot_a / "boot" / TestConfiguration.INITRD_PREFIX - initrd_symlink.symlink_to( - f"{TestConfiguration.INITRD_PREFIX}-{TestConfiguration.KERNEL_VERSION}" - ) + + try: + vmlinuz_symlink.symlink_to( + f"{TestConfiguration.KERNEL_PREFIX}-{TestConfiguration.KERNEL_VERSION}" + ) + initrd_symlink.symlink_to( + f"{TestConfiguration.INITRD_PREFIX}-{TestConfiguration.KERNEL_VERSION}" + ) + except FileExistsError: + pass # prepare slot_b slot_b = tmp_path_factory.mktemp("slot_b") @@ -165,6 +184,8 @@ def ab_slots(tmp_path_factory: pytest.TempPathFactory) -> SlotMeta: slot_b_boot_dev = tmp_path_factory.mktemp("slot_b_boot") slot_b_boot_dir = slot_b_boot_dev / "boot" slot_b_boot_dir.mkdir() + (slot_b_boot_dir / "grub").mkdir() + return SlotMeta( slot_a=str(slot_a), slot_b=str(slot_b), diff --git a/tests/keys/gen_certs.sh b/tests/keys/gen_certs.sh index e9efca02a..0506527ad 100644 --- a/tests/keys/gen_certs.sh +++ b/tests/keys/gen_certs.sh @@ -2,7 +2,7 @@ # https://stackoverflow.com/questions/52500165/problem-verifying-a-self-created-openssl-root-intermediate-and-end-user-certifi -set -e +set -eux # Root CA: openssl ecparam -out root.key -name prime256v1 -genkey @@ -53,4 +53,4 @@ openssl x509 -req \ -out sign.pem \ -sha256 -CAcreateserial -rm root.key interm.key interm.csr sign.csr *.srl +rm -f root.key interm.key interm.csr sign.csr *.srl diff --git a/tests/test_ota_proxy/test_ota_proxy_server.py b/tests/test_ota_proxy/test_ota_proxy_server.py index cfc65b496..c30a5e06a 100644 --- a/tests/test_ota_proxy/test_ota_proxy_server.py +++ b/tests/test_ota_proxy/test_ota_proxy_server.py @@ -60,7 +60,7 @@ class TestOTAProxyServer(ThreadpoolExecutorFixtureMixin): OTA_IMAGE_URL = f"http://{cfg.OTA_IMAGE_SERVER_ADDR}:{cfg.OTA_IMAGE_SERVER_PORT}" OTA_PROXY_URL = f"http://{cfg.OTA_PROXY_SERVER_ADDR}:{cfg.OTA_PROXY_SERVER_PORT}" REGULARS_TXT_PATH = f"{cfg.OTA_IMAGE_DIR}/regulars.txt" - CLIENTS_NUM = 6 + CLIENTS_NUM = 3 @pytest.fixture( params=[ @@ -152,11 +152,12 @@ async def launch_ota_proxy_server(self, setup_ota_proxy_server): await asyncio.sleep(1) # wait before otaproxy server is ready yield finally: - shutil.rmtree(self.ota_cache_dir, ignore_errors=True) try: await self.otaproxy_inst.shutdown() except Exception: pass # ignore exp on shutting down + finally: + shutil.rmtree(self.ota_cache_dir, ignore_errors=True) @pytest.fixture(scope="class") def parse_regulars(self) -> list[RegularInf]: @@ -229,16 +230,17 @@ async def ota_image_downloader( async with aiohttp.ClientSession() as session: await sync_event.wait() await asyncio.sleep(random.randrange(100, 200) // 100) + for entry in regular_entries: url = urljoin( cfg.OTA_IMAGE_URL, quote(f'/data/{entry.relative_to("/")}') ) - _retry_count_for_exceed_hard_limit = 0 - # NOTE: for space_availability==exceed_hard_limit, + _retry_count = 0 + _max_retry = 6 + # NOTE: for space_availability==exceed_hard_limit or below_hard_limit, # it is normal that transition is interrupted when - # space_availability transfered from below_hard_limit to exceed_hard_limit. - # Another try is allowed for this case. + # space_availability status transfered. while True: async with session.get( url, @@ -253,13 +255,13 @@ async def ota_image_downloader( assert hash_f.digest() == entry.sha256hash break except AssertionError: - _retry_count_for_exceed_hard_limit += 1 - if ( - self.space_availability == "exceed_hard_limit" - and _retry_count_for_exceed_hard_limit <= 1 - ): - continue - raise + _retry_count += 1 + if _retry_count > _max_retry: + logger.error(f"failed on {entry}") + raise + logger.warning( + f"failed on {entry}, {_retry_count=}, still retry..." + ) async def test_multiple_clients_download_ota_image( self, parse_regulars: list[RegularInf] From 3224a5bcc6dbf1e00bf35f1f586d0aa166ec7420 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Aug 2024 09:42:18 +0900 Subject: [PATCH 100/193] build(deps): Update aiohttp requirement (#373) Updates the requirements on [aiohttp](https://github.com/aio-libs/aiohttp) to permit the latest version. - [Release notes](https://github.com/aio-libs/aiohttp/releases) - [Changelog](https://github.com/aio-libs/aiohttp/blob/master/CHANGES.rst) - [Commits](https://github.com/aio-libs/aiohttp/compare/v3.9.5...v3.10.1) --- updated-dependencies: - dependency-name: aiohttp dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0c3486255..59b857fa3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ dynamic = [ ] dependencies = [ "aiofiles<25,>=24.1", - "aiohttp<3.10,>=3.9.5", + "aiohttp>=3.9.5,<3.11", "cryptography>=42.0.4,<44", "grpcio<1.54,>=1.53.2", "protobuf<4.22,>=4.21.12", From 6b82fd40a6b6f261799bb9f60bbaa0f17296b884 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 6 Aug 2024 09:44:11 +0900 Subject: [PATCH 101/193] [pre-commit.ci] pre-commit autoupdate (#371) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pycqa/flake8: 7.1.0 → 7.1.1](https://github.com/pycqa/flake8/compare/7.1.0...7.1.1) - [github.com/tox-dev/pyproject-fmt: 2.1.3 → 2.2.1](https://github.com/tox-dev/pyproject-fmt/compare/2.1.3...2.2.1) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Bodong Yang <86948717+Bodong-Yang@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e4d10418d..a44ba29b6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,7 +22,7 @@ repos: hooks: - id: isort - repo: https://github.com/pycqa/flake8 - rev: 7.1.0 + rev: 7.1.1 hooks: - id: flake8 additional_dependencies: @@ -30,7 +30,7 @@ repos: - flake8-comprehensions - flake8-simplify - repo: https://github.com/tox-dev/pyproject-fmt - rev: "2.1.3" + rev: "2.2.1" hooks: - id: pyproject-fmt # https://pyproject-fmt.readthedocs.io/en/latest/#calculating-max-supported-python-version From 21a165403f2739ea85d1e3a758e15fdb3dc6c88f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Aug 2024 09:48:22 +0900 Subject: [PATCH 102/193] build(deps): Bump zstandard from >=0.22,<0.23 to >=0.22,<0.24 (#356) * build(deps): Bump zstandard from >=0.22,<0.23 to >=0.22,<0.24 [zstandard](https://github.com/indygreg/python-zstandard) 0.23.0 release. - [Release notes](https://github.com/indygreg/python-zstandard/releases) - [Changelog](https://github.com/indygreg/python-zstandard/blob/main/docs/news.rst) - [Commits](https://github.com/indygreg/python-zstandard/compare/0.22.0...0.23.0) --- updated-dependencies: - dependency-name: zstandard dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Bodong Yang <86948717+Bodong-Yang@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 59b857fa3..08d423a84 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,7 @@ dependencies = [ "typing-extensions>=4.6.3", "urllib3<2.3,>=2.2.2", "uvicorn[standard]<0.31,>=0.30", - "zstandard<0.23,>=0.22", + "zstandard<0.24,>=0.22", ] optional-dependencies.dev = [ "black", From ae52f67e2ffea01b46d04b8ad208de617d451d36 Mon Sep 17 00:00:00 2001 From: Bodong Yang <86948717+Bodong-Yang@users.noreply.github.com> Date: Wed, 7 Aug 2024 13:11:37 +0900 Subject: [PATCH 103/193] refactor(otaproxy): refine the implementation of cache streaming (#370) This PR refines the implementation of otaproxy.cache_streaming module, especially the CacheTracker implementation. The refined implementation is simpler with less complexities than previous implementation, and fixes some minor issues related to some edge conditions. Major changes * cache_streaming: now tracker labels itself as finished as soon as the caching is finished to speed up releasing the subscribers. * cache_streaming: now provider sets the tracker ready at the first chunk being received. * cache_streaming: simplify the use of weakref for tracker auto finalization. * cache_streaming: remove the use of ambiguous sleep_with_backoff, now it normally wait with backoff, and exit on reaching maximum retries. * ota_cache: retrieve_file_by_cache: now wait for at most 0.255 seconds for the cache file becomes ready. Other change * switch to use simple-sqlite3-orm from pypi. --- pyproject.toml | 2 +- src/ota_proxy/cache_streaming.py | 421 +++++++----------- src/ota_proxy/config.py | 3 - src/ota_proxy/ota_cache.py | 60 +-- src/ota_proxy/utils.py | 23 - tests/test_ota_proxy/test_cache_streaming.py | 161 +++++++ tests/test_ota_proxy/test_ota_cache.py | 122 +---- ..._proxy_server.py => test_ota_proxy_e2e.py} | 0 8 files changed, 367 insertions(+), 425 deletions(-) create mode 100644 tests/test_ota_proxy/test_cache_streaming.py rename tests/test_ota_proxy/{test_ota_proxy_server.py => test_ota_proxy_e2e.py} (100%) diff --git a/pyproject.toml b/pyproject.toml index 08d423a84..765cb45f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ dependencies = [ "pyopenssl<25,>=24.1", "pyyaml<7,>=6.0.1", "requests<2.33,>=2.32", - "simple-sqlite3-orm @ https://github.com/pga2rn/simple-sqlite3-orm/releases/download/v0.2.1/simple_sqlite3_orm-0.2.1-py3-none-any.whl", + "simple-sqlite3-orm<0.3,>=0.2", "typing-extensions>=4.6.3", "urllib3<2.3,>=2.2.2", "uvicorn[standard]<0.31,>=0.30", diff --git a/src/ota_proxy/cache_streaming.py b/src/ota_proxy/cache_streaming.py index 7a519326e..48237dc0d 100644 --- a/src/ota_proxy/cache_streaming.py +++ b/src/ota_proxy/cache_streaming.py @@ -24,22 +24,13 @@ import weakref from concurrent.futures import Executor from pathlib import Path -from typing import ( - AsyncGenerator, - AsyncIterator, - Callable, - Coroutine, - Generic, - List, - MutableMapping, - Optional, - Tuple, - TypeVar, - Union, -) +from typing import AsyncGenerator, AsyncIterator, Callable, Coroutine import aiofiles +from otaclient_common.common import get_backoff +from otaclient_common.typing import StrOrPath + from .config import config as cfg from .db import CacheMeta from .errors import ( @@ -48,16 +39,18 @@ CacheStreamingInterrupt, StorageReachHardLimit, ) -from .utils import wait_with_backoff logger = logging.getLogger(__name__) -_WEAKREF = TypeVar("_WEAKREF") - # cache tracker -class CacheTracker(Generic[_WEAKREF]): +def _tracker_finalizer(tmp_fpath: Path): + """Finalizer that cleans up the tmp file when this tracker is gced.""" + tmp_fpath.unlink(missing_ok=True) + + +class CacheTracker: """A tracker for an ongoing cache entry. This tracker represents an ongoing cache entry under the , @@ -82,7 +75,10 @@ class CacheTracker(Generic[_WEAKREF]): finish the caching. """ - READER_SUBSCRIBE_WAIT_PROVIDER_TIMEOUT = 2 + SUBSCRIBER_WAIT_PROVIDER_READY_MAX_RETRY = 16 # 9s + SUBSCRIBER_WAIT_NEXT_CHUNK_MAX_RETRY = 16 # 9s + SUBSCRIBER_WAIT_BACKOFF_FACTOR = 0.01 + SUBSCRIBER_WAIT_BACKOFF_MAX = 1 FNAME_PART_SEPARATOR = "_" @classmethod @@ -102,51 +98,33 @@ def _tmp_file_naming(cls, cache_identifier: str) -> str: def __init__( self, cache_identifier: str, - ref_holder: _WEAKREF, *, - base_dir: Union[str, Path], + base_dir: StrOrPath, + commit_cache_cb: _CACHE_ENTRY_REGISTER_CALLBACK, executor: Executor, - callback: _CACHE_ENTRY_REGISTER_CALLBACK, below_hard_limit_event: threading.Event, ): self.fpath = Path(base_dir) / self._tmp_file_naming(cache_identifier) - self.meta: CacheMeta | None = None - self.cache_identifier = cache_identifier self.save_path = Path(base_dir) / cache_identifier + self.cache_meta: CacheMeta | None = None + self._commit_cache_cb = commit_cache_cb + self._writer_ready = asyncio.Event() self._writer_finished = asyncio.Event() self._writer_failed = asyncio.Event() - self._ref = ref_holder - self._subscriber_ref_holder: List[_WEAKREF] = [] + self._executor = executor - self._cache_commit_cb = callback self._space_availability_event = below_hard_limit_event self._bytes_written = 0 - self._cache_write_gen: Optional[AsyncGenerator[int, bytes]] = None # self-register the finalizer to this tracker weakref.finalize( self, - self.finalizer, - fpath=self.fpath, + _tracker_finalizer, + self.fpath, ) - async def _finalize_caching(self): - """Commit cache entry to db and rename tmp cached file with sha256hash - to fialize the caching.""" - # if the file with the same sha256has is already presented, skip the hardlink - # NOTE: no need to clean the tmp file, it will be done by the cache tracker. - assert self.meta is not None - await self._cache_commit_cb(self.meta) - if not self.save_path.is_file(): - self.fpath.link_to(self.save_path) - - @staticmethod - def finalizer(*, fpath: Union[str, Path]): - """Finalizer that cleans up the tmp file when this tracker is gced.""" - Path(fpath).unlink(missing_ok=True) - @property def writer_failed(self) -> bool: return self._writer_failed.is_set() @@ -155,64 +133,64 @@ def writer_failed(self) -> bool: def writer_finished(self) -> bool: return self._writer_finished.is_set() - @property - def is_cache_valid(self) -> bool: - """Indicates whether the temp cache entry for this tracker is valid.""" - return ( - self.meta is not None - and self._writer_finished.is_set() - and not self._writer_failed.is_set() - ) - - def get_cache_write_gen(self) -> Optional[AsyncGenerator[int, bytes]]: - if self._cache_write_gen: - return self._cache_write_gen - logger.warning(f"tracker for {self.meta} is not ready, skipped") + def set_writer_failed(self) -> None: # pragma: no cover + self._writer_finished.set() + self._writer_failed.set() - async def _provider_write_cache(self) -> AsyncGenerator[int, bytes]: + async def _provider_write_cache( + self, cache_meta: CacheMeta + ) -> AsyncGenerator[int, bytes]: """Provider writes data chunks from upper caller to tmp cache file. If cache writing failed, this method will exit and tracker.writer_failed and tracker.writer_finished will be set. """ - logger.debug(f"start to cache for {self.meta=}...") + logger.debug(f"start to cache for {cache_meta=}...") try: - if not self.meta: - raise ValueError("called before provider tracker is ready, abort") - async with aiofiles.open(self.fpath, "wb", executor=self._executor) as f: - # let writer become ready when file is open successfully - self._writer_ready.set() - _written = 0 while _data := (yield _written): if not self._space_availability_event.is_set(): - logger.warning( - f"abort writing cache for {self.meta=}: {StorageReachHardLimit.__name__}" - ) - self._writer_failed.set() - return + _err_msg = f"abort writing cache for {cache_meta=}: {StorageReachHardLimit.__name__}" + logger.warning(_err_msg) + raise StorageReachHardLimit(_err_msg) _written = await f.write(_data) - self._bytes_written += _written - logger.debug( - "cache write finished, total bytes written" - f"({self._bytes_written}) for {self.meta=}" - ) - self.meta.cache_size = self._bytes_written + # set the writer is ready on first chunk of data written, + # signal the subcriber that the cache stream starts. + if not self._writer_ready.is_set(): + self._writer_ready.set() - # NOTE: no need to track the cache commit, - # as writer_finish is meant to set on cache file created. - asyncio.create_task(self._finalize_caching()) + self._bytes_written += _written + + # logger.debug( + # "cache write finished, total bytes written" + # f"({self._bytes_written}) for {self.meta=}" + # ) + # NOTE(20240805): mark the writer succeeded in advance to release the + # subscriber faster. Whether the database entry is committed or not + # doesn't matter here, the subscriber doesn't need to fail if caching + # finished but db commit failed. + self._writer_finished.set() + cache_meta.cache_size = self._bytes_written + + # commit the cache meta to the database + await self._commit_cache_cb(cache_meta) + # finalize the cache file, skip finalize if the target file is + # already presented. + if not self.save_path.is_file(): + os.link(self.fpath, self.save_path) except Exception as e: - logger.exception(f"failed to write cache for {self.meta=}: {e!r}") + logger.warning(f"failed to write cache for {cache_meta=}: {e!r}") self._writer_failed.set() finally: + # NOTE: always unblocked the subscriber waiting for writer ready/finished self._writer_finished.set() - self._ref = None + self._writer_ready.set() + self = None # remove the ref to tracker - async def _subscribe_cache_streaming(self) -> AsyncIterator[bytes]: + async def _subscriber_stream_cache(self) -> AsyncIterator[bytes]: """Subscriber keeps polling chunks from ongoing tmp cache file. Subscriber will keep polling until the provider fails or @@ -222,71 +200,49 @@ async def _subscribe_cache_streaming(self) -> AsyncIterator[bytes]: CacheMultipleStreamingFailed if provider failed or timeout reading data chunk from tmp cache file(might be caused by a dead provider). """ + err_count, _bytes_read = 0, 0 try: - err_count, _bytes_read = 0, 0 async with aiofiles.open(self.fpath, "rb", executor=self._executor) as f: - while not (self.writer_finished and _bytes_read == self._bytes_written): - if self.writer_failed: - raise CacheMultiStreamingFailed( - f"provider aborted for {self.meta}" + while ( + not self._writer_finished.is_set() + or _bytes_read < self._bytes_written + ): + if self._writer_failed.is_set(): + raise CacheStreamingInterrupt( + f"abort reading stream on provider failed: {self.cache_meta}" ) - _bytes_read += len(_chunk := await f.read(cfg.CHUNK_SIZE)) - if _chunk: + + if _chunk := await f.read(cfg.CHUNK_SIZE): err_count = 0 + _bytes_read += len(_chunk) yield _chunk continue - err_count += 1 - if not await wait_with_backoff( - err_count, - _backoff_factor=cfg.STREAMING_BACKOFF_FACTOR, - _backoff_max=cfg.STREAMING_BACKOFF_MAX, - ): + # no data chunk is read, wait with backoff for the next + # data chunk written by the provider. + if err_count > self.SUBSCRIBER_WAIT_NEXT_CHUNK_MAX_RETRY: # abort caching due to potential dead streaming coro - _err_msg = f"failed to stream({self.meta=}): timeout getting data, partial read might happen" - logger.error(_err_msg) - # signal streamer to stop streaming + _err_msg = ( + f"failed to read stream for {self.cache_meta}: " + "timeout getting data, partial read might happen" + ) + logger.warning(_err_msg) raise CacheMultiStreamingFailed(_err_msg) - finally: - # unsubscribe on finish - self._subscriber_ref_holder.pop() - - async def _read_cache(self) -> AsyncIterator[bytes]: - """Directly open the tmp cache entry and yield data chunks from it. - Raises: - CacheMultipleStreamingFailed if fails to read from the - cached file, this might indicate a partial written cache file. - """ - _bytes_read, _retry_count = 0, 0 - async with aiofiles.open(self.fpath, "rb", executor=self._executor) as f: - while _bytes_read < self._bytes_written: - if _data := await f.read(cfg.CHUNK_SIZE): - _retry_count = 0 - _bytes_read += len(_data) - yield _data - continue - - # no data is read from the cache entry, - # retry sometimes to ensure all data is acquired - _retry_count += 1 - if not await wait_with_backoff( - _retry_count, - _backoff_factor=cfg.STREAMING_BACKOFF_FACTOR, - _backoff_max=cfg.STREAMING_CACHED_TMP_TIMEOUT, - ): - # abort caching due to potential dead streaming coro - _err_msg = ( - f"open_cached_tmp failed for ({self.meta=}): " - "timeout getting more data, partial written cache file detected" + await asyncio.sleep( + get_backoff( + err_count, + self.SUBSCRIBER_WAIT_BACKOFF_FACTOR, + self.SUBSCRIBER_WAIT_BACKOFF_MAX, + ) ) - logger.debug(_err_msg) - # signal streamer to stop streaming - raise CacheMultiStreamingFailed(_err_msg) + err_count += 1 + finally: + self = None # del the ref to the tracker on finished # exposed API - async def provider_start(self, meta: CacheMeta): + async def start_provider(self, cache_meta: CacheMeta) -> AsyncGenerator[int, bytes]: """Register meta to the Tracker, create tmp cache entry and get ready. Check _provider_write_cache for more details. @@ -295,57 +251,40 @@ async def provider_start(self, meta: CacheMeta): meta: inst of CacheMeta for the requested file tracked by this tracker. This meta is created by open_remote() method. """ - self.meta = meta - self._cache_write_gen = self._provider_write_cache() - await self._cache_write_gen.asend(None) # type: ignore - - async def provider_on_finished(self): - if not self.writer_finished and self._cache_write_gen: - with contextlib.suppress(StopAsyncIteration): - await self._cache_write_gen.asend(b"") - self._writer_finished.set() - self._ref = None - - async def provider_on_failed(self): - """Manually fail and stop the caching.""" - if not self.writer_finished and self._cache_write_gen: - logger.warning(f"interrupt writer coroutine for {self.meta=}") - with contextlib.suppress(StopAsyncIteration, CacheStreamingInterrupt): - await self._cache_write_gen.athrow(CacheStreamingInterrupt) - - self._writer_failed.set() - self._writer_finished.set() - self._ref = None - - async def subscriber_subscribe_tracker(self) -> Optional[AsyncIterator[bytes]]: - """Reader subscribe this tracker and get a file descriptor to get data chunks.""" + self.cache_meta = cache_meta + _gen = self._provider_write_cache(cache_meta) + # kick start the generator + await _gen.asend(None) # type: ignore + return _gen + + async def subscribe_tracker(self) -> tuple[AsyncIterator[bytes], CacheMeta] | None: + """Subscribe to this tracker and get the cache stream and cache_meta.""" _wait_count = 0 while not self._writer_ready.is_set(): + if _wait_count > self.SUBSCRIBER_WAIT_PROVIDER_READY_MAX_RETRY: + logger.warning(f"timeout waiting provider for {self.cache_meta}, abort") + return + if self._writer_failed.is_set(): + return # early break on failed provider + + await asyncio.sleep( + get_backoff( + _wait_count, + self.SUBSCRIBER_WAIT_BACKOFF_FACTOR, + self.SUBSCRIBER_WAIT_BACKOFF_MAX, + ) + ) _wait_count += 1 - if self.writer_failed or not await wait_with_backoff( - _wait_count, - _backoff_factor=cfg.STREAMING_BACKOFF_FACTOR, - _backoff_max=self.READER_SUBSCRIBE_WAIT_PROVIDER_TIMEOUT, - ): - return # timeout waiting for provider to become ready - - # subscribe on an ongoing cache - if not self.writer_finished and isinstance(self._ref, _Weakref): - self._subscriber_ref_holder.append(self._ref) - return self._subscribe_cache_streaming() - # caching just finished, try to directly read the finished cache entry - elif self.is_cache_valid: - return self._read_cache() + + if self._writer_failed.is_set() or self.cache_meta is None: + return # try to subscribe a failed stream + return self._subscriber_stream_cache(), self.cache_meta # a callback that register the cache entry indicates by input CacheMeta inst to the cache_db _CACHE_ENTRY_REGISTER_CALLBACK = Callable[[CacheMeta], Coroutine[None, None, None]] -class _Weakref: - pass - - class CachingRegister: """A tracker register that manages cache trackers. @@ -356,68 +295,39 @@ class CachingRegister: The later comes callers will become the subscriber to this tracker. """ - def __init__(self, base_dir: Union[str, Path]): - self._base_dir = Path(base_dir) - self._id_ref_dict: MutableMapping[str, _Weakref] = weakref.WeakValueDictionary() - self._ref_tracker_dict: MutableMapping[_Weakref, CacheTracker] = ( - weakref.WeakKeyDictionary() + def __init__(self): + self._id_tracker: weakref.WeakValueDictionary[str, CacheTracker] = ( + weakref.WeakValueDictionary() ) - async def get_tracker( - self, - cache_identifier: str, - *, - executor: Executor, - callback: _CACHE_ENTRY_REGISTER_CALLBACK, - below_hard_limit_event: threading.Event, - ) -> Tuple[CacheTracker, bool]: - """Get an inst of CacheTracker for the cache_identifier. - - Returns: - An inst of tracker, and a bool indicates the caller is provider(True), - or subscriber(False). + def get_tracker(self, cache_identifier: str) -> CacheTracker | None: + """Get an inst of CacheTracker for the cache_identifier if existed. + If the tracker doesn't exist, return a lock for tracker registeration. """ - _new_ref = _Weakref() - _ref = self._id_ref_dict.setdefault(cache_identifier, _new_ref) - - # subscriber - if ( - _tracker := self._ref_tracker_dict.get(_ref) - ) and not _tracker.writer_failed: - return _tracker, False - - # provider, or override a failed provider - if _ref is not _new_ref: # override a failed tracker - self._id_ref_dict[cache_identifier] = _new_ref - _ref = _new_ref - - _tracker = CacheTracker( - cache_identifier, - _ref, - base_dir=self._base_dir, - executor=executor, - callback=callback, - below_hard_limit_event=below_hard_limit_event, - ) - self._ref_tracker_dict[_ref] = _tracker - return _tracker, True + _tracker = self._id_tracker.get(cache_identifier) + if _tracker and not _tracker.writer_failed: + return _tracker + + def register_tracker(self, cache_identifier: str, tracker: CacheTracker) -> None: + """Create a register a new tracker into the register.""" + self._id_tracker[cache_identifier] = tracker async def cache_streaming( fd: AsyncIterator[bytes], - meta: CacheMeta, tracker: CacheTracker, + cache_meta: CacheMeta, ) -> AsyncIterator[bytes]: """A cache streamer that get data chunk from and tees to multiple destination. Data chunk yielded from will be teed to: 1. upper uvicorn otaproxy APP to send back to client, - 2. cache_tracker cache_write_gen for caching. + 2. cache_tracker cache_write_gen for caching to local. Args: fd: opened connection to a remote file. - meta: meta data of the requested resource. tracker: an inst of ongoing cache tracker bound to this request. + cache_meta: meta data of the requested resource. Returns: A bytes async iterator to yield data chunk from, for upper otaproxy uvicorn APP. @@ -425,33 +335,44 @@ async def cache_streaming( Raises: CacheStreamingFailed if any exception happens during retrieving. """ - - async def _inner(): - _cache_write_gen = tracker.get_cache_write_gen() - try: - # tee the incoming chunk to two destinations - async for chunk in fd: - # NOTE: for aiohttp, when HTTP chunk encoding is enabled, - # an empty chunk will be sent to indicate the EOF of stream, - # we MUST handle this empty chunk. - if not chunk: # skip if empty chunk is read from remote - continue - # to caching generator - if _cache_write_gen and not tracker.writer_finished: - try: - await _cache_write_gen.asend(chunk) - except Exception as e: - await tracker.provider_on_failed() # signal tracker - logger.error( - f"cache write coroutine failed for {meta=}, abort caching: {e!r}" - ) - # to uvicorn thread - yield chunk - await tracker.provider_on_finished() - except Exception as e: - logger.exception(f"cache tee failed for {meta=}") - await tracker.provider_on_failed() - raise CacheStreamingFailed from e - - await tracker.provider_start(meta) - return _inner() + try: + _cache_write_gen = await tracker.start_provider(cache_meta) + _cache_writer_failed = False + + # tee the incoming chunk to two destinations + async for chunk in fd: + # NOTE: for aiohttp, when HTTP chunk encoding is enabled, + # an empty chunk will be sent to indicate the EOF of stream, + # we MUST handle this empty chunk. + if not chunk: # skip if empty chunk is read from remote + continue + + # to caching generator, if the tracker is still working + if not _cache_writer_failed: + try: + await _cache_write_gen.asend(chunk) + except Exception as e: + logger.error( + f"cache write coroutine failed for {cache_meta=}, abort caching: {e!r}" + ) + _cache_writer_failed = True + + # to uvicorn thread + yield chunk + + # signal provider on finish, no more data chunk will be sent + with contextlib.suppress(StopAsyncIteration): + await _cache_write_gen.asend(b"") + except Exception as e: + _err_msg = f"cache tee failed for {cache_meta=}: {e!r}" + logger.warning(_err_msg) + raise CacheStreamingFailed(_err_msg) from e + finally: + # force terminate the generator in all condition at exit, this + # can ensure the generator being gced after cache_streaming exits. + with contextlib.suppress(StopAsyncIteration): + await _cache_write_gen.athrow(StopAsyncIteration) + + # remove the refs + fd, tracker = None, None # type: ignore + _cache_write_gen = None diff --git a/src/ota_proxy/config.py b/src/ota_proxy/config.py index 61ab3a2a4..a6c483877 100644 --- a/src/ota_proxy/config.py +++ b/src/ota_proxy/config.py @@ -49,9 +49,6 @@ class Config: # cache streaming behavior AIOHTTP_SOCKET_READ_TIMEOUT = 60 # second - STREAMING_BACKOFF_MAX = 30 # seconds - STREAMING_BACKOFF_FACTOR = 0.01 # second - STREAMING_CACHED_TMP_TIMEOUT = 10 # second TMP_FILE_PREFIX = "tmp" URL_BASED_HASH_PREFIX = "URL_" diff --git a/src/ota_proxy/ota_cache.py b/src/ota_proxy/ota_cache.py index ad7fded61..c68ff7731 100644 --- a/src/ota_proxy/ota_cache.py +++ b/src/ota_proxy/ota_cache.py @@ -28,11 +28,12 @@ import aiohttp from multidict import CIMultiDictProxy +from otaclient_common.common import get_backoff from otaclient_common.typing import StrOrPath from ._consts import HEADER_CONTENT_ENCODING, HEADER_OTA_FILE_CACHE_CONTROL from .cache_control_header import OTAFileCacheControl -from .cache_streaming import CachingRegister, cache_streaming +from .cache_streaming import CacheTracker, CachingRegister, cache_streaming from .config import config as cfg from .db import CacheMeta, check_db, init_db from .errors import BaseOTACacheError @@ -194,7 +195,7 @@ async def start(self): thread_nums=cfg.DB_THREADS, thread_wait_timeout=cfg.DB_THREAD_WAIT_TIMEOUT, ) - self._on_going_caching = CachingRegister(self._base_dir) + self._on_going_caching = CachingRegister() if self._upper_proxy: # if upper proxy presented, force disable https @@ -418,11 +419,12 @@ async def _retrieve_file_by_cache( # NOTE(20240729): there is an edge condition that the finished cached file is not yet renamed, # but the database entry has already been inserted. Here we wait for 3 rounds for # cache_commit_callback to rename the tmp file. - _retry_count_max, _factor = 3, 0.01 + _retry_count_max, _factor, _backoff_max = 6, 0.01, 0.1 # 0.255s in total for _retry_count in range(_retry_count_max): if cache_file.is_file(): break - await asyncio.sleep(_factor * _retry_count) # give away the event loop + + await asyncio.sleep(get_backoff(_retry_count, _factor, _backoff_max)) if not cache_file.is_file(): logger.warning( @@ -542,38 +544,42 @@ async def retrieve_file( return _res # --- case 4: no cache available, streaming remote file and cache --- # - tracker, is_writer = await self._on_going_caching.get_tracker( - cache_identifier, + # a online tracker is available for this requrest + if (tracker := self._on_going_caching.get_tracker(cache_identifier)) and ( + subscription := await tracker.subscribe_tracker() + ): + # logger.debug(f"reader subscribe for {tracker.meta=}") + stream_fd, cache_meta = subscription + return stream_fd, cache_meta.export_headers_to_client() + + # no valid online tracker is available for this request, create a new one and + # promote the caller to be the provider. + # NOTE: register the tracker before open the remote fd! + tracker = CacheTracker( + cache_identifier=cache_identifier, + base_dir=self._base_dir, executor=self._executor, - callback=self._commit_cache_callback, + commit_cache_cb=self._commit_cache_callback, below_hard_limit_event=self._storage_below_hard_limit_event, ) - if is_writer: - try: - remote_fd, resp_headers = await self._retrieve_file_by_downloading( - raw_url, headers=headers_from_client - ) - except Exception: - await tracker.provider_on_failed() - raise + self._on_going_caching.register_tracker(cache_identifier, tracker) + # caller is the provider of the requested resource + try: + remote_fd, resp_headers = await self._retrieve_file_by_downloading( + raw_url, headers=headers_from_client + ) cache_meta = create_cachemeta_for_request( raw_url, cache_identifier, compression_alg, resp_headers_from_upper=resp_headers, ) - # start caching - wrapped_fd = await cache_streaming( - fd=remote_fd, - meta=cache_meta, - tracker=tracker, - ) + wrapped_fd = cache_streaming(remote_fd, tracker, cache_meta) return wrapped_fd, resp_headers - - else: - stream_fd = await tracker.subscriber_subscribe_tracker() - if stream_fd and tracker.meta: - logger.debug(f"reader subscribe for {tracker.meta=}") - return stream_fd, tracker.meta.export_headers_to_client() + except Exception: + tracker.set_writer_failed() + raise + finally: + tracker = None # remove ref diff --git a/src/ota_proxy/utils.py b/src/ota_proxy/utils.py index 76e379450..a852b1597 100644 --- a/src/ota_proxy/utils.py +++ b/src/ota_proxy/utils.py @@ -1,6 +1,5 @@ from __future__ import annotations -import asyncio from concurrent.futures import Executor from hashlib import sha256 from os import PathLike @@ -11,28 +10,6 @@ from .config import config as cfg -def get_backoff(n: int, factor: float, _max: float) -> float: - return min(_max, factor * (2 ** (n - 1))) - - -async def wait_with_backoff( - _retry_cnt: int, *, _backoff_factor: float, _backoff_max: float -) -> bool: - """ - Returns: - A bool indicates whether upper caller should keep retry. - """ - _timeout = get_backoff( - _retry_cnt, - _backoff_factor, - _backoff_max, - ) - if _timeout <= _backoff_max: - await asyncio.sleep(_timeout) - return True - return False - - async def read_file(fpath: PathLike, *, executor: Executor) -> AsyncIterator[bytes]: """Open and read a file asynchronously with aiofiles.""" async with aiofiles.open(fpath, "rb", executor=executor) as f: diff --git a/tests/test_ota_proxy/test_cache_streaming.py b/tests/test_ota_proxy/test_cache_streaming.py new file mode 100644 index 000000000..8efdbcd1c --- /dev/null +++ b/tests/test_ota_proxy/test_cache_streaming.py @@ -0,0 +1,161 @@ +# Copyright 2022 TIER IV, INC. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import annotations + +import asyncio +import logging +import random +from pathlib import Path +from typing import Coroutine, Optional + +import pytest + +from ota_proxy.cache_streaming import CacheTracker, CachingRegister +from ota_proxy.db import CacheMeta + +logger = logging.getLogger(__name__) + + +class TestOngoingCachingRegister: + """ + Testing multiple access to a single resource at the same time with ongoing_cache control. + + NOTE; currently this test only testing the weakref implementation part, + the file descriptor management part is tested in test_ota_proxy_server + """ + + URL = "common_url" + WORKS_NUM = 128 + + @pytest.fixture(autouse=True) + async def setup_test(self, tmp_path: Path): + self.base_dir = tmp_path / "base_dir" + self.base_dir.mkdir(parents=True, exist_ok=True) + self.register = CachingRegister() + + # events + # NOTE: we don't have Barrier in asyncio lib, so + # use Semaphore to simulate one + self.register_finish = asyncio.Semaphore(self.WORKS_NUM) + self.sync_event = asyncio.Event() + self.writer_done_event = asyncio.Event() + + async def _wait_for_registeration_finish(self): + while not self.register_finish.locked(): + await asyncio.sleep(0.16) + logger.info("registeration finished") + + async def _worker( + self, + idx: int, + ) -> tuple[bool, Optional[CacheMeta]]: + """ + Returns tuple of bool indicates whether the worker is writter, and CacheMeta + from tracker. + """ + # simulate multiple works subscribing the register + await self.sync_event.wait() + await asyncio.sleep(random.randrange(100, 200) // 100) + + _tracker = self.register.get_tracker(self.URL) + # is subscriber + if _tracker: + logger.debug(f"#{idx} is subscriber") + await self.register_finish.acquire() + + while not _tracker.writer_finished: # simulating cache streaming + await asyncio.sleep(0.1) + return False, _tracker.cache_meta + + # is provider + logger.info(f"#{idx} is provider") + + # NOTE: register the tracker before open the remote fd! + _tracker = CacheTracker( + cache_identifier=self.URL, + base_dir=self.base_dir, + executor=None, # type: ignore + commit_cache_cb=None, # type: ignore + below_hard_limit_event=None, # type: ignore + ) + self.register.register_tracker(self.URL, _tracker) + + # NOTE: use last_access field to store worker index + # NOTE 2: bypass provider_start method, directly set tracker property + cache_meta = CacheMeta( + last_access=idx, + url="some_url", + file_sha256="some_filesha256_value", + ) + _tracker.cache_meta = cache_meta # normally it was set by start_provider + + # NOTE: we are not actually start the caching, so not setting + # executor, commit_cache_cb and below_hard_limit_event + await self.register_finish.acquire() + + # manually set the tracker to be started + _tracker._writer_ready.set() + + # simulate waiting for writer finished downloading + _tracker.fpath.touch() + await self.writer_done_event.wait() + + # finished + _tracker._writer_finished.set() + logger.info(f"writer #{idx} finished") + return True, _tracker.cache_meta + + async def test_ongoing_cache_register(self): + """ + Test multiple access to single resource with ongoing_cache control mechanism. + """ + coros: list[Coroutine] = [] + for idx in range(self.WORKS_NUM): + coros.append(self._worker(idx)) + + random.shuffle(coros) # shuffle the corotines to simulate unordered access + tasks = [asyncio.create_task(c) for c in coros] + logger.info(f"{self.WORKS_NUM} workers have been dispatched") + + # start all the worker, all the workers will now access the same resouce. + self.sync_event.set() + logger.info("all workers start to subscribe to the register") + await self._wait_for_registeration_finish() # wait for all workers finish subscribing + self.writer_done_event.set() # writer finished + + ###### check the test result ###### + meta_set, writer_meta = set(), None + for _fut in asyncio.as_completed(tasks): + is_writer, meta = await _fut + if meta is None: + logger.warning( + "encount edge condition that subscriber subscribes " + "on closed tracker, ignored" + ) + continue + meta_set.add(meta) + if is_writer: + writer_meta = meta + # ensure only one meta presented in the set, and it should be + # the meta from the writer/provider, all the subscriber should use + # the meta from the writer/provider. + assert len(meta_set) == 1 and writer_meta in meta_set + + # ensure that the entry in the register is garbage collected + assert len(self.register._id_tracker) == 0 + + # ensure no tmp files are leftover + assert len(list(self.base_dir.glob("tmp_*"))) == 0 diff --git a/tests/test_ota_proxy/test_ota_cache.py b/tests/test_ota_proxy/test_ota_cache.py index 734911533..d00baa181 100644 --- a/tests/test_ota_proxy/test_ota_cache.py +++ b/tests/test_ota_proxy/test_ota_cache.py @@ -15,13 +15,10 @@ from __future__ import annotations -import asyncio import bisect import logging import random import sqlite3 -from pathlib import Path -from typing import Coroutine, Optional import pytest import pytest_asyncio @@ -29,7 +26,7 @@ from ota_proxy import config as cfg from ota_proxy.db import CacheMeta -from ota_proxy.ota_cache import CachingRegister, LRUCacheHelper +from ota_proxy.ota_cache import LRUCacheHelper from ota_proxy.utils import url_based_hash logger = logging.getLogger(__name__) @@ -142,120 +139,3 @@ async def test_rotate_cache(self, lru_helper: LRUCacheHelper): for target_bucket in list(cfg.BUCKET_FILE_SIZE_DICT)[1:-1]: entries_to_be_removed = await lru_helper.rotate_cache(target_bucket) assert entries_to_be_removed is not None and len(entries_to_be_removed) != 0 - - -class TestOngoingCachingRegister: - """ - Testing multiple access to a single resource at the same time with ongoing_cache control. - - NOTE; currently this test only testing the weakref implementation part, - the file descriptor management part is tested in test_ota_proxy_server - """ - - URL = "common_url" - WORKS_NUM = 128 - - @pytest.fixture(autouse=True) - async def setup_test(self, tmp_path: Path): - self.base_dir = tmp_path / "base_dir" - self.base_dir.mkdir(parents=True, exist_ok=True) - self.register = CachingRegister(self.base_dir) - - # events - # NOTE: we don't have Barrier in asyncio lib, so - # use Semaphore to simulate one - self.register_finish = asyncio.Semaphore(self.WORKS_NUM) - self.sync_event = asyncio.Event() - self.writer_done_event = asyncio.Event() - - async def _wait_for_registeration_finish(self): - while not self.register_finish.locked(): - await asyncio.sleep(0.16) - logger.info("registeration finished") - - async def _worker( - self, - idx: int, - ) -> tuple[bool, Optional[CacheMeta]]: - """ - Returns tuple of bool indicates whether the worker is writter, and CacheMeta - from tracker. - """ - # simulate multiple works subscribing the register - await self.sync_event.wait() - await asyncio.sleep(random.randrange(100, 200) // 100) - - _tracker, _is_writer = await self.register.get_tracker( - self.URL, - executor=None, # type: ignore - callback=None, # type: ignore - below_hard_limit_event=None, # type: ignore - ) - await self.register_finish.acquire() - - if _is_writer: - logger.info(f"#{idx} is provider") - # NOTE: use last_access field to store worker index - # NOTE 2: bypass provider_start method, directly set tracker property - _tracker.meta = CacheMeta( - last_access=idx, - url="some_url", - file_sha256="some_filesha256_value", - ) - _tracker._writer_ready.set() - # simulate waiting for writer finished downloading - await self.writer_done_event.wait() - logger.info(f"writer #{idx} finished") - # finished - _tracker._writer_finished.set() - _tracker._ref = None - return True, _tracker.meta - else: - logger.debug(f"#{idx} is subscriber") - _tracker._subscriber_ref_holder.append(_tracker._ref) - while not _tracker.writer_finished: # simulating cache streaming - await asyncio.sleep(0.1) - # NOTE: directly pop the ref - _tracker._subscriber_ref_holder.pop() - return False, _tracker.meta - - async def test_ongoing_cache_register(self): - """ - Test multiple access to single resource with ongoing_cache control mechanism. - """ - coros: list[Coroutine] = [] - for idx in range(self.WORKS_NUM): - coros.append(self._worker(idx)) - - random.shuffle(coros) # shuffle the corotines to simulate unordered access - tasks = [asyncio.create_task(c) for c in coros] - logger.info(f"{self.WORKS_NUM} workers have been dispatched") - - # start all the worker, all the workers will now access the same resouce. - self.sync_event.set() - logger.info("all workers start to subscribe to the register") - await self._wait_for_registeration_finish() # wait for all workers finish subscribing - self.writer_done_event.set() # writer finished - - ###### check the test result ###### - meta_set, writer_meta = set(), None - for _fut in asyncio.as_completed(tasks): - is_writer, meta = await _fut - if meta is None: - logger.warning( - "encount edge condition that subscriber subscribes " - "on closed tracker, ignored" - ) - continue - meta_set.add(meta) - if is_writer: - writer_meta = meta - # ensure only one meta presented in the set, and it should be - # the meta from the writer/provider, all the subscriber should use - # the meta from the writer/provider. - assert len(meta_set) == 1 and writer_meta in meta_set - - # ensure that the entry in the register is garbage collected - assert ( - len(self.register._id_ref_dict) == len(self.register._ref_tracker_dict) == 0 - ) diff --git a/tests/test_ota_proxy/test_ota_proxy_server.py b/tests/test_ota_proxy/test_ota_proxy_e2e.py similarity index 100% rename from tests/test_ota_proxy/test_ota_proxy_server.py rename to tests/test_ota_proxy/test_ota_proxy_e2e.py From 7c74ad87c378b294d993d6ef1e11ff4a22519aa3 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 8 Aug 2024 10:05:30 +0000 Subject: [PATCH 104/193] prepare for merging the upstream main --- src/otaclient/{app => }/boot_control/__init__.py | 0 src/otaclient/{app => }/boot_control/_common.py | 0 src/otaclient/{app => }/boot_control/_grub.py | 0 src/otaclient/{app => }/boot_control/_jetson_cboot.py | 0 src/otaclient/{app => }/boot_control/_jetson_common.py | 0 src/otaclient/{app => }/boot_control/_jetson_uefi.py | 0 src/otaclient/{app => }/boot_control/_rpi_boot.py | 0 src/otaclient/{app => }/boot_control/configs.py | 0 src/otaclient/{app => }/boot_control/protocol.py | 0 src/otaclient/{app => }/boot_control/selecter.py | 0 10 files changed, 0 insertions(+), 0 deletions(-) rename src/otaclient/{app => }/boot_control/__init__.py (100%) rename src/otaclient/{app => }/boot_control/_common.py (100%) rename src/otaclient/{app => }/boot_control/_grub.py (100%) rename src/otaclient/{app => }/boot_control/_jetson_cboot.py (100%) rename src/otaclient/{app => }/boot_control/_jetson_common.py (100%) rename src/otaclient/{app => }/boot_control/_jetson_uefi.py (100%) rename src/otaclient/{app => }/boot_control/_rpi_boot.py (100%) rename src/otaclient/{app => }/boot_control/configs.py (100%) rename src/otaclient/{app => }/boot_control/protocol.py (100%) rename src/otaclient/{app => }/boot_control/selecter.py (100%) diff --git a/src/otaclient/app/boot_control/__init__.py b/src/otaclient/boot_control/__init__.py similarity index 100% rename from src/otaclient/app/boot_control/__init__.py rename to src/otaclient/boot_control/__init__.py diff --git a/src/otaclient/app/boot_control/_common.py b/src/otaclient/boot_control/_common.py similarity index 100% rename from src/otaclient/app/boot_control/_common.py rename to src/otaclient/boot_control/_common.py diff --git a/src/otaclient/app/boot_control/_grub.py b/src/otaclient/boot_control/_grub.py similarity index 100% rename from src/otaclient/app/boot_control/_grub.py rename to src/otaclient/boot_control/_grub.py diff --git a/src/otaclient/app/boot_control/_jetson_cboot.py b/src/otaclient/boot_control/_jetson_cboot.py similarity index 100% rename from src/otaclient/app/boot_control/_jetson_cboot.py rename to src/otaclient/boot_control/_jetson_cboot.py diff --git a/src/otaclient/app/boot_control/_jetson_common.py b/src/otaclient/boot_control/_jetson_common.py similarity index 100% rename from src/otaclient/app/boot_control/_jetson_common.py rename to src/otaclient/boot_control/_jetson_common.py diff --git a/src/otaclient/app/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py similarity index 100% rename from src/otaclient/app/boot_control/_jetson_uefi.py rename to src/otaclient/boot_control/_jetson_uefi.py diff --git a/src/otaclient/app/boot_control/_rpi_boot.py b/src/otaclient/boot_control/_rpi_boot.py similarity index 100% rename from src/otaclient/app/boot_control/_rpi_boot.py rename to src/otaclient/boot_control/_rpi_boot.py diff --git a/src/otaclient/app/boot_control/configs.py b/src/otaclient/boot_control/configs.py similarity index 100% rename from src/otaclient/app/boot_control/configs.py rename to src/otaclient/boot_control/configs.py diff --git a/src/otaclient/app/boot_control/protocol.py b/src/otaclient/boot_control/protocol.py similarity index 100% rename from src/otaclient/app/boot_control/protocol.py rename to src/otaclient/boot_control/protocol.py diff --git a/src/otaclient/app/boot_control/selecter.py b/src/otaclient/boot_control/selecter.py similarity index 100% rename from src/otaclient/app/boot_control/selecter.py rename to src/otaclient/boot_control/selecter.py From 37eec257b9b9c122e271aa05b57bd0a195fe2109 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Tue, 20 Aug 2024 07:36:37 +0000 Subject: [PATCH 105/193] jetson-common: firmware_bsp_version file -> bsp_version file, and we assume that we always keep rootfs BSP version the same as firmware BSP version --- src/otaclient/boot_control/_jetson_common.py | 49 ++++++++++---------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_common.py b/src/otaclient/boot_control/_jetson_common.py index 2810d1ba9..8fa3cb503 100644 --- a/src/otaclient/boot_control/_jetson_common.py +++ b/src/otaclient/boot_control/_jetson_common.py @@ -103,16 +103,16 @@ def dump(self) -> str: BeforeValidator(BSPVersion.parse), PlainSerializer(BSPVersion.dump, return_type=str), ] -"""BSPVersion in string representation, used by FirmwareBSPVersion model.""" +"""BSPVersion in string representation.""" -class FirmwareBSPVersion(BaseModel): +class SlotBSPVersion(BaseModel): """ BSP version string schema: Rxx.yy.z """ - slot_a: Optional[BSPVersionStr] = None - slot_b: Optional[BSPVersionStr] = None + slot_a: BSPVersionStr | None = None + slot_b: BSPVersionStr | None = None def set_by_slot(self, slot_id: SlotID, ver: BSPVersion | None) -> None: if slot_id == SLOT_A: @@ -208,37 +208,38 @@ def dump_slots_info(cls, *, target: Optional[NVBootctrlTarget] = None) -> str: return cls._nvbootctrl(cmd, target=target, check_output=True) -class FirmwareBSPVersionControl: - """firmware_bsp_version ota-status file for tracking firmware version. +class BSPVersionControl: + """bsp_version ota-status file for tracking BSP version. - The firmware BSP version is stored in /boot/ota-status/firmware_bsp_version json file, - tracking the firmware BSP version for each slot. + The BSP version is stored in /boot/ota-status/bsp_version json file, + tracking the BSP version for each slot. + NOTE that we should always keep the rootfs BSP version the same as firmware BSP version! - Each slot should keep the same firmware_bsp_version file, this file is passed to standby slot - during OTA update. + Each slot should keep the same bsp_version file, this file is passed to standby slot + during OTA update as it. """ def __init__( self, current_slot: SlotID, - current_slot_firmware_bsp_ver: BSPVersion, + current_slot_bsp_ver: BSPVersion, *, - current_firmware_bsp_vf: Path, + current_bsp_version_file: Path, ) -> None: self.current_slot, self.standby_slot = current_slot, SLOT_FLIP[current_slot] - self._version = FirmwareBSPVersion() + self._version = SlotBSPVersion() try: - self._version = FirmwareBSPVersion.model_validate_json( - current_firmware_bsp_vf.read_text() + self._version = SlotBSPVersion.model_validate_json( + current_bsp_version_file.read_text() ) except Exception as e: - logger.warning(f"invalid or missing firmware_bsp_verion file: {e!r}") - current_firmware_bsp_vf.unlink(missing_ok=True) + logger.warning(f"invalid or missing bsp_verion file: {e!r}") + current_bsp_version_file.unlink(missing_ok=True) # NOTE: only check the standby slot's firmware BSP version info from file, # for current slot, always trust the value from nvbootctrl. - self._version.set_by_slot(current_slot, current_slot_firmware_bsp_ver) + self._version.set_by_slot(current_slot, current_slot_bsp_ver) logger.info(f"loading firmware bsp version completed: {self._version}") def write_to_file(self, fw_bsp_fpath: StrOrPath) -> None: @@ -246,20 +247,20 @@ def write_to_file(self, fw_bsp_fpath: StrOrPath) -> None: write_str_to_file_sync(fw_bsp_fpath, self._version.model_dump_json()) @property - def current_slot_fw_ver(self) -> BSPVersion: + def current_slot_bsp_ver(self) -> BSPVersion: assert (res := self._version.get_by_slot(self.current_slot)) return res - @current_slot_fw_ver.setter - def current_slot_fw_ver(self, bsp_ver: BSPVersion | None): + @current_slot_bsp_ver.setter + def current_slot_bsp_ver(self, bsp_ver: BSPVersion | None): self._version.set_by_slot(self.current_slot, bsp_ver) @property - def standby_slot_fw_ver(self) -> BSPVersion | None: + def standby_slot_bsp_ver(self) -> BSPVersion | None: return self._version.get_by_slot(self.standby_slot) - @standby_slot_fw_ver.setter - def standby_slot_fw_ver(self, bsp_ver: BSPVersion | None): + @standby_slot_bsp_ver.setter + def standby_slot_bsp_ver(self, bsp_ver: BSPVersion | None): self._version.set_by_slot(self.standby_slot, bsp_ver) From 294da97cc3a38a65ba3247d2bf7c4e401f8443b7 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Tue, 20 Aug 2024 07:59:06 +0000 Subject: [PATCH 106/193] UEFIFirmwarUpdater: use FirmwareManifest --- src/otaclient/boot_control/_jetson_uefi.py | 66 ++++++++++++++-------- 1 file changed, 42 insertions(+), 24 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index aaf01b3f7..98367f7be 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -33,6 +33,7 @@ from otaclient.app import errors as ota_errors from otaclient.app.configs import config as cfg +from otaclient.boot_control._firmware_package import FirmwareManifest from otaclient_api.v2 import types as api_types from otaclient_common.common import file_sha256, subprocess_call, write_str_to_file_sync from otaclient_common.typing import StrOrPath @@ -41,7 +42,7 @@ from ._jetson_common import ( SLOT_PAR_MAP, BSPVersion, - FirmwareBSPVersionControl, + BSPVersionControl, NVBootctrlCommon, copy_standby_slot_boot_to_internal_emmc, detect_rootfs_bsp_version, @@ -104,7 +105,7 @@ def get_current_fw_bsp_version(cls) -> BSPVersion: bsp_ver = BSPVersion.parse(bsp_ver_str) if bsp_ver.major_rev == 0: _err_msg = f"invalid BSP version: {bsp_ver_str}, this might indicate an incomplete flash!" - logger.warning(_err_msg) + logger.error(_err_msg) raise ValueError(_err_msg) return bsp_ver @@ -188,7 +189,7 @@ def _detect_esp_dev(boot_parent_devpath: StrOrPath) -> str: class L4TLauncherBSPVersionControl(BaseModel): """ - Schema: : + Schema: : """ bsp_ver: BSPVersion @@ -212,7 +213,9 @@ def __init__( boot_parent_devpath: StrOrPath, standby_slot_mp: StrOrPath, *, - fw_bsp_ver_control: FirmwareBSPVersionControl, + tnspec: str, + bsp_ver_control: BSPVersionControl, + firmware_manifest: FirmwareManifest, ) -> None: """Init an instance of UEFIFirmwareUpdater. @@ -223,9 +226,14 @@ def __init__( ota_image_bsp_ver (BSPVersion): The BSP version of OTA image used to update the standby slot. fw_bsp_ver_control (FirmwareBSPVersionControl): The firmware BSP version of each slots. """ - self.fw_bsp_ver_control = fw_bsp_ver_control - # NOTE: standby slot is updated with OTA image - self.ota_image_bsp_ver = detect_rootfs_bsp_version(standby_slot_mp) + self.standby_slot_bsp_ver = bsp_ver_control.standby_slot_bsp_ver + self.current_slot_bsp_ver = bsp_ver_control.current_slot_bsp_ver + + self.tnspec = tnspec + self.firmware_manifest = firmware_manifest + self.firmware_package_bsp_ver = BSPVersion.parse( + firmware_manifest.firmware_spec.bsp_version + ) # NOTE: use the esp partition at the current booted device # i.e., if we boot from nvme0n1, then bootdev_path is /dev/nvme0n1 and @@ -243,6 +251,7 @@ def __init__( self.bootaa64_at_esp_bak = ( self.esp_boot_dir / f"{boot_cfg.L4TLAUNCHER_FNAME}_bak" ) + """The location to backup current L4TLauncher binary.""" self.standby_slot_mp = Path(standby_slot_mp) self.fw_loc_at_standby_slot = self.standby_slot_mp / Path( @@ -284,8 +293,9 @@ def _prepare_fwupdate_capsule(self) -> bool: def _update_l4tlauncher(self) -> bool: """update L4TLauncher with OTA image's one.""" - ota_image_l4tlauncher_ver = self.ota_image_bsp_ver - logger.warning(f"update the l4tlauncher to version {ota_image_l4tlauncher_ver}") + logger.warning( + f"update the l4tlauncher to version {self.firmware_package_bsp_ver}" + ) # new BOOTAA64.efi is located at /opt/ota_package/BOOTAA64.efi ota_image_bootaa64 = self.fw_loc_at_standby_slot / boot_cfg.L4TLAUNCHER_FNAME @@ -295,10 +305,11 @@ def _update_l4tlauncher(self) -> bool: shutil.copy(self.bootaa64_at_esp, self.bootaa64_at_esp_bak) shutil.copy(ota_image_bootaa64, self.bootaa64_at_esp) + os.sync() # ensure the boot application is written to the disk + write_str_to_file_sync( - self.l4tlauncher_ver_fpath, ota_image_l4tlauncher_ver.dump() + self.l4tlauncher_ver_fpath, self.firmware_package_bsp_ver.dump() ) - os.sync() return True @staticmethod @@ -356,19 +367,17 @@ def _detect_l4tlauncher_version(self) -> BSPVersion: # NOTE(20240624): if we failed to detect the l4tlauncher's version, # we assume that the launcher is the same version as current slot's fw. # This is typically the case of a newly flashed ECU. - current_slot_fw_bsp_ver = self.fw_bsp_ver_control.current_slot_fw_ver - logger.error( + logger.warning( ( "failed to determine the l4tlauncher's version, assuming " - f"version is the same as current slot's fw version: {current_slot_fw_bsp_ver}" + f"version is the same as current slot's fw version: {self.current_slot_bsp_ver}" ) ) - l4tlauncher_bsp_ver = current_slot_fw_bsp_ver _ver_control = L4TLauncherBSPVersionControl( - bsp_ver=l4tlauncher_bsp_ver, sha256_digest=l4tlauncher_sha256_digest + bsp_ver=self.current_slot_bsp_ver, sha256_digest=l4tlauncher_sha256_digest ) write_str_to_file_sync(self.l4tlauncher_ver_fpath, _ver_control.dump()) - return l4tlauncher_bsp_ver + return self.current_slot_bsp_ver # APIs @@ -378,19 +387,29 @@ def firmware_update(self) -> bool: Returns: True if firmware update is configured, False if there is no firmware update. """ - standby_slot_fw_bsp_ver = self.fw_bsp_ver_control.standby_slot_fw_ver + # check BSP version, NVIDIA Jetson device doesn't allow firmware downgrade. if ( - standby_slot_fw_bsp_ver - and standby_slot_fw_bsp_ver >= self.ota_image_bsp_ver + self.standby_slot_bsp_ver + and self.standby_slot_bsp_ver >= self.firmware_package_bsp_ver ): logger.info( ( "standby slot has newer or equal ver of firmware, skip firmware update: " - f"{standby_slot_fw_bsp_ver=}, {self.ota_image_bsp_ver=}" + f"{self.standby_slot_bsp_ver=}, {self.firmware_package_bsp_ver=}" ) ) return False + # check firmware compatibility, this is to prevent failed firmware update beforehand. + if not self.firmware_manifest.check_compat(self.tnspec): + _err_msg = ( + "firmware package is incompatible with this device: " + f"{self.tnspec=}, {self.firmware_manifest.firmware_spec.firmware_compat}, " + "skip firmware update" + ) + logger.warning(_err_msg) + return False + with _ensure_esp_mounted(self.esp_part, self.esp_mp): if not self._prepare_fwupdate_capsule(): logger.info("no firmware file is prepared, skip firmware update") @@ -425,12 +444,11 @@ def l4tlauncher_update(self) -> bool: l4tlauncher_bsp_ver = self._detect_l4tlauncher_version() logger.info(f"finished detect l4tlauncher version: {l4tlauncher_bsp_ver}") - ota_image_l4tlauncher_ver = self.ota_image_bsp_ver - if l4tlauncher_bsp_ver >= ota_image_l4tlauncher_ver: + if l4tlauncher_bsp_ver >= self.firmware_package_bsp_ver: logger.info( ( "installed l4tlauncher has newer or equal version of l4tlauncher to OTA image's one, " - f"{l4tlauncher_bsp_ver=}, {ota_image_l4tlauncher_ver=}, " + f"{l4tlauncher_bsp_ver=}, {self.firmware_package_bsp_ver=}, " "skip l4tlauncher update" ) ) From 661907976e55b40cd8678aee186fcd85e7c76ecd Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Tue, 20 Aug 2024 08:00:00 +0000 Subject: [PATCH 107/193] permit same version firmware update --- src/otaclient/boot_control/_jetson_uefi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index 98367f7be..ac8f45d37 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -390,11 +390,11 @@ def firmware_update(self) -> bool: # check BSP version, NVIDIA Jetson device doesn't allow firmware downgrade. if ( self.standby_slot_bsp_ver - and self.standby_slot_bsp_ver >= self.firmware_package_bsp_ver + and self.standby_slot_bsp_ver > self.firmware_package_bsp_ver ): logger.info( ( - "standby slot has newer or equal ver of firmware, skip firmware update: " + "standby slot has newer ver of firmware, skip firmware update: " f"{self.standby_slot_bsp_ver=}, {self.firmware_package_bsp_ver=}" ) ) From 4cad6b72c7ee20a7ae2865d968e34f5e0b1e1763 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Tue, 20 Aug 2024 08:04:56 +0000 Subject: [PATCH 108/193] jetson-common: implement get_nvbootctrl_conf_tnspec --- src/otaclient/boot_control/_jetson_common.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/otaclient/boot_control/_jetson_common.py b/src/otaclient/boot_control/_jetson_common.py index 8fa3cb503..c7fc3a639 100644 --- a/src/otaclient/boot_control/_jetson_common.py +++ b/src/otaclient/boot_control/_jetson_common.py @@ -307,6 +307,14 @@ def detect_rootfs_bsp_version(rootfs: StrOrPath) -> BSPVersion: raise ValueError(_err_msg) from e +def get_nvbootctrl_conf_tnspec(nvbootctrl_conf: str) -> str | None: + """Get the TNSPEC field from nv_boot_control conf file.""" + for line in nvbootctrl_conf.splitlines(): + if line.strip().startswith("TNSPEC"): + _, tnspec = line.split(" ", maxsplit=1) + return tnspec.strip() + + def update_extlinux_cfg(_input: str, partuuid: str) -> str: """Update input exlinux text with input rootfs .""" partuuid_str = f"PARTUUID={partuuid}" From 8e9e62087e0af76708fafa5f59c8f45293f15599 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Tue, 20 Aug 2024 08:49:08 +0000 Subject: [PATCH 109/193] jetson-uefi: utilize firmware_manifest and firmware_update_request --- src/otaclient/boot_control/_jetson_uefi.py | 73 +++++++++++++++------- 1 file changed, 50 insertions(+), 23 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index ac8f45d37..af0e8b7fb 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -33,8 +33,14 @@ from otaclient.app import errors as ota_errors from otaclient.app.configs import config as cfg -from otaclient.boot_control._firmware_package import FirmwareManifest +from otaclient.boot_control._firmware_package import ( + DigestValue, + FirmwareManifest, + FirmwareUpdateRequest, + PayloadType, +) from otaclient_api.v2 import types as api_types +from otaclient_common import replace_root from otaclient_common.common import file_sha256, subprocess_call, write_str_to_file_sync from otaclient_common.typing import StrOrPath @@ -215,6 +221,7 @@ def __init__( *, tnspec: str, bsp_ver_control: BSPVersionControl, + firmware_update_request: FirmwareUpdateRequest, firmware_manifest: FirmwareManifest, ) -> None: """Init an instance of UEFIFirmwareUpdater. @@ -230,6 +237,7 @@ def __init__( self.current_slot_bsp_ver = bsp_ver_control.current_slot_bsp_ver self.tnspec = tnspec + self.firmware_update_request = firmware_update_request self.firmware_manifest = firmware_manifest self.firmware_package_bsp_ver = BSPVersion.parse( firmware_manifest.firmware_spec.bsp_version @@ -254,10 +262,6 @@ def __init__( """The location to backup current L4TLauncher binary.""" self.standby_slot_mp = Path(standby_slot_mp) - self.fw_loc_at_standby_slot = self.standby_slot_mp / Path( - boot_cfg.CAPSULE_PAYLOAD_AT_ROOTFS - ).relative_to("/") - """where the fw update capsule and l4tlauncher bin located.""" self.esp_part = _detect_esp_dev(boot_parent_devpath) @@ -276,19 +280,30 @@ def _prepare_fwupdate_capsule(self) -> bool: # ------ prepare capsule update payload ------ # firmware_package_configured = False - for capsule_fname in boot_cfg.FIRMWARE_LIST: + for capsule_payload in self.firmware_manifest.get_firmware_packages( + self.firmware_update_request + ): + if capsule_payload.type != PayloadType.UEFI_CAPSULE: + continue + + # NOTE: currently we only support payload indicated by file path. + capsule_fpath = capsule_payload.file_location + assert not isinstance(capsule_fpath, DigestValue) + try: shutil.copy( - src=self.fw_loc_at_standby_slot / capsule_fname, - dst=capsule_dir_at_esp / capsule_fname, + src=replace_root(capsule_fpath, "/", self.standby_slot_mp), + dst=capsule_dir_at_esp / capsule_payload.payload_name, ) firmware_package_configured = True - logger.info(f"copy {capsule_fname} to {capsule_dir_at_esp}") + logger.info( + f"copy {capsule_payload.payload_name} to {capsule_dir_at_esp}" + ) except Exception as e: logger.warning( - f"failed to copy {capsule_fname} from {self.fw_loc_at_standby_slot} to {capsule_dir_at_esp}: {e!r}" + f"failed to copy {capsule_payload.payload_name} to {capsule_dir_at_esp}: {e!r}" ) - logger.warning(f"skip {capsule_fname}") + logger.warning(f"skip prepare {capsule_payload.payload_name}") return firmware_package_configured def _update_l4tlauncher(self) -> bool: @@ -296,21 +311,33 @@ def _update_l4tlauncher(self) -> bool: logger.warning( f"update the l4tlauncher to version {self.firmware_package_bsp_ver}" ) + for capsule_payload in self.firmware_manifest.get_firmware_packages( + self.firmware_update_request + ): + if capsule_payload.type != PayloadType.UEFI_CAPSULE: + continue - # new BOOTAA64.efi is located at /opt/ota_package/BOOTAA64.efi - ota_image_bootaa64 = self.fw_loc_at_standby_slot / boot_cfg.L4TLAUNCHER_FNAME - if not ota_image_bootaa64.is_file(): - logger.warning(f"{ota_image_bootaa64} not found, skip update l4tlauncher") - return False + # NOTE: currently we only support payload indicated by file path. + bootapp_fpath = capsule_payload.file_location + assert not isinstance(bootapp_fpath, DigestValue) - shutil.copy(self.bootaa64_at_esp, self.bootaa64_at_esp_bak) - shutil.copy(ota_image_bootaa64, self.bootaa64_at_esp) - os.sync() # ensure the boot application is written to the disk + # new BOOTAA64.efi is located at /opt/ota_package/BOOTAA64.efi + ota_image_bootaa64 = replace_root(bootapp_fpath, "/", self.standby_slot_mp) + try: + shutil.copy(self.bootaa64_at_esp, self.bootaa64_at_esp_bak) + shutil.copy(ota_image_bootaa64, self.bootaa64_at_esp) + os.sync() # ensure the boot application is written to the disk - write_str_to_file_sync( - self.l4tlauncher_ver_fpath, self.firmware_package_bsp_ver.dump() - ) - return True + write_str_to_file_sync( + self.l4tlauncher_ver_fpath, self.firmware_package_bsp_ver.dump() + ) + return True + except Exception as e: + _err_msg = f"failed to prepare boot application update: {e!r}, skip" + logger.warning(_err_msg) + + logger.info("no boot application update is configured in the request, skip") + return False @staticmethod def _write_magic_efivar() -> None: From b447218c6cd7e989e5413842fab3066fbef85ab1 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Tue, 20 Aug 2024 09:02:08 +0000 Subject: [PATCH 110/193] jetson-uefi: UEFIFirmwareUpdater now won't expose L4TLauncher update as API anymore, we always try to update L4TLauncher if firmware update is configured --- src/otaclient/boot_control/_jetson_uefi.py | 27 ++++++++++------------ 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index af0e8b7fb..d77ce97f1 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -454,22 +454,15 @@ def firmware_update(self) -> bool: ) return False - logger.info("firmware update package prepare finished") - return True - - def l4tlauncher_update(self) -> bool: - """Update l4tlauncher if needed. - - NOTE(20240611): Assume that the new L4TLauncher always keeps backward compatibility to - work with old firmware. This assumption MUST be confirmed on the real ECU. - NOTE(20240611): Only update l4tlauncher but never downgrade it. + logger.warning( + "firmware update package prepare finished" + f"will update firmware to {self.firmware_package_bsp_ver} in next reboot" + ) + logger.info("try to update L4TLauncher ...") - Returns: - True if l4tlauncher is updated, else if there is no l4tlauncher update. - """ with _ensure_esp_mounted(self.esp_part, self.esp_mp): l4tlauncher_bsp_ver = self._detect_l4tlauncher_version() - logger.info(f"finished detect l4tlauncher version: {l4tlauncher_bsp_ver}") + logger.info(f"current l4tlauncher version: {l4tlauncher_bsp_ver}") if l4tlauncher_bsp_ver >= self.firmware_package_bsp_ver: logger.info( @@ -479,8 +472,12 @@ def l4tlauncher_update(self) -> bool: "skip l4tlauncher update" ) ) - return False - return self._update_l4tlauncher() + else: + logger.warning( + f"try to update L4TLauncher to {self.firmware_package_bsp_ver=}" + ) + self._update_l4tlauncher() + return True MINIMUM_SUPPORTED_BSP_VERSION = BSPVersion(35, 2, 0) From afcc5af263c307ee75c1ed6973edbd76265e95ba Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Tue, 20 Aug 2024 09:05:31 +0000 Subject: [PATCH 111/193] boot_control.configs: update firmware update related configs fields --- src/otaclient/boot_control/configs.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/otaclient/boot_control/configs.py b/src/otaclient/boot_control/configs.py index fba967577..192b45225 100644 --- a/src/otaclient/boot_control/configs.py +++ b/src/otaclient/boot_control/configs.py @@ -65,18 +65,17 @@ class JetsonCBootControlConfig(JetsonBootCommon): class JetsonUEFIBootControlConfig(JetsonBootCommon): BOOTLOADER = BootloaderType.JETSON_UEFI TEGRA_COMPAT_PATH = "/sys/firmware/devicetree/base/compatible" - FIRMWARE_LIST = ["bl_only_payload.Cap"] + NVBOOTCTRL_CONF_FPATH = "/etc/nv_boot_control.conf" L4TLAUNCHER_FNAME = "BOOTAA64.efi" ESP_MOUNTPOINT = "/mnt/esp" ESP_PARTLABEL = "esp" UPDATE_TRIGGER_EFIVAR = "OsIndications-8be4df61-93ca-11d2-aa0d-00e098032b8c" MAGIC_BYTES = b"\x07\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00" CAPSULE_PAYLOAD_AT_ESP = "EFI/UpdateCapsule" - CAPSULE_PAYLOAD_AT_ROOTFS = "/opt/ota_package/" L4TLAUNCHER_VER_FNAME = "l4tlauncher_version" - NO_FIRMWARE_UPDATE_HINT_FNAME = ".otaclient_no_firmware_update" - """Skip firmware update if this file is presented.""" + FIRMWARE_UPDATE_REQUEST_FPATH = "/opt/ota/firmware/firmware_update.yaml" + FIRMWARE_MANIFEST_FPATH = "/opt/ota/firmware/firmware_manifest.yaml" @dataclass From 991ca267f07b3e32851d1b2e6727a25517c4386e Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 21 Aug 2024 00:48:02 +0000 Subject: [PATCH 112/193] firmware_package: add load_request and load_manifest API --- .../boot_control/_firmware_package.py | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/otaclient/boot_control/_firmware_package.py b/src/otaclient/boot_control/_firmware_package.py index cdc46268f..33396a878 100644 --- a/src/otaclient/boot_control/_firmware_package.py +++ b/src/otaclient/boot_control/_firmware_package.py @@ -44,13 +44,15 @@ import re from enum import Enum +from pathlib import Path from typing import Any, List, Literal +import yaml from pydantic import BaseModel, BeforeValidator, GetCoreSchemaHandler from pydantic_core import CoreSchema, core_schema from typing_extensions import Annotated -from otaclient_common.typing import gen_strenum_validator +from otaclient_common.typing import StrOrPath, gen_strenum_validator class PayloadType(str, Enum): @@ -186,3 +188,19 @@ class FirmwareUpdateRequest(BaseModel): format_version: Literal[1] = 1 firmware_list: List[str] + + +def load_request(request_fpath: StrOrPath) -> FirmwareUpdateRequest: + """Load update request from .""" + _raw = Path(request_fpath).read_text() + _raw_loaded = yaml.safe_load(_raw) + _res = FirmwareUpdateRequest.model_validate(_raw_loaded) + return _res + + +def load_manifest(manifest_fpath: StrOrPath) -> FirmwareManifest: + """Load manifest from .""" + _raw = Path(manifest_fpath).read_text() + _raw_loaded = yaml.safe_load(_raw) + _res = FirmwareManifest.model_validate(_raw_loaded) + return _res From 4c749d32af63c8db052356cde7290e818d1d4cb3 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 21 Aug 2024 00:55:07 +0000 Subject: [PATCH 113/193] load tnspec when initializing boot control --- src/otaclient/boot_control/_jetson_uefi.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index d77ce97f1..0e388b6e1 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -52,6 +52,7 @@ NVBootctrlCommon, copy_standby_slot_boot_to_internal_emmc, detect_rootfs_bsp_version, + get_nvbootctrl_conf_tnspec, preserve_ota_config_files_to_standby, update_standby_slot_extlinux_cfg, ) @@ -484,7 +485,7 @@ def firmware_update(self) -> bool: """Only after R35.2, the Capsule Firmware update is available.""" -class _UEFIBoot: +class _UEFIBootControl: """Low-level boot control implementation for jetson-uefi.""" def __init__(self): @@ -625,7 +626,16 @@ def __init__(self) -> None: ).relative_to("/") try: - self._uefi_control = uefi_control = _UEFIBoot() + self.tnspec = get_nvbootctrl_conf_tnspec( + Path(boot_cfg.NVBOOTCTRL_CONF_FPATH).read_text() + ) + logger.info(f"{self.tnspec=}") + except Exception as e: + logger.warning(f"failed to load tnspec: {e!r}") + self.tnspec = None + + try: + self._uefi_control = uefi_control = _UEFIBootControl() # mount point prepare self._mp_control = SlotMountHelper( From 55570cd4198f15e112fad2fdfaa9f5b5ac6ac68f Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 21 Aug 2024 01:06:07 +0000 Subject: [PATCH 114/193] let lowlevel boot control module load the tnspec --- src/otaclient/boot_control/_jetson_uefi.py | 24 ++++++++++++++-------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index 0e388b6e1..479a5cd98 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -509,6 +509,21 @@ def __init__(self): raise JetsonUEFIBootControlError(_err_msg) logger.info(f"dev compatibility: {compat_info}") + # load tnspec for firmware update compatibility check + try: + self.tnspec = get_nvbootctrl_conf_tnspec( + Path(boot_cfg.NVBOOTCTRL_CONF_FPATH).read_text() + ) + logger.info(f"firmware compatibility: {self.tnspec}") + except Exception as e: + logger.warning( + ( + f"failed to load tnspec: {e!r}, " + "this will result in firmware update being skipped!" + ) + ) + self.tnspec = None + # ------ check current slot BSP version ------ # # check current slot firmware BSP version try: @@ -625,15 +640,6 @@ def __init__(self) -> None: boot_cfg.OTA_STATUS_DIR ).relative_to("/") - try: - self.tnspec = get_nvbootctrl_conf_tnspec( - Path(boot_cfg.NVBOOTCTRL_CONF_FPATH).read_text() - ) - logger.info(f"{self.tnspec=}") - except Exception as e: - logger.warning(f"failed to load tnspec: {e!r}") - self.tnspec = None - try: self._uefi_control = uefi_control = _UEFIBootControl() From f07785c29fb610a97fc98cdd80b0149c20dbd278 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 21 Aug 2024 01:06:42 +0000 Subject: [PATCH 115/193] issue a warning when the rootfs BSP version is not the same as firmware BSP version --- src/otaclient/boot_control/_jetson_uefi.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index 479a5cd98..c055f8ab2 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -556,6 +556,16 @@ def __init__(self): ) ) + if rootfs_bsp_version != fw_bsp_version: + logger.warning( + ( + f"current slot has rootfs BSP version {rootfs_bsp_version}, " + f"while the firmware version is {fw_bsp_version}, " + "rootfs BSP version and firmware version are MISMATCHED! " + "This might result in nvbootctrl not working as expected!" + ) + ) + # ------ sanity check, currently jetson-uefi only supports >= R35.2 ----- # if fw_bsp_version < MINIMUM_SUPPORTED_BSP_VERSION: _err_msg = f"jetson-uefi only supports BSP version >= R35.2, but get {fw_bsp_version=}. " From fb0973e3e33e7e57c11f1c7f72e58a465523a325 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 21 Aug 2024 01:21:30 +0000 Subject: [PATCH 116/193] FirmwareBSPVersionControl: when firmware_bsp_version file is not presented, assume that both slots are running the same version of firmware --- src/otaclient/boot_control/_jetson_common.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_common.py b/src/otaclient/boot_control/_jetson_common.py index c7fc3a639..434cb2b6a 100644 --- a/src/otaclient/boot_control/_jetson_common.py +++ b/src/otaclient/boot_control/_jetson_common.py @@ -208,14 +208,18 @@ def dump_slots_info(cls, *, target: Optional[NVBootctrlTarget] = None) -> str: return cls._nvbootctrl(cmd, target=target, check_output=True) -class BSPVersionControl: - """bsp_version ota-status file for tracking BSP version. +class FirmwareBSPVersionControl: + """firmware_bsp_version ota-status file for tracking BSP version. - The BSP version is stored in /boot/ota-status/bsp_version json file, - tracking the BSP version for each slot. - NOTE that we should always keep the rootfs BSP version the same as firmware BSP version! + The BSP version is stored in /boot/ota-status/firmware_bsp_version json file, + tracking the firmware BSP version for each slot. + NOTE that we only cares about firmware BSP version. - Each slot should keep the same bsp_version file, this file is passed to standby slot + Unfortunately, we don't have method to detect standby slot's firwmare vesrion, + when the firmware_bsp_version file is not presented(typical case when we have newly setup device), + we ASSUME that both slots are running the same BSP version of firmware. + + Each slot should keep the same firmware_bsp_version file, this file is passed to standby slot during OTA update as it. """ @@ -236,6 +240,10 @@ def __init__( except Exception as e: logger.warning(f"invalid or missing bsp_verion file: {e!r}") current_bsp_version_file.unlink(missing_ok=True) + logger.warning( + "assume standby slot is running the same version of firmware" + ) + self._version.set_by_slot(self.standby_slot, current_slot_bsp_ver) # NOTE: only check the standby slot's firmware BSP version info from file, # for current slot, always trust the value from nvbootctrl. From a64402e7b8de16f28489dc25a21f79bfd6dd041a Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 21 Aug 2024 01:29:36 +0000 Subject: [PATCH 117/193] jetson-uefi: cleanup duplicated logic related to bsp version control, always update bsp_version_file on startup --- src/otaclient/boot_control/_jetson_uefi.py | 27 ++++++++-------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index c055f8ab2..10ee69f8b 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -48,7 +48,7 @@ from ._jetson_common import ( SLOT_PAR_MAP, BSPVersion, - BSPVersionControl, + FirmwareBSPVersionControl, NVBootctrlCommon, copy_standby_slot_boot_to_internal_emmc, detect_rootfs_bsp_version, @@ -221,7 +221,7 @@ def __init__( standby_slot_mp: StrOrPath, *, tnspec: str, - bsp_ver_control: BSPVersionControl, + fw_bsp_ver_control: FirmwareBSPVersionControl, firmware_update_request: FirmwareUpdateRequest, firmware_manifest: FirmwareManifest, ) -> None: @@ -234,8 +234,8 @@ def __init__( ota_image_bsp_ver (BSPVersion): The BSP version of OTA image used to update the standby slot. fw_bsp_ver_control (FirmwareBSPVersionControl): The firmware BSP version of each slots. """ - self.standby_slot_bsp_ver = bsp_ver_control.standby_slot_bsp_ver - self.current_slot_bsp_ver = bsp_ver_control.current_slot_bsp_ver + self.standby_slot_bsp_ver = fw_bsp_ver_control.standby_slot_bsp_ver + self.current_slot_bsp_ver = fw_bsp_ver_control.current_slot_bsp_ver self.tnspec = tnspec self.firmware_update_request = firmware_update_request @@ -665,12 +665,14 @@ def __init__(self) -> None: current_fw_bsp_ver_fpath = ( current_ota_status_dir / boot_cfg.FIRMWARE_BSP_VERSION_FNAME ) - - self._firmware_ver_control = fw_bsp_ver = FirmwareBSPVersionControl( + self._firmware_bsp_ver_control = FirmwareBSPVersionControl( current_slot=uefi_control.current_slot, - current_slot_firmware_bsp_ver=uefi_control.fw_bsp_version, - current_firmware_bsp_vf=current_fw_bsp_ver_fpath, + current_slot_bsp_ver=uefi_control.fw_bsp_version, + current_bsp_version_file=current_fw_bsp_ver_fpath, ) + # always update the bsp_version_file on startup to reflect + # the up-to-date current slot BSP version + self._firmware_bsp_ver_control.write_to_file(current_fw_bsp_ver_fpath) # init ota-status files self._ota_status_control = OTAStatusFilesControl( @@ -681,15 +683,6 @@ def __init__(self) -> None: standby_ota_status_dir=standby_ota_status_dir, finalize_switching_boot=self._finalize_switching_boot, ) - - # post starting up, write the firmware bsp version to current slot - # NOTE 1: we always update and refer to ONLY current slot's firmware bsp version file. - # NOTE 2: if OTA status is failure, always assume the firmware update on standby slot failed, - # and clear the standby slot's fw bsp version record. - if self._ota_status_control._ota_status == api_types.StatusOta.FAILURE: - fw_bsp_ver.standby_slot_fw_ver = None - fw_bsp_ver.current_slot_fw_ver = uefi_control.fw_bsp_version - fw_bsp_ver.write_to_file(current_fw_bsp_ver_fpath) except Exception as e: _err_msg = f"failed to start jetson-uefi controller: {e!r}" raise ota_errors.BootControlStartupFailed(_err_msg, module=__name__) from e From e3d85d22f27d485fc95e747222728b9a55cd8204 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 21 Aug 2024 01:37:44 +0000 Subject: [PATCH 118/193] print out BSP version on startup --- src/otaclient/boot_control/_jetson_uefi.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index 10ee69f8b..4d4e222ff 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -665,7 +665,7 @@ def __init__(self) -> None: current_fw_bsp_ver_fpath = ( current_ota_status_dir / boot_cfg.FIRMWARE_BSP_VERSION_FNAME ) - self._firmware_bsp_ver_control = FirmwareBSPVersionControl( + self._firmware_bsp_ver_control = bsp_ver_ctrl = FirmwareBSPVersionControl( current_slot=uefi_control.current_slot, current_slot_bsp_ver=uefi_control.fw_bsp_version, current_bsp_version_file=current_fw_bsp_ver_fpath, @@ -673,6 +673,10 @@ def __init__(self) -> None: # always update the bsp_version_file on startup to reflect # the up-to-date current slot BSP version self._firmware_bsp_ver_control.write_to_file(current_fw_bsp_ver_fpath) + logger.info( + f"\ncurrent slot firmware BSP version: {uefi_control.fw_bsp_version}\n" + f"standby slot firmware BSP version: {bsp_ver_ctrl.standby_slot_bsp_ver}" + ) # init ota-status files self._ota_status_control = OTAStatusFilesControl( @@ -683,6 +687,7 @@ def __init__(self) -> None: standby_ota_status_dir=standby_ota_status_dir, finalize_switching_boot=self._finalize_switching_boot, ) + logger.info("jetson-uefi boot control start up finished") except Exception as e: _err_msg = f"failed to start jetson-uefi controller: {e!r}" raise ota_errors.BootControlStartupFailed(_err_msg, module=__name__) from e From efbeb8d7791e512d87f67a59a5045617a1669ad6 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 21 Aug 2024 02:39:15 +0000 Subject: [PATCH 119/193] otaclient_common: minor typing update --- src/otaclient_common/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/otaclient_common/__init__.py b/src/otaclient_common/__init__.py index a73c58270..cd6e93f73 100644 --- a/src/otaclient_common/__init__.py +++ b/src/otaclient_common/__init__.py @@ -44,7 +44,9 @@ def get_file_size( return ceil(swapfile_fpath.stat().st_size / _multiplier[units]) -def replace_root(path: str | Path, old_root: str | Path, new_root: str | Path) -> str: +def replace_root( + path: str | Path, old_root: str | Path | Literal["/"], new_root: str | Path +) -> str: """Replace a relative to to . For example, if path="/abc", old_root="/", new_root="/new_root", From b270e27ed3a901a15f363a258dc41159eba85c53 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 21 Aug 2024 02:40:58 +0000 Subject: [PATCH 120/193] jetson-uefi: since bsp_version_control doesn't depends on ota_status anymore, now we can load BSP version control file after ota_status loading --- src/otaclient/boot_control/_jetson_uefi.py | 33 ++++++++++++---------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index 4d4e222ff..d616d17df 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -644,12 +644,6 @@ class JetsonUEFIBootControl(BootControllerProtocol): """BootControllerProtocol implementation for jetson-uefi.""" def __init__(self) -> None: - current_ota_status_dir = Path(boot_cfg.OTA_STATUS_DIR) - current_ota_status_dir.mkdir(exist_ok=True, parents=True) - standby_ota_status_dir = Path(cfg.MOUNT_POINT) / Path( - boot_cfg.OTA_STATUS_DIR - ).relative_to("/") - try: self._uefi_control = uefi_control = _UEFIBootControl() @@ -661,6 +655,24 @@ def __init__(self) -> None: active_slot_mount_point=cfg.ACTIVE_ROOT_MOUNT_POINT, ) + # init ota-status files + current_ota_status_dir = Path(boot_cfg.OTA_STATUS_DIR) + standby_ota_status_dir = Path( + replace_root( + boot_cfg.OTA_STATUS_DIR, + "/", + cfg.MOUNT_POINT, + ) + ) + self._ota_status_control = OTAStatusFilesControl( + active_slot=str(self._uefi_control.current_slot), + standby_slot=str(self._uefi_control.standby_slot), + current_ota_status_dir=current_ota_status_dir, + # NOTE: might not yet be populated before OTA update applied! + standby_ota_status_dir=standby_ota_status_dir, + finalize_switching_boot=self._finalize_switching_boot, + ) + # load firmware BSP version current_fw_bsp_ver_fpath = ( current_ota_status_dir / boot_cfg.FIRMWARE_BSP_VERSION_FNAME @@ -678,15 +690,6 @@ def __init__(self) -> None: f"standby slot firmware BSP version: {bsp_ver_ctrl.standby_slot_bsp_ver}" ) - # init ota-status files - self._ota_status_control = OTAStatusFilesControl( - active_slot=str(self._uefi_control.current_slot), - standby_slot=str(self._uefi_control.standby_slot), - current_ota_status_dir=current_ota_status_dir, - # NOTE: might not yet be populated before OTA update applied! - standby_ota_status_dir=standby_ota_status_dir, - finalize_switching_boot=self._finalize_switching_boot, - ) logger.info("jetson-uefi boot control start up finished") except Exception as e: _err_msg = f"failed to start jetson-uefi controller: {e!r}" From cf13f726c20773c2418969bc18f9b970680df239 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 21 Aug 2024 02:41:36 +0000 Subject: [PATCH 121/193] jetson-uefi: finish up integration of new firmware update package control --- src/otaclient/boot_control/_jetson_uefi.py | 88 +++++++++++++--------- 1 file changed, 52 insertions(+), 36 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index d616d17df..4a22a7be8 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -38,6 +38,8 @@ FirmwareManifest, FirmwareUpdateRequest, PayloadType, + load_manifest, + load_request, ) from otaclient_api.v2 import types as api_types from otaclient_common import replace_root @@ -708,42 +710,63 @@ def _finalize_switching_boot(self) -> bool: logger.warning(f"nvbootctrl verify failed: {e!r}") return True - def _capsule_firmware_update(self) -> bool: + def _firmware_update(self) -> bool: """Perform firmware update with UEFI Capsule update if needed. - If standby slot is known to have newer bootable firmware, skip firmware update. - Returns: True if there is firmware update configured, False for no firmware update. """ logger.info("jetson-uefi: checking if we need to do firmware update ...") - standby_bootloader_slot = self._uefi_control.standby_slot - # ------ check if we need to skip firmware update ------ # - skip_firmware_update_hint_file = ( - self._mp_control.standby_slot_mount_point - / Path(boot_cfg.CAPSULE_PAYLOAD_AT_ROOTFS).relative_to("/") - / Path(boot_cfg.NO_FIRMWARE_UPDATE_HINT_FNAME) - ) - if skip_firmware_update_hint_file.is_file(): - logger.warning( - "target image is configured to not doing firmware update, skip" - ) + # ------ check if we need to do firmware update ------ # + if not (tnspec := self._uefi_control.tnspec): + logger.warning("tnspec is not defined, skip firmware update") return False + # only perform update when we have a request file + firmware_update_request_fpath = Path( + replace_root( + boot_cfg.FIRMWARE_UPDATE_REQUEST_FPATH, + "/", + self._mp_control.standby_slot_mount_point, + ), + ) try: - ota_image_bsp_ver = detect_rootfs_bsp_version( - self._mp_control.standby_slot_mount_point + firmware_update_request = load_request(firmware_update_request_fpath) + except FileNotFoundError: + logger.warning("no firmware update request file presented, skip") + return False + except Exception as e: + logger.warning(f"invalid request file: {e!r}") + return False + + # if firmware package doesn't have a manifest file, skip update + firmware_manifest_fpath = Path( + replace_root( + boot_cfg.FIRMWARE_MANIFEST_FPATH, + "/", + self._mp_control.standby_slot_mount_point, ) + ) + try: + firmware_manifest = load_manifest(firmware_manifest_fpath) + except FileNotFoundError: + logger.warning("no firmware manifest file presented, skip") + return False except Exception as e: - logger.warning(f"failed to detect new image's BSP version: {e!r}") - logger.info("skip firmware update due to new image BSP version unknown") + logger.warning(f"invalid manifest file: {e!r}") return False - standby_firmware_bsp_ver = self._firmware_ver_control.standby_slot_fw_ver - if standby_firmware_bsp_ver and standby_firmware_bsp_ver >= ota_image_bsp_ver: + # if firmware package has older version than standby slot, skip update + fw_update_bsp_ver = BSPVersion.parse( + firmware_manifest.firmware_spec.bsp_version + ) + logger.info(f"firmware update package BSP version: {fw_update_bsp_ver}") + + standby_slot_bsp_ver = self._firmware_bsp_ver_control.standby_slot_bsp_ver + if standby_slot_bsp_ver and fw_update_bsp_ver < standby_slot_bsp_ver: logger.info( - f"{standby_bootloader_slot=} has newer or equal ver of firmware, skip firmware update" + "firmware package has older version than current standby slot, skip update" ) return False @@ -751,19 +774,12 @@ def _capsule_firmware_update(self) -> bool: firmware_updater = UEFIFirmwareUpdater( boot_parent_devpath=self._uefi_control.parent_devpath, standby_slot_mp=self._mp_control.standby_slot_mount_point, - fw_bsp_ver_control=self._firmware_ver_control, + tnspec=tnspec, + fw_bsp_ver_control=self._firmware_bsp_ver_control, + firmware_update_request=firmware_update_request, + firmware_manifest=firmware_manifest, ) - if firmware_updater.firmware_update(): - firmware_updater.l4tlauncher_update() - - logger.info( - ( - f"will update to new firmware version in next reboot: {ota_image_bsp_ver=}, \n" - f"will switch to Slot({standby_bootloader_slot}) on successful firmware update" - ) - ) - return True - return False + return firmware_updater.firmware_update() # APIs @@ -806,7 +822,7 @@ def post_update(self) -> Generator[None, None, None]: self._ota_status_control.standby_ota_status_dir / boot_cfg.FIRMWARE_BSP_VERSION_FNAME ) - self._firmware_ver_control.write_to_file(standby_fw_bsp_ver_fpath) + self._firmware_bsp_ver_control.write_to_file(standby_fw_bsp_ver_fpath) # ------ preserve /boot/ota folder to standby rootfs ------ # preserve_ota_config_files_to_standby( @@ -818,8 +834,8 @@ def post_update(self) -> Generator[None, None, None]: / "ota", ) - # ------ switch boot to standby ------ # - firmware_update_triggered = self._capsule_firmware_update() + # ------ firmware update & switch boot to standby ------ # + firmware_update_triggered = self._firmware_update() # NOTE: manual switch boot will cancel the scheduled firmware update! if not firmware_update_triggered: self._uefi_control.switch_boot_to_standby() From d83a4e3192da309d36ff2aa7f3b3d2fa6af7b1f8 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 21 Aug 2024 02:43:40 +0000 Subject: [PATCH 122/193] test_firmware_package: minor fix to typing --- .../test_otaclient/test_boot_control/test_firmware_package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_otaclient/test_boot_control/test_firmware_package.py b/tests/test_otaclient/test_boot_control/test_firmware_package.py index a5cf8a5ff..202f5396f 100644 --- a/tests/test_otaclient/test_boot_control/test_firmware_package.py +++ b/tests/test_otaclient/test_boot_control/test_firmware_package.py @@ -50,7 +50,7 @@ ), ), ) -def test_digest_value_parsing(_in, _expected): +def test_digest_value_parsing(_in, _expected: list[str]): _parsed = DigestValue(_in) assert _parsed.algorithm == _expected[0] assert _parsed.digest == _expected[1] @@ -72,7 +72,7 @@ def test_digest_value_parsing(_in, _expected): ), ), ) -def test_payload_file_location(_in, _expected): +def test_payload_file_location(_in, _expected: list[str] | list[str | DigestValue]): _parsed = PayloadFileLocation(_in) assert _parsed.location_type == _expected[0] assert _parsed.location_path == _expected[1] From a370f6e5632946b35aebd62f902fd66560c3dcfe Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 21 Aug 2024 02:46:28 +0000 Subject: [PATCH 123/193] boot_ctrl: regulate boot_control configs --- src/otaclient/boot_control/configs.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/otaclient/boot_control/configs.py b/src/otaclient/boot_control/configs.py index 192b45225..9f62ce65e 100644 --- a/src/otaclient/boot_control/configs.py +++ b/src/otaclient/boot_control/configs.py @@ -35,19 +35,24 @@ class GrubControlConfig(BaseConfig): class JetsonBootCommon: + # ota_status related OTA_STATUS_DIR = "/boot/ota-status" FIRMWARE_BSP_VERSION_FNAME = "firmware_bsp_version" + + # boot control related EXTLINUX_FILE = "/boot/extlinux/extlinux.conf" - FIRMWARE_DPATH = "/opt/ota_package" - """Refer to standby slot rootfs.""" MODEL_FPATH = "/proc/device-tree/model" - - NV_TEGRA_RELEASE_FPATH = "/etc/nv_tegra_release" - SEPARATE_BOOT_MOUNT_POINT = "/mnt/standby_boot" - MMCBLK_DEV_PREFIX = "mmcblk" # internal emmc NVMESSD_DEV_PREFIX = "nvme" # external nvme ssd INTERNAL_EMMC_DEVNAME = "mmcblk0" + NV_TEGRA_RELEASE_FPATH = "/etc/nv_tegra_release" + SEPARATE_BOOT_MOUNT_POINT = "/mnt/standby_boot" + + # firmware update related + NVBOOTCTRL_CONF_FPATH = "/etc/nv_boot_control.conf" + FIRMWARE_DPATH = "/opt/ota/firmware" + FIRMWARE_UPDATE_REQUEST_FPATH = f"{FIRMWARE_DPATH}/firmware_update.yaml" + FIRMWARE_MANIFEST_FPATH = f"{FIRMWARE_DPATH}/firmware_manifest.yaml" class JetsonCBootControlConfig(JetsonBootCommon): @@ -65,7 +70,6 @@ class JetsonCBootControlConfig(JetsonBootCommon): class JetsonUEFIBootControlConfig(JetsonBootCommon): BOOTLOADER = BootloaderType.JETSON_UEFI TEGRA_COMPAT_PATH = "/sys/firmware/devicetree/base/compatible" - NVBOOTCTRL_CONF_FPATH = "/etc/nv_boot_control.conf" L4TLAUNCHER_FNAME = "BOOTAA64.efi" ESP_MOUNTPOINT = "/mnt/esp" ESP_PARTLABEL = "esp" @@ -74,9 +78,6 @@ class JetsonUEFIBootControlConfig(JetsonBootCommon): CAPSULE_PAYLOAD_AT_ESP = "EFI/UpdateCapsule" L4TLAUNCHER_VER_FNAME = "l4tlauncher_version" - FIRMWARE_UPDATE_REQUEST_FPATH = "/opt/ota/firmware/firmware_update.yaml" - FIRMWARE_MANIFEST_FPATH = "/opt/ota/firmware/firmware_manifest.yaml" - @dataclass class RPIBootControlConfig(BaseConfig): From c21ccd7ded11ffaa6bd90c6759941f8a81396411 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 21 Aug 2024 03:38:52 +0000 Subject: [PATCH 124/193] firmware_package: extend to support BUP --- src/otaclient/boot_control/_firmware_package.py | 7 ++++--- .../test_boot_control/test_firmware_package.py | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/otaclient/boot_control/_firmware_package.py b/src/otaclient/boot_control/_firmware_package.py index 33396a878..c7dce77df 100644 --- a/src/otaclient/boot_control/_firmware_package.py +++ b/src/otaclient/boot_control/_firmware_package.py @@ -58,6 +58,7 @@ class PayloadType(str, Enum): UEFI_CAPSULE = "UEFI-CAPSULE" UEFI_BOOT_APP = "UEFI-BOOT-APP" + BUP = "BUP" DIGEST_PATTERN = re.compile(r"^(?P[\w\-+ ]+):(?P[a-zA-Z0-9]+)$") @@ -97,8 +98,8 @@ def check_compat(self, _tnspec: str) -> bool: ) -class NVIDIAUEFIFirmwareSpec(BaseModel): - # NOTE: let the jetson-uefi module parse the bsp_version +class NVIDIAFirmwareSpec(BaseModel): + # NOTE: let the boot control module parse BSP version bsp_version: str firmware_compat: NVIDIAFirmwareCompat @@ -158,7 +159,7 @@ class FirmwareManifest(BaseModel): ] hardware_series: str hardware_model: str - firmware_spec: NVIDIAUEFIFirmwareSpec + firmware_spec: NVIDIAFirmwareSpec firmware_packages: List[FirmwarePackage] def check_compat(self, _tnspec: str) -> bool: diff --git a/tests/test_otaclient/test_boot_control/test_firmware_package.py b/tests/test_otaclient/test_boot_control/test_firmware_package.py index 202f5396f..05ddc0c44 100644 --- a/tests/test_otaclient/test_boot_control/test_firmware_package.py +++ b/tests/test_otaclient/test_boot_control/test_firmware_package.py @@ -25,7 +25,7 @@ FirmwareUpdateRequest, HardwareType, NVIDIAFirmwareCompat, - NVIDIAUEFIFirmwareSpec, + NVIDIAFirmwareSpec, PayloadFileLocation, PayloadType, ) @@ -140,7 +140,7 @@ def test_check_compat(_spec, _compat_str, _expected): hardware_type=HardwareType("nvidia_jetson"), hardware_series="ADLINK RQX", hardware_model="rqx580", - firmware_spec=NVIDIAUEFIFirmwareSpec( + firmware_spec=NVIDIAFirmwareSpec( bsp_version="r35.4.1", firmware_compat=NVIDIAFirmwareCompat( board_id="2888", From b0392fadf339b9c0859da3ab7e5525fa645ea7d7 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 21 Aug 2024 03:43:06 +0000 Subject: [PATCH 125/193] jetson-cboot: NVUpdateEngine: use firmware package --- src/otaclient/boot_control/_jetson_cboot.py | 59 ++++++++++++++++++--- 1 file changed, 51 insertions(+), 8 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_cboot.py b/src/otaclient/boot_control/_jetson_cboot.py index e551ef121..06c796be1 100644 --- a/src/otaclient/boot_control/_jetson_cboot.py +++ b/src/otaclient/boot_control/_jetson_cboot.py @@ -146,11 +146,57 @@ def _nv_update_engine_unified_ab(cls, payload: Path | str): ) ) - @classmethod - def apply_firmware_update(cls, payload: Path | str, *, unified_ab: bool) -> None: - if unified_ab: - return cls._nv_update_engine_unified_ab(payload) - return cls._nv_update_engine(payload) + def __init__( + self, + tnspec: str, + fw_bsp_ver_control: FirmwareBSPVersionControl, + firmware_update_request: FirmwareUpdateRequest, + firmware_manifest: FirmwareManifest, + unify_ab: bool, + ) -> None: + self._tnspec = tnspec + self._fw_bsp_ver_control = fw_bsp_ver_control + self._firmware_update_request = firmware_update_request + self._firmware_manifest = firmware_manifest + self._unify_ab = unify_ab + + def firmware_update(self) -> bool: + """Perform firmware update if needed. + + Returns: + True if firmware update is performed, False if there is no firmware update. + """ + # check firmware compatibility, this is to prevent failed firmware update beforehand. + if not self._firmware_manifest.check_compat(self._tnspec): + _err_msg = ( + "firmware package is incompatible with this device: " + f"{self._tnspec=}, {self._firmware_manifest.firmware_spec.firmware_compat}, " + "skip firmware update" + ) + logger.warning(_err_msg) + return False + + firmware_update_executed = False + for update_payload in self._firmware_manifest.get_firmware_packages( + self._firmware_update_request + ): + if update_payload.type != PayloadType.BUP: + continue + + # NOTE: currently we only support payload indicated by file path. + bup_fpath = update_payload.file_location + assert not isinstance(bup_fpath, DigestValue) + + if not Path(bup_fpath).is_file(): + logger.warning(f"{bup_fpath=} doesn't exist! skip...") + continue + + if self._unify_ab: + self._nv_update_engine_unified_ab(bup_fpath) + else: + self._nv_update_engine(bup_fpath) + firmware_update_executed = True + return firmware_update_executed @classmethod def verify_update(cls) -> subprocess.CompletedProcess[bytes]: @@ -167,9 +213,6 @@ def verify_update(cls) -> subprocess.CompletedProcess[bytes]: class _CBootControl: - MMCBLK_DEV_PREFIX = "mmcblk" # internal emmc - NVMESSD_DEV_PREFIX = "nvme" # external nvme ssd - INTERNAL_EMMC_DEVNAME = "mmcblk0" _slot_id_partid = {SlotID("0"): "1", SlotID("1"): "2"} def __init__(self): From b0169657116c072a349c09ba8681f602a181d076 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 21 Aug 2024 06:43:36 +0000 Subject: [PATCH 126/193] minor update to NVUpdateEngine --- src/otaclient/boot_control/_jetson_cboot.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_cboot.py b/src/otaclient/boot_control/_jetson_cboot.py index 06c796be1..33d4f7bd9 100644 --- a/src/otaclient/boot_control/_jetson_cboot.py +++ b/src/otaclient/boot_control/_jetson_cboot.py @@ -148,6 +148,7 @@ def _nv_update_engine_unified_ab(cls, payload: Path | str): def __init__( self, + *, tnspec: str, fw_bsp_ver_control: FirmwareBSPVersionControl, firmware_update_request: FirmwareUpdateRequest, @@ -176,6 +177,12 @@ def firmware_update(self) -> bool: logger.warning(_err_msg) return False + update_execute_func = ( + self._nv_update_engine_unified_ab + if self._unify_ab + else self._nv_update_engine + ) + firmware_update_executed = False for update_payload in self._firmware_manifest.get_firmware_packages( self._firmware_update_request @@ -191,10 +198,7 @@ def firmware_update(self) -> bool: logger.warning(f"{bup_fpath=} doesn't exist! skip...") continue - if self._unify_ab: - self._nv_update_engine_unified_ab(bup_fpath) - else: - self._nv_update_engine(bup_fpath) + update_execute_func(bup_fpath) firmware_update_executed = True return firmware_update_executed From e3cf7c9440c7bf85eeb464aa4bbd450ef974ec61 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 21 Aug 2024 07:02:37 +0000 Subject: [PATCH 127/193] jetson-cboot: simplify logic of detecting unified ab --- src/otaclient/boot_control/_jetson_cboot.py | 44 +++++++++------------ 1 file changed, 18 insertions(+), 26 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_cboot.py b/src/otaclient/boot_control/_jetson_cboot.py index 33d4f7bd9..ca74fbe8d 100644 --- a/src/otaclient/boot_control/_jetson_cboot.py +++ b/src/otaclient/boot_control/_jetson_cboot.py @@ -27,17 +27,25 @@ from otaclient.app import errors as ota_errors from otaclient.app.configs import config as cfg +from otaclient.boot_control._firmware_package import ( + DigestValue, + FirmwareManifest, + FirmwareUpdateRequest, + PayloadType, +) from otaclient_api.v2 import types as api_types +from otaclient_common import replace_root from otaclient_common.common import subprocess_run_wrapper from ._common import CMDHelperFuncs, OTAStatusFilesControl, SlotMountHelper from ._jetson_common import ( + BSPVersion, FirmwareBSPVersionControl, NVBootctrlCommon, NVBootctrlTarget, SlotID, copy_standby_slot_boot_to_internal_emmc, - parse_bsp_version, + detect_rootfs_bsp_version, preserve_ota_config_files_to_standby, update_standby_slot_extlinux_cfg, ) @@ -97,9 +105,9 @@ def is_unified_enabled(cls) -> bool | None: except subprocess.CalledProcessError as e: if e.returncode == 70: return False - elif e.returncode == 69: + if e.returncode == 69: return - raise ValueError(f"{cmd} returns unexpected result: {e.returncode=}, {e!r}") + logger.warning(f"{cmd} returns unexpected result: {e.returncode=}, {e!r}") class NVUpdateEngine: @@ -230,19 +238,19 @@ def __init__(self): # ------ check BSP version ------ # try: - self.bsp_version = bsp_version = parse_bsp_version( - Path(boot_cfg.NV_TEGRA_RELEASE_FPATH).read_text() + self.rootfs_bsp_version = rootfs_bsp_version = detect_rootfs_bsp_version( + rootfs=cfg.ACTIVE_ROOTFS_PATH ) except Exception as e: _err_msg = f"failed to detect BSP version: {e!r}" logger.error(_err_msg) raise JetsonCBootContrlError(_err_msg) - logger.info(f"{bsp_version=}") + logger.info(f"{rootfs_bsp_version=}") # ------ sanity check, jetson-cboot is not used after BSP R34 ------ # - if not bsp_version < (34, 0, 0): + if rootfs_bsp_version >= (34, 0, 0): _err_msg = ( - f"jetson-cboot only supports BSP version < R34, but get {bsp_version=}. " + f"jetson-cboot only supports BSP version < R34, but get {rootfs_bsp_version=}. " "Please use jetson-uefi bootloader type for device with BSP >= R34." ) logger.error(_err_msg) @@ -251,27 +259,11 @@ def __init__(self): # ------ check if unified A/B is enabled ------ # # NOTE: mismatch rootfs BSP version and bootloader firmware BSP version # is NOT supported and MUST not occur. - unified_ab_enabled = False - if bsp_version >= (32, 6, 0): - # NOTE: unified A/B is supported starting from r32.6 - unified_ab_enabled = _NVBootctrl.is_unified_enabled() - if unified_ab_enabled is None: - _err_msg = "rootfs A/B is not enabled!" - logger.error(_err_msg) - raise JetsonCBootContrlError(_err_msg) - else: # R32.5 and below doesn't support unified A/B - try: - _NVBootctrl.get_current_slot(target="rootfs") - except subprocess.CalledProcessError: - _err_msg = "rootfs A/B is not enabled!" - logger.error(_err_msg) - raise JetsonCBootContrlError(_err_msg) - self.unified_ab_enabled = unified_ab_enabled - - if unified_ab_enabled: + if unified_ab_enabled := _NVBootctrl.is_unified_enabled(): logger.info( "unified A/B is enabled, rootfs and bootloader will be switched together" ) + self.unified_ab_enabled = unified_ab_enabled # ------ check A/B slots ------ # self.current_bootloader_slot = current_bootloader_slot = ( From 8e3990db30ac97111d39f38955ca95c732631b08 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 21 Aug 2024 07:50:12 +0000 Subject: [PATCH 128/193] jetson-cboot: integrate to use new firmware update package control --- src/otaclient/boot_control/_jetson_cboot.py | 172 +++++++++++--------- 1 file changed, 95 insertions(+), 77 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_cboot.py b/src/otaclient/boot_control/_jetson_cboot.py index ca74fbe8d..31d787698 100644 --- a/src/otaclient/boot_control/_jetson_cboot.py +++ b/src/otaclient/boot_control/_jetson_cboot.py @@ -32,6 +32,8 @@ FirmwareManifest, FirmwareUpdateRequest, PayloadType, + load_manifest, + load_request, ) from otaclient_api.v2 import types as api_types from otaclient_common import replace_root @@ -39,13 +41,13 @@ from ._common import CMDHelperFuncs, OTAStatusFilesControl, SlotMountHelper from ._jetson_common import ( - BSPVersion, FirmwareBSPVersionControl, NVBootctrlCommon, NVBootctrlTarget, SlotID, copy_standby_slot_boot_to_internal_emmc, detect_rootfs_bsp_version, + get_nvbootctrl_conf_tnspec, preserve_ota_config_files_to_standby, update_standby_slot_extlinux_cfg, ) @@ -237,6 +239,9 @@ def __init__(self): raise JetsonCBootContrlError(_err_msg) # ------ check BSP version ------ # + # NOTE(20240821): unfortunately, we don't have proper method to detect + # the firmware BSP version, so we assume that the rootfs BSP version is the + # same as the firmware BSP version. try: self.rootfs_bsp_version = rootfs_bsp_version = detect_rootfs_bsp_version( rootfs=cfg.ACTIVE_ROOTFS_PATH @@ -341,6 +346,21 @@ def __init__(self): f"nvbootctrl -t rootfs dump-slots-info: \n{_NVBootctrl.dump_slots_info(target='rootfs')}" ) + # load tnspec for firmware update compatibility check + try: + self.tnspec = get_nvbootctrl_conf_tnspec( + Path(boot_cfg.NVBOOTCTRL_CONF_FPATH).read_text() + ) + logger.info(f"firmware compatibility: {self.tnspec}") + except Exception as e: + logger.warning( + ( + f"failed to load tnspec: {e!r}, " + "this will result in firmware update being skipped!" + ) + ) + self.tnspec = None + # API @property @@ -374,7 +394,7 @@ class JetsonCBootControl(BootControllerProtocol): def __init__(self) -> None: try: # startup boot controller - self._cboot_control = _CBootControl() + self._cboot_control = cboot_control = _CBootControl() # mount point prepare self._mp_control = SlotMountHelper( @@ -383,20 +403,16 @@ def __init__(self) -> None: active_slot_dev=self._cboot_control.curent_rootfs_devpath, active_slot_mount_point=cfg.ACTIVE_ROOT_MOUNT_POINT, ) - current_ota_status_dir = Path(boot_cfg.OTA_STATUS_DIR) - standby_ota_status_dir = self._mp_control.standby_slot_mount_point / Path( - boot_cfg.OTA_STATUS_DIR - ).relative_to("/") - - # load firmware BSP version from current rootfs slot - self._firmware_ver_control = FirmwareBSPVersionControl( - current_firmware_bsp_vf=current_ota_status_dir - / boot_cfg.FIRMWARE_BSP_VERSION_FNAME, - standby_firmware_bsp_vf=standby_ota_status_dir - / boot_cfg.FIRMWARE_BSP_VERSION_FNAME, - ) # init ota-status files + current_ota_status_dir = Path(boot_cfg.OTA_STATUS_DIR) + standby_ota_status_dir = Path( + replace_root( + boot_cfg.OTA_STATUS_DIR, + "/", + cfg.MOUNT_POINT, + ) + ) self._ota_status_control = OTAStatusFilesControl( active_slot=str(self._cboot_control.current_rootfs_slot), standby_slot=str(self._cboot_control.standby_rootfs_slot), @@ -405,6 +421,26 @@ def __init__(self) -> None: standby_ota_status_dir=standby_ota_status_dir, finalize_switching_boot=self._finalize_switching_boot, ) + + # load firmware BSP version + current_fw_bsp_ver_fpath = ( + current_ota_status_dir / boot_cfg.FIRMWARE_BSP_VERSION_FNAME + ) + self._firmware_bsp_ver_control = bsp_ver_ctrl = FirmwareBSPVersionControl( + current_slot=cboot_control.current_bootloader_slot, + # NOTE: see comments at L240-242 + current_slot_bsp_ver=cboot_control.rootfs_bsp_version, + current_bsp_version_file=current_fw_bsp_ver_fpath, + ) + # always update the bsp_version_file on startup to reflect + # the up-to-date current slot BSP version + self._firmware_bsp_ver_control.write_to_file(current_fw_bsp_ver_fpath) + logger.info( + f"\ncurrent slot firmware BSP version: {bsp_ver_ctrl.current_slot_bsp_ver}\n" + f"standby slot firmware BSP version: {bsp_ver_ctrl.standby_slot_bsp_ver}" + ) + + logger.info("jetson-cboot boot control start up finished") except Exception as e: _err_msg = f"failed to start jetson-cboot controller: {e!r}" raise ota_errors.BootControlStartupFailed(_err_msg, module=__name__) from e @@ -416,7 +452,6 @@ def _finalize_switching_boot(self) -> bool: Also if unified A/B is NOT enabled and everything is alright, execute mark-boot-success to mark the current booted rootfs boots successfully. """ - current_boot_slot = self._cboot_control.current_bootloader_slot current_rootfs_slot = self._cboot_control.current_rootfs_slot update_result = NVUpdateEngine.verify_update() @@ -428,10 +463,6 @@ def _finalize_switching_boot(self) -> bool: "failing the OTA and clear firmware version due to new bootloader slot boot failed." ) logger.error(_err_msg) - - # NOTE: always only change current slots firmware_bsp_version file here. - self._firmware_ver_control.set_version_by_slot(current_boot_slot, None) - self._firmware_ver_control.write_current_firmware_bsp_version() return False # NOTE(20240417): rootfs slot is manually switched by set-active-boot-slot, @@ -444,82 +475,69 @@ def _finalize_switching_boot(self) -> bool: ) return True - def _nv_firmware_update(self) -> Optional[bool]: - """Perform firmware update with nv_update_engine. + def _firmware_update(self) -> bool | None: + """Perform firmware update with nv_update_engine if needed. Returns: True if firmware update applied, False for failed firmware update, None for no firmware update occurs. """ logger.info("jetson-cboot: entering nv firmware update ...") - standby_bootloader_slot = self._cboot_control.standby_bootloader_slot - standby_firmware_bsp_ver = self._firmware_ver_control.get_version_by_slot( - standby_bootloader_slot - ) - logger.info(f"{standby_bootloader_slot=} BSP ver: {standby_firmware_bsp_ver}") # ------ check if we need to do firmware update ------ # - _new_bsp_v_fpath = self._mp_control.standby_slot_mount_point / Path( - boot_cfg.NV_TEGRA_RELEASE_FPATH - ).relative_to("/") + if not (tnspec := self._cboot_control.tnspec): + logger.warning("tnspec is not defined, skip firmware update") + return + + # only perform update when we have a request file + firmware_update_request_fpath = Path( + replace_root( + boot_cfg.FIRMWARE_UPDATE_REQUEST_FPATH, + "/", + self._mp_control.standby_slot_mount_point, + ), + ) try: - new_bsp_v = parse_bsp_version(_new_bsp_v_fpath.read_text()) + firmware_update_request = load_request(firmware_update_request_fpath) + except FileNotFoundError: + logger.warning("no firmware update request file presented, skip") + return except Exception as e: - logger.warning(f"failed to detect new image's BSP version: {e!r}") - logger.warning("skip firmware update due to new image BSP version unknown") + logger.warning(f"invalid request file: {e!r}") return - logger.info(f"BUP package version: {new_bsp_v=}") - if standby_firmware_bsp_ver and standby_firmware_bsp_ver >= new_bsp_v: - logger.info( - f"{standby_bootloader_slot=} has newer or equal ver of firmware, skip firmware update" + # if firmware package doesn't have a manifest file, skip update + firmware_manifest_fpath = Path( + replace_root( + boot_cfg.FIRMWARE_MANIFEST_FPATH, + "/", + self._mp_control.standby_slot_mount_point, ) + ) + try: + firmware_manifest = load_manifest(firmware_manifest_fpath) + except FileNotFoundError: + logger.warning("no firmware manifest file presented, skip") + return + except Exception as e: + logger.warning(f"invalid manifest file: {e!r}") return + standby_bootloader_slot = self._cboot_control.standby_bootloader_slot + standby_firmware_bsp_ver = self._firmware_bsp_ver_control.standby_slot_bsp_ver + logger.info(f"{standby_bootloader_slot=} BSP ver: {standby_firmware_bsp_ver}") + # ------ preform firmware update ------ # - firmware_dpath = self._mp_control.standby_slot_mount_point / Path( - boot_cfg.FIRMWARE_DPATH - ).relative_to("/") - - _firmware_applied = False - for firmware_fname in boot_cfg.FIRMWARE_LIST: - if (firmware_fpath := firmware_dpath / firmware_fname).is_file(): - logger.info(f"nv_firmware: apply {firmware_fpath} ...") - try: - NVUpdateEngine.apply_firmware_update( - firmware_fpath, - unified_ab=bool(self._cboot_control.unified_ab_enabled), - ) - _firmware_applied = True - except subprocess.CalledProcessError as e: - _err_msg = ( - f"failed to apply BUP {firmware_fpath}: {e!r}, \n" - f"stderr={e.stderr.decode()}, \n" - f"stdout={e.stdout.decode()}" - ) - logger.error(_err_msg) - logger.warning("firmware update interrupted, failing OTA...") - - # if the firmware update is interrupted halfway(some of the BUP is applied), - # revert bootloader slot switch - if _firmware_applied: - logger.warning( - "revert bootloader slot switch to current active slot" - ) - _NVBootctrl.set_active_boot_slot( - self._cboot_control.current_bootloader_slot - ) - return False + firmware_updater = NVUpdateEngine( + tnspec=tnspec, + fw_bsp_ver_control=self._firmware_bsp_ver_control, + firmware_update_request=firmware_update_request, + firmware_manifest=firmware_manifest, + unify_ab=bool(self._cboot_control.unified_ab_enabled), + ) # ------ register new firmware version ------ # - if _firmware_applied: - logger.info( - f"nv_firmware: successfully apply firmware to {self._cboot_control.standby_rootfs_slot=}" - ) - self._firmware_ver_control.set_version_by_slot( - standby_bootloader_slot, new_bsp_v - ) - return True + return firmware_updater.firmware_update() # APIs From 82e28731e18f90656ad94e3a01d1ff4f0f9b1fc2 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 21 Aug 2024 07:53:13 +0000 Subject: [PATCH 129/193] jetson-cboot: finish up integrating the new firmware package control --- src/otaclient/boot_control/_jetson_cboot.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_cboot.py b/src/otaclient/boot_control/_jetson_cboot.py index 31d787698..ed9efe0c4 100644 --- a/src/otaclient/boot_control/_jetson_cboot.py +++ b/src/otaclient/boot_control/_jetson_cboot.py @@ -585,16 +585,20 @@ def post_update(self) -> Generator[None, None, None]: ) # ------ firmware update ------ # - firmware_update_result = self._nv_firmware_update() - if firmware_update_result: - self._firmware_ver_control.write_current_firmware_bsp_version() - elif firmware_update_result is None: + firmware_update_result = self._firmware_update() + if firmware_update_result is None: logger.info("no firmware update occurs") - else: + elif firmware_update_result is False: raise JetsonCBootContrlError("firmware update failed") + else: + logger.info("new firmware is written to the standby slot") # ------ preserve BSP version files to standby slot ------ # - self._firmware_ver_control.write_standby_firmware_bsp_version() + standby_fw_bsp_ver_fpath = ( + self._ota_status_control.standby_ota_status_dir + / boot_cfg.FIRMWARE_BSP_VERSION_FNAME + ) + self._firmware_bsp_ver_control.write_to_file(standby_fw_bsp_ver_fpath) # ------ preserve /boot/ota folder to standby rootfs ------ # preserve_ota_config_files_to_standby( From c2c1d9f20f18581bc7848b32b8e38b41e629a1e1 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 21 Aug 2024 07:59:59 +0000 Subject: [PATCH 130/193] firmware_package: add helper load_firmware_package API --- .../boot_control/_firmware_package.py | 46 +++++++++++++------ 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/src/otaclient/boot_control/_firmware_package.py b/src/otaclient/boot_control/_firmware_package.py index c7dce77df..1b718c491 100644 --- a/src/otaclient/boot_control/_firmware_package.py +++ b/src/otaclient/boot_control/_firmware_package.py @@ -42,6 +42,7 @@ from __future__ import annotations +import logging import re from enum import Enum from pathlib import Path @@ -54,6 +55,8 @@ from otaclient_common.typing import StrOrPath, gen_strenum_validator +logger = logging.getLogger(__name__) + class PayloadType(str, Enum): UEFI_CAPSULE = "UEFI-CAPSULE" @@ -191,17 +194,32 @@ class FirmwareUpdateRequest(BaseModel): firmware_list: List[str] -def load_request(request_fpath: StrOrPath) -> FirmwareUpdateRequest: - """Load update request from .""" - _raw = Path(request_fpath).read_text() - _raw_loaded = yaml.safe_load(_raw) - _res = FirmwareUpdateRequest.model_validate(_raw_loaded) - return _res - - -def load_manifest(manifest_fpath: StrOrPath) -> FirmwareManifest: - """Load manifest from .""" - _raw = Path(manifest_fpath).read_text() - _raw_loaded = yaml.safe_load(_raw) - _res = FirmwareManifest.model_validate(_raw_loaded) - return _res +def load_firmware_package( + *, + firmware_update_request_fpath: StrOrPath, + firmware_manifest_fpath: StrOrPath, +) -> tuple[FirmwareUpdateRequest, FirmwareManifest] | None: + """Parse firmware update package.""" + try: + firmware_update_request = FirmwareUpdateRequest.model_validate( + yaml.safe_load(Path(firmware_update_request_fpath).read_text()) + ) + except FileNotFoundError: + logger.info("firmware update request file not found, skip firmware update") + return + except Exception as e: + logger.warning(f"invalid request file: {e!r}") + return + + try: + firmware_manifest = FirmwareManifest.model_validate( + yaml.safe_load(Path(firmware_manifest_fpath).read_text()) + ) + except FileNotFoundError: + logger.warning("no firmware manifest file presented, skip firmware update!") + return + except Exception as e: + logger.warning(f"invalid manifest file: {e!r}") + return + + return firmware_update_request, firmware_manifest From d11b6f2861cd1ff388531883935e9000ed6abeb7 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 21 Aug 2024 08:05:26 +0000 Subject: [PATCH 131/193] jetson-cboot: use load_firmware_package API --- src/otaclient/boot_control/_jetson_cboot.py | 43 +++++---------------- 1 file changed, 9 insertions(+), 34 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_cboot.py b/src/otaclient/boot_control/_jetson_cboot.py index ed9efe0c4..8451864a0 100644 --- a/src/otaclient/boot_control/_jetson_cboot.py +++ b/src/otaclient/boot_control/_jetson_cboot.py @@ -32,8 +32,7 @@ FirmwareManifest, FirmwareUpdateRequest, PayloadType, - load_manifest, - load_request, + load_firmware_package, ) from otaclient_api.v2 import types as api_types from otaclient_common import replace_root @@ -208,6 +207,7 @@ def firmware_update(self) -> bool: logger.warning(f"{bup_fpath=} doesn't exist! skip...") continue + logger.warning(f"apply BUP {bup_fpath} to standby slot ...") update_execute_func(bup_fpath) firmware_update_executed = True return firmware_update_executed @@ -483,49 +483,26 @@ def _firmware_update(self) -> bool | None: None for no firmware update occurs. """ logger.info("jetson-cboot: entering nv firmware update ...") - - # ------ check if we need to do firmware update ------ # if not (tnspec := self._cboot_control.tnspec): logger.warning("tnspec is not defined, skip firmware update") return - # only perform update when we have a request file - firmware_update_request_fpath = Path( - replace_root( + firmware_package_meta = load_firmware_package( + firmware_update_request_fpath=replace_root( boot_cfg.FIRMWARE_UPDATE_REQUEST_FPATH, "/", self._mp_control.standby_slot_mount_point, ), - ) - try: - firmware_update_request = load_request(firmware_update_request_fpath) - except FileNotFoundError: - logger.warning("no firmware update request file presented, skip") - return - except Exception as e: - logger.warning(f"invalid request file: {e!r}") - return - - # if firmware package doesn't have a manifest file, skip update - firmware_manifest_fpath = Path( - replace_root( + firmware_manifest_fpath=replace_root( boot_cfg.FIRMWARE_MANIFEST_FPATH, "/", self._mp_control.standby_slot_mount_point, - ) + ), ) - try: - firmware_manifest = load_manifest(firmware_manifest_fpath) - except FileNotFoundError: - logger.warning("no firmware manifest file presented, skip") + if firmware_package_meta is None: + logger.info("skip firmware update ...") return - except Exception as e: - logger.warning(f"invalid manifest file: {e!r}") - return - - standby_bootloader_slot = self._cboot_control.standby_bootloader_slot - standby_firmware_bsp_ver = self._firmware_bsp_ver_control.standby_slot_bsp_ver - logger.info(f"{standby_bootloader_slot=} BSP ver: {standby_firmware_bsp_ver}") + firmware_update_request, firmware_manifest = firmware_package_meta # ------ preform firmware update ------ # firmware_updater = NVUpdateEngine( @@ -535,8 +512,6 @@ def _firmware_update(self) -> bool | None: firmware_manifest=firmware_manifest, unify_ab=bool(self._cboot_control.unified_ab_enabled), ) - - # ------ register new firmware version ------ # return firmware_updater.firmware_update() # APIs From ece7e95ac81e1275264c942d2a01dd95d532c694 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 21 Aug 2024 08:10:46 +0000 Subject: [PATCH 132/193] jetson-uefi: use new load_firmware_package API; remove duplicated logic --- src/otaclient/boot_control/_jetson_uefi.py | 44 ++++------------------ 1 file changed, 8 insertions(+), 36 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index 4a22a7be8..f61400e08 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -38,8 +38,7 @@ FirmwareManifest, FirmwareUpdateRequest, PayloadType, - load_manifest, - load_request, + load_firmware_package, ) from otaclient_api.v2 import types as api_types from otaclient_common import replace_root @@ -717,59 +716,32 @@ def _firmware_update(self) -> bool: True if there is firmware update configured, False for no firmware update. """ logger.info("jetson-uefi: checking if we need to do firmware update ...") - - # ------ check if we need to do firmware update ------ # if not (tnspec := self._uefi_control.tnspec): logger.warning("tnspec is not defined, skip firmware update") return False - # only perform update when we have a request file - firmware_update_request_fpath = Path( - replace_root( + firmware_package_meta = load_firmware_package( + firmware_update_request_fpath=replace_root( boot_cfg.FIRMWARE_UPDATE_REQUEST_FPATH, "/", self._mp_control.standby_slot_mount_point, ), - ) - try: - firmware_update_request = load_request(firmware_update_request_fpath) - except FileNotFoundError: - logger.warning("no firmware update request file presented, skip") - return False - except Exception as e: - logger.warning(f"invalid request file: {e!r}") - return False - - # if firmware package doesn't have a manifest file, skip update - firmware_manifest_fpath = Path( - replace_root( + firmware_manifest_fpath=replace_root( boot_cfg.FIRMWARE_MANIFEST_FPATH, "/", self._mp_control.standby_slot_mount_point, - ) + ), ) - try: - firmware_manifest = load_manifest(firmware_manifest_fpath) - except FileNotFoundError: - logger.warning("no firmware manifest file presented, skip") - return False - except Exception as e: - logger.warning(f"invalid manifest file: {e!r}") + if firmware_package_meta is None: + logger.info("skip firmware update ...") return False + firmware_update_request, firmware_manifest = firmware_package_meta - # if firmware package has older version than standby slot, skip update fw_update_bsp_ver = BSPVersion.parse( firmware_manifest.firmware_spec.bsp_version ) logger.info(f"firmware update package BSP version: {fw_update_bsp_ver}") - standby_slot_bsp_ver = self._firmware_bsp_ver_control.standby_slot_bsp_ver - if standby_slot_bsp_ver and fw_update_bsp_ver < standby_slot_bsp_ver: - logger.info( - "firmware package has older version than current standby slot, skip update" - ) - return False - # ------ prepare firmware update ------ # firmware_updater = UEFIFirmwareUpdater( boot_parent_devpath=self._uefi_control.parent_devpath, From 35153438ce86f8c33293e9b1b5619efba4d3ab2e Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 21 Aug 2024 08:11:14 +0000 Subject: [PATCH 133/193] jetson-cboot: minor fix --- src/otaclient/boot_control/_jetson_cboot.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_cboot.py b/src/otaclient/boot_control/_jetson_cboot.py index 8451864a0..30e6e1839 100644 --- a/src/otaclient/boot_control/_jetson_cboot.py +++ b/src/otaclient/boot_control/_jetson_cboot.py @@ -40,6 +40,7 @@ from ._common import CMDHelperFuncs, OTAStatusFilesControl, SlotMountHelper from ._jetson_common import ( + BSPVersion, FirmwareBSPVersionControl, NVBootctrlCommon, NVBootctrlTarget, @@ -159,13 +160,11 @@ def __init__( self, *, tnspec: str, - fw_bsp_ver_control: FirmwareBSPVersionControl, firmware_update_request: FirmwareUpdateRequest, firmware_manifest: FirmwareManifest, unify_ab: bool, ) -> None: self._tnspec = tnspec - self._fw_bsp_ver_control = fw_bsp_ver_control self._firmware_update_request = firmware_update_request self._firmware_manifest = firmware_manifest self._unify_ab = unify_ab @@ -505,9 +504,13 @@ def _firmware_update(self) -> bool | None: firmware_update_request, firmware_manifest = firmware_package_meta # ------ preform firmware update ------ # + fw_update_bsp_ver = BSPVersion.parse( + firmware_manifest.firmware_spec.bsp_version + ) + logger.info(f"firmware update package BSP version: {fw_update_bsp_ver}") + firmware_updater = NVUpdateEngine( tnspec=tnspec, - fw_bsp_ver_control=self._firmware_bsp_ver_control, firmware_update_request=firmware_update_request, firmware_manifest=firmware_manifest, unify_ab=bool(self._cboot_control.unified_ab_enabled), From e778e05aab78dac56ec2a679dac02092e4e955c2 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 21 Aug 2024 08:33:48 +0000 Subject: [PATCH 134/193] jetson-uefi: minor fix --- src/otaclient/boot_control/_jetson_uefi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index f61400e08..5d6782523 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -102,7 +102,7 @@ class NVBootctrlJetsonUEFI(NVBootctrlCommon): def get_current_fw_bsp_version(cls) -> BSPVersion: """Get current boot chain's firmware BSP version with nvbootctrl.""" _raw = cls.dump_slots_info() - pa = re.compile(r"\s*Current version:\s*(?P[\.\d]+)\s*") + pa = re.compile(r"Current version:\s*(?P[\.\d]+)") if not (ma := pa.search(_raw)): _err_msg = "nvbootctrl failed to report BSP version" From c7e64eff13771ac3ff10d53e3af45f4c6c4c477e Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 21 Aug 2024 08:48:53 +0000 Subject: [PATCH 135/193] jetson-uefi: minor fix --- src/otaclient/boot_control/_jetson_uefi.py | 29 +++++++++++----------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index 5d6782523..f66585c5f 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -184,7 +184,7 @@ def _detect_esp_dev(boot_parent_devpath: StrOrPath) -> str: raise JetsonUEFIBootControlError("no ESP partition presented") for _esp_part in esp_parts: - if _esp_part.find(str(boot_parent_devpath)) != -1: + if _esp_part.strip().startswith(str(boot_parent_devpath)): logger.info(f"find esp partition at {_esp_part}") esp_part = _esp_part break @@ -196,7 +196,8 @@ def _detect_esp_dev(boot_parent_devpath: StrOrPath) -> str: class L4TLauncherBSPVersionControl(BaseModel): - """ + """Track the L4TLauncher binary BSP version. + Schema: : """ @@ -232,11 +233,12 @@ def __init__( boot_parent_devpath (StrOrPath): The parent dev of current rootfs device. standby_slot_mp (StrOrPath): The mount point of updated standby slot. The firmware update package is located at /opt/ota_package folder. - ota_image_bsp_ver (BSPVersion): The BSP version of OTA image used to update the standby slot. - fw_bsp_ver_control (FirmwareBSPVersionControl): The firmware BSP version of each slots. + tnspec (str): The firmware compatibility string. + fw_bsp_ver_control (FirmwareBSPVersionControl): The firmware BSP version control for each slots. + firmware_update_request (FirmwareUpdateRequest) + firmware_manifest (FirmwareBSPVersionControl) """ self.standby_slot_bsp_ver = fw_bsp_ver_control.standby_slot_bsp_ver - self.current_slot_bsp_ver = fw_bsp_ver_control.current_slot_bsp_ver self.tnspec = tnspec self.firmware_update_request = firmware_update_request @@ -245,6 +247,7 @@ def __init__( firmware_manifest.firmware_spec.bsp_version ) + self.esp_part = _detect_esp_dev(boot_parent_devpath) # NOTE: use the esp partition at the current booted device # i.e., if we boot from nvme0n1, then bootdev_path is /dev/nvme0n1 and # we use the esp at nvme0n1. @@ -254,25 +257,23 @@ def __init__( self.esp_boot_dir = self.esp_mp / "EFI" / "BOOT" self.l4tlauncher_ver_fpath = self.esp_boot_dir / boot_cfg.L4TLAUNCHER_VER_FNAME """A plain text file stores the BSP version string.""" - self.bootaa64_at_esp = self.esp_boot_dir / boot_cfg.L4TLAUNCHER_FNAME """The canonical location of L4TLauncher, called by UEFI.""" - self.bootaa64_at_esp_bak = ( self.esp_boot_dir / f"{boot_cfg.L4TLAUNCHER_FNAME}_bak" ) """The location to backup current L4TLauncher binary.""" - - self.standby_slot_mp = Path(standby_slot_mp) - - self.esp_part = _detect_esp_dev(boot_parent_devpath) - self.capsule_dir_at_esp = self.esp_mp / boot_cfg.CAPSULE_PAYLOAD_AT_ESP """The location to put UEFI capsule to. The UEFI will use the capsule in this location.""" + self.standby_slot_mp = Path(standby_slot_mp) + def _prepare_fwupdate_capsule(self) -> bool: """Copy the Capsule update payloads to specific location at esp partition. + The UEFI framework will look for the firmware update capsule there and start + the firmware update. + Returns: True if at least one of the update capsule is prepared, False if no update capsule is available and configured. @@ -305,7 +306,7 @@ def _prepare_fwupdate_capsule(self) -> bool: logger.warning( f"failed to copy {capsule_payload.payload_name} to {capsule_dir_at_esp}: {e!r}" ) - logger.warning(f"skip prepare {capsule_payload.payload_name}") + logger.warning(f"skip preparing {capsule_payload.payload_name}") return firmware_package_configured def _update_l4tlauncher(self) -> bool: @@ -316,7 +317,7 @@ def _update_l4tlauncher(self) -> bool: for capsule_payload in self.firmware_manifest.get_firmware_packages( self.firmware_update_request ): - if capsule_payload.type != PayloadType.UEFI_CAPSULE: + if capsule_payload.type != PayloadType.UEFI_BOOT_APP: continue # NOTE: currently we only support payload indicated by file path. From bd7419e908d455f35cfa758e189656e9da26147c Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 21 Aug 2024 08:51:49 +0000 Subject: [PATCH 136/193] jetson-uefi: minor update --- src/otaclient/boot_control/_jetson_uefi.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index f66585c5f..350cc32b1 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -238,6 +238,7 @@ def __init__( firmware_update_request (FirmwareUpdateRequest) firmware_manifest (FirmwareBSPVersionControl) """ + self.current_slot_bsp_ver = fw_bsp_ver_control.current_slot_bsp_ver self.standby_slot_bsp_ver = fw_bsp_ver_control.standby_slot_bsp_ver self.tnspec = tnspec @@ -254,14 +255,12 @@ def __init__( self.esp_mp = Path(boot_cfg.ESP_MOUNTPOINT) self.esp_mp.mkdir(exist_ok=True) - self.esp_boot_dir = self.esp_mp / "EFI" / "BOOT" - self.l4tlauncher_ver_fpath = self.esp_boot_dir / boot_cfg.L4TLAUNCHER_VER_FNAME + esp_boot_dir = self.esp_mp / "EFI" / "BOOT" + self.l4tlauncher_ver_fpath = esp_boot_dir / boot_cfg.L4TLAUNCHER_VER_FNAME """A plain text file stores the BSP version string.""" - self.bootaa64_at_esp = self.esp_boot_dir / boot_cfg.L4TLAUNCHER_FNAME + self.bootaa64_at_esp = esp_boot_dir / boot_cfg.L4TLAUNCHER_FNAME """The canonical location of L4TLauncher, called by UEFI.""" - self.bootaa64_at_esp_bak = ( - self.esp_boot_dir / f"{boot_cfg.L4TLAUNCHER_FNAME}_bak" - ) + self.bootaa64_at_esp_bak = esp_boot_dir / f"{boot_cfg.L4TLAUNCHER_FNAME}_bak" """The location to backup current L4TLauncher binary.""" self.capsule_dir_at_esp = self.esp_mp / boot_cfg.CAPSULE_PAYLOAD_AT_ESP """The location to put UEFI capsule to. The UEFI will use the capsule in this location.""" From bc6a8e5a27e69142f627bb3c999b5ca43bcaea4d Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 21 Aug 2024 09:10:12 +0000 Subject: [PATCH 137/193] jetson-uefi: refine UEFIFirmwareUpdater --- src/otaclient/boot_control/_jetson_uefi.py | 54 ++++++++++++---------- 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index 350cc32b1..0273fc912 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -310,15 +310,16 @@ def _prepare_fwupdate_capsule(self) -> bool: def _update_l4tlauncher(self) -> bool: """update L4TLauncher with OTA image's one.""" - logger.warning( - f"update the l4tlauncher to version {self.firmware_package_bsp_ver}" - ) for capsule_payload in self.firmware_manifest.get_firmware_packages( self.firmware_update_request ): if capsule_payload.type != PayloadType.UEFI_BOOT_APP: continue + logger.warning( + f"update the l4tlauncher to version {self.firmware_package_bsp_ver} ..." + ) + # NOTE: currently we only support payload indicated by file path. bootapp_fpath = capsule_payload.file_location assert not isinstance(bootapp_fpath, DigestValue) @@ -411,12 +412,19 @@ def _detect_l4tlauncher_version(self) -> BSPVersion: # APIs def firmware_update(self) -> bool: - """Trigger firmware update in next boot. + """Trigger firmware update in next boot if configured. + + Only when the following conditions met the firmware update will be configured: + 1. a firmware_update_request file is presented and valid in the OTA image. + 2. the firmware_manifest is presented and valid in the OTA image. + 3. the firmware package is compatible with the device. + 4. firmware specified in firmware_update_request is valid. + 5. the standby slot's firmware is older than the OTA image's one. Returns: True if firmware update is configured, False if there is no firmware update. """ - # check BSP version, NVIDIA Jetson device doesn't allow firmware downgrade. + # check BSP version, NVIDIA Jetson device with R34 or newer doesn't allow firmware downgrade. if ( self.standby_slot_bsp_ver and self.standby_slot_bsp_ver > self.firmware_package_bsp_ver @@ -439,11 +447,28 @@ def firmware_update(self) -> bool: logger.warning(_err_msg) return False + # copy the firmware update capsule to specific location with _ensure_esp_mounted(self.esp_part, self.esp_mp): if not self._prepare_fwupdate_capsule(): logger.info("no firmware file is prepared, skip firmware update") return False + logger.info("on capsule prepared, try to update L4TLauncher ...") + l4tlauncher_bsp_ver = self._detect_l4tlauncher_version() + logger.info(f"current l4tlauncher version: {l4tlauncher_bsp_ver}") + + if l4tlauncher_bsp_ver >= self.firmware_package_bsp_ver: + logger.info( + ( + "installed l4tlauncher has newer or equal version of l4tlauncher to OTA image's one, " + f"{l4tlauncher_bsp_ver=}, {self.firmware_package_bsp_ver=}, " + "skip l4tlauncher update" + ) + ) + else: + self._update_l4tlauncher() + + # write special UEFI variable to trigger firmware update on next reboot with _ensure_efivarfs_mounted(): try: self._write_magic_efivar() @@ -460,25 +485,6 @@ def firmware_update(self) -> bool: "firmware update package prepare finished" f"will update firmware to {self.firmware_package_bsp_ver} in next reboot" ) - logger.info("try to update L4TLauncher ...") - - with _ensure_esp_mounted(self.esp_part, self.esp_mp): - l4tlauncher_bsp_ver = self._detect_l4tlauncher_version() - logger.info(f"current l4tlauncher version: {l4tlauncher_bsp_ver}") - - if l4tlauncher_bsp_ver >= self.firmware_package_bsp_ver: - logger.info( - ( - "installed l4tlauncher has newer or equal version of l4tlauncher to OTA image's one, " - f"{l4tlauncher_bsp_ver=}, {self.firmware_package_bsp_ver=}, " - "skip l4tlauncher update" - ) - ) - else: - logger.warning( - f"try to update L4TLauncher to {self.firmware_package_bsp_ver=}" - ) - self._update_l4tlauncher() return True From 6999243045530751383b44e99af0bc1b77abd8a6 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Wed, 21 Aug 2024 09:17:33 +0000 Subject: [PATCH 138/193] jetson-uefi: allow working on BSP R34, but only allow firmware update on R35.2 and newer --- src/otaclient/boot_control/_jetson_uefi.py | 30 ++++++++++++++++------ 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index 0273fc912..827d5b462 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -13,8 +13,8 @@ # limitations under the License. """Boot control implementation for NVIDIA Jetson device boots with UEFI. -jetson-uefi module currently support BSP version >= R35.2. -NOTE: R34~R35.1 is not supported as Capsule update is only available after R35.2. +jetson-uefi module currently support BSP version >= R34(which UEFI is introduced). +But firmware update is only supported after BSP R35.2. """ @@ -62,6 +62,11 @@ logger = logging.getLogger(__name__) +MINIMUM_SUPPORTED_BSP_VERSION = BSPVersion(34, 0, 0) +"""Only after R34, UEFI is introduced.""" +FIRMWARE_UPDATE_MINIMUM_SUPPORTED_BSP_VERSION = BSPVersion(35, 2, 0) +"""Only after R35.2, UEFI Capsule firmware update is introduced.""" + L4TLAUNCHER_BSP_VER_SHA256_MAP: dict[str, BSPVersion] = { "b14fa3623f4078d05573d9dcf2a0b46ea2ae07d6b75d9843f9da6ff24db13718": BSPVersion( 36, 3, 0 @@ -424,6 +429,19 @@ def firmware_update(self) -> bool: Returns: True if firmware update is configured, False if there is no firmware update. """ + # sanity check, UEFI Capsule firmware update is only supported after R35.2. + if ( + self.standby_slot_bsp_ver + and self.standby_slot_bsp_ver + < FIRMWARE_UPDATE_MINIMUM_SUPPORTED_BSP_VERSION + ): + _err_msg = ( + f"UEFI Capsule firmware update is introduced since {FIRMWARE_UPDATE_MINIMUM_SUPPORTED_BSP_VERSION}, " + f"but get {self.standby_slot_bsp_ver}, abort" + ) + logger.error(_err_msg) + return False + # check BSP version, NVIDIA Jetson device with R34 or newer doesn't allow firmware downgrade. if ( self.standby_slot_bsp_ver @@ -488,10 +506,6 @@ def firmware_update(self) -> bool: return True -MINIMUM_SUPPORTED_BSP_VERSION = BSPVersion(35, 2, 0) -"""Only after R35.2, the Capsule Firmware update is available.""" - - class _UEFIBootControl: """Low-level boot control implementation for jetson-uefi.""" @@ -573,9 +587,9 @@ def __init__(self): ) ) - # ------ sanity check, currently jetson-uefi only supports >= R35.2 ----- # + # ------ sanity check, jetson-uefi only supports >= R34 ----- # if fw_bsp_version < MINIMUM_SUPPORTED_BSP_VERSION: - _err_msg = f"jetson-uefi only supports BSP version >= R35.2, but get {fw_bsp_version=}. " + _err_msg = f"jetson-uefi only supports BSP version >= {MINIMUM_SUPPORTED_BSP_VERSION}, but get {fw_bsp_version=}. " logger.error(_err_msg) raise JetsonUEFIBootControlError(_err_msg) From b80e8317842ae38cc77d459c1c311807553d7958 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 22 Aug 2024 03:09:25 +0000 Subject: [PATCH 139/193] jetson-common: implement detect_external_rootdev and get_partition_devpath helper functions --- src/otaclient/boot_control/_jetson_common.py | 39 ++++++++++++++++++++ src/otaclient/boot_control/configs.py | 7 +++- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_common.py b/src/otaclient/boot_control/_jetson_common.py index 434cb2b6a..fe67f478d 100644 --- a/src/otaclient/boot_control/_jetson_common.py +++ b/src/otaclient/boot_control/_jetson_common.py @@ -418,3 +418,42 @@ def update_standby_slot_extlinux_cfg( standby_slot_partuuid, ), ) + + +def detect_external_rootdev(parent_devpath: StrOrPath) -> bool: + """Check whether the ECU is using external device as root device or not. + + Returns: + True if device is booted from external NVMe SSD, False if device is booted + from internal emmc device. + """ + parent_devname = Path(parent_devpath).name + if parent_devname.startswith(jetson_common_cfg.INTERNAL_EMMC_DEVNAME): + logger.info(f"device boots from internal emmc: {parent_devpath}") + return False + logger.info(f"device boots from external device: {parent_devpath}") + return True + + +def get_partition_devpath(parent_devpath: StrOrPath, partition_id: int) -> str: + """Get partition devpath from and . + + For internal emmc like /dev/mmcblk0 with partition_id 1, we will get: + /dev/mmcblk0p1 + For external NVMe SSD like /dev/nvme0n1 with partition_id 1, we will get: + /dev/nvme0n1p1 + For other types of device, including USB drive, like /dev/sda with partition_id 1, + we will get: /dev/sda1 + """ + parent_devpath = str(parent_devpath).strip().rstrip("/") + + parent_devname = Path(parent_devpath).name + if parent_devname.startswith( + jetson_common_cfg.MMCBLK_DEV_PREFIX + ) or parent_devname.startswith(jetson_common_cfg.NVMESSD_DEV_PREFIX): + return f"{parent_devpath}p{partition_id}" + if parent_devname.startswith(jetson_common_cfg.SDX_DEV_PREFIX): + return f"{parent_devpath}{partition_id}" + + logger.warning(f"unexpected {parent_devname=}, treat it the same as sdx type") + return f"{parent_devpath}{partition_id}" diff --git a/src/otaclient/boot_control/configs.py b/src/otaclient/boot_control/configs.py index 9f62ce65e..852f2854a 100644 --- a/src/otaclient/boot_control/configs.py +++ b/src/otaclient/boot_control/configs.py @@ -42,11 +42,14 @@ class JetsonBootCommon: # boot control related EXTLINUX_FILE = "/boot/extlinux/extlinux.conf" MODEL_FPATH = "/proc/device-tree/model" + NV_TEGRA_RELEASE_FPATH = "/etc/nv_tegra_release" + SEPARATE_BOOT_MOUNT_POINT = "/mnt/standby_boot" + + # boot device related MMCBLK_DEV_PREFIX = "mmcblk" # internal emmc NVMESSD_DEV_PREFIX = "nvme" # external nvme ssd + SDX_DEV_PREFIX = "sd" # non-specific device name INTERNAL_EMMC_DEVNAME = "mmcblk0" - NV_TEGRA_RELEASE_FPATH = "/etc/nv_tegra_release" - SEPARATE_BOOT_MOUNT_POINT = "/mnt/standby_boot" # firmware update related NVBOOTCTRL_CONF_FPATH = "/etc/nv_boot_control.conf" From 29c8f9faf553907ca9914cf5acfedd0d9eef8a30 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 22 Aug 2024 03:13:52 +0000 Subject: [PATCH 140/193] jetson-uefi: use new detect_external_rootdev and get_partition_devpath helper functions --- src/otaclient/boot_control/_jetson_uefi.py | 29 ++++++++-------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index 827d5b462..68e5014de 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -52,8 +52,10 @@ FirmwareBSPVersionControl, NVBootctrlCommon, copy_standby_slot_boot_to_internal_emmc, + detect_external_rootdev, detect_rootfs_bsp_version, get_nvbootctrl_conf_tnspec, + get_partition_devpath, preserve_ota_config_files_to_standby, update_standby_slot_extlinux_cfg, ) @@ -589,7 +591,7 @@ def __init__(self): # ------ sanity check, jetson-uefi only supports >= R34 ----- # if fw_bsp_version < MINIMUM_SUPPORTED_BSP_VERSION: - _err_msg = f"jetson-uefi only supports BSP version >= {MINIMUM_SUPPORTED_BSP_VERSION}, but get {fw_bsp_version=}. " + _err_msg = f"jetson-uefi only supports BSP version >= {MINIMUM_SUPPORTED_BSP_VERSION}, but get {fw_bsp_version=}" logger.error(_err_msg) raise JetsonUEFIBootControlError(_err_msg) @@ -610,23 +612,11 @@ def __init__(self): ) # --- detect boot device --- # - self.external_rootfs = False - - parent_devname = parent_devpath.name - if parent_devname.startswith(boot_cfg.MMCBLK_DEV_PREFIX): - logger.info(f"device boots from internal emmc: {parent_devpath}") - elif parent_devname.startswith(boot_cfg.NVMESSD_DEV_PREFIX): - logger.info(f"device boots from external nvme ssd: {parent_devpath}") - self.external_rootfs = True - else: - _err_msg = f"currently we don't support booting from {parent_devpath=}" - logger.error(_err_msg) - raise JetsonUEFIBootControlError(_err_msg) from NotImplementedError( - f"unsupported bootdev {parent_devpath}" - ) + self.external_rootfs = detect_external_rootdev(parent_devpath) - self.standby_rootfs_devpath = ( - f"/dev/{parent_devname}p{SLOT_PAR_MAP[standby_slot]}" + self.standby_rootfs_devpath = get_partition_devpath( + parent_devpath=parent_devpath, + partition_id=SLOT_PAR_MAP[standby_slot], ) self.standby_rootfs_dev_partuuid = CMDHelperFuncs.get_attrs_by_dev( "PARTUUID", self.standby_rootfs_devpath @@ -641,8 +631,9 @@ def __init__(self): f"standby_rootfs(slot {standby_slot}): {self.standby_rootfs_devpath=}, {self.standby_rootfs_dev_partuuid=}" ) - self.standby_internal_emmc_devpath = ( - f"/dev/{boot_cfg.INTERNAL_EMMC_DEVNAME}p{SLOT_PAR_MAP[standby_slot]}" + self.standby_internal_emmc_devpath = get_partition_devpath( + parent_devpath=f"/dev/{boot_cfg.INTERNAL_EMMC_DEVNAME}", + partition_id=SLOT_PAR_MAP[standby_slot], ) logger.info("finished jetson-uefi boot control startup") From 2acfe4d66abfde44d5d70d12eee3cd0234047cdd Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 22 Aug 2024 06:18:58 +0000 Subject: [PATCH 141/193] jetson-cboot: switch to use detect_external_rootdev and get_partition_devpath --- src/otaclient/boot_control/_jetson_cboot.py | 38 ++++++++------------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_cboot.py b/src/otaclient/boot_control/_jetson_cboot.py index 30e6e1839..80f4670bc 100644 --- a/src/otaclient/boot_control/_jetson_cboot.py +++ b/src/otaclient/boot_control/_jetson_cboot.py @@ -40,14 +40,17 @@ from ._common import CMDHelperFuncs, OTAStatusFilesControl, SlotMountHelper from ._jetson_common import ( + SLOT_PAR_MAP, BSPVersion, FirmwareBSPVersionControl, NVBootctrlCommon, NVBootctrlTarget, SlotID, copy_standby_slot_boot_to_internal_emmc, + detect_external_rootdev, detect_rootfs_bsp_version, get_nvbootctrl_conf_tnspec, + get_partition_devpath, preserve_ota_config_files_to_standby, update_standby_slot_extlinux_cfg, ) @@ -226,8 +229,6 @@ def verify_update(cls) -> subprocess.CompletedProcess[bytes]: class _CBootControl: - _slot_id_partid = {SlotID("0"): "1", SlotID("1"): "2"} - def __init__(self): # ------ sanity check, confirm we are at jetson device ------ # if not os.path.exists(boot_cfg.TEGRA_CHIP_ID_PATH): @@ -239,7 +240,7 @@ def __init__(self): # ------ check BSP version ------ # # NOTE(20240821): unfortunately, we don't have proper method to detect - # the firmware BSP version, so we assume that the rootfs BSP version is the + # the firmware BSP version < R34, so we assume that the rootfs BSP version is the # same as the firmware BSP version. try: self.rootfs_bsp_version = rootfs_bsp_version = detect_rootfs_bsp_version( @@ -304,30 +305,19 @@ def __init__(self): CMDHelperFuncs.get_parent_dev(current_rootfs_devpath) ) - self._external_rootfs = False - parent_devname = parent_devpath.name - if parent_devname.startswith(boot_cfg.MMCBLK_DEV_PREFIX): - logger.info(f"device boots from internal emmc: {parent_devpath}") - elif parent_devname.startswith(boot_cfg.NVMESSD_DEV_PREFIX): - logger.info(f"device boots from external nvme ssd: {parent_devpath}") - self._external_rootfs = True - else: - _err_msg = f"we don't support boot from {parent_devpath=} currently" - logger.error(_err_msg) - raise JetsonCBootContrlError(_err_msg) from NotImplementedError( - f"unsupported bootdev {parent_devpath}" - ) + self._external_rootfs = detect_external_rootdev(parent_devpath) # rootfs partition - self.standby_rootfs_devpath = ( - f"/dev/{parent_devname}p{self._slot_id_partid[standby_rootfs_slot]}" + self.standby_rootfs_devpath = get_partition_devpath( + parent_devpath=parent_devpath, + partition_id=SLOT_PAR_MAP[standby_rootfs_slot], ) self.standby_rootfs_dev_partuuid = CMDHelperFuncs.get_attrs_by_dev( - "PARTUUID", f"{self.standby_rootfs_devpath}" - ) + "PARTUUID", self.standby_rootfs_devpath + ).strip() current_rootfs_dev_partuuid = CMDHelperFuncs.get_attrs_by_dev( "PARTUUID", current_rootfs_devpath - ) + ).strip() logger.info( "finish detecting rootfs devs: \n" @@ -336,8 +326,10 @@ def __init__(self): ) # internal emmc partition - self.standby_internal_emmc_devpath = f"/dev/{boot_cfg.INTERNAL_EMMC_DEVNAME}p{self._slot_id_partid[standby_rootfs_slot]}" - + self.standby_internal_emmc_devpath = get_partition_devpath( + parent_devpath=f"/dev/{boot_cfg.INTERNAL_EMMC_DEVNAME}", + partition_id=SLOT_PAR_MAP[standby_rootfs_slot], + ) logger.info(f"finished cboot control init: {current_rootfs_slot=}") logger.info(f"nvbootctrl dump-slots-info: \n{_NVBootctrl.dump_slots_info()}") if not unified_ab_enabled: From 909dec6873d7b2bcce4efd5dbc2741f15f8e30b3 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 22 Aug 2024 06:38:38 +0000 Subject: [PATCH 142/193] jetson_common: detect_rootfs_bsp_version: use replace_root --- src/otaclient/boot_control/_jetson_common.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_common.py b/src/otaclient/boot_control/_jetson_common.py index fe67f478d..723340532 100644 --- a/src/otaclient/boot_control/_jetson_common.py +++ b/src/otaclient/boot_control/_jetson_common.py @@ -29,6 +29,7 @@ from pydantic import BaseModel, BeforeValidator, PlainSerializer from typing_extensions import Annotated, Literal, Self +from otaclient_common import replace_root from otaclient_common.common import copytree_identical, write_str_to_file_sync from otaclient_common.typing import StrOrPath @@ -304,11 +305,13 @@ def detect_rootfs_bsp_version(rootfs: StrOrPath) -> BSPVersion: Returns: BSPversion of the . """ - nv_tegra_release_fpath = Path(rootfs) / Path( - jetson_common_cfg.NV_TEGRA_RELEASE_FPATH - ).relative_to("/") + nv_tegra_release_fpath = replace_root( + jetson_common_cfg.NV_TEGRA_RELEASE_FPATH, + "/", + rootfs, + ) try: - return parse_nv_tegra_release(nv_tegra_release_fpath.read_text()) + return parse_nv_tegra_release(Path(nv_tegra_release_fpath).read_text()) except Exception as e: _err_msg = f"failed to detect rootfs BSP version at: {rootfs}: {e!r}" logger.error(_err_msg) @@ -343,9 +346,9 @@ def _replace(ma: re.Match, repl: str): def copy_standby_slot_boot_to_internal_emmc( *, - internal_emmc_mp: Path | str, - internal_emmc_devpath: Path | str, - standby_slot_boot_dirpath: Path | str, + internal_emmc_mp: StrOrPath, + internal_emmc_devpath: StrOrPath, + standby_slot_boot_dirpath: StrOrPath, ) -> None: """Copy the standby slot's /boot to internal emmc dev. From ea83171b24a4741c4cc5aae73b507c334395d5eb Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 22 Aug 2024 06:39:17 +0000 Subject: [PATCH 143/193] test_jetson_common: fix up, implement tests for newly implemented helper methods --- .../test_boot_control/test_jetson_common.py | 118 +++++++++++++----- 1 file changed, 87 insertions(+), 31 deletions(-) diff --git a/tests/test_otaclient/test_boot_control/test_jetson_common.py b/tests/test_otaclient/test_boot_control/test_jetson_common.py index 69578d99e..a6cc197b9 100644 --- a/tests/test_otaclient/test_boot_control/test_jetson_common.py +++ b/tests/test_otaclient/test_boot_control/test_jetson_common.py @@ -21,13 +21,16 @@ import pytest -from otaclient.app.boot_control._jetson_common import ( +from otaclient.boot_control._jetson_common import ( SLOT_A, SLOT_B, BSPVersion, - FirmwareBSPVersion, FirmwareBSPVersionControl, + SlotBSPVersion, SlotID, + detect_external_rootdev, + get_nvbootctrl_conf_tnspec, + get_partition_devpath, parse_nv_tegra_release, update_extlinux_cfg, ) @@ -95,41 +98,41 @@ def test_dump(self, _in: BSPVersion, _expect: str): assert _in.dump() == _expect -class TestFirmwareBSPVersion: +class TestSlotBSPVersion: @pytest.mark.parametrize( "_in, _slot, _bsp_ver, _expect", ( ( - FirmwareBSPVersion(), + SlotBSPVersion(), SLOT_A, BSPVersion(32, 6, 1), - FirmwareBSPVersion(slot_a=BSPVersion(32, 6, 1)), + SlotBSPVersion(slot_a=BSPVersion(32, 6, 1)), ), ( - FirmwareBSPVersion( + SlotBSPVersion( slot_a=BSPVersion(32, 5, 1), slot_b=BSPVersion(32, 6, 1) ), SLOT_B, None, - FirmwareBSPVersion(slot_a=BSPVersion(32, 5, 1), slot_b=None), + SlotBSPVersion(slot_a=BSPVersion(32, 5, 1), slot_b=None), ), ( - FirmwareBSPVersion( + SlotBSPVersion( slot_a=BSPVersion(32, 5, 1), slot_b=BSPVersion(32, 6, 1) ), SLOT_A, None, - FirmwareBSPVersion(slot_a=None, slot_b=BSPVersion(32, 6, 1)), + SlotBSPVersion(slot_a=None, slot_b=BSPVersion(32, 6, 1)), ), ), ) def test_set_by_slot( self, - _in: FirmwareBSPVersion, + _in: SlotBSPVersion, _slot: SlotID, _bsp_ver: BSPVersion | None, - _expect: FirmwareBSPVersion, + _expect: SlotBSPVersion, ): _in.set_by_slot(_slot, _bsp_ver) assert _in == _expect @@ -138,19 +141,19 @@ def test_set_by_slot( "_in, _slot, _expect", ( ( - FirmwareBSPVersion(), + SlotBSPVersion(), SLOT_A, None, ), ( - FirmwareBSPVersion( + SlotBSPVersion( slot_a=BSPVersion(32, 5, 1), slot_b=BSPVersion(32, 6, 1) ), SLOT_B, BSPVersion(32, 6, 1), ), ( - FirmwareBSPVersion( + SlotBSPVersion( slot_a=BSPVersion(32, 5, 1), slot_b=BSPVersion(32, 6, 1) ), SLOT_A, @@ -160,7 +163,7 @@ def test_set_by_slot( ) def test_get_by_slot( self, - _in: FirmwareBSPVersion, + _in: SlotBSPVersion, _slot: SlotID, _expect: BSPVersion | None, ): @@ -169,17 +172,13 @@ def test_get_by_slot( @pytest.mark.parametrize( "_in", ( - (FirmwareBSPVersion()), - (FirmwareBSPVersion(slot_a=BSPVersion(32, 5, 1))), - ( - FirmwareBSPVersion( - slot_a=BSPVersion(35, 4, 1), slot_b=BSPVersion(35, 5, 0) - ) - ), + (SlotBSPVersion()), + (SlotBSPVersion(slot_a=BSPVersion(32, 5, 1))), + (SlotBSPVersion(slot_a=BSPVersion(35, 4, 1), slot_b=BSPVersion(35, 5, 0))), ), ) - def test_load_and_dump(self, _in: FirmwareBSPVersion): - assert FirmwareBSPVersion.model_validate_json(_in.model_dump_json()) == _in + def test_load_and_dump(self, _in: SlotBSPVersion): + assert SlotBSPVersion.model_validate_json(_in.model_dump_json()) == _in class TestFirmwareBSPVersionControl: @@ -192,33 +191,33 @@ def setup_test(self, tmp_path: Path): def test_init(self): self.test_fw_bsp_vf.write_text( - FirmwareBSPVersion(slot_b=self.slot_b_ver).model_dump_json() + SlotBSPVersion(slot_b=self.slot_b_ver).model_dump_json() ) loaded = FirmwareBSPVersionControl( SLOT_A, self.slot_a_ver, - current_firmware_bsp_vf=self.test_fw_bsp_vf, + current_bsp_version_file=self.test_fw_bsp_vf, ) # NOTE: FirmwareBSPVersionControl will not use the information for current slot. - assert loaded.current_slot_fw_ver == self.slot_a_ver - assert loaded.standby_slot_fw_ver == self.slot_b_ver + assert loaded.current_slot_bsp_ver == self.slot_a_ver + assert loaded.standby_slot_bsp_ver == self.slot_b_ver def test_write_to_file(self): self.test_fw_bsp_vf.write_text( - FirmwareBSPVersion(slot_b=self.slot_b_ver).model_dump_json() + SlotBSPVersion(slot_b=self.slot_b_ver).model_dump_json() ) loaded = FirmwareBSPVersionControl( SLOT_A, self.slot_a_ver, - current_firmware_bsp_vf=self.test_fw_bsp_vf, + current_bsp_version_file=self.test_fw_bsp_vf, ) loaded.write_to_file(self.test_fw_bsp_vf) assert ( self.test_fw_bsp_vf.read_text() - == FirmwareBSPVersion( + == SlotBSPVersion( slot_a=self.slot_a_ver, slot_b=self.slot_b_ver ).model_dump_json() ) @@ -264,3 +263,60 @@ def test_update_extlinux_conf(_template_f: Path, _updated_f: Path, partuuid: str _in = (TEST_DATA_DIR / _template_f).read_text() _expected = (TEST_DATA_DIR / _updated_f).read_text() assert update_extlinux_cfg(_in, partuuid) == _expected + + +@pytest.mark.parametrize( + "parent_devpath, is_external_rootdev", + ( + ("/dev/mmcblk0", False), + ("/dev/mmcblk1", True), + ("/dev/sda", True), + ("/dev/nvme0n1", True), + ), +) +def test_detect_external_rootdev(parent_devpath, is_external_rootdev): + assert detect_external_rootdev(parent_devpath) is is_external_rootdev + + +@pytest.mark.parametrize( + "parent_devpath, partition_id, expected", + ( + ("/dev/mmcblk0", 1, "/dev/mmcblk0p1"), + ("/dev/mmcblk1", 1, "/dev/mmcblk1p1"), + ("/dev/sda", 1, "/dev/sda1"), + ("/dev/nvme0n1", 1, "/dev/nvme0n1p1"), + ), +) +def test_get_partition_devpath(parent_devpath, partition_id, expected): + assert get_partition_devpath(parent_devpath, partition_id) == expected + + +@pytest.mark.parametrize( + "nvbooctrl_conf, expected", + ( + ( + """\ +TNSPEC 2888-400-0004-M.0-1-2-jetson-xavier-rqx580- +TEGRA_CHIPID 0x19 +TEGRA_OTA_BOOT_DEVICE /dev/mmcblk0boot0 +TEGRA_OTA_GPT_DEVICE /dev/mmcblk0boot1 +""", + "2888-400-0004-M.0-1-2-jetson-xavier-rqx580-", + ), + ( + """\ +TNSPEC 3701-500-0005-A.0-1-0-cti-orin-agx-agx201-00- +TEGRA_CHIPID 0x23 +TEGRA_OTA_BOOT_DEVICE /dev/mtdblock0 +TEGRA_OTA_GPT_DEVICE /dev/mtdblock0 +""", + "3701-500-0005-A.0-1-0-cti-orin-agx-agx201-00-", + ), + ( + "not a nvbooctrl file", + None, + ), + ), +) +def test_get_nvbootctrl_conf_tnspec(nvbooctrl_conf, expected): + assert get_nvbootctrl_conf_tnspec(nvbooctrl_conf) == expected From 70ae6d946bd0be4a6a14ebc39b91f7a266c52ce9 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 22 Aug 2024 09:04:36 +0000 Subject: [PATCH 144/193] add test for replace_root --- tests/test_otaclient_common/test_common.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_otaclient_common/test_common.py b/tests/test_otaclient_common/test_common.py index 28589fe22..2d8765c9b 100644 --- a/tests/test_otaclient_common/test_common.py +++ b/tests/test_otaclient_common/test_common.py @@ -26,6 +26,7 @@ import pytest +from otaclient_common import replace_root from otaclient_common.common import ( copytree_identical, ensure_otaproxy_start, @@ -347,3 +348,24 @@ def test_subprocess_check_output_succeeded(self): output = subprocess_check_output(cmd, raise_exception=True) assert output == self.TEST_FILE_CONTENTS + + +@pytest.mark.parametrize( + "path, old_root, new_root, expected", + ( + ( + "/a/canonical/fpath", + "/", + "/mnt/standby_mp", + "/mnt/standby_mp/a/canonical/fpath", + ), + ( + "/a/canonical/dpath/", + "/", + "/mnt/standby_mp/", + "/mnt/standby_mp/a/canonical/dpath/", + ), + ), +) +def get_replace_root(path, old_root, new_root, expected): + assert replace_root(path, old_root, new_root) == expected From 29e32bb0e6b521534acbbf89caaf549480dd1323 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 22 Aug 2024 09:06:37 +0000 Subject: [PATCH 145/193] minor fix to jetson_common --- src/otaclient/boot_control/_jetson_common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_common.py b/src/otaclient/boot_control/_jetson_common.py index 723340532..d7860c4f4 100644 --- a/src/otaclient/boot_control/_jetson_common.py +++ b/src/otaclient/boot_control/_jetson_common.py @@ -112,8 +112,8 @@ class SlotBSPVersion(BaseModel): BSP version string schema: Rxx.yy.z """ - slot_a: BSPVersionStr | None = None - slot_b: BSPVersionStr | None = None + slot_a: Optional[BSPVersionStr] = None + slot_b: Optional[BSPVersionStr] = None def set_by_slot(self, slot_id: SlotID, ver: BSPVersion | None) -> None: if slot_id == SLOT_A: From 202571fe98fda59112b9e9321d8f1c87ee4aee7a Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 22 Aug 2024 09:37:32 +0000 Subject: [PATCH 146/193] jetson-*: fix payload location --- src/otaclient/boot_control/_jetson_cboot.py | 15 +++------------ src/otaclient/boot_control/_jetson_uefi.py | 5 +++-- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_cboot.py b/src/otaclient/boot_control/_jetson_cboot.py index 80f4670bc..a6a47c6f3 100644 --- a/src/otaclient/boot_control/_jetson_cboot.py +++ b/src/otaclient/boot_control/_jetson_cboot.py @@ -20,7 +20,6 @@ from __future__ import annotations import logging -import os import subprocess from pathlib import Path from typing import Generator, Optional @@ -28,7 +27,6 @@ from otaclient.app import errors as ota_errors from otaclient.app.configs import config as cfg from otaclient.boot_control._firmware_package import ( - DigestValue, FirmwareManifest, FirmwareUpdateRequest, PayloadType, @@ -202,8 +200,9 @@ def firmware_update(self) -> bool: continue # NOTE: currently we only support payload indicated by file path. - bup_fpath = update_payload.file_location - assert not isinstance(bup_fpath, DigestValue) + bup_flocation = update_payload.file_location + assert bup_flocation.location_type == "file" + bup_fpath = bup_flocation.location_path if not Path(bup_fpath).is_file(): logger.warning(f"{bup_fpath=} doesn't exist! skip...") @@ -230,14 +229,6 @@ def verify_update(cls) -> subprocess.CompletedProcess[bytes]: class _CBootControl: def __init__(self): - # ------ sanity check, confirm we are at jetson device ------ # - if not os.path.exists(boot_cfg.TEGRA_CHIP_ID_PATH): - _err_msg = ( - f"not a jetson device, {boot_cfg.TEGRA_CHIP_ID_PATH} doesn't exist" - ) - logger.error(_err_msg) - raise JetsonCBootContrlError(_err_msg) - # ------ check BSP version ------ # # NOTE(20240821): unfortunately, we don't have proper method to detect # the firmware BSP version < R34, so we assume that the rootfs BSP version is the diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index 68e5014de..67510785c 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -296,8 +296,9 @@ def _prepare_fwupdate_capsule(self) -> bool: continue # NOTE: currently we only support payload indicated by file path. - capsule_fpath = capsule_payload.file_location - assert not isinstance(capsule_fpath, DigestValue) + capsule_flocation = capsule_payload.file_location + assert capsule_flocation.location_type == "file" + capsule_fpath = capsule_flocation.location_path try: shutil.copy( From 5a5e531112c474e8b033e20ca82b5a71fcce06e4 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 22 Aug 2024 10:37:07 +0000 Subject: [PATCH 147/193] jetson-uefi: add helper method to detect whether OTA_BOOTDEV is QSPI --- src/otaclient/boot_control/_jetson_uefi.py | 33 ++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index 67510785c..77e819efc 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -202,6 +202,39 @@ def _detect_esp_dev(boot_parent_devpath: StrOrPath) -> str: return esp_part +def _detect_ota_bootdev_is_qspi(nvboot_ctrl_conf: str) -> bool | None: + """Detect whether the device is using QSPI or not. + + This is required for determining the way to trigger UEFI Capsule update. + If the TEGRA_OTA_BOOT_DEVICE is mtdblock device, then the device is using QSPI. + If is mmcblk0bootx, then the device is using internal emmc. + + If we cannot determine the TEGRA_OTA_BOOT_DEVICE, we MUST NOT execute the firmware update. + + Returns: + True if device is using QSPI flash, False for device is using internal emmc, + None for unrecognized device or missing TEGRA_OTA_BOOT_DEVICE field in conf. + """ + for line in nvboot_ctrl_conf.splitlines(): + if line.strip().startswith("TEGRA_OTA_BOOT_DEVICE"): + _, _ota_bootdev = line.split(" ", maxsplit=1) + ota_bootdev = _ota_bootdev.strip() + break + else: + logger.warning("no TEGRA_OTA_BOOT_DEVICE field found in nv_boot_ctrl.conf") + return + + if ota_bootdev.startswith("/dev/mmcblk0"): + logger.info("device doesn't use QSPI flash as TEGRA_OTA_BOOT_DEVICE") + return False + + if ota_bootdev.startswith("/dev/mtdblock0"): + logger.info("device uses QSPI flash as TEGRA_OTA_BOOT_DEVICE") + return True + + logger.warning(f"device uses unknown TEGRA_OTA_BOOT_DEVICE: {ota_bootdev}") + + class L4TLauncherBSPVersionControl(BaseModel): """Track the L4TLauncher binary BSP version. From b5451a9e9011b34e34339659236564e14444b724 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 22 Aug 2024 11:14:26 +0000 Subject: [PATCH 148/193] jetson-uefi: implement _trigger_capsule_update_qspi_ota_bootdev --- src/otaclient/boot_control/_jetson_uefi.py | 24 ++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index 77e819efc..26a6023f2 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -163,6 +163,30 @@ def _ensure_efivarfs_mounted() -> Generator[None, Any, None]: ) from e +def _trigger_capsule_update_qspi_ota_bootdev() -> bool: + """Write magic efivar to trigger firmware Capsule update in next boot. + + This method is for device using QSPI flash as TEGRA_OTA_BOOT_DEVICE. + """ + with _ensure_efivarfs_mounted(): + try: + magic_efivar_fpath = ( + Path(EFIVARS_SYS_MOUNT_POINT) / boot_cfg.UPDATE_TRIGGER_EFIVAR + ) + magic_efivar_fpath.write_bytes(boot_cfg.MAGIC_BYTES) + os.sync() + + return True + except Exception as e: + logger.warning( + ( + f"failed to configure capsule update by write magic value: {e!r}\n" + "firmware update might be skipped!" + ) + ) + return False + + @contextlib.contextmanager def _ensure_esp_mounted( esp_dev: StrOrPath, mount_point: StrOrPath From aed03abc82cc08dfcae8058ee17c5b42adc2b8bd Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 22 Aug 2024 11:21:30 +0000 Subject: [PATCH 149/193] jetson-uefi: implement _trigger_capsule_update_non_qspi_ota_bootdev --- src/otaclient/boot_control/_jetson_uefi.py | 40 +++++++++++++++------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index 26a6023f2..e8a5f154a 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -187,6 +187,33 @@ def _trigger_capsule_update_qspi_ota_bootdev() -> bool: return False +def _trigger_capsule_update_non_qspi_ota_bootdev(esp_mp: StrOrPath) -> bool: + """Write magic efivar to trigger firmware Capsule update in next boot. + + Write a special file with magic value into specific location at internal emmc ESP partition. + + This method is for device NOT using QSPI flash as TEGRA_OTA_BOOT_DEVICE. + """ + uefi_variable_dir = Path(esp_mp) / "EFI/NVDA/Variables" + magic_variable_fpath = uefi_variable_dir / boot_cfg.UPDATE_TRIGGER_EFIVAR + + try: + emmc_esp_partition = _detect_esp_dev(f"/dev/{boot_cfg.INTERNAL_EMMC_DEVNAME}") + with _ensure_esp_mounted(emmc_esp_partition, esp_mp): + magic_variable_fpath.write_bytes(boot_cfg.MAGIC_BYTES) + os.sync() + + return True + except Exception as e: + logger.warning( + ( + f"failed to configure capsule update: {e!r}\n" + "firmware update might be skipped!" + ) + ) + return False + + @contextlib.contextmanager def _ensure_esp_mounted( esp_dev: StrOrPath, mount_point: StrOrPath @@ -407,19 +434,6 @@ def _update_l4tlauncher(self) -> bool: logger.info("no boot application update is configured in the request, skip") return False - @staticmethod - def _write_magic_efivar() -> None: - """Write magic efivar to trigger firmware Capsule update in next boot. - - Raises: - JetsonUEFIBootControlError on failed Capsule update preparing. - """ - magic_efivar_fpath = ( - Path(EFIVARS_SYS_MOUNT_POINT) / boot_cfg.UPDATE_TRIGGER_EFIVAR - ) - magic_efivar_fpath.write_bytes(boot_cfg.MAGIC_BYTES) - os.sync() - def _detect_l4tlauncher_version(self) -> BSPVersion: """Try to determine the current in use l4tlauncher version.""" l4tlauncher_sha256_digest = file_sha256(self.bootaa64_at_esp) From 9aaef3720aa8dcf778d8a17e7373e9d9fa48a6f7 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 22 Aug 2024 11:28:02 +0000 Subject: [PATCH 150/193] jetson-uefi: explicitly requires nv_boot_ctrl.conf file presented --- src/otaclient/boot_control/_jetson_uefi.py | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index e8a5f154a..f0cc1f2d1 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -604,20 +604,14 @@ def __init__(self): raise JetsonUEFIBootControlError(_err_msg) logger.info(f"dev compatibility: {compat_info}") - # load tnspec for firmware update compatibility check - try: - self.tnspec = get_nvbootctrl_conf_tnspec( - Path(boot_cfg.NVBOOTCTRL_CONF_FPATH).read_text() - ) - logger.info(f"firmware compatibility: {self.tnspec}") - except Exception as e: - logger.warning( - ( - f"failed to load tnspec: {e!r}, " - "this will result in firmware update being skipped!" - ) - ) - self.tnspec = None + # load nvbootctrl config file + if not ( + nvbootctrl_conf_fpath := Path(boot_cfg.NVBOOTCTRL_CONF_FPATH) + ).is_file(): + _err_msg = "nv_boot_ctrl.conf is missing!" + logger.error(_err_msg) + raise JetsonUEFIBootControlError(_err_msg) + self.nvbootctrl_conf = nvbootctrl_conf_fpath.read_text() # ------ check current slot BSP version ------ # # check current slot firmware BSP version From c7a1d8f86b633ad2ac6dfa421b3db9d14cf677f3 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 22 Aug 2024 11:33:18 +0000 Subject: [PATCH 151/193] jetson-uefi: choose different firmware update trigger method for qspi and non-qspi device --- src/otaclient/boot_control/_jetson_uefi.py | 53 +++++++++++----------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index f0cc1f2d1..f05fc1c1e 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -313,7 +313,7 @@ def __init__( boot_parent_devpath: StrOrPath, standby_slot_mp: StrOrPath, *, - tnspec: str, + nvbootctrl_conf: str, fw_bsp_ver_control: FirmwareBSPVersionControl, firmware_update_request: FirmwareUpdateRequest, firmware_manifest: FirmwareManifest, @@ -324,7 +324,7 @@ def __init__( boot_parent_devpath (StrOrPath): The parent dev of current rootfs device. standby_slot_mp (StrOrPath): The mount point of updated standby slot. The firmware update package is located at /opt/ota_package folder. - tnspec (str): The firmware compatibility string. + nvbootctrl_conf (str): The contents of nv_boot_ctrl.conf file. fw_bsp_ver_control (FirmwareBSPVersionControl): The firmware BSP version control for each slots. firmware_update_request (FirmwareUpdateRequest) firmware_manifest (FirmwareBSPVersionControl) @@ -332,7 +332,7 @@ def __init__( self.current_slot_bsp_ver = fw_bsp_ver_control.current_slot_bsp_ver self.standby_slot_bsp_ver = fw_bsp_ver_control.standby_slot_bsp_ver - self.tnspec = tnspec + self.nvbootctrl_conf = nvbootctrl_conf self.firmware_update_request = firmware_update_request self.firmware_manifest = firmware_manifest self.firmware_package_bsp_ver = BSPVersion.parse( @@ -530,10 +530,17 @@ def firmware_update(self) -> bool: return False # check firmware compatibility, this is to prevent failed firmware update beforehand. - if not self.firmware_manifest.check_compat(self.tnspec): + tnspec = get_nvbootctrl_conf_tnspec(self.nvbootctrl_conf) + if not tnspec: + logger.warning( + "TNSPEC is not defined in nvbootctrl config file, skip firmware update!" + ) + return False + + if not self.firmware_manifest.check_compat(tnspec): _err_msg = ( "firmware package is incompatible with this device: " - f"{self.tnspec=}, {self.firmware_manifest.firmware_spec.firmware_compat}, " + f"{tnspec=}, {self.firmware_manifest.firmware_spec.firmware_compat}, " "skip firmware update" ) logger.warning(_err_msg) @@ -561,23 +568,20 @@ def firmware_update(self) -> bool: self._update_l4tlauncher() # write special UEFI variable to trigger firmware update on next reboot - with _ensure_efivarfs_mounted(): - try: - self._write_magic_efivar() - except Exception as e: - logger.warning( - ( - f"failed to configure capsule update by write magic value: {e!r}\n" - "firmware update might be skipped!" - ) - ) - return False + firmware_update_triggerred = False + if _detect_ota_bootdev_is_qspi(self.nvbootctrl_conf): + firmware_update_triggerred = _trigger_capsule_update_qspi_ota_bootdev() + else: + firmware_update_triggerred = _trigger_capsule_update_non_qspi_ota_bootdev( + self.esp_mp + ) - logger.warning( - "firmware update package prepare finished" - f"will update firmware to {self.firmware_package_bsp_ver} in next reboot" - ) - return True + if firmware_update_triggerred: + logger.warning( + "firmware update package prepare finished" + f"will update firmware to {self.firmware_package_bsp_ver} in next reboot" + ) + return firmware_update_triggerred class _UEFIBootControl: @@ -612,6 +616,7 @@ def __init__(self): logger.error(_err_msg) raise JetsonUEFIBootControlError(_err_msg) self.nvbootctrl_conf = nvbootctrl_conf_fpath.read_text() + logger.info(f"nvboot_ctrl_conf: \n{self.nvbootctrl_conf}") # ------ check current slot BSP version ------ # # check current slot firmware BSP version @@ -793,10 +798,6 @@ def _firmware_update(self) -> bool: True if there is firmware update configured, False for no firmware update. """ logger.info("jetson-uefi: checking if we need to do firmware update ...") - if not (tnspec := self._uefi_control.tnspec): - logger.warning("tnspec is not defined, skip firmware update") - return False - firmware_package_meta = load_firmware_package( firmware_update_request_fpath=replace_root( boot_cfg.FIRMWARE_UPDATE_REQUEST_FPATH, @@ -823,10 +824,10 @@ def _firmware_update(self) -> bool: firmware_updater = UEFIFirmwareUpdater( boot_parent_devpath=self._uefi_control.parent_devpath, standby_slot_mp=self._mp_control.standby_slot_mount_point, - tnspec=tnspec, fw_bsp_ver_control=self._firmware_bsp_ver_control, firmware_update_request=firmware_update_request, firmware_manifest=firmware_manifest, + nvbootctrl_conf=self._uefi_control.nvbootctrl_conf, ) return firmware_updater.firmware_update() From 58644827710babb935ef09008b544aec10a38817 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 22 Aug 2024 11:35:32 +0000 Subject: [PATCH 152/193] jetson-cboot: explicitly require nvbootctrl config file presented --- src/otaclient/boot_control/_jetson_cboot.py | 28 +++++++++------------ 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_cboot.py b/src/otaclient/boot_control/_jetson_cboot.py index a6a47c6f3..62bf400df 100644 --- a/src/otaclient/boot_control/_jetson_cboot.py +++ b/src/otaclient/boot_control/_jetson_cboot.py @@ -328,20 +328,15 @@ def __init__(self): f"nvbootctrl -t rootfs dump-slots-info: \n{_NVBootctrl.dump_slots_info(target='rootfs')}" ) - # load tnspec for firmware update compatibility check - try: - self.tnspec = get_nvbootctrl_conf_tnspec( - Path(boot_cfg.NVBOOTCTRL_CONF_FPATH).read_text() - ) - logger.info(f"firmware compatibility: {self.tnspec}") - except Exception as e: - logger.warning( - ( - f"failed to load tnspec: {e!r}, " - "this will result in firmware update being skipped!" - ) - ) - self.tnspec = None + # load nvbootctrl config file + if not ( + nvbootctrl_conf_fpath := Path(boot_cfg.NVBOOTCTRL_CONF_FPATH) + ).is_file(): + _err_msg = "nv_boot_ctrl.conf is missing!" + logger.error(_err_msg) + raise JetsonCBootContrlError(_err_msg) + self.nvbootctrl_conf = nvbootctrl_conf_fpath.read_text() + logger.info(f"nvboot_ctrl_conf: \n{self.nvbootctrl_conf}") # API @@ -465,8 +460,9 @@ def _firmware_update(self) -> bool | None: None for no firmware update occurs. """ logger.info("jetson-cboot: entering nv firmware update ...") - if not (tnspec := self._cboot_control.tnspec): - logger.warning("tnspec is not defined, skip firmware update") + tnspec = get_nvbootctrl_conf_tnspec(self._cboot_control.nvbootctrl_conf) + if not tnspec: + logger.warning("tnspec is not defined, skip firmware update!") return firmware_package_meta = load_firmware_package( From 0463d68c14355bf6ed6311ed85fcecf514a62c30 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 22 Aug 2024 11:36:25 +0000 Subject: [PATCH 153/193] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_otaclient/test_boot_control/test_jetson_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_otaclient/test_boot_control/test_jetson_common.py b/tests/test_otaclient/test_boot_control/test_jetson_common.py index a6cc197b9..8a749dd84 100644 --- a/tests/test_otaclient/test_boot_control/test_jetson_common.py +++ b/tests/test_otaclient/test_boot_control/test_jetson_common.py @@ -308,7 +308,7 @@ def test_get_partition_devpath(parent_devpath, partition_id, expected): TNSPEC 3701-500-0005-A.0-1-0-cti-orin-agx-agx201-00- TEGRA_CHIPID 0x23 TEGRA_OTA_BOOT_DEVICE /dev/mtdblock0 -TEGRA_OTA_GPT_DEVICE /dev/mtdblock0 +TEGRA_OTA_GPT_DEVICE /dev/mtdblock0 """, "3701-500-0005-A.0-1-0-cti-orin-agx-agx201-00-", ), From 89a16616c02b24cc545ba3bcb5ed7be9990d407d Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 22 Aug 2024 11:38:02 +0000 Subject: [PATCH 154/193] add some logs --- src/otaclient/boot_control/_jetson_uefi.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index f05fc1c1e..af885c766 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -570,8 +570,10 @@ def firmware_update(self) -> bool: # write special UEFI variable to trigger firmware update on next reboot firmware_update_triggerred = False if _detect_ota_bootdev_is_qspi(self.nvbootctrl_conf): + logger.info("device is using QSPI flash as TEGRA_OTA_BOOT_DEVICE") firmware_update_triggerred = _trigger_capsule_update_qspi_ota_bootdev() else: + logger.info("device is using internal EMMC flash as TEGRA_OTA_BOOT_DEVICE") firmware_update_triggerred = _trigger_capsule_update_non_qspi_ota_bootdev( self.esp_mp ) From 6e1121bf70048bd5d880e9a42bf6e729240cdfdf Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Thu, 22 Aug 2024 11:42:51 +0000 Subject: [PATCH 155/193] minor update --- src/otaclient/boot_control/_jetson_uefi.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index af885c766..57a3e191e 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -276,7 +276,7 @@ def _detect_ota_bootdev_is_qspi(nvboot_ctrl_conf: str) -> bool | None: return if ota_bootdev.startswith("/dev/mmcblk0"): - logger.info("device doesn't use QSPI flash as TEGRA_OTA_BOOT_DEVICE") + logger.info("device is using internal emmc flash as TEGRA_OTA_BOOT_DEVICE") return False if ota_bootdev.startswith("/dev/mtdblock0"): @@ -570,10 +570,8 @@ def firmware_update(self) -> bool: # write special UEFI variable to trigger firmware update on next reboot firmware_update_triggerred = False if _detect_ota_bootdev_is_qspi(self.nvbootctrl_conf): - logger.info("device is using QSPI flash as TEGRA_OTA_BOOT_DEVICE") firmware_update_triggerred = _trigger_capsule_update_qspi_ota_bootdev() else: - logger.info("device is using internal EMMC flash as TEGRA_OTA_BOOT_DEVICE") firmware_update_triggerred = _trigger_capsule_update_non_qspi_ota_bootdev( self.esp_mp ) From f003534ed3f09870c1d5b74587b9ea933215643a Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Fri, 23 Aug 2024 00:46:06 +0000 Subject: [PATCH 156/193] jetson-uefi: mark functions and classes that are not meant to be measured coverage rate --- src/otaclient/boot_control/_jetson_uefi.py | 24 ++++++++++++---------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index 57a3e191e..2654af337 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -82,7 +82,7 @@ } -class JetsonUEFIBootControlError(Exception): +class JetsonUEFIBootControlError(Exception): # pragma: no cover """Exception type for covering jetson-uefi related errors.""" @@ -125,7 +125,7 @@ def get_current_fw_bsp_version(cls) -> BSPVersion: return bsp_ver @classmethod - def verify(cls) -> str: + def verify(cls) -> str: # pragma: no cover """Verify the bootloader and rootfs boot.""" return cls._nvbootctrl("verify", check_output=True) @@ -135,7 +135,7 @@ def verify(cls) -> str: @contextlib.contextmanager -def _ensure_efivarfs_mounted() -> Generator[None, Any, None]: +def _ensure_efivarfs_mounted() -> Generator[None, Any, None]: # pragma: no cover """Ensure the efivarfs is mounted as rw, and then umount it.""" if CMDHelperFuncs.is_target_mounted(EFIVARS_SYS_MOUNT_POINT): options = "remount,rw,nosuid,nodev,noexec,relatime" @@ -163,7 +163,7 @@ def _ensure_efivarfs_mounted() -> Generator[None, Any, None]: ) from e -def _trigger_capsule_update_qspi_ota_bootdev() -> bool: +def _trigger_capsule_update_qspi_ota_bootdev() -> bool: # pragma: no cover """Write magic efivar to trigger firmware Capsule update in next boot. This method is for device using QSPI flash as TEGRA_OTA_BOOT_DEVICE. @@ -187,7 +187,9 @@ def _trigger_capsule_update_qspi_ota_bootdev() -> bool: return False -def _trigger_capsule_update_non_qspi_ota_bootdev(esp_mp: StrOrPath) -> bool: +def _trigger_capsule_update_non_qspi_ota_bootdev( + esp_mp: StrOrPath, +) -> bool: # pragma: no cover """Write magic efivar to trigger firmware Capsule update in next boot. Write a special file with magic value into specific location at internal emmc ESP partition. @@ -217,7 +219,7 @@ def _trigger_capsule_update_non_qspi_ota_bootdev(esp_mp: StrOrPath) -> bool: @contextlib.contextmanager def _ensure_esp_mounted( esp_dev: StrOrPath, mount_point: StrOrPath -) -> Generator[None, Any, None]: +) -> Generator[None, Any, None]: # pragma: no cover """Mount the esp partition and then umount it.""" mount_point = Path(mount_point) mount_point.mkdir(exist_ok=True, parents=True) @@ -714,7 +716,7 @@ def __init__(self): # API - def switch_boot_to_standby(self) -> None: + def switch_boot_to_standby(self) -> None: # pragma: no cover target_slot = self.standby_slot logger.info(f"switch boot to standby slot({target_slot})") @@ -833,10 +835,10 @@ def _firmware_update(self) -> bool: # APIs - def get_standby_slot_path(self) -> Path: + def get_standby_slot_path(self) -> Path: # pragma: no cover return self._mp_control.standby_slot_mount_point - def get_standby_boot_dir(self) -> Path: + def get_standby_boot_dir(self) -> Path: # pragma: no cover return self._mp_control.standby_boot_dir def pre_update(self, version: str, *, standby_as_ref: bool, erase_standby: bool): @@ -952,8 +954,8 @@ def on_operation_failure(self): self._ota_status_control.on_failure() self._mp_control.umount_all(ignore_error=True) - def load_version(self) -> str: + def load_version(self) -> str: # pragma: no cover return self._ota_status_control.load_active_slot_version() - def get_booted_ota_status(self) -> api_types.StatusOta: + def get_booted_ota_status(self) -> api_types.StatusOta: # pragma: no cover return self._ota_status_control.booted_ota_status From f0b0a8f86524c3164daa048d931a045f741741f4 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Fri, 23 Aug 2024 00:54:04 +0000 Subject: [PATCH 157/193] WIP: implement test_jetson_uefi --- .../test_boot_control/test_jetson_uefi.py | 132 ++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 tests/test_otaclient/test_boot_control/test_jetson_uefi.py diff --git a/tests/test_otaclient/test_boot_control/test_jetson_uefi.py b/tests/test_otaclient/test_boot_control/test_jetson_uefi.py new file mode 100644 index 000000000..9538e37bc --- /dev/null +++ b/tests/test_otaclient/test_boot_control/test_jetson_uefi.py @@ -0,0 +1,132 @@ +# Copyright 2022 TIER IV, INC. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import annotations + +import pytest +from pytest_mock import MockerFixture + +from otaclient.boot_control import _jetson_uefi +from otaclient.boot_control._jetson_common import BSPVersion +from otaclient.boot_control._jetson_uefi import ( + JetsonUEFIBootControlError, + NVBootctrlJetsonUEFI, + _detect_esp_dev, +) + +MODULE = _jetson_uefi.__name__ + + +class TestNVBootctrlJetsonUEFI: + + @pytest.mark.parametrize( + "_input, expected", + ( + ( + """\ +Current version: 35.4.1 +Capsule update status: 1 +Current bootloader slot: A +Active bootloader slot: A +num_slots: 2 +slot: 0, status: normal +slot: 1, status: normal + """, + BSPVersion.parse("35.4.1"), + ), + ( + """\ +Current version: 36.3.0 +Capsule update status: 0 +Current bootloader slot: A +Active bootloader slot: A +num_slots: 2 +slot: 0, status: normal +slot: 1, status: normal + """, + BSPVersion.parse("36.3.0"), + ), + ( + """\ +Current version: 0.0.0 +Current bootloader slot: A +Active bootloader slot: A +num_slots: 2 +slot: 0, status: normal +slot: 1, status: normal + """, + ValueError(), + ), + ), + ) + def get_current_fw_bsp_version( + self, _input: str, expected: BSPVersion | Exception, mocker: MockerFixture + ): + mocker.patch.object( + NVBootctrlJetsonUEFI, + "dump_slots_info", + mocker.MagicMock(return_value=_input), + ) + + if isinstance(expected, Exception): + with pytest.raises(type(expected)): + NVBootctrlJetsonUEFI.get_current_fw_bsp_version() + else: + assert NVBootctrlJetsonUEFI.get_current_fw_bsp_version() == expected + + +@pytest.mark.parametrize( + "all_esp_devs, boot_parent_devpath, expected", + ( + ( + ["/dev/mmcblk0p41", "/dev/nvme0n1p41"], + "/dev/nvme0n1", + "/dev/nvme0n1p41", + ), + ( + ["/dev/mmcblk0p41"], + "/dev/mmcblk0", + "/dev/mmcblk0p41", + ), + ( + ["/dev/sda23", "/dev/mmcblk0p41"], + "/dev/nvme0n1", + JetsonUEFIBootControlError(), + ), + ( + [], + "/dev/nvme0n1", + JetsonUEFIBootControlError(), + ), + ), +) +def test__detect_esp_dev( + all_esp_devs, boot_parent_devpath, expected, mocker: MockerFixture +): + mocker.patch( + f"{MODULE}.CMDHelperFuncs.get_dev_by_token", + mocker.MagicMock(return_value=all_esp_devs), + ) + + if isinstance(expected, Exception): + with pytest.raises(type(expected)): + _detect_esp_dev(boot_parent_devpath) + else: + assert _detect_esp_dev(boot_parent_devpath) == expected + + +class TestDetectDeviceOTABOOTDEV: + + pass From 5c511aeedc872fb210ab153e19d881c9d648bb1e Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Fri, 23 Aug 2024 01:17:01 +0000 Subject: [PATCH 158/193] jetson-uefi: skip firmware update if failed to detect TEGRA_OTA_BOOTDEV --- src/otaclient/boot_control/_jetson_uefi.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index 2654af337..386376bfe 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -570,8 +570,12 @@ def firmware_update(self) -> bool: self._update_l4tlauncher() # write special UEFI variable to trigger firmware update on next reboot - firmware_update_triggerred = False - if _detect_ota_bootdev_is_qspi(self.nvbootctrl_conf): + device_uses_qspi = _detect_ota_bootdev_is_qspi(self.nvbootctrl_conf) + if device_uses_qspi is None: + logger.warning("failed to detect OTA_BOOTDEV, skip firmware update") + return False + + if device_uses_qspi: firmware_update_triggerred = _trigger_capsule_update_qspi_ota_bootdev() else: firmware_update_triggerred = _trigger_capsule_update_non_qspi_ota_bootdev( From 5e82ab83e8e9e1b40f00ab89d0676dd57f569648 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Fri, 23 Aug 2024 01:27:50 +0000 Subject: [PATCH 159/193] WIP: test_jetson-uefi: implement test detect_ota_bootdev_is_qspi and TestL4TLauncherBSPVersionControl --- .../test_boot_control/test_jetson_uefi.py | 69 ++++++++++++++++++- 1 file changed, 67 insertions(+), 2 deletions(-) diff --git a/tests/test_otaclient/test_boot_control/test_jetson_uefi.py b/tests/test_otaclient/test_boot_control/test_jetson_uefi.py index 9538e37bc..5b1bcc9c4 100644 --- a/tests/test_otaclient/test_boot_control/test_jetson_uefi.py +++ b/tests/test_otaclient/test_boot_control/test_jetson_uefi.py @@ -17,13 +17,16 @@ import pytest from pytest_mock import MockerFixture +from typing_extensions import LiteralString from otaclient.boot_control import _jetson_uefi from otaclient.boot_control._jetson_common import BSPVersion from otaclient.boot_control._jetson_uefi import ( JetsonUEFIBootControlError, + L4TLauncherBSPVersionControl, NVBootctrlJetsonUEFI, _detect_esp_dev, + _detect_ota_bootdev_is_qspi, ) MODULE = _jetson_uefi.__name__ @@ -127,6 +130,68 @@ def test__detect_esp_dev( assert _detect_esp_dev(boot_parent_devpath) == expected -class TestDetectDeviceOTABOOTDEV: +@pytest.mark.parametrize( + "nvbootctrl_conf, expected", + ( + ( + """\ +TNSPEC 3701-500-0005-A.0-1-0-cti-orin-agx-agx201-00- +TEGRA_CHIPID 0x23 +TEGRA_OTA_BOOT_DEVICE /dev/mtdblock0 +TEGRA_OTA_GPT_DEVICE /dev/mtdblock0 +""", + True, + ), + ( + """\ +TNSPEC 2888-400-0004-M.0-1-2-jetson-xavier-rqx580- +TEGRA_CHIPID 0x19 +TEGRA_OTA_BOOT_DEVICE /dev/mmcblk0boot0 +TEGRA_OTA_GPT_DEVICE /dev/mmcblk0boot1 +""", + False, + ), + ("", None), + ), +) +def test__detect_ota_bootdev_is_qspi(nvbootctrl_conf, expected): + assert _detect_ota_bootdev_is_qspi(nvbootctrl_conf) is expected + + +class TestL4TLauncherBSPVersionControl: + + @pytest.mark.parametrize( + "_in, expected", + _test_case := ( + ( + "R35.4.1:ac17457772666351154a5952e3b87851a6398da2afcf3a38bedfc0925760bb0e", + L4TLauncherBSPVersionControl( + bsp_ver=BSPVersion(35, 4, 1), + sha256_digest="ac17457772666351154a5952e3b87851a6398da2afcf3a38bedfc0925760bb0e", + ), + ), + ( + "R35.5.0:3928e0feb84e37db87dc6705a02eec95a6fca2dccd3467b61fa66ed8e1046b67", + L4TLauncherBSPVersionControl( + bsp_ver=BSPVersion(35, 5, 0), + sha256_digest="3928e0feb84e37db87dc6705a02eec95a6fca2dccd3467b61fa66ed8e1046b67", + ), + ), + ( + "R36.3.0:b14fa3623f4078d05573d9dcf2a0b46ea2ae07d6b75d9843f9da6ff24db13718", + L4TLauncherBSPVersionControl( + bsp_ver=BSPVersion(36, 3, 0), + sha256_digest="b14fa3623f4078d05573d9dcf2a0b46ea2ae07d6b75d9843f9da6ff24db13718", + ), + ), + ), + ) + def test_parse(self, _in, expected): + assert L4TLauncherBSPVersionControl.parse(_in) == expected - pass + @pytest.mark.parametrize( + "to_be_dumped, expected", + tuple((entry[1], entry[0]) for entry in _test_case), + ) + def test_dump(self, to_be_dumped: L4TLauncherBSPVersionControl, expected: str): + assert to_be_dumped.dump() == expected From c05ee003c9abc64d5f3c3b86fc31176a20ae1c5f Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Fri, 23 Aug 2024 01:36:05 +0000 Subject: [PATCH 160/193] jetson-uefi: make detect_l4tlauncher_version as a standalone function --- src/otaclient/boot_control/_jetson_uefi.py | 129 ++++++++++++--------- 1 file changed, 74 insertions(+), 55 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index 386376bfe..e8cf8c795 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -307,6 +307,75 @@ def dump(self) -> str: return f"{self.bsp_ver.dump()}{self.SEP}{self.sha256_digest}" +def _l4tlauncher_version_control( + l4tlauncher_ver_fpath: StrOrPath, + l4tlauncher_at_esp: StrOrPath, + *, + current_slot_bsp_ver: BSPVersion, +) -> BSPVersion: + """Try to determine the current in use l4tlauncher version and update ver control file. + + If the version file is presented and the sha256digest matched, return the BSP version from version file. + + If sha256digest mismatched or version file missing: + 1. try to lookup the L4TLAUNCHER_BSP_VER_SHA256_MAP. + 2. assume that the launcher is the same version of current slot firmware BSP version. + Write new version file after detecting. + """ + l4tlauncher_sha256_digest = file_sha256(l4tlauncher_at_esp) + l4tlauncher_ver_fpath = Path(l4tlauncher_ver_fpath) + + # try to determine the version with version file + try: + _ver_control = L4TLauncherBSPVersionControl.parse( + l4tlauncher_ver_fpath.read_text() + ) + if l4tlauncher_sha256_digest == _ver_control.sha256_digest: + return _ver_control.bsp_ver + + logger.warning( + ( + "l4tlauncher sha256 hash mismatched: " + f"{l4tlauncher_sha256_digest=}, {_ver_control.sha256_digest=}, " + "remove version file" + ) + ) + raise ValueError("sha256 hash mismatched") + except Exception as e: + logger.warning(f"missing or invalid l4tlauncher version file: {e!r}") + l4tlauncher_ver_fpath.unlink(missing_ok=True) + + # try to determine the version by looking up table + # NOTE(20240624): since the number of l4tlauncher version is limited, + # we can lookup against a pre-calculated sha256 digest map. + logger.info( + f"try to determine the l4tlauncher verison by hash: {l4tlauncher_sha256_digest}" + ) + if l4tlauncher_bsp_ver := L4TLAUNCHER_BSP_VER_SHA256_MAP.get( + l4tlauncher_sha256_digest + ): + _ver_control = L4TLauncherBSPVersionControl( + bsp_ver=l4tlauncher_bsp_ver, sha256_digest=l4tlauncher_sha256_digest + ) + write_str_to_file_sync(l4tlauncher_ver_fpath, _ver_control.dump()) + return l4tlauncher_bsp_ver + + # NOTE(20240624): if we failed to detect the l4tlauncher's version, + # we assume that the launcher is the same version as current slot's fw. + # This is typically the case of a newly flashed ECU. + logger.warning( + ( + "failed to determine the l4tlauncher's version, assuming " + f"version is the same as current slot's fw version: {current_slot_bsp_ver}" + ) + ) + _ver_control = L4TLauncherBSPVersionControl( + bsp_ver=current_slot_bsp_ver, sha256_digest=l4tlauncher_sha256_digest + ) + write_str_to_file_sync(l4tlauncher_ver_fpath, _ver_control.dump()) + return current_slot_bsp_ver + + class UEFIFirmwareUpdater: """Firmware update implementation using Capsule update.""" @@ -436,60 +505,6 @@ def _update_l4tlauncher(self) -> bool: logger.info("no boot application update is configured in the request, skip") return False - def _detect_l4tlauncher_version(self) -> BSPVersion: - """Try to determine the current in use l4tlauncher version.""" - l4tlauncher_sha256_digest = file_sha256(self.bootaa64_at_esp) - - # try to determine the version with version file - try: - _ver_control = L4TLauncherBSPVersionControl.parse( - self.l4tlauncher_ver_fpath.read_text() - ) - if l4tlauncher_sha256_digest == _ver_control.sha256_digest: - return _ver_control.bsp_ver - - logger.warning( - ( - "l4tlauncher sha256 hash mismatched: " - f"{l4tlauncher_sha256_digest=}, {_ver_control.sha256_digest=}, " - "remove version file" - ) - ) - raise ValueError("sha256 hash mismatched") - except Exception as e: - logger.warning(f"missing or invalid l4tlauncher version file: {e!r}") - self.l4tlauncher_ver_fpath.unlink(missing_ok=True) - - # try to determine the version by looking up table - # NOTE(20240624): since the number of l4tlauncher version is limited, - # we can lookup against a pre-calculated sha256 digest map. - logger.info( - f"try to determine the l4tlauncher verison by hash: {l4tlauncher_sha256_digest}" - ) - if l4tlauncher_bsp_ver := L4TLAUNCHER_BSP_VER_SHA256_MAP.get( - l4tlauncher_sha256_digest - ): - _ver_control = L4TLauncherBSPVersionControl( - bsp_ver=l4tlauncher_bsp_ver, sha256_digest=l4tlauncher_sha256_digest - ) - write_str_to_file_sync(self.l4tlauncher_ver_fpath, _ver_control.dump()) - return l4tlauncher_bsp_ver - - # NOTE(20240624): if we failed to detect the l4tlauncher's version, - # we assume that the launcher is the same version as current slot's fw. - # This is typically the case of a newly flashed ECU. - logger.warning( - ( - "failed to determine the l4tlauncher's version, assuming " - f"version is the same as current slot's fw version: {self.current_slot_bsp_ver}" - ) - ) - _ver_control = L4TLauncherBSPVersionControl( - bsp_ver=self.current_slot_bsp_ver, sha256_digest=l4tlauncher_sha256_digest - ) - write_str_to_file_sync(self.l4tlauncher_ver_fpath, _ver_control.dump()) - return self.current_slot_bsp_ver - # APIs def firmware_update(self) -> bool: @@ -555,7 +570,11 @@ def firmware_update(self) -> bool: return False logger.info("on capsule prepared, try to update L4TLauncher ...") - l4tlauncher_bsp_ver = self._detect_l4tlauncher_version() + l4tlauncher_bsp_ver = _l4tlauncher_version_control( + l4tlauncher_ver_fpath=self.l4tlauncher_ver_fpath, + l4tlauncher_at_esp=self.bootaa64_at_esp, + current_slot_bsp_ver=self.current_slot_bsp_ver, + ) logger.info(f"current l4tlauncher version: {l4tlauncher_bsp_ver}") if l4tlauncher_bsp_ver >= self.firmware_package_bsp_ver: From 24689a02b92fcd2c786e28ffa5df9b57bd57b23b Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Fri, 23 Aug 2024 02:15:36 +0000 Subject: [PATCH 161/193] WIP: test_jetson-uefi: implement test__l4tlauncher_version_control --- src/otaclient/boot_control/_jetson_uefi.py | 7 +- .../test_boot_control/test_jetson_uefi.py | 66 +++++++++++++++++++ 2 files changed, 71 insertions(+), 2 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index e8cf8c795..b444ed8a8 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -313,14 +313,17 @@ def _l4tlauncher_version_control( *, current_slot_bsp_ver: BSPVersion, ) -> BSPVersion: - """Try to determine the current in use l4tlauncher version and update ver control file. + """Try to determine the current in use l4tlauncher version and update ver control file if needed. If the version file is presented and the sha256digest matched, return the BSP version from version file. If sha256digest mismatched or version file missing: 1. try to lookup the L4TLAUNCHER_BSP_VER_SHA256_MAP. 2. assume that the launcher is the same version of current slot firmware BSP version. - Write new version file after detecting. + Write new version file after new detecting. + + Returns: + The detected L4TLauncher BSP version. """ l4tlauncher_sha256_digest = file_sha256(l4tlauncher_at_esp) l4tlauncher_ver_fpath = Path(l4tlauncher_ver_fpath) diff --git a/tests/test_otaclient/test_boot_control/test_jetson_uefi.py b/tests/test_otaclient/test_boot_control/test_jetson_uefi.py index 5b1bcc9c4..9cbef68f5 100644 --- a/tests/test_otaclient/test_boot_control/test_jetson_uefi.py +++ b/tests/test_otaclient/test_boot_control/test_jetson_uefi.py @@ -27,6 +27,7 @@ NVBootctrlJetsonUEFI, _detect_esp_dev, _detect_ota_bootdev_is_qspi, + _l4tlauncher_version_control, ) MODULE = _jetson_uefi.__name__ @@ -195,3 +196,68 @@ def test_parse(self, _in, expected): ) def test_dump(self, to_be_dumped: L4TLauncherBSPVersionControl, expected: str): assert to_be_dumped.dump() == expected + + +@pytest.mark.parametrize( + "ver_ctrl, l4tlauncher_digest, current_bsp_ver, expected_bsp_ver, expected_ver_ctrl", + ( + # valid version file, hash matched + ( + "R36.3.0:b14fa3623f4078d05573d9dcf2a0b46ea2ae07d6b75d9843f9da6ff24db13718", + "b14fa3623f4078d05573d9dcf2a0b46ea2ae07d6b75d9843f9da6ff24db13718", + BSPVersion(1, 2, 3), + BSPVersion(36, 3, 0), + "R36.3.0:b14fa3623f4078d05573d9dcf2a0b46ea2ae07d6b75d9843f9da6ff24db13718", + ), + # no version file, lookup table hit + ( + "invalid_version_file", + "b14fa3623f4078d05573d9dcf2a0b46ea2ae07d6b75d9843f9da6ff24db13718", + BSPVersion(1, 2, 3), + BSPVersion(36, 3, 0), + "R36.3.0:b14fa3623f4078d05573d9dcf2a0b46ea2ae07d6b75d9843f9da6ff24db13718", + ), + # valid version file, hash mismatched, use slot BSP version + ( + "R36.3.0:b14fa3623f4078d05573d9dcf2a0b46ea2ae07d6b75d9843f9da6ff24db13718", + "hash_mismatched", + BSPVersion(1, 2, 3), + BSPVersion(1, 2, 3), + "R1.2.3:hash_mismatched", + ), + # no version file, lookup table unhit + ( + "invalid_version_file", + "not_recorded_hash", + BSPVersion(1, 2, 3), + BSPVersion(1, 2, 3), + "R1.2.3:not_recorded_hash", + ), + ), +) +def test__l4tlauncher_version_control( + ver_ctrl, + l4tlauncher_digest, + current_bsp_ver, + expected_bsp_ver, + expected_ver_ctrl, + tmp_path, + mocker: MockerFixture, +): + ver_control_f = tmp_path / "l4tlauncher_ver_control" + ver_control_f.write_text(ver_ctrl) + + mocker.patch( + f"{MODULE}.file_sha256", mocker.MagicMock(return_value=l4tlauncher_digest) + ) + + assert ( + _l4tlauncher_version_control( + ver_control_f, + "any", + current_slot_bsp_ver=current_bsp_ver, + ) + == expected_bsp_ver + ) + # ensure the version control file is expected + assert ver_control_f.read_text() == expected_ver_ctrl From 6d58f880110bb58a70fc77096647e2165474fa77 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Fri, 23 Aug 2024 02:35:11 +0000 Subject: [PATCH 162/193] firmware_package: PayloadFileLocation and DigestValue are still pydantic models --- .../boot_control/_firmware_package.py | 49 +++++++++---------- 1 file changed, 22 insertions(+), 27 deletions(-) diff --git a/src/otaclient/boot_control/_firmware_package.py b/src/otaclient/boot_control/_firmware_package.py index 1b718c491..0f3c9396a 100644 --- a/src/otaclient/boot_control/_firmware_package.py +++ b/src/otaclient/boot_control/_firmware_package.py @@ -46,11 +46,10 @@ import re from enum import Enum from pathlib import Path -from typing import Any, List, Literal +from typing import List, Literal, Union import yaml -from pydantic import BaseModel, BeforeValidator, GetCoreSchemaHandler -from pydantic_core import CoreSchema, core_schema +from pydantic import BaseModel, BeforeValidator from typing_extensions import Annotated from otaclient_common.typing import StrOrPath, gen_strenum_validator @@ -67,24 +66,18 @@ class PayloadType(str, Enum): DIGEST_PATTERN = re.compile(r"^(?P[\w\-+ ]+):(?P[a-zA-Z0-9]+)$") -def _pydantic_str_schema( - cls, source_type: Any, handler: GetCoreSchemaHandler -) -> CoreSchema: - """Pydantic schema adapter for str.""" - return core_schema.no_info_after_validator_function(cls, handler(str)) - - -class DigestValue(str): +class DigestValue(BaseModel): """Implementation of digest value schema :.""" - def __init__(self, _in: str) -> None: + algorithm: str + digest: str + + @classmethod + def parse(cls, _in: str) -> DigestValue: _in = _in.strip() if not (ma := DIGEST_PATTERN.match(_in)): raise ValueError(f"invalid digest value: {_in}") - self.algorithm = ma.group("algorithm") - self.digest = ma.group("digest") - - __get_pydantic_core_schema__ = classmethod(_pydantic_str_schema) + return DigestValue(algorithm=ma.group("algorithm"), digest=ma.group("digest")) class NVIDIAFirmwareCompat(BaseModel): @@ -107,7 +100,7 @@ class NVIDIAFirmwareSpec(BaseModel): firmware_compat: NVIDIAFirmwareCompat -class PayloadFileLocation(str): +class PayloadFileLocation(BaseModel): """Specifying the payload file location. It supports file URL or digest value. @@ -119,26 +112,28 @@ class PayloadFileLocation(str): """ location_type: Literal["blob", "file"] - location_path: str | DigestValue + location_path: Union[str, DigestValue] - def __init__(self, _in: str) -> None: + @classmethod + def parse(cls, _in: str) -> PayloadFileLocation: if _in.startswith("file://"): - self.location_type = "file" - self.location_path = _in.replace("file://", "", 1) + location_type = "file" + location_path = _in.replace("file://", "", 1) else: - self.location_type = "blob" - self.location_path = DigestValue(_in) - - __get_pydantic_core_schema__ = classmethod(_pydantic_str_schema) + location_type = "blob" + location_path = DigestValue.parse(_in) + return cls(location_type=location_type, location_path=location_path) class FirmwarePackage(BaseModel): """Metadata of a firmware update package payload.""" payload_name: str - file_location: Annotated[PayloadFileLocation, BeforeValidator(PayloadFileLocation)] + file_location: Annotated[ + PayloadFileLocation, BeforeValidator(PayloadFileLocation.parse) + ] type: Annotated[PayloadType, BeforeValidator(gen_strenum_validator(PayloadType))] - digest: DigestValue + digest: Annotated[DigestValue, BeforeValidator(DigestValue.parse)] class HardwareType(str, Enum): From e21b09e87191d94b6544c540c0684613eb83da99 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Fri, 23 Aug 2024 02:49:32 +0000 Subject: [PATCH 163/193] fix up test_firmware_package --- .../test_boot_control/test_firmware_package.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/test_otaclient/test_boot_control/test_firmware_package.py b/tests/test_otaclient/test_boot_control/test_firmware_package.py index 05ddc0c44..5302cc1a1 100644 --- a/tests/test_otaclient/test_boot_control/test_firmware_package.py +++ b/tests/test_otaclient/test_boot_control/test_firmware_package.py @@ -51,7 +51,7 @@ ), ) def test_digest_value_parsing(_in, _expected: list[str]): - _parsed = DigestValue(_in) + _parsed = DigestValue.parse(_in) assert _parsed.algorithm == _expected[0] assert _parsed.digest == _expected[1] @@ -67,13 +67,13 @@ def test_digest_value_parsing(_in, _expected: list[str]): _in := "sha256:32baa6f7e96661d50fb78e5d7149763e3a0fe70c51c37c6bea92c3c27cd2472d", [ "blob", - DigestValue(_in), + DigestValue.parse(_in), ], ), ), ) def test_payload_file_location(_in, _expected: list[str] | list[str | DigestValue]): - _parsed = PayloadFileLocation(_in) + _parsed = PayloadFileLocation.parse(_in) assert _parsed.location_type == _expected[0] assert _parsed.location_path == _expected[1] @@ -153,19 +153,21 @@ def test_check_compat(_spec, _compat_str, _expected): firmware_packages=[ FirmwarePackage( payload_name="bl_only_payload.Cap", - file_location=PayloadFileLocation( + file_location=PayloadFileLocation.parse( "file:///opt/ota/firmware/bl_only_payload.Cap" ), type=PayloadType("UEFI-CAPSULE"), - digest=DigestValue( + digest=DigestValue.parse( "sha256:32baa6f7e96661d50fb78e5d7149763e3a0fe70c51c37c6bea92c3c27cd2472d" ), ), FirmwarePackage( payload_name="BOOTAA64.efi", - file_location=PayloadFileLocation("file:///opt/ota/firmware/BOOTAA64.efi"), + file_location=PayloadFileLocation.parse( + "file:///opt/ota/firmware/BOOTAA64.efi" + ), type=PayloadType("UEFI-BOOT-APP"), - digest=DigestValue( + digest=DigestValue.parse( "sha256:ac17457772666351154a5952e3b87851a6398da2afcf3a38bedfc0925760bb0e" ), ), From adededfa5336df6c897dcf727bd941e77f16adf5 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Fri, 23 Aug 2024 02:53:58 +0000 Subject: [PATCH 164/193] minor fix to firmware_package --- .../boot_control/_firmware_package.py | 40 ++++++++++++------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/src/otaclient/boot_control/_firmware_package.py b/src/otaclient/boot_control/_firmware_package.py index 0f3c9396a..431fc290a 100644 --- a/src/otaclient/boot_control/_firmware_package.py +++ b/src/otaclient/boot_control/_firmware_package.py @@ -46,7 +46,7 @@ import re from enum import Enum from pathlib import Path -from typing import List, Literal, Union +from typing import Any, List, Literal, Union import yaml from pydantic import BaseModel, BeforeValidator @@ -73,11 +73,18 @@ class DigestValue(BaseModel): digest: str @classmethod - def parse(cls, _in: str) -> DigestValue: - _in = _in.strip() - if not (ma := DIGEST_PATTERN.match(_in)): - raise ValueError(f"invalid digest value: {_in}") - return DigestValue(algorithm=ma.group("algorithm"), digest=ma.group("digest")) + def parse(cls, _in: str | DigestValue | Any) -> DigestValue: + if isinstance(_in, DigestValue): + return _in + + if isinstance(_in, str): + _in = _in.strip() + if not (ma := DIGEST_PATTERN.match(_in)): + raise ValueError(f"invalid digest value: {_in}") + return DigestValue( + algorithm=ma.group("algorithm"), digest=ma.group("digest") + ) + raise TypeError(f"expect instance of str or {cls}, get {type(_in)}") class NVIDIAFirmwareCompat(BaseModel): @@ -115,14 +122,19 @@ class PayloadFileLocation(BaseModel): location_path: Union[str, DigestValue] @classmethod - def parse(cls, _in: str) -> PayloadFileLocation: - if _in.startswith("file://"): - location_type = "file" - location_path = _in.replace("file://", "", 1) - else: - location_type = "blob" - location_path = DigestValue.parse(_in) - return cls(location_type=location_type, location_path=location_path) + def parse(cls, _in: str | PayloadFileLocation | Any) -> PayloadFileLocation: + if isinstance(_in, PayloadFileLocation): + return _in + + if isinstance(_in, str): + if _in.startswith("file://"): + location_type = "file" + location_path = _in.replace("file://", "", 1) + else: + location_type = "blob" + location_path = DigestValue.parse(_in) + return cls(location_type=location_type, location_path=location_path) + raise TypeError(f"expect instance of str of {cls}, get {type(_in)}") class FirmwarePackage(BaseModel): From 09c3b71fd22fb4382cfabd7029fde79bb1c19d2c Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Fri, 23 Aug 2024 02:57:02 +0000 Subject: [PATCH 165/193] test_firmware_package: add one more test case --- .../test_boot_control/test_firmware_package.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_otaclient/test_boot_control/test_firmware_package.py b/tests/test_otaclient/test_boot_control/test_firmware_package.py index 5302cc1a1..0a5884e32 100644 --- a/tests/test_otaclient/test_boot_control/test_firmware_package.py +++ b/tests/test_otaclient/test_boot_control/test_firmware_package.py @@ -133,6 +133,10 @@ def test_check_compat(_spec, _compat_str, _expected): file_location: file:///opt/ota/firmware/BOOTAA64.efi type: UEFI-BOOT-APP digest: "sha256:ac17457772666351154a5952e3b87851a6398da2afcf3a38bedfc0925760bb0e" + - payload_name: some_payload_in_blob_storage + file_location: "sha256:55f91b2fc9c397cc83ca7c23e627d09b542ae02381db3ca480e0242fca14e935" + type: UEFI-CAPSULE + digest: "sha256:55f91b2fc9c397cc83ca7c23e627d09b542ae02381db3ca480e0242fca14e935" """ EXAMPLE_FIRMWARE_MANIFEST_PARSED = FirmwareManifest( @@ -171,6 +175,16 @@ def test_check_compat(_spec, _compat_str, _expected): "sha256:ac17457772666351154a5952e3b87851a6398da2afcf3a38bedfc0925760bb0e" ), ), + FirmwarePackage( + payload_name="some_payload_in_blob_storage", + file_location=PayloadFileLocation.parse( + "sha256:55f91b2fc9c397cc83ca7c23e627d09b542ae02381db3ca480e0242fca14e935" + ), + type=PayloadType("UEFI-CAPSULE"), + digest=DigestValue.parse( + "sha256:55f91b2fc9c397cc83ca7c23e627d09b542ae02381db3ca480e0242fca14e935" + ), + ), ], ) From 77b600599ae7e296d87bb56589439ca6735f12e2 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Fri, 23 Aug 2024 03:02:05 +0000 Subject: [PATCH 166/193] jetson-uefi: update according to firmware_package changes; fix update_l4tlauncher ver control --- src/otaclient/boot_control/_jetson_uefi.py | 26 +++++++++++++++------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index b444ed8a8..b93a1a89a 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -34,7 +34,6 @@ from otaclient.app import errors as ota_errors from otaclient.app.configs import config as cfg from otaclient.boot_control._firmware_package import ( - DigestValue, FirmwareManifest, FirmwareUpdateRequest, PayloadType, @@ -455,8 +454,10 @@ def _prepare_fwupdate_capsule(self) -> bool: # NOTE: currently we only support payload indicated by file path. capsule_flocation = capsule_payload.file_location - assert capsule_flocation.location_type == "file" capsule_fpath = capsule_flocation.location_path + assert capsule_flocation.location_type == "file" and isinstance( + capsule_fpath, str + ) try: shutil.copy( @@ -476,10 +477,10 @@ def _prepare_fwupdate_capsule(self) -> bool: def _update_l4tlauncher(self) -> bool: """update L4TLauncher with OTA image's one.""" - for capsule_payload in self.firmware_manifest.get_firmware_packages( + for _payload in self.firmware_manifest.get_firmware_packages( self.firmware_update_request ): - if capsule_payload.type != PayloadType.UEFI_BOOT_APP: + if _payload.type != PayloadType.UEFI_BOOT_APP: continue logger.warning( @@ -487,18 +488,27 @@ def _update_l4tlauncher(self) -> bool: ) # NOTE: currently we only support payload indicated by file path. - bootapp_fpath = capsule_payload.file_location - assert not isinstance(bootapp_fpath, DigestValue) + bootapp_flocation = _payload.file_location + bootapp_fpath, bootapp_ftype = ( + bootapp_flocation.location_path, + bootapp_flocation.location_type, + ) + assert bootapp_ftype == "file" and isinstance(bootapp_fpath, str) - # new BOOTAA64.efi is located at /opt/ota_package/BOOTAA64.efi + # new BOOTAA64.efi is located at OTA image ota_image_bootaa64 = replace_root(bootapp_fpath, "/", self.standby_slot_mp) + new_l4tlauncher_ver_ctrl = L4TLauncherBSPVersionControl( + bsp_ver=self.firmware_package_bsp_ver, + sha256_digest=_payload.digest.digest, + ) try: shutil.copy(self.bootaa64_at_esp, self.bootaa64_at_esp_bak) shutil.copy(ota_image_bootaa64, self.bootaa64_at_esp) os.sync() # ensure the boot application is written to the disk write_str_to_file_sync( - self.l4tlauncher_ver_fpath, self.firmware_package_bsp_ver.dump() + self.l4tlauncher_ver_fpath, + new_l4tlauncher_ver_ctrl.dump(), ) return True except Exception as e: From 67b046a1add53236009f96e1d8fe349c90d78e85 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Fri, 23 Aug 2024 03:05:49 +0000 Subject: [PATCH 167/193] minor fix to test_jetson_uefi --- tests/test_otaclient/test_boot_control/test_jetson_uefi.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_otaclient/test_boot_control/test_jetson_uefi.py b/tests/test_otaclient/test_boot_control/test_jetson_uefi.py index 9cbef68f5..28b5c712d 100644 --- a/tests/test_otaclient/test_boot_control/test_jetson_uefi.py +++ b/tests/test_otaclient/test_boot_control/test_jetson_uefi.py @@ -15,9 +15,10 @@ from __future__ import annotations +from pathlib import Path + import pytest from pytest_mock import MockerFixture -from typing_extensions import LiteralString from otaclient.boot_control import _jetson_uefi from otaclient.boot_control._jetson_common import BSPVersion @@ -241,7 +242,7 @@ def test__l4tlauncher_version_control( current_bsp_ver, expected_bsp_ver, expected_ver_ctrl, - tmp_path, + tmp_path: Path, mocker: MockerFixture, ): ver_control_f = tmp_path / "l4tlauncher_ver_control" From e5a778b1a30a981c664d704deb60301606d5c9b6 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Fri, 23 Aug 2024 05:31:34 +0000 Subject: [PATCH 168/193] jetson-cboot: fix bup update --- src/otaclient/boot_control/_jetson_cboot.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/otaclient/boot_control/_jetson_cboot.py b/src/otaclient/boot_control/_jetson_cboot.py index 62bf400df..9a56f6f24 100644 --- a/src/otaclient/boot_control/_jetson_cboot.py +++ b/src/otaclient/boot_control/_jetson_cboot.py @@ -35,6 +35,7 @@ from otaclient_api.v2 import types as api_types from otaclient_common import replace_root from otaclient_common.common import subprocess_run_wrapper +from otaclient_common.typing import StrOrPath from ._common import CMDHelperFuncs, OTAStatusFilesControl, SlotMountHelper from ._jetson_common import ( @@ -57,6 +58,12 @@ logger = logging.getLogger(__name__) +MAXIMUM_SUPPORTED_BSP_VERSION_EXCLUDE = BSPVersion(34, 0, 0) +"""After R34, cboot is replaced by UEFI. + +Also, cboot firmware update with nvupdate engine is supported up to this version(exclude). +""" + class JetsonCBootContrlError(Exception): """Exception types for covering jetson-cboot related errors.""" @@ -159,12 +166,14 @@ def _nv_update_engine_unified_ab(cls, payload: Path | str): def __init__( self, + standby_slot_mp: StrOrPath, *, tnspec: str, firmware_update_request: FirmwareUpdateRequest, firmware_manifest: FirmwareManifest, unify_ab: bool, ) -> None: + self._standby_slot_mp = standby_slot_mp self._tnspec = tnspec self._firmware_update_request = firmware_update_request self._firmware_manifest = firmware_manifest @@ -201,9 +210,15 @@ def firmware_update(self) -> bool: # NOTE: currently we only support payload indicated by file path. bup_flocation = update_payload.file_location - assert bup_flocation.location_type == "file" bup_fpath = bup_flocation.location_path + assert bup_flocation.location_type == "file" and isinstance(bup_fpath, str) + # bup is located at the OTA image + bup_fpath = replace_root( + bup_fpath, + "/", + self._standby_slot_mp, + ) if not Path(bup_fpath).is_file(): logger.warning(f"{bup_fpath=} doesn't exist! skip...") continue @@ -489,6 +504,7 @@ def _firmware_update(self) -> bool | None: logger.info(f"firmware update package BSP version: {fw_update_bsp_ver}") firmware_updater = NVUpdateEngine( + standby_slot_mp=self._mp_control.standby_slot_mount_point, tnspec=tnspec, firmware_update_request=firmware_update_request, firmware_manifest=firmware_manifest, From 9e7a1a90c37ab97e2812cc97358702c47e76048b Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Fri, 23 Aug 2024 05:35:12 +0000 Subject: [PATCH 169/193] jetson-cboot: prevent firmware update if firmware_package BSP ver >= R34 --- src/otaclient/boot_control/_jetson_cboot.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_cboot.py b/src/otaclient/boot_control/_jetson_cboot.py index 9a56f6f24..66b5599ea 100644 --- a/src/otaclient/boot_control/_jetson_cboot.py +++ b/src/otaclient/boot_control/_jetson_cboot.py @@ -185,6 +185,18 @@ def firmware_update(self) -> bool: Returns: True if firmware update is performed, False if there is no firmware update. """ + firmware_bsp_ver = BSPVersion.parse( + self._firmware_manifest.firmware_spec.bsp_version + ) + if firmware_bsp_ver >= MAXIMUM_SUPPORTED_BSP_VERSION_EXCLUDE: + logger.warning( + f"firmware package has {firmware_bsp_ver}, " + f"which newer or equal to {MAXIMUM_SUPPORTED_BSP_VERSION_EXCLUDE}. " + f"firmware update to {MAXIMUM_SUPPORTED_BSP_VERSION_EXCLUDE} or newer is NOT supported, " + "skip firmware update" + ) + return False + # check firmware compatibility, this is to prevent failed firmware update beforehand. if not self._firmware_manifest.check_compat(self._tnspec): _err_msg = ( @@ -259,10 +271,11 @@ def __init__(self): logger.info(f"{rootfs_bsp_version=}") # ------ sanity check, jetson-cboot is not used after BSP R34 ------ # - if rootfs_bsp_version >= (34, 0, 0): + if rootfs_bsp_version >= MAXIMUM_SUPPORTED_BSP_VERSION_EXCLUDE: _err_msg = ( - f"jetson-cboot only supports BSP version < R34, but get {rootfs_bsp_version=}. " - "Please use jetson-uefi bootloader type for device with BSP >= R34." + f"jetson-cboot only supports BSP version < {MAXIMUM_SUPPORTED_BSP_VERSION_EXCLUDE}," + f" but get {rootfs_bsp_version=}. " + f"Please use jetson-uefi control for device with BSP >= {MAXIMUM_SUPPORTED_BSP_VERSION_EXCLUDE}." ) logger.error(_err_msg) raise JetsonCBootContrlError(_err_msg) From 065f501e10606874ae85276e8a98502c36e31326 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Fri, 23 Aug 2024 05:37:35 +0000 Subject: [PATCH 170/193] jetson-uefi: prevent firmware update to BSP version < R35.2 --- src/otaclient/boot_control/_jetson_uefi.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index b93a1a89a..06d495ec7 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -546,6 +546,17 @@ def firmware_update(self) -> bool: logger.error(_err_msg) return False + if ( + self.firmware_package_bsp_ver + < FIRMWARE_UPDATE_MINIMUM_SUPPORTED_BSP_VERSION + ): + _err_msg = ( + f"update firmware to BSP version < {FIRMWARE_UPDATE_MINIMUM_SUPPORTED_BSP_VERSION} " + "is not supported, abort" + ) + logger.warning(_err_msg) + return False + # check BSP version, NVIDIA Jetson device with R34 or newer doesn't allow firmware downgrade. if ( self.standby_slot_bsp_ver From e2ad3a6ac1573e73420485323d55defdee2b93ef Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Fri, 23 Aug 2024 05:49:10 +0000 Subject: [PATCH 171/193] common: implement file_digest, use file_digest to implement file_sha256 --- src/otaclient_common/common.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/otaclient_common/common.py b/src/otaclient_common/common.py index 49faf21c8..20bbd446f 100644 --- a/src/otaclient_common/common.py +++ b/src/otaclient_common/common.py @@ -20,12 +20,13 @@ from __future__ import annotations +import hashlib import logging import os import shutil import subprocess import time -from hashlib import sha256 +from functools import partial from pathlib import Path from typing import Optional, Union from urllib.parse import urljoin @@ -33,6 +34,7 @@ import requests from otaclient_common.linux import subprocess_run_wrapper +from otaclient_common.typing import StrOrPath logger = logging.getLogger(__name__) @@ -52,17 +54,17 @@ def wait_with_backoff(_retry_cnt: int, *, _backoff_factor: float, _backoff_max: # file verification -def file_sha256( - filename: Union[Path, str], *, chunk_size: int = 1 * 1024 * 1024 -) -> str: - with open(filename, "rb") as f: - m = sha256() - while True: - d = f.read(chunk_size) - if len(d) == 0: - break - m.update(d) - return m.hexdigest() +def file_digest(fpath: StrOrPath, *, algorithm: str, chunk_size: int = 1 * 1024 * 1024): + """Generate file digest with .""" + with open(fpath, "rb") as f: + hasher = hashlib.new(algorithm) + while d := f.read(chunk_size): + hasher.update(d) + return hasher.hexdigest() + + +file_sha256 = partial(file_digest, algorithm="sha256") +file_sha256.__doc__ = "Generate file digest with sha256." def verify_file(fpath: Path, fhash: str, fsize: Optional[int]) -> bool: From 0e329c9f3a152437be674118ad3a738e1d5dc3fe Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Fri, 23 Aug 2024 05:56:25 +0000 Subject: [PATCH 172/193] jetson-uefi: validate payload before firmware update --- src/otaclient/boot_control/_jetson_uefi.py | 32 ++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index 06d495ec7..194c18bc9 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -41,7 +41,12 @@ ) from otaclient_api.v2 import types as api_types from otaclient_common import replace_root -from otaclient_common.common import file_sha256, subprocess_call, write_str_to_file_sync +from otaclient_common.common import ( + file_digest, + file_sha256, + subprocess_call, + write_str_to_file_sync, +) from otaclient_common.typing import StrOrPath from ._common import CMDHelperFuncs, OTAStatusFilesControl, SlotMountHelper @@ -459,9 +464,20 @@ def _prepare_fwupdate_capsule(self) -> bool: capsule_fpath, str ) + capsule_fpath = Path(replace_root(capsule_fpath, "/", self.standby_slot_mp)) + capsule_digest_alg, capsule_digest_value = ( + capsule_payload.digest.algorithm, + capsule_payload.digest.digest, + ) + try: + assert ( + _digest := file_digest(capsule_fpath, algorithm=capsule_digest_alg) + == capsule_digest_value + ), f"{capsule_digest_alg} validation failed, expect {capsule_digest_value}, get {_digest}" + shutil.copy( - src=replace_root(capsule_fpath, "/", self.standby_slot_mp), + src=capsule_fpath, dst=capsule_dir_at_esp / capsule_payload.payload_name, ) firmware_package_configured = True @@ -497,11 +513,23 @@ def _update_l4tlauncher(self) -> bool: # new BOOTAA64.efi is located at OTA image ota_image_bootaa64 = replace_root(bootapp_fpath, "/", self.standby_slot_mp) + payload_digest_alg, payload_digest_value = ( + _payload.digest.algorithm, + _payload.digest.digest, + ) + new_l4tlauncher_ver_ctrl = L4TLauncherBSPVersionControl( bsp_ver=self.firmware_package_bsp_ver, sha256_digest=_payload.digest.digest, ) try: + assert ( + _digest := file_digest( + ota_image_bootaa64, algorithm=payload_digest_alg + ) + == payload_digest_value + ), f"{payload_digest_alg} validation failed, expect {payload_digest_value}, get {_digest}" + shutil.copy(self.bootaa64_at_esp, self.bootaa64_at_esp_bak) shutil.copy(ota_image_bootaa64, self.bootaa64_at_esp) os.sync() # ensure the boot application is written to the disk From 068d550f591903db32a77ea77f459bf3ee53dca8 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Fri, 23 Aug 2024 06:00:21 +0000 Subject: [PATCH 173/193] jetson-cboot: validate payload before firmware update --- src/otaclient/boot_control/_jetson_cboot.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/otaclient/boot_control/_jetson_cboot.py b/src/otaclient/boot_control/_jetson_cboot.py index 66b5599ea..934d3e9c2 100644 --- a/src/otaclient/boot_control/_jetson_cboot.py +++ b/src/otaclient/boot_control/_jetson_cboot.py @@ -34,7 +34,7 @@ ) from otaclient_api.v2 import types as api_types from otaclient_common import replace_root -from otaclient_common.common import subprocess_run_wrapper +from otaclient_common.common import file_digest, subprocess_run_wrapper from otaclient_common.typing import StrOrPath from ._common import CMDHelperFuncs, OTAStatusFilesControl, SlotMountHelper @@ -225,6 +225,11 @@ def firmware_update(self) -> bool: bup_fpath = bup_flocation.location_path assert bup_flocation.location_type == "file" and isinstance(bup_fpath, str) + payload_digest_alg, payload_digest_value = ( + update_payload.digest.algorithm, + update_payload.digest.digest, + ) + # bup is located at the OTA image bup_fpath = replace_root( bup_fpath, @@ -235,6 +240,15 @@ def firmware_update(self) -> bool: logger.warning(f"{bup_fpath=} doesn't exist! skip...") continue + _digest = file_digest(bup_fpath, algorithm=payload_digest_alg) + if _digest != payload_digest_value: + logger.warning( + f"{payload_digest_alg} validation failed, " + f"expect {payload_digest_value}, get {_digest}, " + f"skip apply {update_payload.payload_name}" + ) + continue + logger.warning(f"apply BUP {bup_fpath} to standby slot ...") update_execute_func(bup_fpath) firmware_update_executed = True From 6a9d11f8cf842969be8787d8a8eafad09c5ed04f Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Fri, 23 Aug 2024 06:18:34 +0000 Subject: [PATCH 174/193] minor fix --- src/otaclient/boot_control/_jetson_uefi.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index 194c18bc9..c1122016e 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -471,9 +471,9 @@ def _prepare_fwupdate_capsule(self) -> bool: ) try: + _digest = file_digest(capsule_fpath, algorithm=capsule_digest_alg) assert ( - _digest := file_digest(capsule_fpath, algorithm=capsule_digest_alg) - == capsule_digest_value + _digest == capsule_digest_value ), f"{capsule_digest_alg} validation failed, expect {capsule_digest_value}, get {_digest}" shutil.copy( @@ -523,11 +523,9 @@ def _update_l4tlauncher(self) -> bool: sha256_digest=_payload.digest.digest, ) try: + _digest = file_digest(ota_image_bootaa64, algorithm=payload_digest_alg) assert ( - _digest := file_digest( - ota_image_bootaa64, algorithm=payload_digest_alg - ) - == payload_digest_value + _digest == payload_digest_value ), f"{payload_digest_alg} validation failed, expect {payload_digest_value}, get {_digest}" shutil.copy(self.bootaa64_at_esp, self.bootaa64_at_esp_bak) From 56f18b3f44074ecfa68a6616ed773dbc5dec3ee6 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Fri, 23 Aug 2024 08:05:54 +0000 Subject: [PATCH 175/193] minor cleanup --- src/otaclient/boot_control/_jetson_common.py | 1 - src/otaclient/boot_control/_jetson_uefi.py | 12 ++++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_common.py b/src/otaclient/boot_control/_jetson_common.py index d7860c4f4..5acf679e0 100644 --- a/src/otaclient/boot_control/_jetson_common.py +++ b/src/otaclient/boot_control/_jetson_common.py @@ -249,7 +249,6 @@ def __init__( # NOTE: only check the standby slot's firmware BSP version info from file, # for current slot, always trust the value from nvbootctrl. self._version.set_by_slot(current_slot, current_slot_bsp_ver) - logger.info(f"loading firmware bsp version completed: {self._version}") def write_to_file(self, fw_bsp_fpath: StrOrPath) -> None: """Write firmware_bsp_version from memory to firmware_bsp_version file.""" diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index c1122016e..760d9ae0f 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -653,7 +653,7 @@ def firmware_update(self) -> bool: if firmware_update_triggerred: logger.warning( - "firmware update package prepare finished" + "firmware update package prepare finished. " f"will update firmware to {self.firmware_package_bsp_ver} in next reboot" ) return firmware_update_triggerred @@ -705,7 +705,6 @@ def __init__(self): _err_msg = f"failed to detect BSP version: {e!r}" logger.error(_err_msg) raise JetsonUEFIBootControlError(_err_msg) - logger.info(f"{fw_bsp_version=}") # check current slot rootfs BSP version try: @@ -937,8 +936,13 @@ def post_update(self) -> Generator[None, None, None]: # ------ update extlinux.conf ------ # update_standby_slot_extlinux_cfg( active_slot_extlinux_fpath=Path(boot_cfg.EXTLINUX_FILE), - standby_slot_extlinux_fpath=self._mp_control.standby_slot_mount_point - / Path(boot_cfg.EXTLINUX_FILE).relative_to("/"), + standby_slot_extlinux_fpath=Path( + replace_root( + boot_cfg.EXTLINUX_FILE, + "/", + self._mp_control.standby_slot_mount_point, + ) + ), standby_slot_partuuid=self._uefi_control.standby_rootfs_dev_partuuid, ) From 56260188b2e211ee4cc90cc6ac43b5be07d3c691 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Sat, 31 Aug 2024 14:08:29 +0000 Subject: [PATCH 176/193] updates according to real ECU test: the current slot's UEFI framework does the firmware update --- src/otaclient/boot_control/_jetson_uefi.py | 25 ++++++++++------------ 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index 92fe72309..6865c37f2 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -408,7 +408,6 @@ def __init__( firmware_manifest (FirmwareBSPVersionControl) """ self.current_slot_bsp_ver = fw_bsp_ver_control.current_slot_bsp_ver - self.standby_slot_bsp_ver = fw_bsp_ver_control.standby_slot_bsp_ver self.nvbootctrl_conf = nvbootctrl_conf self.firmware_update_request = firmware_update_request @@ -554,20 +553,20 @@ def firmware_update(self) -> bool: 2. the firmware_manifest is presented and valid in the OTA image. 3. the firmware package is compatible with the device. 4. firmware specified in firmware_update_request is valid. - 5. the standby slot's firmware is older than the OTA image's one. + 5. the firmware package's BSP version is the same or newer than current slot's firmware BSP version. + + NOTE: firmware CANNOT be downgraded with UEFI Capsule update mechanism. + NOTE(20240826): the firmware update is executed by the CURRENT slot's UEFI framework, so + we need to check the firmware package's version against CURRENT slot's firmware BSP version. Returns: True if firmware update is configured, False if there is no firmware update. """ # sanity check, UEFI Capsule firmware update is only supported after R35.2. - if ( - self.standby_slot_bsp_ver - and self.standby_slot_bsp_ver - < FIRMWARE_UPDATE_MINIMUM_SUPPORTED_BSP_VERSION - ): + if self.current_slot_bsp_ver < FIRMWARE_UPDATE_MINIMUM_SUPPORTED_BSP_VERSION: _err_msg = ( f"UEFI Capsule firmware update is introduced since {FIRMWARE_UPDATE_MINIMUM_SUPPORTED_BSP_VERSION}, " - f"but get {self.standby_slot_bsp_ver}, abort" + f"but get {self.current_slot_bsp_ver=}, abort" ) logger.error(_err_msg) return False @@ -584,14 +583,12 @@ def firmware_update(self) -> bool: return False # check BSP version, NVIDIA Jetson device with R34 or newer doesn't allow firmware downgrade. - if ( - self.standby_slot_bsp_ver - and self.standby_slot_bsp_ver > self.firmware_package_bsp_ver - ): + if self.current_slot_bsp_ver > self.firmware_package_bsp_ver: logger.info( ( - "standby slot has newer ver of firmware, skip firmware update: " - f"{self.standby_slot_bsp_ver=}, {self.firmware_package_bsp_ver=}" + "current slot's firmware BSP version is newer than firmware package's," + " skip firmware update: " + f"{self.current_slot_bsp_ver=}, {self.firmware_package_bsp_ver=}" ) ) return False From 9c5070bf09e92bf89890e65d380dee237894e49e Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Sat, 31 Aug 2024 14:17:26 +0000 Subject: [PATCH 177/193] jetson-common: SlotID now also takes 'A' or 'B' --- src/otaclient/boot_control/_jetson_common.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_common.py b/src/otaclient/boot_control/_jetson_common.py index 5acf679e0..ba3fd5791 100644 --- a/src/otaclient/boot_control/_jetson_common.py +++ b/src/otaclient/boot_control/_jetson_common.py @@ -42,19 +42,22 @@ class SlotID(str): """slot_id for A/B slots. - On NVIDIA Jetson device, slot_a has slot_id=0, slot_b has slot_id=1. - For slot_a, the slot partition name suffix is "" or "_a". - For slot_b, the slot partition name suffix is "_b". + slot_a has slot_id=0, slot_b has slot_id=1. """ VALID_SLOTS = ["0", "1"] + VALID_SLOTS_CHAR = ["A", "B"] def __new__(cls, _in: str | Self) -> Self: if isinstance(_in, cls): return _in if _in in cls.VALID_SLOTS: return str.__new__(cls, _in) - raise ValueError(f"{_in=} is not valid slot num, should be '0' or '1'.") + if _in in cls.VALID_SLOTS_CHAR: + return str.__new__(cls, "0") if _in == "A" else str.__new__(cls, "1") + raise ValueError( + f"{_in=} is not valid slot num, should be '0'('A') or '1'('B')." + ) SLOT_A, SLOT_B = SlotID("0"), SlotID("1") From 45745e2bd0541b554cdf3436fbbbeba677f5f402 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Sat, 31 Aug 2024 14:20:29 +0000 Subject: [PATCH 178/193] jetson-common: BSPVersion.parse now only raises ValueError on invalid input. --- src/otaclient/boot_control/_jetson_common.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_common.py b/src/otaclient/boot_control/_jetson_common.py index ba3fd5791..5b92e9ddb 100644 --- a/src/otaclient/boot_control/_jetson_common.py +++ b/src/otaclient/boot_control/_jetson_common.py @@ -82,12 +82,17 @@ class BSPVersion(NamedTuple): @classmethod def parse(cls, _in: str | BSPVersion | Any) -> Self: - """Parse BSP version string into BSPVersion.""" + """Parse BSP version string into BSPVersion. + + Raises: + ValueError on invalid input. + """ if isinstance(_in, cls): return _in if isinstance(_in, str): ma = BSP_VERSION_STR_PA.match(_in) - assert ma, f"not a valid bsp version string: {_in}" + if not ma: + raise ValueError(f"not a valid bsp version string: {_in}") major_ver, major_rev, minor_rev = ( ma.group("major_ver"), From a01d3b263702502a3a031e2b30419d3ec96f29f5 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Sat, 31 Aug 2024 14:29:02 +0000 Subject: [PATCH 179/193] jetson-common: minor update --- src/otaclient/boot_control/_jetson_common.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_common.py b/src/otaclient/boot_control/_jetson_common.py index 5b92e9ddb..5f67bb586 100644 --- a/src/otaclient/boot_control/_jetson_common.py +++ b/src/otaclient/boot_control/_jetson_common.py @@ -167,7 +167,7 @@ def _nvbootctrl( _cmd: str, _slot_id: Optional[SlotID] = None, *, - check_output, + check_output: bool, target: Optional[NVBootctrlTarget] = None, ) -> Any: cmd = [cls.NVBOOTCTRL] @@ -190,7 +190,6 @@ def get_current_slot(cls, *, target: Optional[NVBootctrlTarget] = None) -> SlotI """Prints currently running SLOT.""" cmd = "get-current-slot" res = cls._nvbootctrl(cmd, check_output=True, target=target) - assert isinstance(res, str), f"invalid output from get-current-slot: {res}" return SlotID(res.strip()) @classmethod From 4cd188cb6157e06819b8310d6938acf66c3ca56b Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Sat, 31 Aug 2024 14:31:32 +0000 Subject: [PATCH 180/193] jetson-uefi.NVBootctrlJetsonUEFI: implement get_active_bootloader_slot --- src/otaclient/boot_control/_jetson_uefi.py | 23 ++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index 6865c37f2..178a257b7 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -128,6 +128,29 @@ def get_current_fw_bsp_version(cls) -> BSPVersion: raise ValueError(_err_msg) return bsp_ver + @classmethod + def get_active_bootloader_slot(cls) -> SlotID | None: + """Get the active bootloader slot. + + Returns: + SlotID of active bootloader slot, or None if failed to detect. + """ + _raw = cls.dump_slots_info() + pa = re.compile(r"Active bootloader slot:\s*(?P[AB])") + + if not (ma := pa.search(_raw)): + _err_msg = f"nvbootctrl report invalid active slot: \n{_raw=}" + logger.error(_err_msg) + return + + slot_id_char = ma.group("slot_id_char") + try: + return SlotID(slot_id_char) + except ValueError as e: + _err_msg = f"failed to detect active bootloader slot: {e!r}" + logger.error(_err_msg) + return + @classmethod def verify(cls) -> str: # pragma: no cover """Verify the bootloader and rootfs boot.""" From 7b268990870b11b780ea6113f3decb1d66825132 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Sat, 31 Aug 2024 14:40:34 +0000 Subject: [PATCH 181/193] jetson-uefi: in post update, check if the active boot slot is the same of current boot slot, if not, try to fix it before entering firmware update --- src/otaclient/boot_control/_jetson_uefi.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index 178a257b7..ab6c116f5 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -55,6 +55,7 @@ BSPVersion, FirmwareBSPVersionControl, NVBootctrlCommon, + SlotID, copy_standby_slot_boot_to_internal_emmc, detect_external_rootdev, detect_rootfs_bsp_version, @@ -766,6 +767,7 @@ def __init__(self): # ------ check A/B slots ------ # self.current_slot = current_slot = NVBootctrlJetsonUEFI.get_current_slot() self.standby_slot = standby_slot = NVBootctrlJetsonUEFI.get_standby_slot() + self.active_bootloader_slot = NVBootctrlJetsonUEFI.get_active_bootloader_slot() logger.info(f"{current_slot=}, {standby_slot=}") # ------ detect rootfs_dev and parent_dev ------ # @@ -984,6 +986,26 @@ def post_update(self) -> Generator[None, None, None]: ) # ------ firmware update & switch boot to standby ------ # + if ( + self._uefi_control.active_bootloader_slot + and self._uefi_control.current_slot + != self._uefi_control.active_bootloader_slot + ): + _err_msg = ( + "warning, active bootloader slot is mismatched with current slot: " + f"current_slot={self._uefi_control.current_slot}, " + f"active_bootloader_slot={self._uefi_control.active_bootloader_slot}. " + "this migth indicate an previous interrupted OTA, and this mismatch " + "will cancel the firmware update if configured, " + "correct this mismatch ..." + ) + NVBootctrlJetsonUEFI.set_active_boot_slot( + self._uefi_control.current_slot + ) + self._uefi_control.active_bootloader_slot = ( + NVBootctrlJetsonUEFI.get_active_bootloader_slot() + ) + firmware_update_triggered = self._firmware_update() # NOTE: manual switch boot will cancel the scheduled firmware update! if not firmware_update_triggered: From 15bf4d8dcb96b33b9fdde289741ad8d5037f7ca4 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Sat, 31 Aug 2024 14:41:47 +0000 Subject: [PATCH 182/193] jetson-uefi: get_current_fw_bsp_version now raises JetsonUEFIBootControlError only --- src/otaclient/boot_control/_jetson_uefi.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index ab6c116f5..4fcaf0015 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -112,21 +112,30 @@ class NVBootctrlJetsonUEFI(NVBootctrlCommon): @classmethod def get_current_fw_bsp_version(cls) -> BSPVersion: - """Get current boot chain's firmware BSP version with nvbootctrl.""" + """Get current boot chain's firmware BSP version with nvbootctrl. + + Raises: + JetsonUEFIBootControlError if failed to detect fw bsp version, + or the reported version doesn't make sense. + """ _raw = cls.dump_slots_info() pa = re.compile(r"Current version:\s*(?P[\.\d]+)") if not (ma := pa.search(_raw)): - _err_msg = "nvbootctrl failed to report BSP version" + _err_msg = f"nvbootctrl reports invalid BSP version: \n{_raw=}" logger.error(_err_msg) raise JetsonUEFIBootControlError(_err_msg) - bsp_ver_str = ma.group("bsp_ver") - bsp_ver = BSPVersion.parse(bsp_ver_str) + try: + bsp_ver_str = ma.group("bsp_ver") + bsp_ver = BSPVersion.parse(bsp_ver_str) + except ValueError as e: + raise JetsonUEFIBootControlError(f"invalid bsp version: {e!r}") from e + if bsp_ver.major_rev == 0: _err_msg = f"invalid BSP version: {bsp_ver_str}, this might indicate an incomplete flash!" logger.error(_err_msg) - raise ValueError(_err_msg) + raise JetsonUEFIBootControlError(_err_msg) return bsp_ver @classmethod @@ -999,6 +1008,7 @@ def post_update(self) -> Generator[None, None, None]: "will cancel the firmware update if configured, " "correct this mismatch ..." ) + logger.warning(_err_msg) NVBootctrlJetsonUEFI.set_active_boot_slot( self._uefi_control.current_slot ) From 832880387bace3ffe7b5d9eb68f563552a8dc27d Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Sat, 31 Aug 2024 14:47:16 +0000 Subject: [PATCH 183/193] test jetson-uefi: add test_get_active_bootloader_slot --- .../test_boot_control/test_jetson_uefi.py | 43 ++++++++++++++++++- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/tests/test_otaclient/test_boot_control/test_jetson_uefi.py b/tests/test_otaclient/test_boot_control/test_jetson_uefi.py index 28b5c712d..d63bae2d9 100644 --- a/tests/test_otaclient/test_boot_control/test_jetson_uefi.py +++ b/tests/test_otaclient/test_boot_control/test_jetson_uefi.py @@ -21,7 +21,7 @@ from pytest_mock import MockerFixture from otaclient.boot_control import _jetson_uefi -from otaclient.boot_control._jetson_common import BSPVersion +from otaclient.boot_control._jetson_common import BSPVersion, SlotID from otaclient.boot_control._jetson_uefi import ( JetsonUEFIBootControlError, L4TLauncherBSPVersionControl, @@ -76,7 +76,7 @@ class TestNVBootctrlJetsonUEFI: ), ), ) - def get_current_fw_bsp_version( + def test_get_current_fw_bsp_version( self, _input: str, expected: BSPVersion | Exception, mocker: MockerFixture ): mocker.patch.object( @@ -91,6 +91,45 @@ def get_current_fw_bsp_version( else: assert NVBootctrlJetsonUEFI.get_current_fw_bsp_version() == expected + @pytest.mark.parametrize( + "_input, expected", + ( + ( + """\ +Current version: 35.4.1 +Capsule update status: 1 +Current bootloader slot: A +Active bootloader slot: A +num_slots: 2 +slot: 0, status: normal +slot: 1, status: normal + """, + SlotID("0"), + ), + ( + """\ +Current version: 35.4.1 +Capsule update status: 1 +Current bootloader slot: A +Active bootloader slot: B +num_slots: 2 +slot: 0, status: normal +slot: 1, status: normal + """, + SlotID("1"), + ), + ), + ) + def test_get_active_bootloader_slot( + self, _input: str, expected: SlotID, mocker: MockerFixture + ): + mocker.patch.object( + NVBootctrlJetsonUEFI, + "dump_slots_info", + mocker.MagicMock(return_value=_input), + ) + assert NVBootctrlJetsonUEFI.get_active_bootloader_slot() == expected + @pytest.mark.parametrize( "all_esp_devs, boot_parent_devpath, expected", From 54a8fac73d3f43816db1045bf20342e86ac0d334 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Sat, 31 Aug 2024 14:53:14 +0000 Subject: [PATCH 184/193] jetson-common: define NVBootctrlExecError, defined API now raises this error instead --- src/otaclient/boot_control/_jetson_common.py | 48 +++++++++++++++++--- 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_common.py b/src/otaclient/boot_control/_jetson_common.py index 5f67bb586..196f0a349 100644 --- a/src/otaclient/boot_control/_jetson_common.py +++ b/src/otaclient/boot_control/_jetson_common.py @@ -143,6 +143,10 @@ def get_by_slot(self, slot_id: SlotID) -> BSPVersion | None: NVBootctrlTarget = Literal["bootloader", "rootfs"] +class NVBootctrlExecError(Exception): + """Raised when nvbootctrl command execution failed.""" + + class NVBootctrlCommon: """Helper for calling nvbootctrl commands. @@ -170,6 +174,10 @@ def _nvbootctrl( check_output: bool, target: Optional[NVBootctrlTarget] = None, ) -> Any: + """ + Raises: + CalledProcessError on return code not equal to 0. + """ cmd = [cls.NVBOOTCTRL] if target: cmd.extend(["-t", target]) @@ -187,16 +195,26 @@ def _nvbootctrl( @classmethod def get_current_slot(cls, *, target: Optional[NVBootctrlTarget] = None) -> SlotID: - """Prints currently running SLOT.""" + """Prints currently running SLOT. + + Raises: + NVBootctrlExecError on failed to get current slot. + """ cmd = "get-current-slot" - res = cls._nvbootctrl(cmd, check_output=True, target=target) - return SlotID(res.strip()) + try: + res = cls._nvbootctrl(cmd, check_output=True, target=target) + return SlotID(res.strip()) + except Exception as e: + raise NVBootctrlExecError from e @classmethod def get_standby_slot(cls, *, target: Optional[NVBootctrlTarget] = None) -> SlotID: """Prints standby SLOT. NOTE: this method is implemented with nvbootctrl get-current-slot. + + Raises: + NVBootctrlExecError on failed to get current slot. """ current_slot = cls.get_current_slot(target=target) return SLOT_FLIP[current_slot] @@ -205,15 +223,31 @@ def get_standby_slot(cls, *, target: Optional[NVBootctrlTarget] = None) -> SlotI def set_active_boot_slot( cls, slot_id: SlotID, *, target: Optional[NVBootctrlTarget] = None ) -> None: - """On next boot, load and execute SLOT.""" + """On next boot, load and execute SLOT. + + Raises: + NVBootctrlExecError on nvbootctrl call failed. + """ cmd = "set-active-boot-slot" - return cls._nvbootctrl(cmd, SlotID(slot_id), check_output=False, target=target) + try: + return cls._nvbootctrl( + cmd, SlotID(slot_id), check_output=False, target=target + ) + except subprocess.CalledProcessError as e: + raise NVBootctrlExecError from e @classmethod def dump_slots_info(cls, *, target: Optional[NVBootctrlTarget] = None) -> str: - """Prints info for slots.""" + """Prints info for slots. + + Raises: + NVBootctrlExecError on nvbootctrl call failed. + """ cmd = "dump-slots-info" - return cls._nvbootctrl(cmd, target=target, check_output=True) + try: + return cls._nvbootctrl(cmd, target=target, check_output=True) + except subprocess.CalledProcessError as e: + raise NVBootctrlExecError from e class FirmwareBSPVersionControl: From 8bb617e6097eefa3d0cf0a3673f46b5cdf20bf3e Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Sat, 31 Aug 2024 14:59:36 +0000 Subject: [PATCH 185/193] jetson-uefi: nvbootctrl API now raises NVBootctrlExecError instead --- src/otaclient/boot_control/_jetson_uefi.py | 24 +++++++++++++--------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index 4fcaf0015..19cb5d0f6 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -25,6 +25,7 @@ import os import re import shutil +import subprocess from pathlib import Path from typing import Any, ClassVar, Generator, Literal @@ -55,6 +56,7 @@ BSPVersion, FirmwareBSPVersionControl, NVBootctrlCommon, + NVBootctrlExecError, SlotID, copy_standby_slot_boot_to_internal_emmc, detect_external_rootdev, @@ -115,7 +117,7 @@ def get_current_fw_bsp_version(cls) -> BSPVersion: """Get current boot chain's firmware BSP version with nvbootctrl. Raises: - JetsonUEFIBootControlError if failed to detect fw bsp version, + NVBootctrlExecError if failed to detect fw bsp version, or the reported version doesn't make sense. """ _raw = cls.dump_slots_info() @@ -124,18 +126,18 @@ def get_current_fw_bsp_version(cls) -> BSPVersion: if not (ma := pa.search(_raw)): _err_msg = f"nvbootctrl reports invalid BSP version: \n{_raw=}" logger.error(_err_msg) - raise JetsonUEFIBootControlError(_err_msg) + raise NVBootctrlExecError(_err_msg) try: bsp_ver_str = ma.group("bsp_ver") bsp_ver = BSPVersion.parse(bsp_ver_str) except ValueError as e: - raise JetsonUEFIBootControlError(f"invalid bsp version: {e!r}") from e + raise NVBootctrlExecError(f"invalid bsp version: {e!r}") from e if bsp_ver.major_rev == 0: _err_msg = f"invalid BSP version: {bsp_ver_str}, this might indicate an incomplete flash!" logger.error(_err_msg) - raise JetsonUEFIBootControlError(_err_msg) + raise NVBootctrlExecError(_err_msg) return bsp_ver @classmethod @@ -162,9 +164,13 @@ def get_active_bootloader_slot(cls) -> SlotID | None: return @classmethod - def verify(cls) -> str: # pragma: no cover + def verify(cls) -> str | None: # pragma: no cover """Verify the bootloader and rootfs boot.""" - return cls._nvbootctrl("verify", check_output=True) + try: + return cls._nvbootctrl("verify", check_output=True) + except subprocess.CalledProcessError as e: + logger.warning(f"nvbootctrl verify call failed: {e!r}") + return EFIVARS_FSTYPE = "efivarfs" @@ -889,11 +895,9 @@ def _finalize_switching_boot(self) -> bool: NOTE that if capsule firmware update failed, we must be booted back to the previous slot, so actually we don't need to do any checks here. """ - try: - fw_update_verify = NVBootctrlJetsonUEFI.verify() + fw_update_verify = NVBootctrlJetsonUEFI.verify() + if fw_update_verify: logger.info(f"nvbootctrl verify: {fw_update_verify}") - except Exception as e: - logger.warning(f"nvbootctrl verify failed: {e!r}") return True def _firmware_update(self) -> bool: From a00994a6fb5da8a0eb8eaae33abd936c5578f5ab Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Sat, 31 Aug 2024 15:01:50 +0000 Subject: [PATCH 186/193] do not calculate coverage rate over APIs that only calls nvbootctrl directly --- src/otaclient/boot_control/_jetson_cboot.py | 6 +++--- src/otaclient/boot_control/_jetson_common.py | 16 +++++++++++----- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_cboot.py b/src/otaclient/boot_control/_jetson_cboot.py index 79f4f523b..4e7df795b 100644 --- a/src/otaclient/boot_control/_jetson_cboot.py +++ b/src/otaclient/boot_control/_jetson_cboot.py @@ -79,7 +79,7 @@ class _NVBootctrl(NVBootctrlCommon): @classmethod def mark_boot_successful( cls, slot_id: SlotID, *, target: Optional[NVBootctrlTarget] = None - ) -> None: + ) -> None: # pragma: no cover """Mark current slot as GOOD.""" cmd = "mark-boot-successful" cls._nvbootctrl(cmd, slot_id, check_output=False, target=target) @@ -87,13 +87,13 @@ def mark_boot_successful( @classmethod def set_slot_as_unbootable( cls, slot_id: SlotID, *, target: Optional[NVBootctrlTarget] = None - ) -> None: + ) -> None: # pragma: no cover """Mark SLOT as invalid.""" cmd = "set-slot-as-unbootable" return cls._nvbootctrl(cmd, SlotID(slot_id), check_output=False, target=target) @classmethod - def is_unified_enabled(cls) -> bool | None: + def is_unified_enabled(cls) -> bool | None: # pragma: no cover """Returns 0 only if unified a/b is enabled. NOTE: this command is available after BSP R32.6.1. diff --git a/src/otaclient/boot_control/_jetson_common.py b/src/otaclient/boot_control/_jetson_common.py index 196f0a349..43f22844f 100644 --- a/src/otaclient/boot_control/_jetson_common.py +++ b/src/otaclient/boot_control/_jetson_common.py @@ -173,7 +173,7 @@ def _nvbootctrl( *, check_output: bool, target: Optional[NVBootctrlTarget] = None, - ) -> Any: + ) -> Any: # pragma: no cover """ Raises: CalledProcessError on return code not equal to 0. @@ -194,7 +194,9 @@ def _nvbootctrl( return res.stdout.decode() @classmethod - def get_current_slot(cls, *, target: Optional[NVBootctrlTarget] = None) -> SlotID: + def get_current_slot( + cls, *, target: Optional[NVBootctrlTarget] = None + ) -> SlotID: # pragma: no cover """Prints currently running SLOT. Raises: @@ -208,7 +210,9 @@ def get_current_slot(cls, *, target: Optional[NVBootctrlTarget] = None) -> SlotI raise NVBootctrlExecError from e @classmethod - def get_standby_slot(cls, *, target: Optional[NVBootctrlTarget] = None) -> SlotID: + def get_standby_slot( + cls, *, target: Optional[NVBootctrlTarget] = None + ) -> SlotID: # pragma: no cover """Prints standby SLOT. NOTE: this method is implemented with nvbootctrl get-current-slot. @@ -222,7 +226,7 @@ def get_standby_slot(cls, *, target: Optional[NVBootctrlTarget] = None) -> SlotI @classmethod def set_active_boot_slot( cls, slot_id: SlotID, *, target: Optional[NVBootctrlTarget] = None - ) -> None: + ) -> None: # pragma: no cover """On next boot, load and execute SLOT. Raises: @@ -237,7 +241,9 @@ def set_active_boot_slot( raise NVBootctrlExecError from e @classmethod - def dump_slots_info(cls, *, target: Optional[NVBootctrlTarget] = None) -> str: + def dump_slots_info( + cls, *, target: Optional[NVBootctrlTarget] = None + ) -> str: # pragma: no cover """Prints info for slots. Raises: From 7d0fba5d70ce7e0f3185f93b6960e670b29bbed7 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Sat, 31 Aug 2024 15:04:10 +0000 Subject: [PATCH 187/193] jetson-cboot: for non-critical nvbootctrl call, do not raise exception --- src/otaclient/boot_control/_jetson_cboot.py | 44 ++++++++++++++------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_cboot.py b/src/otaclient/boot_control/_jetson_cboot.py index 4e7df795b..53669fb1c 100644 --- a/src/otaclient/boot_control/_jetson_cboot.py +++ b/src/otaclient/boot_control/_jetson_cboot.py @@ -69,7 +69,7 @@ class JetsonCBootContrlError(Exception): """Exception types for covering jetson-cboot related errors.""" -class _NVBootctrl(NVBootctrlCommon): +class NVBootctrlJetsonCBOOT(NVBootctrlCommon): """Helper for calling nvbootctrl commands. For BSP version < R34. @@ -82,7 +82,10 @@ def mark_boot_successful( ) -> None: # pragma: no cover """Mark current slot as GOOD.""" cmd = "mark-boot-successful" - cls._nvbootctrl(cmd, slot_id, check_output=False, target=target) + try: + cls._nvbootctrl(cmd, slot_id, check_output=False, target=target) + except subprocess.CalledProcessError as e: + logger.warning(f"nvbootctrl {cmd} call failed: {e!r}") @classmethod def set_slot_as_unbootable( @@ -90,7 +93,12 @@ def set_slot_as_unbootable( ) -> None: # pragma: no cover """Mark SLOT as invalid.""" cmd = "set-slot-as-unbootable" - return cls._nvbootctrl(cmd, SlotID(slot_id), check_output=False, target=target) + try: + return cls._nvbootctrl( + cmd, SlotID(slot_id), check_output=False, target=target + ) + except subprocess.CalledProcessError as e: + logger.warning(f"nvbootctrl {cmd} call failed: {e!r}") @classmethod def is_unified_enabled(cls) -> bool | None: # pragma: no cover @@ -297,7 +305,7 @@ def __init__(self): # ------ check if unified A/B is enabled ------ # # NOTE: mismatch rootfs BSP version and bootloader firmware BSP version # is NOT supported and MUST not occur. - if unified_ab_enabled := _NVBootctrl.is_unified_enabled(): + if unified_ab_enabled := NVBootctrlJetsonCBOOT.is_unified_enabled(): logger.info( "unified A/B is enabled, rootfs and bootloader will be switched together" ) @@ -305,17 +313,17 @@ def __init__(self): # ------ check A/B slots ------ # self.current_bootloader_slot = current_bootloader_slot = ( - _NVBootctrl.get_current_slot() + NVBootctrlJetsonCBOOT.get_current_slot() ) self.standby_bootloader_slot = standby_bootloader_slot = ( - _NVBootctrl.get_standby_slot() + NVBootctrlJetsonCBOOT.get_standby_slot() ) if not unified_ab_enabled: self.current_rootfs_slot = current_rootfs_slot = ( - _NVBootctrl.get_current_slot(target="rootfs") + NVBootctrlJetsonCBOOT.get_current_slot(target="rootfs") ) self.standby_rootfs_slot = standby_rootfs_slot = ( - _NVBootctrl.get_standby_slot(target="rootfs") + NVBootctrlJetsonCBOOT.get_standby_slot(target="rootfs") ) else: self.current_rootfs_slot = current_rootfs_slot = current_bootloader_slot @@ -364,10 +372,12 @@ def __init__(self): partition_id=SLOT_PAR_MAP[standby_rootfs_slot], ) logger.info(f"finished cboot control init: {current_rootfs_slot=}") - logger.info(f"nvbootctrl dump-slots-info: \n{_NVBootctrl.dump_slots_info()}") + logger.info( + f"nvbootctrl dump-slots-info: \n{NVBootctrlJetsonCBOOT.dump_slots_info()}" + ) if not unified_ab_enabled: logger.info( - f"nvbootctrl -t rootfs dump-slots-info: \n{_NVBootctrl.dump_slots_info(target='rootfs')}" + f"nvbootctrl -t rootfs dump-slots-info: \n{NVBootctrlJetsonCBOOT.dump_slots_info(target='rootfs')}" ) # load nvbootctrl config file @@ -392,7 +402,9 @@ def external_rootfs_enabled(self) -> bool: return self._external_rootfs def set_standby_rootfs_unbootable(self): - _NVBootctrl.set_slot_as_unbootable(self.standby_rootfs_slot, target="rootfs") + NVBootctrlJetsonCBOOT.set_slot_as_unbootable( + self.standby_rootfs_slot, target="rootfs" + ) def switch_boot_to_standby(self) -> None: # NOTE(20240412): we always try to align bootloader slot with rootfs. @@ -400,11 +412,11 @@ def switch_boot_to_standby(self) -> None: logger.info(f"switch boot to standby slot({target_slot})") if not self.unified_ab_enabled: - _NVBootctrl.set_active_boot_slot(target_slot, target="rootfs") + NVBootctrlJetsonCBOOT.set_active_boot_slot(target_slot, target="rootfs") # when unified_ab enabled, switching bootloader slot will also switch # the rootfs slot. - _NVBootctrl.set_active_boot_slot(target_slot) + NVBootctrlJetsonCBOOT.set_active_boot_slot(target_slot) class JetsonCBootControl(BootControllerProtocol): @@ -487,7 +499,9 @@ def _finalize_switching_boot(self) -> bool: # NOTE(20240417): rootfs slot is manually switched by set-active-boot-slot, # so we need to manually set the slot as success after first reboot. if not self._cboot_control.unified_ab_enabled: - _NVBootctrl.mark_boot_successful(current_rootfs_slot, target="rootfs") + NVBootctrlJetsonCBOOT.mark_boot_successful( + current_rootfs_slot, target="rootfs" + ) logger.info( f"nv_update_engine verify succeeded: \n{update_result.stdout.decode()}" @@ -632,7 +646,7 @@ def post_update(self) -> Generator[None, None, None]: # ------ prepare to reboot ------ # self._mp_control.umount_all(ignore_error=True) - logger.info(f"[post-update]: \n{_NVBootctrl.dump_slots_info()}") + logger.info(f"[post-update]: \n{NVBootctrlJetsonCBOOT.dump_slots_info()}") logger.info("post update finished, wait for reboot ...") yield # hand over control back to otaclient CMDHelperFuncs.reboot() From b94fbe577d62088d3cd259015e5dbc39d2051ffe Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Sat, 31 Aug 2024 15:12:32 +0000 Subject: [PATCH 188/193] minor fix test --- .../test_boot_control/test_jetson_common.py | 2 +- .../test_otaclient/test_boot_control/test_jetson_uefi.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/test_otaclient/test_boot_control/test_jetson_common.py b/tests/test_otaclient/test_boot_control/test_jetson_common.py index 8a749dd84..884447c53 100644 --- a/tests/test_otaclient/test_boot_control/test_jetson_common.py +++ b/tests/test_otaclient/test_boot_control/test_jetson_common.py @@ -70,7 +70,7 @@ class TestBSPVersion: ("32.6.1", BSPVersion(32, 6, 1), None), ("R35.4.1", BSPVersion(35, 4, 1), None), ("1.22.333", BSPVersion(1, 22, 333), None), - ("not_a_valid_bsp_ver", None, AssertionError), + ("not_a_valid_bsp_ver", None, ValueError), (123, None, ValueError), ), ) diff --git a/tests/test_otaclient/test_boot_control/test_jetson_uefi.py b/tests/test_otaclient/test_boot_control/test_jetson_uefi.py index d63bae2d9..2816bb6aa 100644 --- a/tests/test_otaclient/test_boot_control/test_jetson_uefi.py +++ b/tests/test_otaclient/test_boot_control/test_jetson_uefi.py @@ -21,7 +21,11 @@ from pytest_mock import MockerFixture from otaclient.boot_control import _jetson_uefi -from otaclient.boot_control._jetson_common import BSPVersion, SlotID +from otaclient.boot_control._jetson_common import ( + BSPVersion, + NVBootctrlExecError, + SlotID, +) from otaclient.boot_control._jetson_uefi import ( JetsonUEFIBootControlError, L4TLauncherBSPVersionControl, @@ -72,7 +76,7 @@ class TestNVBootctrlJetsonUEFI: slot: 0, status: normal slot: 1, status: normal """, - ValueError(), + NVBootctrlExecError(), ), ), ) From d56b6e0715e7cfe25d52113553d56b1b3351f70a Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Sat, 31 Aug 2024 15:15:00 +0000 Subject: [PATCH 189/193] minor update --- src/otaclient/boot_control/_jetson_common.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/otaclient/boot_control/_jetson_common.py b/src/otaclient/boot_control/_jetson_common.py index 43f22844f..cf13e06e6 100644 --- a/src/otaclient/boot_control/_jetson_common.py +++ b/src/otaclient/boot_control/_jetson_common.py @@ -440,7 +440,6 @@ def preserve_ota_config_files_to_standby( f"{active_slot_ota_dirpath} doesn't exist, skip preserve /boot/ota folder." ) return - # TODO: (20240411) reconsidering should we preserve /boot/ota? copytree_identical(active_slot_ota_dirpath, standby_slot_ota_dirpath) From 05d9596b2c81c04b0e17589d43e63e8c385645c4 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Sat, 31 Aug 2024 15:47:53 +0000 Subject: [PATCH 190/193] jetson-uefi: fine-grained uefi bootctrl startup exception handling --- src/otaclient/boot_control/_jetson_uefi.py | 30 +++++++++++++++------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index 19cb5d0f6..d3803f86d 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -780,18 +780,30 @@ def __init__(self): logger.info("unified A/B is enabled") # ------ check A/B slots ------ # - self.current_slot = current_slot = NVBootctrlJetsonUEFI.get_current_slot() - self.standby_slot = standby_slot = NVBootctrlJetsonUEFI.get_standby_slot() - self.active_bootloader_slot = NVBootctrlJetsonUEFI.get_active_bootloader_slot() + try: + self.current_slot = current_slot = NVBootctrlJetsonUEFI.get_current_slot() + self.standby_slot = standby_slot = NVBootctrlJetsonUEFI.get_standby_slot() + self.active_bootloader_slot = ( + NVBootctrlJetsonUEFI.get_active_bootloader_slot() + ) + except NVBootctrlExecError as e: + _err_msg = f"failed to get slot info: {e!r}" + logger.error(_err_msg) + raise JetsonUEFIBootControlError(_err_msg) from e logger.info(f"{current_slot=}, {standby_slot=}") # ------ detect rootfs_dev and parent_dev ------ # - self.curent_rootfs_devpath = current_rootfs_devpath = ( - CMDHelperFuncs.get_current_rootfs_dev() - ) - self.parent_devpath = parent_devpath = Path( - CMDHelperFuncs.get_parent_dev(current_rootfs_devpath) - ) + try: + self.curent_rootfs_devpath = current_rootfs_devpath = ( + CMDHelperFuncs.get_current_rootfs_dev() + ) + self.parent_devpath = parent_devpath = Path( + CMDHelperFuncs.get_parent_dev(current_rootfs_devpath) + ) + except Exception as e: + _err_msg = f"failed to detect rootfs dev: {e!r}" + logger.error(_err_msg) + raise JetsonUEFIBootControlError(_err_msg) from e # --- detect boot device --- # self.external_rootfs = detect_external_rootdev(parent_devpath) From 7504261ad91e201bd1dde0d1a55510b4a455c95b Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Sat, 31 Aug 2024 15:52:29 +0000 Subject: [PATCH 191/193] jetson-cboot: fine-grained cboot bootctrl startup exception handling --- src/otaclient/boot_control/_jetson_cboot.py | 76 +++++++++++++-------- 1 file changed, 46 insertions(+), 30 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_cboot.py b/src/otaclient/boot_control/_jetson_cboot.py index 53669fb1c..8ed7c026f 100644 --- a/src/otaclient/boot_control/_jetson_cboot.py +++ b/src/otaclient/boot_control/_jetson_cboot.py @@ -43,6 +43,7 @@ BSPVersion, FirmwareBSPVersionControl, NVBootctrlCommon, + NVBootctrlExecError, NVBootctrlTarget, SlotID, copy_standby_slot_boot_to_internal_emmc, @@ -312,22 +313,27 @@ def __init__(self): self.unified_ab_enabled = unified_ab_enabled # ------ check A/B slots ------ # - self.current_bootloader_slot = current_bootloader_slot = ( - NVBootctrlJetsonCBOOT.get_current_slot() - ) - self.standby_bootloader_slot = standby_bootloader_slot = ( - NVBootctrlJetsonCBOOT.get_standby_slot() - ) - if not unified_ab_enabled: - self.current_rootfs_slot = current_rootfs_slot = ( - NVBootctrlJetsonCBOOT.get_current_slot(target="rootfs") + try: + self.current_bootloader_slot = current_bootloader_slot = ( + NVBootctrlJetsonCBOOT.get_current_slot() ) - self.standby_rootfs_slot = standby_rootfs_slot = ( - NVBootctrlJetsonCBOOT.get_standby_slot(target="rootfs") + self.standby_bootloader_slot = standby_bootloader_slot = ( + NVBootctrlJetsonCBOOT.get_standby_slot() ) - else: - self.current_rootfs_slot = current_rootfs_slot = current_bootloader_slot - self.standby_rootfs_slot = standby_rootfs_slot = standby_bootloader_slot + if not unified_ab_enabled: + self.current_rootfs_slot = current_rootfs_slot = ( + NVBootctrlJetsonCBOOT.get_current_slot(target="rootfs") + ) + self.standby_rootfs_slot = standby_rootfs_slot = ( + NVBootctrlJetsonCBOOT.get_standby_slot(target="rootfs") + ) + else: + self.current_rootfs_slot = current_rootfs_slot = current_bootloader_slot + self.standby_rootfs_slot = standby_rootfs_slot = standby_bootloader_slot + except NVBootctrlExecError as e: + _err_msg = f"failed to detect slot info: {e!r}" + logger.error(_err_msg) + raise JetsonCBootContrlError(_err_msg) from e # check if rootfs slot and bootloader slot mismatches, this only happens # when unified_ab is not enabled. @@ -339,26 +345,36 @@ def __init__(self): logger.warning("this might indicates a failed previous firmware update") # ------ detect rootfs_dev and parent_dev ------ # - self.curent_rootfs_devpath = current_rootfs_devpath = ( - CMDHelperFuncs.get_current_rootfs_dev() - ) - self.parent_devpath = parent_devpath = Path( - CMDHelperFuncs.get_parent_dev(current_rootfs_devpath) - ) + try: + self.curent_rootfs_devpath = current_rootfs_devpath = ( + CMDHelperFuncs.get_current_rootfs_dev() + ) + self.parent_devpath = parent_devpath = Path( + CMDHelperFuncs.get_parent_dev(current_rootfs_devpath) + ) + except Exception as e: + _err_msg = f"failed to detect rootfs: {e!r}" + logger.error(_err_msg) + raise JetsonCBootContrlError(_err_msg) from e self._external_rootfs = detect_external_rootdev(parent_devpath) # rootfs partition - self.standby_rootfs_devpath = get_partition_devpath( - parent_devpath=parent_devpath, - partition_id=SLOT_PAR_MAP[standby_rootfs_slot], - ) - self.standby_rootfs_dev_partuuid = CMDHelperFuncs.get_attrs_by_dev( - "PARTUUID", self.standby_rootfs_devpath - ).strip() - current_rootfs_dev_partuuid = CMDHelperFuncs.get_attrs_by_dev( - "PARTUUID", current_rootfs_devpath - ).strip() + try: + self.standby_rootfs_devpath = get_partition_devpath( + parent_devpath=parent_devpath, + partition_id=SLOT_PAR_MAP[standby_rootfs_slot], + ) + self.standby_rootfs_dev_partuuid = CMDHelperFuncs.get_attrs_by_dev( + "PARTUUID", self.standby_rootfs_devpath + ).strip() + current_rootfs_dev_partuuid = CMDHelperFuncs.get_attrs_by_dev( + "PARTUUID", current_rootfs_devpath + ).strip() + except Exception as e: + _err_msg = f"failed to detect rootfs dev partuuid: {e!r}" + logger.error(_err_msg) + raise JetsonCBootContrlError(_err_msg) from e logger.info( "finish detecting rootfs devs: \n" From 1e09021844f78848af541ce2ba3fca8805fc7c97 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Sat, 31 Aug 2024 15:56:08 +0000 Subject: [PATCH 192/193] minor update --- src/otaclient/boot_control/_jetson_cboot.py | 20 ++++++++++---------- src/otaclient/boot_control/_jetson_uefi.py | 20 +++++++++++++------- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_cboot.py b/src/otaclient/boot_control/_jetson_cboot.py index 8ed7c026f..1a7e77c19 100644 --- a/src/otaclient/boot_control/_jetson_cboot.py +++ b/src/otaclient/boot_control/_jetson_cboot.py @@ -303,6 +303,16 @@ def __init__(self): logger.error(_err_msg) raise JetsonCBootContrlError(_err_msg) + # ------ load nvbootctrl config file ------ # + if not ( + nvbootctrl_conf_fpath := Path(boot_cfg.NVBOOTCTRL_CONF_FPATH) + ).is_file(): + _err_msg = "nv_boot_ctrl.conf is missing!" + logger.error(_err_msg) + raise JetsonCBootContrlError(_err_msg) + self.nvbootctrl_conf = nvbootctrl_conf_fpath.read_text() + logger.info(f"nvboot_ctrl_conf: \n{self.nvbootctrl_conf}") + # ------ check if unified A/B is enabled ------ # # NOTE: mismatch rootfs BSP version and bootloader firmware BSP version # is NOT supported and MUST not occur. @@ -396,16 +406,6 @@ def __init__(self): f"nvbootctrl -t rootfs dump-slots-info: \n{NVBootctrlJetsonCBOOT.dump_slots_info(target='rootfs')}" ) - # load nvbootctrl config file - if not ( - nvbootctrl_conf_fpath := Path(boot_cfg.NVBOOTCTRL_CONF_FPATH) - ).is_file(): - _err_msg = "nv_boot_ctrl.conf is missing!" - logger.error(_err_msg) - raise JetsonCBootContrlError(_err_msg) - self.nvbootctrl_conf = nvbootctrl_conf_fpath.read_text() - logger.info(f"nvboot_ctrl_conf: \n{self.nvbootctrl_conf}") - # API @property diff --git a/src/otaclient/boot_control/_jetson_uefi.py b/src/otaclient/boot_control/_jetson_uefi.py index d3803f86d..43c8f2f2d 100644 --- a/src/otaclient/boot_control/_jetson_uefi.py +++ b/src/otaclient/boot_control/_jetson_uefi.py @@ -719,7 +719,7 @@ def __init__(self): raise JetsonUEFIBootControlError(_err_msg) logger.info(f"dev compatibility: {compat_info}") - # load nvbootctrl config file + # ------ load nvbootctrl config file ------ # if not ( nvbootctrl_conf_fpath := Path(boot_cfg.NVBOOTCTRL_CONF_FPATH) ).is_file(): @@ -812,12 +812,18 @@ def __init__(self): parent_devpath=parent_devpath, partition_id=SLOT_PAR_MAP[standby_slot], ) - self.standby_rootfs_dev_partuuid = CMDHelperFuncs.get_attrs_by_dev( - "PARTUUID", self.standby_rootfs_devpath - ).strip() - current_rootfs_dev_partuuid = CMDHelperFuncs.get_attrs_by_dev( - "PARTUUID", current_rootfs_devpath - ).strip() + + try: + self.standby_rootfs_dev_partuuid = CMDHelperFuncs.get_attrs_by_dev( + "PARTUUID", self.standby_rootfs_devpath + ).strip() + current_rootfs_dev_partuuid = CMDHelperFuncs.get_attrs_by_dev( + "PARTUUID", current_rootfs_devpath + ).strip() + except Exception as e: + _err_msg = f"failed to detect rootfs PARTUUID: {e!r}" + logger.error(_err_msg) + raise JetsonUEFIBootControlError(_err_msg) from e logger.info( "rootfs device: \n" From 8a1a49dbe748d9cbbfc0480e2e69f939d021f705 Mon Sep 17 00:00:00 2001 From: "bodong.yang" Date: Tue, 3 Sep 2024 03:32:53 +0000 Subject: [PATCH 193/193] jetson-cboot: only check is_unified_enabled when BSP version >= r32.6.1 --- src/otaclient/boot_control/_jetson_cboot.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/otaclient/boot_control/_jetson_cboot.py b/src/otaclient/boot_control/_jetson_cboot.py index 1a7e77c19..f61315d5f 100644 --- a/src/otaclient/boot_control/_jetson_cboot.py +++ b/src/otaclient/boot_control/_jetson_cboot.py @@ -316,11 +316,13 @@ def __init__(self): # ------ check if unified A/B is enabled ------ # # NOTE: mismatch rootfs BSP version and bootloader firmware BSP version # is NOT supported and MUST not occur. - if unified_ab_enabled := NVBootctrlJetsonCBOOT.is_unified_enabled(): - logger.info( - "unified A/B is enabled, rootfs and bootloader will be switched together" - ) - self.unified_ab_enabled = unified_ab_enabled + self.unified_ab_enabled = False + if rootfs_bsp_version >= BSPVersion(32, 6, 1): + if unified_ab_enabled := NVBootctrlJetsonCBOOT.is_unified_enabled(): + logger.info( + "unified A/B is enabled, rootfs and bootloader will be switched together" + ) + self.unified_ab_enabled = unified_ab_enabled # ------ check A/B slots ------ # try: @@ -428,6 +430,9 @@ def switch_boot_to_standby(self) -> None: logger.info(f"switch boot to standby slot({target_slot})") if not self.unified_ab_enabled: + logger.info( + f"unified AB slot is not enabled, also set active rootfs slot to ${target_slot}" + ) NVBootctrlJetsonCBOOT.set_active_boot_slot(target_slot, target="rootfs") # when unified_ab enabled, switching bootloader slot will also switch @@ -586,6 +591,9 @@ def pre_update(self, version: str, *, standby_as_ref: bool, erase_standby: bool) if not self._cboot_control.unified_ab_enabled: # set standby rootfs as unbootable as we are going to update it # this operation not applicable when unified A/B is enabled. + logger.info( + "unified AB slot is not enabled, set standby rootfs slot as unbootable" + ) self._cboot_control.set_standby_rootfs_unbootable() # prepare standby slot dev