diff --git a/src/otaclient/boot_control/_slot_mnt_helper.py b/src/otaclient/boot_control/_slot_mnt_helper.py index 0d1d369ba..868850b57 100644 --- a/src/otaclient/boot_control/_slot_mnt_helper.py +++ b/src/otaclient/boot_control/_slot_mnt_helper.py @@ -16,11 +16,11 @@ from __future__ import annotations +import atexit import logging import shutil +from functools import partial from pathlib import Path -from subprocess import CalledProcessError -from time import sleep from otaclient.configs.cfg import cfg from otaclient_common import cmdhelper, replace_root @@ -28,91 +28,6 @@ logger = logging.getLogger(__name__) -MAX_RETRY_COUNT = 6 -RETRY_INTERVAL = 2 - - -def ensure_mount( - target: StrOrPath, mnt_point: StrOrPath, *, mount_func, raise_exception: bool -) -> None: # pragma: no cover - """Ensure the mounted on by our best. - - Raises: - If is True, raises the last failed attemp's CalledProcessError. - """ - for _retry in range(MAX_RETRY_COUNT + 1): - try: - mount_func(target=target, mount_point=mnt_point) - cmdhelper.is_target_mounted(mnt_point, raise_exception=True) - return - except CalledProcessError as e: - logger.error( - f"retry#{_retry} failed to mount {target} on {mnt_point}: {e!r}" - ) - logger.error(f"{e.stderr=}\n{e.stdout=}") - - if _retry >= MAX_RETRY_COUNT: - logger.error( - f"exceed max retry count mounting {target} on {mnt_point}, abort" - ) - if raise_exception: - raise - return - - sleep(RETRY_INTERVAL) - continue - - -def ensure_umount( - mnt_point: StrOrPath, *, ignore_error: bool -) -> None: # pragma: no cover - """Try to umount the at our best. - - Raises: - If is False, raises the last failed attemp's CalledProcessError. - """ - for _retry in range(MAX_RETRY_COUNT + 1): - try: - if not cmdhelper.is_target_mounted(mnt_point, raise_exception=False): - break - cmdhelper.umount(mnt_point, raise_exception=True) - except CalledProcessError as e: - logger.warning(f"retry#{_retry} failed to umount {mnt_point}: {e!r}") - logger.warning(f"{e.stderr}\n{e.stdout}") - - if _retry >= MAX_RETRY_COUNT: - logger.error(f"reached max retry on umounting {mnt_point}, abort") - if not ignore_error: - raise - return - - sleep(RETRY_INTERVAL) - continue - - -def ensure_mointpoint( - mnt_point: Path, *, ignore_error: bool -) -> None: # pragma: no cover - """Ensure the exists, has no mount on it and ready for mount. - - If the is valid, but we failed to umount any previous mounts on it, - we still keep use the mountpoint as later mount will override the previous one. - """ - if mnt_point.is_symlink() or not mnt_point.is_dir(): - mnt_point.unlink(missing_ok=True) - - if not mnt_point.exists(): - mnt_point.mkdir(exist_ok=True, parents=True) - return - - try: - ensure_umount(mnt_point, ignore_error=ignore_error) - except Exception: - logger.warning( - f"{mnt_point} still has other mounts on it, " - f"but still use {mnt_point} and override the previous mount" - ) - class SlotMountHelper: # pragma: no cover """Helper class that provides methods for mounting slots.""" @@ -141,6 +56,23 @@ def __init__( ) ) + # ensure the each mount points being umounted at termination + atexit.register( + partial( + cmdhelper.ensure_umount, + self.active_slot_mount_point, + ignore_error=True, + ) + ) + atexit.register( + partial( + cmdhelper.ensure_umount, + self.standby_slot_mount_point, + ignore_error=True, + max_retry=3, + ) + ) + def mount_standby(self) -> None: """Mount standby slot dev rw to . @@ -148,10 +80,10 @@ def mount_standby(self) -> None: CalledProcessedError on the last failed attemp. """ logger.debug("mount standby slot rootfs dev...") - ensure_mointpoint(self.standby_slot_mount_point, ignore_error=True) - ensure_umount(self.standby_slot_dev, ignore_error=False) + cmdhelper.ensure_mointpoint(self.standby_slot_mount_point, ignore_error=True) + cmdhelper.ensure_umount(self.standby_slot_dev, ignore_error=False) - ensure_mount( + cmdhelper.ensure_mount( target=self.standby_slot_dev, mnt_point=self.standby_slot_mount_point, mount_func=cmdhelper.mount_rw, @@ -165,8 +97,8 @@ def mount_active(self) -> None: CalledProcessedError on the last failed attemp. """ logger.debug("mount active slot rootfs dev...") - ensure_mointpoint(self.active_slot_mount_point, ignore_error=True) - ensure_mount( + cmdhelper.ensure_mointpoint(self.active_slot_mount_point, ignore_error=True) + cmdhelper.ensure_mount( target=self.active_rootfs, mnt_point=self.active_slot_mount_point, mount_func=cmdhelper.bind_mount_ro, @@ -195,7 +127,7 @@ def prepare_standby_dev( erase_standby: bool = False, fslabel: str | None = None, ) -> None: - ensure_umount(self.standby_slot_dev, ignore_error=True) + cmdhelper.ensure_umount(self.standby_slot_dev, ignore_error=True) if erase_standby: return cmdhelper.mkfs_ext4(self.standby_slot_dev, fslabel=fslabel) @@ -206,5 +138,7 @@ def prepare_standby_dev( def umount_all(self, *, ignore_error: bool = True): logger.debug("unmount standby slot and active slot mount point...") - ensure_umount(self.active_slot_mount_point, ignore_error=ignore_error) - ensure_umount(self.standby_slot_mount_point, ignore_error=ignore_error) + cmdhelper.ensure_umount(self.active_slot_mount_point, ignore_error=ignore_error) + cmdhelper.ensure_umount( + self.standby_slot_mount_point, ignore_error=ignore_error + ) diff --git a/src/otaclient_common/cmdhelper.py b/src/otaclient_common/cmdhelper.py index 5ca2e9ef2..21af23af7 100644 --- a/src/otaclient_common/cmdhelper.py +++ b/src/otaclient_common/cmdhelper.py @@ -23,8 +23,10 @@ import logging import sys +import time +from pathlib import Path from subprocess import CalledProcessError -from typing import Literal, NoReturn +from typing import Literal, NoReturn, Protocol from otaclient_common.common import subprocess_call, subprocess_check_output from otaclient_common.typing import StrOrPath @@ -237,6 +239,103 @@ def set_ext4_fslabel( subprocess_call(cmd, raise_exception=raise_exception) +def mkfs_ext4( + dev: str, + *, + fslabel: str | None = None, + fsuuid: str | None = None, + raise_exception: bool = True, +) -> None: # pragma: no cover + """Create new ext4 formatted filesystem on , optionally with + and/or . + + Args: + dev (str): device to be formatted to ext4. + fslabel (Optional[str], optional): fslabel of the new ext4 filesystem. Defaults to None. + When it is None, this function will try to preserve the previous fslabel. + fsuuid (Optional[str], optional): fsuuid of the new ext4 filesystem. Defaults to None. + When it is None, this function will try to preserve the previous fsuuid. + raise_exception (bool, optional): raise exception on subprocess call failed. + Defaults to True. + """ + cmd = ["mkfs.ext4", "-F"] + + if not fsuuid: + try: + fsuuid = get_attrs_by_dev("UUID", dev) + assert fsuuid + logger.debug(f"reuse previous UUID: {fsuuid}") + except Exception: + pass + if fsuuid: + logger.debug(f"using UUID: {fsuuid}") + cmd.extend(["-U", fsuuid]) + + if not fslabel: + try: + fslabel = get_attrs_by_dev("LABEL", dev) + assert fslabel + logger.debug(f"reuse previous fs LABEL: {fslabel}") + except Exception: + pass + if fslabel: + logger.debug(f"using fs LABEL: {fslabel}") + cmd.extend(["-L", fslabel]) + + cmd.append(dev) + logger.warning(f"format {dev} to ext4: {cmd=}") + subprocess_call(cmd, raise_exception=raise_exception) + + +def reboot(args: list[str] | None = None) -> NoReturn: # pragma: no cover + """Reboot the system, with optional args passed to reboot command. + + This is implemented by calling: + reboot [args[0], args[1], ...] + + NOTE(20230614): this command makes otaclient exit immediately. + NOTE(20240421): rpi_boot's reboot takes args. + + Args: + args (Optional[list[str]], optional): args passed to reboot command. + Defaults to None, not passing any args. + + Raises: + CalledProcessError for the reboot call, or SystemExit on sys.exit(0). + """ + cmd = ["reboot"] + if args: + logger.info(f"will reboot with argument: {args=}") + cmd.extend(args) + + logger.warning("system will reboot now!") + subprocess_call(cmd, raise_exception=True) + sys.exit(0) + + +# +# ------ mount related helpers ------ # +# + +MAX_RETRY_COUNT = 6 +RETRY_INTERVAL = 2 + + +class MountHelper(Protocol): + """Protocol for mount helper functions. + + This is for typing purpose. + """ + + def __call__( + self, + target: StrOrPath, + mount_point: StrOrPath, + *, + raise_exception: bool = True, + ) -> None: ... + + def mount( target: StrOrPath, mount_point: StrOrPath, @@ -267,7 +366,7 @@ def mount( def mount_rw( - target: str, mount_point: StrOrPath, *, raise_exception: bool = True + target: StrOrPath, mount_point: StrOrPath, *, raise_exception: bool = True ) -> None: # pragma: no cover """Mount the to read-write. @@ -278,7 +377,7 @@ def mount_rw( mount events propagation to/from this mount point. Args: - target (str): target to be mounted. + target (StrOrPath): target to be mounted. mount_point (StrOrPath): mount point to mount to. raise_exception (bool, optional): raise exception on subprocess call failed. Defaults to True. @@ -288,7 +387,7 @@ def mount_rw( "mount", "-o", "rw", "--make-private", "--make-unbindable", - target, + str(target), str(mount_point), ] # fmt: on @@ -296,7 +395,7 @@ def mount_rw( def bind_mount_ro( - target: str, mount_point: StrOrPath, *, raise_exception: bool = True + target: StrOrPath, mount_point: StrOrPath, *, raise_exception: bool = True ) -> None: # pragma: no cover """Bind mount the to read-only. @@ -304,7 +403,7 @@ def bind_mount_ro( mount -o bind,ro --make-private --make-unbindable Args: - target (str): target to be mounted. + target (StrOrPath): target to be mounted. mount_point (StrOrPath): mount point to mount to. raise_exception (bool, optional): raise exception on subprocess call failed. Defaults to True. @@ -314,89 +413,15 @@ def bind_mount_ro( "mount", "-o", "bind,ro", "--make-private", "--make-unbindable", - target, + str(target), str(mount_point) ] # fmt: on subprocess_call(cmd, raise_exception=raise_exception) -def umount( - target: StrOrPath, *, raise_exception: bool = True -) -> None: # pragma: no cover - """Try to umount the . - - This is implemented by calling: - umount - - Before calling umount, the will be check whether it is mounted, - if it is not mounted, this function will return directly. - - Args: - target (StrOrPath): target to be umounted. - raise_exception (bool, optional): raise exception on subprocess call failed. - Defaults to True. - """ - # first try to check whether the target(either a mount point or a dev) - # is mounted - if not is_target_mounted(target, raise_exception=False): - return - - # if the target is mounted, try to unmount it. - _cmd = ["umount", str(target)] - subprocess_call(_cmd, raise_exception=raise_exception) - - -def mkfs_ext4( - dev: str, - *, - fslabel: str | None = None, - fsuuid: str | None = None, - raise_exception: bool = True, -) -> None: # pragma: no cover - """Create new ext4 formatted filesystem on , optionally with - and/or . - - Args: - dev (str): device to be formatted to ext4. - fslabel (Optional[str], optional): fslabel of the new ext4 filesystem. Defaults to None. - When it is None, this function will try to preserve the previous fslabel. - fsuuid (Optional[str], optional): fsuuid of the new ext4 filesystem. Defaults to None. - When it is None, this function will try to preserve the previous fsuuid. - raise_exception (bool, optional): raise exception on subprocess call failed. - Defaults to True. - """ - cmd = ["mkfs.ext4", "-F"] - - if not fsuuid: - try: - fsuuid = get_attrs_by_dev("UUID", dev) - assert fsuuid - logger.debug(f"reuse previous UUID: {fsuuid}") - except Exception: - pass - if fsuuid: - logger.debug(f"using UUID: {fsuuid}") - cmd.extend(["-U", fsuuid]) - - if not fslabel: - try: - fslabel = get_attrs_by_dev("LABEL", dev) - assert fslabel - logger.debug(f"reuse previous fs LABEL: {fslabel}") - except Exception: - pass - if fslabel: - logger.debug(f"using fs LABEL: {fslabel}") - cmd.extend(["-L", fslabel]) - - cmd.append(dev) - logger.warning(f"format {dev} to ext4: {cmd=}") - subprocess_call(cmd, raise_exception=raise_exception) - - def mount_ro( - *, target: str, mount_point: StrOrPath, raise_exception: bool = True + target: StrOrPath, mount_point: StrOrPath, *, raise_exception: bool = True ) -> None: # pragma: no cover """Mount to read-only. @@ -404,14 +429,16 @@ def mount_ro( if the target device is not mounted, we directly mount it to the mount_point. Args: - target (str): target to be mounted. + target (StrOrPath): target to be mounted. mount_point (StrOrPath): mount point to mount to. raise_exception (bool, optional): raise exception on subprocess call failed. Defaults to True. """ # NOTE: set raise_exception to false to allow not mounted # not mounted dev will have empty return str - if _active_mount_point := get_mount_point_by_dev(target, raise_exception=False): + if _active_mount_point := get_mount_point_by_dev( + str(target), raise_exception=False + ): bind_mount_ro( _active_mount_point, mount_point, @@ -424,34 +451,139 @@ def mount_ro( "mount", "-o", "ro", "--make-private", "--make-unbindable", - target, + str(target), str(mount_point), ] # fmt: on subprocess_call(cmd, raise_exception=raise_exception) -def reboot(args: list[str] | None = None) -> NoReturn: # pragma: no cover - """Reboot the system, with optional args passed to reboot command. +def umount( + target: StrOrPath, *, raise_exception: bool = True +) -> None: # pragma: no cover + """Try to umount the . This is implemented by calling: - reboot [args[0], args[1], ...] + umount - NOTE(20230614): this command makes otaclient exit immediately. - NOTE(20240421): rpi_boot's reboot takes args. + Before calling umount, the will be check whether it is mounted, + if it is not mounted, this function will return directly. Args: - args (Optional[list[str]], optional): args passed to reboot command. - Defaults to None, not passing any args. + target (StrOrPath): target to be umounted. + raise_exception (bool, optional): raise exception on subprocess call failed. + Defaults to True. + """ + # first try to check whether the target(either a mount point or a dev) + # is mounted + if not is_target_mounted(target, raise_exception=False): + return + + # if the target is mounted, try to unmount it. + _cmd = ["umount", str(target)] + subprocess_call(_cmd, raise_exception=raise_exception) + + +def ensure_mount( + target: StrOrPath, + mnt_point: StrOrPath, + *, + mount_func: MountHelper, + raise_exception: bool, + max_retry: int = MAX_RETRY_COUNT, + retry_interval: int = RETRY_INTERVAL, +) -> None: # pragma: no cover + """Ensure the mounted on by our best. Raises: - CalledProcessError for the reboot call, or SystemExit on sys.exit(0). + If is True, raises the last failed attemp's CalledProcessError. """ - cmd = ["reboot"] - if args: - logger.info(f"will reboot with argument: {args=}") - cmd.extend(args) + for _retry in range(max_retry + 1): + try: + mount_func(target=target, mount_point=mnt_point) + is_target_mounted(mnt_point, raise_exception=True) + return + except CalledProcessError as e: + logger.info( + ( + f"retry#{_retry} failed to mount {target} on {mnt_point}: {e!r}\n" + f"{e.stderr=}\n{e.stdout=}\n" + "retrying another mount ..." + ) + ) + + if _retry >= max_retry: + logger.error( + f"exceed max retry count mounting {target} on {mnt_point}, abort" + ) + if raise_exception: + raise + return + + time.sleep(retry_interval) + continue + + +def ensure_umount( + mnt_point: StrOrPath, + *, + ignore_error: bool, + max_retry: int = MAX_RETRY_COUNT, + retry_interval: int = RETRY_INTERVAL, +) -> None: # pragma: no cover + """Try to umount the at our best. - logger.warning("system will reboot now!") - subprocess_call(cmd, raise_exception=True) - sys.exit(0) + Raises: + If is False, raises the last failed attemp's CalledProcessError. + """ + for _retry in range(max_retry + 1): + try: + if not is_target_mounted(mnt_point, raise_exception=False): + break + umount(mnt_point, raise_exception=True) + except CalledProcessError as e: + logger.info( + ( + f"retry#{_retry} failed to umount {mnt_point}: {e!r}\n" + f"{e.stderr=}\n{e.stdout=}" + ) + ) + + if _retry >= max_retry: + logger.error(f"reached max retry on umounting {mnt_point}, abort") + if not ignore_error: + raise + return + + time.sleep(retry_interval) + continue + + +def ensure_mointpoint( + mnt_point: StrOrPath, *, ignore_error: bool +) -> None: # pragma: no cover + """Ensure the exists, has no mount on it and ready for mount. + + If the is valid, but we failed to umount any previous mounts on it, + we still keep use the mountpoint as later mount will override the previous one. + """ + mnt_point = Path(mnt_point) + if mnt_point.is_symlink() or not mnt_point.is_dir(): + mnt_point.unlink(missing_ok=True) + + if not mnt_point.exists(): + mnt_point.mkdir(exist_ok=True, parents=True) + return + + try: + ensure_umount(mnt_point, ignore_error=False) + except Exception as e: + if not ignore_error: + logger.error(f"failed to prepare {mnt_point=}: {e!r}") + raise + logger.warning( + ( + f"failed to prepare {mnt_point=}: {e!r} \n" + f"But still use {mnt_point} and override the previous mount" + ) + )