Skip to content

Commit

Permalink
feat: add mount related helper funcs, refine slot_mount_helper with t…
Browse files Browse the repository at this point in the history 
…hese new helper funcs and add atexit hooks for umount (#429)

This PR introduces several high-level mount related helper funcs for robust mount/umount, and refines the slot_mount_helper module with the above new high-level mount helper funcs. These high-level mount helpers are originally implement in the slot_mount_helper module. 
Also, now slot_mount_helper will register atexit hook for mount points used by otaclient during OTA to ensure mount points umount at otaclient exits(terminated).

The following funcs are added:
1. `ensure_mount`: adopt the similar mechanism of initrd, mount and ensure the mount point is mounted with at most MAX_RETRY(6 by default) times.
2. `ensure_umount`: umount and ensure the mount point doesn't have any mounts with at most MAX_RETRY(6 by default) times. With this change, we can also handle mount point with multiple layers of mounts.
3. `ensure_mountpoint`: ensure the the target mount point exists, no mounts on it(using `ensure_umount`) and ready for mount.
  • Loading branch information
@Bodong-Yang
Bodong-Yang authored Nov 26, 2024
1 parent 638db15 commit a670e6d
Show file tree
Hide file tree
Showing 2 changed files with 261 additions and 195 deletions.
124 changes: 29 additions & 95 deletions src/otaclient/boot_control/_slot_mnt_helper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,103 +16,18 @@

from __future__ import annotations

import atexit
import logging
import shutil
from functools import partial
from pathlib import Path
from subprocess import CalledProcessError
from time import sleep

from otaclient.configs.cfg import cfg
from otaclient_common import cmdhelper, replace_root
from otaclient_common.typing import StrOrPath

logger = logging.getLogger(__name__)

MAX_RETRY_COUNT = 6
RETRY_INTERVAL = 2


def ensure_mount(
target: StrOrPath, mnt_point: StrOrPath, *, mount_func, raise_exception: bool
) -> None: # pragma: no cover
"""Ensure the <target> mounted on <mnt_point> by our best.
Raises:
If <raise_exception> is True, raises the last failed attemp's CalledProcessError.
"""
for _retry in range(MAX_RETRY_COUNT + 1):
try:
mount_func(target=target, mount_point=mnt_point)
cmdhelper.is_target_mounted(mnt_point, raise_exception=True)
return
except CalledProcessError as e:
logger.error(
f"retry#{_retry} failed to mount {target} on {mnt_point}: {e!r}"
)
logger.error(f"{e.stderr=}\n{e.stdout=}")

if _retry >= MAX_RETRY_COUNT:
logger.error(
f"exceed max retry count mounting {target} on {mnt_point}, abort"
)
if raise_exception:
raise
return

sleep(RETRY_INTERVAL)
continue


def ensure_umount(
mnt_point: StrOrPath, *, ignore_error: bool
) -> None: # pragma: no cover
"""Try to umount the <mnt_point> at our best.
Raises:
If <ignore_error> is False, raises the last failed attemp's CalledProcessError.
"""
for _retry in range(MAX_RETRY_COUNT + 1):
try:
if not cmdhelper.is_target_mounted(mnt_point, raise_exception=False):
break
cmdhelper.umount(mnt_point, raise_exception=True)
except CalledProcessError as e:
logger.warning(f"retry#{_retry} failed to umount {mnt_point}: {e!r}")
logger.warning(f"{e.stderr}\n{e.stdout}")

if _retry >= MAX_RETRY_COUNT:
logger.error(f"reached max retry on umounting {mnt_point}, abort")
if not ignore_error:
raise
return

sleep(RETRY_INTERVAL)
continue


def ensure_mointpoint(
mnt_point: Path, *, ignore_error: bool
) -> None: # pragma: no cover
"""Ensure the <mnt_point> exists, has no mount on it and ready for mount.
If the <mnt_point> is valid, but we failed to umount any previous mounts on it,
we still keep use the mountpoint as later mount will override the previous one.
"""
if mnt_point.is_symlink() or not mnt_point.is_dir():
mnt_point.unlink(missing_ok=True)

if not mnt_point.exists():
mnt_point.mkdir(exist_ok=True, parents=True)
return

try:
ensure_umount(mnt_point, ignore_error=ignore_error)
except Exception:
logger.warning(
f"{mnt_point} still has other mounts on it, "
f"but still use {mnt_point} and override the previous mount"
)


class SlotMountHelper: # pragma: no cover
"""Helper class that provides methods for mounting slots."""
Expand Down Expand Up @@ -141,17 +56,34 @@ def __init__(
)
)

# ensure the each mount points being umounted at termination
atexit.register(
partial(
cmdhelper.ensure_umount,
self.active_slot_mount_point,
ignore_error=True,
)
)
atexit.register(
partial(
cmdhelper.ensure_umount,
self.standby_slot_mount_point,
ignore_error=True,
max_retry=3,
)
)

def mount_standby(self) -> None:
"""Mount standby slot dev rw to <standby_slot_mount_point>.
Raises:
CalledProcessedError on the last failed attemp.
"""
logger.debug("mount standby slot rootfs dev...")
ensure_mointpoint(self.standby_slot_mount_point, ignore_error=True)
ensure_umount(self.standby_slot_dev, ignore_error=False)
cmdhelper.ensure_mointpoint(self.standby_slot_mount_point, ignore_error=True)
cmdhelper.ensure_umount(self.standby_slot_dev, ignore_error=False)

ensure_mount(
cmdhelper.ensure_mount(
target=self.standby_slot_dev,
mnt_point=self.standby_slot_mount_point,
mount_func=cmdhelper.mount_rw,
Expand All @@ -165,8 +97,8 @@ def mount_active(self) -> None:
CalledProcessedError on the last failed attemp.
"""
logger.debug("mount active slot rootfs dev...")
ensure_mointpoint(self.active_slot_mount_point, ignore_error=True)
ensure_mount(
cmdhelper.ensure_mointpoint(self.active_slot_mount_point, ignore_error=True)
cmdhelper.ensure_mount(
target=self.active_rootfs,
mnt_point=self.active_slot_mount_point,
mount_func=cmdhelper.bind_mount_ro,
Expand Down Expand Up @@ -195,7 +127,7 @@ def prepare_standby_dev(
erase_standby: bool = False,
fslabel: str | None = None,
) -> None:
ensure_umount(self.standby_slot_dev, ignore_error=True)
cmdhelper.ensure_umount(self.standby_slot_dev, ignore_error=True)
if erase_standby:
return cmdhelper.mkfs_ext4(self.standby_slot_dev, fslabel=fslabel)

Expand All @@ -206,5 +138,7 @@ def prepare_standby_dev(

def umount_all(self, *, ignore_error: bool = True):
logger.debug("unmount standby slot and active slot mount point...")
ensure_umount(self.active_slot_mount_point, ignore_error=ignore_error)
ensure_umount(self.standby_slot_mount_point, ignore_error=ignore_error)
cmdhelper.ensure_umount(self.active_slot_mount_point, ignore_error=ignore_error)
cmdhelper.ensure_umount(
self.standby_slot_mount_point, ignore_error=ignore_error
)
Loading

1 comment on commit a670e6d

@github-actions
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Coverage

Coverage Report
FileStmtsMissCoverMissing
src/ota_metadata/legacy
   __init__.py10100% 
   parser.py3264386%100, 156, 161, 197–198, 208–209, 212, 224, 277–279, 283–286, 320–323, 392, 395, 403–405, 418, 427–428, 431–432, 597–599, 649–650, 653, 681–683, 737, 740–742
   types.py841384%37, 40–42, 112–116, 122–125
src/ota_metadata/utils
   cert_store.py420100% 
src/ota_proxy
   __init__.py361072%59, 61, 63, 72, 81–82, 102, 104–106
   __main__.py770%16–18, 20, 22–23, 25
   _consts.py150100% 
   cache_control_header.py68494%71, 91, 113, 121
   cache_streaming.py1442284%154–156, 184–186, 211, 225, 229–230, 265–266, 268, 280, 349, 355–356, 359, 367–370
   config.py170100% 
   db.py731875%109, 115, 153, 159–160, 163, 169, 171, 192–199, 201–202
   errors.py50100% 
   lru_cache_helper.py47295%84–85
   ota_cache.py2155972%70–71, 140, 151–152, 184–185, 202, 239–243, 247–249, 251, 253–260, 262–264, 267–268, 272–273, 277, 324, 332–334, 413–416, 430, 433–434, 448–449, 451–453, 457–458, 464–465, 496, 502, 529, 581–583
   server_app.py1393971%76, 79, 85, 101, 103, 162, 171, 213–214, 216–218, 221, 226–228, 231–232, 235, 238, 241, 244, 257–258, 261–262, 264, 267, 293–296, 299, 313–315, 321–323
   utils.py140100% 
src/otaclient
   __init__.py5260%17, 19
   __main__.py110%16
   _logging.py513335%43–44, 46–47, 49–54, 56–57, 59–60, 62–65, 67, 77, 80–82, 84–86, 89–90, 92–96
   _status_monitor.py1611193%46, 48–49, 157, 160, 177, 180, 196–197, 204, 207
   _types.py750100% 
   errors.py120199%97
   main.py492940%30, 32–39, 41–42, 44–45, 47–48, 52, 54, 59, 61, 67–69, 72–73, 77–80, 82
   ota_core.py3009070%97, 124, 126–127, 129, 133, 137–138, 143–144, 150, 152, 191–194, 200, 204, 210, 349, 381–382, 384, 393, 396, 401–402, 405, 411, 413–417, 453–456, 459–466, 502–505, 583–590, 595, 598–605, 638–639, 645, 649–650, 656, 681–683, 685, 760, 788–789, 791–792, 800, 802–808
   utils.py37294%73–74
src/otaclient/boot_control
   __init__.py40100% 
   _firmware_package.py932276%82, 86, 136, 180, 186, 209–210, 213–218, 220–221, 224–229, 231
   _grub.py41812769%214, 262–265, 271–275, 312–313, 320–325, 328–334, 337, 340–341, 346, 348–350, 359–365, 367–368, 370–372, 381–383, 385–387, 466–467, 471–472, 524, 530, 556, 578, 582–583, 598–600, 624–627, 639, 643–645, 647–649, 708–711, 736–739, 762–765, 777–778, 781–782, 817, 823, 843–844, 846, 871–873, 891–894, 919–922, 929–932, 937–945, 950–957
   _jetson_cboot.py2612610%20, 22–25, 27–29, 35–40, 42, 58–60, 62, 64–65, 71, 75, 134, 137, 139–140, 143, 150–151, 159–160, 163, 169–170, 178, 187–191, 193, 199, 202–203, 209, 212–213, 218–219, 221, 227–228, 231–232, 235–237, 239, 245, 250–252, 254–256, 261, 263–266, 268–269, 278–279, 282–283, 288–289, 292–296, 299–300, 305–306, 309, 312–316, 321–324, 327, 330–331, 334, 337–338, 341, 345–350, 354–355, 359, 362–363, 366, 369–372, 374, 377–378, 382, 385, 388–391, 393, 400, 404–405, 408–409, 415–416, 422, 424–425, 429, 431, 433–435, 438, 442, 445, 448–449, 451, 454, 462–463, 470, 480, 483, 491–492, 497–500, 502, 509, 511–513, 519–520, 524–525, 528, 532, 535, 537, 544–548, 550, 562–565, 568, 571, 573, 580, 590–592, 594, 596, 599, 602, 605, 607–608, 611–615, 619–621, 623, 631–635, 637, 640, 644, 647, 658–659, 664, 674, 677–683, 687–693, 697–706, 710–718, 722, 724, 726–728
   _jetson_common.py1724573%132, 140, 288–291, 294, 311, 319, 354, 359–364, 382, 408–409, 411–413, 417–420, 422–423, 425–429, 431, 438–439, 442–443, 453, 456–457, 460, 462, 506–507
   _jetson_uefi.py40427432%124–126, 131–132, 151–153, 158–161, 328, 446, 448–451, 455, 459–460, 462–470, 472, 484–485, 488–489, 492–493, 496–498, 502–503, 508–510, 514, 518–519, 522–523, 526–527, 531, 534–535, 537, 542–543, 547, 550–551, 556, 560–561, 564, 568–570, 572, 576–579, 581–582, 604–605, 609–610, 612, 616, 620–621, 624–625, 632, 635–637, 640, 642–643, 648–649, 652–655, 657–658, 663, 665–666, 674, 677–680, 682–683, 685, 689–690, 694, 702–706, 709–710, 712, 715–719, 722, 725–729, 733–734, 737–742, 745–746, 749–752, 754–755, 762–763, 773–776, 779, 782–785, 788–792, 795–796, 799, 802–805, 808, 810, 815–816, 819, 822–825, 827, 833, 838–839, 858–859, 862, 870–871, 878, 888, 891, 898–899, 904–907, 915–918, 926–927, 939–942, 944, 947, 950, 958, 969–971, 973–975, 977–981, 986–987, 989, 1002, 1006, 1009, 1019, 1024, 1032–1033, 1036, 1040, 1042–1044, 1050–1051, 1056, 1064–1069, 1074–1079, 1084–1092, 1097–1104, 1112–1114
   _ota_status_control.py1021189%117, 122, 127, 240, 244–245, 248, 255, 257–258, 273
   _rpi_boot.py28713353%53, 56, 120–121, 125, 133–136, 150–153, 158–159, 161–162, 167–168, 171–172, 181–182, 222, 228–232, 235, 253–255, 259–261, 266–268, 272–274, 284–285, 288, 291, 293–294, 296–297, 299–301, 307, 310–311, 321–324, 332–336, 338, 340–341, 346–347, 354, 357–362, 393, 395–398, 408–411, 415–416, 418–422, 450–453, 472–475, 501–504, 509–517, 522–529, 544–547, 554–557, 565–567
   _slot_mnt_helper.py100100% 
   configs.py510100% 
   protocol.py60100% 
   selecter.py412929%44–46, 49–50, 54–55, 58–60, 63, 65, 69, 77–79, 81–82, 84–85, 89, 91, 93–94, 96, 98–99, 101, 103
src/otaclient/configs
   __init__.py170100% 
   _cfg_configurable.py490100% 
   _cfg_consts.py47197%97
   _common.py80100% 
   _ecu_info.py56394%59, 64–65
   _proxy_info.py51492%85, 87–88, 90
   cfg.py190100% 
src/otaclient/create_standby
   __init__.py13192%36
   common.py2264480%59, 62–63, 67–69, 71, 75–76, 78, 126, 174–176, 178–180, 182, 185–188, 192, 203, 279–280, 282–287, 299, 339, 367, 370–372, 388–389, 403, 407, 429–430
   interface.py70100% 
   rebuild_mode.py1151091%98–100, 119, 150–155
src/otaclient/grpc
   _otaproxy_ctx.py1175156%68–70, 72–73, 81–84, 87–89, 93, 98–99, 101–102, 105, 107–108, 111–113, 116–117, 120–122, 127–132, 136, 139–143, 145–146, 154–156, 159, 196–198, 203, 239
src/otaclient/grpc/api_v2
   ecu_status.py1531093%75, 78, 81, 147, 171, 173, 300, 370–371, 410
   ecu_tracker.py341944%40–42, 48, 52–54, 61–63, 66, 70–74, 77–79
   servicer.py1273671%85, 174–176, 183, 194–195, 236–237, 240–244, 253–262, 269, 275, 278–281, 287–288, 295, 298, 304, 307
   types.py46295%78–79
src/otaclient_api/v2
   api_caller.py39684%45–47, 83–85
   types.py2562391%86, 89–92, 131, 209–210, 212, 259, 262–263, 506–508, 512–513, 515, 518–519, 522–523, 586
src/otaclient_common
   __init__.py341555%42–44, 61, 63, 68–77
   _io.py64198%41
   cmdhelper.py130100% 
   common.py1061189%44, 148, 151–153, 168, 175–177, 271, 275
   downloader.py1991094%107–108, 126, 153, 369, 424, 428, 516–517, 526
   linux.py611575%51–53, 59, 69, 74, 76, 108–109, 133–134, 190, 195–196, 198
   logging.py29196%55
   persist_file_handling.py1181884%113, 118, 150–152, 163, 192–193, 228–232, 242–244, 246–247
   proto_streamer.py42880%33, 48, 66–67, 72, 81–82, 100
   proto_wrapper.py3984688%87, 165, 172, 184–186, 205, 210, 221, 257, 263, 268, 299, 303, 307, 402, 462, 469, 472, 492, 499, 501, 526, 532, 535, 537, 562, 568, 571, 573, 605, 609, 611, 625, 642, 669, 672, 676, 707, 713, 762–763, 765, 803–805
   retry_task_map.py105595%158–159, 161, 181–182
   typing.py31487%48, 97–98, 100
TOTAL6461163274% 

Tests Skipped Failures Errors Time
235 0 💤 0 ❌ 0 🔥 11m 55s ⏱️

Please sign in to comment.