Merge remote-tracking branch 'origin/main' into refactor/re-architect…

…uring

.github/tools/gen_requirements_txt.py

@@ -0,0 +1,32 @@
+# This script requires python 3.12 to run.
+from __future__ import annotations
+
+import sys
+from typing import Any
+
+import tomllib  # type: ignore[import]
+
+
+def gen_requirements_txt(pyproject_cfg: dict[str, Any]) -> str:
+    _res: list[str] = [
+        "# Automatically generated from pyproject.toml by gen_requirements_txt.py script.",
+        "# DO NOT EDIT! Only for reference use.",
+    ]
+    try:
+        _res.extend(pyproject_cfg["project"]["dependencies"])
+    except KeyError:
+        print("WARNING: no deps are defined in pyproject.toml")
+    _res.append("")
+    return "\n".join(_res)
+
+
+if __name__ == "__main__":
+    script_name, *args = sys.argv
+    if len(args) < 2:
+        print(f"Usage: {script_name} <pyproject.toml> <requirements.txt>")
+        sys.exit(1)
+
+    pyproject_toml, requirements_txt, *_ = args
+    with open(pyproject_toml, "rb") as src, open(requirements_txt, "w") as dst:
+        _parsed = tomllib.load(src)
+        dst.write(gen_requirements_txt(_parsed))

.github/workflows/gen_requirements_txt.yaml

@@ -0,0 +1,52 @@
+name: gen requirements.txt on pyproject.toml file changed
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'pyproject.toml'
+  # allow manual dispatch of this workflow
+  workflow_dispatch:
+
+jobs:
+  gen-requirements-txt:
+    runs-on: ubuntu-latest
+    timeout-minutes: 1
+
+    steps:
+      - name: checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.ref }}
+          # use repo scope deploy key for the later git operations, so that the pushed commit can trigger the
+          #   workflow as expected. The default action token GITHUB_TOKEN cannot trigger new workflows.
+          # For more details about this restriction, please refer to:
+          #   https://github.com/peter-evans/create-pull-request/issues/48 and
+          #   https://github.com/peter-evans/create-pull-request/blob/main/docs/concepts-guidelines.md#triggering-further-workflow-runs
+          ssh_key: ${{ secrets.DEPLOY_KEY }}
+          persist-credentials: true
+
+      - name: setup python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.12'
+
+      - name: generate requirements.txt
+        run: |
+          python .github/tools/gen_requirements_txt.py \
+            pyproject.toml requirements.txt
+
+      - name: commit change if needed
+        run: |
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+
+          if git diff --exit-code requirements.txt; then
+            echo "skip commit as requirements.txt is not changed"
+          else
+            echo "detect requirements.txt updated, commit change ..."
+            git add requirements.txt
+            git commit -m "[GHA] Update requirements.txt on pyproject.toml changed"
+            git push
+          fi

.github/workflows/test.yaml

@@ -4,6 +4,14 @@ on:
   pull_request:
     branches:
       - main
+    types:
+      - opened
+      - synchronize
+      - reopened
+      # NOTE(20241016): this is a workaround for PR with head
+      #   updated by gen_requirements_txt workflow
+      - review_requested
+      - assigned
   push:
     branches:
       - main
@@ -48,7 +56,7 @@ jobs:
       # export the coverage report to the comment!
       - name: Add coverage report to PR comment
         continue-on-error: true
-        uses: MishaKav/[email protected].52
+        uses: MishaKav/[email protected].53
         with:
           pytest-xml-coverage-path: test_result/coverage.xml
           junitxml-path: test_result/pytest.xml

pyproject.toml

@@ -25,8 +25,8 @@ dynamic = [
 ]
 dependencies = [
   "aiofiles<25,>=24.1",
-  "aiohttp>=3.9.5,<3.11",
-  "cryptography>=42.0.4,<44",
+  "aiohttp>=3.10.2,<3.11",
+  "cryptography>=43.0.1,<44",
   "grpcio>=1.53.2,<1.67",
   "protobuf<4.22,>=4.21.12",
   "pydantic<3,>=2.6",
@@ -37,7 +37,7 @@ dependencies = [
   "simple-sqlite3-orm<0.3,>=0.2",
   "typing-extensions>=4.6.3",
   "urllib3<2.3,>=2.2.2",
-  "uvicorn[standard]<0.31,>=0.30",
+  "uvicorn[standard]>=0.30,<0.32",
   "zstandard<0.24,>=0.22",
 ]
 optional-dependencies.dev = [

requirements.txt

@@ -0,0 +1,17 @@
+# Automatically generated from pyproject.toml by gen_requirements_txt.py script.
+# DO NOT EDIT! Only for reference use.
+aiofiles<25,>=24.1
+aiohttp>=3.10.2,<3.11
+cryptography>=43.0.1,<44
+grpcio>=1.53.2,<1.67
+protobuf<4.22,>=4.21.12
+pydantic<3,>=2.6
+pydantic-settings<3,>=2.3
+pyopenssl<25,>=24.1
+pyyaml<7,>=6.0.1
+requests<2.33,>=2.32
+simple-sqlite3-orm<0.3,>=0.2
+typing-extensions>=4.6.3
+urllib3<2.3,>=2.2.2
+uvicorn[standard]>=0.30,<0.32
+zstandard<0.24,>=0.22

src/otaclient/api_v2/ecu_status.py

@@ -26,7 +26,7 @@
 from otaclient.api_v2._types import convert_status
 from otaclient.app.configs import config as cfg
 from otaclient.app.configs import ecu_info, server_cfg
-from otaclient.configs.ecu_info import ECUContact
+from otaclient.configs import ECUContact
 from otaclient.stats_monitor import OTAClientStatsCollector
 from otaclient_api.v2 import types as api_types
 from otaclient_api.v2.api_caller import ECUNoResponse, OTAClientCall

src/otaclient/app/configs.py

@@ -17,8 +17,10 @@
 from logging import INFO
 from typing import Dict, Tuple
 
-from otaclient.configs.ecu_info import ecu_info  # noqa
-from otaclient.configs.proxy_info import proxy_info  # noqa
+from otaclient.configs.cfg import (
+    ecu_info,  # noqa
+    proxy_info,  # noqa
+)
 
 
 class CreateStandbyMechanism(Enum):

src/otaclient/boot_control/_common.py

@@ -26,12 +26,8 @@
 
 from otaclient._types import OTAStatus
 from otaclient.app.configs import config as cfg
-from otaclient_common.common import (
-    read_str_from_file,
-    subprocess_call,
-    subprocess_check_output,
-    write_str_to_file_sync,
-)
+from otaclient_common.common import subprocess_call, subprocess_check_output
+from otaclient_common._io import read_str_from_file, write_str_to_file_atomic
 from otaclient_common.typing import StrOrPath
 
 logger = logging.getLogger(__name__)
@@ -583,36 +579,36 @@ def _load_slot_in_use_file(self):
     # slot_in_use control
 
     def _store_current_slot_in_use(self, _slot: str):
-        write_str_to_file_sync(
+        write_str_to_file_atomic(
             self.current_ota_status_dir / cfg.SLOT_IN_USE_FNAME, _slot
         )
 
     def _store_standby_slot_in_use(self, _slot: str):
-        write_str_to_file_sync(
+        write_str_to_file_atomic(
             self.standby_ota_status_dir / cfg.SLOT_IN_USE_FNAME, _slot
         )
 
     def _load_current_slot_in_use(self) -> Optional[str]:
         if res := read_str_from_file(
-            self.current_ota_status_dir / cfg.SLOT_IN_USE_FNAME, default=""
+            self.current_ota_status_dir / cfg.SLOT_IN_USE_FNAME, _default=""
         ):
             return res
 
     # status control
 
     def _store_current_status(self, _status: OTAStatus):
-        write_str_to_file_sync(
+        write_str_to_file_atomic(
             self.current_ota_status_dir / cfg.OTA_STATUS_FNAME, _status
         )
 
     def _store_standby_status(self, _status: OTAStatus):
-        write_str_to_file_sync(
+        write_str_to_file_atomic(
             self.standby_ota_status_dir / cfg.OTA_STATUS_FNAME, _status
         )
 
     def _load_current_status(self) -> Optional[OTAStatus]:
         if _status_str := read_str_from_file(
-            self.current_ota_status_dir / cfg.OTA_STATUS_FNAME
+            self.current_ota_status_dir / cfg.OTA_STATUS_FNAME, _default=""
         ).upper():
             with contextlib.suppress(KeyError):
                 # invalid status string
@@ -621,7 +617,7 @@ def _load_current_status(self) -> Optional[OTAStatus]:
     # version control
 
     def _store_standby_version(self, _version: str):
-        write_str_to_file_sync(
+        write_str_to_file_atomic(
             self.standby_ota_status_dir / cfg.OTA_VERSION_FNAME,
             _version,
         )
@@ -680,8 +676,7 @@ def pre_rollback_standby(self):
     def load_active_slot_version(self) -> str:
         return read_str_from_file(
             self.current_ota_status_dir / cfg.OTA_VERSION_FNAME,
-            missing_ok=True,
-            default=cfg.DEFAULT_VERSION_STR,
+            _default=cfg.DEFAULT_VERSION_STR,
         )
 
     def on_failure(self):
@@ -794,4 +789,4 @@ def umount_all(self, *, ignore_error: bool = True):
 
 
 def cat_proc_cmdline(target: str = "/proc/cmdline") -> str:
-    return read_str_from_file(target, missing_ok=False)
+    return read_str_from_file(target, _default="")

src/otaclient/boot_control/_grub.py

@@ -44,13 +44,12 @@
 
 from otaclient import errors as ota_errors
 from otaclient._types import OTAStatus
-from otaclient_common.common import (
-    re_symlink_atomic,
+from otaclient_common._io import (
     read_str_from_file,
-    subprocess_call,
-    subprocess_check_output,
-    write_str_to_file_sync,
+    symlink_atomic,
+    write_str_to_file_atomic,
 )
+from otaclient_common.common import subprocess_call, subprocess_check_output
 
 from ._common import (
     CMDHelperFuncs,
@@ -558,8 +557,8 @@ def _prepare_kernel_initrd_links(target_folder: Path):
         if not (kernel and initrd):
             raise ValueError(f"vmlinuz and/or initrd.img not found at {target_folder}")
 
-        re_symlink_atomic(target_folder / GrubHelper.KERNEL_OTA, kernel)
-        re_symlink_atomic(target_folder / GrubHelper.INITRD_OTA, initrd)
+        symlink_atomic(target_folder / GrubHelper.KERNEL_OTA, kernel)
+        symlink_atomic(target_folder / GrubHelper.INITRD_OTA, initrd)
         logger.info(f"finished generate ota symlinks under {target_folder}")
 
     def _grub_update_on_booted_slot(self, *, abort_on_standby_missed=True):
@@ -588,7 +587,7 @@ def _grub_update_on_booted_slot(self, *, abort_on_standby_missed=True):
         # step1: update grub_default file
         _in = grub_default_file.read_text()
         _out = GrubHelper.update_grub_default(_in)
-        write_str_to_file_sync(grub_default_file, _out)
+        write_str_to_file_atomic(grub_default_file, _out)
 
         # step2: generate grub_cfg by grub-mkconfig
         # parse the output and find the active slot boot entry idx
@@ -612,7 +611,7 @@ def _grub_update_on_booted_slot(self, *, abort_on_standby_missed=True):
             default_entry_idx=active_slot_entry_idx,
         )
         logger.debug(f"generated grub_default: {pformat(_out)}")
-        write_str_to_file_sync(grub_default_file, _out)
+        write_str_to_file_atomic(grub_default_file, _out)
 
         # step4: populate new active grub_file
         # update the ota.standby entry's rootfs uuid to standby slot's uuid
@@ -635,7 +634,7 @@ def _grub_update_on_booted_slot(self, *, abort_on_standby_missed=True):
             kernel_ver=GrubHelper.SUFFIX_OTA_STANDBY,
             rootfs_str=f"root={standby_uuid_str}",
         ):
-            write_str_to_file_sync(active_slot_grub_file, grub_cfg_updated)
+            write_str_to_file_atomic(active_slot_grub_file, grub_cfg_updated)
             logger.info(f"standby rootfs: {standby_uuid_str}")
             logger.debug(f"generated grub_cfg: {pformat(grub_cfg_updated)}")
         else:
@@ -649,10 +648,10 @@ def _grub_update_on_booted_slot(self, *, abort_on_standby_missed=True):
 
             logger.warning(msg)
             logger.info(f"generated grub_cfg: {pformat(grub_cfg_content)}")
-            write_str_to_file_sync(active_slot_grub_file, grub_cfg_content)
+            write_str_to_file_atomic(active_slot_grub_file, grub_cfg_content)
 
         # finally, point grub.cfg to active slot's grub.cfg
-        re_symlink_atomic(  # /boot/grub/grub.cfg -> ../ota-partition/grub.cfg
+        symlink_atomic(  # /boot/grub/grub.cfg -> ../ota-partition/grub.cfg
             self.grub_file,
             Path("../") / cfg.BOOT_OTA_PARTITION_FILE / "grub.cfg",
         )
@@ -662,28 +661,28 @@ def _ensure_ota_partition_symlinks(self, active_slot: str):
         """Ensure /boot/{ota_partition,vmlinuz-ota,initrd.img-ota} symlinks from
         specified <active_slot> point's of view."""
         ota_partition_folder = Path(cfg.BOOT_OTA_PARTITION_FILE)  # ota-partition
-        re_symlink_atomic(  # /boot/ota-partition -> ota-partition.<active_slot>
+        symlink_atomic(  # /boot/ota-partition -> ota-partition.<active_slot>
             self.boot_dir / ota_partition_folder,
             ota_partition_folder.with_suffix(f".{active_slot}"),
         )
-        re_symlink_atomic(  # /boot/vmlinuz-ota -> ota-partition/vmlinuz-ota
+        symlink_atomic(  # /boot/vmlinuz-ota -> ota-partition/vmlinuz-ota
             self.boot_dir / GrubHelper.KERNEL_OTA,
             ota_partition_folder / GrubHelper.KERNEL_OTA,
         )
-        re_symlink_atomic(  # /boot/initrd.img-ota -> ota-partition/initrd.img-ota
+        symlink_atomic(  # /boot/initrd.img-ota -> ota-partition/initrd.img-ota
             self.boot_dir / GrubHelper.INITRD_OTA,
             ota_partition_folder / GrubHelper.INITRD_OTA,
         )
 
     def _ensure_standby_slot_boot_files_symlinks(self, standby_slot: str):
         """Ensure boot files symlinks for specified <standby_slot>."""
         ota_partition_folder = Path(cfg.BOOT_OTA_PARTITION_FILE)  # ota-partition
-        re_symlink_atomic(  # /boot/vmlinuz-ota.standby -> ota-partition.<standby_slot>/vmlinuz-ota
+        symlink_atomic(  # /boot/vmlinuz-ota.standby -> ota-partition.<standby_slot>/vmlinuz-ota
             self.boot_dir / GrubHelper.KERNEL_OTA_STANDBY,
             ota_partition_folder.with_suffix(f".{standby_slot}")
             / GrubHelper.KERNEL_OTA,
         )
-        re_symlink_atomic(  # /boot/initrd.img-ota.standby -> ota-partition.<standby_slot>/initrd.img-ota
+        symlink_atomic(  # /boot/initrd.img-ota.standby -> ota-partition.<standby_slot>/initrd.img-ota
             self.boot_dir / GrubHelper.INITRD_OTA_STANDBY,
             ota_partition_folder.with_suffix(f".{standby_slot}")
             / GrubHelper.INITRD_OTA,
@@ -795,7 +794,7 @@ def _update_fstab(self, *, active_slot_fstab: Path, standby_slot_fstab: Path):
         )
 
         # standby partition fstab (to be merged)
-        fstab_standby = read_str_from_file(standby_slot_fstab, missing_ok=False)
+        fstab_standby = read_str_from_file(standby_slot_fstab)
         fstab_standby_dict: Dict[str, re.Match] = {}
 
         for line in fstab_standby.splitlines():
@@ -805,7 +804,7 @@ def _update_fstab(self, *, active_slot_fstab: Path, standby_slot_fstab: Path):
 
         # merge entries
         merged: List[str] = []
-        fstab_active = read_str_from_file(active_slot_fstab, missing_ok=False)
+        fstab_active = read_str_from_file(active_slot_fstab)
         for line in fstab_active.splitlines():
             if ma := fstab_entry_pa.match(line):
                 mp = ma.group("mount_point")
@@ -827,7 +826,7 @@ def _update_fstab(self, *, active_slot_fstab: Path, standby_slot_fstab: Path):
         merged.append("")  # add a new line at the end of file
 
         # write to standby fstab
-        write_str_to_file_sync(standby_slot_fstab, "\n".join(merged))
+        write_str_to_file_atomic(standby_slot_fstab, "\n".join(merged))
 
     def _cleanup_standby_ota_partition_folder(self):
         """Cleanup old files under the standby ota-partition folder."""