refactor: project restruct phase2 (#311)

Project restruct phase2, in this phase most of the changes are restructring the packages to make the project more modularized and decoupling each components as much as possible.

# Major changes
1. ota_metadata becomes a standalone package, current implementation is treated as the implementation for OTA image format legacy version.
2. create new otaclient_api, current version is v2 and all the protobuf pb2 generated code and proto wrappers are grouped under v2.
3. create new otaclient_common, all common shared libs, helper funcs and types are grouped under this package, categorized by functionality.
4. fix according to import paths changes.
5. restruct tests according to the new project layout.

# Other changes
1. cleanup bootstrap, refactor into samples, which provide sample ecu_info.yaml, proxy_info.yaml and otaclient.service for single ECU setup.
2. fix up tools/offline_ota_image_builder and status_monitor package.
3. not use relative imports anymore.

.gitignore

@@ -159,10 +159,6 @@ cython_debug/
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
 
-# build related
-build
-*.egg-info
-
 # local vscode configs
 .devcontainer
 .vscode

README.md

@@ -1,26 +1,23 @@
-# OTA client
+# OTAClient
 
 ## Overview
 
-This OTA client is a client software to perform over-the-air software updates for linux devices.
-To enable updating of software at any layer (kernel, kernel module, user library, user application), the OTA client targets the entire rootfs for updating.
-When the OTA client receives an update request, it downloads a list from the OTA server that contains the file paths and the hash values of the files, etc., to be updated, and compares them with the files in its own storage and if there is a match, that file is used to update the rootfs. By this delta mechanism, it is possible to reduce the download size even if the entire rootfs is targeted and this mechanism does not require any specific server implementation, nor does it require the server to keep a delta for each version of the rootfs.
+OTAClient is software to perform over-the-air software updates for linux devices.
+It provides a set of APIs for user to start the OTA and monitor the progress and status.
+
+It is designed to work with web.auto FMS OTA component.
 
 ## Feature
 
-- Rootfs updating
-- Delta updating
-- Redundant configuration with A/B partition update
-- Arbitrary files can be copied from A to B partition. This can be used to take over individual files.
-- No specific server implementation is required. The server that supports HTTP GET is only required.
-  - TLS connection is also required.
-- Delta management is not required for server side.
-- To restrict access to the server, cookie can be used.
-- All files to be updated are verified by the hash included in the metadata, and the metadata is also verified by X.509 certificate locally installed.
-- Transfer data is encrypted by TLS
-- Multiple ECU(Electronic Control Unit) support
-- By the internal proxy cache mechanism, the cache can be used for the download requests to the same file from multiple ECU.
+- A/B partition update with support for generic x86_64 device, NVIDIA Jetson series based devices and Raspberry Pi device.
+- Full Rootfs update, with delta update support.
+- Local delta calculation, allowing update to any version of OTA image without the need of a pre-generated delta OTA package.
+- Support persist files from active slot to newly updated slot.
+- Verification over OTA image by digital signature and PKI.
+- Support for protected OTA server with cookie.
+- Optional OTA proxy support and OTA cache support.
+- Multiple ECU OTA supports.
 
 ## License
 
-OTA client is licensed under the Apache License, Version 2.0.
+OTAClient is licensed under the Apache License, Version 2.0.

proto/README.md

@@ -0,0 +1,3 @@
+# OTA Service API proto
+
+This folder includes the OTA service API proto file, and a set of tools to generate the python lib from the proto files.

pyproject.toml

@@ -34,7 +34,7 @@ dependencies = [
   "pyopenssl==24.1",
   "pyyaml>=3.12",
   "requests<2.32,>=2.31",
-  "typing-extensions>=4.6.3; python_version<'3.11'",
+  "typing-extensions>=4.6.3",
   "urllib3<2,>=1.26.8",
   "uvicorn[standard]==0.20",
   "zstandard==0.18",
@@ -60,9 +60,14 @@ version-file = "src/_otaclient_version.py"
 [tool.hatch.build.targets.sdist]
 exclude = [
   "/tools",
+  ".github",
 ]
 
 [tool.hatch.build.targets.wheel]
+exclude = [
+  "**/.gitignore",
+  "**/*README.md",
+]
 only-include = [
   "src",
 ]
@@ -98,9 +103,6 @@ log_auto_indent = true
 log_format = "%(asctime)s %(levelname)s %(filename)s %(funcName)s,%(lineno)d %(message)s"
 log_cli = true
 log_cli_level = "INFO"
-pythonpath = [
-  "otaclient",
-]
 testpaths = [
   "./tests",
 ]
@@ -110,6 +112,10 @@ branch = false
 relative_files = true
 source = [
   "otaclient",
+  "otaclient_api",
+  "otaclient_common",
+  "ota_metadata",
+  "ota_proxy",
 ]
 
 [tool.coverage.report]

samples/README.md

@@ -0,0 +1,3 @@
+# OTAClient configuration files samples
+
+This folder contains the sample otaclient configuration files **ecu_info.yaml**, **proxy_info.yaml** and systemd service unit file **otaclient.service** for a single ECU OTA setup.

samples/ecu_info.yaml

@@ -0,0 +1,7 @@
+# This is the sample ecu_info.yaml for a single x86_64 ECU setup.
+# Please check ecu_info.yaml spec for more details: https://tier4.atlassian.net/l/cp/AGmpqFFc.
+format_version: 1
+ecu_id: autoware
+bootloader: grub
+available_ecu_ids:
+  - autoware

bootstrap/root/etc/systemd/system/otaclient.service → samples/otaclient.service

@@ -1,15 +1,13 @@
-# otaclient.service
-
 [Unit]
 Description=OTA Client
 After=network-online.target nss-lookup.target
 Wants=network-online.target
 
 [Service]
 Type=simple
-ExecStart=/bin/bash -c 'source /opt/ota/.venv/bin/activate && PYTHONPATH=/opt/ota python3 -m otaclient'
+ExecStart=/opt/ota/client/venv/bin/python3 -m otaclient
 Restart=always
-RestartSec=10
+RestartSec=16
 
 [Install]
 WantedBy=multi-user.target

samples/proxy_info.yaml

@@ -0,0 +1,9 @@
+# This is the sample proxy_info.yaml for a single ECU setup.
+# Please check proxy_info.yaml spec for more details: https://tier4.atlassian.net/l/cp/qT4N4K0X.
+format_version: 1
+enable_local_ota_proxy: true
+enable_local_ota_proxy_cache: true
+local_ota_proxy_listen_addr: 127.0.0.1
+local_ota_proxy_listen_port: 8082
+# if otaclient-logger is installed locally
+logging_server: "http://127.0.0.1:8083"

src/ota_metadata/README.md

@@ -0,0 +1,3 @@
+# OTA image metadata
+
+Libs for parsing OTA image.

src/otaclient/app/proto/wrapper.py → src/ota_metadata/legacy/__init__.py

@@ -11,9 +11,24 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Modules for registering wrapped compiled protobuf types."""
+"""OTA image metadata, legacy version."""
 
 
-from ._common import *  # noqa: F403, F401
-from ._ota_metafiles_wrapper import *  # noqa: F403, F401
-from ._otaclient_v2_pb2_wrapper import *  # noqa: F403, F401
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+from otaclient_common import import_from_file
+
+SUPORTED_COMPRESSION_TYPES = ("zst", "zstd")
+
+# ------ dynamically import pb2 generated code ------ #
+
+_PROTO_DIR = Path(__file__).parent
+_PB2_FPATH = _PROTO_DIR / "ota_metafiles_pb2.py"
+_PACKAGE_PREFIX = ".".join(__name__.split(".")[:-1])
+
+_module_name, _module = import_from_file(_PB2_FPATH)
+sys.modules[_module_name] = _module
+sys.modules[f"{_PACKAGE_PREFIX}.{_module_name}"] = _module

src/otaclient/app/ota_metadata.py → src/ota_metadata/legacy/parser.py

@@ -72,18 +72,17 @@
 from typing_extensions import Self
 
 from ota_proxy import OTAFileCacheControl
-
-from .common import RetryTaskMap, get_backoff, urljoin_ensure_base
-from .configs import config as cfg
-from .downloader import Downloader
-from .proto.streamer import Uint32LenDelimitedMsgReader, Uint32LenDelimitedMsgWriter
-from .proto.wrapper import (
-    DirectoryInf,
-    MessageWrapper,
-    PersistentInf,
-    RegularInf,
-    SymbolicLinkInf,
+from otaclient_common.common import get_backoff, urljoin_ensure_base
+from otaclient_common.downloader import Downloader
+from otaclient_common.proto_streamer import (
+    Uint32LenDelimitedMsgReader,
+    Uint32LenDelimitedMsgWriter,
 )
+from otaclient_common.proto_wrapper import MessageWrapper
+from otaclient_common.retry_task_map import RetryTaskMap
+
+from . import SUPORTED_COMPRESSION_TYPES
+from .types import DirectoryInf, PersistentInf, RegularInf, SymbolicLinkInf
 
 logger = logging.getLogger(__name__)
 
@@ -592,10 +591,25 @@ class OTAMetadata:
         ),
     }
 
-    def __init__(self, *, url_base: str, downloader: Downloader) -> None:
+    MAX_COCURRENT = 2
+    BACKOFF_FACTOR = 1
+    BACKOFF_MAX = 6
+
+    def __init__(
+        self,
+        *,
+        url_base: str,
+        downloader: Downloader,
+        run_dir: Path,
+        certs_dir: Path,
+        download_max_idle_time: int,
+    ) -> None:
         self.url_base = url_base
         self._downloader = downloader
-        self._tmp_dir = TemporaryDirectory(prefix="ota_metadata", dir=cfg.RUN_DIR)
+        self.run_dir = run_dir
+        self.certs_dir = certs_dir
+        self.download_max_idle_time = download_max_idle_time
+        self._tmp_dir = TemporaryDirectory(prefix="ota_metadata", dir=run_dir)
         self._tmp_dir_path = Path(self._tmp_dir.name)
 
         # download and parse the metadata.jwt
@@ -622,7 +636,7 @@ def _process_metadata_jwt(self) -> _MetadataJWTClaimsLayout:
         """Download, loading and parsing metadata.jwt."""
         logger.debug("process metadata.jwt...")
         # download and parse metadata.jwt
-        with NamedTemporaryFile(prefix="metadata_jwt", dir=cfg.RUN_DIR) as meta_f:
+        with NamedTemporaryFile(prefix="metadata_jwt", dir=self.run_dir) as meta_f:
             _downloaded_meta_f = Path(meta_f.name)
             self._downloader.download_retry_inf(
                 urljoin_ensure_base(self.url_base, self.METADATA_JWT),
@@ -636,13 +650,13 @@ def _process_metadata_jwt(self) -> _MetadataJWTClaimsLayout:
             )
 
             _parser = _MetadataJWTParser(
-                _downloaded_meta_f.read_text(), certs_dir=cfg.CERTS_DIR
+                _downloaded_meta_f.read_text(), certs_dir=self.certs_dir
             )
         # get not yet verified parsed ota_metadata
         _ota_metadata = _parser.get_otametadata()
 
         # download certificate and verify metadata against this certificate
-        with NamedTemporaryFile(prefix="metadata_cert", dir=cfg.RUN_DIR) as cert_f:
+        with NamedTemporaryFile(prefix="metadata_cert", dir=self.run_dir) as cert_f:
             cert_info = _ota_metadata.certificate
             cert_fname, cert_hash = cert_info.file, cert_info.hash
             cert_file = Path(cert_f.name)
@@ -696,11 +710,11 @@ def _process_text_base_otameta_file(_metafile: MetaFile):
 
         last_active_timestamp = int(time.time())
         _mapper = RetryTaskMap(
-            max_concurrent=cfg.MAX_CONCURRENT_DOWNLOAD_TASKS,
+            max_concurrent=self.MAX_COCURRENT,
             backoff_func=partial(
                 get_backoff,
-                factor=cfg.DOWNLOAD_GROUP_BACKOFF_FACTOR,
-                _max=cfg.DOWNLOAD_GROUP_BACKOFF_MAX,
+                factor=self.BACKOFF_FACTOR,
+                _max=self.BACKOFF_MAX,
             ),
             max_retry=0,  # NOTE: we use another strategy below
         )
@@ -718,12 +732,9 @@ def _process_text_base_otameta_file(_metafile: MetaFile):
             last_active_timestamp = max(
                 last_active_timestamp, self._downloader.last_active_timestamp
             )
-            if (
-                int(time.time()) - last_active_timestamp
-                > cfg.DOWNLOAD_GROUP_INACTIVE_TIMEOUT
-            ):
+            if int(time.time()) - last_active_timestamp > self.download_max_idle_time:
                 logger.error(
-                    f"downloader becomes stuck for {cfg.DOWNLOAD_GROUP_INACTIVE_TIMEOUT=} seconds, abort"
+                    f"downloader becomes stuck for {self.download_max_idle_time=} seconds, abort"
                 )
                 _mapper.shutdown(raise_last_exc=True)
 
@@ -753,7 +764,7 @@ def get_download_url(self, reg_inf: RegularInf) -> Tuple[str, Optional[str]]:
         if (
             self.image_compressed_rootfs_url
             and reg_inf.compressed_alg
-            and reg_inf.compressed_alg in cfg.SUPPORTED_COMPRESS_ALG
+            and reg_inf.compressed_alg in SUPORTED_COMPRESSION_TYPES
         ):
             return (
                 urljoin_ensure_base(

src/otaclient/app/proto/_ota_metafiles_wrapper.py → src/ota_metadata/legacy/types.py

@@ -20,9 +20,8 @@
 from pathlib import Path
 from typing import Union
 
-import ota_metafiles_pb2 as ota_metafiles
-
-from ._common import MessageWrapper, calculate_slots
+from ota_metadata.legacy import ota_metafiles_pb2 as ota_metafiles
+from otaclient_common.proto_wrapper import MessageWrapper, calculate_slots
 
 # helper mixin
 

src/ota_proxy/cache_control.py

@@ -18,7 +18,7 @@
 
 from typing_extensions import Self
 
-from otaclient._utils import copy_callable_typehint_to_method
+from otaclient_common.typing import copy_callable_typehint_to_method
 
 _FIELDS = "_fields"
 

src/ota_proxy/server_app.py

@@ -22,7 +22,7 @@
 
 import aiohttp
 
-from otaclient._utils.logging import BurstSuppressFilter
+from otaclient_common.logging import BurstSuppressFilter
 
 from ._consts import (
     BHEADER_AUTHORIZATION,