Merge remote-tracking branch 'origin/main' into refactor/rpi_faster

.github/workflows/test.yaml

@@ -8,7 +8,7 @@ on:
     branches:
       - main
     paths:
-      - "otaclient/**"
+      - "src/**"
       - "tests/**"
       - ".github/workflows/test.yaml"
   # allow the test CI to be manually triggerred
@@ -20,7 +20,7 @@ jobs:
     timeout-minutes: 20
     steps:
       - name: Checkout commit
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         # sonarcloud needs full git histories
         with:
           fetch-depth: 0

.gitignore

@@ -159,10 +159,6 @@ cython_debug/
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
 
-# build related
-build
-*.egg-info
-
 # local vscode configs
 .devcontainer
 .vscode

.pre-commit-config.yaml

@@ -27,7 +27,7 @@ repos:
           - flake8-comprehensions
           - flake8-simplify
   - repo: https://github.com/tox-dev/pyproject-fmt
-    rev: "1.8.0"
+    rev: "2.1.3"
     hooks:
       - id: pyproject-fmt
         # https://pyproject-fmt.readthedocs.io/en/latest/#calculating-max-supported-python-version
@@ -39,7 +39,7 @@ repos:
   #       additional_dependencies:
   #         - tomli
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.40.0
+    rev: v0.41.0
     hooks:
       - id: markdownlint
         args: ["-c", ".markdownlint.yaml", "--fix"]

README.md

@@ -1,26 +1,23 @@
-# OTA client
+# OTAClient
 
 ## Overview
 
-This OTA client is a client software to perform over-the-air software updates for linux devices.
-To enable updating of software at any layer (kernel, kernel module, user library, user application), the OTA client targets the entire rootfs for updating.
-When the OTA client receives an update request, it downloads a list from the OTA server that contains the file paths and the hash values of the files, etc., to be updated, and compares them with the files in its own storage and if there is a match, that file is used to update the rootfs. By this delta mechanism, it is possible to reduce the download size even if the entire rootfs is targeted and this mechanism does not require any specific server implementation, nor does it require the server to keep a delta for each version of the rootfs.
+OTAClient is software to perform over-the-air software updates for linux devices.
+It provides a set of APIs for user to start the OTA and monitor the progress and status.
+
+It is designed to work with web.auto FMS OTA component.
 
 ## Feature
 
-- Rootfs updating
-- Delta updating
-- Redundant configuration with A/B partition update
-- Arbitrary files can be copied from A to B partition. This can be used to take over individual files.
-- No specific server implementation is required. The server that supports HTTP GET is only required.
-  - TLS connection is also required.
-- Delta management is not required for server side.
-- To restrict access to the server, cookie can be used.
-- All files to be updated are verified by the hash included in the metadata, and the metadata is also verified by X.509 certificate locally installed.
-- Transfer data is encrypted by TLS
-- Multiple ECU(Electronic Control Unit) support
-- By the internal proxy cache mechanism, the cache can be used for the download requests to the same file from multiple ECU.
+- A/B partition update with support for generic x86_64 device, NVIDIA Jetson series based devices and Raspberry Pi device.
+- Full Rootfs update, with delta update support.
+- Local delta calculation, allowing update to any version of OTA image without the need of a pre-generated delta OTA package.
+- Support persist files from active slot to newly updated slot.
+- Verification over OTA image by digital signature and PKI.
+- Support for protected OTA server with cookie.
+- Optional OTA proxy support and OTA cache support.
+- Multiple ECU OTA supports.
 
 ## License
 
-OTA client is licensed under the Apache License, Version 2.0.
+OTAClient is licensed under the Apache License, Version 2.0.

proto/README.md

@@ -0,0 +1,3 @@
+# OTA Service API proto
+
+This folder includes the OTA service API proto file, and a set of tools to generate the python lib from the proto files.

pyproject.toml

@@ -25,22 +25,21 @@ dynamic = [
 ]
 dependencies = [
   "aiofiles==22.1",
-  "aiohttp<3.10.0,>=3.9.5",
-  "cryptography<43.0.0,>=42.0.4",
-  "grpcio<1.54.0,>=1.53.2",
-  "protobuf<4.22.0,>=4.21.12",
+  "aiohttp<3.10,>=3.9.5",
+  "cryptography<43,>=42.0.4",
+  "grpcio<1.54,>=1.53.2",
+  "protobuf<4.22,>=4.21.12",
   "pydantic==2.7",
   "pydantic-settings==2.2.1",
-  "pyOpenSSL==24.1",
-  "PyYAML>=3.12",
-  "requests<2.32.0,>=2.31",
-  'typing_extensions>=4.6.3; python_version < "3.11"',
-  "urllib3<2.0.0,>=1.26.8",
+  "pyopenssl==24.1",
+  "pyyaml>=3.12",
+  "requests<2.32,>=2.31",
+  "typing-extensions>=4.6.3",
+  "urllib3<2,>=1.26.8",
   "uvicorn[standard]==0.20",
   "zstandard==0.18",
 ]
-[project.optional-dependencies]
-dev = [
+optional-dependencies.dev = [
   "black",
   "coverage",
   "flake8",
@@ -50,8 +49,7 @@ dev = [
   "pytest-mock==3.8.2",
   "requests-mock",
 ]
-[project.urls]
-Source = "https://github.com/tier4/ota-client"
+urls.Source = "https://github.com/tier4/ota-client"
 
 [tool.hatch.version]
 source = "vcs"
@@ -60,43 +58,71 @@ source = "vcs"
 version-file = "src/_otaclient_version.py"
 
 [tool.hatch.build.targets.sdist]
-exclude = ["/tools"]
+exclude = [
+  "/tools",
+  ".github",
+]
 
 [tool.hatch.build.targets.wheel]
-only-include = ["src"]
-sources = ["src"]
+exclude = [
+  "**/.gitignore",
+  "**/*README.md",
+]
+only-include = [
+  "src",
+]
+sources = [
+  "src",
+]
 
 [tool.hatch.envs.dev]
 type = "virtual"
-features = ["dev"]
+features = [
+  "dev",
+]
 
 [tool.black]
 line-length = 88
-target-version = ['py38']
+target-version = [
+  'py38',
+]
 extend-exclude = '''(
   ^.*(_pb2.pyi?|_pb2_grpc.pyi?)$
 )'''
 
 [tool.isort]
 profile = "black"
-extend_skip_glob = ["*_pb2.py*", "_pb2_grpc.py*"]
+extend_skip_glob = [
+  "*_pb2.py*",
+  "_pb2_grpc.py*",
+]
 
 [tool.pytest.ini_options]
 asyncio_mode = "auto"
 log_auto_indent = true
 log_format = "%(asctime)s %(levelname)s %(filename)s %(funcName)s,%(lineno)d %(message)s"
 log_cli = true
 log_cli_level = "INFO"
-pythonpath = ["otaclient"]
-testpaths = ["./tests"]
+testpaths = [
+  "./tests",
+]
 
 [tool.coverage.run]
 branch = false
 relative_files = true
-source = ["otaclient"]
+source = [
+  "otaclient",
+  "otaclient_api",
+  "otaclient_common",
+  "ota_metadata",
+  "ota_proxy",
+]
 
 [tool.coverage.report]
-omit = ["**/*_pb2.py*", "**/*_pb2_grpc.py*"]
+omit = [
+  "**/*_pb2.py*",
+  "**/*_pb2_grpc.py*",
+]
 exclude_also = [
   "def __repr__",
   "if __name__ == .__main__.:",
@@ -108,6 +134,11 @@ skip_empty = true
 skip_covered = true
 
 [tool.pyright]
-exclude = ["**/__pycache__"]
-ignore = ["**/*_pb2.py*", "**/*_pb2_grpc.py*"]
+exclude = [
+  "**/__pycache__",
+]
+ignore = [
+  "**/*_pb2.py*",
+  "**/*_pb2_grpc.py*",
+]
 pythonVersion = "3.8"

samples/README.md

@@ -0,0 +1,3 @@
+# OTAClient configuration files samples
+
+This folder contains the sample otaclient configuration files **ecu_info.yaml**, **proxy_info.yaml** and systemd service unit file **otaclient.service** for a single ECU OTA setup.

samples/ecu_info.yaml

@@ -0,0 +1,7 @@
+# This is the sample ecu_info.yaml for a single x86_64 ECU setup.
+# Please check ecu_info.yaml spec for more details: https://tier4.atlassian.net/l/cp/AGmpqFFc.
+format_version: 1
+ecu_id: autoware
+bootloader: grub
+available_ecu_ids:
+  - autoware

bootstrap/root/etc/systemd/system/otaclient.service → samples/otaclient.service

@@ -1,15 +1,13 @@
-# otaclient.service
-
 [Unit]
 Description=OTA Client
 After=network-online.target nss-lookup.target
 Wants=network-online.target
 
 [Service]
 Type=simple
-ExecStart=/bin/bash -c 'source /opt/ota/.venv/bin/activate && PYTHONPATH=/opt/ota python3 -m otaclient'
+ExecStart=/opt/ota/client/venv/bin/python3 -m otaclient
 Restart=always
-RestartSec=10
+RestartSec=16
 
 [Install]
 WantedBy=multi-user.target

samples/proxy_info.yaml

@@ -0,0 +1,9 @@
+# This is the sample proxy_info.yaml for a single ECU setup.
+# Please check proxy_info.yaml spec for more details: https://tier4.atlassian.net/l/cp/qT4N4K0X.
+format_version: 1
+enable_local_ota_proxy: true
+enable_local_ota_proxy_cache: true
+local_ota_proxy_listen_addr: 127.0.0.1
+local_ota_proxy_listen_port: 8082
+# if otaclient-logger is installed locally
+logging_server: "http://127.0.0.1:8083"

sonar-project.properties

@@ -1,7 +1,7 @@
 sonar.organization=tier4
 sonar.projectKey=tier4_ota-client
 sonar.python.coverage.reportPaths=test_result/coverage.xml
-sonar.sources=./otaclient
+sonar.sources=./src
 sonar.tests=tests
 sonar.sourceEncoding=UTF-8
 sonar.python.version=3.8,3.9,3.10,3.11

src/ota_metadata/README.md

@@ -0,0 +1,3 @@
+# OTA image metadata
+
+Libs for parsing OTA image.

tools/emulator/path_loader.py → src/ota_metadata/legacy/__init__.py

@@ -11,19 +11,24 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+"""OTA image metadata, legacy version."""
 
 
-# NOTE: this file should only be loaded once by the program entry!
+from __future__ import annotations
 
+import sys
+from pathlib import Path
 
-###### load path ######
-def _path_load():
-    import sys
-    from pathlib import Path
+from otaclient_common import import_from_file
 
-    project_base = Path(__file__).absolute().parent.parent
-    sys.path.extend([str(project_base), str(project_base / "app")])
+SUPORTED_COMPRESSION_TYPES = ("zst", "zstd")
 
+# ------ dynamically import pb2 generated code ------ #
 
-_path_load()
-######
+_PROTO_DIR = Path(__file__).parent
+_PB2_FPATH = _PROTO_DIR / "ota_metafiles_pb2.py"
+_PACKAGE_PREFIX = ".".join(__name__.split(".")[:-1])
+
+_module_name, _module = import_from_file(_PB2_FPATH)
+sys.modules[_module_name] = _module
+sys.modules[f"{_PACKAGE_PREFIX}.{_module_name}"] = _module