From e3e573fa20cd1a8f791d73fce97429b29dd341c2 Mon Sep 17 00:00:00 2001 From: Bodong YANG Date: Mon, 25 Dec 2023 02:40:32 +0000 Subject: [PATCH 01/16] tools: rename offline_ota_image_builder to image_packer --- tools/images_packer/README.md | 175 ++++++++++++++++++++++++ tools/images_packer/__init__.py | 13 ++ tools/images_packer/__main__.py | 166 ++++++++++++++++++++++ tools/images_packer/builder.py | 234 ++++++++++++++++++++++++++++++++ tools/images_packer/configs.py | 33 +++++ tools/images_packer/manifest.py | 48 +++++++ tools/images_packer/utils.py | 29 ++++ 7 files changed, 698 insertions(+) create mode 100644 tools/images_packer/README.md create mode 100644 tools/images_packer/__init__.py create mode 100644 tools/images_packer/__main__.py create mode 100644 tools/images_packer/builder.py create mode 100644 tools/images_packer/configs.py create mode 100644 tools/images_packer/manifest.py create mode 100644 tools/images_packer/utils.py diff --git a/tools/images_packer/README.md b/tools/images_packer/README.md new file mode 100644 index 000000000..b61d5970e --- /dev/null +++ b/tools/images_packer/README.md @@ -0,0 +1,175 @@ +# Offline OTA image builder + +`tools.offline_ota_image_builder` is a helper package for bundling offline OTA image from the OTA images build by ota-metadata(or images compatible with OTA image specification). +Built image can be used as external cache source for otaproxy, or used by full offline OTA helper util to trigger fully offline OTA update. + +**NOTE**: full offline OTA helper util is not yet implemented. + +It provides the following features: + +1. build offline OTA image from the OTA images built by ota-metadata(or images compatible with OTA image specification), +2. export built offline OTA image as tar archive, +3. prepare and export built image onto specific block device as external cache source device for otaproxy. + + + +## Expected image layout for input OTA images + +This package currently only recognizes the OTA images generated by [ota-metadata@a711013](https://github.com/tier4/ota-metadata/commit/a711013), or the image layout compatible with the ota-metadata and OTA image layout specification as follow. + +```text +Example OTA image layout: +. +├── data +│ ├── <...files with full paths...> +│ └── ... +├── data.zst +│ ├── <... compressed OTA file ...> +│ └── <... naming: .zst ...> +├── certificate.pem +├── dirs.txt +├── metadata.jwt +├── persistents.txt +├── regulars.txt +└── symlinks.txt +``` + +Please also refere to [ota-metadata](https://github.com/tier4/ota-metadata) repo for the implementation of OTA image layout specification. + +## Offline OTA image layout + +The rootfs of the built image has the following layout: + +```text +. +├── manifest.json +├── data +│ ├── # uncompressed file +│ ├── .zst # zst compressed file +│ └── ... +└── meta + ├── # corresponding to the order in manifest.image_meta + │ └── + └── ... + └── + +``` + +This layout is compatible with external cache source's layout, which means any offline OTA image can natually directly be used as external cache source recognized by otaproxy(but user still needs to prepare the device either by themselves and extract the offline OTA image rootfs onto the prepared device, or prepare the device with this **offline_ota_image_builder** package). + + + +## Offline OTA image Manifest schema + +Offline OTA image's rootfs contains a `manifest.json` file with a single JSON object in it. This file includes the information related to this offline OTA image, including the bundled OTA images' metadata. + +```json +{ + "schema_version": 1, + "image_layout_version": 1, + "build_timestamp": 1693291679, + "data_size": 1122334455, + "data_files_num": 12345, + "meta_size": 112233, + "image_meta": [ + { + "ecu_id": "", + "image_version": "", + "ota_metadata_version": 1, + }, + ... + ], +} +``` + + + +## How to use + +### Prepare the dependencies + +This image builder requires latest ota-client to be installed/accessable. The recommended way to install dependencies is to init a virtual environment and install the ota-client into this venv, and then execute the image_builder with this venv. + +1. git clone the latest ota-client repository: + + ```bash + $ git clone https://github.com/tier4/ota-client.git + # image builder is located at ota-client/tools/offline_ota_image_builder + ``` + +2. prepare the virtual environment and activate it: + + ```bash + $ python3 -m venv venv + $ . venv/bin/active + (venv) $ + ``` + +3. install the ota-client into the virtual environment: + + ```bash + (venv) $ pip install ota-client + ``` + +### Build image and export + +#### Builder usage + +```text +usage: offline_ota_image_builder [-h] --image :[:] + [-o ] [-w ] [--confirm-write-to] + +Helper script that builds offline OTA image with given OTA image(s) as external cache source or for +offline OTA use. + +options: + -h, --help show this help message and exit + --image :[:] + OTA image for as tar archive(compressed or uncompressed), this + option can be used multiple times to include multiple images. + -o , --output + save the generated image rootfs into tar archive to . + -w , --write-to + write the image to and prepare the device as external cache source + device. + --force-write-to prepare as external cache source device without inter-active confirmation, + only valid when used with -w option. +``` + +Execute the image builder by directly calling it from the source code. This package requires `root` permission to run(required by extracting OTA image and preparing external cache source device). + +Option `--image :[:]` is used to specify OTA image to be included. This option can be used multiple times to specify multiple OTA images. + +Option `--write-to ` is used to prepare external cache source device used by otaproxy. The specified device will be formatted as `ext4`, fslabel with `ota_cache_src`, and be exported with the built offline OTA image's rootfs. If this package is used in a non-interactive script, option `--force-write-to` can be used to bypass interactive confirmation. + +Option `--output ` specify where to save the exported tar archive of built image. + +User must at least specifies one of `--write-to` and `--output`, or specifies them together. + +#### Usage 1: Build offline OTA image and export it as tar archive + +```bash +# current folder layout: venv ota-client +(venv) $ cd ota-client +(venv) $ sudo -E env PATH=$PATH python3 -m tools.offline_ota_image_builder --image=p1:p1_image.tgz:ver_20230808 --image=p2:p2_image.tgz:ver_20230808 --output=t2.tar +``` + +This will build the offline OTA image with `p1_image.tgz` and `p2_image.tgz`, which are for ECU `p1` and `p2`, and export the built image as `t2.tar` tar archive. + +### Usage 2: Build the offline OTA image and create external cache source dev + +```bash +# current folder layout: venv ota-client +(venv) $ cd ota-client +(venv) $ sudo -E env PATH=$PATH python3 -m tools.offline_ota_image_builder --image=p1:p1_image.tgz:ver_20230808 --image=p2:p2_image.tgz:ver_20230808 --write-to=/dev/ +``` + +This will build the offline OTA image with `p1_image.tgz` and `p2_image.tgz`, which are for ECU `p1` and `p2`, and then prepare the `/dev/` as external cache source device(ext4 filesystem labelled with `ota_cache_src`) with image rootfs exported to the filesystem on `/dev/`. + +### Usage 3: Build the offline OTA image, export it as tar archive and prepare external cache source dev + +```bash +# current folder layout: venv ota-client +(venv) $ cd ota-client +(venv) $ sudo -E env PATH=$PATH python3 -m tools.offline_ota_image_builder --image=p1:p1_image.tgz:ver_20230808 --image=p2:p2_image.tgz:ver_20230808 --output=t2.tar --write-to=/dev/ +``` diff --git a/tools/images_packer/__init__.py b/tools/images_packer/__init__.py new file mode 100644 index 000000000..bcfd866ad --- /dev/null +++ b/tools/images_packer/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022 TIER IV, INC. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tools/images_packer/__main__.py b/tools/images_packer/__main__.py new file mode 100644 index 000000000..5e316b471 --- /dev/null +++ b/tools/images_packer/__main__.py @@ -0,0 +1,166 @@ +# Copyright 2022 TIER IV, INC. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""External cache source builder helper. + +Please refere to https://github.com/tier4/ota-metadata for +OTA image layout specification. +This tool is implemented based on ota-metadata@a711013. + +Example OTA image layout: +. +├── data +│ ├── <...files with full paths...> +│ └── ... +├── data.zst +│ ├── <... compressed OTA file ...> +│ └── <... naming: .zst ...> +├── certificate.pem +├── dirs.txt +├── metadata.jwt +├── persistents.txt +├── regulars.txt +└── symlinks.txt + +Check README.md for the offline OTA image layout specification. +Please refer to OTA cache design doc for more details. +""" + + +import argparse +import errno +import logging +import sys +import tempfile +from pathlib import Path + +from .configs import cfg +from .builder import build +from .manifest import ImageMetadata + +logger = logging.getLogger(__name__) + + +def main(args): + # ------ parse input image options ------ # + image_metas = [] + image_files = {} + + for raw_pair in args.image: + _parsed = str(raw_pair).split(":") + if len(_parsed) >= 2: + _ecu_id, _image_fpath, _image_version, *_ = *_parsed, "" + image_metas.append( + ImageMetadata( + ecu_id=_ecu_id, + image_version=_image_version, + ) + ) + image_files[_ecu_id] = _image_fpath + else: + logger.warning(f"ignore illegal image pair: {raw_pair}") + + if not image_metas: + print("ERR: at least one valid image should be given, abort") + sys.exit(errno.EINVAL) + + # ------ parse export options ------ # + output_fpath, write_to_dev = args.output, args.write_to + + if write_to_dev and not Path(write_to_dev).is_block_device(): + print(f"{write_to_dev} is not a block device, abort") + sys.exit(errno.EINVAL) + + if write_to_dev and not args.force_write_to: + _confirm_write_to = input( + f"WARNING: generated image will be written to {write_to_dev}, \n" + f"\t all data in {write_to_dev} will be lost, continue(type [Y] or [N]): " + ) + if _confirm_write_to != "Y": + print(f"decline writing to {write_to_dev}, abort") + sys.exit(errno.EINVAL) + elif write_to_dev and args.force_write_to: + logger.warning( + f"generated image will be written to {write_to_dev}," + f"all data in {write_to_dev} will be lost" + ) + + # ------ build image ------ # + with tempfile.TemporaryDirectory(prefix="offline_OTA_image_builder") as workdir: + build( + image_metas, + image_files, + workdir=workdir, + output=output_fpath, + write_to_dev=write_to_dev, + ) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format=cfg.LOGGING_FORMAT, force=True) + parser = argparse.ArgumentParser( + prog="offline_ota_image_builder", + description=( + "Helper script that builds offline OTA image " + "with given OTA image(s) as external cache source or for offline OTA use." + ), + ) + parser.add_argument( + "--image", + help=( + "OTA image for as tar archive(compressed or uncompressed), " + "this option can be used multiple times to include multiple images." + ), + required=True, + metavar=":[:]", + action="append", + ) + parser.add_argument( + "-o", + "--output", + help="save the generated image rootfs into tar archive to .", + metavar="", + ) + parser.add_argument( + "-w", + "--write-to", + help=( + "write the image to and prepare the device as " + "external cache source device." + ), + metavar="", + ) + parser.add_argument( + "--force-write-to", + help=( + "prepare as external cache source device without inter-active confirmation, " + "only valid when used with -w option." + ), + action="store_true", + ) + args = parser.parse_args() + + # basic options check + if args.output and (output := Path(args.output)).is_file(): + print(f"ERR: {output} exists, abort") + sys.exit(errno.EINVAL) + + if not (args.output or args.write_to): + print("ERR: at least one export option should be specified") + sys.exit(errno.EINVAL) + + try: + main(args) + except KeyboardInterrupt: + print("ERR: aborted by user") + raise diff --git a/tools/images_packer/builder.py b/tools/images_packer/builder.py new file mode 100644 index 000000000..42c77d7f5 --- /dev/null +++ b/tools/images_packer/builder.py @@ -0,0 +1,234 @@ +# Copyright 2022 TIER IV, INC. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging +import os +import pprint +import shutil +import tempfile +import time +from contextlib import contextmanager +from pathlib import Path +from typing import Mapping, Optional, Sequence + +from otaclient.app.common import subprocess_call +from otaclient.app import ota_metadata +from .configs import cfg +from .manifest import ImageMetadata, Manifest +from .utils import StrPath, InputImageProcessError, ExportError + +logger = logging.getLogger(__name__) + +PROC_MOUNTS = "/proc/mounts" + + +def _check_if_mounted(dev: StrPath): + for line in Path(PROC_MOUNTS).read_text().splitlines(): + if line.find(str(dev)) != -1: + return True + return False + + +@contextmanager +def _unarchive_image(image_fpath: StrPath, *, workdir: StrPath): + _start_time = time.time() + with tempfile.TemporaryDirectory(dir=workdir) as unarchive_dir: + cmd = f"tar xf {image_fpath} -C {unarchive_dir}" + try: + subprocess_call(cmd) + except Exception as e: + _err_msg = f"failed to process input image {image_fpath}: {e!r}, {cmd=}" + logger.error(_err_msg) + raise InputImageProcessError(_err_msg) from e + + logger.info( + f"finish unarchiving {image_fpath}, takes {time.time() - _start_time:.2f}s" + ) + yield unarchive_dir + + +def _process_ota_image(ota_image_dir: StrPath, *, data_dir: StrPath, meta_dir: StrPath): + """Processing OTA image under and update and .""" + _start_time = time.time() + data_dir = Path(data_dir) + # statistics + saved_files, saved_files_size = 0, 0 + + ota_image_dir = Path(ota_image_dir) + + # ------ process OTA image metadata ------ # + metadata_jwt_fpath = ota_image_dir / ota_metadata.OTAMetadata.METADATA_JWT + # NOTE: we don't need to do certificate verification here, so set certs_dir to empty + metadata_jwt = ota_metadata._MetadataJWTParser( + metadata_jwt_fpath.read_text(), certs_dir="" + ).get_otametadata() + + rootfs_dir = ota_image_dir / metadata_jwt.rootfs_directory + compressed_rootfs_dir = ota_image_dir / metadata_jwt.compressed_rootfs_directory + + # ------ update data_dir with the contents of this OTA image ------ # + with open(ota_image_dir / metadata_jwt.regular.file, "r") as f: + for line in f: + reg_inf = ota_metadata.parse_regulars_from_txt(line) + ota_file_sha256 = reg_inf.sha256hash.hex() + + if reg_inf.compressed_alg == cfg.OTA_IMAGE_COMPRESSION_ALG: + _zst_ota_fname = f"{ota_file_sha256}.{cfg.OTA_IMAGE_COMPRESSION_ALG}" + src = compressed_rootfs_dir / _zst_ota_fname + dst = data_dir / _zst_ota_fname + # NOTE: multiple OTA files might have the same hash + # NOTE: squash OTA file permission before moving + if not dst.is_file() and src.is_file(): + src.chmod(0o644) + saved_files += 1 + shutil.move(str(src), dst) + saved_files_size += dst.stat().st_size + + else: + src = rootfs_dir / os.path.relpath(reg_inf.path, "/") + dst = data_dir / ota_file_sha256 + if not dst.is_file() and src.is_file(): + src.chmod(0o644) + saved_files += 1 + shutil.move(str(src), dst) + saved_files_size += dst.stat().st_size + + # ------ update meta_dir with the OTA meta files in this image ------ # + # copy OTA metafiles to image specific meta folder + for _metaf in ota_metadata.MetafilesV1: + shutil.move(str(ota_image_dir / _metaf.value), meta_dir) + # copy certificate and metadata.jwt + shutil.move(str(ota_image_dir / metadata_jwt.certificate.file), meta_dir) + shutil.move(str(metadata_jwt_fpath), meta_dir) + + logger.info(f"finish processing OTA image, takes {time.time() - _start_time:.2f}s") + return saved_files, saved_files_size + + +def _create_image_tar(image_rootfs: StrPath, output_fpath: StrPath): + """Export generated image rootfs as tar ball.""" + cmd = f"tar cf {output_fpath} -C {image_rootfs} ." + try: + logger.info(f"exporting external cache source image to {output_fpath} ...") + subprocess_call(cmd) + except Exception as e: + _err_msg = f"failed to tar generated image to {output_fpath=}: {e!r}, {cmd=}" + logger.error(_err_msg) + raise ExportError(_err_msg) from e + + +def _write_image_to_dev(image_rootfs: StrPath, dev: StrPath, *, workdir: StrPath): + """Prepare and export generated image's rootfs to it.""" + _start_time = time.time() + dev = Path(dev) + if not dev.is_block_device(): + logger.warning(f"{dev=} is not a block device, skip") + return + if _check_if_mounted(dev): + logger.warning(f"{dev=} is mounted, skip") + return + + # prepare device + format_device_cmd = f"mkfs.ext4 -L {cfg.EXTERNAL_CACHE_DEV_FSLABEL} {dev}" + try: + logger.warning(f"formatting {dev} to ext4: {format_device_cmd}") + # NOTE: label offline OTA image as external cache source + subprocess_call(format_device_cmd) + except Exception as e: + _err_msg = f"failed to formatting {dev} to ext4: {e!r}" + logger.error(_err_msg) + raise ExportError(_err_msg) from e + + # mount and copy + mount_point = Path(workdir) / "mnt" + mount_point.mkdir(exist_ok=True) + cp_cmd = f"cp -rT {image_rootfs} {mount_point}" + try: + logger.info(f"copying image rootfs to {dev=}@{mount_point=}...") + subprocess_call(f"mount --make-private --make-unbindable {dev} {mount_point}") + subprocess_call(cp_cmd) + logger.info(f"finish copying, takes {time.time()-_start_time:.2f}s") + except Exception as e: + _err_msg = f"failed to export to image rootfs to {dev=}@{mount_point=}: {e!r}, {cp_cmd=}" + logger.error(_err_msg) + raise ExportError(_err_msg) from e + finally: + subprocess_call(f"umount -l {dev}", raise_exception=False) + + +def build( + image_metas: Sequence[ImageMetadata], + image_files: Mapping[str, StrPath], + *, + workdir: StrPath, + output: Optional[StrPath], + write_to_dev: Optional[StrPath], +): + _start_time = time.time() + logger.info(f"job started at {int(_start_time)}") + + output_workdir = Path(workdir) / cfg.OUTPUT_WORKDIR + output_workdir.mkdir(parents=True, exist_ok=True) + output_data_dir = output_workdir / cfg.OUTPUT_DATA_DIR + output_data_dir.mkdir(exist_ok=True, parents=True) + output_meta_dir = output_workdir / cfg.OUTPUT_META_DIR + output_meta_dir.mkdir(exist_ok=True, parents=True) + + # ------ generate image ------ # + manifest = Manifest(image_meta=list(image_metas)) + # process all inpu OTA images + images_unarchiving_work_dir = Path(workdir) / cfg.IMAGE_UNARCHIVE_WORKDIR + images_unarchiving_work_dir.mkdir(parents=True, exist_ok=True) + for idx, image_meta in enumerate(manifest.image_meta): + ecu_id = image_meta.ecu_id + image_file = image_files[ecu_id] + # image specific metadata dir + _meta_dir = output_meta_dir / str(idx) + _meta_dir.mkdir(exist_ok=True, parents=True) + + logger.info(f"{ecu_id=}: unarchive {image_file} ...") + with _unarchive_image( + image_file, workdir=images_unarchiving_work_dir + ) as unarchived_image_dir: + logger.info(f"{ecu_id=}: processing OTA image ...") + _saved_files_num, _saved_files_size = _process_ota_image( + unarchived_image_dir, + data_dir=output_data_dir, + meta_dir=_meta_dir, + ) + # update manifest after this image is processed + manifest.data_size += _saved_files_size + manifest.data_files_num += _saved_files_num + + # process metadata + for cur_path, _, fnames in os.walk(output_meta_dir): + _cur_path = Path(cur_path) + for _fname in fnames: + _fpath = _cur_path / _fname + if _fpath.is_file(): + manifest.meta_size += _fpath.stat().st_size + + # write offline OTA image manifest + manifest.build_timestamp = int(time.time()) + Path(output_workdir / cfg.MANIFEST_JSON).write_text(manifest.export_to_json()) + logger.info(f"build manifest: {pprint.pformat(manifest)}") + + # ------ export image ------ # + os.sync() # make sure all changes are saved to disk before export + if output: + _create_image_tar(output_workdir, output_fpath=output) + if write_to_dev: + _write_image_to_dev(output_workdir, dev=write_to_dev, workdir=workdir) + logger.info(f"job finished, takes {time.time() - _start_time:.2f}s") diff --git a/tools/images_packer/configs.py b/tools/images_packer/configs.py new file mode 100644 index 000000000..8f4b569ae --- /dev/null +++ b/tools/images_packer/configs.py @@ -0,0 +1,33 @@ +# Copyright 2022 TIER IV, INC. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class BaseConfig: + LOGGING_FORMAT = "[%(asctime)s][%(levelname)s]: %(message)s" + IMAGE_LAYOUT_VERSION = 1 + + MANIFEST_JSON = "manifest.json" + MANIFEST_SCHEMA_VERSION = 1 + OUTPUT_WORKDIR = "output" + OUTPUT_DATA_DIR = "data" + OUTPUT_META_DIR = "meta" + IMAGE_UNARCHIVE_WORKDIR = "images" + + DEFAULT_OTA_METADATA_VERSION = 1 + OTA_IMAGE_COMPRESSION_ALG = "zst" + + EXTERNAL_CACHE_DEV_FSLABEL = "ota_cache_src" + + +cfg = BaseConfig() diff --git a/tools/images_packer/manifest.py b/tools/images_packer/manifest.py new file mode 100644 index 000000000..3341ceffe --- /dev/null +++ b/tools/images_packer/manifest.py @@ -0,0 +1,48 @@ +# Copyright 2022 TIER IV, INC. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""External cache source image manifest definition + +A JSON file manifest.json that contains the basic information of built +external cache source image will be placed at the image rootfs. + +Check README.md for the spec. +""" + + +import json +from dataclasses import dataclass, field, asdict +from typing import Dict, List + +from .configs import cfg + + +@dataclass +class ImageMetadata: + ecu_id: str = "" + image_version: str = "" + ota_metadata_version: int = cfg.DEFAULT_OTA_METADATA_VERSION + + +@dataclass +class Manifest: + schema_version: int = cfg.MANIFEST_SCHEMA_VERSION + image_layout_version: int = cfg.IMAGE_LAYOUT_VERSION + build_timestamp: int = 0 + data_size: int = 0 + data_files_num: int = 0 + meta_size: int = 0 + image_meta: List[ImageMetadata] = field(default_factory=list) + + def export_to_json(self) -> str: + return json.dumps(asdict(self)) diff --git a/tools/images_packer/utils.py b/tools/images_packer/utils.py new file mode 100644 index 000000000..e3d6667df --- /dev/null +++ b/tools/images_packer/utils.py @@ -0,0 +1,29 @@ +# Copyright 2022 TIER IV, INC. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Shared utils among package.""" + + +from os import PathLike +from typing import Union +from typing_extensions import TypeAlias + +StrPath: TypeAlias = Union[str, PathLike] + + +class InputImageProcessError(Exception): + ... + + +class ExportError(Exception): + ... From 02c4399b227d170b428f5af6ea024fe78a9b4591 Mon Sep 17 00:00:00 2001 From: Bodong YANG Date: Mon, 25 Dec 2023 03:38:48 +0000 Subject: [PATCH 02/16] tools.images_packer: layout the subcommands --- tools/images_packer/__main__.py | 145 +++++++++--- tools/offline_ota_image_builder/README.md | 175 --------------- tools/offline_ota_image_builder/__init__.py | 13 -- tools/offline_ota_image_builder/__main__.py | 166 -------------- tools/offline_ota_image_builder/builder.py | 234 -------------------- tools/offline_ota_image_builder/configs.py | 33 --- tools/offline_ota_image_builder/manifest.py | 48 ---- tools/offline_ota_image_builder/utils.py | 29 --- 8 files changed, 114 insertions(+), 729 deletions(-) delete mode 100644 tools/offline_ota_image_builder/README.md delete mode 100644 tools/offline_ota_image_builder/__init__.py delete mode 100644 tools/offline_ota_image_builder/__main__.py delete mode 100644 tools/offline_ota_image_builder/builder.py delete mode 100644 tools/offline_ota_image_builder/configs.py delete mode 100644 tools/offline_ota_image_builder/manifest.py delete mode 100644 tools/offline_ota_image_builder/utils.py diff --git a/tools/images_packer/__main__.py b/tools/images_packer/__main__.py index 5e316b471..a42e8361a 100644 --- a/tools/images_packer/__main__.py +++ b/tools/images_packer/__main__.py @@ -37,12 +37,14 @@ """ +from __future__ import annotations import argparse import errno import logging import sys import tempfile from pathlib import Path +from typing import Callable from .configs import cfg from .builder import build @@ -51,7 +53,12 @@ logger = logging.getLogger(__name__) -def main(args): +def main_build_offline_ota_image_bundle(args: argparse.Namespace): + # ------ args check ------ # + if not (args.output or args.write_to): + print("ERR: at least one export option should be specified") + sys.exit(errno.EINVAL) + # ------ parse input image options ------ # image_metas = [] image_files = {} @@ -106,61 +113,137 @@ def main(args): ) -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO, format=cfg.LOGGING_FORMAT, force=True) - parser = argparse.ArgumentParser( - prog="offline_ota_image_builder", - description=( - "Helper script that builds offline OTA image " - "with given OTA image(s) as external cache source or for offline OTA use." - ), +def main_build_external_cache_src(args: argparse.Namespace): + pass + + +# +# ------ subcommands definition ------ # +# + + +def command_build_cache_src( + subparsers: argparse._SubParsersAction, +) -> tuple[str, argparse.ArgumentParser]: + cmd = "build-cache-src" + subparser: argparse.ArgumentParser = subparsers.add_parser( + name=cmd, + description="build external OTA cache source recognized and used otaproxy.", ) - parser.add_argument( + subparser.add_argument( "--image", help=( - "OTA image for as tar archive(compressed or uncompressed), " - "this option can be used multiple times to include multiple images." + "(Optional) OTA image to be included in the external cache source, " + "this argument can be specified multiple times to include multiple images." ), - required=True, - metavar=":[:]", + required=False, + metavar="", action="append", ) - parser.add_argument( - "-o", - "--output", - help="save the generated image rootfs into tar archive to .", - metavar="", + subparser.add_argument( + "--image-dir", + help="(Optional) Specify a dir of OTA images to be included in the cache source.", + required=False, + metavar="", ) - parser.add_argument( + subparser.add_argument( "-w", "--write-to", help=( - "write the image to and prepare the device as " - "external cache source device." + "(Optional) write the external cache source image to and prepare the device, " + "and then prepare the device to be used as external cache source storage." ), + required=False, metavar="", ) - parser.add_argument( + subparser.add_argument( "--force-write-to", help=( - "prepare as external cache source device without inter-active confirmation, " + "(Optional) prepare as external cache source device without inter-active confirmation, " "only valid when used with -w option." ), + required=False, action="store_true", ) - args = parser.parse_args() + return cmd, subparser - # basic options check - if args.output and (output := Path(args.output)).is_file(): - print(f"ERR: {output} exists, abort") + +def command_build_offline_ota_image_bundle( + subparsers: argparse._SubParsersAction, +) -> tuple[str, argparse.ArgumentParser]: + cmd = "build-offline-ota-imgs-bundle" + subparser: argparse.ArgumentParser = subparsers.add_parser( + name=cmd, + description="build OTA image bundle for offline OTA use.", + ) + subparser.add_argument( + "--image", + help=( + "OTA image for as tar archive(compressed or uncompressed), " + "this option can be used multiple times to include multiple images." + ), + required=True, + metavar=":[:]", + action="append", + ) + return cmd, subparser + + +def register_handler( + _cmd: str, subparser: argparse.ArgumentParser, *, handler: Callable +): + subparser.set_defaults(handler=handler) + + +def get_handler(args: argparse.Namespace) -> Callable[[argparse.Namespace], None]: + try: + return args.handler + except AttributeError: + print("ERR: image packing mode is not specified, check -h for more details") sys.exit(errno.EINVAL) - if not (args.output or args.write_to): - print("ERR: at least one export option should be specified") + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format=cfg.LOGGING_FORMAT, force=True) + + # ------ init main parser ------ # + main_parser = argparse.ArgumentParser( + prog="images_packer", + description=( + "Helper script that builds OTA images bundle from given OTA image(s), " + "start from choosing different image packing modes." + ), + ) + # shared args + main_parser.add_argument( + "-o", + "--output", + help="save the generated image bundle tar archive to .", + metavar="", + ) + + # ------ define subcommands ------ # + subparsers = main_parser.add_subparsers( + title="Image packing modes", + ) + register_handler( + *command_build_cache_src(subparsers), + handler=main_build_external_cache_src, + ) + register_handler( + *command_build_offline_ota_image_bundle(subparsers), + handler=main_build_offline_ota_image_bundle, + ) + + args = main_parser.parse_args() + + # shared args check + if args.output and (output := Path(args.output)).exists(): + print(f"ERR: {output} exists, abort") sys.exit(errno.EINVAL) try: - main(args) + get_handler(args)(args) except KeyboardInterrupt: print("ERR: aborted by user") raise diff --git a/tools/offline_ota_image_builder/README.md b/tools/offline_ota_image_builder/README.md deleted file mode 100644 index b61d5970e..000000000 --- a/tools/offline_ota_image_builder/README.md +++ /dev/null @@ -1,175 +0,0 @@ -# Offline OTA image builder - -`tools.offline_ota_image_builder` is a helper package for bundling offline OTA image from the OTA images build by ota-metadata(or images compatible with OTA image specification). -Built image can be used as external cache source for otaproxy, or used by full offline OTA helper util to trigger fully offline OTA update. - -**NOTE**: full offline OTA helper util is not yet implemented. - -It provides the following features: - -1. build offline OTA image from the OTA images built by ota-metadata(or images compatible with OTA image specification), -2. export built offline OTA image as tar archive, -3. prepare and export built image onto specific block device as external cache source device for otaproxy. - - - -## Expected image layout for input OTA images - -This package currently only recognizes the OTA images generated by [ota-metadata@a711013](https://github.com/tier4/ota-metadata/commit/a711013), or the image layout compatible with the ota-metadata and OTA image layout specification as follow. - -```text -Example OTA image layout: -. -├── data -│ ├── <...files with full paths...> -│ └── ... -├── data.zst -│ ├── <... compressed OTA file ...> -│ └── <... naming: .zst ...> -├── certificate.pem -├── dirs.txt -├── metadata.jwt -├── persistents.txt -├── regulars.txt -└── symlinks.txt -``` - -Please also refere to [ota-metadata](https://github.com/tier4/ota-metadata) repo for the implementation of OTA image layout specification. - -## Offline OTA image layout - -The rootfs of the built image has the following layout: - -```text -. -├── manifest.json -├── data -│ ├── # uncompressed file -│ ├── .zst # zst compressed file -│ └── ... -└── meta - ├── # corresponding to the order in manifest.image_meta - │ └── - └── ... - └── - -``` - -This layout is compatible with external cache source's layout, which means any offline OTA image can natually directly be used as external cache source recognized by otaproxy(but user still needs to prepare the device either by themselves and extract the offline OTA image rootfs onto the prepared device, or prepare the device with this **offline_ota_image_builder** package). - - - -## Offline OTA image Manifest schema - -Offline OTA image's rootfs contains a `manifest.json` file with a single JSON object in it. This file includes the information related to this offline OTA image, including the bundled OTA images' metadata. - -```json -{ - "schema_version": 1, - "image_layout_version": 1, - "build_timestamp": 1693291679, - "data_size": 1122334455, - "data_files_num": 12345, - "meta_size": 112233, - "image_meta": [ - { - "ecu_id": "", - "image_version": "", - "ota_metadata_version": 1, - }, - ... - ], -} -``` - - - -## How to use - -### Prepare the dependencies - -This image builder requires latest ota-client to be installed/accessable. The recommended way to install dependencies is to init a virtual environment and install the ota-client into this venv, and then execute the image_builder with this venv. - -1. git clone the latest ota-client repository: - - ```bash - $ git clone https://github.com/tier4/ota-client.git - # image builder is located at ota-client/tools/offline_ota_image_builder - ``` - -2. prepare the virtual environment and activate it: - - ```bash - $ python3 -m venv venv - $ . venv/bin/active - (venv) $ - ``` - -3. install the ota-client into the virtual environment: - - ```bash - (venv) $ pip install ota-client - ``` - -### Build image and export - -#### Builder usage - -```text -usage: offline_ota_image_builder [-h] --image :[:] - [-o ] [-w ] [--confirm-write-to] - -Helper script that builds offline OTA image with given OTA image(s) as external cache source or for -offline OTA use. - -options: - -h, --help show this help message and exit - --image :[:] - OTA image for as tar archive(compressed or uncompressed), this - option can be used multiple times to include multiple images. - -o , --output - save the generated image rootfs into tar archive to . - -w , --write-to - write the image to and prepare the device as external cache source - device. - --force-write-to prepare as external cache source device without inter-active confirmation, - only valid when used with -w option. -``` - -Execute the image builder by directly calling it from the source code. This package requires `root` permission to run(required by extracting OTA image and preparing external cache source device). - -Option `--image :[:]` is used to specify OTA image to be included. This option can be used multiple times to specify multiple OTA images. - -Option `--write-to ` is used to prepare external cache source device used by otaproxy. The specified device will be formatted as `ext4`, fslabel with `ota_cache_src`, and be exported with the built offline OTA image's rootfs. If this package is used in a non-interactive script, option `--force-write-to` can be used to bypass interactive confirmation. - -Option `--output ` specify where to save the exported tar archive of built image. - -User must at least specifies one of `--write-to` and `--output`, or specifies them together. - -#### Usage 1: Build offline OTA image and export it as tar archive - -```bash -# current folder layout: venv ota-client -(venv) $ cd ota-client -(venv) $ sudo -E env PATH=$PATH python3 -m tools.offline_ota_image_builder --image=p1:p1_image.tgz:ver_20230808 --image=p2:p2_image.tgz:ver_20230808 --output=t2.tar -``` - -This will build the offline OTA image with `p1_image.tgz` and `p2_image.tgz`, which are for ECU `p1` and `p2`, and export the built image as `t2.tar` tar archive. - -### Usage 2: Build the offline OTA image and create external cache source dev - -```bash -# current folder layout: venv ota-client -(venv) $ cd ota-client -(venv) $ sudo -E env PATH=$PATH python3 -m tools.offline_ota_image_builder --image=p1:p1_image.tgz:ver_20230808 --image=p2:p2_image.tgz:ver_20230808 --write-to=/dev/ -``` - -This will build the offline OTA image with `p1_image.tgz` and `p2_image.tgz`, which are for ECU `p1` and `p2`, and then prepare the `/dev/` as external cache source device(ext4 filesystem labelled with `ota_cache_src`) with image rootfs exported to the filesystem on `/dev/`. - -### Usage 3: Build the offline OTA image, export it as tar archive and prepare external cache source dev - -```bash -# current folder layout: venv ota-client -(venv) $ cd ota-client -(venv) $ sudo -E env PATH=$PATH python3 -m tools.offline_ota_image_builder --image=p1:p1_image.tgz:ver_20230808 --image=p2:p2_image.tgz:ver_20230808 --output=t2.tar --write-to=/dev/ -``` diff --git a/tools/offline_ota_image_builder/__init__.py b/tools/offline_ota_image_builder/__init__.py deleted file mode 100644 index bcfd866ad..000000000 --- a/tools/offline_ota_image_builder/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2022 TIER IV, INC. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tools/offline_ota_image_builder/__main__.py b/tools/offline_ota_image_builder/__main__.py deleted file mode 100644 index 5e316b471..000000000 --- a/tools/offline_ota_image_builder/__main__.py +++ /dev/null @@ -1,166 +0,0 @@ -# Copyright 2022 TIER IV, INC. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""External cache source builder helper. - -Please refere to https://github.com/tier4/ota-metadata for -OTA image layout specification. -This tool is implemented based on ota-metadata@a711013. - -Example OTA image layout: -. -├── data -│ ├── <...files with full paths...> -│ └── ... -├── data.zst -│ ├── <... compressed OTA file ...> -│ └── <... naming: .zst ...> -├── certificate.pem -├── dirs.txt -├── metadata.jwt -├── persistents.txt -├── regulars.txt -└── symlinks.txt - -Check README.md for the offline OTA image layout specification. -Please refer to OTA cache design doc for more details. -""" - - -import argparse -import errno -import logging -import sys -import tempfile -from pathlib import Path - -from .configs import cfg -from .builder import build -from .manifest import ImageMetadata - -logger = logging.getLogger(__name__) - - -def main(args): - # ------ parse input image options ------ # - image_metas = [] - image_files = {} - - for raw_pair in args.image: - _parsed = str(raw_pair).split(":") - if len(_parsed) >= 2: - _ecu_id, _image_fpath, _image_version, *_ = *_parsed, "" - image_metas.append( - ImageMetadata( - ecu_id=_ecu_id, - image_version=_image_version, - ) - ) - image_files[_ecu_id] = _image_fpath - else: - logger.warning(f"ignore illegal image pair: {raw_pair}") - - if not image_metas: - print("ERR: at least one valid image should be given, abort") - sys.exit(errno.EINVAL) - - # ------ parse export options ------ # - output_fpath, write_to_dev = args.output, args.write_to - - if write_to_dev and not Path(write_to_dev).is_block_device(): - print(f"{write_to_dev} is not a block device, abort") - sys.exit(errno.EINVAL) - - if write_to_dev and not args.force_write_to: - _confirm_write_to = input( - f"WARNING: generated image will be written to {write_to_dev}, \n" - f"\t all data in {write_to_dev} will be lost, continue(type [Y] or [N]): " - ) - if _confirm_write_to != "Y": - print(f"decline writing to {write_to_dev}, abort") - sys.exit(errno.EINVAL) - elif write_to_dev and args.force_write_to: - logger.warning( - f"generated image will be written to {write_to_dev}," - f"all data in {write_to_dev} will be lost" - ) - - # ------ build image ------ # - with tempfile.TemporaryDirectory(prefix="offline_OTA_image_builder") as workdir: - build( - image_metas, - image_files, - workdir=workdir, - output=output_fpath, - write_to_dev=write_to_dev, - ) - - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO, format=cfg.LOGGING_FORMAT, force=True) - parser = argparse.ArgumentParser( - prog="offline_ota_image_builder", - description=( - "Helper script that builds offline OTA image " - "with given OTA image(s) as external cache source or for offline OTA use." - ), - ) - parser.add_argument( - "--image", - help=( - "OTA image for as tar archive(compressed or uncompressed), " - "this option can be used multiple times to include multiple images." - ), - required=True, - metavar=":[:]", - action="append", - ) - parser.add_argument( - "-o", - "--output", - help="save the generated image rootfs into tar archive to .", - metavar="", - ) - parser.add_argument( - "-w", - "--write-to", - help=( - "write the image to and prepare the device as " - "external cache source device." - ), - metavar="", - ) - parser.add_argument( - "--force-write-to", - help=( - "prepare as external cache source device without inter-active confirmation, " - "only valid when used with -w option." - ), - action="store_true", - ) - args = parser.parse_args() - - # basic options check - if args.output and (output := Path(args.output)).is_file(): - print(f"ERR: {output} exists, abort") - sys.exit(errno.EINVAL) - - if not (args.output or args.write_to): - print("ERR: at least one export option should be specified") - sys.exit(errno.EINVAL) - - try: - main(args) - except KeyboardInterrupt: - print("ERR: aborted by user") - raise diff --git a/tools/offline_ota_image_builder/builder.py b/tools/offline_ota_image_builder/builder.py deleted file mode 100644 index 42c77d7f5..000000000 --- a/tools/offline_ota_image_builder/builder.py +++ /dev/null @@ -1,234 +0,0 @@ -# Copyright 2022 TIER IV, INC. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import logging -import os -import pprint -import shutil -import tempfile -import time -from contextlib import contextmanager -from pathlib import Path -from typing import Mapping, Optional, Sequence - -from otaclient.app.common import subprocess_call -from otaclient.app import ota_metadata -from .configs import cfg -from .manifest import ImageMetadata, Manifest -from .utils import StrPath, InputImageProcessError, ExportError - -logger = logging.getLogger(__name__) - -PROC_MOUNTS = "/proc/mounts" - - -def _check_if_mounted(dev: StrPath): - for line in Path(PROC_MOUNTS).read_text().splitlines(): - if line.find(str(dev)) != -1: - return True - return False - - -@contextmanager -def _unarchive_image(image_fpath: StrPath, *, workdir: StrPath): - _start_time = time.time() - with tempfile.TemporaryDirectory(dir=workdir) as unarchive_dir: - cmd = f"tar xf {image_fpath} -C {unarchive_dir}" - try: - subprocess_call(cmd) - except Exception as e: - _err_msg = f"failed to process input image {image_fpath}: {e!r}, {cmd=}" - logger.error(_err_msg) - raise InputImageProcessError(_err_msg) from e - - logger.info( - f"finish unarchiving {image_fpath}, takes {time.time() - _start_time:.2f}s" - ) - yield unarchive_dir - - -def _process_ota_image(ota_image_dir: StrPath, *, data_dir: StrPath, meta_dir: StrPath): - """Processing OTA image under and update and .""" - _start_time = time.time() - data_dir = Path(data_dir) - # statistics - saved_files, saved_files_size = 0, 0 - - ota_image_dir = Path(ota_image_dir) - - # ------ process OTA image metadata ------ # - metadata_jwt_fpath = ota_image_dir / ota_metadata.OTAMetadata.METADATA_JWT - # NOTE: we don't need to do certificate verification here, so set certs_dir to empty - metadata_jwt = ota_metadata._MetadataJWTParser( - metadata_jwt_fpath.read_text(), certs_dir="" - ).get_otametadata() - - rootfs_dir = ota_image_dir / metadata_jwt.rootfs_directory - compressed_rootfs_dir = ota_image_dir / metadata_jwt.compressed_rootfs_directory - - # ------ update data_dir with the contents of this OTA image ------ # - with open(ota_image_dir / metadata_jwt.regular.file, "r") as f: - for line in f: - reg_inf = ota_metadata.parse_regulars_from_txt(line) - ota_file_sha256 = reg_inf.sha256hash.hex() - - if reg_inf.compressed_alg == cfg.OTA_IMAGE_COMPRESSION_ALG: - _zst_ota_fname = f"{ota_file_sha256}.{cfg.OTA_IMAGE_COMPRESSION_ALG}" - src = compressed_rootfs_dir / _zst_ota_fname - dst = data_dir / _zst_ota_fname - # NOTE: multiple OTA files might have the same hash - # NOTE: squash OTA file permission before moving - if not dst.is_file() and src.is_file(): - src.chmod(0o644) - saved_files += 1 - shutil.move(str(src), dst) - saved_files_size += dst.stat().st_size - - else: - src = rootfs_dir / os.path.relpath(reg_inf.path, "/") - dst = data_dir / ota_file_sha256 - if not dst.is_file() and src.is_file(): - src.chmod(0o644) - saved_files += 1 - shutil.move(str(src), dst) - saved_files_size += dst.stat().st_size - - # ------ update meta_dir with the OTA meta files in this image ------ # - # copy OTA metafiles to image specific meta folder - for _metaf in ota_metadata.MetafilesV1: - shutil.move(str(ota_image_dir / _metaf.value), meta_dir) - # copy certificate and metadata.jwt - shutil.move(str(ota_image_dir / metadata_jwt.certificate.file), meta_dir) - shutil.move(str(metadata_jwt_fpath), meta_dir) - - logger.info(f"finish processing OTA image, takes {time.time() - _start_time:.2f}s") - return saved_files, saved_files_size - - -def _create_image_tar(image_rootfs: StrPath, output_fpath: StrPath): - """Export generated image rootfs as tar ball.""" - cmd = f"tar cf {output_fpath} -C {image_rootfs} ." - try: - logger.info(f"exporting external cache source image to {output_fpath} ...") - subprocess_call(cmd) - except Exception as e: - _err_msg = f"failed to tar generated image to {output_fpath=}: {e!r}, {cmd=}" - logger.error(_err_msg) - raise ExportError(_err_msg) from e - - -def _write_image_to_dev(image_rootfs: StrPath, dev: StrPath, *, workdir: StrPath): - """Prepare and export generated image's rootfs to it.""" - _start_time = time.time() - dev = Path(dev) - if not dev.is_block_device(): - logger.warning(f"{dev=} is not a block device, skip") - return - if _check_if_mounted(dev): - logger.warning(f"{dev=} is mounted, skip") - return - - # prepare device - format_device_cmd = f"mkfs.ext4 -L {cfg.EXTERNAL_CACHE_DEV_FSLABEL} {dev}" - try: - logger.warning(f"formatting {dev} to ext4: {format_device_cmd}") - # NOTE: label offline OTA image as external cache source - subprocess_call(format_device_cmd) - except Exception as e: - _err_msg = f"failed to formatting {dev} to ext4: {e!r}" - logger.error(_err_msg) - raise ExportError(_err_msg) from e - - # mount and copy - mount_point = Path(workdir) / "mnt" - mount_point.mkdir(exist_ok=True) - cp_cmd = f"cp -rT {image_rootfs} {mount_point}" - try: - logger.info(f"copying image rootfs to {dev=}@{mount_point=}...") - subprocess_call(f"mount --make-private --make-unbindable {dev} {mount_point}") - subprocess_call(cp_cmd) - logger.info(f"finish copying, takes {time.time()-_start_time:.2f}s") - except Exception as e: - _err_msg = f"failed to export to image rootfs to {dev=}@{mount_point=}: {e!r}, {cp_cmd=}" - logger.error(_err_msg) - raise ExportError(_err_msg) from e - finally: - subprocess_call(f"umount -l {dev}", raise_exception=False) - - -def build( - image_metas: Sequence[ImageMetadata], - image_files: Mapping[str, StrPath], - *, - workdir: StrPath, - output: Optional[StrPath], - write_to_dev: Optional[StrPath], -): - _start_time = time.time() - logger.info(f"job started at {int(_start_time)}") - - output_workdir = Path(workdir) / cfg.OUTPUT_WORKDIR - output_workdir.mkdir(parents=True, exist_ok=True) - output_data_dir = output_workdir / cfg.OUTPUT_DATA_DIR - output_data_dir.mkdir(exist_ok=True, parents=True) - output_meta_dir = output_workdir / cfg.OUTPUT_META_DIR - output_meta_dir.mkdir(exist_ok=True, parents=True) - - # ------ generate image ------ # - manifest = Manifest(image_meta=list(image_metas)) - # process all inpu OTA images - images_unarchiving_work_dir = Path(workdir) / cfg.IMAGE_UNARCHIVE_WORKDIR - images_unarchiving_work_dir.mkdir(parents=True, exist_ok=True) - for idx, image_meta in enumerate(manifest.image_meta): - ecu_id = image_meta.ecu_id - image_file = image_files[ecu_id] - # image specific metadata dir - _meta_dir = output_meta_dir / str(idx) - _meta_dir.mkdir(exist_ok=True, parents=True) - - logger.info(f"{ecu_id=}: unarchive {image_file} ...") - with _unarchive_image( - image_file, workdir=images_unarchiving_work_dir - ) as unarchived_image_dir: - logger.info(f"{ecu_id=}: processing OTA image ...") - _saved_files_num, _saved_files_size = _process_ota_image( - unarchived_image_dir, - data_dir=output_data_dir, - meta_dir=_meta_dir, - ) - # update manifest after this image is processed - manifest.data_size += _saved_files_size - manifest.data_files_num += _saved_files_num - - # process metadata - for cur_path, _, fnames in os.walk(output_meta_dir): - _cur_path = Path(cur_path) - for _fname in fnames: - _fpath = _cur_path / _fname - if _fpath.is_file(): - manifest.meta_size += _fpath.stat().st_size - - # write offline OTA image manifest - manifest.build_timestamp = int(time.time()) - Path(output_workdir / cfg.MANIFEST_JSON).write_text(manifest.export_to_json()) - logger.info(f"build manifest: {pprint.pformat(manifest)}") - - # ------ export image ------ # - os.sync() # make sure all changes are saved to disk before export - if output: - _create_image_tar(output_workdir, output_fpath=output) - if write_to_dev: - _write_image_to_dev(output_workdir, dev=write_to_dev, workdir=workdir) - logger.info(f"job finished, takes {time.time() - _start_time:.2f}s") diff --git a/tools/offline_ota_image_builder/configs.py b/tools/offline_ota_image_builder/configs.py deleted file mode 100644 index 8f4b569ae..000000000 --- a/tools/offline_ota_image_builder/configs.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2022 TIER IV, INC. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -class BaseConfig: - LOGGING_FORMAT = "[%(asctime)s][%(levelname)s]: %(message)s" - IMAGE_LAYOUT_VERSION = 1 - - MANIFEST_JSON = "manifest.json" - MANIFEST_SCHEMA_VERSION = 1 - OUTPUT_WORKDIR = "output" - OUTPUT_DATA_DIR = "data" - OUTPUT_META_DIR = "meta" - IMAGE_UNARCHIVE_WORKDIR = "images" - - DEFAULT_OTA_METADATA_VERSION = 1 - OTA_IMAGE_COMPRESSION_ALG = "zst" - - EXTERNAL_CACHE_DEV_FSLABEL = "ota_cache_src" - - -cfg = BaseConfig() diff --git a/tools/offline_ota_image_builder/manifest.py b/tools/offline_ota_image_builder/manifest.py deleted file mode 100644 index 3341ceffe..000000000 --- a/tools/offline_ota_image_builder/manifest.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright 2022 TIER IV, INC. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""External cache source image manifest definition - -A JSON file manifest.json that contains the basic information of built -external cache source image will be placed at the image rootfs. - -Check README.md for the spec. -""" - - -import json -from dataclasses import dataclass, field, asdict -from typing import Dict, List - -from .configs import cfg - - -@dataclass -class ImageMetadata: - ecu_id: str = "" - image_version: str = "" - ota_metadata_version: int = cfg.DEFAULT_OTA_METADATA_VERSION - - -@dataclass -class Manifest: - schema_version: int = cfg.MANIFEST_SCHEMA_VERSION - image_layout_version: int = cfg.IMAGE_LAYOUT_VERSION - build_timestamp: int = 0 - data_size: int = 0 - data_files_num: int = 0 - meta_size: int = 0 - image_meta: List[ImageMetadata] = field(default_factory=list) - - def export_to_json(self) -> str: - return json.dumps(asdict(self)) diff --git a/tools/offline_ota_image_builder/utils.py b/tools/offline_ota_image_builder/utils.py deleted file mode 100644 index e3d6667df..000000000 --- a/tools/offline_ota_image_builder/utils.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2022 TIER IV, INC. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Shared utils among package.""" - - -from os import PathLike -from typing import Union -from typing_extensions import TypeAlias - -StrPath: TypeAlias = Union[str, PathLike] - - -class InputImageProcessError(Exception): - ... - - -class ExportError(Exception): - ... From 5592763c2b93f3795660051b7315ba11b39dfdf8 Mon Sep 17 00:00:00 2001 From: Bodong YANG Date: Mon, 25 Dec 2023 03:55:38 +0000 Subject: [PATCH 03/16] tools.images_packer: implement subcommand handler for build_cache_src --- tools/images_packer/__main__.py | 56 +++++++++++++++++++++++++++------ tools/images_packer/builder.py | 2 +- 2 files changed, 47 insertions(+), 11 deletions(-) diff --git a/tools/images_packer/__main__.py b/tools/images_packer/__main__.py index a42e8361a..a313fdfda 100644 --- a/tools/images_packer/__main__.py +++ b/tools/images_packer/__main__.py @@ -54,11 +54,6 @@ def main_build_offline_ota_image_bundle(args: argparse.Namespace): - # ------ args check ------ # - if not (args.output or args.write_to): - print("ERR: at least one export option should be specified") - sys.exit(errno.EINVAL) - # ------ parse input image options ------ # image_metas = [] image_files = {} @@ -81,6 +76,52 @@ def main_build_offline_ota_image_bundle(args: argparse.Namespace): print("ERR: at least one valid image should be given, abort") sys.exit(errno.EINVAL) + # ------ build image ------ # + with tempfile.TemporaryDirectory(prefix="offline_OTA_image_builder") as workdir: + build( + image_metas, + image_files, + workdir=workdir, + output=args.output, + ) + + +def main_build_external_cache_src(args: argparse.Namespace): + # ------ args check ------ # + if not (args.output or args.write_to): + logger.error("ERR: at least one export option should be specified") + sys.exit(errno.EINVAL) + + # ------ parse args ------ # + image_metas: list[ImageMetadata] = [] + image_files: dict[str, Path] = {} + count = 0 + + # retrieves images from --image arg + for _image_fpath in args.image: + count_str = str(count) + image_metas.append(ImageMetadata(ecu_id=count_str)) + image_files[count_str] = _image_fpath + count += 1 + + # retrieves images from --image-dir arg + _image_store_dpath = Path(args.image_dir) + if not _image_store_dpath.is_dir(): + logger.error( + f"ERR: specified ={_image_store_dpath} doesn't exist, abort" + ) + sys.exit(errno.EINVAL) + + for _image_fpath in _image_store_dpath.glob("*"): + count_str = str(count) + image_metas.append(ImageMetadata(ecu_id=count_str)) + image_files[count_str] = _image_fpath + count += 1 + + if not image_metas: + print("ERR: at least one valid image should be given, abort") + sys.exit(errno.EINVAL) + # ------ parse export options ------ # output_fpath, write_to_dev = args.output, args.write_to @@ -109,14 +150,9 @@ def main_build_offline_ota_image_bundle(args: argparse.Namespace): image_files, workdir=workdir, output=output_fpath, - write_to_dev=write_to_dev, ) -def main_build_external_cache_src(args: argparse.Namespace): - pass - - # # ------ subcommands definition ------ # # diff --git a/tools/images_packer/builder.py b/tools/images_packer/builder.py index 42c77d7f5..b30b25261 100644 --- a/tools/images_packer/builder.py +++ b/tools/images_packer/builder.py @@ -174,7 +174,7 @@ def build( *, workdir: StrPath, output: Optional[StrPath], - write_to_dev: Optional[StrPath], + write_to_dev: Optional[StrPath] = None, ): _start_time = time.time() logger.info(f"job started at {int(_start_time)}") From ffecddde288d2971c88ba6467124c8ccb14abc41 Mon Sep 17 00:00:00 2001 From: Bodong YANG Date: Mon, 25 Dec 2023 06:21:07 +0000 Subject: [PATCH 04/16] minor update --- tools/images_packer/manifest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/images_packer/manifest.py b/tools/images_packer/manifest.py index 3341ceffe..9c4e4fa9a 100644 --- a/tools/images_packer/manifest.py +++ b/tools/images_packer/manifest.py @@ -22,7 +22,7 @@ import json from dataclasses import dataclass, field, asdict -from typing import Dict, List +from typing import List from .configs import cfg From a9644422e4cbc1a6b178d011ee9bc593cc757a4b Mon Sep 17 00:00:00 2001 From: Bodong YANG Date: Mon, 25 Dec 2023 06:42:24 +0000 Subject: [PATCH 05/16] tools.images_packer: check over write_to_dev is performed before image build --- tools/images_packer/__main__.py | 14 ++++++++++---- tools/images_packer/builder.py | 15 --------------- tools/images_packer/utils.py | 19 +++++++++++++++++++ 3 files changed, 29 insertions(+), 19 deletions(-) diff --git a/tools/images_packer/__main__.py b/tools/images_packer/__main__.py index a313fdfda..6ffd1695f 100644 --- a/tools/images_packer/__main__.py +++ b/tools/images_packer/__main__.py @@ -49,6 +49,7 @@ from .configs import cfg from .builder import build from .manifest import ImageMetadata +from .utils import check_if_mounted, umount logger = logging.getLogger(__name__) @@ -92,6 +93,15 @@ def main_build_external_cache_src(args: argparse.Namespace): logger.error("ERR: at least one export option should be specified") sys.exit(errno.EINVAL) + if args.write_to: + write_to_dev = Path(args.write_to) + if not write_to_dev.is_block_device(): + logger.error(f"{write_to_dev=} is not a block device, abort") + sys.exit(errno.EINVAL) + if check_if_mounted(write_to_dev): + logger.warning(f"{write_to_dev=} is mounted, try to umount it...") + umount(write_to_dev) + # ------ parse args ------ # image_metas: list[ImageMetadata] = [] image_files: dict[str, Path] = {} @@ -125,10 +135,6 @@ def main_build_external_cache_src(args: argparse.Namespace): # ------ parse export options ------ # output_fpath, write_to_dev = args.output, args.write_to - if write_to_dev and not Path(write_to_dev).is_block_device(): - print(f"{write_to_dev} is not a block device, abort") - sys.exit(errno.EINVAL) - if write_to_dev and not args.force_write_to: _confirm_write_to = input( f"WARNING: generated image will be written to {write_to_dev}, \n" diff --git a/tools/images_packer/builder.py b/tools/images_packer/builder.py index b30b25261..aa0b850d7 100644 --- a/tools/images_packer/builder.py +++ b/tools/images_packer/builder.py @@ -31,15 +31,6 @@ logger = logging.getLogger(__name__) -PROC_MOUNTS = "/proc/mounts" - - -def _check_if_mounted(dev: StrPath): - for line in Path(PROC_MOUNTS).read_text().splitlines(): - if line.find(str(dev)) != -1: - return True - return False - @contextmanager def _unarchive_image(image_fpath: StrPath, *, workdir: StrPath): @@ -133,12 +124,6 @@ def _write_image_to_dev(image_rootfs: StrPath, dev: StrPath, *, workdir: StrPath """Prepare and export generated image's rootfs to it.""" _start_time = time.time() dev = Path(dev) - if not dev.is_block_device(): - logger.warning(f"{dev=} is not a block device, skip") - return - if _check_if_mounted(dev): - logger.warning(f"{dev=} is mounted, skip") - return # prepare device format_device_cmd = f"mkfs.ext4 -L {cfg.EXTERNAL_CACHE_DEV_FSLABEL} {dev}" diff --git a/tools/images_packer/utils.py b/tools/images_packer/utils.py index e3d6667df..3e9449dc0 100644 --- a/tools/images_packer/utils.py +++ b/tools/images_packer/utils.py @@ -15,9 +15,12 @@ from os import PathLike +from pathlib import Path from typing import Union from typing_extensions import TypeAlias +from otaclient.app.common import subprocess_call + StrPath: TypeAlias = Union[str, PathLike] @@ -27,3 +30,19 @@ class InputImageProcessError(Exception): class ExportError(Exception): ... + + +PROC_MOUNTS = "/proc/mounts" + + +def check_if_mounted(dev: StrPath) -> bool: + """Perform a fast check to see if is mounted.""" + for line in Path(PROC_MOUNTS).read_text().splitlines(): + if line.find(str(dev)) != -1: + return True + return False + + +def umount(dev: StrPath): + _cmd = f"umount {dev}" + subprocess_call(_cmd, raise_exception=True) From d0555d804cfea74411159dd8c3c95dcfe3f6fa4f Mon Sep 17 00:00:00 2001 From: Bodong YANG Date: Mon, 25 Dec 2023 07:40:37 +0000 Subject: [PATCH 06/16] list -o args for each subcommands --- tools/images_packer/__main__.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/tools/images_packer/__main__.py b/tools/images_packer/__main__.py index 6ffd1695f..b8be70e59 100644 --- a/tools/images_packer/__main__.py +++ b/tools/images_packer/__main__.py @@ -188,6 +188,12 @@ def command_build_cache_src( required=False, metavar="", ) + subparser.add_argument( + "-o", + "--output", + help="save the generated image bundle tar archive to .", + metavar="", + ) subparser.add_argument( "-w", "--write-to", @@ -228,6 +234,12 @@ def command_build_offline_ota_image_bundle( metavar=":[:]", action="append", ) + subparser.add_argument( + "-o", + "--output", + help="save the generated image bundle tar archive to .", + metavar="", + ) return cmd, subparser @@ -256,13 +268,6 @@ def get_handler(args: argparse.Namespace) -> Callable[[argparse.Namespace], None "start from choosing different image packing modes." ), ) - # shared args - main_parser.add_argument( - "-o", - "--output", - help="save the generated image bundle tar archive to .", - metavar="", - ) # ------ define subcommands ------ # subparsers = main_parser.add_subparsers( @@ -279,7 +284,7 @@ def get_handler(args: argparse.Namespace) -> Callable[[argparse.Namespace], None args = main_parser.parse_args() - # shared args check + # check common args if args.output and (output := Path(args.output)).exists(): print(f"ERR: {output} exists, abort") sys.exit(errno.EINVAL) From 7e2c77b6b233b86f7858749951080a1948ea824d Mon Sep 17 00:00:00 2001 From: Bodong YANG Date: Mon, 25 Dec 2023 07:45:36 +0000 Subject: [PATCH 07/16] update README.md --- tools/images_packer/README.md | 125 ++++++++++++++++++++-------------- 1 file changed, 73 insertions(+), 52 deletions(-) diff --git a/tools/images_packer/README.md b/tools/images_packer/README.md index b61d5970e..5867ec55f 100644 --- a/tools/images_packer/README.md +++ b/tools/images_packer/README.md @@ -1,15 +1,28 @@ -# Offline OTA image builder +# OTA images packer -`tools.offline_ota_image_builder` is a helper package for bundling offline OTA image from the OTA images build by ota-metadata(or images compatible with OTA image specification). -Built image can be used as external cache source for otaproxy, or used by full offline OTA helper util to trigger fully offline OTA update. +`tools.images_packer` is a helper package for bundling multiple images into one single images bundle. +The input OTA images are expected to be built by ota-metadata(or images compatible with OTA image specification). -**NOTE**: full offline OTA helper util is not yet implemented. +This package provide the following two packing mode, each for different purposes: +1. `build-cache-src`: with this mode, the built image can be recognized and utilized by otaproxy to reduce network downloading during OTA update. -It provides the following features: + The following features are provided in this mode: -1. build offline OTA image from the OTA images built by ota-metadata(or images compatible with OTA image specification), -2. export built offline OTA image as tar archive, -3. prepare and export built image onto specific block device as external cache source device for otaproxy. + 1. build external cache source image, + + 2. export the built image as tar archive, + + 3. prepare and export built image onto specific block device as external cache source device recognized by otaproxy. + +2. `build-offline-ota-imgs-bundle`: with this mode, the built image can be used by the `fully offline OTA helper` script to trigger fullly offline OTA locally without any network. + + The following feature are provided in this mode: + + 1. build offline OTA image, + + 2. export the built image as tar archive. + + **NOTE**: full offline OTA helper util is not yet implemented. @@ -36,7 +49,7 @@ Example OTA image layout: Please also refere to [ota-metadata](https://github.com/tier4/ota-metadata) repo for the implementation of OTA image layout specification. -## Offline OTA image layout +## OTA image bundle layout The rootfs of the built image has the following layout: @@ -55,11 +68,13 @@ The rootfs of the built image has the following layout: ``` -This layout is compatible with external cache source's layout, which means any offline OTA image can natually directly be used as external cache source recognized by otaproxy(but user still needs to prepare the device either by themselves and extract the offline OTA image rootfs onto the prepared device, or prepare the device with this **offline_ota_image_builder** package). +Both external cache source image and offline OTA image bundle use this image bundle layout. +It means that offline OTA image can also be used as external cache source recognized by otaproxy(but user still needs to prepare the device either by themselves and extract the offline OTA image rootfs onto the prepared device, or prepare the device with this **images_packer** package). + -## Offline OTA image Manifest schema +## OTA image bundle Manifest schema Offline OTA image's rootfs contains a `manifest.json` file with a single JSON object in it. This file includes the information related to this offline OTA image, including the bundled OTA images' metadata. @@ -84,7 +99,7 @@ Offline OTA image's rootfs contains a `manifest.json` file with a single JSON ob -## How to use +## How to use `images_packer` ### Prepare the dependencies @@ -111,65 +126,71 @@ This image builder requires latest ota-client to be installed/accessable. The re (venv) $ pip install ota-client ``` -### Build image and export +### Examples of building and exporting image -#### Builder usage - -```text -usage: offline_ota_image_builder [-h] --image :[:] - [-o ] [-w ] [--confirm-write-to] +#### Usage 1: Build external cache source image and export it as tar archive -Helper script that builds offline OTA image with given OTA image(s) as external cache source or for -offline OTA use. - -options: - -h, --help show this help message and exit - --image :[:] - OTA image for as tar archive(compressed or uncompressed), this - option can be used multiple times to include multiple images. - -o , --output - save the generated image rootfs into tar archive to . - -w , --write-to - write the image to and prepare the device as external cache source - device. - --force-write-to prepare as external cache source device without inter-active confirmation, - only valid when used with -w option. +```bash +# current folder layout: venv ota-client +(venv) $ cd ota-client +(venv) $ sudo -E env PATH=$PATH python3 -m tools.image_packer build-cache-src --image-dir=./images_dir --output=t2.tar ``` -Execute the image builder by directly calling it from the source code. This package requires `root` permission to run(required by extracting OTA image and preparing external cache source device). - -Option `--image :[:]` is used to specify OTA image to be included. This option can be used multiple times to specify multiple OTA images. - -Option `--write-to ` is used to prepare external cache source device used by otaproxy. The specified device will be formatted as `ext4`, fslabel with `ota_cache_src`, and be exported with the built offline OTA image's rootfs. If this package is used in a non-interactive script, option `--force-write-to` can be used to bypass interactive confirmation. +This will build the external cache source image with images listed in `./images_dir`, and export the built image as `t2.tar` tar archive. -Option `--output ` specify where to save the exported tar archive of built image. - -User must at least specifies one of `--write-to` and `--output`, or specifies them together. - -#### Usage 1: Build offline OTA image and export it as tar archive +#### Usage 2: Build offline OTA images bundle and export it as tar archive ```bash # current folder layout: venv ota-client (venv) $ cd ota-client -(venv) $ sudo -E env PATH=$PATH python3 -m tools.offline_ota_image_builder --image=p1:p1_image.tgz:ver_20230808 --image=p2:p2_image.tgz:ver_20230808 --output=t2.tar +(venv) $ sudo -E env PATH=$PATH python3 -m tools.image_packer build-offline-ota-imgs-bundle --image=p1:p1_image.tgz:ver_20230808 --image=p2:p2_image.tgz:ver_20230808 --output=t2.tar ``` This will build the offline OTA image with `p1_image.tgz` and `p2_image.tgz`, which are for ECU `p1` and `p2`, and export the built image as `t2.tar` tar archive. -### Usage 2: Build the offline OTA image and create external cache source dev +#### Usage 3: Build the external cache source image and create external cache source dev ```bash # current folder layout: venv ota-client (venv) $ cd ota-client -(venv) $ sudo -E env PATH=$PATH python3 -m tools.offline_ota_image_builder --image=p1:p1_image.tgz:ver_20230808 --image=p2:p2_image.tgz:ver_20230808 --write-to=/dev/ +(venv) $ sudo -E env PATH=$PATH python3 -m tools.image_packer build-cache-src --image-dir=./images_dir --write-to=/dev/ ``` -This will build the offline OTA image with `p1_image.tgz` and `p2_image.tgz`, which are for ECU `p1` and `p2`, and then prepare the `/dev/` as external cache source device(ext4 filesystem labelled with `ota_cache_src`) with image rootfs exported to the filesystem on `/dev/`. +This will build the external cache source image OTA images listed in the `./images_dir`, and prepare the `/dev/` as external cache source device(ext4 filesystem labelled with `ota_cache_src`) with image rootfs exported to the filesystem on `/dev/`. -### Usage 3: Build the offline OTA image, export it as tar archive and prepare external cache source dev +## CLI Usage reference -```bash -# current folder layout: venv ota-client -(venv) $ cd ota-client -(venv) $ sudo -E env PATH=$PATH python3 -m tools.offline_ota_image_builder --image=p1:p1_image.tgz:ver_20230808 --image=p2:p2_image.tgz:ver_20230808 --output=t2.tar --write-to=/dev/ +### `build-cache-src` subcommand + +```text +usage: images_packer build-cache-src [-h] [--image ] [--image-dir ] [-o ] [-w ] [--force-write-to] + +build external OTA cache source recognized and used otaproxy. + +optional arguments: + -h, --help show this help message and exit + --image + (Optional) OTA image to be included in the external cache source, this argument can be specified multiple times to include multiple images. + --image-dir + (Optional) Specify a dir of OTA images to be included in the cache source. + -o , --output + save the generated image bundle tar archive to . + -w , --write-to + (Optional) write the external cache source image to and prepare the device, and then prepare the device to be used as external cache source storage. + --force-write-to (Optional) prepare as external cache source device without inter-active confirmation, only valid when used with -w option. ``` + +### `build-offline-ota-imgs-bundle` subcommand + +```text +usage: images_packer build-offline-ota-imgs-bundle [-h] --image :[:] [-o ] + +build OTA image bundle for offline OTA use. + +optional arguments: + -h, --help show this help message and exit + --image :[:] + OTA image for as tar archive(compressed or uncompressed), this option can be used multiple times to include multiple images. + -o , --output + save the generated image bundle tar archive to . +``` \ No newline at end of file From 6141adbba14166e0bfab19c4f336df457f1a10ed Mon Sep 17 00:00:00 2001 From: Bodong YANG Date: Mon, 25 Dec 2023 07:50:58 +0000 Subject: [PATCH 08/16] minor update to README.md --- tools/images_packer/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/images_packer/README.md b/tools/images_packer/README.md index 5867ec55f..6fc53a094 100644 --- a/tools/images_packer/README.md +++ b/tools/images_packer/README.md @@ -4,6 +4,7 @@ The input OTA images are expected to be built by ota-metadata(or images compatible with OTA image specification). This package provide the following two packing mode, each for different purposes: + 1. `build-cache-src`: with this mode, the built image can be recognized and utilized by otaproxy to reduce network downloading during OTA update. The following features are provided in this mode: @@ -14,7 +15,7 @@ This package provide the following two packing mode, each for different purposes 3. prepare and export built image onto specific block device as external cache source device recognized by otaproxy. -2. `build-offline-ota-imgs-bundle`: with this mode, the built image can be used by the `fully offline OTA helper` script to trigger fullly offline OTA locally without any network. +2. `build-offline-ota-imgs-bundle`: with this mode, the built image can be used by the `fully offline OTA helper` script to trigger fullly offline OTA locally without any network. The following feature are provided in this mode: @@ -71,7 +72,6 @@ The rootfs of the built image has the following layout: Both external cache source image and offline OTA image bundle use this image bundle layout. It means that offline OTA image can also be used as external cache source recognized by otaproxy(but user still needs to prepare the device either by themselves and extract the offline OTA image rootfs onto the prepared device, or prepare the device with this **images_packer** package). - ## OTA image bundle Manifest schema @@ -193,4 +193,4 @@ optional arguments: OTA image for as tar archive(compressed or uncompressed), this option can be used multiple times to include multiple images. -o , --output save the generated image bundle tar archive to . -``` \ No newline at end of file +``` From 6214184d438812a69b971198edd145c70f2c2a4a Mon Sep 17 00:00:00 2001 From: Bodong YANG Date: Mon, 25 Dec 2023 08:13:03 +0000 Subject: [PATCH 09/16] fix image specifying args --- tools/images_packer/__main__.py | 34 +++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/tools/images_packer/__main__.py b/tools/images_packer/__main__.py index b8be70e59..278c7e356 100644 --- a/tools/images_packer/__main__.py +++ b/tools/images_packer/__main__.py @@ -108,25 +108,27 @@ def main_build_external_cache_src(args: argparse.Namespace): count = 0 # retrieves images from --image arg - for _image_fpath in args.image: - count_str = str(count) - image_metas.append(ImageMetadata(ecu_id=count_str)) - image_files[count_str] = _image_fpath - count += 1 + if args.image: + for _image_fpath in args.image: + count_str = str(count) + image_metas.append(ImageMetadata(ecu_id=count_str)) + image_files[count_str] = _image_fpath + count += 1 # retrieves images from --image-dir arg - _image_store_dpath = Path(args.image_dir) - if not _image_store_dpath.is_dir(): - logger.error( - f"ERR: specified ={_image_store_dpath} doesn't exist, abort" - ) - sys.exit(errno.EINVAL) + if args.image_dir: + _image_store_dpath = Path(args.image_dir) + if not _image_store_dpath.is_dir(): + logger.error( + f"ERR: specified ={_image_store_dpath} doesn't exist, abort" + ) + sys.exit(errno.EINVAL) - for _image_fpath in _image_store_dpath.glob("*"): - count_str = str(count) - image_metas.append(ImageMetadata(ecu_id=count_str)) - image_files[count_str] = _image_fpath - count += 1 + for _image_fpath in _image_store_dpath.glob("*"): + count_str = str(count) + image_metas.append(ImageMetadata(ecu_id=count_str)) + image_files[count_str] = _image_fpath + count += 1 if not image_metas: print("ERR: at least one valid image should be given, abort") From b3f0f3ee9a72deea3a0c1968e72b1714763a5cdc Mon Sep 17 00:00:00 2001 From: Bodong YANG Date: Mon, 25 Dec 2023 08:17:36 +0000 Subject: [PATCH 10/16] minor fix --- tools/images_packer/__main__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/images_packer/__main__.py b/tools/images_packer/__main__.py index 278c7e356..4ea905463 100644 --- a/tools/images_packer/__main__.py +++ b/tools/images_packer/__main__.py @@ -173,6 +173,7 @@ def command_build_cache_src( subparser: argparse.ArgumentParser = subparsers.add_parser( name=cmd, description="build external OTA cache source recognized and used otaproxy.", + argument_default=None, ) subparser.add_argument( "--image", @@ -193,6 +194,7 @@ def command_build_cache_src( subparser.add_argument( "-o", "--output", + required=False, help="save the generated image bundle tar archive to .", metavar="", ) @@ -225,6 +227,7 @@ def command_build_offline_ota_image_bundle( subparser: argparse.ArgumentParser = subparsers.add_parser( name=cmd, description="build OTA image bundle for offline OTA use.", + argument_default=None, ) subparser.add_argument( "--image", @@ -239,6 +242,7 @@ def command_build_offline_ota_image_bundle( subparser.add_argument( "-o", "--output", + required=True, help="save the generated image bundle tar archive to .", metavar="", ) From d5416d01f177f9d0c3341f35be7be747f6925a16 Mon Sep 17 00:00:00 2001 From: Bodong YANG Date: Mon, 25 Dec 2023 08:21:54 +0000 Subject: [PATCH 11/16] use images_packer as tmp prefix --- tools/images_packer/__main__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/images_packer/__main__.py b/tools/images_packer/__main__.py index 4ea905463..867d3f144 100644 --- a/tools/images_packer/__main__.py +++ b/tools/images_packer/__main__.py @@ -78,7 +78,7 @@ def main_build_offline_ota_image_bundle(args: argparse.Namespace): sys.exit(errno.EINVAL) # ------ build image ------ # - with tempfile.TemporaryDirectory(prefix="offline_OTA_image_builder") as workdir: + with tempfile.TemporaryDirectory(prefix="images_packer") as workdir: build( image_metas, image_files, @@ -152,7 +152,7 @@ def main_build_external_cache_src(args: argparse.Namespace): ) # ------ build image ------ # - with tempfile.TemporaryDirectory(prefix="offline_OTA_image_builder") as workdir: + with tempfile.TemporaryDirectory(prefix="images_packer") as workdir: build( image_metas, image_files, From a9edc6bcabd49d881e17700e266eb214f03b789f Mon Sep 17 00:00:00 2001 From: Bodong YANG Date: Tue, 26 Dec 2023 05:21:08 +0000 Subject: [PATCH 12/16] images_packer.build_offline_ota_imgs_bundle: add a check for multiple target OTA image being specified to specific ECU --- tools/images_packer/__main__.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tools/images_packer/__main__.py b/tools/images_packer/__main__.py index 867d3f144..c8779720f 100644 --- a/tools/images_packer/__main__.py +++ b/tools/images_packer/__main__.py @@ -69,6 +69,13 @@ def main_build_offline_ota_image_bundle(args: argparse.Namespace): image_version=_image_version, ) ) + if _ecu_id in image_files: + logger.warning( + ( + f"override previously set OTA target image for ECU@{_ecu_id} " + f"from {image_files[_ecu_id]} to {_image_fpath}" + ) + ) image_files[_ecu_id] = _image_fpath else: logger.warning(f"ignore illegal image pair: {raw_pair}") @@ -233,7 +240,9 @@ def command_build_offline_ota_image_bundle( "--image", help=( "OTA image for as tar archive(compressed or uncompressed), " - "this option can be used multiple times to include multiple images." + "this option can be used multiple times to include multiple images. \n" + "NOTE: if multiple OTA target image is specified for the same ECU, the later one " + "will override the previous set one." ), required=True, metavar=":[:]", From 9abd24f0256917bf24808fa21c29a9aa84acbfea Mon Sep 17 00:00:00 2001 From: Bodong YANG Date: Tue, 26 Dec 2023 05:22:31 +0000 Subject: [PATCH 13/16] update README.md --- tools/images_packer/README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tools/images_packer/README.md b/tools/images_packer/README.md index 6fc53a094..e0be9e8db 100644 --- a/tools/images_packer/README.md +++ b/tools/images_packer/README.md @@ -133,7 +133,7 @@ This image builder requires latest ota-client to be installed/accessable. The re ```bash # current folder layout: venv ota-client (venv) $ cd ota-client -(venv) $ sudo -E env PATH=$PATH python3 -m tools.image_packer build-cache-src --image-dir=./images_dir --output=t2.tar +(venv) $ sudo -E env PATH=$PATH python3 -m tools.images_packer build-cache-src --image-dir=./images_dir --output=t2.tar ``` This will build the external cache source image with images listed in `./images_dir`, and export the built image as `t2.tar` tar archive. @@ -143,17 +143,17 @@ This will build the external cache source image with images listed in `./images_ ```bash # current folder layout: venv ota-client (venv) $ cd ota-client -(venv) $ sudo -E env PATH=$PATH python3 -m tools.image_packer build-offline-ota-imgs-bundle --image=p1:p1_image.tgz:ver_20230808 --image=p2:p2_image.tgz:ver_20230808 --output=t2.tar +(venv) $ sudo -E env PATH=$PATH python3 -m tools.images_packer build-offline-ota-imgs-bundle --image=p1:p1_image.tgz:ver_20230808 --image=p2:p2_image.tgz:ver_20230808 --output=t2.tar ``` This will build the offline OTA image with `p1_image.tgz` and `p2_image.tgz`, which are for ECU `p1` and `p2`, and export the built image as `t2.tar` tar archive. -#### Usage 3: Build the external cache source image and create external cache source dev +#### Usage 3: Build the external cache source image and create external cache source device ```bash # current folder layout: venv ota-client (venv) $ cd ota-client -(venv) $ sudo -E env PATH=$PATH python3 -m tools.image_packer build-cache-src --image-dir=./images_dir --write-to=/dev/ +(venv) $ sudo -E env PATH=$PATH python3 -m tools.images_packer build-cache-src --image-dir=./images_dir --write-to=/dev/ ``` This will build the external cache source image OTA images listed in the `./images_dir`, and prepare the `/dev/` as external cache source device(ext4 filesystem labelled with `ota_cache_src`) with image rootfs exported to the filesystem on `/dev/`. @@ -183,7 +183,7 @@ optional arguments: ### `build-offline-ota-imgs-bundle` subcommand ```text -usage: images_packer build-offline-ota-imgs-bundle [-h] --image :[:] [-o ] +usage: images_packer build-offline-ota-imgs-bundle [-h] --image :[:] -o build OTA image bundle for offline OTA use. @@ -191,6 +191,7 @@ optional arguments: -h, --help show this help message and exit --image :[:] OTA image for as tar archive(compressed or uncompressed), this option can be used multiple times to include multiple images. + NOTE: if multiple OTA target image is specified for the same ECU, the later one will override the previous set one. -o , --output save the generated image bundle tar archive to . ``` From 707857c889682e8d5faaf3f63deba242e798fca5 Mon Sep 17 00:00:00 2001 From: "hiroyuki.obinata" Date: Tue, 26 Dec 2023 14:34:58 +0900 Subject: [PATCH 14/16] add bash script for external cache --- tools/images_packer/build-cache.sh | 61 ++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100755 tools/images_packer/build-cache.sh diff --git a/tools/images_packer/build-cache.sh b/tools/images_packer/build-cache.sh new file mode 100755 index 000000000..54c15a00f --- /dev/null +++ b/tools/images_packer/build-cache.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +set -eu + +usage() { + echo "\ +Usage: + sudo $0 (images_dir|image_path) device +images_dir: Directory that contains rootfs tgz images. +images_path: rootfs tgz image file. +device: External cache storage device to be created. e.g. /dev/sdc." +} + +if [ $(id -u) -ne 0 ]; then + echo "Error: \"sudo\" must be specified." + usage + exit 1 +fi + +if [ $# -ne 2 ]; then + echo "Error: invalid argument" + usage + exit 1 +fi + +IMAGE=$1 +DEVICE=$2 + +UUID=$(lsblk ${DEVICE} -n -o UUID || true) +if [ "${UUID}" = "" ]; then + echo "Error: device(=${DEVICE})" + usage + exit 1 +fi + +if grep -q ${UUID} /etc/fstab; then + echo "Error: device(=${DEVICE}) is a internal device" + exit 1 +fi + +if grep -q ${DEVICE} /etc/fstab; then + echo "Error: device(=${DEVICE}) is a internal device" + exit 1 +fi + +SCRIPT_DIR=$(dirname $(realpath $0)) +ROOT_DIR=${SCRIPT_DIR}/../.. + +cd ${ROOT_DIR} +python3 -m venv venv +source venv/bin/activate + +pip install . -q + +if [ -d ${IMAGE} ]; then + python -m tools.images_packer build-cache-src --image-dir=${IMAGE} --write-to=${DEVICE} --force-write-to +elif [ -f ${IMAGE} ]; then + python -m tools.images_packer build-cache-src --image=${IMAGE} --write-to=${DEVICE} --force-write-to +else + usage +fi From 786ef4d160af649bf21ca82f42b5e30b34eca644 Mon Sep 17 00:00:00 2001 From: Bodong YANG Date: Tue, 26 Dec 2023 05:52:19 +0000 Subject: [PATCH 15/16] explicitly reject setting multiple OTA target image to the same ECU --- tools/images_packer/__main__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/images_packer/__main__.py b/tools/images_packer/__main__.py index c8779720f..b03a3e34e 100644 --- a/tools/images_packer/__main__.py +++ b/tools/images_packer/__main__.py @@ -70,12 +70,14 @@ def main_build_offline_ota_image_bundle(args: argparse.Namespace): ) ) if _ecu_id in image_files: - logger.warning( + logger.error( ( - f"override previously set OTA target image for ECU@{_ecu_id} " - f"from {image_files[_ecu_id]} to {_image_fpath}" + f"ERR: OTA target image for ECU@{_ecu_id} is already set to" + f"from {image_files[_ecu_id]}, please check the params" ) ) + sys.exit(errno.EINVAL) + image_files[_ecu_id] = _image_fpath else: logger.warning(f"ignore illegal image pair: {raw_pair}") From a13f3bbfeb5975d82c6a8319670969f2cb39c32f Mon Sep 17 00:00:00 2001 From: Bodong YANG Date: Tue, 26 Dec 2023 05:54:14 +0000 Subject: [PATCH 16/16] update usage --- tools/images_packer/README.md | 2 +- tools/images_packer/__main__.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/images_packer/README.md b/tools/images_packer/README.md index e0be9e8db..a308c977b 100644 --- a/tools/images_packer/README.md +++ b/tools/images_packer/README.md @@ -191,7 +191,7 @@ optional arguments: -h, --help show this help message and exit --image :[:] OTA image for as tar archive(compressed or uncompressed), this option can be used multiple times to include multiple images. - NOTE: if multiple OTA target image is specified for the same ECU, the later one will override the previous set one. + NOTE: multiple OTA target image specified for the same ECU is unexpected behavior. -o , --output save the generated image bundle tar archive to . ``` diff --git a/tools/images_packer/__main__.py b/tools/images_packer/__main__.py index b03a3e34e..bd3f3f5ab 100644 --- a/tools/images_packer/__main__.py +++ b/tools/images_packer/__main__.py @@ -73,7 +73,7 @@ def main_build_offline_ota_image_bundle(args: argparse.Namespace): logger.error( ( f"ERR: OTA target image for ECU@{_ecu_id} is already set to" - f"from {image_files[_ecu_id]}, please check the params" + f"{image_files[_ecu_id]}, please check the --image params" ) ) sys.exit(errno.EINVAL) @@ -243,8 +243,7 @@ def command_build_offline_ota_image_bundle( help=( "OTA image for as tar archive(compressed or uncompressed), " "this option can be used multiple times to include multiple images. \n" - "NOTE: if multiple OTA target image is specified for the same ECU, the later one " - "will override the previous set one." + "NOTE: multiple OTA target image specified for the same ECU is unexpected behavior" ), required=True, metavar=":[:]",