From 33497b163f1395179e761145e7ac6760b28198fb Mon Sep 17 00:00:00 2001 From: Bodong Yang <86948717+Bodong-Yang@users.noreply.github.com> Date: Tue, 26 Dec 2023 15:02:12 +0900 Subject: [PATCH] [Chore] tools.images_packer: refine previous offline_ota_image_builder to support 2 image packing mode (#267) This PR refines the tools.offline_ota_image_builder(renamed to images_packer) to explicitly provides two image packing mode(each requires different command args), which: * build-cache-src: build external cache source image and (optionally) prepare specific device as external cache source device, * build-offline-ota-imgs-bundle: build offline OTA image bundle for offline OTA. --- tools/images_packer/README.md | 197 +++++++++++ .../__init__.py | 0 tools/images_packer/__main__.py | 312 ++++++++++++++++++ tools/images_packer/build-cache.sh | 61 ++++ .../builder.py | 17 +- .../configs.py | 0 .../manifest.py | 2 +- .../utils.py | 19 ++ tools/offline_ota_image_builder/README.md | 175 ---------- tools/offline_ota_image_builder/__main__.py | 166 ---------- 10 files changed, 591 insertions(+), 358 deletions(-) create mode 100644 tools/images_packer/README.md rename tools/{offline_ota_image_builder => images_packer}/__init__.py (100%) create mode 100644 tools/images_packer/__main__.py create mode 100755 tools/images_packer/build-cache.sh rename tools/{offline_ota_image_builder => images_packer}/builder.py (95%) rename tools/{offline_ota_image_builder => images_packer}/configs.py (100%) rename tools/{offline_ota_image_builder => images_packer}/manifest.py (97%) rename tools/{offline_ota_image_builder => images_packer}/utils.py (65%) delete mode 100644 tools/offline_ota_image_builder/README.md delete mode 100644 tools/offline_ota_image_builder/__main__.py diff --git a/tools/images_packer/README.md b/tools/images_packer/README.md new file mode 100644 index 000000000..a308c977b --- /dev/null +++ b/tools/images_packer/README.md @@ -0,0 +1,197 @@ +# OTA images packer + +`tools.images_packer` is a helper package for bundling multiple images into one single images bundle. +The input OTA images are expected to be built by ota-metadata(or images compatible with OTA image specification). + +This package provide the following two packing mode, each for different purposes: + +1. `build-cache-src`: with this mode, the built image can be recognized and utilized by otaproxy to reduce network downloading during OTA update. + + The following features are provided in this mode: + + 1. build external cache source image, + + 2. export the built image as tar archive, + + 3. prepare and export built image onto specific block device as external cache source device recognized by otaproxy. + +2. `build-offline-ota-imgs-bundle`: with this mode, the built image can be used by the `fully offline OTA helper` script to trigger fullly offline OTA locally without any network. + + The following feature are provided in this mode: + + 1. build offline OTA image, + + 2. export the built image as tar archive. + + **NOTE**: full offline OTA helper util is not yet implemented. + + + +## Expected image layout for input OTA images + +This package currently only recognizes the OTA images generated by [ota-metadata@a711013](https://github.com/tier4/ota-metadata/commit/a711013), or the image layout compatible with the ota-metadata and OTA image layout specification as follow. + +```text +Example OTA image layout: +. +├── data +│ ├── <...files with full paths...> +│ └── ... +├── data.zst +│ ├── <... compressed OTA file ...> +│ └── <... naming: .zst ...> +├── certificate.pem +├── dirs.txt +├── metadata.jwt +├── persistents.txt +├── regulars.txt +└── symlinks.txt +``` + +Please also refere to [ota-metadata](https://github.com/tier4/ota-metadata) repo for the implementation of OTA image layout specification. + +## OTA image bundle layout + +The rootfs of the built image has the following layout: + +```text +. +├── manifest.json +├── data +│ ├── # uncompressed file +│ ├── .zst # zst compressed file +│ └── ... +└── meta + ├── # corresponding to the order in manifest.image_meta + │ └── + └── ... + └── + +``` + +Both external cache source image and offline OTA image bundle use this image bundle layout. +It means that offline OTA image can also be used as external cache source recognized by otaproxy(but user still needs to prepare the device either by themselves and extract the offline OTA image rootfs onto the prepared device, or prepare the device with this **images_packer** package). + + + +## OTA image bundle Manifest schema + +Offline OTA image's rootfs contains a `manifest.json` file with a single JSON object in it. This file includes the information related to this offline OTA image, including the bundled OTA images' metadata. + +```json +{ + "schema_version": 1, + "image_layout_version": 1, + "build_timestamp": 1693291679, + "data_size": 1122334455, + "data_files_num": 12345, + "meta_size": 112233, + "image_meta": [ + { + "ecu_id": "", + "image_version": "", + "ota_metadata_version": 1, + }, + ... + ], +} +``` + + + +## How to use `images_packer` + +### Prepare the dependencies + +This image builder requires latest ota-client to be installed/accessable. The recommended way to install dependencies is to init a virtual environment and install the ota-client into this venv, and then execute the image_builder with this venv. + +1. git clone the latest ota-client repository: + + ```bash + $ git clone https://github.com/tier4/ota-client.git + # image builder is located at ota-client/tools/offline_ota_image_builder + ``` + +2. prepare the virtual environment and activate it: + + ```bash + $ python3 -m venv venv + $ . venv/bin/active + (venv) $ + ``` + +3. install the ota-client into the virtual environment: + + ```bash + (venv) $ pip install ota-client + ``` + +### Examples of building and exporting image + +#### Usage 1: Build external cache source image and export it as tar archive + +```bash +# current folder layout: venv ota-client +(venv) $ cd ota-client +(venv) $ sudo -E env PATH=$PATH python3 -m tools.images_packer build-cache-src --image-dir=./images_dir --output=t2.tar +``` + +This will build the external cache source image with images listed in `./images_dir`, and export the built image as `t2.tar` tar archive. + +#### Usage 2: Build offline OTA images bundle and export it as tar archive + +```bash +# current folder layout: venv ota-client +(venv) $ cd ota-client +(venv) $ sudo -E env PATH=$PATH python3 -m tools.images_packer build-offline-ota-imgs-bundle --image=p1:p1_image.tgz:ver_20230808 --image=p2:p2_image.tgz:ver_20230808 --output=t2.tar +``` + +This will build the offline OTA image with `p1_image.tgz` and `p2_image.tgz`, which are for ECU `p1` and `p2`, and export the built image as `t2.tar` tar archive. + +#### Usage 3: Build the external cache source image and create external cache source device + +```bash +# current folder layout: venv ota-client +(venv) $ cd ota-client +(venv) $ sudo -E env PATH=$PATH python3 -m tools.images_packer build-cache-src --image-dir=./images_dir --write-to=/dev/ +``` + +This will build the external cache source image OTA images listed in the `./images_dir`, and prepare the `/dev/` as external cache source device(ext4 filesystem labelled with `ota_cache_src`) with image rootfs exported to the filesystem on `/dev/`. + +## CLI Usage reference + +### `build-cache-src` subcommand + +```text +usage: images_packer build-cache-src [-h] [--image ] [--image-dir ] [-o ] [-w ] [--force-write-to] + +build external OTA cache source recognized and used otaproxy. + +optional arguments: + -h, --help show this help message and exit + --image + (Optional) OTA image to be included in the external cache source, this argument can be specified multiple times to include multiple images. + --image-dir + (Optional) Specify a dir of OTA images to be included in the cache source. + -o , --output + save the generated image bundle tar archive to . + -w , --write-to + (Optional) write the external cache source image to and prepare the device, and then prepare the device to be used as external cache source storage. + --force-write-to (Optional) prepare as external cache source device without inter-active confirmation, only valid when used with -w option. +``` + +### `build-offline-ota-imgs-bundle` subcommand + +```text +usage: images_packer build-offline-ota-imgs-bundle [-h] --image :[:] -o + +build OTA image bundle for offline OTA use. + +optional arguments: + -h, --help show this help message and exit + --image :[:] + OTA image for as tar archive(compressed or uncompressed), this option can be used multiple times to include multiple images. + NOTE: multiple OTA target image specified for the same ECU is unexpected behavior. + -o , --output + save the generated image bundle tar archive to . +``` diff --git a/tools/offline_ota_image_builder/__init__.py b/tools/images_packer/__init__.py similarity index 100% rename from tools/offline_ota_image_builder/__init__.py rename to tools/images_packer/__init__.py diff --git a/tools/images_packer/__main__.py b/tools/images_packer/__main__.py new file mode 100644 index 000000000..bd3f3f5ab --- /dev/null +++ b/tools/images_packer/__main__.py @@ -0,0 +1,312 @@ +# Copyright 2022 TIER IV, INC. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""External cache source builder helper. + +Please refere to https://github.com/tier4/ota-metadata for +OTA image layout specification. +This tool is implemented based on ota-metadata@a711013. + +Example OTA image layout: +. +├── data +│ ├── <...files with full paths...> +│ └── ... +├── data.zst +│ ├── <... compressed OTA file ...> +│ └── <... naming: .zst ...> +├── certificate.pem +├── dirs.txt +├── metadata.jwt +├── persistents.txt +├── regulars.txt +└── symlinks.txt + +Check README.md for the offline OTA image layout specification. +Please refer to OTA cache design doc for more details. +""" + + +from __future__ import annotations +import argparse +import errno +import logging +import sys +import tempfile +from pathlib import Path +from typing import Callable + +from .configs import cfg +from .builder import build +from .manifest import ImageMetadata +from .utils import check_if_mounted, umount + +logger = logging.getLogger(__name__) + + +def main_build_offline_ota_image_bundle(args: argparse.Namespace): + # ------ parse input image options ------ # + image_metas = [] + image_files = {} + + for raw_pair in args.image: + _parsed = str(raw_pair).split(":") + if len(_parsed) >= 2: + _ecu_id, _image_fpath, _image_version, *_ = *_parsed, "" + image_metas.append( + ImageMetadata( + ecu_id=_ecu_id, + image_version=_image_version, + ) + ) + if _ecu_id in image_files: + logger.error( + ( + f"ERR: OTA target image for ECU@{_ecu_id} is already set to" + f"{image_files[_ecu_id]}, please check the --image params" + ) + ) + sys.exit(errno.EINVAL) + + image_files[_ecu_id] = _image_fpath + else: + logger.warning(f"ignore illegal image pair: {raw_pair}") + + if not image_metas: + print("ERR: at least one valid image should be given, abort") + sys.exit(errno.EINVAL) + + # ------ build image ------ # + with tempfile.TemporaryDirectory(prefix="images_packer") as workdir: + build( + image_metas, + image_files, + workdir=workdir, + output=args.output, + ) + + +def main_build_external_cache_src(args: argparse.Namespace): + # ------ args check ------ # + if not (args.output or args.write_to): + logger.error("ERR: at least one export option should be specified") + sys.exit(errno.EINVAL) + + if args.write_to: + write_to_dev = Path(args.write_to) + if not write_to_dev.is_block_device(): + logger.error(f"{write_to_dev=} is not a block device, abort") + sys.exit(errno.EINVAL) + if check_if_mounted(write_to_dev): + logger.warning(f"{write_to_dev=} is mounted, try to umount it...") + umount(write_to_dev) + + # ------ parse args ------ # + image_metas: list[ImageMetadata] = [] + image_files: dict[str, Path] = {} + count = 0 + + # retrieves images from --image arg + if args.image: + for _image_fpath in args.image: + count_str = str(count) + image_metas.append(ImageMetadata(ecu_id=count_str)) + image_files[count_str] = _image_fpath + count += 1 + + # retrieves images from --image-dir arg + if args.image_dir: + _image_store_dpath = Path(args.image_dir) + if not _image_store_dpath.is_dir(): + logger.error( + f"ERR: specified ={_image_store_dpath} doesn't exist, abort" + ) + sys.exit(errno.EINVAL) + + for _image_fpath in _image_store_dpath.glob("*"): + count_str = str(count) + image_metas.append(ImageMetadata(ecu_id=count_str)) + image_files[count_str] = _image_fpath + count += 1 + + if not image_metas: + print("ERR: at least one valid image should be given, abort") + sys.exit(errno.EINVAL) + + # ------ parse export options ------ # + output_fpath, write_to_dev = args.output, args.write_to + + if write_to_dev and not args.force_write_to: + _confirm_write_to = input( + f"WARNING: generated image will be written to {write_to_dev}, \n" + f"\t all data in {write_to_dev} will be lost, continue(type [Y] or [N]): " + ) + if _confirm_write_to != "Y": + print(f"decline writing to {write_to_dev}, abort") + sys.exit(errno.EINVAL) + elif write_to_dev and args.force_write_to: + logger.warning( + f"generated image will be written to {write_to_dev}," + f"all data in {write_to_dev} will be lost" + ) + + # ------ build image ------ # + with tempfile.TemporaryDirectory(prefix="images_packer") as workdir: + build( + image_metas, + image_files, + workdir=workdir, + output=output_fpath, + ) + + +# +# ------ subcommands definition ------ # +# + + +def command_build_cache_src( + subparsers: argparse._SubParsersAction, +) -> tuple[str, argparse.ArgumentParser]: + cmd = "build-cache-src" + subparser: argparse.ArgumentParser = subparsers.add_parser( + name=cmd, + description="build external OTA cache source recognized and used otaproxy.", + argument_default=None, + ) + subparser.add_argument( + "--image", + help=( + "(Optional) OTA image to be included in the external cache source, " + "this argument can be specified multiple times to include multiple images." + ), + required=False, + metavar="", + action="append", + ) + subparser.add_argument( + "--image-dir", + help="(Optional) Specify a dir of OTA images to be included in the cache source.", + required=False, + metavar="", + ) + subparser.add_argument( + "-o", + "--output", + required=False, + help="save the generated image bundle tar archive to .", + metavar="", + ) + subparser.add_argument( + "-w", + "--write-to", + help=( + "(Optional) write the external cache source image to and prepare the device, " + "and then prepare the device to be used as external cache source storage." + ), + required=False, + metavar="", + ) + subparser.add_argument( + "--force-write-to", + help=( + "(Optional) prepare as external cache source device without inter-active confirmation, " + "only valid when used with -w option." + ), + required=False, + action="store_true", + ) + return cmd, subparser + + +def command_build_offline_ota_image_bundle( + subparsers: argparse._SubParsersAction, +) -> tuple[str, argparse.ArgumentParser]: + cmd = "build-offline-ota-imgs-bundle" + subparser: argparse.ArgumentParser = subparsers.add_parser( + name=cmd, + description="build OTA image bundle for offline OTA use.", + argument_default=None, + ) + subparser.add_argument( + "--image", + help=( + "OTA image for as tar archive(compressed or uncompressed), " + "this option can be used multiple times to include multiple images. \n" + "NOTE: multiple OTA target image specified for the same ECU is unexpected behavior" + ), + required=True, + metavar=":[:]", + action="append", + ) + subparser.add_argument( + "-o", + "--output", + required=True, + help="save the generated image bundle tar archive to .", + metavar="", + ) + return cmd, subparser + + +def register_handler( + _cmd: str, subparser: argparse.ArgumentParser, *, handler: Callable +): + subparser.set_defaults(handler=handler) + + +def get_handler(args: argparse.Namespace) -> Callable[[argparse.Namespace], None]: + try: + return args.handler + except AttributeError: + print("ERR: image packing mode is not specified, check -h for more details") + sys.exit(errno.EINVAL) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format=cfg.LOGGING_FORMAT, force=True) + + # ------ init main parser ------ # + main_parser = argparse.ArgumentParser( + prog="images_packer", + description=( + "Helper script that builds OTA images bundle from given OTA image(s), " + "start from choosing different image packing modes." + ), + ) + + # ------ define subcommands ------ # + subparsers = main_parser.add_subparsers( + title="Image packing modes", + ) + register_handler( + *command_build_cache_src(subparsers), + handler=main_build_external_cache_src, + ) + register_handler( + *command_build_offline_ota_image_bundle(subparsers), + handler=main_build_offline_ota_image_bundle, + ) + + args = main_parser.parse_args() + + # check common args + if args.output and (output := Path(args.output)).exists(): + print(f"ERR: {output} exists, abort") + sys.exit(errno.EINVAL) + + try: + get_handler(args)(args) + except KeyboardInterrupt: + print("ERR: aborted by user") + raise diff --git a/tools/images_packer/build-cache.sh b/tools/images_packer/build-cache.sh new file mode 100755 index 000000000..54c15a00f --- /dev/null +++ b/tools/images_packer/build-cache.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +set -eu + +usage() { + echo "\ +Usage: + sudo $0 (images_dir|image_path) device +images_dir: Directory that contains rootfs tgz images. +images_path: rootfs tgz image file. +device: External cache storage device to be created. e.g. /dev/sdc." +} + +if [ $(id -u) -ne 0 ]; then + echo "Error: \"sudo\" must be specified." + usage + exit 1 +fi + +if [ $# -ne 2 ]; then + echo "Error: invalid argument" + usage + exit 1 +fi + +IMAGE=$1 +DEVICE=$2 + +UUID=$(lsblk ${DEVICE} -n -o UUID || true) +if [ "${UUID}" = "" ]; then + echo "Error: device(=${DEVICE})" + usage + exit 1 +fi + +if grep -q ${UUID} /etc/fstab; then + echo "Error: device(=${DEVICE}) is a internal device" + exit 1 +fi + +if grep -q ${DEVICE} /etc/fstab; then + echo "Error: device(=${DEVICE}) is a internal device" + exit 1 +fi + +SCRIPT_DIR=$(dirname $(realpath $0)) +ROOT_DIR=${SCRIPT_DIR}/../.. + +cd ${ROOT_DIR} +python3 -m venv venv +source venv/bin/activate + +pip install . -q + +if [ -d ${IMAGE} ]; then + python -m tools.images_packer build-cache-src --image-dir=${IMAGE} --write-to=${DEVICE} --force-write-to +elif [ -f ${IMAGE} ]; then + python -m tools.images_packer build-cache-src --image=${IMAGE} --write-to=${DEVICE} --force-write-to +else + usage +fi diff --git a/tools/offline_ota_image_builder/builder.py b/tools/images_packer/builder.py similarity index 95% rename from tools/offline_ota_image_builder/builder.py rename to tools/images_packer/builder.py index 42c77d7f5..aa0b850d7 100644 --- a/tools/offline_ota_image_builder/builder.py +++ b/tools/images_packer/builder.py @@ -31,15 +31,6 @@ logger = logging.getLogger(__name__) -PROC_MOUNTS = "/proc/mounts" - - -def _check_if_mounted(dev: StrPath): - for line in Path(PROC_MOUNTS).read_text().splitlines(): - if line.find(str(dev)) != -1: - return True - return False - @contextmanager def _unarchive_image(image_fpath: StrPath, *, workdir: StrPath): @@ -133,12 +124,6 @@ def _write_image_to_dev(image_rootfs: StrPath, dev: StrPath, *, workdir: StrPath """Prepare and export generated image's rootfs to it.""" _start_time = time.time() dev = Path(dev) - if not dev.is_block_device(): - logger.warning(f"{dev=} is not a block device, skip") - return - if _check_if_mounted(dev): - logger.warning(f"{dev=} is mounted, skip") - return # prepare device format_device_cmd = f"mkfs.ext4 -L {cfg.EXTERNAL_CACHE_DEV_FSLABEL} {dev}" @@ -174,7 +159,7 @@ def build( *, workdir: StrPath, output: Optional[StrPath], - write_to_dev: Optional[StrPath], + write_to_dev: Optional[StrPath] = None, ): _start_time = time.time() logger.info(f"job started at {int(_start_time)}") diff --git a/tools/offline_ota_image_builder/configs.py b/tools/images_packer/configs.py similarity index 100% rename from tools/offline_ota_image_builder/configs.py rename to tools/images_packer/configs.py diff --git a/tools/offline_ota_image_builder/manifest.py b/tools/images_packer/manifest.py similarity index 97% rename from tools/offline_ota_image_builder/manifest.py rename to tools/images_packer/manifest.py index 3341ceffe..9c4e4fa9a 100644 --- a/tools/offline_ota_image_builder/manifest.py +++ b/tools/images_packer/manifest.py @@ -22,7 +22,7 @@ import json from dataclasses import dataclass, field, asdict -from typing import Dict, List +from typing import List from .configs import cfg diff --git a/tools/offline_ota_image_builder/utils.py b/tools/images_packer/utils.py similarity index 65% rename from tools/offline_ota_image_builder/utils.py rename to tools/images_packer/utils.py index e3d6667df..3e9449dc0 100644 --- a/tools/offline_ota_image_builder/utils.py +++ b/tools/images_packer/utils.py @@ -15,9 +15,12 @@ from os import PathLike +from pathlib import Path from typing import Union from typing_extensions import TypeAlias +from otaclient.app.common import subprocess_call + StrPath: TypeAlias = Union[str, PathLike] @@ -27,3 +30,19 @@ class InputImageProcessError(Exception): class ExportError(Exception): ... + + +PROC_MOUNTS = "/proc/mounts" + + +def check_if_mounted(dev: StrPath) -> bool: + """Perform a fast check to see if is mounted.""" + for line in Path(PROC_MOUNTS).read_text().splitlines(): + if line.find(str(dev)) != -1: + return True + return False + + +def umount(dev: StrPath): + _cmd = f"umount {dev}" + subprocess_call(_cmd, raise_exception=True) diff --git a/tools/offline_ota_image_builder/README.md b/tools/offline_ota_image_builder/README.md deleted file mode 100644 index b61d5970e..000000000 --- a/tools/offline_ota_image_builder/README.md +++ /dev/null @@ -1,175 +0,0 @@ -# Offline OTA image builder - -`tools.offline_ota_image_builder` is a helper package for bundling offline OTA image from the OTA images build by ota-metadata(or images compatible with OTA image specification). -Built image can be used as external cache source for otaproxy, or used by full offline OTA helper util to trigger fully offline OTA update. - -**NOTE**: full offline OTA helper util is not yet implemented. - -It provides the following features: - -1. build offline OTA image from the OTA images built by ota-metadata(or images compatible with OTA image specification), -2. export built offline OTA image as tar archive, -3. prepare and export built image onto specific block device as external cache source device for otaproxy. - - - -## Expected image layout for input OTA images - -This package currently only recognizes the OTA images generated by [ota-metadata@a711013](https://github.com/tier4/ota-metadata/commit/a711013), or the image layout compatible with the ota-metadata and OTA image layout specification as follow. - -```text -Example OTA image layout: -. -├── data -│ ├── <...files with full paths...> -│ └── ... -├── data.zst -│ ├── <... compressed OTA file ...> -│ └── <... naming: .zst ...> -├── certificate.pem -├── dirs.txt -├── metadata.jwt -├── persistents.txt -├── regulars.txt -└── symlinks.txt -``` - -Please also refere to [ota-metadata](https://github.com/tier4/ota-metadata) repo for the implementation of OTA image layout specification. - -## Offline OTA image layout - -The rootfs of the built image has the following layout: - -```text -. -├── manifest.json -├── data -│ ├── # uncompressed file -│ ├── .zst # zst compressed file -│ └── ... -└── meta - ├── # corresponding to the order in manifest.image_meta - │ └── - └── ... - └── - -``` - -This layout is compatible with external cache source's layout, which means any offline OTA image can natually directly be used as external cache source recognized by otaproxy(but user still needs to prepare the device either by themselves and extract the offline OTA image rootfs onto the prepared device, or prepare the device with this **offline_ota_image_builder** package). - - - -## Offline OTA image Manifest schema - -Offline OTA image's rootfs contains a `manifest.json` file with a single JSON object in it. This file includes the information related to this offline OTA image, including the bundled OTA images' metadata. - -```json -{ - "schema_version": 1, - "image_layout_version": 1, - "build_timestamp": 1693291679, - "data_size": 1122334455, - "data_files_num": 12345, - "meta_size": 112233, - "image_meta": [ - { - "ecu_id": "", - "image_version": "", - "ota_metadata_version": 1, - }, - ... - ], -} -``` - - - -## How to use - -### Prepare the dependencies - -This image builder requires latest ota-client to be installed/accessable. The recommended way to install dependencies is to init a virtual environment and install the ota-client into this venv, and then execute the image_builder with this venv. - -1. git clone the latest ota-client repository: - - ```bash - $ git clone https://github.com/tier4/ota-client.git - # image builder is located at ota-client/tools/offline_ota_image_builder - ``` - -2. prepare the virtual environment and activate it: - - ```bash - $ python3 -m venv venv - $ . venv/bin/active - (venv) $ - ``` - -3. install the ota-client into the virtual environment: - - ```bash - (venv) $ pip install ota-client - ``` - -### Build image and export - -#### Builder usage - -```text -usage: offline_ota_image_builder [-h] --image :[:] - [-o ] [-w ] [--confirm-write-to] - -Helper script that builds offline OTA image with given OTA image(s) as external cache source or for -offline OTA use. - -options: - -h, --help show this help message and exit - --image :[:] - OTA image for as tar archive(compressed or uncompressed), this - option can be used multiple times to include multiple images. - -o , --output - save the generated image rootfs into tar archive to . - -w , --write-to - write the image to and prepare the device as external cache source - device. - --force-write-to prepare as external cache source device without inter-active confirmation, - only valid when used with -w option. -``` - -Execute the image builder by directly calling it from the source code. This package requires `root` permission to run(required by extracting OTA image and preparing external cache source device). - -Option `--image :[:]` is used to specify OTA image to be included. This option can be used multiple times to specify multiple OTA images. - -Option `--write-to ` is used to prepare external cache source device used by otaproxy. The specified device will be formatted as `ext4`, fslabel with `ota_cache_src`, and be exported with the built offline OTA image's rootfs. If this package is used in a non-interactive script, option `--force-write-to` can be used to bypass interactive confirmation. - -Option `--output ` specify where to save the exported tar archive of built image. - -User must at least specifies one of `--write-to` and `--output`, or specifies them together. - -#### Usage 1: Build offline OTA image and export it as tar archive - -```bash -# current folder layout: venv ota-client -(venv) $ cd ota-client -(venv) $ sudo -E env PATH=$PATH python3 -m tools.offline_ota_image_builder --image=p1:p1_image.tgz:ver_20230808 --image=p2:p2_image.tgz:ver_20230808 --output=t2.tar -``` - -This will build the offline OTA image with `p1_image.tgz` and `p2_image.tgz`, which are for ECU `p1` and `p2`, and export the built image as `t2.tar` tar archive. - -### Usage 2: Build the offline OTA image and create external cache source dev - -```bash -# current folder layout: venv ota-client -(venv) $ cd ota-client -(venv) $ sudo -E env PATH=$PATH python3 -m tools.offline_ota_image_builder --image=p1:p1_image.tgz:ver_20230808 --image=p2:p2_image.tgz:ver_20230808 --write-to=/dev/ -``` - -This will build the offline OTA image with `p1_image.tgz` and `p2_image.tgz`, which are for ECU `p1` and `p2`, and then prepare the `/dev/` as external cache source device(ext4 filesystem labelled with `ota_cache_src`) with image rootfs exported to the filesystem on `/dev/`. - -### Usage 3: Build the offline OTA image, export it as tar archive and prepare external cache source dev - -```bash -# current folder layout: venv ota-client -(venv) $ cd ota-client -(venv) $ sudo -E env PATH=$PATH python3 -m tools.offline_ota_image_builder --image=p1:p1_image.tgz:ver_20230808 --image=p2:p2_image.tgz:ver_20230808 --output=t2.tar --write-to=/dev/ -``` diff --git a/tools/offline_ota_image_builder/__main__.py b/tools/offline_ota_image_builder/__main__.py deleted file mode 100644 index 5e316b471..000000000 --- a/tools/offline_ota_image_builder/__main__.py +++ /dev/null @@ -1,166 +0,0 @@ -# Copyright 2022 TIER IV, INC. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""External cache source builder helper. - -Please refere to https://github.com/tier4/ota-metadata for -OTA image layout specification. -This tool is implemented based on ota-metadata@a711013. - -Example OTA image layout: -. -├── data -│ ├── <...files with full paths...> -│ └── ... -├── data.zst -│ ├── <... compressed OTA file ...> -│ └── <... naming: .zst ...> -├── certificate.pem -├── dirs.txt -├── metadata.jwt -├── persistents.txt -├── regulars.txt -└── symlinks.txt - -Check README.md for the offline OTA image layout specification. -Please refer to OTA cache design doc for more details. -""" - - -import argparse -import errno -import logging -import sys -import tempfile -from pathlib import Path - -from .configs import cfg -from .builder import build -from .manifest import ImageMetadata - -logger = logging.getLogger(__name__) - - -def main(args): - # ------ parse input image options ------ # - image_metas = [] - image_files = {} - - for raw_pair in args.image: - _parsed = str(raw_pair).split(":") - if len(_parsed) >= 2: - _ecu_id, _image_fpath, _image_version, *_ = *_parsed, "" - image_metas.append( - ImageMetadata( - ecu_id=_ecu_id, - image_version=_image_version, - ) - ) - image_files[_ecu_id] = _image_fpath - else: - logger.warning(f"ignore illegal image pair: {raw_pair}") - - if not image_metas: - print("ERR: at least one valid image should be given, abort") - sys.exit(errno.EINVAL) - - # ------ parse export options ------ # - output_fpath, write_to_dev = args.output, args.write_to - - if write_to_dev and not Path(write_to_dev).is_block_device(): - print(f"{write_to_dev} is not a block device, abort") - sys.exit(errno.EINVAL) - - if write_to_dev and not args.force_write_to: - _confirm_write_to = input( - f"WARNING: generated image will be written to {write_to_dev}, \n" - f"\t all data in {write_to_dev} will be lost, continue(type [Y] or [N]): " - ) - if _confirm_write_to != "Y": - print(f"decline writing to {write_to_dev}, abort") - sys.exit(errno.EINVAL) - elif write_to_dev and args.force_write_to: - logger.warning( - f"generated image will be written to {write_to_dev}," - f"all data in {write_to_dev} will be lost" - ) - - # ------ build image ------ # - with tempfile.TemporaryDirectory(prefix="offline_OTA_image_builder") as workdir: - build( - image_metas, - image_files, - workdir=workdir, - output=output_fpath, - write_to_dev=write_to_dev, - ) - - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO, format=cfg.LOGGING_FORMAT, force=True) - parser = argparse.ArgumentParser( - prog="offline_ota_image_builder", - description=( - "Helper script that builds offline OTA image " - "with given OTA image(s) as external cache source or for offline OTA use." - ), - ) - parser.add_argument( - "--image", - help=( - "OTA image for as tar archive(compressed or uncompressed), " - "this option can be used multiple times to include multiple images." - ), - required=True, - metavar=":[:]", - action="append", - ) - parser.add_argument( - "-o", - "--output", - help="save the generated image rootfs into tar archive to .", - metavar="", - ) - parser.add_argument( - "-w", - "--write-to", - help=( - "write the image to and prepare the device as " - "external cache source device." - ), - metavar="", - ) - parser.add_argument( - "--force-write-to", - help=( - "prepare as external cache source device without inter-active confirmation, " - "only valid when used with -w option." - ), - action="store_true", - ) - args = parser.parse_args() - - # basic options check - if args.output and (output := Path(args.output)).is_file(): - print(f"ERR: {output} exists, abort") - sys.exit(errno.EINVAL) - - if not (args.output or args.write_to): - print("ERR: at least one export option should be specified") - sys.exit(errno.EINVAL) - - try: - main(args) - except KeyboardInterrupt: - print("ERR: aborted by user") - raise