From 376118d6de7a7e1949b5196f70943e462df2a797 Mon Sep 17 00:00:00 2001 From: Paymaun Date: Tue, 6 Sep 2022 17:16:42 -0700 Subject: [PATCH] Add experimental ADU manifest init command. (#571) * Improve help, add additional validation case. --- HISTORY.rst | 2 + azext_iot/deviceupdate/_help.py | 41 ++++ azext_iot/deviceupdate/command_map.py | 6 + azext_iot/deviceupdate/commands_update.py | 182 ++++++++++++++++- azext_iot/deviceupdate/params.py | 78 ++++++++ azext_iot/deviceupdate/providers/base.py | 34 +++- .../deviceupdate/test_adu_manifest_int.py | 186 ++++++++++++++++++ 7 files changed, 523 insertions(+), 6 deletions(-) create mode 100644 azext_iot/tests/deviceupdate/test_adu_manifest_int.py diff --git a/HISTORY.rst b/HISTORY.rst index 3b4fd2270..95e2c85f0 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -19,6 +19,8 @@ unreleased * The command `az iot device-update device class list` adds support for `--filter` when no `--group-id` is provided. * The parameters `--account`, `--instance`, and `--resource-group` support setting default overridable values via config. Use `az config set` i.e. `az config set defaults.adu_account=` or `az configure` i.e. `az configure --defaults adu_account=`. +* Introducing the experimental command `az iot device-update update init v5` for initializing (or generating) an import manifest + with the desired state. * Improved built-in documentation. diff --git a/azext_iot/deviceupdate/_help.py b/azext_iot/deviceupdate/_help.py index 46d09c9cb..60303ed67 100644 --- a/azext_iot/deviceupdate/_help.py +++ b/azext_iot/deviceupdate/_help.py @@ -822,3 +822,44 @@ def load_deviceupdate_help(): text: > az iot device-update device show -n {account_name} -i {instance_name} -d {device_id} """ + + helps["iot device-update update init"] = """ + type: group + short-summary: Utilities for initializing update import manifests. + """ + + helps["iot device-update update init v5"] = """ + type: command + short-summary: Initialize a v5 import manifest with the desired state. + long-summary: This command supports all attributes of the v5 import manifest. Review examples + and parameter descriptions for details on how to fully utilize the operation. Note that + there is positional sensitivity between --step and --file, as well as --file and --related-file. + Review parameter help for more details. + + examples: + - name: Initialize a minimum content import manifest. + text: > + az iot device-update update init v5 --update-provider Microsoft --update-name AptUpdate --update-version 1.0.0 + --description "My minimum update" + --compat deviceManufacturer=Contoso deviceModel=Vacuum + --step handler=microsoft/apt:1 --file path=/my/apt/manifest/file + + - name: Initialize a non-deployable leaf update to be referenced in a bundled update. + text: > + az iot device-update update init v5 --update-provider Microsoft --update-name SwUpdate --update-version 1.1 + --compat deviceManufacturer=Contoso deviceModel=Microphone --step handler=microsoft/swupdate:1 description="Deploy Update" + --file path=/my/update/image/file1 --file path=/my/update/image/file2 + --is-deployable false + + - name: Initialize a bundled update referencing a leaf update as well as defining independent steps. Inline json examples + compatible with bash-like shells. + text: > + az iot device-update update init v5 --update-provider Microsoft --update-name Bundled --update-version 2.0 + --compat deviceManufacturer=Contoso deviceModel=SpaceStation + --step handler=microsoft/script:1 properties='{"arguments": "--pre"}' description="Pre-install script" + --file path=/my/update/scripts/preinstall.sh downloadHandler=microsoft/delta:1 + --related-file path=/my/update/scripts/related_preinstall.json properties='{"microsoft.sourceFileHashAlgorithm": "sha256"}' + --step updateId.provider=Microsoft updateId.name=SwUpdate updateId.version=1.1 + --step handler=microsoft/script:1 properties='{"arguments": "--post"}' description="Post-install script" + --file path=/my/update/scripts/postinstall.sh + """ diff --git a/azext_iot/deviceupdate/command_map.py b/azext_iot/deviceupdate/command_map.py index 60c522485..e6e0b630c 100644 --- a/azext_iot/deviceupdate/command_map.py +++ b/azext_iot/deviceupdate/command_map.py @@ -105,6 +105,12 @@ def load_deviceupdate_commands(self, _): cmd_group.command("list", "list_update_files") cmd_group.show_command("show", "show_update_file") + with self.command_group( + "iot device-update update init", + command_type=deviceupdate_update_ops, + ) as cmd_group: + cmd_group.command("v5", "manifest_init_v5", is_experimental=True) + with self.command_group( "iot device-update device", command_type=deviceupdate_device_ops, diff --git a/azext_iot/deviceupdate/commands_update.py b/azext_iot/deviceupdate/commands_update.py index a07097eec..c5951951b 100644 --- a/azext_iot/deviceupdate/commands_update.py +++ b/azext_iot/deviceupdate/commands_update.py @@ -12,7 +12,7 @@ AzureError, ARMPolling, ) -from typing import Optional, List, Union +from typing import Optional, List, Union, Dict logger = get_logger(__name__) @@ -137,7 +137,7 @@ def import_update( if not size or not hashes: client_calculated_meta = data_manager.calculate_manifest_metadata(url) - hashes = data_manager.assemble_hashes(hash_list=hashes) or {"sha256": client_calculated_meta.hash} + hashes = data_manager.assemble_nargs_to_dict(hash_list=hashes) or {"sha256": client_calculated_meta.hash} size = size or client_calculated_meta.bytes manifest_metadata = DeviceUpdateDataModels.ImportManifestMetadata(url=url, size_in_bytes=size, hashes=hashes) @@ -210,3 +210,181 @@ def delete_update( return data_manager.data_client.device_update.begin_delete_update( name=update_name, provider=update_provider, version=update_version ) + + +def manifest_init_v5( + cmd, + update_name: str, + update_provider: str, + update_version: str, + compatibility: List[List[str]], + steps: List[List[str]], + files: List[List[str]] = None, + related_files: List[List[str]] = None, + description: str = None, + deployable: bool = None, +): + import json + from datetime import datetime + from pathlib import PurePath + from azure.cli.core.azclierror import ArgumentUsageError + + def _sanitize_safe_params(safe_params: list, keep: list) -> list: + ''' + Intended to filter un-related params, + leaving only related params with inherent positional indexing + to be used by the _associate_related function. + ''' + result: List[str] = [] + if not safe_params: + return result + for param in safe_params: + if param in keep: + result.append(param) + return result + + def _associate_related(sanitized_params: list, key: str) -> dict: + ''' + Intended to associate related param indexes. For example + associate --file with the nearest --step or associate --related-file + with the nearest --file. + ''' + result: Dict[int, list] = {} + if not sanitized_params: + return result + params_len = len(sanitized_params) + key_index = 0 + related_key_index = 0 + for i in range(params_len): + if sanitized_params[i] == key: + result[key_index] = [] + for j in range(i + 1, params_len): + if sanitized_params[j] == key: + break + result[key_index].append(related_key_index) + related_key_index = related_key_index + 1 + key_index = key_index + 1 + return result + + payload = {} + payload["manifestVersion"] = "5.0" + payload["createdDateTime"] = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") + payload["updateId"] = {} + payload["updateId"]["name"] = update_name + payload["updateId"]["provider"] = update_provider + payload["updateId"]["version"] = update_version + if deployable is False: + payload["isDeployable"] = False + if description: + payload["description"] = description + processed_compatibility = [] + for compat in compatibility: + if not compat or not compat[0]: + continue + processed_compatibility.append(DeviceUpdateDataManager.assemble_nargs_to_dict(compat)) + payload["compatibility"] = processed_compatibility + + safe_params = cmd.cli_ctx.data.get("safe_params", []) + processed_steps = [] + for s in range(len(steps)): + if not steps[s] or not steps[s][0]: + continue + + step_file_params = _sanitize_safe_params(safe_params, ["--step", "--file"]) + related_step_file_map = _associate_related(step_file_params, "--step") + + assembled_step = DeviceUpdateDataManager.assemble_nargs_to_dict(steps[s]) + step = {} + if all(k in assembled_step for k in ("updateId.provider", "updateId.name", "updateId.version")): + # reference step + step = { + "type": "reference", + "updateId": { + "provider": assembled_step["updateId.provider"], + "name": assembled_step["updateId.name"], + "version": assembled_step["updateId.version"], + }, + } + elif "handler" in assembled_step: + # inline step + step = { + "type": "inline", + "handler": assembled_step["handler"], + } + step["files"] = [f.strip() for f in assembled_step["files"].split(",")] if "files" in assembled_step else [] + if not step["files"]: + derived_step_files = [] + for f in related_step_file_map[s]: + step_file = files[f] + if not step_file or not step_file[0]: + continue + assembled_step_file = DeviceUpdateDataManager.assemble_nargs_to_dict(step_file) + if "path" in assembled_step_file: + derived_step_files.append(PurePath(assembled_step_file["path"]).name) + step["files"] = derived_step_files + + if "properties" in assembled_step: + step["handlerProperties"] = json.loads(assembled_step["properties"]) + + if not step: + raise ArgumentUsageError( + "Usage of --step requires at least an entry of handler= for an inline step or " + "all of updateId.provider=, updateId.name=, updateId.version= for a reference step.") + + step_desc = assembled_step.get("description") or assembled_step.get("desc") + if step_desc: + step["description"] = step_desc + processed_steps.append(step) + + payload["instructions"] = {} + payload["instructions"]["steps"] = processed_steps + + if files: + file_params = _sanitize_safe_params(safe_params, ["--file", "--related-file"]) + related_file_map = _associate_related(file_params, "--file") + + processed_files = [] + for f in range(len(files)): + if not files[f] or not files[f][0]: + continue + processed_file = {} + assembled_file = DeviceUpdateDataManager.assemble_nargs_to_dict(files[f]) + if "path" not in assembled_file: + raise ArgumentUsageError("When using --file path is required.") + assembled_file_metadata = DeviceUpdateDataManager.calculate_file_metadata(assembled_file["path"]) + processed_file["hashes"] = {"sha256": assembled_file_metadata.hash} + processed_file["filename"] = assembled_file_metadata.name + processed_file["sizeInBytes"] = assembled_file_metadata.bytes + + if "downloadHandler" in assembled_file: + processed_file["downloadHandler"] = {"id": assembled_file["downloadHandler"]} + + processed_related_files = [] + for r in related_file_map[f]: + related_file = related_files[r] + if not related_file or not related_file[0]: + continue + processed_related_file = {} + assembled_related_file = DeviceUpdateDataManager.assemble_nargs_to_dict(related_file) + if "path" not in assembled_related_file: + raise ArgumentUsageError("When using --related-file path is required.") + related_file_metadata = DeviceUpdateDataManager.calculate_file_metadata(assembled_related_file["path"]) + processed_related_file["hashes"] = {"sha256": related_file_metadata.hash} + processed_related_file["filename"] = related_file_metadata.name + processed_related_file["sizeInBytes"] = related_file_metadata.bytes + + if "properties" in assembled_related_file: + processed_related_file["properties"] = json.loads(assembled_related_file["properties"]) + + if processed_related_file: + processed_related_files.append(processed_related_file) + + if processed_related_files: + processed_file["relatedFiles"] = processed_related_files + + if processed_file: + processed_files.append(processed_file) + + payload["files"] = processed_files + + return payload diff --git a/azext_iot/deviceupdate/params.py b/azext_iot/deviceupdate/params.py index 9f301c549..f88c22347 100644 --- a/azext_iot/deviceupdate/params.py +++ b/azext_iot/deviceupdate/params.py @@ -440,3 +440,81 @@ def load_deviceupdate_arguments(self, _): options_list=["--description"], help="Description for the log collection operation.", ) + + with self.argument_context("iot device-update update init") as context: + context.argument( + "update_provider", + options_list=["--update-provider"], + help="The update provider as a component of updateId.", + ) + context.argument( + "update_name", + options_list=["--update-name"], + help="The update name as a component of updateId.", + ) + context.argument( + "update_version", + options_list=["--update-version"], + help="The update version as a component of updateId.", + ) + context.argument( + "description", + options_list=["--description"], + help="Description for the import update manifest.", + ) + context.argument( + "deployable", + options_list=["--is-deployable"], + arg_type=get_three_state_flag(), + help="Indicates whether the update is independently deployable.", + ) + context.argument( + "compatibility", + options_list=["--compat"], + nargs="+", + action="append", + help="Space-separated key=value pairs corresponding to properties of a device this update is compatible with. " + "Typically used for defining properties such as deviceManufacturer and deviceModel. " + "--compat can be used 1 or more times. ", + ) + context.argument( + "steps", + options_list=["--step"], + nargs="+", + action="append", + help="Space-separated key=value pairs corresponding to 'instructions.steps' element properties. " + "The client will determine if a step is an inline or reference step based on the provided " + "key value pairs. If either inline or reference step can be satisfied, the reference step will be prioritized. " + "Usage of --file will be associated with the nearest inline --step entry, deriving the value for 'files'. " + "The following reference step keys are supported: " + "`updateId.provider`, `updateId.name` `updateId.version` and `description`." + "The following inline step keys are supported: " + "`handler` (ex: 'microsoft/script:1' or 'microsoft/swupdate:1' or 'microsoft/apt:1'), " + "`properties` (in-line json object the agent will pass to the handler) and `description`. " + "--step can be used 1 or more times.", + ) + context.argument( + "files", + options_list=["--file"], + nargs="+", + action="append", + help="Space-separated key=value pairs corresponding to 'files' element properties. " + "A --file entry can include the closest --related-file entries if provided. " + "The following keys are supported: " + "`path` [required] local file path to update file, " + "`downloadHandler` (ex: 'microsoft/delta:1') download handler for utilizing related files " + "to download payload file. " + "--file can be used 1 or more times." + ) + context.argument( + "related_files", + options_list=["--related-file"], + nargs="+", + action="append", + help="Space-separated key=value pairs corresponding to 'files[*].relatedFiles' element properties. " + "A --related-file entry will be associated to the closest --file entry if it exists. " + "The following keys are supported: " + "`path` [required] local file path to related update file, " + "`properties` in-line json object passed to the download handler. " + "--related-file can be used 1 or more times." + ) diff --git a/azext_iot/deviceupdate/providers/base.py b/azext_iot/deviceupdate/providers/base.py index e5aac0aad..cbe43e82e 100644 --- a/azext_iot/deviceupdate/providers/base.py +++ b/azext_iot/deviceupdate/providers/base.py @@ -44,6 +44,13 @@ class UpdateManifestMeta(NamedTuple): hash: str +class FileMetadata(NamedTuple): + bytes: int + hash: str + name: str + path: str + + __all__ = [ "DeviceUpdateClientHandler", "DeviceUpdateAccountManager", @@ -53,6 +60,7 @@ class UpdateManifestMeta(NamedTuple): "parse_account_rg", "AccountContainer", "UpdateManifestMeta", + "FileMetadata", "ARMPolling", "AzureError", "HttpResponseError", @@ -231,15 +239,33 @@ def calculate_manifest_metadata(self, url: str) -> UpdateManifestMeta: The hash value is a base64 representation of a sha256 digest. """ from urllib.request import urlopen - from base64 import b64encode - from hashlib import sha256 with urlopen(url) as f: file_content: bytes = f.read() - hash = b64encode(sha256(file_content).digest()).decode("utf8") + hash = self.calculate_hash_from_bytes(file_content) return UpdateManifestMeta(len(file_content), hash) - def assemble_hashes(self, hash_list: List[str]) -> Dict[str, str]: + @classmethod + def calculate_file_metadata(cls, file_path) -> FileMetadata: + from pathlib import PurePath + + file_pure_path = PurePath(file_path) + with open(file_pure_path.as_posix(), "rb") as file_path: + logger.debug("Attempting to read file %s as binary", file_path) + raw_bytes = file_path.read() + size_in_bytes = len(raw_bytes) + hash = cls.calculate_hash_from_bytes(raw_bytes) + return FileMetadata(size_in_bytes, hash, file_pure_path.name, file_pure_path) + + @classmethod + def calculate_hash_from_bytes(cls, raw_bytes: bytes) -> str: + from base64 import b64encode + from hashlib import sha256 + + return b64encode(sha256(raw_bytes).digest()).decode("utf8") + + @classmethod + def assemble_nargs_to_dict(cls, hash_list: List[str]) -> Dict[str, str]: result = {} if not hash_list: return result diff --git a/azext_iot/tests/deviceupdate/test_adu_manifest_int.py b/azext_iot/tests/deviceupdate/test_adu_manifest_int.py new file mode 100644 index 000000000..c406ec2c7 --- /dev/null +++ b/azext_iot/tests/deviceupdate/test_adu_manifest_int.py @@ -0,0 +1,186 @@ +# coding=utf-8 +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import pytest +from knack.log import get_logger +from azext_iot.common.embedded_cli import EmbeddedCLI +from azext_iot.tests.conftest import get_context_path + +cli = EmbeddedCLI() + +logger = get_logger(__name__) + + +@pytest.mark.parametrize( + "options, expected", + [ + ( + "--update-provider digimaun0 --update-name simpleaptupdate --update-version 1.0.0 " + "--compat deviceManufacturer=Contoso deviceModel=Vacuum " + "--compat deviceManufacturer=Contoso deviceModel=Radio " + "--step handler=microsoft/apt:1 " + f"--file path=\"{get_context_path(__file__, 'manifests', 'libcurl4-doc-apt-manifest.json')}\"", + { + "updateId": {"provider": "digimaun0", "name": "simpleaptupdate", "version": "1.0.0"}, + "compatibility": [ + {"deviceManufacturer": "Contoso", "deviceModel": "Vacuum"}, + {"deviceManufacturer": "Contoso", "deviceModel": "Radio"}, + ], + "instructions": { + "steps": [{"handler": "microsoft/apt:1", "files": ["libcurl4-doc-apt-manifest.json"], "type": "inline"}] + }, + "files": [ + { + "filename": "libcurl4-doc-apt-manifest.json", + "sizeInBytes": 163, + "hashes": {"sha256": "iFWTIaxp33tf5BR1w0fMmnnHpjsUjLRQ9eZFjw74LbU="}, + } + ], + "manifestVersion": "5.0", + }, + ), + ( + "--update-provider digimaun1 --update-name Microphone --update-version 2.0.0 " + "--compat deviceManufacturer=Contoso deviceModel=Microphone " + "--step handler=microsoft/swupdate:1 " + f"--file path=\"{get_context_path(__file__, 'manifests', 'surface15', 'action.sh')}\" " + f"--file path=\"{get_context_path(__file__, 'manifests', 'surface15', 'install.sh')}\" " + "--is-deployable false", + { + "updateId": {"provider": "digimaun1", "name": "Microphone", "version": "2.0.0"}, + "compatibility": [ + {"deviceManufacturer": "Contoso", "deviceModel": "Microphone"}, + ], + "instructions": { + "steps": [{"handler": "microsoft/swupdate:1", "files": ["action.sh", "install.sh"], "type": "inline"}] + }, + "files": [ + { + "filename": "action.sh", + "sizeInBytes": 33, + "hashes": {"sha256": "n+KGjLjSGr7LVKsgWiExUDeU6Z2ZTJu0tpAWxkmYKxA="}, + }, + { + "filename": "install.sh", + "sizeInBytes": 23, + "hashes": {"sha256": "u6QdeTFImuTiReJ4WP9RlnYABdpd0cs8kuCz2zrHW28="}, + }, + ], + "manifestVersion": "5.0", + "isDeployable": False, + }, + ), + ( + "--update-provider digimaun2 --update-name Toaster --update-version 1.0.1 " + "--compat deviceManufacturer=Contoso deviceModel=Toaster " + '--step handler=microsoft/script:1 properties=\'{"args": "--pre"}\' description="Pre-install script" files=image ' + f"--file path=\"{get_context_path(__file__, 'manifests', 'surface15', 'install.sh')}\" " + "downloadHandler=microsoft/delta:1 " + f"--related-file path=\"{get_context_path(__file__, 'manifests', 'simple_apt_manifest_v5.json')}\" " + 'properties=\'{"microsoft.sourceFileHashAlgorithm":"sha256", ' + '"microsoft.sourceFileHash":"YmFYwnEUddq2nZsBAn5v7gCRKdHx+TUntMz5tLwU+24="}\' ' + '--step updateId.provider=digimaun updateId.name=Microphone updateId.version=1.3 description="Microphone Firmware" ' + '--step updateId.provider=digimaun updateId.name=Speaker updateId.version=0.9 description="Speaker Firmware" ' + '--step handler=microsoft/script:1 properties=\'{"args":"--post"}\' description="Post-install script" ' + f"--file path=\"{get_context_path(__file__, 'manifests', 'surface15', 'action.sh')}\" ", + { + "updateId": {"provider": "digimaun2", "name": "Toaster", "version": "1.0.1"}, + "compatibility": [ + {"deviceManufacturer": "Contoso", "deviceModel": "Toaster"}, + ], + "instructions": { + "steps": [ + { + "type": "inline", + "handler": "microsoft/script:1", + "handlerProperties": {"args": "--pre"}, + "files": ["image"], + "description": "Pre-install script", + }, + { + "type": "reference", + "updateId": {"provider": "digimaun", "name": "Microphone", "version": "1.3"}, + "description": "Microphone Firmware", + }, + { + "type": "reference", + "updateId": {"provider": "digimaun", "name": "Speaker", "version": "0.9"}, + "description": "Speaker Firmware", + }, + { + "type": "inline", + "handler": "microsoft/script:1", + "handlerProperties": {"args": "--post"}, + "files": ["action.sh"], + "description": "Post-install script", + }, + ] + }, + "files": [ + { + "filename": "install.sh", + "sizeInBytes": 23, + "hashes": {"sha256": "u6QdeTFImuTiReJ4WP9RlnYABdpd0cs8kuCz2zrHW28="}, + "downloadHandler": {"id": "microsoft/delta:1"}, + "relatedFiles": [ + { + "filename": "simple_apt_manifest_v5.json", + "sizeInBytes": 1031, + "hashes": {"sha256": "L+ZKmOOT3xRfHsFK7pcTXBLjeI2OFCW0855qIcV5sts="}, + "properties": { + "microsoft.sourceFileHash": "YmFYwnEUddq2nZsBAn5v7gCRKdHx+TUntMz5tLwU+24=", + "microsoft.sourceFileHashAlgorithm": "sha256", + }, + } + ], + }, + { + "filename": "action.sh", + "sizeInBytes": 33, + "hashes": {"sha256": "n+KGjLjSGr7LVKsgWiExUDeU6Z2ZTJu0tpAWxkmYKxA="}, + }, + ], + "manifestVersion": "5.0", + }, + ), + ], +) +def test_adu_manifest_init_v5(options, expected): + result = cli.invoke(f"iot device-update update init v5 {options}").as_json() + del result["createdDateTime"] + assert result == expected + + +@pytest.mark.parametrize( + "options", + [ + ( + # path key is required for --file + "--update-provider digimaun --update-name invalid --update-version 1.0.0 " + "--compat deviceManufacturer=Contoso deviceModel=Vacuum " + "--step handler=microsoft/apt:1 " + "--file downhandler=abcd/123", + ), + ( + # path key is required for --related-file + "--update-provider digimaun --update-name invalid --update-version 1.0.0 " + "--compat deviceManufacturer=Contoso deviceModel=Vacuum " + "--step handler=microsoft/apt:1 " + f"--file path=\"{get_context_path(__file__, 'manifests', 'libcurl4-doc-apt-manifest.json')}\" " + "--related-file properties='{\"a\": 1}'" + ), + ( + # Usage of --step requires at least an entry of handler= for an inline step or + # all of updateId.provider=, updateId.name=, updateId.version= for a reference step. + "--update-provider digimaun --update-name invalid --update-version 1.0.0 " + "--compat deviceManufacturer=Contoso deviceModel=Vacuum " + "--step stuff=things " + f"--file path=\"{get_context_path(__file__, 'manifests', 'libcurl4-doc-apt-manifest.json')}\" " + ), + ], +) +def test_adu_manifest_init_v5_invalid_path_required(options): + assert not cli.invoke(f"iot device-update update init v5 {options}").success()