Add experimental ADU manifest init command. (#571)

* Improve help, add additional validation case.

HISTORY.rst

@@ -19,6 +19,8 @@ unreleased
 * The command `az iot device-update device class list` adds support for `--filter` when no `--group-id` is provided.
 * The parameters `--account`, `--instance`, and `--resource-group` support setting default overridable values via config.
   Use `az config set` i.e. `az config set defaults.adu_account=<name>` or `az configure` i.e. `az configure --defaults adu_account=<name>`.
+* Introducing the experimental command `az iot device-update update init v5` for initializing (or generating) an import manifest
+  with the desired state.
 * Improved built-in documentation.
 
 

azext_iot/deviceupdate/_help.py

@@ -822,3 +822,44 @@ def load_deviceupdate_help():
           text: >
             az iot device-update device show -n {account_name} -i {instance_name} -d {device_id}
     """
+
+    helps["iot device-update update init"] = """
+        type: group
+        short-summary: Utilities for initializing update import manifests.
+    """
+
+    helps["iot device-update update init v5"] = """
+        type: command
+        short-summary: Initialize a v5 import manifest with the desired state.
+        long-summary: This command supports all attributes of the v5 import manifest. Review examples
+          and parameter descriptions for details on how to fully utilize the operation. Note that
+          there is positional sensitivity between --step and --file, as well as --file and --related-file.
+          Review parameter help for more details.
+
+        examples:
+        - name: Initialize a minimum content import manifest.
+          text: >
+            az iot device-update update init v5 --update-provider Microsoft --update-name AptUpdate --update-version 1.0.0
+            --description "My minimum update"
+            --compat deviceManufacturer=Contoso deviceModel=Vacuum
+            --step handler=microsoft/apt:1 --file path=/my/apt/manifest/file
+
+        - name: Initialize a non-deployable leaf update to be referenced in a bundled update.
+          text: >
+            az iot device-update update init v5 --update-provider Microsoft --update-name SwUpdate --update-version 1.1
+            --compat deviceManufacturer=Contoso deviceModel=Microphone --step handler=microsoft/swupdate:1 description="Deploy Update"
+            --file path=/my/update/image/file1 --file path=/my/update/image/file2
+            --is-deployable false
+
+        - name: Initialize a bundled update referencing a leaf update as well as defining independent steps. Inline json examples
+            compatible with bash-like shells.
+          text: >
+            az iot device-update update init v5 --update-provider Microsoft --update-name Bundled --update-version 2.0
+            --compat deviceManufacturer=Contoso deviceModel=SpaceStation
+            --step handler=microsoft/script:1 properties='{"arguments": "--pre"}' description="Pre-install script"
+            --file path=/my/update/scripts/preinstall.sh downloadHandler=microsoft/delta:1
+            --related-file path=/my/update/scripts/related_preinstall.json properties='{"microsoft.sourceFileHashAlgorithm": "sha256"}'
+            --step updateId.provider=Microsoft updateId.name=SwUpdate updateId.version=1.1
+            --step handler=microsoft/script:1 properties='{"arguments": "--post"}' description="Post-install script"
+            --file path=/my/update/scripts/postinstall.sh
+    """

azext_iot/deviceupdate/command_map.py

@@ -105,6 +105,12 @@ def load_deviceupdate_commands(self, _):
         cmd_group.command("list", "list_update_files")
         cmd_group.show_command("show", "show_update_file")
 
+    with self.command_group(
+        "iot device-update update init",
+        command_type=deviceupdate_update_ops,
+    ) as cmd_group:
+        cmd_group.command("v5", "manifest_init_v5", is_experimental=True)
+
     with self.command_group(
         "iot device-update device",
         command_type=deviceupdate_device_ops,

azext_iot/deviceupdate/commands_update.py

@@ -12,7 +12,7 @@
     AzureError,
     ARMPolling,
 )
-from typing import Optional, List, Union
+from typing import Optional, List, Union, Dict
 
 logger = get_logger(__name__)
 
@@ -137,7 +137,7 @@ def import_update(
     if not size or not hashes:
         client_calculated_meta = data_manager.calculate_manifest_metadata(url)
 
-    hashes = data_manager.assemble_hashes(hash_list=hashes) or {"sha256": client_calculated_meta.hash}
+    hashes = data_manager.assemble_nargs_to_dict(hash_list=hashes) or {"sha256": client_calculated_meta.hash}
     size = size or client_calculated_meta.bytes
 
     manifest_metadata = DeviceUpdateDataModels.ImportManifestMetadata(url=url, size_in_bytes=size, hashes=hashes)
@@ -210,3 +210,181 @@ def delete_update(
     return data_manager.data_client.device_update.begin_delete_update(
         name=update_name, provider=update_provider, version=update_version
     )
+
+
+def manifest_init_v5(
+    cmd,
+    update_name: str,
+    update_provider: str,
+    update_version: str,
+    compatibility: List[List[str]],
+    steps: List[List[str]],
+    files: List[List[str]] = None,
+    related_files: List[List[str]] = None,
+    description: str = None,
+    deployable: bool = None,
+):
+    import json
+    from datetime import datetime
+    from pathlib import PurePath
+    from azure.cli.core.azclierror import ArgumentUsageError
+
+    def _sanitize_safe_params(safe_params: list, keep: list) -> list:
+        '''
+        Intended to filter un-related params,
+        leaving only related params with inherent positional indexing
+        to be used by the _associate_related function.
+        '''
+        result: List[str] = []
+        if not safe_params:
+            return result
+        for param in safe_params:
+            if param in keep:
+                result.append(param)
+        return result
+
+    def _associate_related(sanitized_params: list, key: str) -> dict:
+        '''
+        Intended to associate related param indexes. For example
+        associate --file with the nearest --step or associate --related-file
+        with the nearest --file.
+        '''
+        result: Dict[int, list] = {}
+        if not sanitized_params:
+            return result
+        params_len = len(sanitized_params)
+        key_index = 0
+        related_key_index = 0
+        for i in range(params_len):
+            if sanitized_params[i] == key:
+                result[key_index] = []
+                for j in range(i + 1, params_len):
+                    if sanitized_params[j] == key:
+                        break
+                    result[key_index].append(related_key_index)
+                    related_key_index = related_key_index + 1
+                key_index = key_index + 1
+        return result
+
+    payload = {}
+    payload["manifestVersion"] = "5.0"
+    payload["createdDateTime"] = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
+    payload["updateId"] = {}
+    payload["updateId"]["name"] = update_name
+    payload["updateId"]["provider"] = update_provider
+    payload["updateId"]["version"] = update_version
+    if deployable is False:
+        payload["isDeployable"] = False
+    if description:
+        payload["description"] = description
+    processed_compatibility = []
+    for compat in compatibility:
+        if not compat or not compat[0]:
+            continue
+        processed_compatibility.append(DeviceUpdateDataManager.assemble_nargs_to_dict(compat))
+    payload["compatibility"] = processed_compatibility
+
+    safe_params = cmd.cli_ctx.data.get("safe_params", [])
+    processed_steps = []
+    for s in range(len(steps)):
+        if not steps[s] or not steps[s][0]:
+            continue
+
+        step_file_params = _sanitize_safe_params(safe_params, ["--step", "--file"])
+        related_step_file_map = _associate_related(step_file_params, "--step")
+
+        assembled_step = DeviceUpdateDataManager.assemble_nargs_to_dict(steps[s])
+        step = {}
+        if all(k in assembled_step for k in ("updateId.provider", "updateId.name", "updateId.version")):
+            # reference step
+            step = {
+                "type": "reference",
+                "updateId": {
+                    "provider": assembled_step["updateId.provider"],
+                    "name": assembled_step["updateId.name"],
+                    "version": assembled_step["updateId.version"],
+                },
+            }
+        elif "handler" in assembled_step:
+            # inline step
+            step = {
+                "type": "inline",
+                "handler": assembled_step["handler"],
+            }
+            step["files"] = [f.strip() for f in assembled_step["files"].split(",")] if "files" in assembled_step else []
+            if not step["files"]:
+                derived_step_files = []
+                for f in related_step_file_map[s]:
+                    step_file = files[f]
+                    if not step_file or not step_file[0]:
+                        continue
+                    assembled_step_file = DeviceUpdateDataManager.assemble_nargs_to_dict(step_file)
+                    if "path" in assembled_step_file:
+                        derived_step_files.append(PurePath(assembled_step_file["path"]).name)
+                step["files"] = derived_step_files
+
+            if "properties" in assembled_step:
+                step["handlerProperties"] = json.loads(assembled_step["properties"])
+
+        if not step:
+            raise ArgumentUsageError(
+                "Usage of --step requires at least an entry of handler=<value> for an inline step or "
+                "all of updateId.provider=<value>, updateId.name=<value>, updateId.version=<value> for a reference step.")
+
+        step_desc = assembled_step.get("description") or assembled_step.get("desc")
+        if step_desc:
+            step["description"] = step_desc
+        processed_steps.append(step)
+
+    payload["instructions"] = {}
+    payload["instructions"]["steps"] = processed_steps
+
+    if files:
+        file_params = _sanitize_safe_params(safe_params, ["--file", "--related-file"])
+        related_file_map = _associate_related(file_params, "--file")
+
+        processed_files = []
+        for f in range(len(files)):
+            if not files[f] or not files[f][0]:
+                continue
+            processed_file = {}
+            assembled_file = DeviceUpdateDataManager.assemble_nargs_to_dict(files[f])
+            if "path" not in assembled_file:
+                raise ArgumentUsageError("When using --file path is required.")
+            assembled_file_metadata = DeviceUpdateDataManager.calculate_file_metadata(assembled_file["path"])
+            processed_file["hashes"] = {"sha256": assembled_file_metadata.hash}
+            processed_file["filename"] = assembled_file_metadata.name
+            processed_file["sizeInBytes"] = assembled_file_metadata.bytes
+
+            if "downloadHandler" in assembled_file:
+                processed_file["downloadHandler"] = {"id": assembled_file["downloadHandler"]}
+
+            processed_related_files = []
+            for r in related_file_map[f]:
+                related_file = related_files[r]
+                if not related_file or not related_file[0]:
+                    continue
+                processed_related_file = {}
+                assembled_related_file = DeviceUpdateDataManager.assemble_nargs_to_dict(related_file)
+                if "path" not in assembled_related_file:
+                    raise ArgumentUsageError("When using --related-file path is required.")
+                related_file_metadata = DeviceUpdateDataManager.calculate_file_metadata(assembled_related_file["path"])
+                processed_related_file["hashes"] = {"sha256": related_file_metadata.hash}
+                processed_related_file["filename"] = related_file_metadata.name
+                processed_related_file["sizeInBytes"] = related_file_metadata.bytes
+
+                if "properties" in assembled_related_file:
+                    processed_related_file["properties"] = json.loads(assembled_related_file["properties"])
+
+                if processed_related_file:
+                    processed_related_files.append(processed_related_file)
+
+            if processed_related_files:
+                processed_file["relatedFiles"] = processed_related_files
+
+            if processed_file:
+                processed_files.append(processed_file)
+
+        payload["files"] = processed_files
+
+    return payload

azext_iot/deviceupdate/params.py

@@ -440,3 +440,81 @@ def load_deviceupdate_arguments(self, _):
             options_list=["--description"],
             help="Description for the log collection operation.",
         )
+
+    with self.argument_context("iot device-update update init") as context:
+        context.argument(
+            "update_provider",
+            options_list=["--update-provider"],
+            help="The update provider as a component of updateId.",
+        )
+        context.argument(
+            "update_name",
+            options_list=["--update-name"],
+            help="The update name as a component of updateId.",
+        )
+        context.argument(
+            "update_version",
+            options_list=["--update-version"],
+            help="The update version as a component of updateId.",
+        )
+        context.argument(
+            "description",
+            options_list=["--description"],
+            help="Description for the import update manifest.",
+        )
+        context.argument(
+            "deployable",
+            options_list=["--is-deployable"],
+            arg_type=get_three_state_flag(),
+            help="Indicates whether the update is independently deployable.",
+        )
+        context.argument(
+            "compatibility",
+            options_list=["--compat"],
+            nargs="+",
+            action="append",
+            help="Space-separated key=value pairs corresponding to properties of a device this update is compatible with. "
+            "Typically used for defining properties such as deviceManufacturer and deviceModel. "
+            "--compat can be used 1 or more times. ",
+        )
+        context.argument(
+            "steps",
+            options_list=["--step"],
+            nargs="+",
+            action="append",
+            help="Space-separated key=value pairs corresponding to 'instructions.steps' element properties. "
+            "The client will determine if a step is an inline or reference step based on the provided "
+            "key value pairs. If either inline or reference step can be satisfied, the reference step will be prioritized. "
+            "Usage of --file will be associated with the nearest inline --step entry, deriving the value for 'files'. "
+            "The following reference step keys are supported: "
+            "`updateId.provider`, `updateId.name` `updateId.version` and `description`."
+            "The following inline step keys are supported: "
+            "`handler` (ex: 'microsoft/script:1' or 'microsoft/swupdate:1' or 'microsoft/apt:1'), "
+            "`properties` (in-line json object the agent will pass to the handler) and `description`. "
+            "--step can be used 1 or more times.",
+        )
+        context.argument(
+            "files",
+            options_list=["--file"],
+            nargs="+",
+            action="append",
+            help="Space-separated key=value pairs corresponding to 'files' element properties. "
+            "A --file entry can include the closest --related-file entries if provided. "
+            "The following keys are supported: "
+            "`path` [required] local file path to update file, "
+            "`downloadHandler` (ex: 'microsoft/delta:1') download handler for utilizing related files "
+            "to download payload file. "
+            "--file can be used 1 or more times."
+        )
+        context.argument(
+            "related_files",
+            options_list=["--related-file"],
+            nargs="+",
+            action="append",
+            help="Space-separated key=value pairs corresponding to 'files[*].relatedFiles' element properties. "
+            "A --related-file entry will be associated to the closest --file entry if it exists. "
+            "The following keys are supported: "
+            "`path` [required] local file path to related update file, "
+            "`properties` in-line json object passed to the download handler. "
+            "--related-file can be used 1 or more times."
+        )