From 27b9b3de922041495ce938f8863c68254d07d637 Mon Sep 17 00:00:00 2001 From: Paymaun Date: Thu, 29 Jul 2021 11:12:00 -0700 Subject: [PATCH] Azure namespace import utility (#396) * Azure namespace package shim. --- .azure-devops/templates/run-tests.yml | 4 +- CONTRIBUTING.md | 16 ++----- README.md | 6 ++- azext_iot/common/shared.py | 10 ---- azext_iot/common/utility.py | 46 ++++++++++++++----- azext_iot/digitaltwins/providers/model.py | 2 +- azext_iot/digitaltwins/providers/twin.py | 4 +- azext_iot/operations/dps.py | 4 +- .../tests/utility/test_iot_utility_unit.py | 30 +++++++++++- 9 files changed, 78 insertions(+), 44 deletions(-) diff --git a/.azure-devops/templates/run-tests.yml b/.azure-devops/templates/run-tests.yml index dec6cc9f9..833d17824 100644 --- a/.azure-devops/templates/run-tests.yml +++ b/.azure-devops/templates/run-tests.yml @@ -36,7 +36,7 @@ steps: - ${{ if eq(parameters.runUnitTests, 'true') }}: - script: | - pytest -vv ${{ parameters.path }} -k "_unit" --cov=azext_iot --cov-config .coveragerc --junitxml=junit/test-iotext-unit-${{ parameters.name }}.xml + pytest -s -vv ${{ parameters.path }} -k "_unit" --cov=azext_iot --cov-config .coveragerc --junitxml=junit/test-iotext-unit-${{ parameters.name }}.xml displayName: '${{ parameters.name }} unit tests' env: COVERAGE_FILE: .coverage.${{ parameters.name }} @@ -51,7 +51,7 @@ steps: scriptLocation: inlineScript inlineScript: | export COVERAGE_FILE=.coverage.${{ parameters.name }} - pytest -vv ${{ parameters.path }} -k "_int" --cov=azext_iot --cov-config .coveragerc --junitxml=junit/test-iotext-int-${{ parameters.name }}.xml + pytest -s -vv ${{ parameters.path }} -k "_int" --cov=azext_iot --cov-config .coveragerc --junitxml=junit/test-iotext-int-${{ parameters.name }}.xml - task: PublishBuildArtifacts@1 inputs: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8301c0439..69a3f5332 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,7 +14,7 @@ You must fork and clone the repositories below. Follow the videos and instructio > IMPORTANT: When cloning the repositories and environments, ensure they are all siblings to each other. This makes things much easier down the line. -``` +```text source-directory/ |-- azure-cli/ |-- azure-iot-cli-extension/ @@ -31,8 +31,6 @@ After following the videos, ensure you have: #### Environment Variables -It is recommended that you set the following environment variables in a way such that they are persisted through machine restarts. - You can run this setup in `bash` or `cmd` environments, this documentation just show the `powershell` flavor. 1. Create a directory for your development extensions to live in @@ -47,20 +45,12 @@ You can run this setup in `bash` or `cmd` environments, this documentation just $env:AZURE_EXTENSION_DIR="path/to/source/extensions" ``` -3. Set `PYTHONPATH` to the following. Order matters here so be careful. - - ```powershell - $env:PYTHONPATH="path/to/source/azure-iot-cli-extension;path/to/source/extensions/azure-iot" - ``` - -Restart any PowerShell windows you may have open and reactivate your python environment. Check that the environment variables created above have persisted. - #### azdev Steps -Similar to the video, just execute the following command. +Similar to the video, have your virtual environment activated then execute the following command ```powershell -azdev setup -c path/to/source/azure-cli +(.env3) azdev setup -c path/to/source/azure-cli ``` #### Install dev extension diff --git a/README.md b/README.md index 6dc128549..15cf51a89 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,8 @@ The **Azure IoT extension for Azure CLI** aims to accelerate the development, management and automation of Azure IoT solutions. It does this via addition of rich features and functionality to the official [Azure CLI](https://docs.microsoft.com/en-us/cli/azure). ## News -- Starting with version `0.10.13` of the IoT extension, you will need an Azure CLI core version of `2.17.1` or higher. IoT extension version `0.10.11` remains on the extension index to support environments that cannot upgrade core CLI versions. + +- Starting with version `0.10.13` of the IoT extension, you will need an Azure CLI core version of `2.17.1` or higher. IoT extension version `0.10.11` remains on the extension index to support environments that cannot upgrade core CLI versions. - Azure CLI `2.24.0` requires an `azure-iot` extension update to `0.10.11` or later for IoT Hub commands to work properly. This can be done with `az extension update --name azure-iot`. A common error that arises when using an older `azure-iot` with Azure CLI `2.24.0` looks like `AttributeError: 'IotHubResourceOperations' object has no attribute 'config'`. @@ -15,7 +16,8 @@ The **Azure IoT extension for Azure CLI** aims to accelerate the development, ma Uninstall the legacy extension with the following command: `az extension remove --name azure-cli-iot-ext`. Related - if you see an error with a stacktrace similar to: - ``` + + ```text ... azure-cli-iot-ext/azext_iot/common/_azure.py, ln 90, in get_iot_hub_connection_string client = iot_hub_service_factory(cmd.cli_ctx) diff --git a/azext_iot/common/shared.py b/azext_iot/common/shared.py index be947ca06..a5fc3b3b5 100644 --- a/azext_iot/common/shared.py +++ b/azext_iot/common/shared.py @@ -152,16 +152,6 @@ class DistributedTracingSamplingModeType(Enum): on = "on" -class PnPModelType(Enum): - """ - Type of PnP Model. - """ - - any = "any" - interface = "Interface" - capabilityModel = "capabilityModel" - - class ConfigType(Enum): """ Type of configuration deployment. diff --git a/azext_iot/common/utility.py b/azext_iot/common/utility.py index fb632553a..197f58764 100644 --- a/azext_iot/common/utility.py +++ b/azext_iot/common/utility.py @@ -115,7 +115,7 @@ def validate_key_value_pairs(string): def process_json_arg(content, argument_name, preserve_order=False): - """ Primary processor of json input """ + """Primary processor of json input""" json_from_file = None @@ -142,9 +142,9 @@ def process_json_arg(content, argument_name, preserve_order=False): def shell_safe_json_parse(json_or_dict_string, preserve_order=False): - """ Allows the passing of JSON or Python dictionary strings. This is needed because certain + """Allows the passing of JSON or Python dictionary strings. This is needed because certain JSON strings in CMD shell are not received in main's argv. This allows the user to specify - the alternative notation, which does not have this problem (but is technically not JSON). """ + the alternative notation, which does not have this problem (but is technically not JSON).""" try: if not preserve_order: return json.loads(json_or_dict_string) @@ -188,15 +188,15 @@ def read_file_content(file_path, allow_binary=False): def trim_from_start(s, substring): - """ Trims a substring from the target string (if it exists) returning the trimmed string. - Otherwise returns original target string. """ + """Trims a substring from the target string (if it exists) returning the trimmed string. + Otherwise returns original target string.""" if s.startswith(substring): s = s[len(substring) :] return s def validate_min_python_version(major, minor, error_msg=None, exit_on_fail=True): - """ If python version does not match AT LEAST requested values, will throw non 0 exit code.""" + """If python version does not match AT LEAST requested values, will throw non 0 exit code.""" version = sys.version_info result = False if version.major > major: @@ -219,7 +219,7 @@ def validate_min_python_version(major, minor, error_msg=None, exit_on_fail=True) def unicode_binary_map(target): - """ Decode binary keys and values of map to unicode.""" + """Decode binary keys and values of map to unicode.""" # Assumes no iteritems() result = {} @@ -311,7 +311,7 @@ def url_encode_str(s, plus=False): def test_import(package): - """ Used to determine if a dependency is loading correctly """ + """Used to determine if a dependency is loading correctly""" import importlib try: @@ -332,7 +332,7 @@ def unpack_pnp_http_error(e): def unpack_msrest_error(e): - """ Obtains full response text from an msrest error """ + """Obtains full response text from an msrest error""" op_err = None try: @@ -345,7 +345,7 @@ def unpack_msrest_error(e): def dict_transform_lower_case_key(d): - """ Converts a dictionary to an identical one with all lower case keys """ + """Converts a dictionary to an identical one with all lower case keys""" return {k.lower(): v for k, v in d.items()} @@ -381,7 +381,7 @@ def init_monitoring(cmd, timeout, properties, enqueued_time, repair, yes): def dict_clean(d): - """ Remove None from dictionary """ + """Remove None from dictionary""" if not isinstance(d, dict): return d return dict((k, dict_clean(v)) for k, v in d.items() if v is not None) @@ -440,6 +440,7 @@ def is_iso8601_time(self, to_validate: str) -> bool: def ensure_iothub_sdk_min_version(min_ver): from packaging import version + try: from azure.mgmt.iothub import __version__ as iot_sdk_version except ImportError: @@ -500,3 +501,26 @@ def generate_key(byte_length=32): token_bytes = secrets.token_bytes(byte_length) return base64.b64encode(token_bytes).decode("utf8") + + +def ensure_azure_namespace_path(): + """ + Run prior to importing azure namespace packages (azure.*) to ensure the + extension root path is configured for package import. + """ + from azure.cli.core.extension import get_extension_path + from azext_iot.constants import EXTENSION_NAME + + ext_path = get_extension_path(EXTENSION_NAME) + + ext_azure_dir = os.path.join(ext_path, "azure") + if os.path.isdir(ext_azure_dir): + import azure + + if getattr(azure, "__path__", None) and ext_azure_dir not in azure.__path__: + azure.__path__.append(ext_azure_dir) # _NamespacePath /w PEP420 + + if sys.path and sys.path[0] != ext_path: + sys.path.insert(0, ext_path) + + return diff --git a/azext_iot/digitaltwins/providers/model.py b/azext_iot/digitaltwins/providers/model.py index 0130ca1ba..26fbda768 100644 --- a/azext_iot/digitaltwins/providers/model.py +++ b/azext_iot/digitaltwins/providers/model.py @@ -190,7 +190,7 @@ def delete_parents(model_id, model_dict): try: self.delete(model_id) except CLIError as e: - logger.warn(f"Could not delete model {model_id}; error is {e}") + logger.warning(f"Could not delete model {model_id}; error is {e}") while len(parsed_models) > 0: model_id = next(iter(parsed_models)) diff --git a/azext_iot/digitaltwins/providers/twin.py b/azext_iot/digitaltwins/providers/twin.py index e465a355d..0b28abe49 100644 --- a/azext_iot/digitaltwins/providers/twin.py +++ b/azext_iot/digitaltwins/providers/twin.py @@ -127,7 +127,7 @@ def delete_all(self, only_relationships=False): if not only_relationships: self.delete(twin_id=twin["$dtId"]) except CLIError as e: - logger.warn(f"Could not delete twin {twin['$dtId']}. The error is {e}") + logger.warning(f"Could not delete twin {twin['$dtId']}. The error is {e}") def add_relationship( self, @@ -260,7 +260,7 @@ def delete_all_relationship(self, twin_id): relationship_id=relationship.relationship_id ) except CLIError as e: - logger.warn(f"Could not delete relationship {relationship}. The error is {e}.") + logger.warning(f"Could not delete relationship {relationship}. The error is {e}.") def get_component(self, twin_id, component_path): try: diff --git a/azext_iot/operations/dps.py b/azext_iot/operations/dps.py index df1885663..71d6c5b8a 100644 --- a/azext_iot/operations/dps.py +++ b/azext_iot/operations/dps.py @@ -78,7 +78,7 @@ def iot_dps_device_enrollment_get( ).response.json() enrollment["attestation"] = attestation else: - logger.warn( + logger.warning( "--show-keys argument was provided, but requested enrollment has an attestation type of '{}'." " Currently, --show-keys is only supported for symmetric key enrollments".format( enrollment_type @@ -325,7 +325,7 @@ def iot_dps_device_enrollment_group_get( ).response.json() enrollment_group["attestation"] = attestation else: - logger.warn( + logger.warning( "--show-keys argument was provided, but requested enrollment group has an attestation type of '{}'." " Currently, --show-keys is only supported for symmetric key enrollment groups".format( enrollment_type diff --git a/azext_iot/tests/utility/test_iot_utility_unit.py b/azext_iot/tests/utility/test_iot_utility_unit.py index 9719ff3b5..54b30aa11 100644 --- a/azext_iot/tests/utility/test_iot_utility_unit.py +++ b/azext_iot/tests/utility/test_iot_utility_unit.py @@ -7,6 +7,7 @@ import json import pytest import os +import sys from unittest import mock from knack.util import CLIError @@ -375,7 +376,34 @@ def _validate_directory(path): invalid_directories = [] for directory in directory_structure: if directory_structure[directory] is None: - invalid_directories.append("Directory: '{}' missing __init__.py".format(directory)) + invalid_directories.append( + "Directory: '{}' missing __init__.py".format(directory) + ) if invalid_directories: pytest.fail(", ".join(invalid_directories)) + + def test_ensure_azure_namespace_path(self): + import azure + from azext_iot.common.utility import ensure_azure_namespace_path + from azure.cli.core.extension import get_extension_path + from azext_iot.constants import EXTENSION_NAME + + ext_path = get_extension_path(EXTENSION_NAME) + + original_sys_path = list(sys.path) + original_azure_namespace_path = list(azure.__path__) + ext_azure_dir = os.path.join(ext_path, "azure") + + os.makedirs(ext_azure_dir, exist_ok=True) + + ensure_azure_namespace_path() + modified_sys_path = list(sys.path) + modified_azure_namespace_path = list(azure.__path__) + + original_azure_namespace_path.append(ext_azure_dir) + assert set(original_azure_namespace_path) == set(modified_azure_namespace_path) + + original_sys_path.insert(0, ext_path) + assert set(original_sys_path) == set(modified_sys_path) + assert modified_sys_path[0] == ext_path