From 2c5e705eeebfb323e8f4f45d75e123c339d61a59 Mon Sep 17 00:00:00 2001 From: Ryan K Date: Tue, 10 Jan 2023 15:05:14 -0800 Subject: [PATCH] Edge device creation, configuration, and device bundles (#591) Adds `az iot edge devices create` command that allows users to create and configure various edge device scenarios and deployments using a configuration file or inline device arguments. --- .gitignore | 4 +- CredScanSuppressions.json | 15 + HISTORY.rst | 9 + azext_iot/common/certops.py | 247 +++- azext_iot/common/fileops.py | 53 + azext_iot/common/shared.py | 8 +- azext_iot/common/utility.py | 85 +- azext_iot/deviceupdate/commands_update.py | 14 +- azext_iot/deviceupdate/providers/base.py | 18 +- azext_iot/iothub/_help.py | 47 + azext_iot/iothub/command_map.py | 12 +- azext_iot/iothub/commands_device_identity.py | 73 + azext_iot/iothub/common.py | 39 +- azext_iot/iothub/params.py | 98 +- azext_iot/iothub/providers/device_identity.py | 442 ++++++ .../iothub/providers/helpers/__init__.py | 5 + .../providers/helpers/edge_device_config.py | 583 ++++++++ azext_iot/operations/hub.py | 2 +- azext_iot/tests/conftest.py | 48 +- .../deployments/deploymentLowerLayer.json | 57 + .../deployments/deploymentTopLayer.json | 82 ++ .../devices/device_configs/device_config.toml | 45 + .../edge_devices_min_config.yml | 15 + .../fake_edge_container_auth.json | 5 + .../invalid/duplicate_device_config.yml | 11 + .../invalid/invalid_deployment.json | 3 + .../invalid/missing_deployment.yml | 17 + .../invalid/missing_device_id.yml | 8 + .../device_configs/nested_edge_config.json | 59 + .../device_configs/nested_edge_config.yml | 39 + .../nested_edge_config_secondary.yaml | 18 + .../test_iot_edge_devices_create_int.py | 397 ++++++ .../devices/test_iot_edge_devices_unit.py | 1267 +++++++++++++++++ .../tests/utility/test_iot_file_operations.py | 65 + .../tests/utility/test_iot_utility_unit.py | 10 +- docs/samples/sample_devices_config.yaml | 31 + setup.py | 3 + thirdpartynotice | 75 + 38 files changed, 3934 insertions(+), 75 deletions(-) create mode 100644 azext_iot/common/fileops.py create mode 100644 azext_iot/iothub/commands_device_identity.py create mode 100644 azext_iot/iothub/providers/device_identity.py create mode 100644 azext_iot/iothub/providers/helpers/__init__.py create mode 100644 azext_iot/iothub/providers/helpers/edge_device_config.py create mode 100644 azext_iot/tests/iothub/devices/device_configs/deployments/deploymentLowerLayer.json create mode 100644 azext_iot/tests/iothub/devices/device_configs/deployments/deploymentTopLayer.json create mode 100644 azext_iot/tests/iothub/devices/device_configs/device_config.toml create mode 100644 azext_iot/tests/iothub/devices/device_configs/edge_devices_min_config.yml create mode 100644 azext_iot/tests/iothub/devices/device_configs/fake_edge_container_auth.json create mode 100644 azext_iot/tests/iothub/devices/device_configs/invalid/duplicate_device_config.yml create mode 100644 azext_iot/tests/iothub/devices/device_configs/invalid/invalid_deployment.json create mode 100644 azext_iot/tests/iothub/devices/device_configs/invalid/missing_deployment.yml create mode 100644 azext_iot/tests/iothub/devices/device_configs/invalid/missing_device_id.yml create mode 100644 azext_iot/tests/iothub/devices/device_configs/nested_edge_config.json create mode 100644 azext_iot/tests/iothub/devices/device_configs/nested_edge_config.yml create mode 100644 azext_iot/tests/iothub/devices/device_configs/nested_edge_config_secondary.yaml create mode 100644 azext_iot/tests/iothub/devices/test_iot_edge_devices_create_int.py create mode 100644 azext_iot/tests/iothub/devices/test_iot_edge_devices_unit.py create mode 100644 azext_iot/tests/utility/test_iot_file_operations.py create mode 100644 docs/samples/sample_devices_config.yaml diff --git a/.gitignore b/.gitignore index 7be037157..e21ece30b 100644 --- a/.gitignore +++ b/.gitignore @@ -289,8 +289,8 @@ __pycache__/ # Virtual environment env/ -env27/ -env36/ +env2*/ +env3*/ .python-version # PTVS analysis diff --git a/CredScanSuppressions.json b/CredScanSuppressions.json index faa999933..9ae2791ed 100644 --- a/CredScanSuppressions.json +++ b/CredScanSuppressions.json @@ -66,6 +66,21 @@ { "file": "azext_iot\\tests\\central\\json\\x509_verified_certiciate.pem", "_justification": "Completely made up x509 certificate for IoT central integration tests." + }, + { + "file": "azext_iot\\tests\\iothub\\devices\\device_configs\\nested_edge_config.json", + "placeholder": "$credential$", + "_justification": "Nested Edge configuration file with fake credentials" + }, + { + "file": "azext_iot\\tests\\iothub\\devices\\device_configs\\fake_edge_container_auth.json", + "placeholder": "$credential$", + "_justification": "Container auth test file with fake credentials" + }, + { + "file": "azext_iot\\tests\\iothub\\devices\\test_iot_edge_devices_create_int.py", + "placeholder": "$credential$", + "_justification": "Fake credentials for container auth" } ] } \ No newline at end of file diff --git a/HISTORY.rst b/HISTORY.rst index bc85ae55a..19d56393a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -39,6 +39,15 @@ unreleased * Some minor improvements to command documentation involving managed identities. +**IoT Edge updates** + +* Introduces a new experimental command `az iot edge devices create` that enables advanced IoT Edge device creation and configuration. + This command allows users to specify either multiple inline arguments (`--device property=value`) or a [configuration file](https://aka.ms/aziotcli-edge-devices-config) + to create multiple edge devices (including nested device scenarios) and configure their deployments. + + If an output path is specified, this command will also create tar files containing each device's certificate bundle, an IoT Edge + `config.toml` config file and an installation script to configure a target Edge device with these settings. + 0.18.3 +++++++++++++++ diff --git a/azext_iot/common/certops.py b/azext_iot/common/certops.py index d47e217b9..d0368cce6 100644 --- a/azext_iot/common/certops.py +++ b/azext_iot/common/certops.py @@ -9,23 +9,28 @@ """ import datetime -from os.path import exists, join +from os.path import exists import base64 -from typing import Dict +from typing import Dict, List, Optional from cryptography import x509 from cryptography.x509.oid import NameOID from cryptography.hazmat.primitives import serialization, hashes from cryptography.hazmat.primitives.asymmetric import rsa +from azext_iot.common.fileops import write_content_to_file +from azext_iot.common.utility import read_file_content +from azure.cli.core.azclierror import FileOperationError from azext_iot.common.shared import SHAHashVersions def create_self_signed_certificate( subject: str, - valid_days: int, - cert_output_dir: str, + valid_days: int = 365, + cert_output_dir: str = None, + key_size: int = 2048, cert_only: bool = False, file_prefix: str = None, sha_version: int = SHAHashVersions.SHA1.value, + v3_extensions: bool = False, ) -> Dict[str, str]: """ Function used to create a basic self-signed certificate with no extensions. @@ -44,8 +49,8 @@ def create_self_signed_certificate( result (dict): dict with certificate value, private key and thumbprint. """ # create a key pair - key = rsa.generate_private_key(public_exponent=65537, key_size=2048) - + key = rsa.generate_private_key(public_exponent=65537, key_size=key_size) + serial = x509.random_serial_number() # create a self-signed cert subject_name = x509.Name( [ @@ -57,19 +62,52 @@ def create_self_signed_certificate( .subject_name(subject_name) .issuer_name(subject_name) .public_key(key.public_key()) - .serial_number(x509.random_serial_number()) + .serial_number(serial) .not_valid_before(datetime.datetime.utcnow()) .not_valid_after( datetime.datetime.utcnow() + datetime.timedelta(days=valid_days) ) - .sign(key, hashes.SHA256()) ) + # v3_ca extensions + if v3_extensions: + subject_key_id = x509.SubjectKeyIdentifier.from_public_key(key.public_key()) + authority_key_id = x509.AuthorityKeyIdentifier( + authority_cert_issuer=[x509.DirectoryName(subject_name)], + authority_cert_serial_number=serial, + key_identifier=subject_key_id.digest + ) + basic = x509.BasicConstraints(ca=True, path_length=None) + key_usage = x509.KeyUsage( + digital_signature=True, + crl_sign=True, + key_cert_sign=True, + content_commitment=False, + data_encipherment=False, + decipher_only=False, + encipher_only=False, + key_agreement=False, + key_encipherment=False, + ) + cert = ( + cert + .add_extension(subject_key_id, critical=False) + .add_extension(authority_key_id, critical=False) + .add_extension(basic, critical=True) + .add_extension(key_usage, critical=True) + ) + + # sign + cert = cert.sign(key, hashes.SHA256()) + + # private key key_dump = key.private_bytes( encoding=serialization.Encoding.PEM, format=serialization.PrivateFormat.TraditionalOpenSSL, encryption_algorithm=serialization.NoEncryption(), ).decode("utf-8") + + # certificate string cert_dump = cert.public_bytes(serialization.Encoding.PEM).decode("utf-8") hash = None @@ -80,18 +118,26 @@ def create_self_signed_certificate( else: raise ValueError("Only SHA1 and SHA256 supported for now.") + # thumbprint thumbprint = cert.fingerprint(hash).hex().upper() if cert_output_dir and exists(cert_output_dir): cert_file = (file_prefix or subject) + "-cert.pem" key_file = (file_prefix or subject) + "-key.pem" - - with open(join(cert_output_dir, cert_file), "wt", encoding="utf-8") as f: - f.write(cert_dump) + write_content_to_file( + content=cert_dump, + destination=cert_output_dir, + file_name=cert_file, + overwrite=True, + ) if not cert_only: - with open(join(cert_output_dir, key_file), "wt", encoding="utf-8") as f: - f.write(key_dump) + write_content_to_file( + content=key_dump, + destination=cert_output_dir, + file_name=key_file, + overwrite=True, + ) result = { "certificate": cert_dump, @@ -122,3 +168,178 @@ def open_certificate(certificate_path: str) -> str: certificate = base64.b64encode(certificate).decode("utf-8") # Remove trailing white space from the certificate content return certificate.rstrip() + + +def create_ca_signed_certificate( + subject: str, + ca_public_key: str, + ca_private_key: str, + cert_output_dir: Optional[str] = None, + cert_file: Optional[str] = None, + key_size: int = 4096, + valid_days: int = 365, +) -> Dict[str, str]: + """ + Function used to create a new X.509 v3 certificate signed by an existing CA cert. + + Args: + subject (str): Certificate common name field. + ca_public (str): Signing CA public key + ca_private (str): Signing CA private key + cert_output_dir (str): string value of output directory. + cert_file (bool): Certificate file name if it needs to be different from the subject. + key_size (str): The size of the generated private key + valid_days (int): number of days certificate is valid for; used to calculate + certificate expiry. + + Returns: + result (dict): dict with certificate value, private key and thumbprint. + """ + + private_key = rsa.generate_private_key(public_exponent=65537, key_size=key_size) + ca_public_key = ca_public_key.encode("utf-8") + ca_private_key = ca_private_key.encode("utf-8") + ca_key = serialization.load_pem_private_key(ca_private_key, password=None) + ca_cert = x509.load_pem_x509_certificate(ca_public_key) + + # v3 certificate extensions + subject_key_id = x509.SubjectKeyIdentifier.from_public_key(ca_key.public_key()) + auth_subject_key = ca_cert.extensions.get_extension_for_class(x509.SubjectKeyIdentifier) or subject_key_id + authority_key_id = x509.AuthorityKeyIdentifier( + authority_cert_issuer=[x509.DirectoryName(ca_cert.subject)], + authority_cert_serial_number=ca_cert.serial_number, + key_identifier=auth_subject_key.value.digest + ) + basic_constraints = x509.BasicConstraints(ca=True, path_length=None) + key_usage = x509.KeyUsage( + digital_signature=True, + crl_sign=True, + key_cert_sign=True, + content_commitment=False, + data_encipherment=False, + decipher_only=False, + encipher_only=False, + key_agreement=False, + key_encipherment=False, + ) + subject_name = x509.Name( + [ + x509.NameAttribute(NameOID.COMMON_NAME, subject), + ] + ) + cert = ( + x509.CertificateBuilder() + .subject_name(subject_name) + .issuer_name(ca_cert.subject) + .public_key(private_key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(datetime.datetime.utcnow()) + .not_valid_after( + datetime.datetime.utcnow() + datetime.timedelta(days=valid_days) + ) + .add_extension(subject_key_id, False) + .add_extension(authority_key_id, False) + .add_extension(basic_constraints, True) + .add_extension(key_usage, True) + .sign(ca_key, hashes.SHA256()) + ) + certificate = cert.public_bytes(serialization.Encoding.PEM).decode("utf-8") + thumbprint = cert.fingerprint(hashes.SHA256()).hex().upper() + privateKey = private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + ).decode("utf-8") + + if cert_output_dir and exists(cert_output_dir): + write_content_to_file( + content=certificate, + destination=cert_output_dir, + file_name=f"{cert_file or subject}.cert.pem", + ) + write_content_to_file( + content=privateKey, + destination=cert_output_dir, + file_name=f"{cert_file or subject}.key.pem", + ) + return { + "certificate": certificate, + "thumbprint": thumbprint, + "privateKey": privateKey, + } + + +def load_ca_cert_info( + cert_path: str, key_path: str, password: Optional[str] = None +) -> Dict[str, str]: + """ + Function used to load CA certificate public and private key content + into our certificate / thumprint / privateKey format. + + Args: + cert_path (str): Path to certificate public key file. + key_path (str): Path to the certificate private key file. + password (str): Optional password used to unlock the private key. + + Returns: + result (dict): dict with certificate value, private key and thumbprint. + """ + for path in [cert_path, key_path]: + if not exists(path): + raise FileOperationError( + f"Error loading certificates. No file found at path '{path}'" + ) + # open cert files and get string contents + key_str = read_file_content(key_path).encode("utf-8") + cert_str = read_file_content(cert_path).encode("utf-8") + + # load certificates + try: + cert_obj = x509.load_pem_x509_certificate(cert_str) + key_obj = serialization.load_pem_private_key( + key_str, password=(password.encode("utf-8") if password else None) + ) + except Exception as ex: + raise FileOperationError(f"Error loading certificate info:\n{ex}") + + # create correctly stringified versions + key = key_obj.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + ).decode("utf-8") + thumbprint = cert_obj.fingerprint(hashes.SHA256()).hex().upper() + cert_dump = cert_obj.public_bytes(serialization.Encoding.PEM).decode("utf-8") + + return { + "certificate": cert_dump, + "thumbprint": thumbprint, + "privateKey": key, + } + + +def make_cert_chain( + certs: List[str], + output_dir: Optional[str] = None, + output_file: Optional[str] = None, +) -> str: + """ + Function used to create a simple chain certificate file on disk. + + Args: + certs List[str]: List of certificate contents (strings) to write to the file. + output_dir str: The output directory to write the chained cert to. + output_file str: The file name of the written certificate chain file. + + Returns: + cert_content str: String content of chained certs + """ + cert_content = "".join(certs) + if output_dir and exists(output_dir) and len(certs): + write_content_to_file( + content=cert_content, + destination=output_dir, + file_name=output_file or "cert-chain.pem", + overwrite=True, + ) + return cert_content diff --git a/azext_iot/common/fileops.py b/azext_iot/common/fileops.py new file mode 100644 index 000000000..7439600eb --- /dev/null +++ b/azext_iot/common/fileops.py @@ -0,0 +1,53 @@ +# coding=utf-8 +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +from os import makedirs, remove, listdir +from os.path import exists, join +from pathlib import PurePath +from typing import Union +from azure.cli.core.azclierror import FileOperationError + + +""" +fileops: Functions for working with files. +""" + + +def write_content_to_file( + content: Union[str, bytes], + destination: str, + file_name: str, + overwrite: bool = False, +): + dest_path = PurePath(destination) + file_path = dest_path.joinpath(file_name) + + if exists(file_path) and not overwrite: + raise FileOperationError(f"File already exists at path: {file_path}") + if overwrite and destination: + makedirs(destination, exist_ok=True) + write_content = bytes(content, "utf-8") if isinstance(content, str) else content + with open(file_path, "wb") as f: + f.write(write_content) + + +def tar_directory( + target_directory: str, + tarfile_path: str, + tarfile_name: str, + overwrite: bool = False, +): + full_path = join(tarfile_path, f"{tarfile_name}.tgz") + if exists(full_path): + if not overwrite: + raise FileOperationError(f"File {full_path} already exists") + remove(full_path) + if not exists(tarfile_path): + makedirs(tarfile_path, exist_ok=overwrite) + import tarfile + with tarfile.open(full_path, "w:gz") as tar: + for file_name in listdir(target_directory): + tar.add(join(target_directory, file_name), file_name) diff --git a/azext_iot/common/shared.py b/azext_iot/common/shared.py index 5157976be..1aa12f6f7 100644 --- a/azext_iot/common/shared.py +++ b/azext_iot/common/shared.py @@ -263,6 +263,7 @@ class IoTDPSStateType(Enum): """ IoT Hub Device Provisioning Service State Property """ + Activating = "Activating" ActivationFailed = "ActivationFailed" Active = "Active" @@ -279,12 +280,13 @@ class IoTDPSStateType(Enum): class ConnectionStringParser(Enum): """ - All connection string parser with respective functions + All connection string parser with respective functions """ + from azext_iot.common._azure import ( parse_iot_device_connection_string, parse_iot_device_module_connection_string, - parse_iot_hub_connection_string + parse_iot_hub_connection_string, ) Module = parse_iot_device_module_connection_string @@ -296,6 +298,7 @@ class DiscoveryResourceType(Enum): """ Resource types supported by discovery. """ + IoTHub = "IoT Hub" DPS = "IoT Hub Device Provisioning Service" @@ -304,5 +307,6 @@ class SHAHashVersions(Enum): """ Supported SHA types for generating the certificate thumbprint. """ + SHA1 = 1 SHA256 = 256 diff --git a/azext_iot/common/utility.py b/azext_iot/common/utility.py index 40a764feb..d0d7fda08 100644 --- a/azext_iot/common/utility.py +++ b/azext_iot/common/utility.py @@ -19,10 +19,11 @@ import hmac import hashlib +from typing import Any, Optional, List, Dict + from threading import Event, Thread from datetime import datetime from knack.log import get_logger -from typing import Optional from azure.cli.core.azclierror import ( CLIInternalError, FileOperationError, @@ -119,7 +120,9 @@ def validate_key_value_pairs(string): return result -def process_json_arg(content: str, argument_name: str = "content", preserve_order=False): +def process_json_arg( + content: str, argument_name: str = "content", preserve_order=False +): """Primary processor of json input""" json_from_file = None @@ -146,6 +149,48 @@ def process_json_arg(content: str, argument_name: str = "content", preserve_orde ) +_file_location_error = "{0} file not found - Please ensure the path '{1}' is correct." +_file_parse_error = "Failed to parse {0} file located at '{1}' with exception:\n{2}" + + +def process_yaml_arg(path: str) -> Dict[str, Any]: + """Primary processor of yaml file input""" + + if not os.path.exists(path): + raise FileOperationError( + _file_location_error.format("YAML", path) + ) + + try: + import yaml + + with open(path, "rb") as f: + return yaml.load(f, Loader=yaml.SafeLoader) + except Exception as ex: + raise InvalidArgumentValueError( + _file_parse_error.format("YAML", path, ex) + ) + + +def process_toml_arg(path: str) -> Dict[str, Any]: + """Primary processor of TOML file input""" + + if not os.path.exists(path): + raise FileOperationError( + _file_location_error.format("TOML", path) + ) + + try: + import tomli + + with open(path, "rb") as f: + return tomli.load(f) + except Exception as ex: + raise InvalidArgumentValueError( + _file_parse_error.format("TOML", path, ex) + ) + + def shell_safe_json_parse(json_or_dict_string, preserve_order=False): """Allows the passing of JSON or Python dictionary strings. This is needed because certain JSON strings in CMD shell are not received in main's argv. This allows the user to specify @@ -412,7 +457,15 @@ def calculate_millisec_since_unix_epoch_utc(offset_seconds: int = 0): return int(1000 * ((now - epoch).total_seconds() + offset_seconds)) -def init_monitoring(cmd, timeout, properties, enqueued_time, repair, yes, message_count: Optional[int] = None): +def init_monitoring( + cmd, + timeout, + properties, + enqueued_time, + repair, + yes, + message_count: Optional[int] = None, +): from azext_iot.common.deps import ensure_uamqp if timeout < 0: @@ -422,9 +475,7 @@ def init_monitoring(cmd, timeout, properties, enqueued_time, repair, yes, messag timeout = timeout * 1000 if message_count and message_count <= 0: - raise InvalidArgumentValueError( - "Message count must be greater than 0." - ) + raise InvalidArgumentValueError("Message count must be greater than 0.") config = cmd.cli_ctx.config output = cmd.cli_ctx.invocation.data.get("output", None) @@ -649,3 +700,25 @@ def generate_storage_account_sas_token( ) return sas_token + + +def assemble_nargs_to_dict(hash_list: List[str]) -> Dict[str, str]: + result = {} + if not hash_list: + return result + for hash in hash_list: + if "=" not in hash: + logger.warning( + "Skipping processing of '%s', input format is key=value | key='value value'.", + hash, + ) + continue + split_hash = hash.split("=", 1) + result[split_hash[0]] = split_hash[1] + for key in result: + if not result.get(key): + logger.warning( + "No value assigned to key '%s', input format is key=value | key='value value'.", + key, + ) + return result diff --git a/azext_iot/deviceupdate/commands_update.py b/azext_iot/deviceupdate/commands_update.py index c73db4305..e7794c19a 100644 --- a/azext_iot/deviceupdate/commands_update.py +++ b/azext_iot/deviceupdate/commands_update.py @@ -5,7 +5,7 @@ # -------------------------------------------------------------------------------------------- from knack.log import get_logger -from azext_iot.common.utility import handle_service_exception +from azext_iot.common.utility import handle_service_exception, assemble_nargs_to_dict from azext_iot.deviceupdate.providers.base import ( DeviceUpdateDataModels, DeviceUpdateDataManager, @@ -141,7 +141,7 @@ def import_update( if not size or not hashes: client_calculated_meta = data_manager.calculate_manifest_metadata(url) - hashes = data_manager.assemble_nargs_to_dict(hash_list=hashes) or {"sha256": client_calculated_meta.hash} + hashes = assemble_nargs_to_dict(hash_list=hashes) or {"sha256": client_calculated_meta.hash} size = size or client_calculated_meta.bytes manifest_metadata = DeviceUpdateDataModels.ImportManifestMetadata(url=url, size_in_bytes=size, hashes=hashes) @@ -292,7 +292,7 @@ def _associate_related(sanitized_params: list, key: str) -> dict: for compat in compatibility: if not compat or not compat[0]: continue - processed_compatibility.append(DeviceUpdateDataManager.assemble_nargs_to_dict(compat)) + processed_compatibility.append(assemble_nargs_to_dict(compat)) payload["compatibility"] = processed_compatibility safe_params = cmd.cli_ctx.data.get("safe_params", []) @@ -304,7 +304,7 @@ def _associate_related(sanitized_params: list, key: str) -> dict: step_file_params = _sanitize_safe_params(safe_params, ["--step", "--file"]) related_step_file_map = _associate_related(step_file_params, "--step") - assembled_step = DeviceUpdateDataManager.assemble_nargs_to_dict(steps[s]) + assembled_step = assemble_nargs_to_dict(steps[s]) step = {} if all(k in assembled_step for k in ("updateId.provider", "updateId.name", "updateId.version")): # reference step @@ -336,7 +336,7 @@ def _associate_related(sanitized_params: list, key: str) -> dict: step_file = files[f] if not step_file or not step_file[0]: continue - assembled_step_file = DeviceUpdateDataManager.assemble_nargs_to_dict(step_file) + assembled_step_file = assemble_nargs_to_dict(step_file) if "path" in assembled_step_file: step_filename = PurePath(assembled_step_file["path"]).name if step_filename not in derived_step_files: @@ -381,7 +381,7 @@ def _associate_related(sanitized_params: list, key: str) -> dict: if not files[f] or not files[f][0]: continue processed_file = {} - assembled_file = DeviceUpdateDataManager.assemble_nargs_to_dict(files[f]) + assembled_file = assemble_nargs_to_dict(files[f]) if "path" not in assembled_file: raise ArgumentUsageError("When using --file path is required.") assembled_file_metadata = DeviceUpdateDataManager.calculate_file_metadata(assembled_file["path"]) @@ -402,7 +402,7 @@ def _associate_related(sanitized_params: list, key: str) -> dict: if not related_file or not related_file[0]: continue processed_related_file = {} - assembled_related_file = DeviceUpdateDataManager.assemble_nargs_to_dict(related_file) + assembled_related_file = assemble_nargs_to_dict(related_file) if "path" not in assembled_related_file: raise ArgumentUsageError("When using --related-file path is required.") related_file_metadata = DeviceUpdateDataManager.calculate_file_metadata(assembled_related_file["path"]) diff --git a/azext_iot/deviceupdate/providers/base.py b/azext_iot/deviceupdate/providers/base.py index 7f9070534..4eb5df71b 100644 --- a/azext_iot/deviceupdate/providers/base.py +++ b/azext_iot/deviceupdate/providers/base.py @@ -26,7 +26,7 @@ from azure.core.exceptions import AzureError, HttpResponseError from msrest.serialization import Model from pathlib import Path, PurePath -from typing import Any, NamedTuple, Union, List, Dict, Tuple, Optional +from typing import Any, NamedTuple, Union, List, Tuple, Optional import json import os @@ -262,22 +262,6 @@ def calculate_hash_from_bytes(cls, raw_bytes: bytes) -> str: return b64encode(sha256(raw_bytes).digest()).decode("utf8") - @classmethod - def assemble_nargs_to_dict(cls, hash_list: List[str]) -> Dict[str, str]: - result = {} - if not hash_list: - return result - for hash in hash_list: - if "=" not in hash: - logger.warning("Skipping processing of '%s', input format is key=value | key='value value'.", hash) - continue - split_hash = hash.split("=", 1) - result[split_hash[0]] = split_hash[1] - for key in result: - if not result.get(key): - logger.warning("No value assigned to key '%s', input format is key=value | key='value value'.", key) - return result - def assemble_files(self, file_list_col: List[List[str]]) -> Union[DeviceUpdateDataModels.FileImportMetadata, None]: if not file_list_col: return diff --git a/azext_iot/iothub/_help.py b/azext_iot/iothub/_help.py index a74ffa1e6..e8e63238e 100644 --- a/azext_iot/iothub/_help.py +++ b/azext_iot/iothub/_help.py @@ -316,6 +316,53 @@ def load_iothub_help(): short-summary: Upload a local file as a device to a pre-configured blob storage container. """ + helps["iot edge devices"] = """ + type: group + short-summary: Commands to manage IoT Edge devices. + """ + + helps["iot edge devices create"] = """ + type: command + short-summary: Create and configure multiple edge devices in an IoT Hub. + long-summary: | + This operation accepts inline device arguments or an edge devices configuration file in YAML or JSON format. + Inline command args (like '--device-auth') will take precedence and override configuration file properties if they are provided. + A sample configuration file can be found here: https://aka.ms/aziotcli-edge-devices-config + Review examples and parameter descriptions for details on how to fully utilize this operation. + + examples: + - name: Create a couple of edge devices using symmetric key auth (default) + text: | + az iot edge devices create -n {hub_name} --device id=device_1 --device id=device_2 + + - name: Create a flat list of edge devices using self-signed certificate authentication with various edge property configurations, using inline arguments. + text: | + az iot edge devices create -n {hub_name} --device-auth x509_thumbprint --default-edge-agent "mcr.microsoft.com/azureiotedge-agent:1.4" + --device id=device_1 hostname={FQDN} + --device id=device_2 edge_agent={agent_image} + --device id=parent hostname={FQDN} edge_agent={agent_image} container_auth={path_or_json_string} + + - name: Delete all existing device-identities on a hub and create new devices based on a configuration file (with progress bars and visualization output). + text: > + az iot edge devices create -n {hub_name} --cfg path/to/config_yml_or_json -c -v + + - name: Create a group of nested edge devices with custom module deployments - containing 2 parent devices with 1 child device each, using inline arguments. + Also specifies output path for device certificate bundles. + text: | + az iot edge devices create -n {hub_name} --out {device_bundle_path} + --device id=parent_1 deployment=/path/to/parentDeployment_1.json + --device id=child_1 parent=parent_1 deployment=/path/to/child_deployment_1.json + --device id=parent_2 deployment=/path/to/parentDeployment_2.json + --device id=child_2 parent=parent_2 deployment=/path/to/child_deployment_2.json + + - name: Create a simple nested edge device configuration with an existing root CA, using x509 auth, and specify a custom device bundle output path. + text: | + az iot edge devices create -n {hub_name} --out {device_bundle_path} + --root-cert "root_cert.pem" --root-key "root_key.pem" --device-auth x509_thumbprint + --device id=parent1 + --device id=child1 parent=parent1 + """ + helps[ "iot hub message-endpoint" ] = """ diff --git a/azext_iot/iothub/command_map.py b/azext_iot/iothub/command_map.py index 261fdcc32..9360be688 100644 --- a/azext_iot/iothub/command_map.py +++ b/azext_iot/iothub/command_map.py @@ -19,6 +19,9 @@ device_messaging_ops = CliCommandType( operations_tmpl="azext_iot.iothub.commands_device_messaging#{}" ) +device_identity_ops = CliCommandType( + operations_tmpl="azext_iot.iothub.commands_device_identity#{}" +) iothub_resource_ops = CliCommandType( operations_tmpl="azext_iot.iothub.commands_certificate#{}" ) @@ -46,7 +49,9 @@ def load_iothub_commands(self, _): cmd_group.command("list", "job_list") cmd_group.command("cancel", "job_cancel") - with self.command_group("iot hub digital-twin", command_type=pnp_runtime_ops) as cmd_group: + with self.command_group( + "iot hub digital-twin", command_type=pnp_runtime_ops + ) as cmd_group: cmd_group.command("invoke-command", "invoke_device_command") cmd_group.show_command("show", "get_digital_twin") cmd_group.command("update", "patch_digital_twin") @@ -133,6 +138,11 @@ def load_iothub_commands(self, _): cmd_group.command("send", "iot_c2d_message_send") cmd_group.command("purge", "iot_c2d_message_purge") + with self.command_group( + "iot edge devices", command_type=device_identity_ops + ) as cmd_group: + cmd_group.command("create", "iot_edge_devices_create", is_experimental=True) + with self.command_group( "iot hub certificate root-authority", command_type=iothub_resource_ops, is_experimental=True ) as cmd_group: diff --git a/azext_iot/iothub/commands_device_identity.py b/azext_iot/iothub/commands_device_identity.py new file mode 100644 index 000000000..d1ebc0f10 --- /dev/null +++ b/azext_iot/iothub/commands_device_identity.py @@ -0,0 +1,73 @@ +# coding=utf-8 +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +from typing import List, Optional +from azext_iot.iothub.providers.device_identity import DeviceIdentityProvider +from knack.log import get_logger + +logger = get_logger(__name__) + + +def iot_edge_devices_create( + cmd, + devices: Optional[List[List[str]]] = None, + config_file: Optional[str] = None, + visualize: bool = False, + clean: bool = False, + yes: bool = False, + device_auth_type: Optional[str] = None, + default_edge_agent: Optional[str] = None, + device_config_template: Optional[str] = None, + root_cert_path: Optional[str] = None, + root_key_path: Optional[str] = None, + root_cert_password: Optional[str] = None, + bundle_output_path: Optional[str] = None, + hub_name: Optional[str] = None, + resource_group_name: Optional[str] = None, + login: Optional[str] = None, + auth_type_dataplane: Optional[str] = None, +): + device_identity_provider = DeviceIdentityProvider( + cmd=cmd, + hub_name=hub_name, + rg=resource_group_name, + login=login, + auth_type_dataplane=auth_type_dataplane, + ) + return device_identity_provider.create_edge_devices( + devices=devices, + config_file=config_file, + clean=clean, + yes=yes, + visualize=visualize, + auth_type=device_auth_type, + default_edge_agent=default_edge_agent, + device_config_template=device_config_template, + root_cert_path=root_cert_path, + root_key_path=root_key_path, + root_cert_password=root_cert_password, + output_path=bundle_output_path, + ) + + +def iot_delete_devices( + cmd, + device_ids: List[str], + hub_name: Optional[str] = None, + resource_group_name: Optional[str] = None, + login: Optional[str] = None, + auth_type_dataplane: Optional[str] = None, +): + device_identity_provider = DeviceIdentityProvider( + cmd=cmd, + hub_name=hub_name, + rg=resource_group_name, + login=login, + auth_type_dataplane=auth_type_dataplane, + ) + return device_identity_provider.delete_device_identities( + device_ids=device_ids + ) diff --git a/azext_iot/iothub/common.py b/azext_iot/iothub/common.py index 31f163092..8ba7d7e15 100644 --- a/azext_iot/iothub/common.py +++ b/azext_iot/iothub/common.py @@ -3,12 +3,47 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- - """ -shared: Define shared data types(enums) and constant strings. +common: Define shared data types(enums) and constant strings. """ from enum import Enum +from typing import NamedTuple, Optional, List, Dict +from azext_iot.sdk.iothub.service.models import ConfigurationContent + + +class EdgeContainerAuth(NamedTuple): + """ + Edge container authentication datatype + """ + serveraddress: str + username: str + password: str + + +class EdgeDeviceConfig(NamedTuple): + """ + Individual Edge device configuration data format. + """ + device_id: str + deployment: Optional[ConfigurationContent] = None + parent_id: Optional[str] = None + hostname: Optional[str] = None + parent_hostname: Optional[str] = None + edge_agent: Optional[str] = None + container_auth: Optional[EdgeContainerAuth] = None + + +class EdgeDevicesConfig(NamedTuple): + """ + Edge device configuration file data format. + """ + version: str + auth_method: str + root_cert: Dict[str, str] + devices: List[EdgeDeviceConfig] + template_config_path: Optional[str] = None + default_edge_agent: Optional[str] = None SYSTEM_ASSIGNED_IDENTITY = '[system]' diff --git a/azext_iot/iothub/params.py b/azext_iot/iothub/params.py index 5f95cb116..8e5d69ab2 100644 --- a/azext_iot/iothub/params.py +++ b/azext_iot/iothub/params.py @@ -6,7 +6,7 @@ from azext_iot.iothub.common import CertificateAuthorityVersions from azure.cli.core.commands.parameters import get_enum_type, get_three_state_flag -from azext_iot.common.shared import SettleType, ProtocolType, AckType +from azext_iot.common.shared import DeviceAuthType, SettleType, ProtocolType, AckType from azext_iot.assets.user_messages import info_param_properties_device from azext_iot._params import hub_auth_type_dataplane_param_type from azext_iot.iothub.common import EncodingFormat, EndpointType, RouteSourceType @@ -44,14 +44,14 @@ def load_iothub_arguments(self, _): type=int, options_list=["--connect-timeout", "--cto"], help="Maximum interval of time, in seconds, that IoT Hub will attempt to connect to the device.", - arg_group="Timeout" + arg_group="Timeout", ) context.argument( "response_timeout", type=int, options_list=["--response-timeout", "--rto"], help="Maximum interval of time, in seconds, that the digital twin command will wait for the result.", - arg_group="Timeout" + arg_group="Timeout", ) with self.argument_context("iot device") as context: @@ -249,6 +249,98 @@ def load_iothub_arguments(self, _): help="MIME Type of file.", ) + with self.argument_context("iot edge devices") as context: + context.argument( + "devices", + options_list=["--device", "-d"], + nargs="+", + action="append", + help="Space-separated key=value pairs corresponding to properties of the edge device to create. " + "The following key values are supported: `id` (device_id), `deployment` (inline json or path to file), `hostname`, " + "`parent` (device_id), `edge_agent` (image URL), and `container_auth` (inline json or path to file). " + "--device can be used 1 or more times. Review help examples for full parameter usage - these parameters also refer " + "to their corresponding values in our sample configuration file: " + "https://aka.ms/aziotcli-edge-devices-config" + ) + context.argument( + "clean", + options_list=["--clean", "-c"], + arg_type=get_three_state_flag(), + help="Deletes all devices in target hub before creating new devices.", + ) + context.argument( + "visualize", + options_list=["--visualize", "--vis", "-v"], + arg_type=get_three_state_flag(), + help="Shows visualizations of devices and progress of various tasks " + "(device creation, setting parents, updating configs, etc).", + ) + context.argument( + "config_file", + options_list=["--config-file", "--config", "--cfg"], + help="Path to devices configuration file. Sample configuration file: " + "https://aka.ms/aziotcli-edge-devices-config", + ) + context.argument( + "device_auth_type", + arg_type=get_enum_type( + [DeviceAuthType.shared_private_key.value, DeviceAuthType.x509_thumbprint.value] + ), + options_list=["--device-auth-type", "--device-auth"], + help="Device to hub authorization mechanism.", + ) + context.argument( + "default_edge_agent", + options_list=["--default-edge-agent", "--default-agent", "--dea"], + help="Default edge agent for created Edge devices if not specified individually.", + ) + context.argument( + "device_config_template", + options_list=["--device-config-template", "--dct"], + help="Path to IoT Edge config.toml file to use as a basis for edge device configs.", + ) + context.argument( + "bundle_output_path", + options_list=[ + "--output-path", + "--out", + ], + help="Directory path to output device configuration bundles. " + "If this value is not specified, no file output will be created.", + ) + context.argument( + "root_cert_path", + options_list=[ + "--root-cert", + "--rc", + ], + help="Path to root public key certificate to sign nested edge device certs.", + arg_group="Root Certificate" + ) + context.argument( + "root_key_path", + options_list=[ + "--root-key", + "--rk", + ], + help="Path to root private key to sign nested edge device certs.", + arg_group="Root Certificate" + ) + context.argument( + "root_cert_password", + options_list=[ + "--root-pass", + "--rp", + ], + help="Root key password", + arg_group="Root Certificate" + ) + context.argument( + "yes", + options_list=['--yes', '-y'], + help='Do not prompt for confirmation when --clean switch is used to delete existing hub devices.', + ) + with self.argument_context("iot hub message-endpoint") as context: context.argument( "hub_name", diff --git a/azext_iot/iothub/providers/device_identity.py b/azext_iot/iothub/providers/device_identity.py new file mode 100644 index 000000000..4f3e9e290 --- /dev/null +++ b/azext_iot/iothub/providers/device_identity.py @@ -0,0 +1,442 @@ +# coding=utf-8 +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- +from pathlib import PurePath +from knack.prompting import prompt_y_n +from os import makedirs +from os.path import exists, abspath +from shutil import rmtree +from azext_iot.common.certops import ( + create_self_signed_certificate, + create_ca_signed_certificate, + make_cert_chain, +) + +from azext_iot.common.fileops import tar_directory, write_content_to_file +from azext_iot.iothub.providers.helpers.edge_device_config import ( + DEVICE_README, + EDGE_ROOT_CERTIFICATE_FILENAME, + MAX_DEVICE_SCOPE_RETRIES, + create_edge_device_config, + process_edge_devices_config_args, + process_edge_devices_config_file_content, + create_edge_device_config_script, +) +from tqdm import tqdm +from time import sleep +from typing import Dict, List +from knack.log import get_logger +from typing import Optional +from azext_iot.common.shared import ( + DeviceAuthType, + SdkType, +) +from azext_iot.iothub.common import ( + EdgeDevicesConfig, +) +from azext_iot.iothub.providers.base import IoTHubProvider +from azext_iot.common.utility import ( + process_json_arg, + process_yaml_arg, +) +from azext_iot.operations.generic import _execute_query +from azure.cli.core.azclierror import ( + AzureResponseError, + InvalidArgumentValueError, + ManualInterrupt, + MutuallyExclusiveArgumentError, +) +from azext_iot.operations.hub import _assemble_device +from azext_iot.sdk.iothub.service.models import Device + +logger = get_logger(__name__) + + +class DeviceIdentityProvider(IoTHubProvider): + def __init__( + self, + cmd, + hub_name: Optional[str] = None, + rg: Optional[str] = None, + login: Optional[str] = None, + auth_type_dataplane: Optional[str] = None, + ): + super(DeviceIdentityProvider, self).__init__( + cmd=cmd, + hub_name=hub_name, + rg=rg, + login=login, + auth_type_dataplane=auth_type_dataplane, + ) + self.service_sdk = self.get_sdk(sdk_type=SdkType.service_sdk) + + def create_edge_devices( + self, + devices: Optional[List[List[str]]] = None, + config_file: Optional[str] = None, + visualize: bool = False, + clean: bool = False, + yes: bool = False, + auth_type: Optional[str] = None, + default_edge_agent: Optional[str] = None, + device_config_template: Optional[str] = None, + root_cert_path: Optional[str] = None, + root_key_path: Optional[str] = None, + root_cert_password: Optional[str] = None, + output_path: Optional[str] = None, + ): + from treelib import Tree + from treelib.exceptions import ( + NodeIDAbsentError, + LoopError, + DuplicatedNodeIdError, + ) + + config: EdgeDevicesConfig = None + + # configuration for root cert and output directories + root_cert_name = EDGE_ROOT_CERTIFICATE_FILENAME + bundle_output_directory = None + if output_path: + if not exists(output_path): + makedirs(output_path, exist_ok=True) + bundle_output_directory = PurePath(output_path) + + # If user has provided a path to a configuration file + if config_file: + # Cannot process both config file and --devices nargs* + if devices: + raise MutuallyExclusiveArgumentError( + "Please either use a --config-file argument or inline --device arguments, both were provided." + ) + + config_content = None + # Process Edge Config file into object dictionary + if config_file.endswith((".yaml", ".yml")): + config_content = process_yaml_arg(config_file) + elif config_file.endswith(".json"): + config_content = process_json_arg(config_file) + else: + raise InvalidArgumentValueError("Config file must be JSON or YAML") + + # path arguments in config file are relative to the config file's path + config_path = PurePath(config_file).parent.as_posix() + + config = process_edge_devices_config_file_content( + config_path=config_path, + content=config_content, + override_auth_type=auth_type, + override_root_cert_path=root_cert_path, + override_root_key_path=root_key_path, + override_default_edge_agent=default_edge_agent, + override_device_config_template=device_config_template, + ) + elif devices: + config = process_edge_devices_config_args( + device_args=devices, + auth_type=auth_type, + default_edge_agent=default_edge_agent, + device_config_template=device_config_template, + root_cert_path=root_cert_path, + root_key_path=root_key_path, + root_cert_password=root_cert_password, + ) + + if not config or not len(config.devices): + raise InvalidArgumentValueError( + "No devices found in input. " + "Please check your input arguments or config file and try the command again" + ) + tree = Tree() + tree_root_node_id = "|root|" + tree.create_node("Devices", tree_root_node_id) + hub_cert_auth = config.auth_method == DeviceAuthType.x509_thumbprint.value + + # dict of device parents by ID + device_to_parent_dict: Dict[str, str] = {} + # device configs by id + device_config_dict: Dict[str, EdgeDevicesConfig] = {} + for device_config in config.devices: + device_config_dict[device_config.device_id] = device_config + + # first pass to create devices in flat tree + config_devices_iterator = ( + tqdm( + config.devices, + desc="Creating device structure and certificates", + ) + if visualize + else config.devices + ) + for device_config in config_devices_iterator: + device_id = device_config.device_id + # add to flat tree + try: + tree.create_node(device_id, device_id, parent=tree_root_node_id) + except DuplicatedNodeIdError: + raise InvalidArgumentValueError( + f"Duplicate deviceId '{device_id}' detected" + ) + + # second pass to move nodes and check hierarchy + for device_config in config.devices: + device_id = device_config.device_id + + # Move nodes to their correct parents, track device->parent in dict + device_parent = device_config.parent_id or tree_root_node_id + if device_parent != tree_root_node_id: + device_to_parent_dict[device_id] = device_parent + try: + tree.update_node(device_id, data=device_parent) + tree.move_node(device_id, device_parent) + except NodeIDAbsentError: + raise InvalidArgumentValueError( + f"Error building device hierarchy, missing parent '{device_parent}'" + ) + except LoopError: + raise InvalidArgumentValueError( + "Error building device hierarchy, found a loop between " + f"devices '{device_id}' and '{device_parent}'." + ) + + # Show the device tree + if visualize: + tree.show() + + # Query existing devices + query_args = ["SELECT deviceId FROM devices"] + query_method = self.service_sdk.query.get_twins + existing_devices = _execute_query(query_args, query_method) + existing_device_ids = [x["deviceId"] for x in existing_devices] + + # Clear devices if necessary + if clean and existing_device_ids: + if not yes and not prompt_y_n(msg=f"Confirm you want to delete all devices in '{self.hub_name}'", default='n'): + raise ManualInterrupt("Operation was aborted, existing device deletion was not confirmed.") + delete_iterator = ( + tqdm(existing_device_ids, "Deleting existing device identities") + if visualize + else existing_device_ids + ) + self.delete_device_identities(delete_iterator) + if self.service_sdk.devices.get_devices(): + raise AzureResponseError( + "An error has occurred - Not all devices were deleted." + ) + else: + # If not cleaning the hub, ensure no duplicate device ids + duplicates = list( + filter(lambda id: id in device_config_dict, existing_device_ids) + ) + if any(duplicates): + raise InvalidArgumentValueError( + f"The following devices already exist on hub '{self.hub_name}': {duplicates}. " + "To clear all devices before creating the hierarchy, please utilize the `--clean` switch." + ) + + # Create all devices and configs + device_iterator = ( + tqdm(config.devices, desc="Creating device identities and configs") + if visualize + else config.devices + ) + + for device in device_iterator: + device_id = device.device_id + device_cert_output_directory = None + if bundle_output_directory: + device_cert_output_directory = bundle_output_directory.joinpath(device_id) + # if the device's folder already exists, remove it + if exists(device_cert_output_directory): + rmtree(device_cert_output_directory) + # create fresh device folder + makedirs(device_cert_output_directory) + + # signed device cert + signed_device_cert = create_ca_signed_certificate( + subject=f"{device_id}.deviceca", + ca_public_key=config.root_cert["certificate"], + ca_private_key=config.root_cert["privateKey"], + cert_output_dir=device_cert_output_directory, + cert_file=device_id, + ) + + device_pk = None + device_sk = None + # if using x509 device auth + if hub_cert_auth: + # hub auth cert for device + device_hub_cert = create_self_signed_certificate( + subject=device_id, + valid_days=365, + key_size=4096, + sha_version=256, + v3_extensions=True + ) + device_pk = signed_device_cert["thumbprint"] + device_sk = device_hub_cert["thumbprint"] + + # create device object for service + assembled_device = _assemble_device( + is_update=False, + device_id=device_id, + auth_method=config.auth_method, + pk=device_pk, + sk=device_sk, + edge_enabled=True, + ) + # create device identity + device_result: Device = self.service_sdk.devices.create_or_update_identity( + id=device_id, device=assembled_device + ) + + # write all device bundle content + if device_cert_output_directory: + if hub_cert_auth: + # hub auth cert + write_content_to_file( + content=device_hub_cert["certificate"], + destination=device_cert_output_directory, + file_name=f"{device_id}.hub-auth-cert.pem", + overwrite=True + ) + # hub auth key + write_content_to_file( + content=device_hub_cert["privateKey"], + destination=device_cert_output_directory, + file_name=f"{device_id}.hub-auth-key.pem", + overwrite=True + ) + else: + device_keys = device_result.authentication.symmetric_key + device_pk = device_keys.primary_key if device_keys else None + + # edge device config + create_edge_device_config( + device_id=device_id, + hub_hostname=self.target["entity"], + auth_method=config.auth_method, + default_edge_agent=config.default_edge_agent, + device_config=device_config_dict[device_id], + device_config_path=config.template_config_path, + device_pk=device_pk, + output_path=device_cert_output_directory, + ) + # root cert + write_content_to_file( + content=config.root_cert["certificate"], + destination=device_cert_output_directory, + file_name=root_cert_name, + ) + # full-chain cert + make_cert_chain( + certs=[ + signed_device_cert["certificate"], + config.root_cert["certificate"], + ], + output_dir=device_cert_output_directory, + output_file=f"{device_id}.full-chain.cert.pem", + ) + # write install script + write_content_to_file( + content=create_edge_device_config_script( + device_id=device_id, + hub_auth=hub_cert_auth, + hostname=device.hostname, + has_parent=(device.parent_id is not None), + parent_hostname=device.parent_hostname, + ), + destination=device_cert_output_directory, + file_name="install.sh", + overwrite=True, + ) + # write device readme + write_content_to_file( + content=DEVICE_README, + destination=device_cert_output_directory, + file_name="README.md", + overwrite=True, + ) + # create archive + tar_directory( + target_directory=device_cert_output_directory, + tarfile_path=bundle_output_directory, + tarfile_name=device_id, + overwrite=True, + ) + # delete uncompressed files + rmtree(device_cert_output_directory) + + # Get all device ids and scopes (inconsistent timing, hence sleep) + scope_retries = 0 + query_args = ["SELECT deviceId, deviceScope FROM devices"] + query_method = self.service_sdk.query.get_twins + all_hub_devices = _execute_query(query_args, query_method) + + # Ensure we retrieve all device scopes + while len(all_hub_devices) < len(config.devices) and scope_retries < MAX_DEVICE_SCOPE_RETRIES: + sleep(3) + scope_retries += 1 + logger.info("Retrying device scope query - attempt {} of {}" + .format(scope_retries, MAX_DEVICE_SCOPE_RETRIES)) + all_hub_devices = _execute_query(query_args, query_method) + + if len(all_hub_devices) < len(config.devices): + raise AzureResponseError( + "An error occurred - Failed to fetch device scopes for all devices after {} retries" + .format(scope_retries) + ) + + # set all device scopes + scope_dict: Dict[str, str] = {} + for device in all_hub_devices: + id = device["deviceId"] + if device_config_dict.get(id, None): + scope_dict[id] = device["deviceScope"] + + # Set parent / child relationships + device_to_parent_iterator = ( + tqdm(device_to_parent_dict, desc="Setting device parents") + if visualize + else device_to_parent_dict + ) + for device_id in device_to_parent_iterator: + # get device properties + device = self.service_sdk.devices.get_identity(id=device_id) + parent_id = device_to_parent_dict[device_id] + parent_scope = scope_dict[parent_id] + # set new parent scope + device.parent_scopes = [parent_scope] + # update device + self.service_sdk.devices.create_or_update_identity( + id=device_id, device=device, if_match="*" + ) + + # update edge config / set-modules + devices_config_iterator = ( + tqdm(config.devices, desc="Setting edge module content") + if visualize + else config.devices + ) + for device_config in devices_config_iterator: + device_id = device_config.device_id + deployment_content = device_config.deployment + if deployment_content: + self.service_sdk.configuration.apply_on_edge_device( + id=device_id, content=deployment_content + ) + + # Print device bundle details after other visuals + if bundle_output_directory: + num_bundles = len(config.devices) + bundle_plural = '' if num_bundles == 1 else 's' + print(f"{num_bundles} device bundle{bundle_plural} created in folder: {abspath(bundle_output_directory)}") + + def delete_device_identities(self, device_ids: List[str]): + for id in device_ids: + try: + self.service_sdk.devices.delete_identity(id=id, if_match="*") + except Exception as err: + raise AzureResponseError(err) diff --git a/azext_iot/iothub/providers/helpers/__init__.py b/azext_iot/iothub/providers/helpers/__init__.py new file mode 100644 index 000000000..55614acbf --- /dev/null +++ b/azext_iot/iothub/providers/helpers/__init__.py @@ -0,0 +1,5 @@ +# coding=utf-8 +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- diff --git a/azext_iot/iothub/providers/helpers/edge_device_config.py b/azext_iot/iothub/providers/helpers/edge_device_config.py new file mode 100644 index 000000000..9f3d9d529 --- /dev/null +++ b/azext_iot/iothub/providers/helpers/edge_device_config.py @@ -0,0 +1,583 @@ +# coding=utf-8 +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- +"""This module defines common values and functions for processing edge device configurations""" + +from pathlib import PurePath +from os import getcwd +from typing import Optional, List, Dict, Any +from azext_iot.common.fileops import write_content_to_file +from azext_iot.common.certops import create_self_signed_certificate, load_ca_cert_info +from azext_iot.common.shared import ( + ConfigType, + DeviceAuthType, +) +from azext_iot.iothub.common import ( + EdgeDevicesConfig, + EdgeDeviceConfig, + EdgeContainerAuth, +) +from azext_iot.common.utility import process_json_arg, process_toml_arg, assemble_nargs_to_dict + +from azure.cli.core.azclierror import ( + CLIInternalError, + FileOperationError, + InvalidArgumentValueError, + RequiredArgumentMissingError, +) + +from azext_iot.operations.hub import _process_config_content +from azext_iot.sdk.iothub.service.models import ConfigurationContent +from knack.log import get_logger + +logger = get_logger(__name__) + +MAX_DEVICE_SCOPE_RETRIES = 5 + +DEVICE_CONFIG_SCHEMA_VALID_VERSIONS: Dict[str, Any] = {} + +DEVICE_CONFIG_SCHEMA_VALID_VERSIONS["1.0"] = { + "type": "object", + "required": ["configVersion", "iotHub", "edgeConfiguration", "edgeDevices"], + "properties": { + "configVersion": {"type": "string"}, + "iotHub": { + "type": "object", + "required": ["authenticationMethod"], + "properties": { + "authenticationMethod": { + "type": "string", + "enum": ["symmetricKey", "x509Certificate"] + }, + } + }, + "certificates": { + "type": "object", + "required": ["rootCACertPath", "rootCACertKeyPath"], + "properties": { + "rootCACertPath": {"type": "string"}, + "rootCACertKeyPath": {"type": "string"} + } + }, + "edgeConfiguration": { + "type": "object", + "properties": { + "templateConfigPath": {"type": "string"}, + "defaultEdgeAgent": {"type": "string"} + }, + "required": ["defaultEdgeAgent"] + }, + "edgeDevices": { + "type": "array", + "items": {"$ref": "#/$defs/edgeDevice"} + }, + }, + "$defs": { + "edgeDevice": { + "type": "object", + "properties": { + "deviceId": {"type": "string"}, + "hostname": {"type": "string"}, + "edgeAgent": {"type": "string"}, + "deployment": {"type": "string"}, + "containerAuth": { + "type": "object", + "properties": { + "serverAddress": {"type": "string"}, + "username": {"type": "string"}, + "password": {"type": "string"} + } + }, + "children": { + "type": "array", + "items": {"$ref": "#/$defs/edgeDevice"}, + "minItems": 1, + } + }, + "required": ["deviceId"] + } + } +} + +# Edge device TOML default values +DEVICE_CONFIG_TOML = { + "hostname": "", + "provisioning": { + "device_id": "", + "iothub_hostname": "", + "source": "manual", + "authentication": { + "device_id_pk": "", + "method": "sas", + "trust_bundle_cert": "", + }, + }, + "edge_ca": {"cert": "file:///", "pk": "file:///"}, + "agent": {"config": {"image": ""}, "name": "edgeAgent", "type": "docker"}, + "connect": { + "management_uri": "unix:///var/run/iotedge/mgmt.sock", + "workload_uri": "unix:///var/run/iotedge/workload.sock", + }, + "listen": { + "management_uri": "fd://aziot-edged.mgmt.socket", + "workload_uri": "fd://aziot-edged.workload.socket", + }, + "moby_runtime": {"network": "azure-iot-edge", "uri": "unix:///var/run/docker.sock"}, +} + +EDGE_ROOT_CERTIFICATE_FILENAME = "iotedge_config_cli_root.pem" + +EDGE_CONFIG_SCRIPT_HEADERS = """ +# This script will attempt to configure a pre-installed iotedge as a nested node. +# It must be run as sudo, and will modify the ca + +device_id="{}" +cp config.toml /etc/aziot/config.toml +""" +EDGE_CONFIG_SCRIPT_HOSTNAME = """ +# ======================= Set Hostname ======================================= + +read -p "Enter the hostname to use: " hostname +if [ -z "$hostname" ] +then + echo "Invalid hostname $hostname" + exit 1 +fi + +sed -i "s/{{HOSTNAME}}/$hostname/" /etc/aziot/config.toml +""" +EDGE_CONFIG_SCRIPT_PARENT_HOSTNAME = """ +# ======================= Set Parent Hostname ======================================= + +read -p "Enter the parent hostname to use: " parent_hostname +if [ -z "$parent_hostname" ] +then + echo "Invalid parent hostname $parent_hostname" + exit 1 +fi + +sed -i "s/{{PARENT_HOSTNAME}}/$parent_hostname/" /etc/aziot/config.toml +""" +EDGE_CONFIG_SCRIPT_CA_CERTS = f""" +# ======================= Install nested root CA ======================================= +if [ -f /etc/os-release ] +then + . /etc/os-release + if [[ "$NAME" == "Common Base Linux Mariner"* ]]; + then + cp {EDGE_ROOT_CERTIFICATE_FILENAME} /etc/pki/ca-trust/source/anchors/{EDGE_ROOT_CERTIFICATE_FILENAME}.crt + update-ca-trust + else + cp {EDGE_ROOT_CERTIFICATE_FILENAME} /usr/local/share/ca-certificates/{EDGE_ROOT_CERTIFICATE_FILENAME}.crt + update-ca-certificates + fi +else + cp {EDGE_ROOT_CERTIFICATE_FILENAME} /usr/local/share/ca-certificates/{EDGE_ROOT_CERTIFICATE_FILENAME}.crt + update-ca-certificates +fi + +systemctl restart docker + +# ======================= Copy device certs ======================================= +cert_dir="/etc/aziot/certificates" +mkdir -p $cert_dir +cp "{EDGE_ROOT_CERTIFICATE_FILENAME}" "$cert_dir/{EDGE_ROOT_CERTIFICATE_FILENAME}" +cp "$device_id.full-chain.cert.pem" "$cert_dir/$device_id.full-chain.cert.pem" +cp "$device_id.key.pem" "$cert_dir/$device_id.key.pem" +""" +EDGE_CONFIG_SCRIPT_HUB_AUTH_CERTS = """ +# ======================= Copy hub auth certs ======================================= +cert_dir="/etc/aziot/certificates" +mkdir -p $cert_dir +cp "$device_id.hub-auth-cert.pem" "$cert_dir/$device_id.hub-auth-cert.pem" +cp "$device_id.hub-auth-key.pem" "$cert_dir/$device_id.hub-auth-key.pem" +""" +EDGE_CONFIG_SCRIPT_APPLY = """ +# ======================= Read User Input ======================================= +iotedge config apply -c /etc/aziot/config.toml + +echo "To check the edge runtime status, run 'iotedge system status'. To validate the configuration, run 'sudo iotedge check'" +""" + +EDGE_SUPPORTED_OS_LINK = "https://aka.ms/iotedge-supported-systems" +EDGE_LINUX_TUTORIAL_LINK = "https://aka.ms/iotedge-provision-linux-device" +EDGE_WINDOWS_TUTORIAL_LINK = "https://aka.ms/iotedge-provision-windows" + +DEVICE_README = f""" +# Prerequisites +Each device must have IoT Edge (must be v1.2 or later) installed. +Pick a [supported OS]({EDGE_SUPPORTED_OS_LINK}) and follow the corresponding tutorial to install Azure IoT Edge: + - [Linux on Windows]({EDGE_WINDOWS_TUTORIAL_LINK}) + - [Linux]({EDGE_LINUX_TUTORIAL_LINK}) + +# Steps + +1. Copy the bundle for each created device (device_id.tgz) onto the device. +2. Extract the bundle file by running following command: + +```Extract + tar zxvf ~//[[device-id]].tgz +``` + +3. Run the install script: + +```Run + sudo bash ./install.sh +``` + +4. If hostnames were not provided in the configuration file, the script will prompt for hostnames. + - Follow the prompt by entering the device and/or parent hostname (FQDN or IP address). + - On the parent device, it will prompt for its own hostname. + - On a child device, it may prompt the hostname of both the child and parent devices. + +""" + + +EDGE_ROOT_CERTIFICATE_SUBJECT = "Azure_IoT_CLI_Extension_Cert" + + +def create_edge_device_config( + hub_hostname: str, + device_id: str, + auth_method: DeviceAuthType, + device_config: EdgeDeviceConfig, + default_edge_agent: str, + device_config_path: Optional[str] = None, + device_pk: Optional[str] = None, + output_path: Optional[str] = None, +): + # load default device TOML object or custom path + device_toml = ( + process_toml_arg(device_config_path) + if device_config_path + else DEVICE_CONFIG_TOML + ) + + device_toml[ + "trust_bundle_cert" + ] = f"file:///etc/aziot/certificates/{EDGE_ROOT_CERTIFICATE_FILENAME}" + # Dynamic, AlwaysOnStartup, OnErrorOnly + device_toml["auto_reprovisioning_mode"] = "Dynamic" + device_toml["hostname"] = ( + device_config.hostname if device_config.hostname else "{{HOSTNAME}}" + ) + if device_config.parent_id: + device_toml["parent_hostname"] = ( + device_config.parent_hostname + if device_config.parent_hostname + else "{{PARENT_HOSTNAME}}" + ) + device_toml["provisioning"] = { + "device_id": device_id, + "iothub_hostname": hub_hostname, + "source": "manual", + "authentication": {"device_id_pk": {"value": device_pk}, "method": "sas"} + if auth_method == DeviceAuthType.shared_private_key.value + else { + "method": "x509", + "identity_cert": f"file:///etc/aziot/certificates/{device_id}.hub-auth-cert.pem", + "identity_pk": f"file:///etc/aziot/certificates/{device_id}.hub-auth-key.pem", + }, + } + device_toml["edge_ca"] = { + "cert": f"file:///etc/aziot/certificates/{device_id}.full-chain.cert.pem", + "pk": f"file:///etc/aziot/certificates/{device_id}.key.pem", + } + device_toml["agent"]["config"] = { + "image": device_config.edge_agent or default_edge_agent or '', + "auth": { + "serveraddress": device_config.container_auth.serveraddress, + "username": device_config.container_auth.username, + "password": device_config.container_auth.password, + } + if device_config.container_auth + else {}, + } + if output_path: + import tomli_w + write_content_to_file( + tomli_w.dumps(device_toml), + output_path, + "config.toml", + overwrite=True, + ) + return device_toml + + +def process_edge_devices_config_file_content( + content: dict, + config_path: Optional[str] = None, + override_auth_type: Optional[str] = None, + override_root_cert_path: Optional[str] = None, + override_root_key_path: Optional[str] = None, + override_root_password: Optional[str] = None, + override_default_edge_agent: Optional[str] = None, + override_device_config_template: Optional[str] = None, +) -> EdgeDevicesConfig: + """ + Process edge config file schema dictionary + """ + + # Use current directory if no config file path + config_path = config_path or getcwd() + + # Warn about override values + for value, name in [ + (override_auth_type, "Authentication Type"), + (override_root_cert_path, "Root certificate"), + (override_root_key_path, "Root certificate key"), + (override_default_edge_agent, "Default edge agent"), + (override_device_config_template, "Device config template"), + ]: + if value: + logger.info( + f"Overriding configuration file property `{name}` " + f"with command argument value: `{value}`" + ) + + version = content.get("configVersion", None) + if not version: + raise InvalidArgumentValueError("'configVersion' property missing from device configuration file.") + from jsonschema import validate + from jsonschema.exceptions import ValidationError + try: + validate(content, DEVICE_CONFIG_SCHEMA_VALID_VERSIONS[version]) + except ValidationError as err: + raise InvalidArgumentValueError(f"Invalid devices config file schema:\n{err.message}") + + hub_config = content.get("iotHub", {}) + devices_config = content.get("edgeDevices", []) + + # edge root CA + root_cert = None + certificates = content.get("certificates", None) + if certificates or any( + [override_root_cert_path, override_root_key_path, override_root_password] + ): + root_ca_cert = override_root_cert_path or certificates.get( + "rootCACertPath", None + ) + root_ca_key = override_root_key_path or certificates.get( + "rootCACertKeyPath", None + ) + if not all([root_ca_cert, root_ca_key]): + raise InvalidArgumentValueError( + "Please check your config file to ensure values are provided " + "for both `rootCACertPath` and `rootCACertKeyPath`." + ) + root_cert = load_ca_cert_info( + root_ca_cert, root_ca_key, password=override_root_password + ) + else: + root_cert = create_self_signed_certificate( + subject=EDGE_ROOT_CERTIFICATE_SUBJECT, + key_size=4096, + sha_version=256, + v3_extensions=True + ) + + # device auth + # default to symmetric key + device_authentication_method = DeviceAuthType.shared_private_key.value + auth_value = hub_config.get("authenticationMethod", None) + if override_auth_type: + device_authentication_method = override_auth_type + else: + device_authentication_method = ( + DeviceAuthType.x509_thumbprint.value + if auth_value == "x509Certificate" + else DeviceAuthType.shared_private_key.value + ) + + # edge config + edge_config = content.get("edgeConfiguration", None) + if edge_config or any( + [override_default_edge_agent, override_device_config_template] + ): + # do not use path relative to config file if overridden from CLI context + if override_device_config_template: + template_config_path = override_device_config_template + else: + template_config_path = edge_config.get("templateConfigPath", None) + if template_config_path: # relative path to config file to device.toml + template_config_path = PurePath(config_path, template_config_path).as_posix() + + default_edge_agent = override_default_edge_agent or edge_config.get( + "defaultEdgeAgent", None + ) + all_devices = [] + + def _process_edge_config_device(device: dict, parent_id=None, parent_hostname=None): + device_id = device.get("deviceId", None) + if not device_id: + raise InvalidArgumentValueError( + "A device parameter is missing required attribute 'device_id'" + ) + deployment = device.get("deployment", None) + if deployment: + # relative path from config file to deployment.json + deployment = PurePath(config_path, deployment).as_posix() + deployment = try_parse_valid_deployment_config(deployment) + + child_devices = device.get("children", []) + container_auth = device.get("containerAuth", {}) + hostname = device.get("hostname", None) + edge_agent = device.get("edgeAgent", None) + device_config = EdgeDeviceConfig( + device_id=device_id, + deployment=deployment, + parent_id=parent_id, + parent_hostname=parent_hostname, + container_auth=EdgeContainerAuth( + serveraddress=container_auth.get("serverAddress", None), + username=container_auth.get("username", None), + password=container_auth.get("password", None), + ) + if container_auth + else None, + hostname=hostname, + edge_agent=edge_agent, + ) + all_devices.append(device_config) + for child_device in child_devices: + _process_edge_config_device( + child_device, parent_id=device_id, parent_hostname=hostname + ) + + for device in devices_config: + _process_edge_config_device(device) + return EdgeDevicesConfig( + version=version, + auth_method=device_authentication_method, + root_cert=root_cert, + devices=all_devices, + template_config_path=template_config_path, + default_edge_agent=default_edge_agent, + ) + + +def create_edge_device_config_script( + device_id: str, + hub_auth: bool = False, + hostname: Optional[str] = None, + has_parent: bool = False, + parent_hostname: Optional[str] = None, +): + return "\n".join( + [EDGE_CONFIG_SCRIPT_HEADERS.format(device_id)] + + ([EDGE_CONFIG_SCRIPT_HOSTNAME] if not hostname else []) + + ( + [EDGE_CONFIG_SCRIPT_PARENT_HOSTNAME] + if (has_parent and not parent_hostname) + else [] + ) + + [EDGE_CONFIG_SCRIPT_CA_CERTS] + + ([EDGE_CONFIG_SCRIPT_HUB_AUTH_CERTS] if hub_auth else []) + + [EDGE_CONFIG_SCRIPT_APPLY] + ) + + +def try_parse_valid_deployment_config(deployment_path: str): + try: + deployment_content = process_json_arg( + deployment_path, argument_name="deployment" + ) + processed_content = _process_config_content( + deployment_content, config_type=ConfigType.edge + ) + return ConfigurationContent(**processed_content) + except CLIInternalError: + raise FileOperationError( + f"Please ensure a deployment file exists at path: '{deployment_path}'" + ) + except Exception as ex: + logger.warning(f"Error processing config file at '{deployment_path}'") + raise InvalidArgumentValueError(ex) + + +def process_edge_devices_config_args( + device_args: List[List[str]], + auth_type: str, + default_edge_agent: Optional[str] = None, + device_config_template: Optional[str] = None, + root_cert_path: Optional[str] = None, + root_key_path: Optional[str] = None, + root_cert_password: Optional[str] = None, +) -> EdgeDevicesConfig: + # raise error if only key or cert provided + if (root_cert_path is not None) ^ (root_key_path is not None): + raise RequiredArgumentMissingError( + "You must provide a path to both the root cert public and private keys." + ) + # create cert if one isn't provided + root_cert = ( + load_ca_cert_info(root_cert_path, root_key_path, root_cert_password) + if all([root_cert_path, root_key_path]) + else create_self_signed_certificate( + subject=EDGE_ROOT_CERTIFICATE_SUBJECT, + key_size=4096, + sha_version=256, + v3_extensions=True + ) + ) + + config = EdgeDevicesConfig( + version="1.0", + auth_method=(auth_type or DeviceAuthType.shared_private_key.value), + default_edge_agent=default_edge_agent, + template_config_path=device_config_template, + devices=[], + root_cert=root_cert, + ) + # Process --device arguments + all_devices: Dict[str, Dict[str, str]] = {} + for device_input in device_args: + # assemble device params from nArgs strings + device_dict = assemble_nargs_to_dict(device_input) + device_id = device_dict.get("id", None) + if not device_id: + raise InvalidArgumentValueError( + "A device argument is missing required parameter 'id'" + ) + if all_devices.get(device_id, None): + raise InvalidArgumentValueError( + f"Duplicate deviceId '{device_id}' detected" + ) + all_devices[device_id] = device_dict + + for device_id in all_devices: + device_dict = all_devices[device_id] + deployment = device_dict.get("deployment", None) + if deployment: + deployment = try_parse_valid_deployment_config(deployment) + parent_id = device_dict.get("parent", None) + parent_hostname = None + if parent_id: + parent = all_devices.get(parent_id, {}) + parent_hostname = parent.get("hostname", None) + hostname = device_dict.get("hostname", None) + edge_agent = device_dict.get("edge_agent", None) + container_auth_arg = device_dict.get("container_auth", "{}") + container_auth_obj = process_json_arg(container_auth_arg) + container_auth = ( + EdgeContainerAuth( + serveraddress=container_auth_obj.get("serverAddress", None), + username=container_auth_obj.get("username", None), + password=container_auth_obj.get("password", None), + ) + if container_auth_obj + else None + ) + device_config = EdgeDeviceConfig( + device_id=device_id, + deployment=deployment, + parent_id=parent_id, + hostname=hostname, + parent_hostname=parent_hostname, + edge_agent=edge_agent, + container_auth=container_auth, + ) + config.devices.append(device_config) + return config diff --git a/azext_iot/operations/hub.py b/azext_iot/operations/hub.py index 48f788a69..ec505136f 100644 --- a/azext_iot/operations/hub.py +++ b/azext_iot/operations/hub.py @@ -279,7 +279,7 @@ def _assemble_auth(auth_method, pk, sk): def _create_self_signed_cert(subject, valid_days, output_path=None): from azext_iot.common.certops import create_self_signed_certificate - return create_self_signed_certificate(subject, valid_days, output_path) + return create_self_signed_certificate(subject=subject, valid_days=valid_days, cert_output_dir=output_path) def update_iot_device_custom( diff --git a/azext_iot/tests/conftest.py b/azext_iot/tests/conftest.py index f8394f676..178d490ca 100644 --- a/azext_iot/tests/conftest.py +++ b/azext_iot/tests/conftest.py @@ -35,10 +35,14 @@ "azext_iot.operations.hub._iot_hub_monitor_events" ) path_iot_device_show = "azext_iot.operations.hub._iot_device_show" -path_device_messaging_iot_device_show = "azext_iot.iothub.providers.device_messaging._iot_device_show" +path_device_messaging_iot_device_show = ( + "azext_iot.iothub.providers.device_messaging._iot_device_show" +) path_update_device_twin = "azext_iot.operations.hub._iot_device_twin_update" hub_entity = "myhub.azure-devices.net" -path_iot_service_provisioning_factory = "azext_iot._factory.iot_service_provisioning_factory" +path_iot_service_provisioning_factory = ( + "azext_iot._factory.iot_service_provisioning_factory" +) path_gdcs = "azext_iot.dps.providers.discovery.DPSDiscovery.get_target" path_discovery_dps_init = ( "azext_iot.dps.providers.discovery.DPSDiscovery._initialize_client" @@ -60,19 +64,16 @@ # Mock Iot DPS Target mock_dps_target = {} -mock_dps_target['cs'] = 'HostName=mydps;SharedAccessKeyName=name;SharedAccessKey=value' -mock_dps_target['entity'] = 'mydps' -mock_dps_target['primarykey'] = 'rJx/6rJ6rmG4ak890+eW5MYGH+A0uzRvjGNjg3Ve8sfo=' -mock_dps_target['secondarykey'] = 'aCd/6rJ6rmG4ak890+eW5MYGH+A0uzRvjGNjg3Ve8sfo=' -mock_dps_target['policy'] = 'provisioningserviceowner' -mock_dps_target['subscription'] = "5952cff8-bcd1-4235-9554-af2c0348bf23" +mock_dps_target["cs"] = "HostName=mydps;SharedAccessKeyName=name;SharedAccessKey=value" +mock_dps_target["entity"] = "mydps" +mock_dps_target["primarykey"] = "rJx/6rJ6rmG4ak890+eW5MYGH+A0uzRvjGNjg3Ve8sfo=" +mock_dps_target["secondarykey"] = "aCd/6rJ6rmG4ak890+eW5MYGH+A0uzRvjGNjg3Ve8sfo=" +mock_dps_target["policy"] = "provisioningserviceowner" +mock_dps_target["subscription"] = "5952cff8-bcd1-4235-9554-af2c0348bf23" mock_symmetric_key_attestation = { "type": "symmetricKey", - "symmetricKey": { - "primaryKey": "primary_key", - "secondaryKey": "secondary_key" - }, + "symmetricKey": {"primaryKey": "primary_key", "secondaryKey": "secondary_key"}, } generic_cs_template = "HostName={};SharedAccessKeyName={};SharedAccessKey={}" @@ -428,17 +429,30 @@ def fixture_gdcs(mocker): @pytest.fixture() def fixture_dps_sas(mocker): - r = SasTokenAuthentication(mock_dps_target['entity'], - mock_dps_target['policy'], - mock_dps_target['primarykey']) + r = SasTokenAuthentication( + mock_dps_target["entity"], + mock_dps_target["policy"], + mock_dps_target["primarykey"], + ) sas = mocker.patch(path_sas) sas.return_value = r @pytest.fixture def patch_certificate_open(mocker): + patch = mocker.patch("azext_iot.operations.dps.open_certificate") + patch.return_value = "" + return patch + + +@pytest.fixture +def patch_create_edge_root_cert(mocker): patch = mocker.patch( - "azext_iot.operations.dps.open_certificate" + "azext_iot.iothub.providers.helpers.edge_device_config.create_self_signed_certificate", ) - patch.return_value = "" + patch.return_value = { + "certificate": "root_certificate", + "privateKey": "root_private_key", + "thumbprint": "root_thumbprint", + } return patch diff --git a/azext_iot/tests/iothub/devices/device_configs/deployments/deploymentLowerLayer.json b/azext_iot/tests/iothub/devices/device_configs/deployments/deploymentLowerLayer.json new file mode 100644 index 000000000..39081d4ce --- /dev/null +++ b/azext_iot/tests/iothub/devices/device_configs/deployments/deploymentLowerLayer.json @@ -0,0 +1,57 @@ +{ + "modulesContent": { + "$edgeAgent": { + "properties.desired": { + "modules": { + "simulatedTemperatureSensor": { + "settings": { + "image": "$upstream:443/azureiotedge-simulated-temperature-sensor:1.0", + "createOptions": "" + }, + "type": "docker", + "status": "running", + "restartPolicy": "always", + "version": "1.0" + } + }, + "runtime": { + "settings": { + "minDockerVersion": "v1.25" + }, + "type": "docker" + }, + "schemaVersion": "1.1", + "systemModules": { + "edgeAgent": { + "settings": { + "image": "$upstream:443/azureiotedge-agent:1.2", + "createOptions": "" + }, + "type": "docker" + }, + "edgeHub": { + "settings": { + "image": "$upstream:443/azureiotedge-hub:1.2", + "createOptions": "{\"HostConfig\":{\"PortBindings\":{\"443/tcp\":[{\"HostPort\":\"443\"}],\"5671/tcp\":[{\"HostPort\":\"5671\"}],\"8883/tcp\":[{\"HostPort\":\"8883\"}]}}}" + }, + "type": "docker", + "env": {}, + "status": "running", + "restartPolicy": "always" + } + } + } + }, + "$edgeHub": { + "properties.desired": { + "routes": { + "route": "FROM /messages/* INTO $upstream" + }, + "schemaVersion": "1.1", + "storeAndForwardConfiguration": { + "timeToLiveSecs": 7200 + } + } + } + } +} \ No newline at end of file diff --git a/azext_iot/tests/iothub/devices/device_configs/deployments/deploymentTopLayer.json b/azext_iot/tests/iothub/devices/device_configs/deployments/deploymentTopLayer.json new file mode 100644 index 000000000..f1315031e --- /dev/null +++ b/azext_iot/tests/iothub/devices/device_configs/deployments/deploymentTopLayer.json @@ -0,0 +1,82 @@ +{ + "modulesContent": { + "$edgeAgent": { + "properties.desired": { + "modules": { + "registry": { + "settings": { + "image": "registry:latest" + }, + "type": "docker", + "version": "1.0", + "env": { + "REGISTRY_PROXY_REMOTEURL": { + "value": "https://mcr.microsoft.com" + } + }, + "status": "running", + "restartPolicy": "always" + }, + "IoTEdgeAPIProxy": { + "settings": { + "image": "mcr.microsoft.com/azureiotedge-api-proxy:1.0", + "createOptions": "{\"HostConfig\": {\"PortBindings\": {\"443/tcp\": [{\"HostPort\":\"443\"}]}}}" + }, + "type": "docker", + "env": { + "NGINX_DEFAULT_PORT": { + "value": "443" + }, + "DOCKER_REQUEST_ROUTE_ADDRESS": { + "value": "registry:5000" + }, + "BLOB_UPLOAD_ROUTE_ADDRESS": { + "value": "AzureBlobStorageonIoTEdge:11002" + } + }, + "status": "running", + "restartPolicy": "always", + "version": "1.0" + } + }, + "runtime": { + "settings": { + "minDockerVersion": "v1.25" + }, + "type": "docker" + }, + "schemaVersion": "1.1", + "systemModules": { + "edgeAgent": { + "settings": { + "image": "mcr.microsoft.com/azureiotedge-agent:1.2", + "createOptions": "" + }, + "type": "docker" + }, + "edgeHub": { + "settings": { + "image": "mcr.microsoft.com/azureiotedge-hub:1.2", + "createOptions": "{\"HostConfig\":{\"PortBindings\":{\"5671/tcp\":[{\"HostPort\":\"5671\"}],\"8883/tcp\":[{\"HostPort\":\"8883\"}]}}}" + }, + "type": "docker", + "env": {}, + "status": "running", + "restartPolicy": "always" + } + } + } + }, + "$edgeHub": { + "properties.desired": { + "routes": { + "route": "FROM /messages/* INTO $upstream" + }, + "schemaVersion": "1.1", + "storeAndForwardConfiguration": { + "timeToLiveSecs": 7200 + } + } + } + } +} \ No newline at end of file diff --git a/azext_iot/tests/iothub/devices/device_configs/device_config.toml b/azext_iot/tests/iothub/devices/device_configs/device_config.toml new file mode 100644 index 000000000..ad67077d6 --- /dev/null +++ b/azext_iot/tests/iothub/devices/device_configs/device_config.toml @@ -0,0 +1,45 @@ +## The following values will automatically be filled in when installing the configuration bundles generated by the iotedge_config +## If the values are included, they will be overwritten. + +hostname = "" +parent_hostname = "" + +[provisioning] +device_id = "" +iothub_hostname = "" +source = "manual" + +[provisioning.authentication] +device_id_pk = {value = ""} +method = "sas" + +trust_bundle_cert = "" + +[edge_ca] +cert = "file:///" +pk = "file:///" + +[agent.config] +image = "" + +## ==================================================================================================================== + +## Add other parameters that you want to be included in the configuration bundles here +[agent] +name = "edgeAgent" +type = "docker" + +[connect] +management_uri = "unix:///var/run/iotedge/mgmt.sock" +workload_uri = "unix:///var/run/iotedge/workload.sock" + +[listen] +management_uri = "fd://aziot-edged.mgmt.socket" +workload_uri = "fd://aziot-edged.workload.socket" + +[moby_runtime] +network = "azure-iot-edge" +uri = "unix:///var/run/docker.sock" + +[test] +foo = "bar" \ No newline at end of file diff --git a/azext_iot/tests/iothub/devices/device_configs/edge_devices_min_config.yml b/azext_iot/tests/iothub/devices/device_configs/edge_devices_min_config.yml new file mode 100644 index 000000000..433ebbc1e --- /dev/null +++ b/azext_iot/tests/iothub/devices/device_configs/edge_devices_min_config.yml @@ -0,0 +1,15 @@ +configVersion: "1.0" +iotHub: + authenticationMethod: x509Certificate + +edgeConfiguration: + defaultEdgeAgent: "$upstream:443/azureiotedge-agent:1.1" + +edgeDevices: + - deviceId: device_1 + edgeAgent: "mcr.microsoft.com/azureiotedge-agent:1.2" + children: + - deviceId: device_2 + - deviceId: device_3 + edgeAgent: "mcr.microsoft.com/azureiotedge-agent:1.4" + - deviceId: device_4 \ No newline at end of file diff --git a/azext_iot/tests/iothub/devices/device_configs/fake_edge_container_auth.json b/azext_iot/tests/iothub/devices/device_configs/fake_edge_container_auth.json new file mode 100644 index 000000000..e2c818d4d --- /dev/null +++ b/azext_iot/tests/iothub/devices/device_configs/fake_edge_container_auth.json @@ -0,0 +1,5 @@ +{ + "serverAddress": "servername", + "username": "username", + "password": "$credential$" +} \ No newline at end of file diff --git a/azext_iot/tests/iothub/devices/device_configs/invalid/duplicate_device_config.yml b/azext_iot/tests/iothub/devices/device_configs/invalid/duplicate_device_config.yml new file mode 100644 index 000000000..ce1152582 --- /dev/null +++ b/azext_iot/tests/iothub/devices/device_configs/invalid/duplicate_device_config.yml @@ -0,0 +1,11 @@ +configVersion: "1.0" +iotHub: + authenticationMethod: symmetricKey +edgeConfiguration: + templateConfigPath: "./device_configs/device_config.toml" + +edgeDevices: + - deviceId: device_1 + # duplicate deviceID + - deviceId: device_1 + - deviceId: device_2 \ No newline at end of file diff --git a/azext_iot/tests/iothub/devices/device_configs/invalid/invalid_deployment.json b/azext_iot/tests/iothub/devices/device_configs/invalid/invalid_deployment.json new file mode 100644 index 000000000..8a7968762 --- /dev/null +++ b/azext_iot/tests/iothub/devices/device_configs/invalid/invalid_deployment.json @@ -0,0 +1,3 @@ +{ + "foo": "bar" +} \ No newline at end of file diff --git a/azext_iot/tests/iothub/devices/device_configs/invalid/missing_deployment.yml b/azext_iot/tests/iothub/devices/device_configs/invalid/missing_deployment.yml new file mode 100644 index 000000000..32a280d8d --- /dev/null +++ b/azext_iot/tests/iothub/devices/device_configs/invalid/missing_deployment.yml @@ -0,0 +1,17 @@ +configVersion: "1.0" +iotHub: + authenticationMethod: symmetricKey +edgeConfiguration: + templateConfigPath: "./device_configs/device_config.toml" + +edgeDevices: + - deviceId: device_1 + edgeAgent: "mcr.microsoft.com/azureiotedge-agent:1.2" + deployment: "../deploymentTopLayer.json" + children: + - deviceId: device_2 + deployment: "../deploymentLowerLayer.json" + children: + - deviceId: device_3 + # invalid deployment / config does not exist + deployment: "./path/does/not/exist.json" \ No newline at end of file diff --git a/azext_iot/tests/iothub/devices/device_configs/invalid/missing_device_id.yml b/azext_iot/tests/iothub/devices/device_configs/invalid/missing_device_id.yml new file mode 100644 index 000000000..e0c2b4db0 --- /dev/null +++ b/azext_iot/tests/iothub/devices/device_configs/invalid/missing_device_id.yml @@ -0,0 +1,8 @@ +configVersion: "1.0" +iotHub: + authenticationMethod: symmetricKey +edgeConfiguration: + templateConfigPath: "./device_configs/device_config.toml" + +edgeDevices: + - deployment: './fake_deployment.json' \ No newline at end of file diff --git a/azext_iot/tests/iothub/devices/device_configs/nested_edge_config.json b/azext_iot/tests/iothub/devices/device_configs/nested_edge_config.json new file mode 100644 index 000000000..fdf6d47d1 --- /dev/null +++ b/azext_iot/tests/iothub/devices/device_configs/nested_edge_config.json @@ -0,0 +1,59 @@ +{ + "configVersion": "1.0", + "iotHub": { + "authenticationMethod": "symmetricKey" + }, + "edgeConfiguration": { + "templateConfigPath": "device_config.toml", + "defaultEdgeAgent": "$upstream:443/azureiotedge-agent:1.2" + }, + "edgeDevices": [ + { + "deviceId": "device_1", + "edgeAgent": "mcr.microsoft.com/azureiotedge-agent:1.1", + "hostname": "device_1", + "deployment": "./deployments/deploymentTopLayer.json", + "children": [ + { + "deviceId": "device_2", + "hostname": "device_2", + "deployment": "./deployments/deploymentLowerLayer.json", + "children": [ + { + "deviceId": "device_3", + "hostname": "device_3", + "deployment": "./deployments/deploymentLowerLayer.json" + } + ] + }, + { + "deviceId": "device_4", + "hostname": "device_4", + "deployment": "./deployments/deploymentTopLayer.json", + "children": [ + { + "deviceId": "device_5", + "deployment": "./deployments/deploymentLowerLayer.json", + "hostname": "device_5" + }, + { + "deviceId": "device_6", + "hostname": "device_6", + "deployment": "./deployments/deploymentLowerLayer.json" + } + ] + } + ] + }, + { + "deviceId": "device_7", + "hostname": "device_7", + "edgeAgent": "mcr.microsoft.com/azureiotedge-agent:1.2", + "containerAuth": { + "serverAddress": "mcr.microsoft.com", + "username": "test-user", + "password": "$credential$" + } + } + ] + } \ No newline at end of file diff --git a/azext_iot/tests/iothub/devices/device_configs/nested_edge_config.yml b/azext_iot/tests/iothub/devices/device_configs/nested_edge_config.yml new file mode 100644 index 000000000..c0f5e0d09 --- /dev/null +++ b/azext_iot/tests/iothub/devices/device_configs/nested_edge_config.yml @@ -0,0 +1,39 @@ +configVersion: "1.0" + +iotHub: + authenticationMethod: symmetricKey + +edgeConfiguration: + templateConfigPath: "device_config.toml" + defaultEdgeAgent: "$upstream:443/azureiotedge-agent:1.2" + +edgeDevices: + - deviceId: device_1 + edgeAgent: "mcr.microsoft.com/azureiotedge-agent:1.1" + hostname: device_1 + deployment: "./deployments/deploymentTopLayer.json" + children: + - deviceId: device_2 + hostname: device_2 + deployment: "./deployments/deploymentLowerLayer.json" + children: + - deviceId: device_3 + hostname: device_3 + deployment: "./deployments/deploymentLowerLayer.json" + - deviceId: device_4 + hostname: device_4 + deployment: "./deployments/deploymentTopLayer.json" + children: + - deviceId: device_5 + deployment: "./deployments/deploymentLowerLayer.json" + hostname: device_5 + - deviceId: device_6 + hostname: device_6 + deployment: "./deployments/deploymentLowerLayer.json" + - deviceId: device_7 + hostname: device_7 + edgeAgent: "mcr.microsoft.com/azureiotedge-agent:1.2" + containerAuth: + serverAddress: 'mcr.microsoft.com' + username: 'test-user' + password: '$credential$' \ No newline at end of file diff --git a/azext_iot/tests/iothub/devices/device_configs/nested_edge_config_secondary.yaml b/azext_iot/tests/iothub/devices/device_configs/nested_edge_config_secondary.yaml new file mode 100644 index 000000000..8eb74bae5 --- /dev/null +++ b/azext_iot/tests/iothub/devices/device_configs/nested_edge_config_secondary.yaml @@ -0,0 +1,18 @@ +configVersion: "1.0" +iotHub: + authenticationMethod: symmetricKey +edgeConfiguration: + defaultEdgeAgent: "mcr.microsoft.com/azureiotedge-agent:1.2" + templateConfigPath: "device_config.toml" + +edgeDevices: + - deviceId: device_100 + edgeAgent: "mcr.microsoft.com/azureiotedge-agent:1.2" + deployment: "./deployments/deploymentTopLayer.json" + children: + - deviceId: device_200 + children: + - deviceId: device_300 + children: + - deviceId: device_400 + deployment: "./deployments/deploymentLowerLayer.json" \ No newline at end of file diff --git a/azext_iot/tests/iothub/devices/test_iot_edge_devices_create_int.py b/azext_iot/tests/iothub/devices/test_iot_edge_devices_create_int.py new file mode 100644 index 000000000..449f00db3 --- /dev/null +++ b/azext_iot/tests/iothub/devices/test_iot_edge_devices_create_int.py @@ -0,0 +1,397 @@ +# coding=utf-8 +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +from typing import List, NamedTuple, Optional +import pytest +import tarfile +from shutil import rmtree +from os.path import exists +from azext_iot.tests.iothub import IoTLiveScenarioTest + + +class EdgeDevicesTestConfig(NamedTuple): + id: str + parent: str + deployment: str + hostname: Optional[str] + edge_agent: Optional[str] + + +@pytest.mark.usefixtures("set_cwd") +class TestNestedEdgeHierarchy(IoTLiveScenarioTest): + def __init__(self, test_case): + super(TestNestedEdgeHierarchy, self).__init__(test_case) + self.deployment_top = "./device_configs/deployments/deploymentTopLayer.json" + self.deployment_lower = "./device_configs/deployments/deploymentLowerLayer.json" + + self.deployment_top_config = "./deployments/deploymentTopLayer.json" + self.deployment_lower_config = "./deployments/deploymentLowerLayer.json" + + def test_nested_edge_devices_create_nArgs_full(self): + # ├── device_1 (toplayer) + # │ ├── device_2 + # │ │ └── device_3 (lowerLayer) + # │ └── device_4 + # │ ├── device_5 (lowerLayer) + # │ └── device_6 (lowerLayer) + # └── device_7 (toplayer) + + devices: List[EdgeDevicesTestConfig] = [ + EdgeDevicesTestConfig("device1", None, self.deployment_top, None, None), + EdgeDevicesTestConfig("device2", "device1", None, None, None), + EdgeDevicesTestConfig( + "device3", "device2", self.deployment_lower, None, None + ), + EdgeDevicesTestConfig("device4", "device1", None, None, None), + EdgeDevicesTestConfig( + "device5", "device4", self.deployment_lower, None, None + ), + EdgeDevicesTestConfig( + "device6", "device4", self.deployment_lower, None, None + ), + EdgeDevicesTestConfig("device7", None, self.deployment_top, None, None), + ] + + device_arg_string = self._generate_device_arg_string(devices) + self.cmd( + f"iot edge devices create -n {self.entity_name} -g {self.entity_rg} -c -y {device_arg_string} " + "--out bundles --device-auth x509_thumbprint" + ) + + self._validate_results(devices, "bundles", True) + + def test_nested_edge_devices_create_nArgs_partial(self): + # Partial 1 + # ├── device1 (toplayer) + # │ ├── device2 + # │ │ └── device3 (lowerLayer) + + # Partial 2 + # │── device4 (toplayer) + # │ ├── device5 (lowerLayer) + # │ └── device6 (lowerLayer) + + partial_devices_primary = [ + EdgeDevicesTestConfig("device1", None, self.deployment_top, None, None), + EdgeDevicesTestConfig("device2", "device1", None, None, None), + EdgeDevicesTestConfig( + "device3", "device2", self.deployment_lower, None, None + ), + ] + + partial_devices_secondary = [ + EdgeDevicesTestConfig("device4", None, self.deployment_top, None, None), + EdgeDevicesTestConfig( + "device5", "device4", self.deployment_lower, None, None + ), + EdgeDevicesTestConfig( + "device6", "device4", self.deployment_lower, None, None + ), + ] + + # Clean on first run + device_arg_string = self._generate_device_arg_string(partial_devices_primary) + self.cmd( + f"iot edge devices create -n {self.entity_name} -g {self.entity_rg} -c -y {device_arg_string} --out primary" + ) + + self._validate_results(partial_devices_primary, "primary") + + # Second run, no clean + device_arg_string = self._generate_device_arg_string(partial_devices_secondary) + self.cmd( + f"iot edge devices create -n {self.entity_name} -g {self.entity_rg} {device_arg_string}" + ) + # validate results on entire run + full_device_list = partial_devices_primary + partial_devices_secondary + self._validate_results(full_device_list, None) + + def test_nested_edge_devices_create_config_full(self): + # ├── device_1 (toplayer) + # │ ├── device_2 (lowerLayer) + # │ │ └── device_3 (lowerLayer) + # │ └── device_4 (toplayer) + # │ ├── device_5 (lowerLayer) + # │ └── device_6 (lowerLayer) + # └── device_7 + config_path = "./device_configs/nested_edge_config.yml" + devices: List[EdgeDevicesTestConfig] = [ + EdgeDevicesTestConfig( + "device_1", + None, + self.deployment_top_config, + "device_1", + "mcr.microsoft.com/azureiotedge-agent:1.1", + ), + EdgeDevicesTestConfig( + "device_2", "device_1", self.deployment_lower_config, "device_2", None + ), + EdgeDevicesTestConfig( + "device_3", "device_2", self.deployment_lower_config, "device_3", None + ), + EdgeDevicesTestConfig( + "device_4", "device_1", self.deployment_top_config, "device_4", None + ), + EdgeDevicesTestConfig( + "device_5", "device_4", self.deployment_lower_config, "device_5", None + ), + EdgeDevicesTestConfig( + "device_6", "device_4", self.deployment_lower_config, "device_6", None + ), + EdgeDevicesTestConfig( + "device_7", + None, + None, + "device_7", + "mcr.microsoft.com/azureiotedge-agent:1.2", + ), + ] + + self.cmd( + f"iot edge devices create -n {self.entity_name} -g {self.entity_rg} -c -y --cfg {config_path} " + "--out device_bundles --device-auth x509_thumbprint" + ) + + self._validate_results(devices, "device_bundles", True) + + def test_nested_edge_devices_create_config_partial(self): + # Partial 1 + # ├── device_1 (toplayer) + # │ ├── device_2 (lowerLayer) + # │ │ └── device_3 (lowerLayer) + # │ └── device_4 (toplayer) + # │ ├── device_5 (lowerLayer) + # │ └── device_6 (lowerLayer) + # └── device_7 + + # Partial 2 + # └── device_100 (toplayer) + # └── device_200 + # └── device_300 + # └── device_400 (lowerLayer) + + primary_config_path = "./device_configs/nested_edge_config.json" + secondary_config_path = "./device_configs/nested_edge_config_secondary.yaml" + devices_primary = [ + EdgeDevicesTestConfig( + "device_1", None, self.deployment_top_config, None, None + ), + EdgeDevicesTestConfig( + "device_2", "device_1", self.deployment_lower_config, None, None + ), + EdgeDevicesTestConfig( + "device_3", "device_2", self.deployment_lower_config, None, None + ), + EdgeDevicesTestConfig( + "device_4", "device_1", self.deployment_top_config, None, None + ), + EdgeDevicesTestConfig( + "device_5", "device_4", self.deployment_lower_config, None, None + ), + EdgeDevicesTestConfig( + "device_6", "device_4", self.deployment_lower_config, None, None + ), + EdgeDevicesTestConfig("device_7", None, None, None, None), + ] + devices_secondary = [ + EdgeDevicesTestConfig( + "device_100", None, self.deployment_top_config, None, None + ), + EdgeDevicesTestConfig("device_200", "device_100", None, None, None), + EdgeDevicesTestConfig("device_300", "device_200", None, None, None), + EdgeDevicesTestConfig( + "device_400", "device_300", self.deployment_lower_config, None, None + ), + ] + self.cmd( + f"iot edge devices create -n {self.entity_name} -g {self.entity_rg} -c -y --cfg {primary_config_path} --out output" + ) + + self._validate_results(devices_primary, "output") + + self.cmd( + f"iot edge devices create -n {self.entity_name} -g {self.entity_rg} --cfg {secondary_config_path}" + ) + + self._validate_results(devices_primary + devices_secondary, None) + + def test_edge_devices_nArgs_flat_no_output(self): + devices: List[EdgeDevicesTestConfig] = [ + EdgeDevicesTestConfig("device1", None, self.deployment_top, None, None), + EdgeDevicesTestConfig("device2", None, None, None, None), + EdgeDevicesTestConfig("device3", None, self.deployment_lower, None, None), + EdgeDevicesTestConfig("device4", None, None, None, None), + EdgeDevicesTestConfig("device5", None, self.deployment_lower, None, None), + EdgeDevicesTestConfig("device6", None, self.deployment_lower, None, None), + EdgeDevicesTestConfig("device7", None, self.deployment_top, None, None), + ] + + device_arg_string = self._generate_device_arg_string(devices) + self.cmd( + f"iot edge devices create -n {self.entity_name} -g {self.entity_rg} -c -y {device_arg_string}" + ) + + self._validate_results(devices, None) + + def test_edge_devices_create_config_overrides(self): + config_path = "./device_configs/edge_devices_min_config.yml" + override_auth_type = "shared_private_key" + default_edge_agent = "mcr.microsoft.com/azureiotedge-agent:1.3" + config_template = "./device_configs/device_config.toml" + devices: List[EdgeDevicesTestConfig] = [ + EdgeDevicesTestConfig( + "device_1", None, None, None, "mcr.microsoft.com/azureiotedge-agent:1.2" + ), + EdgeDevicesTestConfig( + "device_2", "device_1", None, None, default_edge_agent + ), + EdgeDevicesTestConfig( + "device_3", + "device_1", + None, + None, + "mcr.microsoft.com/azureiotedge-agent:1.4", + ), + EdgeDevicesTestConfig("device_4", None, None, None, default_edge_agent), + ] + + # file has cert auth, call with overrides: keyAuth, custom config path, default edge agent + self.cmd( + f"iot edge devices create -n {self.entity_name} -g {self.entity_rg} -c -y --cfg {config_path} --dct {config_template}" + f" --out device_bundles --device-auth {override_auth_type} --default-edge-agent {default_edge_agent}" + ) + + self._validate_results( + devices, "device_bundles", cert_auth=False, custom_device_template=True + ) + + def _validate_results( + self, + devices: List[EdgeDevicesTestConfig], + output_path: str, + cert_auth: bool = False, + custom_device_template: bool = False, + ): + # get all devices in hub + device_list = self.cmd( + f"iot hub device-identity list -n {self.entity_name} -g {self.entity_rg}" + ).get_output_in_json() + # make sure all devices were created + assert len(device_list) == len(devices) + # validate each device + for device_tuple in devices: + device_id = device_tuple.id + device = self.cmd( + f"iot hub device-identity show -d {device_id} -n {self.entity_name} -g {self.entity_rg}" + ).get_output_in_json() + assert device + parent = device_tuple.parent + if parent: + # validate each device's parent is correct + assert f"ms-azure-iot-edge://{parent}-" in device["parentScopes"][0] + deployment = device_tuple.deployment + if deployment: + checks = [] + + if deployment in [self.deployment_top, self.deployment_top_config]: + checks.extend( + ( + self.exists("properties.desired.modules.IoTEdgeAPIProxy"), + self.check( + "properties.desired.modules.IoTEdgeAPIProxy.env.BLOB_UPLOAD_ROUTE_ADDRESS.value", + "AzureBlobStorageonIoTEdge:11002", + ), + ) + ) + if deployment in [self.deployment_lower, self.deployment_lower_config]: + checks.extend( + ( + self.exists( + "properties.desired.modules.simulatedTemperatureSensor" + ), + self.check( + "properties.desired.modules.simulatedTemperatureSensor.settings.image", + "$upstream:443/azureiotedge-simulated-temperature-sensor:1.0", + ), + ) + ) + # get edgeAgent properties to check module configs + self.cmd( + f"iot hub module-twin show -d {device_id} -n {self.entity_name} -g {self.entity_rg} -m $edgeAgent", + checks=checks, + ) + # check output if specified + if output_path: + # untar target device bundle + bundle_file = f"{output_path}/{device_id}.tgz" + assert exists(bundle_file) + with tarfile.open(bundle_file, "r:gz") as device_tar: + + # check device bundle files + file_names = device_tar.getnames() + for item in [ + f"{device_id}.cert.pem", + f"{device_id}.key.pem", + f"{device_id}.full-chain.cert.pem", + "iotedge_config_cli_root.pem", + "config.toml", + "install.sh", + "README.md", + ]: + assert item in file_names + assert cert_auth == (f"{device_id}.hub-auth-cert.pem" in file_names) + assert cert_auth == (f"{device_id}.hub-auth-key.pem" in file_names) + # check config values + config_toml = device_tar.extractfile("config.toml") + import tomli + + config = tomli.load(config_toml) + + # auth type + assert config["provisioning"]["authentication"]["method"] == ( + "x509" if cert_auth else "sas" + ) + # hub hostname + assert ( + config["provisioning"]["iothub_hostname"] + == f"{self.entity_name}.azure-devices.net" + ) + # device_id + assert config["provisioning"]["device_id"] == device_id + # device hostname + hostname = getattr(device_tuple, "hostname", None) + if hostname: + assert config["hostname"] == hostname + # edge agent + agent = getattr(device_tuple, "edge_agent", None) + if agent: + assert config["agent"]["config"]["image"] == agent + # hacky way to ensure we're loading config from file + if custom_device_template: + assert config["test"]["foo"] == "bar" + + if output_path: + rmtree(output_path) + + def _generate_device_arg_string(self, devices: List[EdgeDevicesTestConfig]): + device_arg_string = "" + for device_tuple in devices: + device_id = device_tuple.id + parent_id = device_tuple.parent + deployment = device_tuple.deployment + hostname = getattr(device_tuple, "hostname", None) + edge_agent = getattr(device_tuple, "edge_agent", None) + args = [f"id={device_id}"] + if parent_id: + args.append(f"parent={parent_id}") + if deployment: + args.append(f"deployment={deployment}") + if hostname: + args.append(f"hostname={hostname}") + if edge_agent: + args.append(f"edgeAgent={edge_agent}") + device_arg_string = f"{device_arg_string} --device {' '.join(args)}" + return device_arg_string diff --git a/azext_iot/tests/iothub/devices/test_iot_edge_devices_unit.py b/azext_iot/tests/iothub/devices/test_iot_edge_devices_unit.py new file mode 100644 index 000000000..d9953f52b --- /dev/null +++ b/azext_iot/tests/iothub/devices/test_iot_edge_devices_unit.py @@ -0,0 +1,1267 @@ +# coding=utf-8 +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +from os import getcwd +from pathlib import PurePath +from unittest.mock import call +import pytest +import json +import responses +import re +from os.path import exists, join +from azext_iot.common.certops import create_self_signed_certificate +from azext_iot.common.fileops import write_content_to_file +from azext_iot.common.shared import DeviceAuthType +from azext_iot.common.utility import process_json_arg, process_yaml_arg +from azext_iot.sdk.iothub.service.models import ConfigurationContent +from azext_iot.iothub import commands_device_identity as subject +from azext_iot.iothub.providers.helpers.edge_device_config import ( + EDGE_CONFIG_SCRIPT_APPLY, + EDGE_CONFIG_SCRIPT_CA_CERTS, + EDGE_CONFIG_SCRIPT_HEADERS, + EDGE_CONFIG_SCRIPT_HOSTNAME, + EDGE_CONFIG_SCRIPT_HUB_AUTH_CERTS, + EDGE_CONFIG_SCRIPT_PARENT_HOSTNAME, + EDGE_ROOT_CERTIFICATE_FILENAME, + EDGE_ROOT_CERTIFICATE_SUBJECT, + create_edge_device_config_script, + process_edge_devices_config_args, + process_edge_devices_config_file_content, + create_edge_device_config, + try_parse_valid_deployment_config, +) +from azext_iot.iothub.common import ( + EdgeContainerAuth, + EdgeDevicesConfig, + EdgeDeviceConfig, +) +from azext_iot.tests.conftest import fixture_cmd, mock_target + +from azure.cli.core.azclierror import ( + AzureResponseError, + FileOperationError, + InvalidArgumentValueError, + MutuallyExclusiveArgumentError, + RequiredArgumentMissingError, +) + +from shutil import rmtree + +hub_name = "myhub" +hub_entity = mock_target["entity"] +resource_group_name = "RESOURCEGROUP" +mock_container_auth = { + "serverAddress": "serverAddress", + "username": "username", + "password": "$credential$", +} + + +test_certs_folder = "./test_certs" +test_root_cert = "root-cert.pem" +test_root_key = "root-key.pem" + +test_device_scopes = [ + {"deviceId": "device_1", "deviceScope": "dev1-scope-value"}, + {"deviceId": "device_2", "deviceScope": "dev2-scope-value"}, + {"deviceId": "device_3", "deviceScope": "dev3-scope-value"}, + {"deviceId": "device_4", "deviceScope": "dev4-scope-value"}, + {"deviceId": "device_5", "deviceScope": "dev5-scope-value"}, + {"deviceId": "device_6", "deviceScope": "dev6-scope-value"}, + {"deviceId": "device_7", "deviceScope": "dev7-scope-value"}, +] +test_path = getcwd() + + +class TestEdgeHierarchyCreateArgs: + @pytest.fixture() + def service_client(self, mocked_response, fixture_ghcs, fixture_sas): + mocked_response.assert_all_requests_are_fired = False + devices_url = f"https://{hub_entity}/devices" + # Query devices + mocked_response.add( + method=responses.POST, + url=f"{devices_url}/query", + body=json.dumps( + [ + {"deviceId": "dev1", "deviceScope": "dev1-scope-value"}, + {"deviceId": "dev2", "deviceScope": "dev2-scope-value"}, + ] + ), + status=200, + content_type="application/json", + match_querystring=False, + ) + + # delete any existing devices + mocked_response.add( + method=responses.DELETE, + url=re.compile(r"{}/dev\d+".format(devices_url)), + body="{}", + status=204, + content_type="application/json", + match_querystring=False, + ) + + # GET existing devices + mocked_response.add( + method=responses.GET, + url=devices_url, + body="[]", + status=200, + content_type="application/json", + match_querystring=False, + ) + + # GET specific devices + mocked_response.add( + method=responses.GET, + url=re.compile(r"{}/dev\d+".format(devices_url)), + body="{}", + status=200, + content_type="application/json", + match_querystring=False, + ) + + # Create / Update device-identity + mocked_response.add( + method=responses.PUT, + url=re.compile(r"{}/dev\d+".format(devices_url)), + body=json.dumps( + {"authentication": {"symmetricKey": {"primaryKey": "devicePrimaryKey"}}} + ), + status=200, + content_type="application/json", + match_querystring=False, + ) + + # Update config content / set modules + mocked_response.add( + method=responses.POST, + url=re.compile(r"{}/dev\d+/applyConfigurationContent".format(devices_url)), + body="{}", + status=200, + content_type="application/json", + match_querystring=False, + ) + + yield mocked_response + + @pytest.mark.parametrize( + "devices, config, visualize, clean, auth, output", + [ + # basic example, default auth, should output no files + ([["id=dev1", "parent=dev2"], ["id=dev2"]], None, False, True, None, None), + # Visualize, no clean, certificate auth, specified output + ( + [["id=dev3"]], + None, + True, + False, + DeviceAuthType.x509_thumbprint.value, + "device_bundles", + ), + # Flex argument processing + ( + [ + [ + "id=dev4", + "hostname=device-hostname", + "deployment=device_configs/deployments/deploymentTopLayer.json", + "edge_agent=my-edge-agent", + f"container_auth={json.dumps(mock_container_auth)}", + ], + [ + "id=dev5", + "hostname=device-hostname", + "deployment=device_configs/deployments/deploymentTopLayer.json", + "edge_agent=my-edge-agent", + "container_auth=device_configs/fake_edge_container_auth.json", + ], + ], + None, + True, + True, + DeviceAuthType.x509_thumbprint.value, + "new_device_bundle_folder", + ), + ], + ) + def test_edge_devices_create_args( + self, + fixture_cmd, + set_cwd, + service_client, + devices, + config, + visualize, + clean, + auth, + output, + ): + subject.iot_edge_devices_create( + cmd=fixture_cmd, + devices=devices, + config_file=config, + visualize=visualize, + clean=clean, + # can't prompt in unit test + yes=clean, + device_auth_type=auth, + bundle_output_path=output, + ) + + if output: + assert exists(output) + for device in devices: + device_id = device[0].split("=")[1] + assert exists(join(output, f"{device_id}.tgz")) + + rmtree(output) + + +class TestHierarchyCreateFailures: + @pytest.mark.parametrize( + "devices, config, root_cert, root_key, exception", + [ + # No devices + (None, None, None, None, InvalidArgumentValueError), + # Missing Device Id + ( + [["parent=dev2"], ["id=dev2"]], + None, + None, + None, + InvalidArgumentValueError, + ), + # Loop + ( + [["id=dev1", "parent=dev2"], ["id=dev2", "parent=dev1"]], + None, + None, + None, + InvalidArgumentValueError, + ), + # duplicate device + ( + [ + ["id=dev1", "parent=dev2"], + ["id=dev2"], + ["id=dev1"], + ], + None, + None, + None, + InvalidArgumentValueError, + ), + # missing parent + ( + [ + ["id=dev1", "parent=dev3"], + ["id=dev2"], + ["id=dev4"], + ], + None, + None, + None, + InvalidArgumentValueError, + ), + # devices AND config + ( + [ + ["id=dev1", "parent=dev2"], + ["id=dev2"], + ], + "path-to-config.yml", + None, + None, + MutuallyExclusiveArgumentError, + ), + # invalid deployment path + ( + [ + ["id=dev1", "parent=dev2", "deployment=path_does_not_exist.json"], + ["id=dev2"], + ], + None, + None, + None, + FileOperationError, + ), + # invalid deployment JSON + ( + [ + [ + "id=dev1", + "parent=dev2", + "deployment=./device_configs/invalid/invalid_deployment.json", + ], + ["id=dev2"], + ], + None, + None, + None, + InvalidArgumentValueError, + ), + # root cert but not key + ( + [ + ["id=dev1"], + ], + None, + "root_cert.pem", + None, + RequiredArgumentMissingError, + ), + # missing cert paths + ( + [ + ["id=dev1"], + ], + None, + "root_cert.pem", + "root_key.pem", + FileOperationError, + ), + ], + ) + def test_edge_devices_create_arg_failures( + self, + fixture_cmd, + fixture_ghcs, + set_cwd, + devices, + config, + root_cert, + root_key, + exception, + ): + with pytest.raises(exception): + subject.iot_edge_devices_create( + cmd=fixture_cmd, + devices=devices, + config_file=config, + root_cert_path=root_cert, + root_key_path=root_key, + ) + + @pytest.mark.parametrize( + "devices, config, exception", + [ + # No devices + (None, None, InvalidArgumentValueError), + # duplicate device + ( + None, + "device_configs/invalid/duplicate_device_config.yml", + InvalidArgumentValueError, + ), + # missing device ID + ( + None, + "device_configs/invalid/missing_device_id.yml", + InvalidArgumentValueError, + ), + # devices AND config + ( + [ + ["id=dev1", "parent=dev2"], + ["id=dev2"], + ], + "device_configs/nested_edge_config.yml", + MutuallyExclusiveArgumentError, + ), + # invalid file format + ( + None, + "device_configs/nested_edge_config.txt", + InvalidArgumentValueError, + ), + ], + ) + def test_edge_devices_create_config_failures( + self, fixture_ghcs, set_cwd, devices, config, exception + ): + with pytest.raises(exception): + subject.iot_edge_devices_create( + cmd=fixture_cmd, + devices=devices, + config_file=config, + ) + + +class TestHierarchyCreateConfig: + @pytest.fixture() + def service_client(self, mocked_response, fixture_ghcs, fixture_sas): + mocked_response.assert_all_requests_are_fired = False + devices_url = f"https://{hub_entity}/devices" + # get scopes (force retry) + for scope_response in [ + [], # Query existing devices + test_device_scopes[:-2], # Get Scopes [missing last 2] + test_device_scopes # On retry, return full list + ]: + mocked_response.add( + method=responses.POST, + url=f"{devices_url}/query", + body=json.dumps(scope_response), + status=200, + content_type="application/json", + match_querystring=False, + ) + + # delete any existing devices + mocked_response.add( + method=responses.DELETE, + url=re.compile(r"{}/device_\d+".format(devices_url)), + body="{}", + status=204, + content_type="application/json", + match_querystring=False, + ) + + # GET existing devices + mocked_response.add( + method=responses.GET, + url=devices_url, + body="[]", + status=200, + content_type="application/json", + match_querystring=False, + ) + + # Create / Update device-identity + mocked_response.add( + method=responses.PUT, + url=re.compile(r"{}/device_\d+".format(devices_url)), + body=json.dumps( + {"authentication": {"symmetricKey": {"primaryKey": "devicePrimaryKey"}}} + ), + status=200, + content_type="application/json", + match_querystring=False, + ) + + # GET specific device + mocked_response.add( + method=responses.GET, + url=re.compile(r"{}/device_\d+".format(devices_url)), + body="{}", + status=200, + content_type="application/json", + match_querystring=False, + ) + + # Update config content / set modules + mocked_response.add( + method=responses.POST, + url=re.compile( + r"{}/device_\d+/applyConfigurationContent".format(devices_url) + ), + body="{}", + status=200, + content_type="application/json", + match_querystring=False, + ) + + yield mocked_response + + @pytest.fixture() + def scope_retry_failed_client(self, mocked_response, fixture_ghcs, fixture_sas): + devices_url = f"https://{hub_entity}/devices" + # always return empty query + mocked_response.add( + method=responses.POST, + url=f"{devices_url}/query", + body="[]", + status=200, + content_type="application/json", + match_querystring=False, + ) + + # Create / Update device-identities + mocked_response.add( + method=responses.PUT, + url=re.compile(r"{}/device_\d+".format(devices_url)), + body=json.dumps( + {"authentication": {"symmetricKey": {"primaryKey": "devicePrimaryKey"}}} + ), + status=200, + content_type="application/json", + match_querystring=False, + ) + return mocked_response + + @pytest.fixture() + def mock_config_parse(self, mocker): + from azext_iot.iothub.providers import device_identity + + return mocker.spy(device_identity, "process_edge_devices_config_file_content") + + @pytest.mark.parametrize( + "devices, config, visualize, clean, out, auth_override, agent_override, " + "template_override, ca_cert_override, ca_key_override", + [ # yaml config + ( + None, + "device_configs/nested_edge_config.yml", + False, + True, + "device_bundles", + None, + None, + None, + None, + None, + ), + # yaml with auth type, device_config, and agent override + ( + None, + "device_configs/nested_edge_config.yml", + False, + True, + "device_bundles", + DeviceAuthType.x509_thumbprint.value, + "custom_edge_agent", + "./device_configs/device_config.toml", + None, + None + ), + # json config + ( + None, + "device_configs/nested_edge_config.json", + True, + True, + "device_bundles_2", + None, + None, + None, + None, + None, + ), + # json config with cert overrides + ( + None, + "device_configs/nested_edge_config.json", + True, + True, + "device_bundles_2", + None, + None, + None, + f"{test_certs_folder}/{test_root_cert}", + f"{test_certs_folder}/{test_root_key}" + ), + # no output + ( + None, + "device_configs/nested_edge_config.json", + True, + True, + None, + None, + None, + None, + None, + None + ), + ], + ) + def test_edge_devices_create_config( + self, + fixture_cmd, + mock_config_parse, + service_client, + set_cwd, + devices, + config, + visualize, + clean, + out, + auth_override, + agent_override, + template_override, + ca_cert_override, + ca_key_override + ): + + # create cert if we need + if ca_cert_override and ca_key_override: + root_cert = create_self_signed_certificate( + subject=EDGE_ROOT_CERTIFICATE_SUBJECT, + key_size=4096, + sha_version=256, + v3_extensions=True + ) + write_content_to_file( + content=root_cert["certificate"], + destination=test_certs_folder, + file_name=test_root_cert, + overwrite=True, + ) + write_content_to_file( + content=root_cert["privateKey"], + destination=test_certs_folder, + file_name=test_root_key, + overwrite=True, + ) + + subject.iot_edge_devices_create( + cmd=fixture_cmd, + devices=devices, + config_file=config, + visualize=visualize, + clean=clean, + # can't prompt in unit test + yes=clean, + bundle_output_path=out, + device_auth_type=auth_override, + device_config_template=template_override, + default_edge_agent=agent_override, + root_cert_path=ca_cert_override, + root_key_path=ca_key_override + ) + + # remove test cert' + if ca_cert_override and ca_key_override: + rmtree(test_certs_folder) + + cfg_obj = ( + process_yaml_arg(config) + if config.endswith(".yml") + else process_json_arg(config) + ) + + # parse called with correct config + assert mock_config_parse.call_args[1]["content"] == cfg_obj + + # assert inline overrides + assert mock_config_parse.call_args[1]["content"] == cfg_obj + assert mock_config_parse.call_args[1]["override_auth_type"] == auth_override + assert ( + mock_config_parse.call_args[1]["override_default_edge_agent"] + == agent_override + ) + assert ( + mock_config_parse.call_args[1]["override_device_config_template"] + == template_override + ) + assert ( + mock_config_parse.call_args[1]["override_root_cert_path"] + == ca_cert_override + ) + assert ( + mock_config_parse.call_args[1]["override_root_key_path"] + == ca_key_override + ) + + expected_devices = [] + + def add_device(device): + expected_devices.append(device["deviceId"]) + for child in device.get("children", []): + add_device(child) + + for device in cfg_obj["edgeDevices"]: + add_device(device) + if out: + assert exists(out) + for device_id in expected_devices: + assert exists(join(out, f"{device_id}.tgz")) + + rmtree(out) + + def test_edge_devices_scope_retry_failure(self, fixture_cmd, scope_retry_failed_client, set_cwd): + with pytest.raises(AzureResponseError): + subject.iot_edge_devices_create( + cmd=fixture_cmd, + devices=None, + config_file="device_configs/nested_edge_config.yml" + ) + + +class TestEdgeHierarchyConfigFunctions: + def create_test_root_cert(self, path): + root_cert = create_self_signed_certificate( + subject=EDGE_ROOT_CERTIFICATE_SUBJECT, + key_size=4096, + sha_version=256, + v3_extensions=True + ) + write_content_to_file( + content=root_cert["certificate"], + destination=path, + file_name=test_root_cert, + overwrite=True, + ) + write_content_to_file( + content=root_cert["privateKey"], + destination=path, + file_name=test_root_key, + overwrite=True, + ) + return root_cert + + test_device_id = "test_device_id" + device_config_with_parent_no_agent = EdgeDeviceConfig( + device_id=test_device_id, + parent_id="parent_device_id", + parent_hostname="parentHostname", + hostname="hostname", + ) + device_config_container_auth_with_agent_no_parent = EdgeDeviceConfig( + device_id=test_device_id, + container_auth=EdgeContainerAuth( + serveraddress="test-container-registry-address", + username="container registry username", + password="container registry password", + ), + edge_agent="special-edge-agent", + ) + + @pytest.mark.parametrize( + "device_id, auth_method, device_config, default_edge_agent, device_config_path, device_pk, output_path", + [ + # load external TOML, key auth + ( + test_device_id, + DeviceAuthType.shared_private_key.value, + device_config_with_parent_no_agent, + "default-edge-agent", + "./device_configs/device_config.toml", + "test-device-pk", + None, + ), + # load default TOML, cert auth + ( + test_device_id, + DeviceAuthType.x509_thumbprint.value, + device_config_container_auth_with_agent_no_parent, + "default-edge-agent", + None, + None, + "output_device_configs", + ), + ], + ) + def test_create_edge_device_config( + self, + set_cwd, + fixture_ghcs, + device_id, + auth_method, + device_config, + default_edge_agent, + device_config_path, + device_pk, + output_path, + ): + device_toml = create_edge_device_config( + device_id=device_id, + hub_hostname=hub_entity, + auth_method=auth_method, + device_config=device_config, + default_edge_agent=default_edge_agent, + device_config_path=device_config_path, + device_pk=device_pk, + output_path=output_path, + ) + + # always assert + assert ( + device_toml["trust_bundle_cert"] + == f"file:///etc/aziot/certificates/{EDGE_ROOT_CERTIFICATE_FILENAME}" + ) + assert device_toml["auto_reprovisioning_mode"] == "Dynamic" + assert device_toml["edge_ca"] == { + "cert": f"file:///etc/aziot/certificates/{device_id}.full-chain.cert.pem", + "pk": f"file:///etc/aziot/certificates/{device_id}.key.pem", + } + + assert device_toml["hostname"] == ( + device_config.hostname if device_config.hostname else "{{HOSTNAME}}" + ) + + # parent hostname config + if device_config.parent_id: + assert device_toml["parent_hostname"] == ( + device_config.parent_hostname + if device_config.parent_hostname + else "{{PARENT_HOSTNAME}}" + ) + + # hub provisioning + assert device_toml["provisioning"]["device_id"] == device_id + assert device_toml["provisioning"]["iothub_hostname"] == hub_entity + assert device_toml["provisioning"]["source"] == "manual" + + auth = device_toml["provisioning"]["authentication"] + if auth_method == DeviceAuthType.x509_thumbprint.value: + assert auth == { + "method": "x509", + "identity_cert": f"file:///etc/aziot/certificates/{device_id}.hub-auth-cert.pem", + "identity_pk": f"file:///etc/aziot/certificates/{device_id}.hub-auth-key.pem", + } + else: + assert auth == {"device_id_pk": {"value": device_pk}, "method": "sas"} + + assert device_toml["agent"]["config"]["image"] == ( + device_config.edge_agent if device_config.edge_agent else default_edge_agent + ) + + if not device_config.container_auth: + assert device_toml["agent"]["config"]["auth"] == {} + else: + assert ( + device_toml["agent"]["config"]["auth"]["serveraddress"] + == device_config.container_auth.serveraddress + ) + assert ( + device_toml["agent"]["config"]["auth"]["username"] + == device_config.container_auth.username + ) + assert ( + device_toml["agent"]["config"]["auth"]["password"] + == device_config.container_auth.password + ) + + if output_path: + import tomli_w + from os.path import join + + path = join(output_path, "config.toml") + with open(path, "rt", encoding="utf-8") as f: + assert tomli_w.dumps(device_toml) == f.read() + rmtree(output_path) + + @pytest.mark.parametrize( + "deployment, error", + [ + ( + "device_configs/invalid/invalid_deployment.json", + InvalidArgumentValueError, + ), + ("path_does_not_exist.json", FileOperationError), + ("device_configs/deployments/deploymentLowerLayer.json", None), + ], + ) + def test_process_edge_config_content(self, set_cwd, deployment, error): + try: + config_content = try_parse_valid_deployment_config(deployment) + assert isinstance(config_content, ConfigurationContent) + except error as ex: + assert isinstance(ex, error) + + @pytest.mark.parametrize( + "content, expected", + [ + ( + { + "configVersion": "1.0", + "iotHub": {"authenticationMethod": "symmetricKey"}, + "edgeConfiguration": { + "templateConfigPath": "template-config-path.toml", + "defaultEdgeAgent": "edge-agent-1", + }, + "edgeDevices": [ + { + "deviceId": "parent-device-id", + "edgeAgent": "test-agent", + "hostname": "parent-hostname", + }, + ], + }, + EdgeDevicesConfig( + version="1.0", + auth_method=DeviceAuthType.shared_private_key.value, + root_cert={ + "certificate": "root_certificate", + "thumbprint": "root_thumbprint", + "privateKey": "root_private_key", + }, + devices=[ + EdgeDeviceConfig( + device_id="parent-device-id", + edge_agent="test-agent", + hostname="parent-hostname", + ), + ], + template_config_path=PurePath(test_path, "template-config-path.toml").as_posix(), + default_edge_agent="edge-agent-1", + ), + ), + ( + { + "configVersion": "1.0", + "iotHub": {"authenticationMethod": "x509Certificate"}, + "edgeConfiguration": {"defaultEdgeAgent": "edge-agent-1"}, + "edgeDevices": [ + { + "deviceId": "parent-device-id", + "edgeAgent": "test-agent", + "hostname": "parent-hostname", + "children": [ + { + "deviceId": "child-device-id", + "edgeAgent": "test-agent2", + "hostname": "child-hostname", + } + ], + } + ], + }, + EdgeDevicesConfig( + version="1.0", + auth_method=DeviceAuthType.x509_thumbprint.value, + root_cert={ + "certificate": "root_certificate", + "thumbprint": "root_thumbprint", + "privateKey": "root_private_key", + }, + devices=[ + EdgeDeviceConfig( + device_id="parent-device-id", + edge_agent="test-agent", + hostname="parent-hostname", + ), + EdgeDeviceConfig( + device_id="child-device-id", + edge_agent="test-agent2", + hostname="child-hostname", + parent_id="parent-device-id", + parent_hostname="parent-hostname", + ), + ], + default_edge_agent="edge-agent-1", + ), + ), + ], + ) + def test_process_edge_devices_config_content( + self, set_cwd, patch_create_edge_root_cert, content, expected + ): + result = process_edge_devices_config_file_content(content=content, config_path=test_path) + assert result == expected + + def test_process_edge_devices_config_load_cert( + self, + set_cwd, + ): + content = { + "configVersion": "1.0", + "iotHub": {"authenticationMethod": "x509Certificate"}, + "edgeConfiguration": {"defaultEdgeAgent": "edge-agent-2"}, + "certificates": { + "rootCACertPath": f"{test_certs_folder}/{test_root_cert}", + "rootCACertKeyPath": f"{test_certs_folder}/{test_root_key}", + }, + "edgeDevices": [{"deviceId": "test"}], + } + cert = self.create_test_root_cert(test_certs_folder) + result = process_edge_devices_config_file_content(content) + rmtree(test_certs_folder) + assert result == EdgeDevicesConfig( + version="1.0", + auth_method=DeviceAuthType.x509_thumbprint.value, + default_edge_agent="edge-agent-2", + root_cert=cert, + devices=[EdgeDeviceConfig(device_id="test")], + ) + + @pytest.mark.parametrize( + "content, error", + [ + # no version + ( + { + "iotHub": {"authentication_method": "symmetricKey"}, + "edgeConfiguration": { + "templateConfigPath": "template-config-path.toml", + "defaultEdgeAgent": "edge-agent-1", + }, + }, + InvalidArgumentValueError, + ), + # No iothub config + ( + { + "configVersion": "1.0", + "edgeConfiguration": { + "templateConfigPath": "template-config-path.toml", + "defaultEdgeAgent": "edge-agent-1", + }, + }, + InvalidArgumentValueError, + ), + # missing root CA key + ( + { + "configVersion": "1.0", + "iotHub": {"authentication_method": "symmetricKey"}, + "edgeConfiguration": { + "templateConfigPath": "template-config-path.toml", + "defaultEdgeAgent": "edge-agent-1", + }, + "certificates": { + "rootCACertPath": "certs/root-cert.pem", + }, + "edgeDevices": [], + }, + InvalidArgumentValueError, + ), + # invalid auth value + ( + { + "configVersion": "1.0", + "iotHub": {"authenticationMethod": "super-duper-auth"}, + "edgeConfiguration": { + "templateConfigPath": "template-config-path.toml", + "defaultEdgeAgent": "edge-agent-1", + }, + "edgeDevices": [], + }, + InvalidArgumentValueError, + ), + ], + ) + def test_process_edge_devices_config_errors(self, content, error): + with pytest.raises(error): + process_edge_devices_config_file_content(content) + + @pytest.mark.parametrize( + "devices, auth, edge_agent, config_template, expected", + [ + # No extra params + ( + [["id=dev1", "parent=dev2"], ["id=dev2"]], + DeviceAuthType.x509_thumbprint.value, + None, + None, + EdgeDevicesConfig( + version="1.0", + auth_method=DeviceAuthType.x509_thumbprint.value, + root_cert={ + "certificate": "root_certificate", + "thumbprint": "root_thumbprint", + "privateKey": "root_private_key", + }, + devices=[ + EdgeDeviceConfig(device_id="dev1", parent_id="dev2"), + EdgeDeviceConfig(device_id="dev2"), + ], + ), + ), + # various edge-agent configs + ( + [ + ["id=dev1", "edge_agent=new-edge-agent"], + ["id=dev2"], + ], + DeviceAuthType.x509_thumbprint.value, + "default-edge-agent", + None, + EdgeDevicesConfig( + version="1.0", + auth_method=DeviceAuthType.x509_thumbprint.value, + default_edge_agent="default-edge-agent", + root_cert={ + "certificate": "root_certificate", + "thumbprint": "root_thumbprint", + "privateKey": "root_private_key", + }, + devices=[ + EdgeDeviceConfig(device_id="dev1", edge_agent="new-edge-agent"), + EdgeDeviceConfig( + device_id="dev2", + ), + ], + ), + ), + # load device config toml + ( + [ + ["id=dev1", "edge_agent=new-edge-agent", "hostname=dev1"], + ["id=dev2", "hostname=dev2"], + ], + DeviceAuthType.x509_thumbprint.value, + None, + "device_config.toml", + EdgeDevicesConfig( + version="1.0", + auth_method=DeviceAuthType.x509_thumbprint.value, + root_cert={ + "certificate": "root_certificate", + "thumbprint": "root_thumbprint", + "privateKey": "root_private_key", + }, + template_config_path="device_config.toml", + devices=[ + EdgeDeviceConfig( + device_id="dev1", + edge_agent="new-edge-agent", + hostname="dev1", + ), + EdgeDeviceConfig(device_id="dev2", hostname="dev2"), + ], + ), + ), + ], + ) + def test_process_edge_devices_config_args( + self, + set_cwd, + patch_create_edge_root_cert, + devices, + auth, + edge_agent, + config_template, + expected, + ): + result = process_edge_devices_config_args( + device_args=devices, + auth_type=auth, + default_edge_agent=edge_agent, + device_config_template=config_template, + ) + assert result == expected + + @pytest.mark.parametrize( + "device_id, hub_auth, hostname, has_parent, parent_hostname, segments", + [ + # device, hub_auth, hostname, parent, parent_hostname + ( + "test_device_id", + True, + "hostname", + True, + "parent_hostname", + [ + EDGE_CONFIG_SCRIPT_HEADERS.format("test_device_id"), + EDGE_CONFIG_SCRIPT_CA_CERTS, + EDGE_CONFIG_SCRIPT_HUB_AUTH_CERTS, + EDGE_CONFIG_SCRIPT_APPLY, + ], + ), + # device, no hub auth, hostname, parent, parent_hostname + ( + "test_device_id", + False, + "hostname", + True, + "parent_hostname", + [ + EDGE_CONFIG_SCRIPT_HEADERS.format("test_device_id"), + EDGE_CONFIG_SCRIPT_CA_CERTS, + EDGE_CONFIG_SCRIPT_APPLY, + ], + ), + # no optional parameters + ( + "test_device_id", + None, + None, + None, + None, + [ + EDGE_CONFIG_SCRIPT_HEADERS.format("test_device_id"), + EDGE_CONFIG_SCRIPT_HOSTNAME, + EDGE_CONFIG_SCRIPT_CA_CERTS, + EDGE_CONFIG_SCRIPT_APPLY, + ], + ), + # parent but no hostnames, no hub auth + ( + "test_device_id", + False, + None, + True, + None, + [ + EDGE_CONFIG_SCRIPT_HEADERS.format("test_device_id"), + EDGE_CONFIG_SCRIPT_HOSTNAME, + EDGE_CONFIG_SCRIPT_PARENT_HOSTNAME, + EDGE_CONFIG_SCRIPT_CA_CERTS, + EDGE_CONFIG_SCRIPT_APPLY, + ], + ), + ], + ) + def test_create_edge_device_config_script( + self, device_id, hub_auth, hostname, has_parent, parent_hostname, segments + ): + script_content = create_edge_device_config_script( + device_id=device_id, + hub_auth=hub_auth, + hostname=hostname, + has_parent=has_parent, + parent_hostname=parent_hostname, + ) + + assert script_content == "\n".join(segments) + + +class TestDevicesDelete: + @pytest.fixture() + def service_client(self, mocked_response, fixture_ghcs, fixture_sas): + mocked_response.assert_all_requests_are_fired = False + devices_url = f"https://{hub_entity}/devices" + # delete devices + mocked_response.add( + method=responses.DELETE, + url=re.compile(r"{}/device_\d+".format(devices_url)), + body="{}", + status=204, + content_type="application/json", + match_querystring=False, + ) + + yield mocked_response + + @pytest.fixture() + def mock_bulk_delete(self, mocker): + from azext_iot.iothub.providers.device_identity import DeviceIdentityProvider + + return mocker.spy(DeviceIdentityProvider, "delete_device_identities") + + @pytest.fixture() + def mock_service_delete(self, mocker): + from azext_iot.sdk.iothub.service.operations.devices_operations import DevicesOperations + + mock = mocker.spy(DevicesOperations, "delete_identity") + mock.metadata = {'url': '/devices/{id}'} + return mock + + @pytest.mark.parametrize( + "devices", + [ + [ + 'device_1', + 'device_2', + 'device_3', + ], + [ + 'device_1' + ], + ] + ) + def test_delete_bulk_devices( + self, + service_client, + mock_bulk_delete, + mock_service_delete, + fixture_ghcs, + fixture_sas, + devices, + ): + subject.iot_delete_devices( + cmd=fixture_cmd, + device_ids=devices + ) + assert mock_bulk_delete.call_args[1]["device_ids"] == devices + mock_self = mock_service_delete.call_args[0][0] + calls = [call(mock_self, id=device, if_match="*") for device in devices] + mock_service_delete.assert_has_calls(calls) diff --git a/azext_iot/tests/utility/test_iot_file_operations.py b/azext_iot/tests/utility/test_iot_file_operations.py new file mode 100644 index 000000000..a26216f6c --- /dev/null +++ b/azext_iot/tests/utility/test_iot_file_operations.py @@ -0,0 +1,65 @@ +# coding=utf-8 +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +from azext_iot.common.fileops import tar_directory, write_content_to_file +import pytest +import os +from os.path import join, exists +from azure.cli.core.azclierror import FileOperationError + +current_or_empty_dirs = ['./', '.', '', None] + + +class TestFileOperations(object): + + @pytest.mark.parametrize( + "content, destination, file_name, overwrite, error, remove_after_test", + [ + # initial create + ("test_file_contents", "./", "new_file.txt", False, None, False), + # fileoperationerror trying to overwrite existing file with overwrite: false + ("new_test_file_contents", "./", "new_file.txt", False, FileOperationError, False), + # force overwrite new file + ("overwrite_file_contents", "./", "new_file.txt", True, None, True), + ("second_test_file_contents", "test_dir", "new_file.txt", True, None, True), + ] + ) + def test_write_content_to_file(self, set_cwd, content, destination, file_name, overwrite, error, remove_after_test): + try: + write_content_to_file(content=content, destination=destination, file_name=file_name, overwrite=overwrite) + with open(join(destination, file_name), "rt", encoding="utf-8") as f: + assert f.read() == content + if remove_after_test: + os.remove(join(destination, file_name)) + if destination not in current_or_empty_dirs: + os.rmdir(destination) + except Exception as ex: + assert (error and isinstance(ex, error)) + + @pytest.mark.parametrize( + "target_directory, tarfile_path, tarfile_name, overwrite, error, delete_after_test", + [ + ("./", "./", "test_tar", False, None, False), + ("./", "./", "test_tar", False, FileOperationError, False), + ("./", "./", "test_tar", True, None, True), + ("./", "new_dir", "test_tar", True, None, True), + ] + ) + def test_tar_directory(self, set_cwd, target_directory, tarfile_path, tarfile_name, overwrite, error, delete_after_test): + try: + tar_directory( + target_directory=target_directory, + tarfile_path=tarfile_path, + tarfile_name=tarfile_name, + overwrite=overwrite + ) + assert exists(join(tarfile_path, f"{tarfile_name}.tgz")) + if delete_after_test: + os.remove(join(tarfile_path, f"{tarfile_name}.tgz")) + if tarfile_path not in current_or_empty_dirs: + os.rmdir(tarfile_path) + except Exception as ex: + assert (error and isinstance(ex, error)) diff --git a/azext_iot/tests/utility/test_iot_utility_unit.py b/azext_iot/tests/utility/test_iot_utility_unit.py index 209a31e78..5cfc8c69f 100644 --- a/azext_iot/tests/utility/test_iot_utility_unit.py +++ b/azext_iot/tests/utility/test_iot_utility_unit.py @@ -338,12 +338,12 @@ def mock_invoke(args, out_file): ( "iot hub device-twin show -n 'abcd' -d 'dcba'", "20a300e5-a444-4130-bb5a-1abd08ad930a", - None + None, ), ( "iot hub device-identity create -n abcd -d dcba", None, - "20a300e5-a444-4130-bb5a-1abd08ad930a" + "20a300e5-a444-4130-bb5a-1abd08ad930a", ), ( "iot hub device-twin show -n 'abcd' -d 'dcba'", @@ -352,7 +352,9 @@ def mock_invoke(args, out_file): ), ], ) - def test_embedded_cli(self, mocker, mocked_azclient, command, user_subscription, subscription): + def test_embedded_cli( + self, mocker, mocked_azclient, command, user_subscription, subscription + ): import shlex cli_ctx = mocker.MagicMock() @@ -465,7 +467,7 @@ class TestHandleServiceException(object): (502, AzureInternalError), (503, AzureInternalError), (504, AzureInternalError), - (None, AzureResponseError) + (None, AzureResponseError), ], ) def test_handle_service_exception(self, mocker, status_code, expected_error): diff --git a/docs/samples/sample_devices_config.yaml b/docs/samples/sample_devices_config.yaml new file mode 100644 index 000000000..91a796761 --- /dev/null +++ b/docs/samples/sample_devices_config.yaml @@ -0,0 +1,31 @@ +# This is a sample edge device creation configuration file. +# The following are sample values used to setup a nested edge scenario with +# one parent device (parentDevice) and two child devices (childDevice and secondChildDevice) + +configVersion: "1.0" + +iotHub: + authenticationMethod: symmetricKey # Options: [symmetricKey, x509Certificate] + +# certificates: # Optional, a self-signed CA will be generated if not provided here or in command arguments +# rootCACertPath: "rootCA.cert.pem" +# rootCACertKeyPath: "rootCA.key.pem" + +# Edge-specific configuration for device config templates and default edge agent. +edgeConfiguration: + templateConfigPath: "./device_configs/config_template.toml" # Optional, an empty device configuration is created if not provided here or in command arguments + defaultEdgeAgent: "$upstream:443/azureiotedge-agent:1.4" # In this nested scenario, these child devices will use an upstream container registry. + +edgeDevices: + - deviceId: "parentDevice" # Hub Device Identity + edgeAgent: "mcr.microsoft.com/azureiotedge-agent:1.2" # Edge agent image URI + hostname: "parentDeviceHostname.local" # FQDN or IP address + deployment: "parentDeviceDeployment.json" # Path to edge deployment configuration JSON + children: + - deviceId: "childDevice" + hostname: "childDeviceHostname.local" + edgeAgent: "$upstream:443/azureiotedge-agent:1.2" # Optional, will use `edgeConfiguration.defaultEdgeAgent` if not provided + deployment: "childDeviceDeployment.json" + - deviceId: "secondChildDevice" + hostname: "secondChildDeviceHostname.local" + deployment: "childDeviceDeployment.json" \ No newline at end of file diff --git a/setup.py b/setup.py index 03a2f1bc8..89ee483b5 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,10 @@ "jsonschema~=3.2.0", "importlib_metadata;python_version<'3.8'", "azure-iot-device~=2.11", + "tomli~=2.0", + "tomli-w~=1.0", "tqdm~=4.62", + "treelib~=1.6", "packaging" ] EXTRAS = {"uamqp": ["uamqp~=1.2"]} diff --git a/thirdpartynotice b/thirdpartynotice index c023006bc..649d0e6e6 100644 --- a/thirdpartynotice +++ b/thirdpartynotice @@ -128,3 +128,78 @@ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +----------------------------------------------------------------------------------------------------------------------- +License notice for treelib, obtained from https://github.com/caesar0301/treelib/blob/master/LICENSE + +treelib - Simple to use for you. +Python 2/3 Tree Implementation + +Copyright (C) 2011 + Brett Alistair Kromkamp - brettkromkamp@gmail.com +Copyright (C) 2012-2017 + Xiaming Chen - chenxm35@gmail.com + and other contributors. +All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +----------------------------------------------------------------------------------------------------------------------- +License notice for tomli, obtained from https://github.com/hukkin/tomli/blob/master/LICENSE + +MIT License + +Copyright (c) 2021 Taneli Hukkinen + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +----------------------------------------------------------------------------------------------------------------------- +License notice for tomli-w, obtained from https://github.com/hukkin/tomli-w/blob/master/LICENSE + +MIT License + +Copyright (c) 2021 Taneli Hukkinen + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE.