From fcbc0ae2e4b27569798813dcf7391371746a4ddf Mon Sep 17 00:00:00 2001 From: Akash Mukhopadhyay <43958099+mukhoakash@users.noreply.github.com> Date: Wed, 26 Jun 2024 01:00:51 -0700 Subject: [PATCH] AKS: Add ephemeral disk volume type and nvme perf tier support to Azure Container Storage (#7730) --- src/aks-preview/HISTORY.rst | 6 + src/aks-preview/azext_aks_preview/_params.py | 36 +++ .../azurecontainerstorage/_consts.py | 5 + .../azurecontainerstorage/_helpers.py | 115 ++++++++-- .../azurecontainerstorage/_validators.py | 207 ++++++++++++++++-- .../azurecontainerstorage/acstor_ops.py | 199 +++++++++++++---- src/aks-preview/azext_aks_preview/custom.py | 4 + .../managed_cluster_decorator.py | 131 ++++++++++- .../tests/latest/test_aks_commands.py | 109 +++++++++ .../tests/latest/test_validators.py | 202 ++++++++++++++--- src/aks-preview/linter_exclusions.yml | 12 + src/aks-preview/setup.py | 2 +- 12 files changed, 904 insertions(+), 124 deletions(-) diff --git a/src/aks-preview/HISTORY.rst b/src/aks-preview/HISTORY.rst index 97b5570e40f..209aa65225f 100644 --- a/src/aks-preview/HISTORY.rst +++ b/src/aks-preview/HISTORY.rst @@ -11,6 +11,11 @@ To release a new version, please select a new version number (usually plus 1 to Pending +++++++ + +5.0.0b2 +++++++++ +* Add option `--ephemeral-disk-volume-type` to `az aks create` and `az aks update` for Azure Container Storage operations. +* Add option `--azure-container-storage-perf-tier` to `az aks create` and `az aks update` to define resource tiers for Azure Container Storage performance. * Vendor new SDK and bump API version to 2024-04-02-preview. 5.0.0b1 @@ -19,6 +24,7 @@ Pending * Update --enable-advanced-network-observability description to note additional costs and add missing flag to create command. * Change default value of `--vm-set-type` to VirtualMachines when `--vm-sizes` is set. + 4.0.0b5 ++++++++ * Add warnings to `az aks mesh` commands for out of support asm revision in use. diff --git a/src/aks-preview/azext_aks_preview/_params.py b/src/aks-preview/azext_aks_preview/_params.py index f4b9d4d34d4..de690a69225 100644 --- a/src/aks-preview/azext_aks_preview/_params.py +++ b/src/aks-preview/azext_aks_preview/_params.py @@ -190,6 +190,11 @@ ) from azext_aks_preview.azurecontainerstorage._consts import ( CONST_ACSTOR_ALL, + CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY, + CONST_DISK_TYPE_PV_WITH_ANNOTATION, + CONST_EPHEMERAL_NVME_PERF_TIER_BASIC, + CONST_EPHEMERAL_NVME_PERF_TIER_PREMIUM, + CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD, CONST_STORAGE_POOL_TYPE_AZURE_DISK, CONST_STORAGE_POOL_TYPE_EPHEMERAL_DISK, CONST_STORAGE_POOL_TYPE_ELASTIC_SAN, @@ -362,6 +367,17 @@ CONST_ACSTOR_ALL, ] +ephemeral_disk_volume_types = [ + CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY, + CONST_DISK_TYPE_PV_WITH_ANNOTATION, +] + +ephemeral_disk_nvme_perf_tiers = [ + CONST_EPHEMERAL_NVME_PERF_TIER_BASIC, + CONST_EPHEMERAL_NVME_PERF_TIER_PREMIUM, + CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD, +] + # consts for guardrails level node_provisioning_modes = [ CONST_NODE_PROVISIONING_MODE_MANUAL, @@ -889,6 +905,16 @@ def load_arguments(self, _): arg_type=get_enum_type(storage_pool_options), help="set ephemeral disk storage pool option for azure container storage", ) + c.argument( + "ephemeral_disk_volume_type", + arg_type=get_enum_type(ephemeral_disk_volume_types), + help="set ephemeral disk volume type for azure container storage", + ) + c.argument( + "ephemeral_disk_nvme_perf_tier", + arg_type=get_enum_type(ephemeral_disk_nvme_perf_tiers), + help="set ephemeral disk volume type for azure container storage", + ) c.argument( "node_provisioning_mode", is_preview=True, @@ -1320,6 +1346,16 @@ def load_arguments(self, _): "azure_container_storage_nodepools", help="define the comma separated nodepool list to install azure container storage", ) + c.argument( + "ephemeral_disk_volume_type", + arg_type=get_enum_type(ephemeral_disk_volume_types), + help="set ephemeral disk volume type for azure container storage", + ) + c.argument( + "ephemeral_disk_nvme_perf_tier", + arg_type=get_enum_type(ephemeral_disk_nvme_perf_tiers), + help="set ephemeral disk volume type for azure container storage", + ) c.argument( "node_provisioning_mode", is_preview=True, diff --git a/src/aks-preview/azext_aks_preview/azurecontainerstorage/_consts.py b/src/aks-preview/azext_aks_preview/azurecontainerstorage/_consts.py index 13f7b05f3c5..b428abff7d5 100644 --- a/src/aks-preview/azext_aks_preview/azurecontainerstorage/_consts.py +++ b/src/aks-preview/azext_aks_preview/azurecontainerstorage/_consts.py @@ -7,6 +7,11 @@ CONST_ACSTOR_IO_ENGINE_LABEL_KEY = "acstor.azure.com/io-engine" CONST_ACSTOR_IO_ENGINE_LABEL_VAL = "acstor" CONST_ACSTOR_K8S_EXTENSION_NAME = "microsoft.azurecontainerstorage" +CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY = "EphemeralVolumeOnly" +CONST_DISK_TYPE_PV_WITH_ANNOTATION = "PersistentVolumeWithAnnotation" +CONST_EPHEMERAL_NVME_PERF_TIER_BASIC = "Basic" +CONST_EPHEMERAL_NVME_PERF_TIER_PREMIUM = "Premium" +CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD = "Standard" CONST_EXT_INSTALLATION_NAME = "azurecontainerstorage" CONST_K8S_EXTENSION_CLIENT_FACTORY_MOD_NAME = "azext_k8s_extension._client_factory" CONST_K8S_EXTENSION_CUSTOM_MOD_NAME = "azext_k8s_extension.custom" diff --git a/src/aks-preview/azext_aks_preview/azurecontainerstorage/_helpers.py b/src/aks-preview/azext_aks_preview/azurecontainerstorage/_helpers.py index 534c4baeeb2..d24cc982074 100644 --- a/src/aks-preview/azext_aks_preview/azurecontainerstorage/_helpers.py +++ b/src/aks-preview/azext_aks_preview/azurecontainerstorage/_helpers.py @@ -10,6 +10,11 @@ CONST_ACSTOR_ALL, CONST_ACSTOR_IO_ENGINE_LABEL_KEY, CONST_ACSTOR_K8S_EXTENSION_NAME, + CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY, + CONST_DISK_TYPE_PV_WITH_ANNOTATION, + CONST_EPHEMERAL_NVME_PERF_TIER_BASIC, + CONST_EPHEMERAL_NVME_PERF_TIER_PREMIUM, + CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD, CONST_EXT_INSTALLATION_NAME, CONST_K8S_EXTENSION_CLIENT_FACTORY_MOD_NAME, CONST_K8S_EXTENSION_CUSTOM_MOD_NAME, @@ -144,7 +149,7 @@ def get_extension_installed_and_cluster_configs( resource_group, cluster_name, agentpool_profiles -) -> Tuple[bool, bool, bool, bool, float]: +) -> Tuple[bool, bool, bool, bool, bool, float, str, str]: client_factory = get_k8s_extension_module(CONST_K8S_EXTENSION_CLIENT_FACTORY_MOD_NAME) client = client_factory.cf_k8s_extension_operation(cmd.cli_ctx) k8s_extension_custom_mod = get_k8s_extension_module(CONST_K8S_EXTENSION_CUSTOM_MOD_NAME) @@ -153,6 +158,8 @@ def get_extension_installed_and_cluster_configs( is_azureDisk_enabled = False is_ephemeralDisk_nvme_enabled = False is_ephemeralDisk_localssd_enabled = False + ephemeral_disk_volume_type = CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY + perf_tier = CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD resource_cpu_value = -1 try: @@ -184,6 +191,25 @@ def get_extension_installed_and_cluster_configs( config_settings.get("global.cli.storagePool.ephemeralDisk.temp.enabled", "False") == "True" ) cpu_value = config_settings.get("global.cli.resources.ioEngine.cpu", "1") + enable_ephemeral_bypass_annotation = ( + config_settings.get( + "global.cli.storagePool.ephemeralDisk.enableEphemeralBypassAnnotation", "False" + ) == "True" + ) + perf_tier = config_settings.get( + "global.cli.storagePool.ephemeralDisk.nvme.perfTier", + CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD + ) + + if perf_tier.lower() == CONST_EPHEMERAL_NVME_PERF_TIER_BASIC.lower(): + perf_tier = CONST_EPHEMERAL_NVME_PERF_TIER_BASIC + elif perf_tier.lower() == CONST_EPHEMERAL_NVME_PERF_TIER_PREMIUM.lower(): + perf_tier = CONST_EPHEMERAL_NVME_PERF_TIER_PREMIUM + else: + perf_tier = CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD + + if enable_ephemeral_bypass_annotation: + ephemeral_disk_volume_type = CONST_DISK_TYPE_PV_WITH_ANNOTATION resource_cpu_value = float(cpu_value) else: # For versions where "global.cli.activeControl" were not set it signifies @@ -192,7 +218,7 @@ def get_extension_installed_and_cluster_configs( is_azureDisk_enabled = is_elasticSan_enabled = is_ephemeralDisk_localssd_enabled = True resource_cpu_value = 1 - # Determine if epehemeral NVMe was active based on the labelled nodepools present in cluster. + # Determine if ephemeral NVMe was active based on the labelled nodepools present in cluster. for agentpool in agentpool_profiles: vm_size = agentpool.vm_size node_labels = agentpool.node_labels @@ -211,7 +237,9 @@ def get_extension_installed_and_cluster_configs( is_elasticSan_enabled, is_ephemeralDisk_localssd_enabled, is_ephemeralDisk_nvme_enabled, - resource_cpu_value + resource_cpu_value, + ephemeral_disk_volume_type, + perf_tier ) @@ -219,6 +247,7 @@ def get_initial_resource_value_args( storage_pool_type, storage_pool_option, nodepool_skus, + ephemeral_nvme_perf_tier, ): core_value = memory_value = hugepages_value = hugepages_number = 0 if (storage_pool_type == CONST_STORAGE_POOL_TYPE_AZURE_DISK or @@ -230,7 +259,7 @@ def get_initial_resource_value_args( hugepages_number = 512 elif (storage_pool_type == CONST_STORAGE_POOL_TYPE_EPHEMERAL_DISK and storage_pool_option == CONST_STORAGE_POOL_OPTION_NVME): - core_value = _get_cpu_value_based_on_vm_size(nodepool_skus) + core_value = _get_ephemeral_nvme_cpu_value_based_on_vm_size_perf_tier(nodepool_skus, ephemeral_nvme_perf_tier) memory_value = 2 hugepages_value = 2 hugepages_number = 1024 @@ -248,6 +277,7 @@ def get_current_resource_value_args( is_elasticSan_enabled, is_ephemeralDisk_localssd_enabled, is_ephemeralDisk_nvme_enabled, + ephemeral_nvme_perf_tier, current_core_value=None, nodepool_skus=None, ): @@ -261,6 +291,7 @@ def get_current_resource_value_args( is_elasticSan_enabled, is_ephemeralDisk_localssd_enabled, is_ephemeralDisk_nvme_enabled, + ephemeral_nvme_perf_tier, current_core_value, nodepool_skus, ) @@ -276,11 +307,13 @@ def get_current_resource_value_args( def get_desired_resource_value_args( storage_pool_type, storage_pool_option, + ephemeral_nvme_perf_tier, current_core_value, is_azureDisk_enabled, is_elasticSan_enabled, is_ephemeralDisk_localssd_enabled, is_ephemeralDisk_nvme_enabled, + perf_tier_updated, nodepool_skus, is_enabling_op, ): @@ -294,6 +327,7 @@ def get_desired_resource_value_args( is_elasticSan_enabled, is_ephemeralDisk_localssd_enabled, is_ephemeralDisk_nvme_enabled, + ephemeral_nvme_perf_tier, current_core_value, nodepool_skus, ) @@ -310,17 +344,23 @@ def get_desired_resource_value_args( updated_hugepages_value = 1 updated_hugepages_number = 512 elif (storage_pool_type == CONST_STORAGE_POOL_TYPE_EPHEMERAL_DISK and - storage_pool_option == CONST_STORAGE_POOL_OPTION_NVME): - updated_core_value = _get_cpu_value_based_on_vm_size(nodepool_skus) + (storage_pool_option == CONST_STORAGE_POOL_OPTION_NVME or + is_ephemeralDisk_nvme_enabled or perf_tier_updated)): + updated_core_value = _get_ephemeral_nvme_cpu_value_based_on_vm_size_perf_tier( + nodepool_skus, + ephemeral_nvme_perf_tier, + ) updated_memory_value = 2 updated_hugepages_value = 2 updated_hugepages_number = 1024 - # We have decided the updated value based on the type we are enabling. - # Now, we compare and check if the current values are already greater - # than that and if so, we preserve the current values. - updated_core_value = current_core_value \ - if current_core_value > updated_core_value else updated_core_value + if not perf_tier_updated: + # For an operation where we are not modifying the perf tiers for nvme, + # we have decided the updated value based on the type we are enabling. + # Now, we compare and check if the current values are already greater + # than that and if so, we preserve the current values. + updated_core_value = current_core_value \ + if current_core_value > updated_core_value else updated_core_value updated_memory_value = current_memory_value \ if current_memory_value > updated_memory_value else updated_memory_value updated_hugepages_value = current_hugepages_value \ @@ -345,8 +385,8 @@ def get_desired_resource_value_args( ) is_ephemeral_nvme_disabled_azureDisk_active = ( storage_pool_type == CONST_STORAGE_POOL_TYPE_EPHEMERAL_DISK and - (storage_pool_option == CONST_STORAGE_POOL_OPTION_NVME and - (is_ephemeralDisk_localssd_enabled or is_azureDisk_enabled) or + ((storage_pool_option == CONST_STORAGE_POOL_OPTION_NVME and + (is_ephemeralDisk_localssd_enabled or is_azureDisk_enabled)) or (storage_pool_option == CONST_ACSTOR_ALL and is_azureDisk_enabled)) ) if is_disabled_type_smaller_than_active_types: @@ -399,24 +439,55 @@ def get_cores_from_sku(vm_size): return cpu_value -def _get_cpu_value_based_on_vm_size(nodepool_skus): +def check_if_new_storagepool_creation_required( + storage_pool_type, + ephemeral_disk_volume_type, + ephemeral_disk_nvme_perf_tier, + existing_ephemeral_disk_volume_type, + existing_ephemeral_nvme_perf_tier, + is_extension_installed, + is_ephemeralDisk_nvme_enabled, + is_ephemeralDisk_localssd_enabled, +) -> bool: + should_create_storagepool = ( + not is_extension_installed or + not (is_ephemeralDisk_localssd_enabled or is_ephemeralDisk_nvme_enabled) or + storage_pool_type != CONST_STORAGE_POOL_TYPE_EPHEMERAL_DISK or + ( + (ephemeral_disk_volume_type is None or + (existing_ephemeral_disk_volume_type.lower() == ephemeral_disk_volume_type.lower())) and + (ephemeral_disk_nvme_perf_tier is None or + (existing_ephemeral_nvme_perf_tier.lower() == ephemeral_disk_nvme_perf_tier.lower())) + ) + ) + + return should_create_storagepool + + +def _get_ephemeral_nvme_cpu_value_based_on_vm_size_perf_tier(nodepool_skus, perf_tier): cpu_value = -1 + multiplication_factor = 0.25 + if perf_tier.lower() == CONST_EPHEMERAL_NVME_PERF_TIER_BASIC.lower(): + multiplication_factor = 0.15 + elif perf_tier.lower() == CONST_EPHEMERAL_NVME_PERF_TIER_PREMIUM.lower(): + multiplication_factor = 0.5 for vm_size in nodepool_skus: number_of_cores = get_cores_from_sku(vm_size) if number_of_cores != -1: if cpu_value == -1: - cpu_value = number_of_cores * 0.25 + cpu_value = number_of_cores * multiplication_factor else: - cpu_value = (number_of_cores * 0.25) if \ - (cpu_value > number_of_cores * 0.25) else \ + cpu_value = (number_of_cores * multiplication_factor) if \ + (cpu_value > number_of_cores * multiplication_factor) else \ cpu_value else: raise UnknownError( f"Unable to determine the number of cores in nodepool of node size: {vm_size}" ) - if cpu_value == -1: - cpu_value = 1 + # In any case when cpu_value = -1 or is lesser than 1, + # set the value to 1. + cpu_value = max(cpu_value, 1) return cpu_value @@ -425,6 +496,7 @@ def _get_current_resource_values( is_elasticSan_enabled, is_ephemeralDisk_localssd_enabled, is_ephemeralDisk_nvme_enabled, + ephemeral_nvme_perf_tier, current_core_value=None, nodepool_skus=None, ): @@ -444,7 +516,10 @@ def _get_current_resource_values( current_hugepages_number = 512 if is_ephemeralDisk_nvme_enabled: if current_core_value is None and nodepool_skus is not None: - core_value = _get_cpu_value_based_on_vm_size(nodepool_skus) + core_value = _get_ephemeral_nvme_cpu_value_based_on_vm_size_perf_tier( + nodepool_skus, + ephemeral_nvme_perf_tier, + ) current_memory_value = 2 current_hugepages_value = 2 current_hugepages_number = 1024 diff --git a/src/aks-preview/azext_aks_preview/azurecontainerstorage/_validators.py b/src/aks-preview/azext_aks_preview/azurecontainerstorage/_validators.py index 0202df2ed9b..44592a41ca6 100644 --- a/src/aks-preview/azext_aks_preview/azurecontainerstorage/_validators.py +++ b/src/aks-preview/azext_aks_preview/azurecontainerstorage/_validators.py @@ -5,6 +5,9 @@ import re +from azext_aks_preview._consts import ( + CONST_DEFAULT_NODE_OS_TYPE +) from azext_aks_preview.azurecontainerstorage._consts import ( CONST_ACSTOR_ALL, CONST_ACSTOR_IO_ENGINE_LABEL_KEY, @@ -38,7 +41,7 @@ logger = get_logger(__name__) -def validate_disable_azure_container_storage_params( +def validate_disable_azure_container_storage_params( # pylint: disable=too-many-branches storage_pool_type, storage_pool_name, storage_pool_sku, @@ -50,6 +53,8 @@ def validate_disable_azure_container_storage_params( is_elasticSan_enabled, is_ephemeralDisk_localssd_enabled, is_ephemeralDisk_nvme_enabled, + ephemeral_disk_volume_type, + ephemeral_disk_nvme_perf_tier, ): if not is_extension_installed: raise InvalidArgumentValueError( @@ -75,6 +80,18 @@ def validate_disable_azure_container_storage_params( 'when --disable-azure-container-storage is set.' ) + if ephemeral_disk_volume_type is not None: + raise MutuallyExclusiveArgumentError( + 'Conflicting flags. Cannot define --ephemeral-disk-volume-type value ' + 'when --disable-azure-container-storage is set.' + ) + + if ephemeral_disk_nvme_perf_tier is not None: + raise MutuallyExclusiveArgumentError( + 'Conflicting flags. Cannot define --ephemeral-disk-nvme-perf-tier value ' + 'when --disable-azure-container-storage is set.' + ) + if storage_pool_option is not None: if storage_pool_type != CONST_STORAGE_POOL_TYPE_EPHEMERAL_DISK: raise ArgumentUsageError( @@ -155,7 +172,7 @@ def validate_disable_azure_container_storage_params( ) -def validate_enable_azure_container_storage_params( +def validate_enable_azure_container_storage_params( # pylint: disable=too-many-locals,too-many-branches storage_pool_type, storage_pool_name, storage_pool_sku, @@ -168,6 +185,10 @@ def validate_enable_azure_container_storage_params( is_elasticSan_enabled, is_ephemeralDisk_localssd_enabled, is_ephemeralDisk_nvme_enabled, + ephemeral_disk_volume_type, + ephemeral_disk_nvme_perf_tier, + existing_ephemeral_disk_volume_type, + existing_ephemeral_disk_nvme_perf_tier, ): if storage_pool_name is not None: pattern = r'[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*' @@ -195,11 +216,107 @@ def validate_enable_azure_container_storage_params( ) if storage_pool_type == CONST_STORAGE_POOL_TYPE_EPHEMERAL_DISK: - if storage_pool_option is None: - raise RequiredArgumentMissingError( - 'Value of --storage-pool-option must be defined when ' - '--enable-azure-container-storage is set to ephemeralDisk.' - ) + if ephemeral_disk_volume_type is None and ephemeral_disk_nvme_perf_tier is None: + if storage_pool_option is None: + raise RequiredArgumentMissingError( + 'Value of --storage-pool-option must be defined when ' + '--enable-azure-container-storage is set to ephemeralDisk.' + ) + else: + required_type_installed_for_disk_vol_type = False + required_type_installed_for_nvme_perf_tier = False + + if ephemeral_disk_volume_type is not None: + required_type_installed_for_disk_vol_type = is_extension_installed and \ + (is_ephemeralDisk_localssd_enabled or is_ephemeralDisk_nvme_enabled) + + if ephemeral_disk_nvme_perf_tier is not None: + required_type_installed_for_nvme_perf_tier = is_extension_installed and \ + is_ephemeralDisk_nvme_enabled + + if storage_pool_option is None: + option_needed_by_dependent_params = [] + supported_option_needed = '' + if ephemeral_disk_volume_type is not None and not required_type_installed_for_disk_vol_type: + option_needed_by_dependent_params.append('--ephemeral-disk-volume-type') + supported_option_needed = f'{CONST_STORAGE_POOL_OPTION_NVME} or {CONST_STORAGE_POOL_OPTION_SSD}' + if ephemeral_disk_nvme_perf_tier is not None and not required_type_installed_for_nvme_perf_tier: + option_needed_by_dependent_params.append('--ephemeral-disk-nvme-perf-tier') + supported_option_needed = {CONST_STORAGE_POOL_OPTION_NVME} + + if len(option_needed_by_dependent_params) > 0: + params_requiring_options = ', '.join(option_needed_by_dependent_params) + raise ArgumentUsageError( + f'Cannot set {params_requiring_options} along with --enable-azure-container-storage ' + f'when storage pool type: {CONST_STORAGE_POOL_TYPE_EPHEMERAL_DISK} option: ' + f'{supported_option_needed} is not enabled for Azure Container Storage. ' + 'Enable the option using --storage-pool-option.' + ) + else: + if ephemeral_disk_nvme_perf_tier is not None and not required_type_installed_for_nvme_perf_tier and \ + storage_pool_option != CONST_STORAGE_POOL_OPTION_NVME: + raise ArgumentUsageError( + 'Cannot set --ephemeral-disk-nvme-perf-tier along with --enable-azure-container-storage ' + f'when storage pool type: {CONST_STORAGE_POOL_TYPE_EPHEMERAL_DISK} option: ' + f'{CONST_STORAGE_POOL_OPTION_NVME} is not enabled for Azure Container Storage. ' + 'Enable the option using --storage-pool-option.' + ) + + if (storage_pool_name is not None or storage_pool_sku is not None or + storage_pool_size is not None or nodepool_list is not None): + # pylint: disable=too-many-boolean-expressions + if (ephemeral_disk_volume_type is not None and + required_type_installed_for_disk_vol_type and ephemeral_disk_nvme_perf_tier is None) or \ + (ephemeral_disk_volume_type is None and + ephemeral_disk_nvme_perf_tier is not None and required_type_installed_for_nvme_perf_tier) or \ + (ephemeral_disk_volume_type is not None and required_type_installed_for_disk_vol_type and + ephemeral_disk_nvme_perf_tier is not None and required_type_installed_for_nvme_perf_tier): + enabled_options_arr = [] + if is_ephemeralDisk_nvme_enabled: + enabled_options_arr.append(CONST_STORAGE_POOL_OPTION_NVME) + if is_ephemeralDisk_localssd_enabled: + enabled_options_arr.append(CONST_STORAGE_POOL_OPTION_SSD) + enabled_options = ', '.join(enabled_options_arr) + + params_defined_arr = [] + if storage_pool_size is not None: + params_defined_arr.append('--storage-pool-size') + if storage_pool_sku is not None: + params_defined_arr.append('--storage-pool-sku') + if storage_pool_name is not None: + params_defined_arr.append('--storage-pool-name') + if nodepool_list is not None: + params_defined_arr.append('--azure-container-storage-nodepools') + params = ', '.join(params_defined_arr) + + flags_defined_arr = [] + if ephemeral_disk_volume_type is not None: + flags_defined_arr.append('--ephemeral-disk-volume-type') + if ephemeral_disk_nvme_perf_tier is not None: + flags_defined_arr.append('--ephemeral-disk-nvme-perf-tier') + + flags_defined = ', '.join(flags_defined_arr) + + raise ArgumentUsageError( + f'Cannot set {params} for creation of a new storage pool while setting value for ' + f'{flags_defined} since {CONST_STORAGE_POOL_TYPE_EPHEMERAL_DISK} storage pool type is ' + f'already enabled for storage pool option {enabled_options}.' + ) + else: + if ephemeral_disk_volume_type is not None and ephemeral_disk_nvme_perf_tier is None and \ + existing_ephemeral_disk_volume_type.lower() == ephemeral_disk_volume_type.lower(): + raise InvalidArgumentValueError( + 'Azure Container Storage is already configured with --ephemeral-disk-volume-type ' + f'value set to {existing_ephemeral_disk_volume_type}.' + ) + + if ephemeral_disk_nvme_perf_tier is not None and ephemeral_disk_volume_type is None and \ + existing_ephemeral_disk_nvme_perf_tier.lower() == ephemeral_disk_nvme_perf_tier.lower(): + raise InvalidArgumentValueError( + 'Azure Container Storage is already configured with --ephemeral-disk-nvme-perf-tier ' + f'value set to {existing_ephemeral_disk_nvme_perf_tier}.' + ) + if storage_pool_option == CONST_ACSTOR_ALL: raise InvalidArgumentValueError( f'Cannot set --storage-pool-option value as {CONST_ACSTOR_ALL} ' @@ -211,6 +328,16 @@ def validate_enable_azure_container_storage_params( 'Cannot set --storage-pool-option when --enable-azure-container-storage is not ephemeralDisk.' ) + if ephemeral_disk_volume_type is not None: + raise ArgumentUsageError( + 'Cannot set --ephemeral-disk-volume-type when --enable-azure-container-storage is not ephemeralDisk.' + ) + + if ephemeral_disk_nvme_perf_tier is not None: + raise ArgumentUsageError( + 'Cannot set --ephemeral-disk-nvme-perf-tier when --enable-azure-container-storage is not ephemeralDisk.' + ) + _validate_storage_pool_size(storage_pool_size, storage_pool_type) _validate_nodepools( @@ -232,9 +359,12 @@ def validate_enable_azure_container_storage_params( f'{storage_pool_type} in the cluster.' ) + # pylint: disable=too-many-boolean-expressions if storage_pool_type == CONST_STORAGE_POOL_TYPE_EPHEMERAL_DISK and \ + ephemeral_disk_volume_type is None and \ ((is_ephemeralDisk_nvme_enabled and - storage_pool_option == CONST_STORAGE_POOL_OPTION_NVME) or + storage_pool_option == CONST_STORAGE_POOL_OPTION_NVME and + ephemeral_disk_nvme_perf_tier is None) or (is_ephemeralDisk_localssd_enabled and storage_pool_option == CONST_STORAGE_POOL_OPTION_SSD)): ephemeral_disk_type_installed = CONST_STORAGE_POOL_OPTION_SSD if \ @@ -280,7 +410,7 @@ def _validate_storage_pool_size(storage_pool_size, storage_pool_type): ) -def _validate_nodepools( +def _validate_nodepools( # pylint: disable=too-many-branches nodepool_list, agentpool_details, storage_pool_type, @@ -301,12 +431,13 @@ def _validate_nodepools( 'in a cluster where Azure Container Storage is already installed.' ) - for agentpool in agentpool_details: - node_labels = agentpool.get("node_labels") - if node_labels is not None and \ - node_labels.get(CONST_ACSTOR_IO_ENGINE_LABEL_KEY) is not None: - nodepool_name = agentpool.get("name") - nodepool_arr.append(nodepool_name) + if agentpool_details is not None: + for agentpool in agentpool_details: + node_labels = agentpool.get("node_labels") + if node_labels is not None and \ + node_labels.get(CONST_ACSTOR_IO_ENGINE_LABEL_KEY) is not None: + nodepool_name = agentpool.get("name") + nodepool_arr.append(nodepool_name) if len(nodepool_arr) == 0: raise ArgumentUsageError( @@ -322,14 +453,43 @@ def _validate_nodepools( 'from the node pool and use node pools which has nodes with 4 or more cores and try again.' ) else: - _validate_nodepool_names(nodepool_list, agentpool_details) + agentpool_names = [] + if agentpool_details is not None: + for details in agentpool_details: + agentpool_names.append(details.get("name")) + if not nodepool_list: + agentpool_names_str = ', '.join(agentpool_names) + raise RequiredArgumentMissingError( + 'Multiple node pools present. Please define the node pools on which you want ' + 'to enable Azure Container Storage using --azure-container-storage-nodepools.' + f'\nNode pools available in the cluster are: {agentpool_names_str}.' + '\nAborting Azure Container Storage operation.' + ) + _validate_nodepool_names(nodepool_list, agentpool_names) nodepool_arr = nodepool_list.split(',') nvme_nodepool_found = False + available_node_count = 0 for nodepool in nodepool_arr: for agentpool in agentpool_details: pool_name = agentpool.get("name") if nodepool == pool_name: + os_type = agentpool.get("os_type") + if os_type is not None and os_type.lower() != CONST_DEFAULT_NODE_OS_TYPE.lower(): + raise InvalidArgumentValueError( + f'Azure Container Storage can be enabled only on {CONST_DEFAULT_NODE_OS_TYPE} nodepools. ' + f'Node pool: {pool_name}, os type: {os_type} does not meet the criteria.' + ) + mode = agentpool.get("mode") + node_taints = agentpool.get("node_taints") + if mode is not None and mode.lower() == "system" and node_taints is not None: + critical_taint = "CriticalAddonsOnly=true:NoSchedule" + if critical_taint.casefold() in (taint.casefold() for taint in node_taints): + raise InvalidArgumentValueError( + f'Unable to install Azure Container Storage on system nodepool: {pool_name} ' + f'since it has a taint {critical_taint}. Remove the taint from the node pool ' + 'and retry the Azure Container Storage operation.' + ) vm_size = agentpool.get("vm_size") if vm_size is not None: cpu_value = get_cores_from_sku(vm_size) @@ -343,6 +503,15 @@ def _validate_nodepools( if vm_size.lower().startswith('standard_l'): nvme_nodepool_found = True + node_count = agentpool.get("count") + if node_count is not None: + available_node_count = available_node_count + node_count + + if available_node_count < 3: + raise UnknownError( + 'Insufficient nodes present. Azure Container Storage requires atleast 3 nodes to be enabled.' + ) + if storage_pool_type == CONST_STORAGE_POOL_TYPE_EPHEMERAL_DISK and \ storage_pool_option == CONST_STORAGE_POOL_OPTION_NVME and \ not nvme_nodepool_found: @@ -355,7 +524,7 @@ def _validate_nodepools( # _validate_nodepool_names validates that the nodepool_list is a comma separated # string consisting of valid nodepool names i.e. a lower alphanumeric # characters and the first character should be lowercase letter. -def _validate_nodepool_names(nodepool_names, agentpool_details): +def _validate_nodepool_names(nodepool_names, agentpool_names): pattern = r'^[a-z][a-z0-9]*(?:,[a-z][a-z0-9]*)*$' if re.fullmatch(pattern, nodepool_names) is None: raise InvalidArgumentValueError( @@ -365,10 +534,6 @@ def _validate_nodepool_names(nodepool_names, agentpool_details): "alphanumeric characters and must begin with a lowercase letter." ) - agentpool_names = [] - for details in agentpool_details: - agentpool_names.append(details.get("name")) - nodepool_list = nodepool_names.split(',') for nodepool in nodepool_list: if nodepool not in agentpool_names: diff --git a/src/aks-preview/azext_aks_preview/azurecontainerstorage/acstor_ops.py b/src/aks-preview/azext_aks_preview/azurecontainerstorage/acstor_ops.py index d78d68b117e..6d04eb61fd1 100644 --- a/src/aks-preview/azext_aks_preview/azurecontainerstorage/acstor_ops.py +++ b/src/aks-preview/azext_aks_preview/azurecontainerstorage/acstor_ops.py @@ -6,7 +6,11 @@ from azure.cli.core.azclierror import UnknownError from azure.cli.core.commands import LongRunningOperation from azext_aks_preview.azurecontainerstorage._consts import ( + CONST_ACSTOR_ALL, CONST_ACSTOR_K8S_EXTENSION_NAME, + CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY, + CONST_DISK_TYPE_PV_WITH_ANNOTATION, + CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD, CONST_EXT_INSTALLATION_NAME, CONST_K8S_EXTENSION_CLIENT_FACTORY_MOD_NAME, CONST_K8S_EXTENSION_CUSTOM_MOD_NAME, @@ -15,12 +19,12 @@ CONST_STORAGE_POOL_OPTION_NVME, CONST_STORAGE_POOL_OPTION_SSD, CONST_STORAGE_POOL_SKU_PREMIUM_LRS, - CONST_ACSTOR_ALL, CONST_STORAGE_POOL_TYPE_AZURE_DISK, CONST_STORAGE_POOL_TYPE_ELASTIC_SAN, CONST_STORAGE_POOL_TYPE_EPHEMERAL_DISK, ) from azext_aks_preview.azurecontainerstorage._helpers import ( + check_if_new_storagepool_creation_required, get_k8s_extension_module, get_current_resource_value_args, get_desired_resource_value_args, @@ -46,7 +50,11 @@ def perform_enable_azure_container_storage( # pylint: disable=too-many-statemen storage_pool_sku, storage_pool_option, acstor_nodepool_skus, + ephemeral_disk_volume_type, + ephemeral_disk_nvme_perf_tier, is_cluster_create, + existing_ephemeral_disk_volume_type=CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY, + existing_ephemeral_nvme_perf_tier=CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD, is_extension_installed=False, is_azureDisk_enabled=False, is_elasticSan_enabled=False, @@ -75,51 +83,88 @@ def perform_enable_azure_container_storage( # pylint: disable=too-many-statemen storage_pool_type, ) - # Step 3: Configure the storagepool parameters - config_settings = [] - if storage_pool_name is None: - storage_pool_name = storage_pool_type.lower() - if storage_pool_type == CONST_STORAGE_POOL_TYPE_EPHEMERAL_DISK: - storage_pool_name = storage_pool_type.lower() + "-" + storage_pool_option.lower() - if storage_pool_size is None: - storage_pool_size = CONST_STORAGE_POOL_DEFAULT_SIZE_ESAN if \ - storage_pool_type == CONST_STORAGE_POOL_TYPE_ELASTIC_SAN else \ - CONST_STORAGE_POOL_DEFAULT_SIZE + is_storagepool_create_op_required = check_if_new_storagepool_creation_required( + storage_pool_type, + ephemeral_disk_volume_type, + ephemeral_disk_nvme_perf_tier, + existing_ephemeral_disk_volume_type, + existing_ephemeral_nvme_perf_tier, + is_extension_installed, + is_ephemeralDisk_nvme_enabled, + is_ephemeralDisk_localssd_enabled, + ) + config_settings = [] azure_disk_enabled = is_azureDisk_enabled if is_extension_installed else False elastic_san_enabled = is_elasticSan_enabled if is_extension_installed else False ephemeral_disk_nvme_enabled = is_ephemeralDisk_nvme_enabled if is_extension_installed else False ephemeral_disk_localssd_enabled = is_ephemeralDisk_localssd_enabled if is_extension_installed else False + # Step 3: Configure the storagepool parameters + if is_storagepool_create_op_required: + if storage_pool_name is None: + storage_pool_name = storage_pool_type.lower() + if storage_pool_type == CONST_STORAGE_POOL_TYPE_EPHEMERAL_DISK: + storage_pool_name = storage_pool_type.lower() + "-" + storage_pool_option.lower() + if storage_pool_size is None: + storage_pool_size = CONST_STORAGE_POOL_DEFAULT_SIZE_ESAN if \ + storage_pool_type == CONST_STORAGE_POOL_TYPE_ELASTIC_SAN else \ + CONST_STORAGE_POOL_DEFAULT_SIZE + + epheremaldisk_type = "" + if storage_pool_type == CONST_STORAGE_POOL_TYPE_EPHEMERAL_DISK: + if storage_pool_option == CONST_STORAGE_POOL_OPTION_NVME: + ephemeral_disk_nvme_enabled = True + elif storage_pool_option == CONST_STORAGE_POOL_OPTION_SSD: + ephemeral_disk_localssd_enabled = True + epheremaldisk_type = storage_pool_option.lower() + else: + if storage_pool_sku is None: + storage_pool_sku = CONST_STORAGE_POOL_SKU_PREMIUM_LRS + if storage_pool_type == CONST_STORAGE_POOL_TYPE_ELASTIC_SAN: + config_settings.append({"global.cli.storagePool.elasticSan.sku": storage_pool_sku}) + elastic_san_enabled = True + elif storage_pool_type == CONST_STORAGE_POOL_TYPE_AZURE_DISK: + config_settings.append({"global.cli.storagePool.azureDisk.sku": storage_pool_sku}) + azure_disk_enabled = True + + config_settings.extend( + [ + {"global.cli.storagePool.install.name": storage_pool_name}, + {"global.cli.storagePool.install.size": storage_pool_size}, + {"global.cli.storagePool.install.type": storage_pool_type}, + {"global.cli.storagePool.install.diskType": epheremaldisk_type}, + ] + ) - epheremaldisk_type = "" - if storage_pool_type == CONST_STORAGE_POOL_TYPE_EPHEMERAL_DISK: - if storage_pool_option == CONST_STORAGE_POOL_OPTION_NVME: - ephemeral_disk_nvme_enabled = True - elif storage_pool_option == CONST_STORAGE_POOL_OPTION_SSD: - ephemeral_disk_localssd_enabled = True - epheremaldisk_type = storage_pool_option.lower() - else: - if storage_pool_sku is None: - storage_pool_sku = CONST_STORAGE_POOL_SKU_PREMIUM_LRS - if storage_pool_type == CONST_STORAGE_POOL_TYPE_ELASTIC_SAN: - config_settings.append({"global.cli.storagePool.elasticSan.sku": storage_pool_sku}) - elastic_san_enabled = True - elif storage_pool_type == CONST_STORAGE_POOL_TYPE_AZURE_DISK: - config_settings.append({"global.cli.storagePool.azureDisk.sku": storage_pool_sku}) - azure_disk_enabled = True + enable_ephemeral_bypass_annotation = ( + (ephemeral_disk_volume_type is None and + existing_ephemeral_nvme_perf_tier.lower() == CONST_DISK_TYPE_PV_WITH_ANNOTATION.lower()) or + (ephemeral_disk_volume_type is not None and + ephemeral_disk_volume_type.lower() == CONST_DISK_TYPE_PV_WITH_ANNOTATION.lower()) + ) + + if existing_ephemeral_nvme_perf_tier is None: + existing_ephemeral_nvme_perf_tier = CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD + if ephemeral_disk_nvme_perf_tier is None: + ephemeral_disk_nvme_perf_tier = existing_ephemeral_nvme_perf_tier + + perf_tier_updated = False + if existing_ephemeral_nvme_perf_tier.lower() != ephemeral_disk_nvme_perf_tier.lower(): + perf_tier_updated = True config_settings.extend( [ {"global.cli.activeControl": True}, - {"global.cli.storagePool.install.create": True}, - {"global.cli.storagePool.install.name": storage_pool_name}, - {"global.cli.storagePool.install.size": storage_pool_size}, - {"global.cli.storagePool.install.type": storage_pool_type}, - {"global.cli.storagePool.install.diskType": epheremaldisk_type}, + {"global.cli.storagePool.install.create": is_storagepool_create_op_required}, {"global.cli.storagePool.azureDisk.enabled": azure_disk_enabled}, {"global.cli.storagePool.elasticSan.enabled": elastic_san_enabled}, {"global.cli.storagePool.ephemeralDisk.nvme.enabled": ephemeral_disk_nvme_enabled}, {"global.cli.storagePool.ephemeralDisk.temp.enabled": ephemeral_disk_localssd_enabled}, + { + "global.cli.storagePool.ephemeralDisk.enableEphemeralBypassAnnotation": + enable_ephemeral_bypass_annotation + }, + {"global.cli.storagePool.ephemeralDisk.nvme.perfTier": ephemeral_disk_nvme_perf_tier}, # Always set cli.storagePool.disable.type to empty # and cli.storagePool.disable.validation to False # during enable operation so that any older disable @@ -137,11 +182,13 @@ def perform_enable_azure_container_storage( # pylint: disable=too-many-statemen resource_args = get_desired_resource_value_args( storage_pool_type, storage_pool_option, + ephemeral_disk_nvme_perf_tier, current_core_value, is_azureDisk_enabled, is_elasticSan_enabled, is_ephemeralDisk_localssd_enabled, is_ephemeralDisk_nvme_enabled, + perf_tier_updated, acstor_nodepool_skus, True, ) @@ -151,6 +198,7 @@ def perform_enable_azure_container_storage( # pylint: disable=too-many-statemen storage_pool_type, storage_pool_option, acstor_nodepool_skus, + ephemeral_disk_nvme_perf_tier, ) config_settings.extend(resource_args) @@ -168,6 +216,8 @@ def perform_enable_azure_container_storage( # pylint: disable=too-many-statemen {"global.cli.storagePool.install.type": ""}, {"global.cli.storagePool.install.diskType": ""}, ] + + update_after_exception = False try: if is_extension_installed: result = k8s_extension_custom_mod.update_k8s_extension( @@ -212,6 +262,7 @@ def perform_enable_azure_container_storage( # pylint: disable=too-many-statemen ) else: if is_extension_installed: + update_after_exception = True logger.error( "AKS update to enable Azure Container Storage pool type %s failed. \n" " Error: %s. Resetting cluster state.", storage_pool_type, ex @@ -225,6 +276,7 @@ def perform_enable_azure_container_storage( # pylint: disable=too-many-statemen is_elasticSan_enabled, is_ephemeralDisk_localssd_enabled, is_ephemeralDisk_nvme_enabled, + existing_ephemeral_nvme_perf_tier, current_core_value, ) @@ -236,23 +288,41 @@ def perform_enable_azure_container_storage( # pylint: disable=too-many-statemen update_settings.append({"global.cli.storagePool.elasticSan.enabled": False}) elif storage_pool_type == CONST_STORAGE_POOL_TYPE_EPHEMERAL_DISK: if storage_pool_option == CONST_STORAGE_POOL_OPTION_NVME: - update_settings.append({"global.cli.storagePool.ephemeralDisk.nvme.enabled": False}) + update_settings.extend( + [ + {"global.cli.storagePool.ephemeralDisk.nvme.enabled": False}, + { + "global.cli.storagePool.ephemeralDisk.nvme.perfTier": + existing_ephemeral_nvme_perf_tier + }, + ] + ) elif storage_pool_option == CONST_STORAGE_POOL_OPTION_SSD: update_settings.append({"global.cli.storagePool.ephemeralDisk.temp.enabled": False}) + enable_ephemeral_bypass_annotation = ( + existing_ephemeral_disk_volume_type.lower() == CONST_DISK_TYPE_PV_WITH_ANNOTATION.lower() + ) + update_settings.append( + { + "global.cli.storagePool.ephemeralDisk.enableEphemeralBypassAnnotation": + enable_ephemeral_bypass_annotation + } + ) else: logger.error("AKS update to enable Azure Container Storage failed.\nError: %s", ex) - k8s_extension_custom_mod.update_k8s_extension( - cmd, - client, - resource_group, - cluster_name, - CONST_EXT_INSTALLATION_NAME, - "managedClusters", - configuration_settings=update_settings, - yes=True, - no_wait=True, - ) + if is_storagepool_create_op_required or update_after_exception: + k8s_extension_custom_mod.update_k8s_extension( + cmd, + client, + resource_group, + cluster_name, + CONST_EXT_INSTALLATION_NAME, + "managedClusters", + configuration_settings=update_settings, + yes=True, + no_wait=True, + ) def perform_disable_azure_container_storage( # pylint: disable=too-many-statements,too-many-locals,too-many-branches @@ -270,11 +340,13 @@ def perform_disable_azure_container_storage( # pylint: disable=too-many-stateme is_ephemeralDisk_localssd_enabled, is_ephemeralDisk_nvme_enabled, current_core_value, + existing_ephemeral_disk_volume_type=CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY, + existing_ephemeral_nvme_perf_tier=CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD, is_called_from_extension=False, ): # This will be set true only when aks-preview extension is used # and we want the aks-preview ManagedClusterDecorator to call the - # perform_enable_azure_container_storage function. + # perform_disable_azure_container_storage function. if not is_called_from_extension: return client_factory = get_k8s_extension_module(CONST_K8S_EXTENSION_CLIENT_FACTORY_MOD_NAME) @@ -304,14 +376,17 @@ def perform_disable_azure_container_storage( # pylint: disable=too-many-stateme else: if is_ephemeralDisk_nvme_enabled: pool_option = CONST_STORAGE_POOL_OPTION_NVME.lower() + storage_pool_option = CONST_STORAGE_POOL_OPTION_NVME elif is_ephemeralDisk_localssd_enabled: pool_option = CONST_STORAGE_POOL_OPTION_SSD.lower() + storage_pool_option = CONST_STORAGE_POOL_OPTION_SSD # Step 1: Perform validation if accepted by user if perform_validation: config_settings = [ {"global.cli.storagePool.disable.validation": True}, {"global.cli.storagePool.disable.type": storage_pool_type}, + {"global.cli.storagePool.disable.diskType": pool_option.lower()}, # Set these values to ensure cluster state incase of # a cluster where cli operation has not yet run or older # version of charts were installed. @@ -322,7 +397,6 @@ def perform_disable_azure_container_storage( # pylint: disable=too-many-stateme ] config_settings.extend(reset_install_settings) - config_settings.append({"global.cli.storagePool.disable.diskType": pool_option.lower()}) try: update_result = k8s_extension_custom_mod.update_k8s_extension( @@ -436,6 +510,10 @@ def perform_disable_azure_container_storage( # pylint: disable=too-many-stateme False ) else: + updated_enable_ephemeral_bypass_annotation = ( + existing_ephemeral_disk_volume_type.lower() == CONST_DISK_TYPE_PV_WITH_ANNOTATION.lower() + ) + updated_ephemeral_nvme_perf_tier = existing_ephemeral_nvme_perf_tier # Disabling a particular type of storagepool. if storage_pool_type == CONST_STORAGE_POOL_TYPE_AZURE_DISK: azure_disk_enabled = False @@ -450,6 +528,13 @@ def perform_disable_azure_container_storage( # pylint: disable=too-many-stateme ephemeral_disk_nvme_enabled = False ephemeral_disk_localssd_enabled = False + # If we are disabling ephemeral nvme, reset the following params: + # 1. ephemeral_disk_volume_type + # 2. ephemeral_disk_nvme_per_tier + if not ephemeral_disk_nvme_enabled: + updated_enable_ephemeral_bypass_annotation = False + updated_ephemeral_nvme_perf_tier = CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD + config_settings = [ {"global.cli.storagePool.disable.validation": False}, {"global.cli.storagePool.disable.type": storage_pool_type}, @@ -462,6 +547,11 @@ def perform_disable_azure_container_storage( # pylint: disable=too-many-stateme {"global.cli.storagePool.elasticSan.enabled": elastic_san_enabled}, {"global.cli.storagePool.ephemeralDisk.nvme.enabled": ephemeral_disk_nvme_enabled}, {"global.cli.storagePool.ephemeralDisk.temp.enabled": ephemeral_disk_localssd_enabled}, + { + "global.cli.storagePool.ephemeralDisk.enableEphemeralBypassAnnotation": + updated_enable_ephemeral_bypass_annotation + }, + {"global.cli.storagePool.ephemeralDisk.nvme.perfTier": updated_ephemeral_nvme_perf_tier}, ] config_settings.extend(reset_install_settings) @@ -470,11 +560,13 @@ def perform_disable_azure_container_storage( # pylint: disable=too-many-stateme resource_args = get_desired_resource_value_args( storage_pool_type, storage_pool_option, + existing_ephemeral_nvme_perf_tier, current_core_value, is_azureDisk_enabled, is_elasticSan_enabled, is_ephemeralDisk_localssd_enabled, is_ephemeralDisk_nvme_enabled, + False, None, False, ) @@ -530,10 +622,25 @@ def perform_disable_azure_container_storage( # pylint: disable=too-many-stateme is_elasticSan_enabled, is_ephemeralDisk_localssd_enabled, is_ephemeralDisk_nvme_enabled, + existing_ephemeral_nvme_perf_tier, current_core_value, ) update_settings.extend(resource_args) + + reset_enable_ephemeral_bypass_annotation = ( + existing_ephemeral_disk_volume_type.lower() == CONST_DISK_TYPE_PV_WITH_ANNOTATION.lower() + ) + update_settings.extend( + [ + { + "global.cli.storagePool.ephemeralDisk.enableEphemeralBypassAnnotation": + reset_enable_ephemeral_bypass_annotation + }, + {"global.cli.storagePool.ephemeralDisk.nvme.perfTier": existing_ephemeral_nvme_perf_tier}, + ] + ) + # Revert back to storagepool type states which was supposed to disabled. azure_disk_enabled = is_azureDisk_enabled elastic_san_enabled = is_elasticSan_enabled diff --git a/src/aks-preview/azext_aks_preview/custom.py b/src/aks-preview/azext_aks_preview/custom.py index 74f8e49beb8..bb5231ed930 100644 --- a/src/aks-preview/azext_aks_preview/custom.py +++ b/src/aks-preview/azext_aks_preview/custom.py @@ -639,6 +639,8 @@ def aks_create( storage_pool_size=None, storage_pool_sku=None, storage_pool_option=None, + ephemeral_disk_volume_type=None, + ephemeral_disk_nvme_perf_tier=None, node_provisioning_mode=None, ssh_access=CONST_SSH_ACCESS_LOCALUSER, # trusted launch @@ -846,6 +848,8 @@ def aks_update( storage_pool_sku=None, storage_pool_option=None, azure_container_storage_nodepools=None, + ephemeral_disk_volume_type=None, + ephemeral_disk_nvme_perf_tier=None, node_provisioning_mode=None, ssh_access=None, cluster_service_load_balancer_health_probe_mode=None, diff --git a/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py b/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py index 96f32c8168d..0a61da70791 100644 --- a/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py +++ b/src/aks-preview/azext_aks_preview/managed_cluster_decorator.py @@ -3347,6 +3347,21 @@ def set_up_azure_container_storage(self, mc: ManagedCluster) -> ManagedCluster: # read the azure container storage values passed pool_type = self.context.raw_param.get("enable_azure_container_storage") enable_azure_container_storage = pool_type is not None + ephemeral_disk_volume_type = self.context.raw_param.get("ephemeral_disk_volume_type") + ephemeral_disk_nvme_perf_tier = self.context.raw_param.get("ephemeral_disk_nvme_perf_tier") + if (ephemeral_disk_volume_type is not None or ephemeral_disk_nvme_perf_tier is not None) and \ + not enable_azure_container_storage: + params_defined_arr = [] + if ephemeral_disk_volume_type is not None: + params_defined_arr.append('--ephemeral-disk-volume-type') + if ephemeral_disk_nvme_perf_tier is not None: + params_defined_arr.append('--ephemeral-disk-nvme-perf-tier') + + params_defined = 'and '.join(params_defined_arr) + raise RequiredArgumentMissingError( + f'Cannot set {params_defined} without the parameter --enable-azure-container-storage.' + ) + if enable_azure_container_storage: pool_name = self.context.raw_param.get("storage_pool_name") pool_option = self.context.raw_param.get("storage_pool_option") @@ -3359,6 +3374,10 @@ def set_up_azure_container_storage(self, mc: ManagedCluster) -> ManagedCluster: pool_details = {} pool_details["name"] = agentpool.name pool_details["vm_size"] = agentpool.vm_size + pool_details["count"] = agentpool.count + pool_details["os_type"] = agentpool.os_type + pool_details["mode"] = agentpool.mode + pool_details["node_taints"] = agentpool.node_taints agentpool_details.append(pool_details) # Marking the only agentpool name as the valid nodepool for # installing Azure Container Storage during `az aks create` @@ -3367,6 +3386,15 @@ def set_up_azure_container_storage(self, mc: ManagedCluster) -> ManagedCluster: from azext_aks_preview.azurecontainerstorage._validators import ( validate_enable_azure_container_storage_params ) + from azext_aks_preview.azurecontainerstorage._consts import ( + CONST_ACSTOR_IO_ENGINE_LABEL_KEY, + CONST_ACSTOR_IO_ENGINE_LABEL_VAL, + CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY, + CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD, + ) + + default_ephemeral_disk_volume_type = CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY + default_ephemeral_disk_nvme_perf_tier = CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD validate_enable_azure_container_storage_params( pool_type, pool_name, @@ -3380,13 +3408,13 @@ def set_up_azure_container_storage(self, mc: ManagedCluster) -> ManagedCluster: False, False, False, + ephemeral_disk_volume_type, + ephemeral_disk_nvme_perf_tier, + default_ephemeral_disk_volume_type, + default_ephemeral_disk_nvme_perf_tier, ) # Setup Azure Container Storage labels on the nodepool - from azext_aks_preview.azurecontainerstorage._consts import ( - CONST_ACSTOR_IO_ENGINE_LABEL_KEY, - CONST_ACSTOR_IO_ENGINE_LABEL_VAL - ) nodepool_labels = agentpool.node_labels if nodepool_labels is None: nodepool_labels = {} @@ -3396,6 +3424,16 @@ def set_up_azure_container_storage(self, mc: ManagedCluster) -> ManagedCluster: # set intermediates self.context.set_intermediate("enable_azure_container_storage", True, overwrite_exists=True) self.context.set_intermediate("azure_container_storage_nodepools", nodepool_list, overwrite_exists=True) + self.context.set_intermediate( + "current_ephemeral_nvme_perf_tier", + default_ephemeral_disk_nvme_perf_tier, + overwrite_exists=True + ) + self.context.set_intermediate( + "existing_ephemeral_disk_volume_type", + default_ephemeral_disk_volume_type, + overwrite_exists=True + ) return mc @@ -3839,6 +3877,10 @@ def postprocessing_after_mc_created(self, cluster: ManagedCluster) -> None: pool_option = self.context.raw_param.get("storage_pool_option") pool_sku = self.context.raw_param.get("storage_pool_sku") pool_size = self.context.raw_param.get("storage_pool_size") + ephemeral_disk_volume_type = self.context.raw_param.get("ephemeral_disk_volume_type") + ephemeral_disk_nvme_perf_tier = self.context.raw_param.get("ephemeral_disk_nvme_perf_tier") + existing_ephemeral_disk_volume_type = self.context.get_intermediate("existing_ephemeral_disk_volume_type") + existing_ephemeral_nvme_perf_tier = self.context.get_intermediate("current_ephemeral_nvme_perf_tier") kubelet_identity_object_id = cluster.identity_profile["kubeletidentity"].object_id node_resource_group = cluster.node_resource_group agent_pool_vm_sizes = [] @@ -3860,7 +3902,11 @@ def postprocessing_after_mc_created(self, cluster: ManagedCluster) -> None: pool_sku, pool_option, agent_pool_vm_sizes, + ephemeral_disk_volume_type, + ephemeral_disk_nvme_perf_tier, True, + existing_ephemeral_disk_volume_type, + existing_ephemeral_nvme_perf_tier, is_called_from_extension=True, ) @@ -4047,12 +4093,27 @@ def update_azure_container_storage(self, mc: ManagedCluster) -> ManagedCluster: enable_azure_container_storage = enable_pool_type is not None disable_azure_container_storage = disable_pool_type is not None nodepool_list = self.context.raw_param.get("azure_container_storage_nodepools") + ephemeral_disk_volume_type = self.context.raw_param.get("ephemeral_disk_volume_type") + ephemeral_disk_nvme_perf_tier = self.context.raw_param.get("ephemeral_disk_nvme_perf_tier") if enable_azure_container_storage and disable_azure_container_storage: raise MutuallyExclusiveArgumentError( 'Conflicting flags. Cannot set --enable-azure-container-storage ' 'and --disable-azure-container-storage together.' ) + if (ephemeral_disk_volume_type is not None or ephemeral_disk_nvme_perf_tier is not None) and \ + not enable_azure_container_storage: + params_defined_arr = [] + if ephemeral_disk_volume_type is not None: + params_defined_arr.append('--ephemeral-disk-volume-type') + if ephemeral_disk_nvme_perf_tier is not None: + params_defined_arr.append('--ephemeral-disk-nvme-perf-tier') + + params_defined = 'and '.join(params_defined_arr) + raise RequiredArgumentMissingError( + f'Cannot set {params_defined} without the parameter --enable-azure-container-storage.' + ) + # pylint: disable=too-many-nested-blocks if enable_azure_container_storage or disable_azure_container_storage: # Require the agent pool profiles for azure container storage @@ -4075,6 +4136,8 @@ def update_azure_container_storage(self, mc: ManagedCluster) -> ManagedCluster: is_ephemeralDisk_localssd_enabled, is_ephemeralDisk_nvme_enabled, current_core_value, + existing_ephemeral_disk_volume_type, + existing_perf_tier, ) = get_extension_installed_and_cluster_configs( self.cmd, self.context.get_resource_group_name(), @@ -4091,8 +4154,12 @@ def update_azure_container_storage(self, mc: ManagedCluster) -> ManagedCluster: for agentpool in mc.agent_pool_profiles: pool_details = {} pool_details["vm_size"] = agentpool.vm_size + pool_details["count"] = agentpool.count node_name = agentpool.name pool_details["name"] = node_name + pool_details["os_type"] = agentpool.os_type + pool_details["mode"] = agentpool.mode + pool_details["node_taints"] = agentpool.node_taints if agentpool.node_labels is not None: node_labels = agentpool.node_labels if node_labels is not None and \ @@ -4109,7 +4176,7 @@ def update_azure_container_storage(self, mc: ManagedCluster) -> ManagedCluster: # one nodepool exists, choose the only nodepool by default. if not is_extension_installed: if nodepool_list is None: - nodepool_list = "nodepool1" + nodepool_list = "" if len(labelled_nodepool_arr) > 0: nodepool_list = ','.join(labelled_nodepool_arr) elif len(agentpool_details) == 1: @@ -4132,8 +4199,37 @@ def update_azure_container_storage(self, mc: ManagedCluster) -> ManagedCluster: is_elasticSan_enabled, is_ephemeralDisk_localssd_enabled, is_ephemeralDisk_nvme_enabled, + ephemeral_disk_volume_type, + ephemeral_disk_nvme_perf_tier, + existing_ephemeral_disk_volume_type, + existing_perf_tier, ) + if is_ephemeralDisk_nvme_enabled and ephemeral_disk_nvme_perf_tier is not None: + # Adding this intermediate and check to ensure that the below + # message prompt doesn't appear twice when aks-preview extension + # is called from both update_mc_profile_preview and update_mc_profile_default. + is_azure_container_storage_perf_tier_op_set = self.context.get_intermediate( + "azure_container_storage_perf_tier_op_set", + default_value="default", + ) + + if is_azure_container_storage_perf_tier_op_set == "default": + msg = ( + "Changing ephemeralDisk NVMe performance tier may result in a temporary " + "interruption to the applications using Azure Container Storage. Do you " + "want to continue with this operation?" + ) + + if not (self.context.get_yes() or prompt_y_n(msg, default="n")): + raise DecoratorEarlyExitException() + + self.context.set_intermediate( + "azure_container_storage_perf_tier_op_set", + True, + overwrite_exists=True + ) + # If the extension is already installed, # we expect that the Azure Container Storage # nodes are already labelled. Use those label @@ -4159,6 +4255,7 @@ def update_azure_container_storage(self, mc: ManagedCluster) -> ManagedCluster: # set intermediates self.context.set_intermediate("azure_container_storage_nodepools", nodepool_list, overwrite_exists=True) self.context.set_intermediate("enable_azure_container_storage", True, overwrite_exists=True) + if disable_azure_container_storage: from azext_aks_preview.azurecontainerstorage._validators import ( validate_disable_azure_container_storage_params @@ -4175,6 +4272,8 @@ def update_azure_container_storage(self, mc: ManagedCluster) -> ManagedCluster: is_elasticSan_enabled, is_ephemeralDisk_localssd_enabled, is_ephemeralDisk_nvme_enabled, + ephemeral_disk_volume_type, + ephemeral_disk_nvme_perf_tier, ) is_pre_disable_validate_set = self.context.get_intermediate( @@ -4222,6 +4321,17 @@ def update_azure_container_storage(self, mc: ManagedCluster) -> ManagedCluster: self.context.set_intermediate("is_extension_installed", is_extension_installed, overwrite_exists=True) self.context.set_intermediate("is_azureDisk_enabled", is_azureDisk_enabled, overwrite_exists=True) self.context.set_intermediate("is_elasticSan_enabled", is_elasticSan_enabled, overwrite_exists=True) + self.context.set_intermediate("current_core_value", current_core_value, overwrite_exists=True) + self.context.set_intermediate( + "current_ephemeral_nvme_perf_tier", + existing_perf_tier, + overwrite_exists=True + ) + self.context.set_intermediate( + "existing_ephemeral_disk_volume_type", + existing_ephemeral_disk_volume_type, + overwrite_exists=True + ) self.context.set_intermediate( "is_ephemeralDisk_nvme_enabled", is_ephemeralDisk_nvme_enabled, @@ -4232,7 +4342,6 @@ def update_azure_container_storage(self, mc: ManagedCluster) -> ManagedCluster: is_ephemeralDisk_localssd_enabled, overwrite_exists=True ) - self.context.set_intermediate("current_core_value", current_core_value, overwrite_exists=True) return mc @@ -5308,6 +5417,8 @@ def postprocessing_after_mc_created(self, cluster: ManagedCluster) -> None: is_ephemeralDisk_localssd_enabled = self.context.get_intermediate("is_ephemeralDisk_localssd_enabled") is_ephemeralDisk_nvme_enabled = self.context.get_intermediate("is_ephemeralDisk_nvme_enabled") current_core_value = self.context.get_intermediate("current_core_value") + existing_ephemeral_disk_volume_type = self.context.get_intermediate("existing_ephemeral_disk_volume_type") + existing_ephemeral_nvme_perf_tier = self.context.get_intermediate("current_ephemeral_nvme_perf_tier") pool_option = self.context.raw_param.get("storage_pool_option") # enable azure container storage @@ -5323,6 +5434,8 @@ def postprocessing_after_mc_created(self, cluster: ManagedCluster) -> None: pool_sku = self.context.raw_param.get("storage_pool_sku") pool_size = self.context.raw_param.get("storage_pool_size") nodepool_list = self.context.get_intermediate("azure_container_storage_nodepools") + ephemeral_disk_volume_type = self.context.raw_param.get("ephemeral_disk_volume_type") + ephemeral_disk_nvme_perf_tier = self.context.raw_param.get("ephemeral_disk_nvme_perf_tier") kubelet_identity_object_id = cluster.identity_profile["kubeletidentity"].object_id acstor_nodepool_skus = [] for agentpool_profile in cluster.agent_pool_profiles: @@ -5342,7 +5455,11 @@ def postprocessing_after_mc_created(self, cluster: ManagedCluster) -> None: pool_sku, pool_option, acstor_nodepool_skus, + ephemeral_disk_volume_type, + ephemeral_disk_nvme_perf_tier, False, + existing_ephemeral_disk_volume_type, + existing_ephemeral_nvme_perf_tier, is_extension_installed, is_azureDisk_enabled, is_elasticSan_enabled, @@ -5372,6 +5489,8 @@ def postprocessing_after_mc_created(self, cluster: ManagedCluster) -> None: is_ephemeralDisk_localssd_enabled, is_ephemeralDisk_nvme_enabled, current_core_value, + existing_ephemeral_disk_volume_type, + existing_ephemeral_nvme_perf_tier, is_called_from_extension=True, ) diff --git a/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py b/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py index 9d39b07fa24..074a4005bfc 100644 --- a/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py +++ b/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py @@ -10937,6 +10937,61 @@ def test_aks_create_with_azurecontainerstorage( ], ) + # live only due to downloading k8s-extension extension + @live_only() + @AllowLargeResponse(8192) + @AKSCustomResourceGroupPreparer( + random_name_length=17, name_prefix="clitest", location="westus2" + ) + def test_aks_create_with_azurecontainerstorage_with_ephemeral_disk_parameters( + self, resource_group, resource_group_location + ): + # reset the count so in replay mode the random names will start with 0 + self.test_resources_count = 0 + # kwargs for string formatting + aks_name = self.create_random_name("cliakstest", 16) + + node_vm_size = "standard_l8s_v3" + self.kwargs.update( + { + "resource_group": resource_group, + "name": aks_name, + "location": resource_group_location, + "resource_type": "Microsoft.ContainerService/ManagedClusters", + "ssh_key_value": self.generate_ssh_keys(), + "node_vm_size": node_vm_size, + } + ) + + # add k8s-extension extension for azurecontainerstorage operations. + self.cmd("extension add --name k8s-extension") + + create_cmd = ( + "aks create --resource-group={resource_group} --name={name} --location={location} --ssh-key-value={ssh_key_value} --node-vm-size={node_vm_size} " + "--node-count 3 --enable-managed-identity --enable-azure-container-storage ephemeralDisk --storage-pool-option NVMe " + "--ephemeral-disk-volume-type EphemeralVolumeOnly --ephemeral-disk-nvme-perf-tier Premium --output=json" + ) + + # enabling azurecontainerstorage will not affect any field in the cluster. + # the only check we should perform is to verify that the cluster is provisioned successfully. + self.cmd( + create_cmd, + checks=[ + self.check("provisioningState", "Succeeded"), + ], + ) + + # delete + cmd = ( + "aks delete --resource-group={resource_group} --name={name} --yes --no-wait" + ) + self.cmd( + cmd, + checks=[ + self.is_empty(), + ], + ) + # live only due to downloading k8s-extension extension @live_only() @AllowLargeResponse(8192) @@ -11164,6 +11219,60 @@ def test_aks_update_with_azurecontainerstorage(self, resource_group, resource_gr self.is_empty(), ]) + @live_only() + @AllowLargeResponse(8192) + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + def test_aks_update_with_azurecontainerstorage_with_ephemeral_disk_parameters(self, resource_group, resource_group_location): + aks_name = self.create_random_name('cliakstest', 16) + node_vm_size = 'standard_l8s_v3' + self.kwargs.update({ + 'resource_group': resource_group, + 'name': aks_name, + 'location': resource_group_location, + 'ssh_key_value': self.generate_ssh_keys(), + 'node_vm_size': node_vm_size, + }) + + # add k8s-extension extension for azurecontainerstorage operations. + self.cmd('extension add --name k8s-extension') + + # create: without enable-azure-container-storage + create_cmd = 'aks create --resource-group={resource_group} --name={name} --location={location} --ssh-key-value={ssh_key_value} --node-vm-size={node_vm_size} --node-count 3 --enable-managed-identity --output=json' + self.cmd(create_cmd, checks=[ + self.check('provisioningState', 'Succeeded'), + ]) + + # enabling or disabling azurecontainerstorage will not affect any field in the cluster. + # the only check we should perform is to verify that the cluster is provisioned successfully. + + # update: enable-azure-container-storage + update_cmd = 'aks update --resource-group={resource_group} --name={name} --yes --output=json ' \ + '--enable-azure-container-storage ephemeralDisk --storage-pool-option NVMe ' \ + '--ephemeral-disk-volume-type PersistentVolumeWithAnnotation ' \ + '--ephemeral-disk-nvme-perf-tier Standard' + + self.cmd(update_cmd, checks=[ + self.check('provisioningState', 'Succeeded'), + ]) + + # Sleep for 5 mins before next operation, + # since azure container storage operations take + # some time to post process. + time.sleep(5 * 60) + + # update: disable-azure-container-storage + update_cmd = 'aks update --resource-group={resource_group} --name={name} --yes --output=json ' \ + '--disable-azure-container-storage all' + self.cmd(update_cmd, checks=[ + self.check('provisioningState', 'Succeeded'), + ]) + + # delete + cmd = 'aks delete --resource-group={resource_group} --name={name} --yes --no-wait' + self.cmd(cmd, checks=[ + self.is_empty(), + ]) + # live only due to workspace is not mocked correctly @AllowLargeResponse() @AKSCustomResourceGroupPreparer( diff --git a/src/aks-preview/azext_aks_preview/tests/latest/test_validators.py b/src/aks-preview/azext_aks_preview/tests/latest/test_validators.py index 6dc89e816df..b3fba1c3fbe 100644 --- a/src/aks-preview/azext_aks_preview/tests/latest/test_validators.py +++ b/src/aks-preview/azext_aks_preview/tests/latest/test_validators.py @@ -14,6 +14,7 @@ InvalidArgumentValueError, MutuallyExclusiveArgumentError, RequiredArgumentMissingError, + UnknownError, ) from azure.cli.core.util import CLIError @@ -717,7 +718,7 @@ def test_disable_when_extension_not_installed(self): ) with self.assertRaises(InvalidArgumentValueError) as cm: acstor_validator.validate_disable_azure_container_storage_params( - None, None, None, None, None, None, is_extension_installed, False, False, False, False + None, None, None, None, None, None, is_extension_installed, False, False, False, False, None, None ) self.assertEqual(str(cm.exception), err) @@ -729,7 +730,7 @@ def test_disable_flag_with_storage_pool_name(self): ) with self.assertRaises(MutuallyExclusiveArgumentError) as cm: acstor_validator.validate_disable_azure_container_storage_params( - None, storage_pool_name, None, None, None, None, True, False, False, False, False + None, storage_pool_name, None, None, None, None, True, False, False, False, False, None, None ) self.assertEqual(str(cm.exception), err) @@ -741,7 +742,7 @@ def test_disable_flag_with_storage_pool_sku(self): ) with self.assertRaises(MutuallyExclusiveArgumentError) as cm: acstor_validator.validate_disable_azure_container_storage_params( - None, None, storage_pool_sku, None, None, None, True, False, False, False, False + None, None, storage_pool_sku, None, None, None, True, False, False, False, False, None, None ) self.assertEqual(str(cm.exception), err) @@ -753,7 +754,34 @@ def test_disable_flag_with_storage_pool_size(self): ) with self.assertRaises(MutuallyExclusiveArgumentError) as cm: acstor_validator.validate_disable_azure_container_storage_params( - None, None, None, None, storage_pool_size, None, True, False, False, False, False + None, None, None, None, storage_pool_size, None, True, False, False, False, False, None, None + ) + self.assertEqual(str(cm.exception), err) + + def test_disable_flag_with_ephemeral_disk_volume_type(self): + storage_pool_size = "5Gi" + ephemeral_disk_volume_type = acstor_consts.CONST_DISK_TYPE_PV_WITH_ANNOTATION + err = ( + "Conflicting flags. Cannot define --ephemeral-disk-volume-type value " + "when --disable-azure-container-storage is set." + ) + with self.assertRaises(MutuallyExclusiveArgumentError) as cm: + acstor_validator.validate_disable_azure_container_storage_params( + None, None, None, None, None, None, True, False, False, False, False, ephemeral_disk_volume_type, None + ) + self.assertEqual(str(cm.exception), err) + + def test_disable_flag_with_ephemeral_disk_nvme_perf_tier(self): + storage_pool_size = "5Gi" + ephemeral_disk_volume_type = acstor_consts.CONST_DISK_TYPE_PV_WITH_ANNOTATION + perf_tier = acstor_consts.CONST_EPHEMERAL_NVME_PERF_TIER_PREMIUM + err = ( + "Conflicting flags. Cannot define --ephemeral-disk-nvme-perf-tier value " + "when --disable-azure-container-storage is set." + ) + with self.assertRaises(MutuallyExclusiveArgumentError) as cm: + acstor_validator.validate_disable_azure_container_storage_params( + None, None, None, None, None, None, True, False, False, False, False, None, perf_tier ) self.assertEqual(str(cm.exception), err) @@ -766,7 +794,7 @@ def test_disable_flag_with_storage_pool_option_not_ephemeralDisk(self): ) with self.assertRaises(ArgumentUsageError) as cm: acstor_validator.validate_disable_azure_container_storage_params( - storage_pool_type, None, None, storage_pool_option, None, None, True, False, False, False, False + storage_pool_type, None, None, storage_pool_option, None, None, True, False, False, False, False, None, None ) self.assertEqual(str(cm.exception), err) @@ -778,7 +806,7 @@ def test_disable_flag_with_storage_pool_option_not_set_both_ephemeralDisk_enable ) with self.assertRaises(RequiredArgumentMissingError) as cm: acstor_validator.validate_disable_azure_container_storage_params( - storage_pool_type, None, None, None, None, None, True, False, False, True, True + storage_pool_type, None, None, None, None, None, True, False, False, True, True, None, None ) self.assertEqual(str(cm.exception), err) @@ -791,7 +819,7 @@ def test_disable_flag_with_nodepool_list(self): ) with self.assertRaises(MutuallyExclusiveArgumentError) as cm: acstor_validator.validate_disable_azure_container_storage_params( - storage_pool_type, None, None, None, None, nodepool_list, True, False, False, False, False + storage_pool_type, None, None, None, None, nodepool_list, True, False, False, False, False, None, None ) self.assertEqual(str(cm.exception), err) @@ -805,7 +833,7 @@ def test_disable_type_when_not_enabled(self): ) with self.assertRaises(ArgumentUsageError) as cm: acstor_validator.validate_disable_azure_container_storage_params( - pool_type, None, None, None, None, None, True, is_azureDisk_enabled, False, False, False + pool_type, None, None, None, None, None, True, is_azureDisk_enabled, False, False, False, None, None ) self.assertEqual(str(cm.exception), err) @@ -818,7 +846,7 @@ def test_disable_only_storage_pool_installed(self): ) with self.assertRaises(ArgumentUsageError) as cm: acstor_validator.validate_disable_azure_container_storage_params( - pool_type, None, None, None, None, None, True, True, False, False, False + pool_type, None, None, None, None, None, True, True, False, False, False, None, None ) self.assertEqual(str(cm.exception), err) @@ -832,14 +860,14 @@ def test_disable_only_storagepool_type_enabled(self): ) with self.assertRaises(ArgumentUsageError) as cm: acstor_validator.validate_disable_azure_container_storage_params( - pool_type, None, None, None, None, None, True, is_azureDisk_enabled, False, False, False + pool_type, None, None, None, None, None, True, is_azureDisk_enabled, False, False, False, None, None ) self.assertEqual(str(cm.exception), err) def test_valid_disable(self): pool_type = acstor_consts.CONST_STORAGE_POOL_TYPE_ELASTIC_SAN acstor_validator.validate_disable_azure_container_storage_params( - pool_type, None, None, None, None, None, True, False, True, True, False + pool_type, None, None, None, None, None, True, False, True, True, False, None, None ) @@ -853,7 +881,7 @@ def test_enable_with_invalid_storage_pool_name(self): ) with self.assertRaises(InvalidArgumentValueError) as cm: acstor_validator.validate_enable_azure_container_storage_params( - None, storage_pool_name, None, None, None, None, None, False, False, False, False, False + None, storage_pool_name, None, None, None, None, None, False, False, False, False, False, None, None, acstor_consts.CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY, acstor_consts.CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD ) self.assertEqual(str(cm.exception), err) @@ -864,7 +892,7 @@ def test_enable_with_sku_and_ephemeral_disk_pool(self): err = "Cannot set --storage-pool-sku when --enable-azure-container-storage is ephemeralDisk." with self.assertRaises(ArgumentUsageError) as cm: acstor_validator.validate_enable_azure_container_storage_params( - storage_pool_type, storage_pool_name, storage_pool_sku, None, None, None, None, False, False, False, False, False + storage_pool_type, storage_pool_name, storage_pool_sku, None, None, None, None, False, False, False, False, False, None, None, acstor_consts.CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY, acstor_consts.CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD ) self.assertEqual(str(cm.exception), err) @@ -886,7 +914,7 @@ def test_enable_with_sku_and_elastic_san_pool(self): ) with self.assertRaises(ArgumentUsageError) as cm: acstor_validator.validate_enable_azure_container_storage_params( - storage_pool_type, storage_pool_name, storage_pool_sku, None, None, None, None, False, False, False, False, False + storage_pool_type, storage_pool_name, storage_pool_sku, None, None, None, None, False, False, False, False, False, None, None, acstor_consts.CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY, acstor_consts.CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD ) self.assertEqual(str(cm.exception), err) @@ -897,7 +925,47 @@ def test_enable_with_option_and_non_ephemeral_disk_pool(self): err = "Cannot set --storage-pool-option when --enable-azure-container-storage is not ephemeralDisk." with self.assertRaises(ArgumentUsageError) as cm: acstor_validator.validate_enable_azure_container_storage_params( - storage_pool_type, storage_pool_name, None, storage_pool_option, None, None, None, False, False, False, False, False + storage_pool_type, storage_pool_name, None, storage_pool_option, None, None, None, False, False, False, False, False, None, None, acstor_consts.CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY, acstor_consts.CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD + ) + self.assertEqual(str(cm.exception), err) + + def test_enable_with_ephemeral_disk_volume_type_and_non_ephemeral_disk_pool(self): + storage_pool_name = "valid-name" + ephemeral_disk_volume_type = acstor_consts.CONST_DISK_TYPE_PV_WITH_ANNOTATION + storage_pool_type = acstor_consts.CONST_STORAGE_POOL_TYPE_AZURE_DISK + err = "Cannot set --ephemeral-disk-volume-type when --enable-azure-container-storage is not ephemeralDisk." + with self.assertRaises(ArgumentUsageError) as cm: + acstor_validator.validate_enable_azure_container_storage_params( + storage_pool_type, storage_pool_name, None, None, None, None, None, False, False, False, False, False, ephemeral_disk_volume_type, None, acstor_consts.CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY, acstor_consts.CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD + ) + self.assertEqual(str(cm.exception), err) + + def test_enable_with_ephemeral_disk_nvme_perf_tier_and_non_ephemeral_disk_pool(self): + storage_pool_name = "valid-name" + perf_tier = acstor_consts.CONST_EPHEMERAL_NVME_PERF_TIER_PREMIUM + storage_pool_type = acstor_consts.CONST_STORAGE_POOL_TYPE_AZURE_DISK + err = ( + "Cannot set --ephemeral-disk-nvme-perf-tier when --enable-azure-container-storage is not ephemeralDisk." + ) + with self.assertRaises(ArgumentUsageError) as cm: + acstor_validator.validate_enable_azure_container_storage_params( + storage_pool_type, storage_pool_name, None, None, None, None, None, False, False, False, False, False, None, perf_tier, acstor_consts.CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY, acstor_consts.CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD + ) + self.assertEqual(str(cm.exception), err) + + def test_enable_with_ephemeral_disk_nvme_perf_tier_and_ephemeral_temp_disk_pool(self): + storage_pool_name = "valid-name" + perf_tier = acstor_consts.CONST_EPHEMERAL_NVME_PERF_TIER_PREMIUM + storage_pool_type = acstor_consts.CONST_STORAGE_POOL_TYPE_EPHEMERAL_DISK + storage_pool_option = acstor_consts.CONST_STORAGE_POOL_OPTION_SSD + err = ( + "Cannot set --ephemeral-disk-nvme-perf-tier along with --enable-azure-container-storage " + "when storage pool type: ephemeralDisk option: NVMe is not enabled for Azure Container Storage. " + "Enable the option using --storage-pool-option." + ) + with self.assertRaises(ArgumentUsageError) as cm: + acstor_validator.validate_enable_azure_container_storage_params( + storage_pool_type, storage_pool_name, None, storage_pool_option, None, None, None, False, False, False, False, False, None, perf_tier, acstor_consts.CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY, acstor_consts.CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD ) self.assertEqual(str(cm.exception), err) @@ -908,7 +976,7 @@ def test_enable_with_option_all_and_ephemeral_disk_pool(self): err = "Cannot set --storage-pool-option value as all when --enable-azure-container-storage is set." with self.assertRaises(InvalidArgumentValueError) as cm: acstor_validator.validate_enable_azure_container_storage_params( - storage_pool_type, storage_pool_name, None, storage_pool_option, None, None, None, False, False, False, False, False + storage_pool_type, storage_pool_name, None, storage_pool_option, None, None, None, False, False, False, False, False, None, None, acstor_consts.CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY, acstor_consts.CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD ) self.assertEqual(str(cm.exception), err) @@ -919,7 +987,7 @@ def test_enable_with_invalid_storage_pool_size(self): err = "Value for --storage-pool-size should be defined with size followed by Gi or Ti e.g. 512Gi or 2Ti." with self.assertRaises(ArgumentUsageError) as cm: acstor_validator.validate_enable_azure_container_storage_params( - storage_pool_type, storage_pool_name, None, None, storage_pool_size, None, None, False, False, False, False, False + storage_pool_type, storage_pool_name, None, None, storage_pool_size, None, None, False, False, False, False, False, None, None, acstor_consts.CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY, acstor_consts.CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD ) self.assertEqual(str(cm.exception), err) @@ -930,7 +998,7 @@ def test_enable_with_invalid_size_for_esan_storage_pool(self): err = "Value for --storage-pool-size must be at least 1Ti when --enable-azure-container-storage is elasticSan." with self.assertRaises(ArgumentUsageError) as cm: acstor_validator.validate_enable_azure_container_storage_params( - storage_pool_type, storage_pool_name, None, None, storage_pool_size, None, None, False, False, False, False, False + storage_pool_type, storage_pool_name, None, None, storage_pool_size, None, None, False, False, False, False, False, None, None, acstor_consts.CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY, acstor_consts.CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD ) self.assertEqual(str(cm.exception), err) @@ -947,7 +1015,7 @@ def test_invalid_comma_separated_nodepool_list(self): ) with self.assertRaises(InvalidArgumentValueError) as cm: acstor_validator.validate_enable_azure_container_storage_params( - storage_pool_type, storage_pool_name, None, None, storage_pool_size, nodepool_list, None, False, False, False, False, False + storage_pool_type, storage_pool_name, None, None, storage_pool_size, nodepool_list, None, False, False, False, False, False, None, None, acstor_consts.CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY, acstor_consts.CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD ) self.assertEqual(str(cm.exception), err) @@ -966,7 +1034,7 @@ def test_missing_nodepool_from_cluster_nodepool_list_single(self): ) with self.assertRaises(InvalidArgumentValueError) as cm: acstor_validator.validate_enable_azure_container_storage_params( - storage_pool_type, storage_pool_name, None, storage_pool_option, storage_pool_size, nodepool_list, agentpools, False, False, False, False, False + storage_pool_type, storage_pool_name, None, storage_pool_option, storage_pool_size, nodepool_list, agentpools, False, False, False, False, False, None, None, acstor_consts.CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY, acstor_consts.CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD ) self.assertEqual(str(cm.exception), err) @@ -985,7 +1053,41 @@ def test_missing_nodepool_from_cluster_nodepool_list_multiple(self): ) with self.assertRaises(InvalidArgumentValueError) as cm: acstor_validator.validate_enable_azure_container_storage_params( - storage_pool_type, storage_pool_name, None, storage_pool_option, storage_pool_size, nodepool_list, agentpools, False, False, False, False, False + storage_pool_type, storage_pool_name, None, storage_pool_option, storage_pool_size, nodepool_list, agentpools, False, False, False, False, False, None, None, acstor_consts.CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY, acstor_consts.CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD + ) + self.assertEqual(str(cm.exception), err) + + def test_system_nodepool_with_taint(self): + storage_pool_name = "valid-name" + storage_pool_size = "5Ti" + storage_pool_type = acstor_consts.CONST_STORAGE_POOL_TYPE_EPHEMERAL_DISK + storage_pool_option = acstor_consts.CONST_STORAGE_POOL_OPTION_SSD + nodepool_list = "nodepool1" + agentpools = [{"name": "nodepool1", "mode": "System", "node_taints": ["CriticalAddonsOnly=true:NoSchedule"]}, {"name": "nodepool2", "count": 1}] + err = ( + 'Unable to install Azure Container Storage on system nodepool: nodepool1 ' + 'since it has a taint CriticalAddonsOnly=true:NoSchedule. Remove the taint from the node pool ' + 'and retry the Azure Container Storage operation.' + ) + with self.assertRaises(InvalidArgumentValueError) as cm: + acstor_validator.validate_enable_azure_container_storage_params( + storage_pool_type, storage_pool_name, None, storage_pool_option, storage_pool_size, nodepool_list, agentpools, False, False, False, False, False, None, None, acstor_consts.CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY, acstor_consts.CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD + ) + self.assertEqual(str(cm.exception), err) + + def test_nodepool_from_cluster_nodepool_list_with_insufficient_count(self): + storage_pool_name = "valid-name" + storage_pool_size = "5Ti" + storage_pool_type = acstor_consts.CONST_STORAGE_POOL_TYPE_EPHEMERAL_DISK + storage_pool_option = acstor_consts.CONST_STORAGE_POOL_OPTION_SSD + nodepool_list = "nodepool1,nodepool2" + agentpools = [{"name": "nodepool1", "count": 1}, {"name": "nodepool2", "count": 1}] + err = ( + "Insufficient nodes present. Azure Container Storage requires atleast 3 nodes to be enabled." + ) + with self.assertRaises(UnknownError) as cm: + acstor_validator.validate_enable_azure_container_storage_params( + storage_pool_type, storage_pool_name, None, storage_pool_option, storage_pool_size, nodepool_list, agentpools, False, False, False, False, False, None, None, acstor_consts.CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY, acstor_consts.CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD ) self.assertEqual(str(cm.exception), err) @@ -995,9 +1097,9 @@ def test_valid_enable_for_azure_disk_pool(self): storage_pool_type = acstor_consts.CONST_STORAGE_POOL_TYPE_AZURE_DISK storage_pool_sku = acstor_consts.CONST_STORAGE_POOL_SKU_PREMIUM_LRS nodepool_list = "nodepool1,nodepool2" - agentpools = [{"name": "nodepool1"}, {"name": "nodepool2"}] + agentpools = [{"name": "nodepool1", "mode": "User", "count": 2}, {"name": "nodepool2", "mode": "System", "count": 1}] acstor_validator.validate_enable_azure_container_storage_params( - storage_pool_type, storage_pool_name, storage_pool_sku, None, storage_pool_size, nodepool_list, agentpools, False, False, False, False, False + storage_pool_type, storage_pool_name, storage_pool_sku, None, storage_pool_size, nodepool_list, agentpools, False, False, False, False, False, None, None, acstor_consts.CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY, acstor_consts.CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD ) def test_valid_enable_for_ephemeral_disk_pool(self): @@ -1006,9 +1108,49 @@ def test_valid_enable_for_ephemeral_disk_pool(self): storage_pool_type = acstor_consts.CONST_STORAGE_POOL_TYPE_EPHEMERAL_DISK storage_pool_option = acstor_consts.CONST_STORAGE_POOL_OPTION_NVME nodepool_list = "nodepool1" - agentpools = [{"name": "nodepool1", "vm_size": "Standard_L8s_v3"}, {"name": "nodepool2", "vm_size": "Standard_L8s_v3"}] + agentpools = [{"name": "nodepool1", "vm_size": "Standard_L8s_v3", "mode": "System", "count": 5}, {"name": "nodepool2", "vm_size": "Standard_L8s_v3"}] + acstor_validator.validate_enable_azure_container_storage_params( + storage_pool_type, storage_pool_name, None, storage_pool_option, storage_pool_size, nodepool_list, agentpools, False, False, False, False, False, None, None, acstor_consts.CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY, acstor_consts.CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD + ) + + def test_valid_enable_for_ephemeral_disk_pool_with_ephemeral_disk_volume_type(self): + storage_pool_name = "valid-name" + storage_pool_size = "5Ti" + storage_pool_type = acstor_consts.CONST_STORAGE_POOL_TYPE_EPHEMERAL_DISK + storage_pool_option = acstor_consts.CONST_STORAGE_POOL_OPTION_NVME + nodepool_list = "nodepool1" + ephemeral_disk_volume_type = acstor_consts.CONST_DISK_TYPE_PV_WITH_ANNOTATION + agentpools = [{"name": "nodepool1", "vm_size": "Standard_L8s_v3", "mode": "System", "count": 3}, {"name": "nodepool2", "vm_size": "Standard_L8s_v3"}] + acstor_validator.validate_enable_azure_container_storage_params( + storage_pool_type, storage_pool_name, None, storage_pool_option, storage_pool_size, nodepool_list, agentpools, False, False, False, False, False, ephemeral_disk_volume_type, None, acstor_consts.CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY, acstor_consts.CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD + ) + + def test_valid_enable_for_ephemeral_disk_pool_with_ephemeral_disk_volume_type_already_installed(self): + storage_pool_type = acstor_consts.CONST_STORAGE_POOL_TYPE_EPHEMERAL_DISK + ephemeral_disk_volume_type = acstor_consts.CONST_DISK_TYPE_PV_WITH_ANNOTATION + agentpools = [{"name": "nodepool1", "node_labels": {"acstor.azure.com/io-engine": "acstor"}, "count": 3}, {"name": "nodepool2"}] + acstor_validator.validate_enable_azure_container_storage_params( + storage_pool_type, None, None, None, None, None, agentpools, True, False, False, True, False, ephemeral_disk_volume_type, None, acstor_consts.CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY, acstor_consts.CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD + ) + + def test_valid_enable_for_ephemeral_disk_pool_with_ephemeral_disk_nvme_perf_tier(self): + storage_pool_name = "valid-name" + storage_pool_size = "5Ti" + storage_pool_type = acstor_consts.CONST_STORAGE_POOL_TYPE_EPHEMERAL_DISK + storage_pool_option = acstor_consts.CONST_STORAGE_POOL_OPTION_NVME + nodepool_list = "nodepool1" + perf_tier = acstor_consts.CONST_EPHEMERAL_NVME_PERF_TIER_PREMIUM + agentpools = [{"name": "nodepool1", "vm_size": "Standard_L8s_v3", "count": 4}, {"name": "nodepool2", "vm_size": "Standard_L8s_v3"}] + acstor_validator.validate_enable_azure_container_storage_params( + storage_pool_type, storage_pool_name, None, storage_pool_option, storage_pool_size, nodepool_list, agentpools, False, False, False, False, False, None, perf_tier, acstor_consts.CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY, acstor_consts.CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD + ) + + def test_valid_enable_for_ephemeral_disk_pool_with_azure_container_storage_per_tier_nvme_already_installed(self): + storage_pool_type = acstor_consts.CONST_STORAGE_POOL_TYPE_EPHEMERAL_DISK + perf_tier = acstor_consts.CONST_EPHEMERAL_NVME_PERF_TIER_PREMIUM + agentpools = [{"name": "nodepool1", "node_labels": {"acstor.azure.com/io-engine": "acstor"}, "count": 3}, {"name": "nodepool2"}] acstor_validator.validate_enable_azure_container_storage_params( - storage_pool_type, storage_pool_name, None, storage_pool_option, storage_pool_size, nodepool_list, agentpools, False, False, False, False, False + storage_pool_type, None, None, None, None, None, agentpools, True, False, False, False, True, None, perf_tier, acstor_consts.CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY, acstor_consts.CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD ) def test_extension_installed_nodepool_list_defined(self): @@ -1021,7 +1163,7 @@ def test_extension_installed_nodepool_list_defined(self): ) with self.assertRaises(ArgumentUsageError) as cm: acstor_validator.validate_enable_azure_container_storage_params( - storage_pool_type, None, None, None, None, nodepool_list, None, True, False, False, False, False + storage_pool_type, None, None, None, None, nodepool_list, None, True, False, False, False, False, None, None, acstor_consts.CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY, acstor_consts.CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD ) self.assertEqual(str(cm.exception), err) @@ -1030,7 +1172,7 @@ def test_extension_installed_storagepool_type_installed(self): storage_pool_size = "5Ti" storage_pool_type = acstor_consts.CONST_STORAGE_POOL_TYPE_AZURE_DISK storage_pool_sku = acstor_consts.CONST_STORAGE_POOL_SKU_PREMIUM_LRS - agentpools = [{"name": "nodepool1", "node_labels": {"acstor.azure.com/io-engine": "acstor"}}, {"name": "nodepool2"}] + agentpools = [{"name": "nodepool1", "node_labels": {"acstor.azure.com/io-engine": "acstor"}, "count": 3}, {"name": "nodepool2"}] err = ( "Invalid --enable-azure-container-storage value. " "Azure Container Storage is already enabled for storagepool type " @@ -1038,7 +1180,7 @@ def test_extension_installed_storagepool_type_installed(self): ) with self.assertRaises(ArgumentUsageError) as cm: acstor_validator.validate_enable_azure_container_storage_params( - storage_pool_type, storage_pool_name, storage_pool_sku, None, storage_pool_size, None, agentpools, True, True, False, False, False + storage_pool_type, storage_pool_name, storage_pool_sku, None, storage_pool_size, None, agentpools, True, True, False, False, False, None, None, acstor_consts.CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY, acstor_consts.CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD ) self.assertEqual(str(cm.exception), err) @@ -1047,9 +1189,9 @@ def test_valid_cluster_update(self): storage_pool_size = "5Ti" storage_pool_type = acstor_consts.CONST_STORAGE_POOL_TYPE_AZURE_DISK storage_pool_sku = acstor_consts.CONST_STORAGE_POOL_SKU_PREMIUM_LRS - agentpools = [{"name": "nodepool1", "node_labels": {"acstor.azure.com/io-engine": "acstor"}}, {"name": "nodepool2"}] + agentpools = [{"name": "nodepool1", "node_labels": {"acstor.azure.com/io-engine": "acstor"}, "mode": "User", "count": 3}, {"name": "nodepool2"}] acstor_validator.validate_enable_azure_container_storage_params( - storage_pool_type, storage_pool_name, storage_pool_sku, None, storage_pool_size, None, agentpools, True, False, False, False, False + storage_pool_type, storage_pool_name, storage_pool_sku, None, storage_pool_size, None, agentpools, True, False, False, False, False, None, None, acstor_consts.CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY, acstor_consts.CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD ) class GatewayPrefixSizeSpace: diff --git a/src/aks-preview/linter_exclusions.yml b/src/aks-preview/linter_exclusions.yml index dd86f272a33..4cea7de5df5 100644 --- a/src/aks-preview/linter_exclusions.yml +++ b/src/aks-preview/linter_exclusions.yml @@ -60,6 +60,12 @@ aks create: enable_azure_container_storage: rule_exclusions: - option_length_too_long + ephemeral_disk_volume_type: + rule_exclusions: + - option_length_too_long + ephemeral_disk_nvme_perf_tier: + rule_exclusions: + - option_length_too_long enable_ai_toolchain_operator: rule_exclusions: - option_length_too_long @@ -173,6 +179,12 @@ aks update: azure_container_storage_nodepools: rule_exclusions: - option_length_too_long + ephemeral_disk_volume_type: + rule_exclusions: + - option_length_too_long + ephemeral_disk_nvme_perf_tier: + rule_exclusions: + - option_length_too_long enable_ai_toolchain_operator: rule_exclusions: - option_length_too_long diff --git a/src/aks-preview/setup.py b/src/aks-preview/setup.py index c565a30a659..9b90d5ff453 100644 --- a/src/aks-preview/setup.py +++ b/src/aks-preview/setup.py @@ -9,7 +9,7 @@ from setuptools import setup, find_packages -VERSION = "5.0.0b1" +VERSION = "5.0.0b2" CLASSIFIERS = [ "Development Status :: 4 - Beta",