AKS: Add ephemeral disk volume type and nvme perf tier support to Azu…

…re Container Storage (#7730)

src/aks-preview/HISTORY.rst

@@ -11,6 +11,11 @@ To release a new version, please select a new version number (usually plus 1 to
 
 Pending
 +++++++
+
+5.0.0b2
+++++++++
+* Add option `--ephemeral-disk-volume-type` to `az aks create` and `az aks update` for Azure Container Storage operations.
+* Add option `--azure-container-storage-perf-tier` to `az aks create` and `az aks update` to define resource tiers for Azure Container Storage performance.
 * Vendor new SDK and bump API version to 2024-04-02-preview.
 
 5.0.0b1
@@ -19,6 +24,7 @@ Pending
 * Update --enable-advanced-network-observability description to note additional costs and add missing flag to create command.
 * Change default value of `--vm-set-type` to VirtualMachines when `--vm-sizes` is set.
 
+
 4.0.0b5
 ++++++++
 * Add warnings to `az aks mesh` commands for out of support asm revision in use.

src/aks-preview/azext_aks_preview/_params.py

@@ -190,6 +190,11 @@
 )
 from azext_aks_preview.azurecontainerstorage._consts import (
     CONST_ACSTOR_ALL,
+    CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY,
+    CONST_DISK_TYPE_PV_WITH_ANNOTATION,
+    CONST_EPHEMERAL_NVME_PERF_TIER_BASIC,
+    CONST_EPHEMERAL_NVME_PERF_TIER_PREMIUM,
+    CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD,
     CONST_STORAGE_POOL_TYPE_AZURE_DISK,
     CONST_STORAGE_POOL_TYPE_EPHEMERAL_DISK,
     CONST_STORAGE_POOL_TYPE_ELASTIC_SAN,
@@ -362,6 +367,17 @@
     CONST_ACSTOR_ALL,
 ]
 
+ephemeral_disk_volume_types = [
+    CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY,
+    CONST_DISK_TYPE_PV_WITH_ANNOTATION,
+]
+
+ephemeral_disk_nvme_perf_tiers = [
+    CONST_EPHEMERAL_NVME_PERF_TIER_BASIC,
+    CONST_EPHEMERAL_NVME_PERF_TIER_PREMIUM,
+    CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD,
+]
+
 # consts for guardrails level
 node_provisioning_modes = [
     CONST_NODE_PROVISIONING_MODE_MANUAL,
@@ -889,6 +905,16 @@ def load_arguments(self, _):
             arg_type=get_enum_type(storage_pool_options),
             help="set ephemeral disk storage pool option for azure container storage",
         )
+        c.argument(
+            "ephemeral_disk_volume_type",
+            arg_type=get_enum_type(ephemeral_disk_volume_types),
+            help="set ephemeral disk volume type for azure container storage",
+        )
+        c.argument(
+            "ephemeral_disk_nvme_perf_tier",
+            arg_type=get_enum_type(ephemeral_disk_nvme_perf_tiers),
+            help="set ephemeral disk volume type for azure container storage",
+        )
         c.argument(
             "node_provisioning_mode",
             is_preview=True,
@@ -1320,6 +1346,16 @@ def load_arguments(self, _):
             "azure_container_storage_nodepools",
             help="define the comma separated nodepool list to install azure container storage",
         )
+        c.argument(
+            "ephemeral_disk_volume_type",
+            arg_type=get_enum_type(ephemeral_disk_volume_types),
+            help="set ephemeral disk volume type for azure container storage",
+        )
+        c.argument(
+            "ephemeral_disk_nvme_perf_tier",
+            arg_type=get_enum_type(ephemeral_disk_nvme_perf_tiers),
+            help="set ephemeral disk volume type for azure container storage",
+        )
         c.argument(
             "node_provisioning_mode",
             is_preview=True,

src/aks-preview/azext_aks_preview/azurecontainerstorage/_consts.py

@@ -7,6 +7,11 @@
 CONST_ACSTOR_IO_ENGINE_LABEL_KEY = "acstor.azure.com/io-engine"
 CONST_ACSTOR_IO_ENGINE_LABEL_VAL = "acstor"
 CONST_ACSTOR_K8S_EXTENSION_NAME = "microsoft.azurecontainerstorage"
+CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY = "EphemeralVolumeOnly"
+CONST_DISK_TYPE_PV_WITH_ANNOTATION = "PersistentVolumeWithAnnotation"
+CONST_EPHEMERAL_NVME_PERF_TIER_BASIC = "Basic"
+CONST_EPHEMERAL_NVME_PERF_TIER_PREMIUM = "Premium"
+CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD = "Standard"
 CONST_EXT_INSTALLATION_NAME = "azurecontainerstorage"
 CONST_K8S_EXTENSION_CLIENT_FACTORY_MOD_NAME = "azext_k8s_extension._client_factory"
 CONST_K8S_EXTENSION_CUSTOM_MOD_NAME = "azext_k8s_extension.custom"

src/aks-preview/azext_aks_preview/azurecontainerstorage/_helpers.py

@@ -10,6 +10,11 @@
     CONST_ACSTOR_ALL,
     CONST_ACSTOR_IO_ENGINE_LABEL_KEY,
     CONST_ACSTOR_K8S_EXTENSION_NAME,
+    CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY,
+    CONST_DISK_TYPE_PV_WITH_ANNOTATION,
+    CONST_EPHEMERAL_NVME_PERF_TIER_BASIC,
+    CONST_EPHEMERAL_NVME_PERF_TIER_PREMIUM,
+    CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD,
     CONST_EXT_INSTALLATION_NAME,
     CONST_K8S_EXTENSION_CLIENT_FACTORY_MOD_NAME,
     CONST_K8S_EXTENSION_CUSTOM_MOD_NAME,
@@ -144,7 +149,7 @@ def get_extension_installed_and_cluster_configs(
     resource_group,
     cluster_name,
     agentpool_profiles
-) -> Tuple[bool, bool, bool, bool, float]:
+) -> Tuple[bool, bool, bool, bool, bool, float, str, str]:
     client_factory = get_k8s_extension_module(CONST_K8S_EXTENSION_CLIENT_FACTORY_MOD_NAME)
     client = client_factory.cf_k8s_extension_operation(cmd.cli_ctx)
     k8s_extension_custom_mod = get_k8s_extension_module(CONST_K8S_EXTENSION_CUSTOM_MOD_NAME)
@@ -153,6 +158,8 @@ def get_extension_installed_and_cluster_configs(
     is_azureDisk_enabled = False
     is_ephemeralDisk_nvme_enabled = False
     is_ephemeralDisk_localssd_enabled = False
+    ephemeral_disk_volume_type = CONST_DISK_TYPE_EPHEMERAL_VOLUME_ONLY
+    perf_tier = CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD
     resource_cpu_value = -1
 
     try:
@@ -184,6 +191,25 @@ def get_extension_installed_and_cluster_configs(
                     config_settings.get("global.cli.storagePool.ephemeralDisk.temp.enabled", "False") == "True"
                 )
                 cpu_value = config_settings.get("global.cli.resources.ioEngine.cpu", "1")
+                enable_ephemeral_bypass_annotation = (
+                    config_settings.get(
+                        "global.cli.storagePool.ephemeralDisk.enableEphemeralBypassAnnotation", "False"
+                    ) == "True"
+                )
+                perf_tier = config_settings.get(
+                    "global.cli.storagePool.ephemeralDisk.nvme.perfTier",
+                    CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD
+                )
+
+                if perf_tier.lower() == CONST_EPHEMERAL_NVME_PERF_TIER_BASIC.lower():
+                    perf_tier = CONST_EPHEMERAL_NVME_PERF_TIER_BASIC
+                elif perf_tier.lower() == CONST_EPHEMERAL_NVME_PERF_TIER_PREMIUM.lower():
+                    perf_tier = CONST_EPHEMERAL_NVME_PERF_TIER_PREMIUM
+                else:
+                    perf_tier = CONST_EPHEMERAL_NVME_PERF_TIER_STANDARD
+
+                if enable_ephemeral_bypass_annotation:
+                    ephemeral_disk_volume_type = CONST_DISK_TYPE_PV_WITH_ANNOTATION
                 resource_cpu_value = float(cpu_value)
             else:
                 # For versions where "global.cli.activeControl" were not set it signifies
@@ -192,7 +218,7 @@ def get_extension_installed_and_cluster_configs(
                 is_azureDisk_enabled = is_elasticSan_enabled = is_ephemeralDisk_localssd_enabled = True
                 resource_cpu_value = 1
 
-                # Determine if epehemeral NVMe was active based on the labelled nodepools present in cluster.
+                # Determine if ephemeral NVMe was active based on the labelled nodepools present in cluster.
                 for agentpool in agentpool_profiles:
                     vm_size = agentpool.vm_size
                     node_labels = agentpool.node_labels
@@ -211,14 +237,17 @@ def get_extension_installed_and_cluster_configs(
         is_elasticSan_enabled,
         is_ephemeralDisk_localssd_enabled,
         is_ephemeralDisk_nvme_enabled,
-        resource_cpu_value
+        resource_cpu_value,
+        ephemeral_disk_volume_type,
+        perf_tier
     )
 
 
 def get_initial_resource_value_args(
     storage_pool_type,
     storage_pool_option,
     nodepool_skus,
+    ephemeral_nvme_perf_tier,
 ):
     core_value = memory_value = hugepages_value = hugepages_number = 0
     if (storage_pool_type == CONST_STORAGE_POOL_TYPE_AZURE_DISK or
@@ -230,7 +259,7 @@ def get_initial_resource_value_args(
         hugepages_number = 512
     elif (storage_pool_type == CONST_STORAGE_POOL_TYPE_EPHEMERAL_DISK and
           storage_pool_option == CONST_STORAGE_POOL_OPTION_NVME):
-        core_value = _get_cpu_value_based_on_vm_size(nodepool_skus)
+        core_value = _get_ephemeral_nvme_cpu_value_based_on_vm_size_perf_tier(nodepool_skus, ephemeral_nvme_perf_tier)
         memory_value = 2
         hugepages_value = 2
         hugepages_number = 1024
@@ -248,6 +277,7 @@ def get_current_resource_value_args(
     is_elasticSan_enabled,
     is_ephemeralDisk_localssd_enabled,
     is_ephemeralDisk_nvme_enabled,
+    ephemeral_nvme_perf_tier,
     current_core_value=None,
     nodepool_skus=None,
 ):
@@ -261,6 +291,7 @@ def get_current_resource_value_args(
         is_elasticSan_enabled,
         is_ephemeralDisk_localssd_enabled,
         is_ephemeralDisk_nvme_enabled,
+        ephemeral_nvme_perf_tier,
         current_core_value,
         nodepool_skus,
     )
@@ -276,11 +307,13 @@ def get_current_resource_value_args(
 def get_desired_resource_value_args(
     storage_pool_type,
     storage_pool_option,
+    ephemeral_nvme_perf_tier,
     current_core_value,
     is_azureDisk_enabled,
     is_elasticSan_enabled,
     is_ephemeralDisk_localssd_enabled,
     is_ephemeralDisk_nvme_enabled,
+    perf_tier_updated,
     nodepool_skus,
     is_enabling_op,
 ):
@@ -294,6 +327,7 @@ def get_desired_resource_value_args(
         is_elasticSan_enabled,
         is_ephemeralDisk_localssd_enabled,
         is_ephemeralDisk_nvme_enabled,
+        ephemeral_nvme_perf_tier,
         current_core_value,
         nodepool_skus,
     )
@@ -310,17 +344,23 @@ def get_desired_resource_value_args(
             updated_hugepages_value = 1
             updated_hugepages_number = 512
         elif (storage_pool_type == CONST_STORAGE_POOL_TYPE_EPHEMERAL_DISK and
-              storage_pool_option == CONST_STORAGE_POOL_OPTION_NVME):
-            updated_core_value = _get_cpu_value_based_on_vm_size(nodepool_skus)
+              (storage_pool_option == CONST_STORAGE_POOL_OPTION_NVME or
+               is_ephemeralDisk_nvme_enabled or perf_tier_updated)):
+            updated_core_value = _get_ephemeral_nvme_cpu_value_based_on_vm_size_perf_tier(
+                nodepool_skus,
+                ephemeral_nvme_perf_tier,
+            )
             updated_memory_value = 2
             updated_hugepages_value = 2
             updated_hugepages_number = 1024
 
-        # We have decided the updated value based on the type we are enabling.
-        # Now, we compare and check if the current values are already greater
-        # than that and if so, we preserve the current values.
-        updated_core_value = current_core_value \
-            if current_core_value > updated_core_value else updated_core_value
+        if not perf_tier_updated:
+            # For an operation where we are not modifying the perf tiers for nvme,
+            # we have decided the updated value based on the type we are enabling.
+            # Now, we compare and check if the current values are already greater
+            # than that and if so, we preserve the current values.
+            updated_core_value = current_core_value \
+                if current_core_value > updated_core_value else updated_core_value
         updated_memory_value = current_memory_value \
             if current_memory_value > updated_memory_value else updated_memory_value
         updated_hugepages_value = current_hugepages_value \
@@ -345,8 +385,8 @@ def get_desired_resource_value_args(
         )
         is_ephemeral_nvme_disabled_azureDisk_active = (
             storage_pool_type == CONST_STORAGE_POOL_TYPE_EPHEMERAL_DISK and
-            (storage_pool_option == CONST_STORAGE_POOL_OPTION_NVME and
-                (is_ephemeralDisk_localssd_enabled or is_azureDisk_enabled) or
+            ((storage_pool_option == CONST_STORAGE_POOL_OPTION_NVME and
+                (is_ephemeralDisk_localssd_enabled or is_azureDisk_enabled)) or
                 (storage_pool_option == CONST_ACSTOR_ALL and is_azureDisk_enabled))
         )
         if is_disabled_type_smaller_than_active_types:
@@ -399,24 +439,55 @@ def get_cores_from_sku(vm_size):
     return cpu_value
 
 
-def _get_cpu_value_based_on_vm_size(nodepool_skus):
+def check_if_new_storagepool_creation_required(
+    storage_pool_type,
+    ephemeral_disk_volume_type,
+    ephemeral_disk_nvme_perf_tier,
+    existing_ephemeral_disk_volume_type,
+    existing_ephemeral_nvme_perf_tier,
+    is_extension_installed,
+    is_ephemeralDisk_nvme_enabled,
+    is_ephemeralDisk_localssd_enabled,
+) -> bool:
+    should_create_storagepool = (
+        not is_extension_installed or
+        not (is_ephemeralDisk_localssd_enabled or is_ephemeralDisk_nvme_enabled) or
+        storage_pool_type != CONST_STORAGE_POOL_TYPE_EPHEMERAL_DISK or
+        (
+            (ephemeral_disk_volume_type is None or
+                (existing_ephemeral_disk_volume_type.lower() == ephemeral_disk_volume_type.lower())) and
+            (ephemeral_disk_nvme_perf_tier is None or
+                (existing_ephemeral_nvme_perf_tier.lower() == ephemeral_disk_nvme_perf_tier.lower()))
+        )
+    )
+
+    return should_create_storagepool
+
+
+def _get_ephemeral_nvme_cpu_value_based_on_vm_size_perf_tier(nodepool_skus, perf_tier):
     cpu_value = -1
+    multiplication_factor = 0.25
+    if perf_tier.lower() == CONST_EPHEMERAL_NVME_PERF_TIER_BASIC.lower():
+        multiplication_factor = 0.15
+    elif perf_tier.lower() == CONST_EPHEMERAL_NVME_PERF_TIER_PREMIUM.lower():
+        multiplication_factor = 0.5
     for vm_size in nodepool_skus:
         number_of_cores = get_cores_from_sku(vm_size)
         if number_of_cores != -1:
             if cpu_value == -1:
-                cpu_value = number_of_cores * 0.25
+                cpu_value = number_of_cores * multiplication_factor
             else:
-                cpu_value = (number_of_cores * 0.25) if \
-                    (cpu_value > number_of_cores * 0.25) else \
+                cpu_value = (number_of_cores * multiplication_factor) if \
+                    (cpu_value > number_of_cores * multiplication_factor) else \
                     cpu_value
         else:
             raise UnknownError(
                 f"Unable to determine the number of cores in nodepool of node size: {vm_size}"
             )
 
-    if cpu_value == -1:
-        cpu_value = 1
+    # In any case when cpu_value = -1 or is lesser than 1,
+    # set the value to 1.
+    cpu_value = max(cpu_value, 1)
     return cpu_value
 
 
@@ -425,6 +496,7 @@ def _get_current_resource_values(
     is_elasticSan_enabled,
     is_ephemeralDisk_localssd_enabled,
     is_ephemeralDisk_nvme_enabled,
+    ephemeral_nvme_perf_tier,
     current_core_value=None,
     nodepool_skus=None,
 ):
@@ -444,7 +516,10 @@ def _get_current_resource_values(
         current_hugepages_number = 512
     if is_ephemeralDisk_nvme_enabled:
         if current_core_value is None and nodepool_skus is not None:
-            core_value = _get_cpu_value_based_on_vm_size(nodepool_skus)
+            core_value = _get_ephemeral_nvme_cpu_value_based_on_vm_size_perf_tier(
+                nodepool_skus,
+                ephemeral_nvme_perf_tier,
+            )
         current_memory_value = 2
         current_hugepages_value = 2
         current_hugepages_number = 1024