{AKS} feat: ga update for azure monitor for metrics addon (managed prometheus)

linter_exclusions.yml

@@ -106,6 +106,21 @@ aks create:
     custom_ca_trust_certificates:
       rule_exclusions:
         - option_length_too_long
+    azure_monitor_workspace_resource_id:
+      rule_exclusions:
+      - option_length_too_long
+    enable_azuremonitormetrics:
+      rule_exclusions:
+      - option_length_too_long
+    ksm_metric_annotations_allow_list:
+      rule_exclusions:
+      - option_length_too_long
+    ksm_metric_labels_allow_list:
+      rule_exclusions:
+      - option_length_too_long
+    enable_windows_recording_rules:
+      rule_exclusions:
+      - option_length_too_long
 aks update:
   parameters:
     aad_admin_group_object_ids:

src/aks-preview/HISTORY.rst

@@ -11,6 +11,10 @@ To release a new version, please select a new version number (usually plus 1 to
 
 Pending
 +++++++
+
+0.5.138
++++++++
+* GA update for Azure Monitor Metrics Addon (managed prometheus metrics) for AKS
 * Vendor new SDK and bump API version to 2023-03-02-preview.
 * fix: don't use current kube_proxy_config on UPDATE
 

src/aks-preview/azext_aks_preview/_help.py

@@ -528,6 +528,24 @@
         - name: --enable-asm --enable-azure-service-mesh
           type: bool
           short-summary: Enable Azure Service Mesh.
+        - name: --enable-azuremonitormetrics
+          type: bool
+          short-summary: Enable Azure Monitor Metrics Profile
+        - name: --azure-monitor-workspace-resource-id
+          type: string
+          short-summary: Resource ID of the Azure Monitor Workspace
+        - name: --ksm-metric-labels-allow-list
+          type: string
+          short-summary: Comma-separated list of additional Kubernetes label keys that will be used in the resource' labels metric. By default the metric contains only name and namespace labels. To include additional labels provide a list of resource names in their plural form and Kubernetes label keys you would like to allow for them (e.g. '=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)'. A single '*' can be provided per resource instead to allow any labels, but that has severe performance implications (e.g. '=pods=[*]').
+        - name: --ksm-metric-annotations-allow-list
+          type: string
+          short-summary: Comma-separated list of additional Kubernetes label keys that will be used in the resource' labels metric. By default the metric contains only name and namespace labels. To include additional labels provide a list of resource names in their plural form and Kubernetes label keys you would like to allow for them (e.g.'=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)'. A single '*' can be provided per resource instead to allow any labels, but that has severe performance implications (e.g. '=pods=[*]').
+        - name: --grafana-resource-id
+          type: string
+          short-summary: Resource ID of the Azure Managed Grafana Workspace
+        - name: --enable-windows-recording-rules
+          type: bool
+          short-summary: Enable Windows Recording Rules when enabling the Azure Monitor Metrics addon
     examples:
         - name: Create a Kubernetes cluster with an existing SSH public key.
           text: az aks create -g MyResourceGroup -n MyManagedCluster --ssh-key-value /path/to/publickey
@@ -595,6 +613,8 @@
           text: az aks create -g MyResourceGroup -n MyManagedCluster --enable-custom-ca-trust
         - name: Create a kubernetes cluster with Azure Service Mesh enabled.
           text: az aks create -g MyResourceGroup -n MyManagedCluster --enable-azure-service-mesh
+        - name: Create a kubernetes cluster with Azure Monitor Metrics enabled.
+          text: az aks create -g MyResourceGroup -n MyManagedCluster --enable-azuremonitormetrics
 
 """.format(sp_cache=AKS_SERVICE_PRINCIPAL_CACHE)
 
@@ -947,7 +967,7 @@
           short-summary: Enable Windows Recording Rules when enabling the Azure Monitor Metrics addon
         - name: --disable-azuremonitormetrics
           type: bool
-          short-summary: Disable Azure Monitor Metrics Profile
+          short-summary: Disable Azure Monitor Metrics Profile. This will delete all DCRA's associated with the cluster, any linked DCRs with the data stream = prometheus-stream and the recording rule groups created by the addon for this AKS cluster.
         - name: --enable-node-restriction
           type: bool
           short-summary: Enable node restriction option on cluster.

src/aks-preview/azext_aks_preview/_params.py

@@ -151,8 +151,6 @@
     validate_defender_disable_and_enable_parameters,
     validate_azuremonitorworkspaceresourceid,
     validate_grafanaresourceid,
-    validate_ksm_labels,
-    validate_ksm_annotations,
     validate_disable_windows_outbound_nat,
     validate_allowed_host_ports,
     validate_application_security_groups,
@@ -419,6 +417,13 @@ def load_arguments(self, _):
         c.argument('nodepool_asg_ids', validator=validate_application_security_groups, is_preview=True, help="application security groups for agentpool")
         c.argument('node_public_ip_tags', arg_type=tags_type, validator=validate_node_public_ip_tags,
                    help='space-separated tags: key[=value] [key[=value] ...].')
+        # azure monitor profile
+        c.argument('enable_azuremonitormetrics', action='store_true')
+        c.argument('azure_monitor_workspace_resource_id', validator=validate_azuremonitorworkspaceresourceid)
+        c.argument('ksm_metric_labels_allow_list')
+        c.argument('ksm_metric_annotations_allow_list')
+        c.argument('grafana_resource_id', validator=validate_grafanaresourceid)
+        c.argument('enable_windows_recording_rules', action='store_true')
 
     with self.argument_context('aks update') as c:
         # managed cluster paramerters
@@ -522,13 +527,13 @@ def load_arguments(self, _):
         c.argument('enable_private_cluster', action='store_true', is_preview=True, help='enable private cluster for apiserver vnet integration')
         c.argument('disable_private_cluster', action='store_true', is_preview=True, help='disable private cluster for apiserver vnet integration')
         c.argument('private_dns_zone', is_preview=True)
-        c.argument('enable_azuremonitormetrics', action='store_true', is_preview=True)
-        c.argument('azure_monitor_workspace_resource_id', validator=validate_azuremonitorworkspaceresourceid, is_preview=True)
-        c.argument('ksm_metric_labels_allow_list', validator=validate_ksm_labels, is_preview=True)
-        c.argument('ksm_metric_annotations_allow_list', validator=validate_ksm_annotations, is_preview=True)
-        c.argument('grafana_resource_id', validator=validate_grafanaresourceid, is_preview=True)
-        c.argument('enable_windows_recording_rules', action='store_true', is_preview=True)
-        c.argument('disable_azuremonitormetrics', action='store_true', is_preview=True)
+        c.argument('enable_azuremonitormetrics', action='store_true')
+        c.argument('azure_monitor_workspace_resource_id', validator=validate_azuremonitorworkspaceresourceid)
+        c.argument('ksm_metric_labels_allow_list')
+        c.argument('ksm_metric_annotations_allow_list')
+        c.argument('grafana_resource_id', validator=validate_grafanaresourceid)
+        c.argument('enable_windows_recording_rules', action='store_true')
+        c.argument('disable_azuremonitormetrics', action='store_true')
         c.argument('enable_vpa', action='store_true', is_preview=True, help="enable vertical pod autoscaler for cluster")
         c.argument('disable_vpa', action='store_true', is_preview=True, help="disable vertical pod autoscaler for cluster")
         c.argument('cluster_snapshot_id', validator=validate_cluster_snapshot_id, is_preview=True)

src/aks-preview/azext_aks_preview/_validators.py

@@ -729,63 +729,6 @@ def validate_grafanaresourceid(namespace):
         raise ArgumentUsageError("--grafana-resource-id not in the correct format. It should match `/subscriptions/<subscriptionId>/resourceGroups/<resourceGroupName>/providers/microsoft.dashboard/grafana/<resourceName>`")
 
 
-def validate_ksm_parameter(ksmparam):
-    labelValueMap = {}
-    ksmStrLength = len(ksmparam)
-    EOF = -1
-    next = ""
-    name = ""
-    firstWordPos = 0
-    for i, v in enumerate(ksmparam):
-        if i + 1 == ksmStrLength:
-            next = EOF
-        else:
-            next = ord(ksmparam[i + 1])
-        if i - 1 >= 0:
-            previous = ord(ksmparam[i - 1])
-        else:
-            previous = v
-        if v == "=":
-            if previous == ord(",") or next != ord("["):
-                raise InvalidArgumentValueError("Please format --metric properly. For eg. : --ksm-metric-labels-allow-list \"=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)\" and --ksm-metric-annotations-allow-list \"namespaces=[kubernetes.io/team,...],pods=[kubernetes.io/team],...\"")
-            name = ksmparam[firstWordPos:i]
-            labelValueMap[name] = []
-            firstWordPos = i + 1
-        elif v == "[":
-            if previous != ord("="):
-                raise InvalidArgumentValueError("Please format --metric properly. For eg. : --ksm-metric-labels-allow-list \"=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)\" and --ksm-metric-annotations-allow-list \"namespaces=[kubernetes.io/team,...],pods=[kubernetes.io/team],...\"")
-            firstWordPos = i + 1
-        elif v == "]":
-            # if after metric group, has char not comma or end.
-            if next != EOF and next != ord(","):
-                raise InvalidArgumentValueError("Please format --metric properly. For eg. : --ksm-metric-labels-allow-list \"=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)\" and --ksm-metric-annotations-allow-list \"namespaces=[kubernetes.io/team,...],pods=[kubernetes.io/team],...\"")
-            if previous != ord("["):
-                labelValueMap[name].append(ksmparam[firstWordPos:i])
-            firstWordPos = i + 1
-        elif v == ",":
-            # if starts or ends with comma
-            if previous == v or next == EOF or next == ord("]"):
-                raise InvalidArgumentValueError("Please format --metric properly. For eg. : --ksm-metric-labels-allow-list \"=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)\" and --ksm-metric-annotations-allow-list \"namespaces=[kubernetes.io/team,...],pods=[kubernetes.io/team],...\"")
-            if previous != ord("]"):
-                labelValueMap[name].append(ksmparam[firstWordPos:i])
-            firstWordPos = i + 1
-    for label in labelValueMap:
-        if (bool(re.match(r'^[a-zA-Z_][A-Za-z0-9_]+$', label))) is False:
-            raise InvalidArgumentValueError("Please format --metric properly. For eg. : --ksm-metric-labels-allow-list \"=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)\" and --ksm-metric-annotations-allow-list \"namespaces=[kubernetes.io/team,...],pods=[kubernetes.io/team],...\"")
-
-
-def validate_ksm_labels(namespace):
-    if namespace.ksm_metric_labels_allow_list is None:
-        return
-    validate_ksm_parameter(namespace.ksm_metric_labels_allow_list)
-
-
-def validate_ksm_annotations(namespace):
-    if namespace.ksm_metric_annotations_allow_list is None:
-        return
-    validate_ksm_parameter(namespace.ksm_metric_annotations_allow_list)
-
-
 def validate_allowed_host_ports(namespace):
     if hasattr(namespace, "nodepool_allowed_host_ports"):
         host_ports = namespace.nodepool_allowed_host_ports

src/aks-preview/azext_aks_preview/azuremonitormetrics/addonput.py

@@ -0,0 +1,36 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+import json
+from azext_aks_preview.azuremonitormetrics.constants import AKS_CLUSTER_API
+from azure.cli.core.profiles import ResourceType
+from azure.cli.core.azclierror import (
+    UnknownError,
+    CLIError
+)
+
+
+def addon_put(cmd, cluster_subscription, cluster_resource_group_name, cluster_name):
+    from azure.cli.core.util import send_raw_request
+    armendpoint = cmd.cli_ctx.cloud.endpoints.resource_manager
+    feature_check_url = f"{armendpoint}/subscriptions/{cluster_subscription}/resourceGroups/{cluster_resource_group_name}/providers/Microsoft.ContainerService/managedClusters/{cluster_name}?api-version={AKS_CLUSTER_API}"
+    try:
+        headers = ['User-Agent=azuremonitormetrics.addon_get']
+        r = send_raw_request(cmd.cli_ctx, "GET", feature_check_url,
+                             body={}, headers=headers)
+    except CLIError as e:
+        raise UnknownError(e)
+    json_response = json.loads(r.text)
+    if "azureMonitorProfile" in json_response["properties"]:
+        if "metrics" in json_response["properties"]["azureMonitorProfile"]:
+            if json_response["properties"]["azureMonitorProfile"]["metrics"]["enabled"] is False:
+                # What if enabled doesn't exist
+                json_response["properties"]["azureMonitorProfile"]["metrics"]["enabled"] = True
+    try:
+        headers = ['User-Agent=azuremonitormetrics.addon_put']
+        body = json.dumps(json_response)
+        r = send_raw_request(cmd.cli_ctx, "PUT", feature_check_url,
+                             body=body, headers=headers)
+    except CLIError as e:
+        raise UnknownError(e)

src/aks-preview/azext_aks_preview/azuremonitormetrics/amg/link.py

@@ -0,0 +1,80 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+import json
+import uuid
+from knack.util import CLIError
+from azext_aks_preview.azuremonitormetrics.constants import (
+    GRAFANA_API,
+    GRAFANA_ROLE_ASSIGNMENT_API,
+    GrafanaLink
+)
+from azext_aks_preview.azuremonitormetrics.helper import sanitize_resource_id
+
+
+def link_grafana_instance(cmd, raw_parameters, azure_monitor_workspace_resource_id):
+    from azure.cli.core.util import send_raw_request
+    # GET grafana principal ID
+    try:
+        grafana_resource_id = raw_parameters.get("grafana_resource_id")
+        if grafana_resource_id is None or grafana_resource_id == "":
+            return GrafanaLink.NOPARAMPROVIDED
+        grafana_resource_id = sanitize_resource_id(grafana_resource_id)
+        grafanaURI = "{0}{1}?api-version={2}".format(
+            cmd.cli_ctx.cloud.endpoints.resource_manager,
+            grafana_resource_id,
+            GRAFANA_API
+        )
+        headers = ['User-Agent=azuremonitormetrics.link_grafana_instance']
+        grafanaArmResponse = send_raw_request(cmd.cli_ctx, "GET", grafanaURI, body={}, headers=headers)
+        servicePrincipalId = grafanaArmResponse.json()["identity"]["principalId"]
+    except CLIError as e:
+        raise CLIError(e)
+    # Add Role Assignment
+    try:
+        MonitoringDataReader = "b0d8363b-8ddd-447d-831f-62ca05bff136"
+        roleDefinitionURI = "{0}{1}/providers/Microsoft.Authorization/roleAssignments/{2}?api-version={3}".format(
+            cmd.cli_ctx.cloud.endpoints.resource_manager,
+            azure_monitor_workspace_resource_id,
+            uuid.uuid4(),
+            GRAFANA_ROLE_ASSIGNMENT_API
+        )
+        roleDefinitionId = "{0}/providers/Microsoft.Authorization/roleDefinitions/{1}".format(
+            azure_monitor_workspace_resource_id,
+            MonitoringDataReader
+        )
+        association_body = json.dumps({"properties": {"roleDefinitionId": roleDefinitionId, "principalId": servicePrincipalId}})
+        headers = ['User-Agent=azuremonitormetrics.add_role_assignment']
+        send_raw_request(cmd.cli_ctx, "PUT", roleDefinitionURI, body=association_body, headers=headers)
+    except CLIError as e:
+        if e.response.status_code != 409:
+            erroString = "Role Assingment failed. Please manually assign the `Monitoring Data Reader` role to the Azure Monitor Workspace ({0}) for the Azure Managed Grafana System Assigned Managed Identity ({1})".format(
+                azure_monitor_workspace_resource_id,
+                servicePrincipalId
+            )
+            print(erroString)
+    # Setting up AMW Integration
+    targetGrafanaArmPayload = grafanaArmResponse.json()
+    if targetGrafanaArmPayload["properties"] is None:
+        raise CLIError("Invalid grafana payload to add AMW integration")
+    if "grafanaIntegrations" not in json.dumps(targetGrafanaArmPayload):
+        targetGrafanaArmPayload["properties"]["grafanaIntegrations"] = {}
+    if "azureMonitorWorkspaceIntegrations" not in json.dumps(targetGrafanaArmPayload):
+        targetGrafanaArmPayload["properties"]["grafanaIntegrations"]["azureMonitorWorkspaceIntegrations"] = []
+    amwIntegrations = targetGrafanaArmPayload["properties"]["grafanaIntegrations"]["azureMonitorWorkspaceIntegrations"]
+    if amwIntegrations != [] and azure_monitor_workspace_resource_id in json.dumps(amwIntegrations).lower():
+        return GrafanaLink.ALREADYPRESENT
+    try:
+        grafanaURI = "{0}{1}?api-version={2}".format(
+            cmd.cli_ctx.cloud.endpoints.resource_manager,
+            grafana_resource_id,
+            GRAFANA_API
+        )
+        targetGrafanaArmPayload["properties"]["grafanaIntegrations"]["azureMonitorWorkspaceIntegrations"].append({"azureMonitorWorkspaceResourceId": azure_monitor_workspace_resource_id})
+        targetGrafanaArmPayload = json.dumps(targetGrafanaArmPayload)
+        headers = ['User-Agent=azuremonitormetrics.setup_amw_grafana_integration', 'Content-Type=application/json']
+        send_raw_request(cmd.cli_ctx, "PUT", grafanaURI, body=targetGrafanaArmPayload, headers=headers)
+    except CLIError as e:
+        raise CLIError(e)
+    return GrafanaLink.SUCCESS

src/aks-preview/azext_aks_preview/azuremonitormetrics/amw/create.py

@@ -0,0 +1,42 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+import json
+
+from azext_aks_preview.azuremonitormetrics.constants import MAC_API
+from azure.cli.command_modules.acs._client_factory import get_resource_groups_client, get_resources_client
+from azure.core.exceptions import HttpResponseError
+from knack.util import CLIError
+from azext_aks_preview.azuremonitormetrics.amw.defaults import get_default_mac_name_and_region
+
+
+def create_default_mac(cmd, cluster_subscription, cluster_region):
+    from azure.cli.core.util import send_raw_request
+    default_mac_name, default_mac_region = get_default_mac_name_and_region(cmd, cluster_region)
+    default_resource_group_name = "DefaultResourceGroup-{0}".format(default_mac_region)
+    azure_monitor_workspace_resource_id = "/subscriptions/{0}/resourceGroups/{1}/providers/microsoft.monitor/accounts/{2}".format(cluster_subscription, default_resource_group_name, default_mac_name)
+    # Check if default resource group exists or not, if it does not then create it
+    resource_groups = get_resource_groups_client(cmd.cli_ctx, cluster_subscription)
+    resources = get_resources_client(cmd.cli_ctx, cluster_subscription)
+
+    if resource_groups.check_existence(default_resource_group_name):
+        try:
+            resource = resources.get_by_id(azure_monitor_workspace_resource_id, MAC_API)
+            # If MAC already exists then return from here
+            return azure_monitor_workspace_resource_id, resource.location
+        except HttpResponseError as ex:
+            if ex.status_code != 404:
+                raise ex
+    else:
+        resource_groups.create_or_update(default_resource_group_name, {"location": default_mac_region})
+    association_body = json.dumps({"location": default_mac_region, "properties": {}})
+    armendpoint = cmd.cli_ctx.cloud.endpoints.resource_manager
+    association_url = f"{armendpoint}{azure_monitor_workspace_resource_id}?api-version={MAC_API}"
+    try:
+        headers = ['User-Agent=azuremonitormetrics.create_default_mac']
+        send_raw_request(cmd.cli_ctx, "PUT", association_url,
+                         body=association_body, headers=headers)
+        return azure_monitor_workspace_resource_id, default_mac_region
+    except CLIError as e:
+        raise e