diff --git a/Dockerfile.interop b/Dockerfile.interop new file mode 100644 index 000000000..73efd12fd --- /dev/null +++ b/Dockerfile.interop @@ -0,0 +1,31 @@ +# Command to build this Dockerfile +# docker build -f Dockerfile -t quay.io/vboulos/acmqe-automation/obs:obs-ginkgo_1_14_2-linux-go . + +FROM quay.io/vboulos/acmqe-automation/ginkgo_1_14_2-linux-go + + +# Copy the Obs repo repo into /tmp/obs folder +RUN mkdir /tmp/obs +WORKDIR /tmp/obs +COPY . . + +# good colors for most applications +ENV TERM=xterm + +# Set required permissions for OpenShift usage +RUN chgrp -R 0 /tmp && \ + chmod -R g=u /tmp + +RUN mkdir -p /go +RUN chgrp -R 0 /go && \ + chmod -R g=u /go + +RUN mkdir -p ~/.kube +RUN chgrp -R 0 ~/.kube && \ + chmod -R g=u ~/.kube + +RUN mkdir -p /alabama/.kube +RUN chgrp -R 0 /alabama/.kube && \ + chmod -R g=u /alabama/.kube + +CMD ["/bin/bash"] \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 000000000..f9d2eb31d --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,122 @@ +pipeline { + options { + buildDiscarder(logRotator(daysToKeepStr: '30')) + } + agent { + docker { + image 'quay.io/vboulos/acmqe-automation/ginkgo_1_14_2-linux-go' + args '--network host -u 0:0' + } + } + parameters { + string(name:'CLOUD_PROVIDER', defaultValue: '', description: 'Cloud provider, OCP and ACM version information, like VMWARE-412-264, AWS-411') + string(name:'HUB_CLUSTER_NAME', defaultValue: '', description: 'Name of Hub cluster') + string(name:'BASE_DOMAIN', defaultValue: '', description: 'Base domain of Hub cluster') + string(name:'OC_CLUSTER_USER', defaultValue: 'kubeadmin', description: 'OCP Hub User Name') + string(name:'OC_HUB_CLUSTER_PASS', defaultValue: '', description: 'OCP Hub Password') + string(name:'OC_HUB_CLUSTER_API_URL', defaultValue: '', description: 'OCP Hub API URL') + string(name:'MANAGED_CLUSTER_NAME', defaultValue: '', description: 'Managed cluster name') + string(name:'MANAGED_CLUSTER_BASE_DOMAIN', defaultValue: '', description: 'Managed cluster base domain') + string(name:'MANAGED_CLUSTER_USER', defaultValue: 'kubeadmin', description: 'Managed Cluster User Name') + string(name:'MANAGED_CLUSTER_PASS', defaultValue: '', description: 'Managed cluster Password') + string(name:'MANAGED_CLUSTER_API_URL', defaultValue: '', description: 'Managed cluster API URL') + string(name:'BUCKET', defaultValue: 'obs-auto-bucket', description: 'Bucket name') + string(name:'REGION', defaultValue: 'us-east-1', description: 'Bucket region') + password(name:'AWS_ACCESS_KEY_ID', defaultValue: '', description: 'AWS access key ID') + password(name:'AWS_SECRET_ACCESS_KEY', defaultValue: '', description: 'AWS secret access key') + string(name:'SKIP_INSTALL_STEP', defaultValue: 'false', description: 'Skip Observability installation') + string(name:'SKIP_UNINSTALL_STEP', defaultValue: 'true', description: 'Skip Observability uninstallation') + string(name:'TAGGING', defaultValue: '', description: 'with tagging value to run the specific test cases') + string(name:'USE_MINIO', defaultValue: 'false', description: 'If no AWS S3 bucket, you could use minio as object storage to instead') + } + environment { + CI = 'true' + AWS_SECRET_ACCESS_KEY = credentials('cqu_aws_secret_access_key') + AWS_ACCESS_KEY_ID = credentials('cqu_aws_access_key') + } + stages { + stage('Test Run') { + steps { + sh """ + export CLOUD_PROVIDER="${params.CLOUD_PROVIDER}" + export OC_CLUSTER_USER="${params.OC_CLUSTER_USER}" + set +x + export OC_HUB_CLUSTER_PASS="${params.OC_HUB_CLUSTER_PASS}" + set -x + export OC_HUB_CLUSTER_API_URL="${params.OC_HUB_CLUSTER_API_URL}" + BASE_DOMAIN=\$(echo \${OC_HUB_CLUSTER_API_URL} | awk -F'https://api\\.|:' '{print \$2}') + HUB_CLUSTER_NAME=\$(echo \$BASE_DOMAIN | cut -d'.' -f1) + echo "BASE_DOMAIN: \$BASE_DOMAIN" + echo "HUB_CLUSTER_NAME: \$HUB_CLUSTER_NAME" + export HUB_CLUSTER_NAME="\$HUB_CLUSTER_NAME" + export BASE_DOMAIN="\$BASE_DOMAIN" + export MANAGED_CLUSTER_NAME="${params.MANAGED_CLUSTER_NAME}" + export MANAGED_CLUSTER_BASE_DOMAIN="${params.MANAGED_CLUSTER_BASE_DOMAIN}" + export MANAGED_CLUSTER_USER="${params.MANAGED_CLUSTER_USER}" + set +x + export MANAGED_CLUSTER_PASS="${params.MANAGED_CLUSTER_PASS}" + set -x + export MANAGED_CLUSTER_API_URL="${params.MANAGED_CLUSTER_API_URL}" + export BUCKET="${params.BUCKET}" + export REGION="${params.REGION}" + export SKIP_INSTALL_STEP="${params.SKIP_INSTALL_STEP}" + export SKIP_UNINSTALL_STEP="${params.SKIP_UNINSTALL_STEP}" + export TAGGING="${params.TAGGING}" + export USE_MINIO="${params.USE_MINIO}" + export IS_CANARY_ENV=true + + if [[ -n "${params.AWS_ACCESS_KEY_ID}" ]]; then + export AWS_ACCESS_KEY_ID="${params.AWS_ACCESS_KEY_ID}" + fi + + if [[ -n "${params.AWS_SECRET_ACCESS_KEY}" ]]; then + export AWS_SECRET_ACCESS_KEY="${params.AWS_SECRET_ACCESS_KEY}" + fi + + if [[ "${params.USE_MINIO}" == true ]]; then + export IS_CANARY_ENV=false + fi + if [[ -z "${OC_CLUSTER_USER}" || -z "${OC_HUB_CLUSTER_PASS}" || -z "${OC_HUB_CLUSTER_API_URL}" ]]; then + echo "Aborting test.. OCP HUB details are required for the test execution" + exit 1 + else + if [[ -n "${params.MANAGED_CLUSTER_USER}" && -n "${params.MANAGED_CLUSTER_PASS}" && -n "${params.MANAGED_CLUSTER_API_URL}" ]]; then + set +x + oc login --insecure-skip-tls-verify -u \$MANAGED_CLUSTER_USER -p \$MANAGED_CLUSTER_PASS \$MANAGED_CLUSTER_API_URL + set -x + oc config view --minify --raw=true > ~/.kube/managed_kubeconfig + export MAKUBECONFIG=~/.kube/managed_kubeconfig + fi + set +x + oc login --insecure-skip-tls-verify -u \$OC_CLUSTER_USER -p \$OC_HUB_CLUSTER_PASS \$OC_HUB_CLUSTER_API_URL + set -x + export KUBECONFIG=~/.kube/config + go mod vendor && ginkgo build ./tests/pkg/tests/ + rm -rf tests/pkg/tests/*.xml + cd tests + cp resources/options.yaml.template resources/options.yaml + /usr/local/bin/yq e -i '.options.hub.name="'"\$HUB_CLUSTER_NAME"'"' resources/options.yaml + /usr/local/bin/yq e -i '.options.hub.baseDomain="'"\$BASE_DOMAIN"'"' resources/options.yaml + /usr/local/bin/yq e -i '.options.clusters.name="'"\$MANAGED_CLUSTER_NAME"'"' resources/options.yaml + /usr/local/bin/yq e -i '.options.clusters.baseDomain="'"\$MANAGED_CLUSTER_BASE_DOMAIN"'"' resources/options.yaml + /usr/local/bin/yq e -i '.options.clusters.kubeconfig="'"\$MAKUBECONFIG"'"' resources/options.yaml + cat resources/options.yaml + if [[ -n "${params.TAGGING}" ]]; then + ginkgo --focus="\$TAGGING" -v pkg/tests/ -- -options=../../resources/options.yaml -v=5 + else + ginkgo -v pkg/tests/ -- -options=../../resources/options.yaml -v=5 + fi + fi + """ + } + } + + + } + post { + always { + archiveArtifacts artifacts: 'tests/pkg/tests/*.xml', followSymlinks: false + junit 'tests/pkg/tests/*.xml' + } + } +} diff --git a/execute_obs_interop_commands.sh b/execute_obs_interop_commands.sh new file mode 100644 index 000000000..be17ed337 --- /dev/null +++ b/execute_obs_interop_commands.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +export PARAM_AWS_SECRET_ACCESS_KEY=${PARAM_AWS_SECRET_ACCESS_KEY:-} +export PARAM_AWS_ACCESS_KEY_ID=${PARAM_AWS_ACCESS_KEY_ID:-} +export CLOUD_PROVIDER=${CLOUD_PROVIDER:-} +export OC_CLUSTER_USER=${OC_CLUSTER_USER:-} +export OC_HUB_CLUSTER_PASS=${OC_HUB_CLUSTER_PASS:-} +export OC_HUB_CLUSTER_API_URL=${OC_HUB_CLUSTER_API_URL:-} +export HUB_CLUSTER_NAME=${HUB_CLUSTER_NAME:-} +export BASE_DOMAIN=${BASE_DOMAIN:-} +export MANAGED_CLUSTER_NAME=${MANAGED_CLUSTER_NAME:-} +export MANAGED_CLUSTER_BASE_DOMAIN=${MANAGED_CLUSTER_BASE_DOMAIN:-} +export MANAGED_CLUSTER_USER=${MANAGED_CLUSTER_USER:-} +export MANAGED_CLUSTER_PASS=${MANAGED_CLUSTER_PASS:-} +export MANAGED_CLUSTER_API_URL=${MANAGED_CLUSTER_API_URL} +export BUCKET=${BUCKET:-'obs-v1'} +export REGION=${REGION:-'us-east-1'} +export USE_MINIO=${USE_MINIO:-'false'} +export SKIP_INSTALL_STEP=${SKIP_INSTALL_STEP:-'false'} +export SKIP_UNINSTALL_STEP=${SKIP_UNINSTALL_STEP:-'true'} +export TAGGING=${TAGGING:-} + +if [[ -n ${PARAM_AWS_ACCESS_KEY_ID} ]]; then + export AWS_ACCESS_KEY_ID=${PARAM_AWS_ACCESS_KEY_ID} +fi + +if [[ -n ${PARAM_AWS_SECRET_ACCESS_KEY} ]]; then + export AWS_SECRET_ACCESS_KEY=${PARAM_AWS_SECRET_ACCESS_KEY} +fi + +# if [[ ${!USE_MINIO} == "false" ]]; then +# export IS_CANARY_ENV=true +# fi + +export IS_CANARY_ENV=true + +if [[ -z ${HUB_CLUSTER_NAME} || -z ${BASE_DOMAIN} || -z ${OC_CLUSTER_USER} || -z ${OC_HUB_CLUSTER_PASS} || -z ${OC_HUB_CLUSTER_API_URL} ]]; then + echo "Aborting test.. OCP HUB details are required for the test execution" + exit 1 +else + if [[ -n ${MANAGED_CLUSTER_USER} && -n ${MANAGED_CLUSTER_PASS} && -n ${MANAGED_CLUSTER_API_URL} ]]; then + oc login --insecure-skip-tls-verify -u $MANAGED_CLUSTER_USER -p $MANAGED_CLUSTER_PASS $MANAGED_CLUSTER_API_URL + oc config view --minify --raw=true >~/.kube/managed_kubeconfig + export MAKUBECONFIG=~/.kube/managed_kubeconfig + fi + set +x + oc login --insecure-skip-tls-verify -u $OC_CLUSTER_USER -p $OC_HUB_CLUSTER_PASS $OC_HUB_CLUSTER_API_URL + set -x + + oc config view --minify --raw=true >userfile + //cat userfile + whoami + rm -rf ~/.kube/config + cp userfile ~/.kube/config + //cat ~/.kube/config + export KUBECONFIG=~/.kube/config + + go mod vendor && ginkgo build ./tests/pkg/tests/ + cd tests + cp resources/options.yaml.template resources/options.yaml + /usr/local/bin/yq e -i '.options.hub.name="'"$HUB_CLUSTER_NAME"'"' resources/options.yaml + /usr/local/bin/yq e -i '.options.hub.baseDomain="'"$BASE_DOMAIN"'"' resources/options.yaml + /usr/local/bin/yq e -i '.options.clusters.name="'"$MANAGED_CLUSTER_NAME"'"' resources/options.yaml + /usr/local/bin/yq e -i '.options.clusters.baseDomain="'"$MANAGED_CLUSTER_BASE_DOMAIN"'"' resources/options.yaml + /usr/local/bin/yq e -i '.options.clusters.kubeconfig="'"$MAKUBECONFIG"'"' resources/options.yaml + cat resources/options.yaml + ginkgo --focus=$TAGGING -v pkg/tests/ -- -options=../../resources/options.yaml -v=5 +fi diff --git a/tests/pkg/tests/observability-e2e-test_suite_test.go b/tests/pkg/tests/observability-e2e-test_suite_test.go index 4575459e7..75a4c13bc 100644 --- a/tests/pkg/tests/observability-e2e-test_suite_test.go +++ b/tests/pkg/tests/observability-e2e-test_suite_test.go @@ -9,6 +9,7 @@ import ( "fmt" "math/rand" "os" + "strings" "testing" "time" @@ -217,19 +218,37 @@ func initVars() { testOptions.KubeConfig = kubeconfig } + cloudProvider := strings.ToLower(os.Getenv("CLOUD_PROVIDER")) + substring1 := "rosa" + substring2 := "hcp" if testOptions.HubCluster.BaseDomain != "" { baseDomain = testOptions.HubCluster.BaseDomain - if testOptions.HubCluster.ClusterServerURL == "" { - testOptions.HubCluster.ClusterServerURL = fmt.Sprintf( - "https://api.%s:6443", - testOptions.HubCluster.BaseDomain, - ) + // TODO: Simplify + if strings.Contains(cloudProvider, substring1) && strings.Contains(cloudProvider, substring2) { + + testOptions.HubCluster.ClusterServerURL = fmt.Sprintf( + "https://api.%s:443", + testOptions.HubCluster.BaseDomain, + ) + } else { + testOptions.HubCluster.ClusterServerURL = fmt.Sprintf( + "https://api.%s:6443", + testOptions.HubCluster.BaseDomain, + ) + } } } else { Expect(baseDomain).NotTo(BeEmpty(), "The `baseDomain` is required.") testOptions.HubCluster.BaseDomain = baseDomain testOptions.HubCluster.ClusterServerURL = fmt.Sprintf("https://api.%s:6443", baseDomain) + // TODO: Simplify + if strings.Contains(cloudProvider, substring1) && strings.Contains(cloudProvider, substring2) { + + testOptions.HubCluster.ClusterServerURL = fmt.Sprintf("https://api.%s:443", baseDomain) + } else { + testOptions.HubCluster.ClusterServerURL = fmt.Sprintf("https://api.%s:6443", baseDomain) + } } if testOptions.HubCluster.User != "" { diff --git a/tests/pkg/tests/observability_addon_test.go b/tests/pkg/tests/observability_addon_test.go index f9ce37591..398417b80 100644 --- a/tests/pkg/tests/observability_addon_test.go +++ b/tests/pkg/tests/observability_addon_test.go @@ -5,13 +5,14 @@ package tests import ( + "context" "fmt" "strings" - "errors" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog" "github.com/stolostron/multicluster-observability-operator/tests/pkg/utils" @@ -43,7 +44,7 @@ var _ = Describe("Observability:", func() { }, EventuallyTimeoutMinute*6, EventuallyIntervalSecond*5).Should(Succeed()) }) - Context("[P2][Sev2][observability] Modifying MCO cr to disable observabilityaddon (addon/g0) -", func() { + Context("RHACM4K-1260: Observability: Verify monitoring operator and deployment status when metrics collection disabled [P2][Sev2][Observability]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @pre-upgrade (addon/g0) -", func() { It("[Stable] Should have resource requirement defined in CR", func() { By("Check addon resource requirement") res, err := utils.GetMCOAddonSpecResources(testOptions) @@ -70,27 +71,17 @@ var _ = Describe("Observability:", func() { By("Waiting for MCO addon components scales to 0") Eventually(func() error { - err, podList := utils.GetPodList( - testOptions, - false, - MCO_ADDON_NAMESPACE, - "component=metrics-collector", - ) + err = utils.CheckAllOBAsDeleted(testOptions) + if err != nil { - return errors.New("Failed to disable observability addon") - } - if len(podList.Items) != 0 { - for _, po := range podList.Items { - if po.Status.Phase == "Running" { - return errors.New("Failed to disable observability addon, there is still metrics-collector pod in Running") - } - } + return fmt.Errorf("Failed to disable observability addon") } return nil }, EventuallyTimeoutMinute*5, EventuallyIntervalSecond*5).Should(Succeed()) + // TODO: do we need this, disabled in core-automation-repo Eventually(func() error { - err = utils.CheckAllOBAsDeleted(testOptions) + err = utils.CheckAllOBADisabled(testOptions) if err != nil { return err } @@ -119,8 +110,7 @@ var _ = Describe("Observability:", func() { return nil }, EventuallyTimeoutMinute*2, EventuallyIntervalSecond*5).Should(Succeed()) }) - - It("[Stable] Modifying MCO cr to enable observabilityaddon", func() { + It("RHACM4K-1418: Observability: Verify clustermanagementaddon CR for Observability - Modifying MCO cr to enable observabilityaddon [P2][Sev2][Stable][Observability]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @pre-upgrade (addon/g0)", func() { Eventually(func() error { return utils.ModifyMCOAddonSpecMetrics(testOptions, true) }, EventuallyTimeoutMinute*1, EventuallyIntervalSecond*5).Should(Succeed()) @@ -139,6 +129,11 @@ var _ = Describe("Observability:", func() { } return false }, EventuallyTimeoutMinute*6, EventuallyIntervalSecond*5).Should(BeTrue()) + }) + It("RHACM4K-1074: Observability: Verify ObservabilityEndpoint operator deployment - Modifying MCO cr to enable observabilityaddon [P2][Sev2][Stable][Observability]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @post-release @pre-upgrade (addon/g0)", func() { + Eventually(func() error { + return utils.ModifyMCOAddonSpecMetrics(testOptions, true) + }, EventuallyTimeoutMinute*1, EventuallyIntervalSecond*5).Should(Succeed()) By("Checking the status in managedclusteraddon reflects the endpoint operator status correctly") Eventually(func() error { @@ -151,7 +146,19 @@ var _ = Describe("Observability:", func() { }) }) - It("[P3][Sev3][observability][Stable] Should not set interval to values beyond scope (addon/g0)", func() { + It("RHACM4K-6923: Observability: Verify default scrap interval change to 5 minutes - [P2][Sev2][Observability][Stable]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @post-release @pre-upgrade (addon/g2)", func() { + By("Check default interval value is 300") + Eventually(func() bool { + mco, getErr := dynClient.Resource(utils.NewMCOGVRV1BETA2()).Get(context.TODO(), MCO_CR_NAME, metav1.GetOptions{}) + if getErr != nil { + panic(getErr.Error()) + } + observabilityAddonSpec := mco.Object["spec"].(map[string]interface{})["observabilityAddonSpec"].(map[string]interface{}) + return observabilityAddonSpec["interval"] == int64(300) + }, EventuallyTimeoutMinute*1, EventuallyIntervalSecond*1).Should(BeTrue()) + }) + + It("RHACM4K-1235: Observability: Verify metrics data global setting on the managed cluster - Should not set interval to values beyond scope [P3][Sev3][Observability][Stable]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @post-release @pre-upgrade (addon/g0)", func() { By("Set interval to 14") Eventually(func() bool { err := utils.ModifyMCOAddonSpecInterval(testOptions, int64(14)) @@ -175,20 +182,38 @@ var _ = Describe("Observability:", func() { }, EventuallyTimeoutMinute*1, EventuallyIntervalSecond*1).Should(BeTrue()) }) - Context("[P2][Sev2][observability] Should not have the expected MCO addon pods when disable observability from managedcluster (addon/g0) -", func() { + It("RHACM4K-1259: Observability: Verify imported cluster is observed [P3][Sev3][Observability][Stable]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore (deploy/g1)", func() { + + Eventually(func() error { + return utils.UpdateObservabilityFromManagedCluster(testOptions, false) + }, EventuallyTimeoutMinute*5, EventuallyIntervalSecond*5).Should(Succeed()) + + klog.V(1).Infof("managedcluster number is <%d>", len(testOptions.ManagedClusters)) + if len(testOptions.ManagedClusters) >= 1 { + Eventually(func() bool { + return true + }, EventuallyTimeoutMinute*5, EventuallyIntervalSecond*5).Should(BeTrue()) + } + }) + + Context("RHACM4K-7518: Observability: Disable the Observability by updating managed cluster label [P2][Sev2][Observability]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore (addon/g1) -", func() { It("[Stable] Modifying managedcluster cr to disable observability", func() { Eventually(func() error { return utils.UpdateObservabilityFromManagedCluster(testOptions, false) }, EventuallyTimeoutMinute*5, EventuallyIntervalSecond*5).Should(Succeed()) - By("Waiting for MCO addon components scales to 0") - Eventually(func() bool { - err, obaNS := utils.GetNamespace(testOptions, false, MCO_ADDON_NAMESPACE) - if err == nil && obaNS == nil { - return true - } - return false - }, EventuallyTimeoutMinute*5, EventuallyIntervalSecond*5).Should(BeTrue()) + klog.V(1).Infof("managedcluster number is <%d>", len(testOptions.ManagedClusters)) + // TODO (jacob): Do we need this given we have a condition for that earlier? + if len(testOptions.ManagedClusters) > 0 { + By("Waiting for MCO addon components scales to 0") + Eventually(func() bool { + err, obaNS := utils.GetNamespace(testOptions, false, MCO_ADDON_NAMESPACE) + if err == nil && obaNS == nil { + return true + } + return false + }, EventuallyTimeoutMinute*5, EventuallyIntervalSecond*5).Should(BeTrue()) + } }) It("[Stable] Remove disable observability label from the managed cluster", func() { diff --git a/tests/pkg/tests/observability_alert_test.go b/tests/pkg/tests/observability_alert_test.go index 0d21d1aa1..c591e8185 100644 --- a/tests/pkg/tests/observability_alert_test.go +++ b/tests/pkg/tests/observability_alert_test.go @@ -14,6 +14,7 @@ import ( "net/http" "net/url" "os" + "os/exec" "reflect" "sort" "strings" @@ -55,7 +56,30 @@ var _ = Describe("Observability:", func() { } secret := "alertmanager-config" - It("@BVT - [P1][Sev1][observability][Stable] Should have the expected statefulsets (alert/g0)", func() { + It("RHACM4K-39481: Observability: Verify PrometheusRule resource(2.9) [P2][Sev2][Observability][Stable]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @post-release (alert/g1)", func() { + By("Checking if PrometheusRule: acm-observability-alert-rules is existed") + + command := "oc" + args := []string{"get", "prometheusrules.monitoring.coreos.com", "acm-observability-alert-rules", "-n", "open-cluster-management-observability"} + + output, err := exec.Command(command, args...).CombinedOutput() + if err != nil { + fmt.Printf("Error executing command: %v\n", err) + fmt.Printf("Command output:\n%s\n", output) + return + } + + prometheusRule := "acm-observability-alert-rules" + if strings.Contains(string(output), prometheusRule) { + fmt.Println("Expected result found.") + } else { + fmt.Println("Expected result not found.") + } + + fmt.Printf("Command output:\n%s\n", output) + + }) + It("RHACM4K-1404: Observability: Verify alert is created and received - Should have the expected statefulsets @BVT - [P1][Sev1][Observability][Stable]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @post-release (alert/g0)", func() { By("Checking if STS: Alertmanager and observability-thanos-rule exist") for _, label := range statefulsetLabels { sts, err := hubClient.AppsV1(). @@ -80,7 +104,7 @@ var _ = Describe("Observability:", func() { } }) - It("[P2][Sev2][observability][Stable] Should have the expected configmap (alert/g0)", func() { + It("RHACM4K-1404: Observability: Verify alert is created and received - Should have the expected configmap [P2][Sev2][Observability][Stable]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @post-release (alert/g0)", func() { By("Checking if CM: thanos-ruler-default-rules is existed") cm, err := hubClient.CoreV1().ConfigMaps(MCO_NAMESPACE).Get(context.TODO(), configmap[0], metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -89,7 +113,43 @@ var _ = Describe("Observability:", func() { klog.V(3).Infof("Configmap %s does exist", configmap[0]) }) - It("[P3][Sev3][observability][Stable] Should not have the CM: thanos-ruler-custom-rules (alert/g0)", func() { + It("RHACM4K-52080: Observability: Verify Endpointmetrics reconcile CMO Config changes [P2][Sev2][Observability][Stable]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @post-release (alert/g2)", func() { + By("Checking if CM: cluster-monitoring-config is existed") + namespace := "openshift-monitoring" + configMapName := "cluster-monitoring-config" + cm, err := hubClient.CoreV1().ConfigMaps(namespace).Get(context.TODO(), configMapName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + Expect(cm.ResourceVersion).ShouldNot(BeEmpty()) + klog.V(3).Infof("Configmap %s does exist", configmap[0]) + + By("Remove additionalAlertmanagerConfigs from the cm cluster-monitoring-config") + configContent := cm.Data["config.yaml"] + if strings.Contains(configContent, "additionalAlertmanagerConfigs:") { + // Find and remove the additionalAlertmanagerConfigs section + startIndex := strings.Index(configContent, "additionalAlertmanagerConfigs:") + endIndex := strings.Index(configContent[startIndex:], "externalLabels:") + if endIndex != -1 { + endIndex += startIndex + removedContent := configContent[startIndex:endIndex] + + // Remove the section from the config.yaml + configContent = strings.Replace(configContent, removedContent, "", 1) + cm.Data["config.yaml"] = configContent + + // Update the ConfigMap + _, err = hubClient.CoreV1().ConfigMaps(namespace).Update(context.TODO(), cm, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred(), "Failed to update ConfigMap") + fmt.Println("Removed additionalAlertmanagerConfigs and updated the ConfigMap.") + } else { + fmt.Println("Could not find the externalLabels section after additionalAlertmanagerConfigs.") + } + } else { + fmt.Println("No additionalAlertmanagerConfigs section found.") + } + }) + + It("RHACM4K-1404: Observability: Verify alert is created and received - Should not have the CM: thanos-ruler-custom-rules [P3][Sev3][Observability][Stable]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @post-release (alert/g0)", func() { By("Checking if CM: thanos-ruler-custom-rules not existed") _, err := hubClient.CoreV1().ConfigMaps(MCO_NAMESPACE).Get(context.TODO(), configmap[1], metav1.GetOptions{}) @@ -102,7 +162,7 @@ var _ = Describe("Observability:", func() { klog.V(3).Infof("Configmap %s does not exist", configmap[1]) }) - It("@BVT - [P1][Sev1][observability][Stable] Should have the expected secret (alert/g0)", func() { + It("RHACM4K-1404: Observability: Verify alert is created and received - Should have the expected secret @BVT - [P1][Sev1][Observability][Stable]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @post-release (alert/g0)", func() { By("Checking if SECRETS: alertmanager-config is existed") secret, err := hubClient.CoreV1().Secrets(MCO_NAMESPACE).Get(context.TODO(), secret, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -111,9 +171,10 @@ var _ = Describe("Observability:", func() { klog.V(3).Infof("Successfully got secret: %s", secret.GetName()) }) - It("@BVT - [P1][Sev1][observability][Stable] Should have the alertmanager configured in rule (alert/g0)", func() { + It("RHACM4K-1404: Observability: Verify alert is created and received - Should have the alertmanager configured in rule @BVT - [P1][Sev1][Observability][Stable]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @post-release (alert/g0)", func() { By(`Checking if --alertmanagers.url or --alertmanager.config or --alertmanagers.config-file is configured in rule`) + rules, err := hubClient.AppsV1().StatefulSets(MCO_NAMESPACE).List(context.TODO(), metav1.ListOptions{ LabelSelector: THANOS_RULE_LABEL, }) @@ -139,7 +200,7 @@ var _ = Describe("Observability:", func() { klog.V(3).Info("Have the alertmanager url configured in rule") }) - It("[P2][Sev2][observability][Stable] Should have custom alert generated (alert/g0)", func() { + It("RHACM4K-1404: Observability: Verify alert is created and received - Should have custom alert generated P2][Sev2][Observability][Stable]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @post-release (alert/g0)", func() { By("Creating custom alert rules") rules, err := hubClient.AppsV1().StatefulSets(MCO_NAMESPACE).List(context.TODO(), metav1.ListOptions{ @@ -220,7 +281,7 @@ var _ = Describe("Observability:", func() { // klog.V(3).Infof("Successfully modified the secret: alertmanager-config") // }) - It("[P2][Sev2][observability][Stable] Should have custom alert updated (alert/g0)", func() { + It("RHACM4K-1668: Observability: Updated alert rule can take effect automatically - Should have custom alert updated [P2][Sev2][Observability][Stable]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @post-release (alert/g0)", func() { By("Updating custom alert rules") // Replace preceding custom alert with new one that cannot fire @@ -260,7 +321,7 @@ var _ = Describe("Observability:", func() { EventuallyIntervalSecond*5).Should(Succeed()) }) - It("[P2][Sev2][observability][Stable] delete the customized rules (alert/g0)", func() { + It("RHACM4K-1668: Observability: Updated alert rule can take effect automatically - delete the customized rules [P2][Sev2][Observability][Stable]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @post-release (alert/g0)", func() { rules, err := hubClient.AppsV1().StatefulSets(MCO_NAMESPACE).List(context.TODO(), metav1.ListOptions{ LabelSelector: THANOS_RULE_LABEL, @@ -301,13 +362,31 @@ var _ = Describe("Observability:", func() { klog.V(3).Infof("Successfully deleted CM: thanos-ruler-custom-rules") }) - It("[P2][Sev2][observability][Integration] Should have alert named Watchdog forwarded to alertmanager (alertforward/g0)", func() { + It("RHACM4K-3457: Observability: Verify managed cluster alert would be forward to hub alert manager - Should have alert named Watchdog forwarded to alertmanager [P2][Sev2][Observability][Integration]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e (alertforward/g0)", func() { + + cloudProvider := strings.ToLower(os.Getenv("CLOUD_PROVIDER")) + substring1 := "rosa" + substring2 := "hcp" + + var amURL *url.URL + + if strings.Contains(cloudProvider, substring1) && strings.Contains(cloudProvider, substring2) { + Skip("skip on rosa-hcp") + amURL = &url.URL{ + Scheme: "https", + Host: "alertmanager-open-cluster-management-observability.apps.rosa." + testOptions.HubCluster.BaseDomain, + Path: "/api/v2/alerts", + } + + } else { + amURL = &url.URL{ + Scheme: "https", + Host: "alertmanager-open-cluster-management-observability.apps." + testOptions.HubCluster.BaseDomain, + Path: "/api/v2/alerts", + } - amURL := url.URL{ - Scheme: "https", - Host: "alertmanager-open-cluster-management-observability.apps." + testOptions.HubCluster.BaseDomain, - Path: "/api/v2/alerts", } + q := amURL.Query() q.Set("filter", "alertname=Watchdog") amURL.RawQuery = q.Encode() @@ -336,9 +415,15 @@ var _ = Describe("Observability:", func() { expectedOCPClusterIDs, err := utils.ListOCPManagedClusterIDs(testOptions, "4.8.0") Expect(err).NotTo(HaveOccurred()) + expectedLocalClusterIDs, err := utils.ListLocalClusterIDs(testOptions) + expectedOCPClusterIDs = append(expectedOCPClusterIDs, expectedLocalClusterIDs...) + klog.V(3).Infof("expectedOCPClusterIDs is %s", expectedOCPClusterIDs) + Expect(err).NotTo(HaveOccurred()) expectedKSClusterNames, err := utils.ListKSManagedClusterNames(testOptions) + klog.V(3).Infof("expectedKSClusterNames is %s", expectedKSClusterNames) Expect(err).NotTo(HaveOccurred()) expectClusterIdentifiers := append(expectedOCPClusterIDs, expectedKSClusterNames...) + klog.V(3).Infof("expectClusterIdentifiers is %s", expectClusterIdentifiers) // install watchdog PrometheusRule to *KS clusters watchDogRuleKustomizationPath := "../../../examples/alerts/watchdog_rule" @@ -395,13 +480,155 @@ var _ = Describe("Observability:", func() { } sort.Strings(clusterIDsInAlerts) + klog.V(3).Infof("clusterIDsInAlerts is %s", clusterIDsInAlerts) sort.Strings(expectClusterIdentifiers) - if !reflect.DeepEqual(clusterIDsInAlerts, expectClusterIdentifiers) { + klog.V(3).Infof("sort.Strings.expectClusterIdentifiers is %s", expectClusterIdentifiers) + klog.V(3).Infof("no sort.Strings.expectedOCPClusterIDs is %s", expectedOCPClusterIDs) + sort.Strings(expectedOCPClusterIDs) + klog.V(3).Infof("sort.Strings.expectedOCPClusterIDs is %s", expectedOCPClusterIDs) + if !reflect.DeepEqual(clusterIDsInAlerts, expectedOCPClusterIDs) { return fmt.Errorf("Not all openshift managedclusters >=4.8.0 forward Watchdog alert to hub cluster") } return nil - }, EventuallyTimeoutMinute*5, EventuallyIntervalSecond*5).Should(Succeed()) + }, EventuallyTimeoutMinute*3, EventuallyIntervalSecond*5).Should(Succeed()) + }) + + It("RHACM4K-22427: Observability: Disable the managedcluster's alerts forward to the Hub [P2][Sev2][Observability][Integration] @e2e (alertforward/g1)", func() { + cloudProvider := strings.ToLower(os.Getenv("CLOUD_PROVIDER")) + substring1 := "rosa" + substring2 := "hcp" + + var amURL *url.URL + + if strings.Contains(cloudProvider, substring1) && strings.Contains(cloudProvider, substring2) { + Skip("skip on rosa-hcp") + amURL = &url.URL{ + Scheme: "https", + Host: "alertmanager-open-cluster-management-observability.apps.rosa." + testOptions.HubCluster.BaseDomain, + Path: "/api/v2/alerts", + } + + } else { + amURL = &url.URL{ + Scheme: "https", + Host: "alertmanager-open-cluster-management-observability.apps." + testOptions.HubCluster.BaseDomain, + Path: "/api/v2/alerts", + } + + } + q := amURL.Query() + q.Set("filter", "alertname=Watchdog") + amURL.RawQuery = q.Encode() + + caCrt, err := utils.GetRouterCA(hubClient) + Expect(err).NotTo(HaveOccurred()) + pool := x509.NewCertPool() + pool.AppendCertsFromPEM(caCrt) + + client := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{RootCAs: pool}, + }, + } + + alertGetReq, err := http.NewRequest("GET", amURL.String(), nil) + Expect(err).NotTo(HaveOccurred()) + + if os.Getenv("IS_KIND_ENV") != "true" { + if BearerToken == "" { + BearerToken, err = utils.FetchBearerToken(testOptions) + Expect(err).NotTo(HaveOccurred()) + } + alertGetReq.Header.Set("Authorization", "Bearer "+BearerToken) + } + + expectedKSClusterNames, err := utils.ListKSManagedClusterNames(testOptions) + Expect(err).NotTo(HaveOccurred()) + + watchDogRuleKustomizationPath := "../../../examples/alerts/watchdog_rule" + yamlB, err := kustomize.Render(kustomize.Options{KustomizationPath: watchDogRuleKustomizationPath}) + Expect(err).NotTo(HaveOccurred()) + for _, ks := range expectedKSClusterNames { + for idx, mc := range testOptions.ManagedClusters { + if mc.Name == ks { + err = utils.Apply( + testOptions.ManagedClusters[idx].ClusterServerURL, + testOptions.ManagedClusters[idx].KubeConfig, + testOptions.ManagedClusters[idx].KubeContext, + yamlB, + ) + Expect(err).NotTo(HaveOccurred()) + } + } + } + + By("Add annotations to disable alert forward") + mco, getErr := dynClient.Resource(utils.NewMCOGVRV1BETA2()).Get(context.TODO(), MCO_CR_NAME, metav1.GetOptions{}) + if getErr != nil { + klog.Errorf("err: %+v\n", err) + } + spec := mco.Object["metadata"].(map[string]interface{}) + annotations, found := spec["annotations"].(map[string]interface{}) + if !found { + annotations = make(map[string]interface{}) + } + annotations["mco-disable-alerting"] = "true" + spec["annotations"] = annotations + _, updateErr := dynClient.Resource(utils.NewMCOGVRV1BETA2()).Update(context.TODO(), mco, metav1.UpdateOptions{}) + if updateErr != nil { + klog.Errorf("err: %+v\n", err) + } + + By("Checking Watchdog alerts is not forwarded to the hub") + Eventually(func() bool { + resp, err := client.Do(alertGetReq) + if err != nil { + klog.Errorf("err: %+v\n", err) + return false + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + klog.Errorf("err: %+v\n", resp) + return false + } + + alertResult, err := io.ReadAll(resp.Body) + if err != nil { + return false + } + + postableAlerts := models.PostableAlerts{} + err = json.Unmarshal(alertResult, &postableAlerts) + if err != nil { + return false + } + klog.V(3).Infof("postableAlerts is %+v", postableAlerts) + + for _, alt := range postableAlerts { + klog.V(3).Infof("alt.Labels is %s", alt.Labels) + if alt.Labels != nil { + klog.V(3).Infof("waiting alerts are disappeared?") + return false + } + } + + return true + }, EventuallyTimeoutMinute*10, EventuallyIntervalSecond*120).Should(BeTrue()) + + klog.V(3).Infof("before enable alert forward - spec is %s", spec) + mco1, getErr := dynClient.Resource(utils.NewMCOGVRV1BETA2()).Get(context.TODO(), MCO_CR_NAME, metav1.GetOptions{}) + if getErr != nil { + klog.Errorf("err: %+v\n", err) + } + spec1 := mco1.Object["metadata"].(map[string]interface{}) + delete(spec1["annotations"].(map[string]interface{}), "mco-disable-alerting") + _, updateErr1 := dynClient.Resource(utils.NewMCOGVRV1BETA2()).Update(context.TODO(), mco1, metav1.UpdateOptions{}) + if updateErr1 != nil { + klog.Errorf("err: %+v\n", updateErr1) + } + klog.V(3).Infof("enable alert forward - spec1 is %s", spec1) }) JustAfterEach(func() { diff --git a/tests/pkg/tests/observability_certrenew_test.go b/tests/pkg/tests/observability_certrenew_test.go index cdebc9b8c..201b5aab7 100644 --- a/tests/pkg/tests/observability_certrenew_test.go +++ b/tests/pkg/tests/observability_certrenew_test.go @@ -33,7 +33,7 @@ var _ = Describe("Observability:", func() { } }) - It("[P1][Sev1][observability][Integration] Should have metrics collector pod restart if cert secret re-generated (certrenew/g0)", func() { + It("RHACM4K-3073: Observability: Verify Observability Certificate rotation - Should have metrics collector pod restart if cert secret re-generated [P1][Sev1][Observability][Integration]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @post-release (certrenew/g0)", func() { By("Waiting for pods ready: observability-observatorium-api, observability-rbac-query-proxy, metrics-collector-deployment") // sleep 30s to wait for installation is ready time.Sleep(30 * time.Second) diff --git a/tests/pkg/tests/observability_config_test.go b/tests/pkg/tests/observability_config_test.go index 10336a7ed..e721997cf 100644 --- a/tests/pkg/tests/observability_config_test.go +++ b/tests/pkg/tests/observability_config_test.go @@ -7,7 +7,8 @@ package tests import ( "context" "fmt" - "os" + "strings" + "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -15,6 +16,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog" + "github.com/stolostron/multicluster-observability-operator/tests/pkg/kustomize" "github.com/stolostron/multicluster-observability-operator/tests/pkg/utils" ) @@ -31,10 +33,85 @@ var _ = Describe("Observability:", func() { testOptions.HubCluster.KubeContext) }) - It("@BVT - [P1][Sev1][observability][Stable] Checking metrics default values on managed cluster (config/g0)", func() { - if os.Getenv("SKIP_INSTALL_STEP") == trueStr { - Skip("Skip the case due to MCO CR was created customized") - } + It("RHACM4K-31474: Observability: Verify memcached setting max_item_size is populated on thanos-store - [P1][Sev1][Observability][Stable]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @post-release(config/g1)", func() { + + By("Updating mco cr to update values in storeMemcached") + yamlB, err := kustomize.Render(kustomize.Options{KustomizationPath: "../../../examples/maxitemsize/updatemcocr"}) + Expect(err).ToNot(HaveOccurred()) + Expect( + utils.Apply( + testOptions.HubCluster.ClusterServerURL, + testOptions.KubeConfig, + testOptions.HubCluster.KubeContext, + yamlB, + )).NotTo(HaveOccurred()) + + time.Sleep(60 * time.Second) + + By("Check the value is effect in the sts observability-thanos-store-shard-0") + Eventually(func() bool { + + thanosStoreMemSts, _ := utils.GetStatefulSet(testOptions, true, "observability-thanos-store-memcached", MCO_NAMESPACE) + //klog.V(3).Infof("STS thanosStoreSts is %s", thanosStoreMemSts) + containers := thanosStoreMemSts.Spec.Template.Spec.Containers + + args := containers[0].Args + //klog.V(3).Infof("args is %s", args) + + argsStr := strings.Join(args, " ") + //klog.V(3).Infof("argsStr is %s", argsStr) + + if !strings.Contains(argsStr, "-I 10m") { + klog.V(3).Infof("maxItemSize is not effect in sts observability-thanos-store-memcached") + return false + } + + klog.V(3).Infof("maxItemSize is effect in sts observability-thanos-store-memcached") + return true + + }, EventuallyTimeoutMinute*1, EventuallyIntervalSecond*10).Should(BeTrue()) + }) + + It("RHACM4K-31475: Observability: Verify memcached setting max_item_size is populated on thanos-query-frontend - [P1][Sev1][Observability][Stable]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @post-release(config/g1)", func() { + + By("Updating mco cr to update values in storeMemcached") + yamlB, err := kustomize.Render(kustomize.Options{KustomizationPath: "../../../examples/maxitemsize/updatemcocr"}) + Expect(err).ToNot(HaveOccurred()) + Expect( + utils.Apply( + testOptions.HubCluster.ClusterServerURL, + testOptions.KubeConfig, + testOptions.HubCluster.KubeContext, + yamlB, + )).NotTo(HaveOccurred()) + + time.Sleep(60 * time.Second) + + By("Check the value is effect in the sts observability-thanos-store-shard-0") + Eventually(func() bool { + + thanosQueFronMemSts, _ := utils.GetStatefulSet(testOptions, true, "observability-thanos-query-frontend-memcached", MCO_NAMESPACE) + //klog.V(3).Infof("STS thanosStoreSts is %s", thanosQueFronMemSts) + containers := thanosQueFronMemSts.Spec.Template.Spec.Containers + + args := containers[0].Args + //klog.V(3).Infof("args is %s", args) + + argsStr := strings.Join(args, " ") + //klog.V(3).Infof("argsStr is %s", argsStr) + + if !strings.Contains(argsStr, "-I 10m") { + klog.V(3).Infof("maxItemSize is not effect in sts observability-thanos-query-frontend-memcached") + return false + } + + klog.V(3).Infof("maxItemSize is effect in sts observability-thanos-query-frontend-memcached") + return true + + }, EventuallyTimeoutMinute*1, EventuallyIntervalSecond*10).Should(BeTrue()) + }) + + It("RHACM4K-1235: Observability: Verify metrics data global setting on the managed cluster @BVT - [P1][Sev1][Observability][Stable]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @post-release(config/g0)", func() { mcoRes, err := dynClient.Resource(utils.NewMCOGVRV1BETA2()). Get(context.TODO(), MCO_CR_NAME, metav1.GetOptions{}) if err != nil { @@ -42,13 +119,10 @@ var _ = Describe("Observability:", func() { } observabilityAddonSpec := mcoRes.Object["spec"].(map[string]interface{})["observabilityAddonSpec"].(map[string]interface{}) Expect(observabilityAddonSpec["enableMetrics"]).To(Equal(true)) - Expect(observabilityAddonSpec["interval"]).To(Equal(int64(30))) + Expect(observabilityAddonSpec["interval"]).To(Equal(int64(300))) }) - It("@BVT - [P1][Sev1][observability][Stable] Checking default value of PVC and StorageClass (config/g0)", func() { - if os.Getenv("SKIP_INSTALL_STEP") == trueStr { - Skip("Skip the case due to MCO CR was created customized") - } + It("RHACM4K-1065: Observability: Verify MCO CR storage class and PVC @BVT - [P1][Sev1][Observability][Stable]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @post-release (config/g0)", func() { mcoSC, err := dynClient.Resource(utils.NewMCOGVRV1BETA2()). Get(context.TODO(), MCO_CR_NAME, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -153,7 +227,7 @@ var _ = Describe("Observability:", func() { }, } - It("@BVT - [P1][Sev1][observability][Integration] Checking replicas in advanced config for each component (config/g0)", func() { + It("RHACM4K-2822: Observability: Verify the replica in advanced config for Observability components @BVT - [P1][Sev1][Observability][Integration] @e2e (config/g0)", func() { mcoRes, err := dynClient.Resource(utils.NewMCOGVRV1BETA2()). Get(context.TODO(), MCO_CR_NAME, metav1.GetOptions{}) @@ -190,9 +264,10 @@ var _ = Describe("Observability:", func() { } }) - It("[P2][Sev2][observability][Integration] Checking resources in advanced config (config/g0)", func() { + It("RHACM4K-3419: Observability: Persist advance values in MCO CR - Checking resources in advanced config [P2][Sev2][Observability][Integration] @e2e @post-release @pre-upgrade (config/g0)", func() { mcoRes, err := dynClient.Resource(utils.NewMCOGVRV1BETA2()). Get(context.TODO(), MCO_CR_NAME, metav1.GetOptions{}) + if err != nil { panic(err.Error()) } @@ -236,7 +311,7 @@ var _ = Describe("Observability:", func() { } }) - It("[P2][Sev2][observability][Integration] Checking service account annotations is set for store/query/rule/compact/receive (config/g0)", func() { + It("RHACM4K-11169: Observability: Verify ACM Observability with Security Service Token credentials - [P2][Sev2][observability][Integration]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @pre-upgrade Checking service account annotations is set for store/query/rule/compact/receive @e2e (config/g0)", func() { mcoRes, err := dynClient.Resource(utils.NewMCOGVRV1BETA2()). Get(context.TODO(), MCO_CR_NAME, metav1.GetOptions{}) diff --git a/tests/pkg/tests/observability_dashboard_test.go b/tests/pkg/tests/observability_dashboard_test.go index b4267ad80..bb5c5c6f7 100644 --- a/tests/pkg/tests/observability_dashboard_test.go +++ b/tests/pkg/tests/observability_dashboard_test.go @@ -33,7 +33,7 @@ var _ = Describe("Observability:", func() { testOptions.HubCluster.KubeContext) }) - It("[P2][Sev2][observability][Stable] Should have custom dashboard which defined in configmap (dashboard/g0)", func() { + It("RHACM4K-1669: Observability: Verify new customized Grafana dashboard - Should have custom dashboard which defined in configmap [P2][Sev2][Observability][Stable]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @post-release @pre-upgrade (dashboard/g0)", func() { By("Creating custom dashboard configmap") yamlB, _ := kustomize.Render( kustomize.Options{KustomizationPath: "../../../examples/dashboards/sample_custom_dashboard"}, @@ -47,10 +47,10 @@ var _ = Describe("Observability:", func() { Eventually(func() bool { _, result := utils.ContainDashboard(testOptions, dashboardTitle) return result - }, EventuallyTimeoutMinute*5, EventuallyIntervalSecond*5).Should(BeTrue()) + }, EventuallyTimeoutMinute*7, EventuallyIntervalSecond*5).Should(BeTrue()) }) - It("[P2][Sev2][observability][Stable] Should have update custom dashboard after configmap updated (dashboard/g0)", func() { + It("RHACM4K-1669: Observability: Verify new customized Grafana dashboard - Should have update custom dashboard after configmap updated [P2][Sev2][Observability][Stable]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @post-release @pre-upgrade (dashboard/g0)", func() { By("Updating custom dashboard configmap") yamlB, _ := kustomize.Render( kustomize.Options{KustomizationPath: "../../../examples/dashboards/update_sample_custom_dashboard"}, @@ -71,7 +71,7 @@ var _ = Describe("Observability:", func() { }, EventuallyTimeoutMinute*7, EventuallyIntervalSecond*5).Should(BeTrue()) }) - It("[P2][Sev2][observability][Stable] Should have no custom dashboard in grafana after related configmap removed (dashboard/g0)", func() { + It("RHACM4K-1669: Observability: Verify new customized Grafana dashboard - Should have no custom dashboard in grafana after related configmap removed [P2][Sev2][Observability][Stable]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @post-release @pre-upgrade (dashboard/g0)", func() { By("Deleting custom dashboard configmap") err = utils.DeleteConfigMap(testOptions, true, dashboardName, MCO_NAMESPACE) Expect(err).ToNot(HaveOccurred()) @@ -81,17 +81,7 @@ var _ = Describe("Observability:", func() { }, EventuallyTimeoutMinute*3, EventuallyIntervalSecond*5).Should(BeFalse()) }) - JustAfterEach(func() { - Expect(utils.IntegrityChecking(testOptions)).NotTo(HaveOccurred()) - }) - - AfterEach(func() { - if CurrentGinkgoTestDescription().Failed { - utils.LogFailingTestStandardDebugInfo(testOptions) - } - testFailed = testFailed || CurrentGinkgoTestDescription().Failed - }) - + // TODO (jacob): Need RHACM4K no It("[P2][Sev2][observability][Stable] Should have default overview dashboards (dashboard/g0)", func() { // Check Original dash exists Eventually(func() bool { diff --git a/tests/pkg/tests/observability_deployment_test.go b/tests/pkg/tests/observability_deployment_test.go new file mode 100644 index 000000000..9e6a9b972 --- /dev/null +++ b/tests/pkg/tests/observability_deployment_test.go @@ -0,0 +1,119 @@ +// Copyright (c) Red Hat, Inc. +// Copyright Contributors to the Open Cluster Management project +// Licensed under the Apache License 2.0 + +package tests + +import ( + "context" + "fmt" + "os" + "strings" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog" + + "github.com/stolostron/multicluster-observability-operator/tests/pkg/utils" +) + +var _ = Describe("", func() { + BeforeEach(func() { + hubClient = utils.NewKubeClient( + testOptions.HubCluster.ClusterServerURL, + testOptions.KubeConfig, + testOptions.HubCluster.KubeContext) + + dynClient = utils.NewKubeClientDynamic( + testOptions.HubCluster.ClusterServerURL, + testOptions.KubeConfig, + testOptions.HubCluster.KubeContext) + }) + + It("RHACM4K-1064: Observability: Verify MCO deployment - [P1][Sev1][Observability][Stable]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @post-release @pre-upgrade (deployment/g0)", func() { + By("Check MCO in ready status") + Eventually(func() error { + err = utils.CheckMCOComponents(testOptions) + if err != nil { + testFailed = true + return err + } + testFailed = false + return nil + }, EventuallyTimeoutMinute*25, EventuallyIntervalSecond*10).Should(Succeed()) + + By("Check clustermanagementaddon CR is created") + Eventually(func() error { + _, err := dynClient.Resource(utils.NewMCOClusterManagementAddonsGVR()).Get(context.TODO(), "observability-controller", metav1.GetOptions{}) + if err != nil { + testFailed = true + return err + } + testFailed = false + return nil + }, EventuallyTimeoutMinute*5, EventuallyIntervalSecond*5).Should(Succeed()) + + }) + + It("RHACM4K-1288: Observability: Verify Observability function working on the hub cluster - [P1][Sev1][Observability][Stable]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @post-release @pre-upgrade (deployment/g0)", func() { + By("Check etrics-collector pod is ready") + Eventually(func() error { + + err, podList := utils.GetPodList( + testOptions, + false, + "open-cluster-management-observability", + "component=metrics-collector", + ) + + if err != nil { + return fmt.Errorf("Failed to get the pod metrics-collector") + } + if len(podList.Items) != 0 { + for _, po := range podList.Items { + if po.Status.Phase == "Running" { + klog.V(1).Infof("metrics-collector pod in Running") + return nil + } + } + } + return nil + /* + err = utils.CheckAllOBAsEnabledLocal(testOptions) + if err != nil { + testFailed = true + return err + } + testFailed = false + return nil + */ + }, EventuallyTimeoutMinute*20, EventuallyIntervalSecond*10).Should(Succeed()) + + }) + + It("RHACM4K-30645: Observability: Verify setting in CM cluster-monitoring-config is not removed after MCO enabled - [P1][Sev1][Observability][Stable]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @post-release @pre-upgrade (deployment/g1)", func() { + By("Check enableUserAlertmanagerConfig value is not replaced in the CM cluster-monitoring-config") + if os.Getenv("SKIP_INSTALL_STEP") == "true" { + Skip("Skip the case due to this case is only available before MCOCR deployment") + } + Eventually(func() bool { + + cm, err := hubClient.CoreV1().ConfigMaps("openshift-monitoring").Get(context.TODO(), "cluster-monitoring-config", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + if strings.Contains(cm.String(), "enableUserAlertmanagerConfig: true") { + return true + } + return false + }, EventuallyTimeoutMinute*5, EventuallyIntervalSecond*5).Should(BeTrue()) + }) + + AfterEach(func() { + if CurrentGinkgoTestDescription().Failed { + utils.LogFailingTestStandardDebugInfo(testOptions) + } + testFailed = testFailed || CurrentGinkgoTestDescription().Failed + }) + +}) diff --git a/tests/pkg/tests/observability_endpoint_preserve_test.go b/tests/pkg/tests/observability_endpoint_preserve_test.go index c79d31332..191347dea 100644 --- a/tests/pkg/tests/observability_endpoint_preserve_test.go +++ b/tests/pkg/tests/observability_endpoint_preserve_test.go @@ -35,7 +35,7 @@ var _ = Describe("Observability:", func() { } }) - Context("[P2][Sev2][observability] Should revert any manual changes on metrics-collector deployment (endpoint_preserve/g0) -", func() { + Context("RHACM4K-1659: Observability: Verify metrics collector is prevent to be configured manually [P2][Sev2][Observability]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @post-release (endpoint_preserve/g0) -", func() { newDep := &appv1.Deployment{} It("[Stable] Deleting metrics-collector deployment for cluster", func() { var ( @@ -118,10 +118,11 @@ var _ = Describe("Observability:", func() { }) }) - It("[P2][Sev2][observability][Stable] Should revert any manual changes on metrics-collector-view clusterolebinding (endpoint_preserve/g0)", func() { - if os.Getenv("IS_KIND_ENV") == trueStr { + It("RHACM4K-1659: Observability: Verify metrics collector is prevent to be configured manually - Should revert any manual changes on metrics-collector-view clusterolebinding [P2][Sev2][Observability][Stable]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @post-release (endpoint_preserve/g0)", func() { + if os.Getenv("IS_KIND_ENV") == "true" { Skip("Skip the case due to run in KinD") } + By("Deleting metrics-collector-view clusterolebinding") err, crb := utils.GetCRB(testOptions, false, "metrics-collector-view") Expect(err).ToNot(HaveOccurred()) @@ -155,8 +156,8 @@ var _ = Describe("Observability:", func() { }, EventuallyTimeoutMinute*1, EventuallyIntervalSecond*1).Should(BeTrue()) }) - It("[P2][Sev2][observability][Stable] Should recreate on metrics-collector-serving-certs-ca-bundle configmap if deleted (endpoint_preserve/g0)", func() { - if os.Getenv("IS_KIND_ENV") == trueStr { + It("RHACM4K-1659: Observability: Verify metrics collector is prevent to be configured manually - Should recreate on metrics-collector-serving-certs-ca-bundle configmap if deleted [P2][Sev2][Observability][Stable]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @post-release (endpoint_preserve/g0)", func() { + if os.Getenv("IS_KIND_ENV") == "true" { Skip("Skip the case due to run in KinD") } @@ -211,6 +212,5 @@ var _ = Describe("Observability:", func() { namespace = MCO_ADDON_NAMESPACE testFailed = testFailed || CurrentGinkgoTestDescription().Failed isHub = false - }) }) diff --git a/tests/pkg/tests/observability_export_test.go b/tests/pkg/tests/observability_export_test.go index 508ca1159..9fda0a333 100644 --- a/tests/pkg/tests/observability_export_test.go +++ b/tests/pkg/tests/observability_export_test.go @@ -38,7 +38,7 @@ var _ = Describe("Observability:", func() { }, EventuallyTimeoutMinute*6, EventuallyIntervalSecond*5).Should(Succeed()) }) - It("[P2][Sev2][observability][Integration] Should have acm_remote_write_requests_total metrics with correct labels/value (export/g0)", func() { + It("RHACM4K-11170: Observability: Verify metrics would be exported to corp tools(2.5)(draft)[P2][Sev2][observability][Integration] Should have acm_remote_write_requests_total metrics with correct labels/value @e2e (export/g0)", func() { By("Adding victoriametrics deployment/service/secret") yamlB, err := kustomize.Render(kustomize.Options{KustomizationPath: "../../../examples/export"}) Expect(err).ToNot(HaveOccurred()) diff --git a/tests/pkg/tests/observability_grafana_dev_test.go b/tests/pkg/tests/observability_grafana_dev_test.go index 13fe02af2..c419166e2 100644 --- a/tests/pkg/tests/observability_grafana_dev_test.go +++ b/tests/pkg/tests/observability_grafana_dev_test.go @@ -20,7 +20,7 @@ var _ = Describe("Observability:", func() { // Do not need to run this case in canary environment // If we really need it in canary, ensure the grafana-dev-test.sh is available // in observability-e2e-test image and all required commands exist - It("[P1][Sev1][observability][Integration] Should run grafana-dev test successfully (grafana_dev/g0)", func() { + It("RHACM4K-1705: Observability: Setup a Grafana develop instance [P1][Sev1][Observability][Integration]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @post-release (grafana_dev/g0)", func() { cmd := exec.Command("../../grafana-dev-test.sh") var out bytes.Buffer cmd.Stdout = &out diff --git a/tests/pkg/tests/observability_grafana_test.go b/tests/pkg/tests/observability_grafana_test.go index 908351401..caf623cd0 100644 --- a/tests/pkg/tests/observability_grafana_test.go +++ b/tests/pkg/tests/observability_grafana_test.go @@ -5,10 +5,16 @@ package tests import ( + "context" + "encoding/json" "fmt" + "strings" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog" "github.com/stolostron/multicluster-observability-operator/tests/pkg/utils" ) @@ -26,7 +32,7 @@ var _ = Describe("Observability:", func() { testOptions.HubCluster.KubeContext) }) - It("@BVT - [P1][Sev1][observability][Stable] Should have metric data in grafana console (grafana/g0)", func() { + It("RHACM4K-1066: Observability: Verify Grafana - Should have metric data in grafana console @BVT - [P1][Sev1][Observability][Stable]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @post-release (grafana/g0)", func() { Eventually(func() error { clusters, err := utils.ListManagedClusters(testOptions) if err != nil { @@ -50,6 +56,57 @@ var _ = Describe("Observability:", func() { }, EventuallyTimeoutMinute*6, EventuallyIntervalSecond*5).Should(Succeed()) }) + It("RHACM4K-23537: Observability: Verify managed cluster labels in Grafana dashboards(2.7) - [P1][Sev1][Observability][Stable]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @post-release (grafana/g1)", func() { + Eventually(func() bool { + clientDynamic := utils.GetKubeClientDynamic(testOptions, true) + objs, err := clientDynamic.Resource(utils.NewOCMManagedClustersGVR()).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + klog.V(1).Infof("Get the managedcluster failed, The err is: %s\n", err) + } + + for _, obj := range objs.Items { + metadata := obj.Object["metadata"].(map[string]interface{}) + name := metadata["name"].(string) + if name == "local-cluster" { + labels := metadata["labels"].(map[string]interface{}) + labels["autolabel"] = "grafanacm" + klog.V(1).Infof("The cluster with new label: %s\n", labels) + _, updateErr := clientDynamic.Resource(utils.NewOCMManagedClustersGVR()).Update(context.TODO(), &obj, metav1.UpdateOptions{}) + if updateErr != nil { + klog.V(1).Infof("Update label failed, updateErr is : %s\n", updateErr) + } + } + + } + + var ( + errcm error + cm *v1.ConfigMap + ) + errcm, cm = utils.GetConfigMap( + testOptions, + false, + "observability-managed-cluster-label-allowlist", + MCO_NAMESPACE, + ) + if errcm != nil { + klog.V(1).Infof("The errcm is: %s\n", errcm) + } + + data, err := json.Marshal(cm) + if err != nil { + klog.V(1).Infof("The err is: %s\n", err) + } + if !strings.Contains(string(data), "autolabel") { + klog.V(1).Infof("new managedcluster label autolabel is NOT added into configmap observability-managed-cluster-label-allowlist") + return false + } else { + klog.V(1).Infof("new managedcluster label autolabel is added into configmap observability-managed-cluster-label-allowlist") + return true + } + }, EventuallyTimeoutMinute*1, EventuallyIntervalSecond*10).Should(BeTrue()) + }) + JustAfterEach(func() { Expect(utils.IntegrityChecking(testOptions)).NotTo(HaveOccurred()) }) diff --git a/tests/pkg/tests/observability_install_test.go b/tests/pkg/tests/observability_install_test.go index 485196ba3..95bbd5b93 100644 --- a/tests/pkg/tests/observability_install_test.go +++ b/tests/pkg/tests/observability_install_test.go @@ -37,6 +37,19 @@ func installMCO() { testOptions.KubeConfig, testOptions.HubCluster.KubeContext) + // TODO (jacob): The test RHACM4K-30645 depends on the below. Should that test maybe be moved here? + By("Deploy CM cluster-monitoring-config") + + yamlBc, _ := kustomize.Render( + kustomize.Options{KustomizationPath: "../../../examples/configmapcmc/cluster-monitoring-config"}, + ) + Expect( + utils.Apply( + testOptions.HubCluster.ClusterServerURL, + testOptions.KubeConfig, + testOptions.HubCluster.KubeContext, + yamlBc)).NotTo(HaveOccurred()) + By("Checking MCO operator is started up and running") podList, err := hubClient.CoreV1().Pods("").List(context.TODO(), metav1.ListOptions{LabelSelector: MCO_LABEL}) Expect(len(podList.Items)).To(Equal(1)) @@ -52,6 +65,33 @@ func installMCO() { Expect(string(pod.Status.Phase)).To(Equal("Running")) } + // print mco logs if MCO installation failed + // defer func(testOptions utils.TestOptions, isHub bool, namespace, podName, containerName string, previous bool, tailLines int64) { + // if testFailed { + // mcoLogs, err := utils.GetPodLogs( + // testOptions, + // isHub, + // namespace, + // podName, + // containerName, + // previous, + // tailLines, + // ) + // Expect(err).NotTo(HaveOccurred()) + // fmt.Fprintf(GinkgoWriter, "[DEBUG] MCO is installed failed, checking MCO operator logs:\n%s\n", mcoLogs) + // } else { + // fmt.Fprintf(GinkgoWriter, "[DEBUG] MCO is installed successfully!\n") + // } + // }( + // testOptions, + // false, + // mcoNs, + // mcoPod, + // "multicluster-observability-operator", + // false, + // 1000, + // ) + By("Checking Required CRDs are created") Eventually(func() error { return utils.HaveCRDs(testOptions.HubCluster, testOptions.KubeConfig, diff --git a/tests/pkg/tests/observability_manifestwork_test.go b/tests/pkg/tests/observability_manifestwork_test.go index 193abd615..ac7497a42 100644 --- a/tests/pkg/tests/observability_manifestwork_test.go +++ b/tests/pkg/tests/observability_manifestwork_test.go @@ -32,7 +32,7 @@ var _ = Describe("Observability:", func() { } }) - Context("[P2][Sev2][observability][Stable] Should be automatically created within 1 minute when delete manifestwork (manifestwork/g0) -", func() { + Context("[P2][Sev2][observability][Stable] Should be automatically created within 1 minute when delete manifestwork @ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @post-release (manifestwork/g0) -", func() { manifestWorkName := "endpoint-observability-work" clientDynamic := utils.GetKubeClientDynamic(testOptions, true) clusterName := utils.GetManagedClusterName(testOptions) diff --git a/tests/pkg/tests/observability_metrics_test.go b/tests/pkg/tests/observability_metrics_test.go index 1b8224b3a..deae1d758 100644 --- a/tests/pkg/tests/observability_metrics_test.go +++ b/tests/pkg/tests/observability_metrics_test.go @@ -50,7 +50,34 @@ var _ = Describe("Observability:", func() { }, EventuallyTimeoutMinute*6, EventuallyIntervalSecond*5).Should(Succeed()) }) - It("[P2][Sev2][observability][Integration] Should have metrics which defined in custom metrics allowlist (metrics/g0)", func() { + // TODO (jacob): exact same as RHACM4K-3339?? + // It("RHACM4K-1449 - Observability - Verify metrics data consistency [P2][Sev2][Observability][Integration]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @post-release @pre-upgrade (metrics/g1)", func() { + // metricList := utils.GetDefaultMetricList(testOptions) + // _, etcdPodList := utils.GetPodList( + // testOptions, + // true, + // "openshift-etcd", + // "app=etcd", + // ) + // + // for _, name := range metricList { + // if _, ok := ignoredMetrics[name]; ok { + // continue + // } + // + // Eventually(func() error { + // res, err := utils.QueryGrafana(testOptions, query) + // if err != nil { + // return err + // } + // if len(res.Data.Result) == 0 { + // return fmt.Errorf("no data found for %s", query) + // } + // }, EventuallyTimeoutMinute*2, EventuallyIntervalSecond*3).Should(Succeed()) + // } + // }) + + It("RHACM4K-1658: Observability: Customized metrics data are collected [P2][Sev2][Observability][Integration]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @post-release @pre-upgrade (metrics/g0)", func() { By("Adding custom metrics allowlist configmap") yamlB, err := kustomize.Render(kustomize.Options{KustomizationPath: "../../../examples/metrics/allowlist"}) Expect(err).ToNot(HaveOccurred()) @@ -82,7 +109,7 @@ var _ = Describe("Observability:", func() { }, EventuallyTimeoutMinute*10, EventuallyIntervalSecond*5).Should(Succeed()) }) - It("[P2][Sev2][observability][Integration] Should have no metrics which have been marked for deletion in names section (metrics/g0)", func() { + It("RHACM4K-3063: Observability: Metrics removal from default allowlist [P2][Sev2][Observability][Integration]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @post-release @pre-upgrade (metrics/g0)", func() { By("Waiting for deleted metrics disappear on grafana console") Eventually(func() error { for _, cluster := range clusters { @@ -104,7 +131,7 @@ var _ = Describe("Observability:", func() { }, EventuallyTimeoutMinute*10, EventuallyIntervalSecond*5).Should(Succeed()) }) - It("[P2][Sev2][observability][Integration] Should have no metrics which have been marked for deletion in matches section (metrics/g0)", func() { + It("RHACM4K-3063: Observability: Metrics removal from default allowlist [P2][Sev2][Observability][Integration]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @post-release @pre-upgrade (metrics/g0)", func() { By("Waiting for deleted metrics disappear on grafana console") Eventually(func() error { for _, cluster := range clusters { @@ -125,7 +152,7 @@ var _ = Describe("Observability:", func() { }, EventuallyTimeoutMinute*10, EventuallyIntervalSecond*5).Should(Succeed()) }) - It("[P2][Sev2][observability][Integration] Should have no metrics after custom metrics allowlist deleted (metrics/g0)", func() { + It("RHACM4K-3063: Observability: Metrics removal from default allowlist [P2][Sev2][Observability][Integration]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @post-release @pre-upgrade (metrics/g0)", func() { By("Deleting custom metrics allowlist configmap") Eventually(func() error { err := hubClient.CoreV1(). @@ -154,6 +181,7 @@ var _ = Describe("Observability:", func() { }, EventuallyTimeoutMinute*10, EventuallyIntervalSecond*5).Should(Succeed()) }) + // TODO Jacob: RHACM4K number // Ensures that the allowList is current by checking that the metrics are being collected It("[P2][Sev2][observability][Integration] Should collect expected metrics from spokes (metrics/g0)", func() { // Get the metrics from the deployed allowList configMap @@ -221,6 +249,30 @@ var _ = Describe("Observability:", func() { } }) + It("RHACM4K-3339: Observability: Verify recording rule - Should have metrics which used grafana dashboard [P2][Sev2][Observability][Integration]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore @e2e @post-release @pre-upgrade (ssli/g1)", func() { + metricList, _ := utils.GetDefaultMetricList(testOptions) + + for _, name := range metricList { + + if _, ok := ignoredMetrics[name]; ok { + continue + } + + Eventually(func() error { + query := fmt.Sprintf("%s", name) + res, err := utils.QueryGrafana(testOptions, query) + + if err != nil { + return err + } + if len(res.Data.Result) == 0 { + return fmt.Errorf("no data found for %s", query) + } + return nil + }, EventuallyTimeoutMinute*2, EventuallyIntervalSecond*3).Should(Succeed()) + } + }) + JustAfterEach(func() { Expect(utils.IntegrityChecking(testOptions)).NotTo(HaveOccurred()) }) diff --git a/tests/pkg/tests/observability_observatorium_preserve_test.go b/tests/pkg/tests/observability_observatorium_preserve_test.go index b1bbd2aab..1a49aa26f 100644 --- a/tests/pkg/tests/observability_observatorium_preserve_test.go +++ b/tests/pkg/tests/observability_observatorium_preserve_test.go @@ -29,7 +29,7 @@ var _ = Describe("Observability:", func() { testOptions.HubCluster.KubeContext) }) - Context("[P1][Sev1][observability] Should revert any manual changes on observatorium cr (observatorium_preserve/g0) -", func() { + Context("RHACM4K-1443: Observability: Verify Observatorium CR configuration compliance [P1][Sev1][Observability]@post-upgrade @post-restore @e2e @post-release (observatorium_preserve/g0) -", func() { It("[Stable] Updating observatorium cr (spec.thanos.compact.retentionResolution1h) should be automatically reverted", func() { oldCRResourceVersion := "" updateRetention := "10d" diff --git a/tests/pkg/tests/observability_reconcile_test.go b/tests/pkg/tests/observability_reconcile_test.go index c37470a64..1dd085138 100644 --- a/tests/pkg/tests/observability_reconcile_test.go +++ b/tests/pkg/tests/observability_reconcile_test.go @@ -7,6 +7,8 @@ package tests import ( "context" "fmt" + "os" + "strings" "time" . "github.com/onsi/ginkgo" @@ -41,7 +43,13 @@ var _ = Describe("Observability:", func() { testOptions.HubCluster.KubeContext) }) - It("[P2][Sev2][observability][Stable] Modifying MCO CR for reconciling (reconcile/g0)", func() { + It("RHACM4K-2881: Observability: Check and tune backup retention settings in MCO CR - tune retention settings in MCO CR [P2][Sev2][Observability][Stable] @e2e @post-release @post-upgrade @post-restore (reconcile/g0)", func() { + cloudProvider := strings.ToLower(os.Getenv("CLOUD_PROVIDER")) + substring1 := "vmware" + substring2 := "ibm" + if strings.Contains(cloudProvider, substring1) || strings.Contains(cloudProvider, substring2) { + Skip("Skip the case due to it's not supported on the VMWARE and IBM") + } By("Modifying MCO CR for reconciling") err := utils.ModifyMCOCR(testOptions) Expect(err).ToNot(HaveOccurred()) @@ -97,7 +105,7 @@ var _ = Describe("Observability:", func() { }, EventuallyTimeoutMinute*10, EventuallyIntervalSecond*5).Should(Succeed()) }) - It("[P2][Sev2][observability][Stable] Checking node selector for all pods (reconcile/g0)", func() { + It("RHACM4K-1655: Observability: Verify nodeSelector setting effects for Observability components [P2][Sev2][Observability][Stable] @e2e @post-release @post-upgrade @post-restore (reconcile/g0)", func() { By("Checking node selector spec in MCO CR") mcoSC, err := dynClient.Resource(utils.NewMCOGVRV1BETA2()). Get(context.TODO(), MCO_CR_NAME, metav1.GetOptions{}) @@ -118,7 +126,7 @@ var _ = Describe("Observability:", func() { }, EventuallyTimeoutMinute*5, EventuallyIntervalSecond*5).Should(Succeed()) }) - It("[P2][Sev2][observability][Stable] Checking podAntiAffinity for all pods (reconcile/g0)", func() { + It("RHACM4K-1657: Observability: Check affinity rule takes effect on Observability components [P2][Sev2][Observability][Stable] @e2e @post-release @post-upgrade @post-restore (reconcile/g0)", func() { By("Checking podAntiAffinity for all pods") Eventually(func() error { err := utils.CheckAllPodsAffinity(testOptions) @@ -129,7 +137,14 @@ var _ = Describe("Observability:", func() { }, EventuallyTimeoutMinute*5, EventuallyIntervalSecond*5).Should(Succeed()) }) - It("[P2][Sev2][observability][Stable] Checking alertmanager storage resize (reconcile/g0)", func() { + It("RHACM4K-2821: Observability: Customize the Observability components storage size [P2][Sev2][Observability][Stable] @e2e @post-release @post-upgrade @post-restore (reconcile/g0)", func() { + cloudProvider := strings.ToLower(os.Getenv("CLOUD_PROVIDER")) + substring1 := "vmware" + substring2 := "ibm" + if strings.Contains(cloudProvider, substring1) || strings.Contains(cloudProvider, substring2) { + //if strings.Contains(string(os.Getenv("CLOUD_PROVIDER")), "VMWARE") { + Skip("Skip the case due to it's not supported on the VMWARE and IBM") + } By("Resizing alertmanager storage") alertmans, _ := hubClient.AppsV1().StatefulSets(MCO_NAMESPACE).List(context.TODO(), metav1.ListOptions{ LabelSelector: ALERTMANAGER_LABEL, @@ -137,7 +152,7 @@ var _ = Describe("Observability:", func() { Expect(len(alertmans.Items)).NotTo(Equal(0)) Eventually(func() error { - err := utils.CheckStorageResize(testOptions, (*alertmans).Items[0].Name, "2Gi") + err := utils.CheckStorageResize(testOptions, (*alertmans).Items[0].Name, "3Gi") if err != nil { return err } @@ -145,7 +160,14 @@ var _ = Describe("Observability:", func() { }, EventuallyTimeoutMinute*5, EventuallyIntervalSecond*5).Should(Succeed()) }) - It("[P2][Sev2][observability][Stable] Revert MCO CR changes (reconcile/g0)", func() { + It("RHACM4K-2881: Observability: Check and tune backup retention settings in MCO CR - Revert MCO CR changes [P2][Sev2][Observability][Stable] @e2e @post-release @post-upgrade @post-restore (reconcile/g0)", func() { + cloudProvider := strings.ToLower(os.Getenv("CLOUD_PROVIDER")) + substring1 := "vmware" + substring2 := "ibm" + if strings.Contains(cloudProvider, substring1) || strings.Contains(cloudProvider, substring2) { + //if strings.Contains(string(os.Getenv("CLOUD_PROVIDER")), "VMWARE") { + Skip("Skip the case due to it's not supported on the VMWARE and IBM") + } advRetentionCon, err := utils.CheckAdvRetentionConfig(testOptions) if !advRetentionCon { Skip("Skip the case since " + err.Error()) diff --git a/tests/pkg/tests/observability_retention_test.go b/tests/pkg/tests/observability_retention_test.go index acd9ea220..c2615ed2f 100644 --- a/tests/pkg/tests/observability_retention_test.go +++ b/tests/pkg/tests/observability_retention_test.go @@ -64,7 +64,7 @@ var _ = Describe("Observability:", func() { } }) - It("[P2][Sev2][observability][Stable] Check compact args (retention/g0):", func() { + It("RHACM4K-2881: Observability: Check and tune backup retention settings in MCO CR - Check compact args [P2][Sev2][Observability][Stable] @e2e @post-release @post-upgrade @post-restore (retention/g0):", func() { By("--delete-delay=" + deleteDelay) Eventually(func() error { compacts, err := hubClient.AppsV1().StatefulSets(MCO_NAMESPACE).List(context.TODO(), metav1.ListOptions{ @@ -83,7 +83,7 @@ var _ = Describe("Observability:", func() { }, EventuallyTimeoutMinute*1, EventuallyIntervalSecond*5).Should(Succeed()) }) - It("[P2][Sev2][observability][Stable] Check store args (retention/g0):", func() { + It("RHACM4K-2881: Observability: Check and tune backup retention settings in MCO CR - Check store args [P2][Sev2][Observability][Stable] @e2e @post-release @post-upgrade @post-restore (retention/g0):", func() { By("--ignore-deletion-marks-delay=" + ignoreDeletionMarksDelay) Eventually(func() error { stores, err := hubClient.AppsV1().StatefulSets(MCO_NAMESPACE).List(context.TODO(), metav1.ListOptions{ @@ -105,7 +105,7 @@ var _ = Describe("Observability:", func() { }, EventuallyTimeoutMinute*1, EventuallyIntervalSecond*5).Should(Succeed()) }) - It("[P2][Sev2][observability][Stable] Check receive args (retention/g0):", func() { + It("RHACM4K-2881: Observability: Check and tune backup retention settings in MCO CR - Check receive args [P2][Sev2][Observability][Stable] @e2e @post-release @post-upgrade @post-restore (retention/g0):", func() { By("--tsdb.retention=" + retentionInLocal) Eventually(func() error { receives, err := hubClient.AppsV1().StatefulSets(MCO_NAMESPACE).List(context.TODO(), metav1.ListOptions{ @@ -127,7 +127,7 @@ var _ = Describe("Observability:", func() { }, EventuallyTimeoutMinute*1, EventuallyIntervalSecond*5).Should(Succeed()) }) - It("[P2][Sev2][observability][Stable] Check rule args (retention/g0):", func() { + It("RHACM4K-2881: Observability: Check and tune backup retention settings in MCO CR - Check rule args [P2][Sev2][Observability][Stable] @e2e @post-release @post-upgrade @post-restore (retention/g0):", func() { By("--tsdb.retention=" + retentionInLocal) Eventually(func() error { rules, err := hubClient.AppsV1().StatefulSets(MCO_NAMESPACE).List(context.TODO(), metav1.ListOptions{ @@ -149,7 +149,7 @@ var _ = Describe("Observability:", func() { }, EventuallyTimeoutMinute*1, EventuallyIntervalSecond*5).Should(Succeed()) }) - It("[P2][Sev2][observability][Stable] Check rule args (retention/g0):", func() { + It("RHACM4K-2881: Observability: Check and tune backup retention settings in MCO CR - Check rule args [P2][Sev2][Observability][Stable] @e2e @post-release @post-upgrade @post-restore (retention/g0):", func() { By("--tsdb.block-duration=" + blockDuration) Eventually(func() error { rules, err := hubClient.AppsV1().StatefulSets(MCO_NAMESPACE).List(context.TODO(), metav1.ListOptions{ diff --git a/tests/pkg/tests/observability_route_test.go b/tests/pkg/tests/observability_route_test.go index 9b01930e9..860c06934 100644 --- a/tests/pkg/tests/observability_route_test.go +++ b/tests/pkg/tests/observability_route_test.go @@ -40,10 +40,25 @@ var _ = Describe("Observability:", func() { testOptions.HubCluster.KubeContext) }) - It("@BVT - [P1][Sev1][observability][Integration] Should access metrics via rbac-query-proxy route (route/g0)", func() { + It("RHACM4K-1693: Observability: Verify Observability working with new OCP API Server certs - @BVT - [P1][Sev1][observability][Integration]@ocpInterop @non-ui-post-restore @non-ui-post-release @non-ui-pre-upgrade @non-ui-post-upgrade @post-upgrade @post-restore Should access metrics via rbac-query-proxy route @e2e (route/g0)", func() { Eventually(func() error { query := "/api/v1/query?query=cluster_version" - url := "https://rbac-query-proxy-open-cluster-management-observability.apps." + testOptions.HubCluster.BaseDomain + query + + cloudProvider := strings.ToLower(os.Getenv("CLOUD_PROVIDER")) + substring1 := "rosa" + substring2 := "hcp" + + var url string + + if strings.Contains(cloudProvider, substring1) && strings.Contains(cloudProvider, substring2) { + Skip("skip on rosa-hcp") + url = "https://rbac-query-proxy-open-cluster-management-observability.apps.rosa." + testOptions.HubCluster.BaseDomain + query + + } else { + url = "https://rbac-query-proxy-open-cluster-management-observability.apps." + testOptions.HubCluster.BaseDomain + query + + } + req, err := http.NewRequest( "GET", url, @@ -63,6 +78,8 @@ var _ = Describe("Observability:", func() { client := &http.Client{} if os.Getenv("IS_KIND_ENV") != trueStr { client.Transport = tr + client.Transport = tr + BearerToken, err = utils.FetchBearerToken(testOptions) req.Header.Set("Authorization", "Bearer "+BearerToken) } @@ -93,6 +110,14 @@ var _ = Describe("Observability:", func() { It("@BVT - [P1][Sev1][observability][Integration] Should access alert via alertmanager route (route/g0)", func() { Eventually(func() error { + cloudProvider := strings.ToLower(os.Getenv("CLOUD_PROVIDER")) + substring1 := "rosa" + substring2 := "hcp" + + if strings.Contains(cloudProvider, substring1) && strings.Contains(cloudProvider, substring2) { + Skip("skip on rosa-hcp") + } + query := "/api/v2/alerts" url := "https://alertmanager-open-cluster-management-observability.apps." + testOptions.HubCluster.BaseDomain + query alertJson := ` @@ -136,6 +161,7 @@ var _ = Describe("Observability:", func() { client := &http.Client{} if os.Getenv("IS_KIND_ENV") != trueStr { client.Transport = tr + BearerToken, err = utils.FetchBearerToken(testOptions) alertPostReq.Header.Set("Authorization", "Bearer "+BearerToken) } if !alertCreated { @@ -164,6 +190,7 @@ var _ = Describe("Observability:", func() { } if os.Getenv("IS_KIND_ENV") != trueStr { + BearerToken, err = utils.FetchBearerToken(testOptions) alertGetReq.Header.Set("Authorization", "Bearer "+BearerToken) } diff --git a/tests/pkg/tests/results.xml b/tests/pkg/tests/results.xml new file mode 100644 index 000000000..642d4bf10 --- /dev/null +++ b/tests/pkg/tests/results.xml @@ -0,0 +1,169 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/pkg/utils/client.go b/tests/pkg/utils/client.go index cb606ee11..2891387f0 100644 --- a/tests/pkg/utils/client.go +++ b/tests/pkg/utils/client.go @@ -7,6 +7,7 @@ package utils import ( "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" + "k8s.io/klog" ) func getKubeClient(opt TestOptions, isHub bool) kubernetes.Interface { @@ -19,6 +20,7 @@ func getKubeClient(opt TestOptions, isHub bool) kubernetes.Interface { opt.ManagedClusters[0].ClusterServerURL, opt.ManagedClusters[0].KubeConfig, opt.ManagedClusters[0].KubeContext) + klog.V(1).Infof("New kubeclient for managedcluster <%v>", opt.ManagedClusters[0].Name) } return clientKube } diff --git a/tests/pkg/utils/cluster_deploy.go b/tests/pkg/utils/cluster_deploy.go new file mode 100644 index 000000000..bed8e8206 --- /dev/null +++ b/tests/pkg/utils/cluster_deploy.go @@ -0,0 +1,44 @@ +// Copyright (c) Red Hat, Inc. +// Copyright Contributors to the Open Cluster Management project +// Licensed under the Apache License 2.0 + +package utils + +// ClusterDeploy defines the data passed to Hive +type ClusterDeploy struct { + Kind string `yaml:"kind"` + APIVersion string `yaml:"apiVersion"` + Items []Items `yaml:"items"` +} + +// Items defines the list of items in the cluster deploy yaml +type Items struct { + Kind string `yaml:"kind"` + Metadata Metadata `yaml:"metadata"` + StringData StringData `yaml:"stringData,omitempty"` + Spec Spec `yaml:"spec,omitempty"` +} + +// Metadata defines the name +type Metadata struct { + Name string `yaml:"name,omitempty"` +} + +// StringData defiines the ssh values +type StringData struct { + Dockerconfigjson string `yaml:".dockerconfigjson,omitempty"` + SSHPrivateKey string `yaml:"ssh-privatekey,omitempty"` +} + +// Spec defines the kube specifications +type Spec struct { + BaseDomain string `yaml:"baseDomain,omitempty"` + ClusterName string `yaml:"clusterName,omitempty"` + Provisioning Provisioning `yaml:"provisioning,omitempty"` +} + +// Provisioning defines the data related to cluster creation +type Provisioning struct { + ReleaseImage string `yaml:"releaseImage,omitempty"` + SSHKnownHosts []string `yaml:"sshKnownHosts,omitempty"` +} diff --git a/tests/pkg/utils/install_config.go b/tests/pkg/utils/install_config.go new file mode 100644 index 000000000..a146f05b4 --- /dev/null +++ b/tests/pkg/utils/install_config.go @@ -0,0 +1,56 @@ +// Copyright (c) Red Hat, Inc. +// Copyright Contributors to the Open Cluster Management project +// Licensed under the Apache License 2.0 + +package utils + +// InstallConfig definition for install config structure from install-config.yaml +type InstallConfig struct { + BaseDomain string `yaml:"baseDomain,omitempty"` + Networking Networking `yaml:"networking,omitempty"` + Metadata Metadata `yaml:"metadata"` + Platform Platform `yaml:"platform,omitempty"` + PullSecret string `yaml:"pullSecret,omitempty"` + SSHKey string `yaml:"sshKey,omitempty"` +} + +// Networking definition +type Networking struct { + NetworkType string `yaml:"networkType"` + MachineCIDR string `yaml:"machineCIDR"` +} + +// Platform definition +type Platform struct { + Baremetal Baremetal `yaml:"baremetal,omitempty"` +} + +// Baremetal specs for target baremetal provisioning +type Baremetal struct { + ExternalBridge string `yaml:"externalBridge,omitempty"` + ProvisioningBridge string `yaml:"provisioningBridge,omitempty"` + LibvirtURI string `yaml:"libvirtURI,omitempty"` + ProvisioningNetworkInterface string `yaml:"provisioningNetworkInterface,omitempty"` + ProvisioningNetworkCIDR string `yaml:"provisioningNetworkCIDR,omitempty"` + APIVIP string `yaml:"apiVIP,omitempty"` + DNSVIP string `yaml:"dnsVIP,omitempty"` + IngressVIP string `yaml:"ingressVIP,omitempty"` + Hosts []Host `yaml:"hosts,omitempty"` + SSHKnownHosts string `yaml:"sshKnownHosts,omitempty"` +} + +// Host is an array of baremetal assets +type Host struct { + Name string `yaml:"name"` + Role string `yaml:"role"` + Bmc Bmc `yaml:"bmc"` + BootMACAddress string `yaml:"bootMACAddress"` + HardwareProfile string `yaml:"hardwareProfile"` +} + +// Bmc definition +type Bmc struct { + Address string `yaml:"address"` + Username string `yaml:"username"` + Password string `yaml:"password"` +} diff --git a/tests/pkg/utils/mco_dashboard.go b/tests/pkg/utils/mco_dashboard.go index 1c81c6d42..2e60698ea 100644 --- a/tests/pkg/utils/mco_dashboard.go +++ b/tests/pkg/utils/mco_dashboard.go @@ -17,6 +17,10 @@ import ( "k8s.io/klog" ) +const ( + trueStr = "true" +) + func ContainDashboard(opt TestOptions, title string) (error, bool) { grafanaConsoleURL := GetGrafanaURL(opt) path := "/api/search?" @@ -30,7 +34,7 @@ func ContainDashboard(opt TestOptions, title string) (error, bool) { } client := &http.Client{} - if os.Getenv("IS_KIND_ENV") != "true" { + if os.Getenv("IS_KIND_ENV") != trueStr { tr := &http.Transport{ // #nosec G402 -- Used in test. TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, diff --git a/tests/pkg/utils/mco_deploy.go b/tests/pkg/utils/mco_deploy.go index 0a48233c0..a44dd8a41 100644 --- a/tests/pkg/utils/mco_deploy.go +++ b/tests/pkg/utils/mco_deploy.go @@ -115,6 +115,18 @@ func GetAllMCOPods(opt TestOptions) ([]corev1.Pod, error) { // ignore non-mco pods mcoPods := []corev1.Pod{} for _, p := range podList.Items { + if strings.Contains(p.GetName(), "metrics-collector") { + continue + } + + if strings.Contains(p.GetName(), "endpoint-observability-operator") { + continue + } + + if strings.Contains(p.GetName(), "uwl-metrics-collector") { + continue + } + if strings.Contains(p.GetName(), "grafana-test") { continue } @@ -385,7 +397,7 @@ func ModifyMCOCR(opt TestOptions) error { } spec := mco.Object["spec"].(map[string]interface{}) storageConfig := spec["storageConfig"].(map[string]interface{}) - storageConfig["alertmanagerStorageSize"] = "2Gi" + storageConfig["alertmanagerStorageSize"] = "3Gi" advRetentionCon, _ := CheckAdvRetentionConfig(opt) if advRetentionCon { @@ -610,6 +622,13 @@ func CreatePullSecret(opt TestOptions, mcoNs string) error { return errGet } + mcopSecret, errGet := clientKube.CoreV1().Secrets(MCO_NAMESPACE).Get(context.TODO(), name, metav1.GetOptions{}) + if mcopSecret != nil { + errDelGet := clientKube.CoreV1().Secrets(MCO_NAMESPACE).Delete(context.TODO(), name, metav1.DeleteOptions{}) + if errGet != nil { + klog.V(1).Infof("Delete existing pullSecret - %s", errDelGet) + } + } pullSecret.ObjectMeta = metav1.ObjectMeta{ Name: name, Namespace: MCO_NAMESPACE, diff --git a/tests/pkg/utils/mco_grafana.go b/tests/pkg/utils/mco_grafana.go index 74d3f5b97..19a1bab56 100644 --- a/tests/pkg/utils/mco_grafana.go +++ b/tests/pkg/utils/mco_grafana.go @@ -4,12 +4,31 @@ package utils +import ( + "os" + "strings" +) + func GetGrafanaURL(opt TestOptions) string { - grafanaConsoleURL := "https://grafana-open-cluster-management-observability.apps." + opt.HubCluster.BaseDomain - if opt.HubCluster.GrafanaURL != "" { - grafanaConsoleURL = opt.HubCluster.GrafanaURL + cloudProvider := strings.ToLower(os.Getenv("CLOUD_PROVIDER")) + substring1 := "rosa" + substring2 := "hcp" + if strings.Contains(cloudProvider, substring1) && strings.Contains(cloudProvider, substring2) { + + grafanaConsoleURL := "https://grafana-open-cluster-management-observability.apps.rosa." + opt.HubCluster.BaseDomain + if opt.HubCluster.GrafanaURL != "" { + grafanaConsoleURL = opt.HubCluster.GrafanaURL + } else { + opt.HubCluster.GrafanaHost = "grafana-open-cluster-management-observability.apps.rosa." + opt.HubCluster.BaseDomain + } + return grafanaConsoleURL } else { - opt.HubCluster.GrafanaHost = "grafana-open-cluster-management-observability.apps." + opt.HubCluster.BaseDomain + grafanaConsoleURL := "https://grafana-open-cluster-management-observability.apps." + opt.HubCluster.BaseDomain + if opt.HubCluster.GrafanaURL != "" { + grafanaConsoleURL = opt.HubCluster.GrafanaURL + } else { + opt.HubCluster.GrafanaHost = "grafana-open-cluster-management-observability.apps." + opt.HubCluster.BaseDomain + } + return grafanaConsoleURL } - return grafanaConsoleURL } diff --git a/tests/pkg/utils/mco_managedcluster.go b/tests/pkg/utils/mco_managedcluster.go index 15c424fdc..ba1eb9ac2 100644 --- a/tests/pkg/utils/mco_managedcluster.go +++ b/tests/pkg/utils/mco_managedcluster.go @@ -11,6 +11,8 @@ import ( goversion "github.com/hashicorp/go-version" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "k8s.io/klog" ) func UpdateObservabilityFromManagedCluster(opt TestOptions, enableObservability bool) error { @@ -144,6 +146,41 @@ func ListOCPManagedClusterIDs(opt TestOptions, minVersionStr string) ([]string, return clusterIDs, nil } +func ListLocalClusterIDs(opt TestOptions) ([]string, error) { + clientDynamic := GetKubeClientDynamic(opt, true) + objs, err := clientDynamic.Resource(NewOCMManagedClustersGVR()).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return nil, err + } + clusterIDs := []string{} + for _, obj := range objs.Items { + metadata := obj.Object["metadata"].(map[string]interface{}) + labels := metadata["labels"].(map[string]interface{}) + if labels != nil { + vendorStr := "" + if vendor, ok := labels["vendor"]; ok { + vendorStr = vendor.(string) + } + + localClusterLabelStr := "" + if localCluster, ok := labels["local-cluster"]; ok { + localClusterLabelStr = localCluster.(string) + } + if vendorStr == "OpenShift" && localClusterLabelStr == "true" { + clusterIDStr := "" + if clusterID, ok := labels["clusterID"]; ok { + clusterIDStr = clusterID.(string) + } + if len(clusterIDStr) > 0 { + clusterIDs = append(clusterIDs, clusterIDStr) + } + } + } + } + klog.V(3).Infof("clusterIDs is %s", clusterIDs) + return clusterIDs, nil +} + func ListKSManagedClusterNames(opt TestOptions) ([]string, error) { clientDynamic := GetKubeClientDynamic(opt, true) objs, err := clientDynamic.Resource(NewOCMManagedClustersGVR()).List(context.TODO(), metav1.ListOptions{}) diff --git a/tests/pkg/utils/mco_pods.go b/tests/pkg/utils/mco_pods.go index 91b6d721c..696c86d8b 100644 --- a/tests/pkg/utils/mco_pods.go +++ b/tests/pkg/utils/mco_pods.go @@ -37,6 +37,17 @@ func GetPodList(opt TestOptions, isHub bool, namespace string, labelSelector str return nil, podList } +// TODO (jacob): delete? +func DeletePod(opt TestOptions, isHub bool, namespace, name string) error { + clientKube := getKubeClient(opt, isHub) + err := clientKube.CoreV1().Pods(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{}) + if err != nil { + klog.Errorf("Failed to delete pod %s in namespace %s due to %v", name, namespace, err) + return err + } + return nil +} + func GetPodLogs( opt TestOptions, isHub bool,