From 8ee05dab606d3324f4694c1d3c0ce1faf529199e Mon Sep 17 00:00:00 2001 From: Tanmay Satam Date: Thu, 6 Jul 2023 14:27:15 -0400 Subject: [PATCH 1/3] Add initial spike implementation for Hive RequestDisallowedByPolicy error handling --- pkg/hive/const.go | 9 +++ pkg/hive/manager.go | 91 +++++++++++++++++++++-- pkg/hive/manager_test.go | 154 +++++++++++++++++++++++++++++++++++++-- 3 files changed, 242 insertions(+), 12 deletions(-) create mode 100644 pkg/hive/const.go diff --git a/pkg/hive/const.go b/pkg/hive/const.go new file mode 100644 index 00000000000..3c4057bf7de --- /dev/null +++ b/pkg/hive/const.go @@ -0,0 +1,9 @@ +package hive + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +const ( + ProvisionFailedReasonInvalidTemplateDeployment = "AzureInvalidTemplateDeployment" + ProvisionFailedReasonUnknownError = "UnknownError" +) diff --git a/pkg/hive/manager.go b/pkg/hive/manager.go index 4ad4d744a7c..5e27d57a5d0 100644 --- a/pkg/hive/manager.go +++ b/pkg/hive/manager.go @@ -5,8 +5,14 @@ package hive import ( "context" + "encoding/json" "fmt" + "net/http" + "regexp" + "sort" + "strings" + mgmtfeatures "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-07-01/features" hivev1 "github.com/openshift/hive/apis/hive/v1" "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" @@ -195,14 +201,9 @@ func (hr *clusterManager) IsClusterInstallationComplete(ctx context.Context, doc return true, nil } - checkFailureConditions := map[hivev1.ClusterDeploymentConditionType]corev1.ConditionStatus{ - hivev1.ProvisionFailedCondition: corev1.ConditionTrue, - } - for _, cond := range cd.Status.Conditions { - conditionStatus, found := checkFailureConditions[cond.Type] - if found && conditionStatus == cond.Status { - return false, fmt.Errorf("clusterdeployment has failed: %s == %s", cond.Type, cond.Status) + if cond.Type == hivev1.ProvisionFailedCondition { + return false, hr.handleProvisionFailed(ctx, cd, cond) } } @@ -237,3 +238,79 @@ func (hr *clusterManager) ResetCorrelationData(ctx context.Context, doc *api.Ope return hr.hiveClientset.Update(ctx, cd) }) } + +func (hr *clusterManager) handleProvisionFailed(ctx context.Context, cd *hivev1.ClusterDeployment, cond hivev1.ClusterDeploymentCondition) error { + if cond.Status != corev1.ConditionTrue { + return nil + } + + switch cond.Reason { + case ProvisionFailedReasonInvalidTemplateDeployment: + // TODO: refactor this case body to dedicated handler. Extract reusable components (install log JSON parsing) + latestProvision, err := hr.latestProvisionForDeployment(ctx, cd) + if err != nil { + return err + } + installLog := *latestProvision.Spec.InstallLog + installLog = strings.TrimSpace(installLog) + installLogLines := strings.Split(installLog, "\n") + lastLine := installLogLines[len(installLogLines)-1] + + regex := regexp.MustCompile(`(\{.*\})`) + responseJson := regex.FindStringSubmatch(lastLine)[1] + + response := &mgmtfeatures.ErrorResponse{} + if err := json.Unmarshal([]byte(responseJson), response); err != nil { + return err + } + + cloudErr := &api.CloudError{ + StatusCode: http.StatusBadRequest, + CloudErrorBody: &api.CloudErrorBody{ + Code: api.CloudErrorCodeDeploymentFailed, + Message: "The deployment failed. Please see details for more information.", + Details: make([]api.CloudErrorBody, len(*response.Details)), + }, + } + + for i, detail := range *response.Details { + cloudErr.CloudErrorBody.Details[i] = api.CloudErrorBody{ + Code: *detail.Code, + Message: *detail.Message, + Target: *detail.Target, + } + } + + return cloudErr + default: + return &api.CloudError{ + StatusCode: http.StatusInternalServerError, + CloudErrorBody: &api.CloudErrorBody{ + Code: api.CloudErrorCodeInternalServerError, + Message: "Deployment failed.", + }, + } + } +} + +func (hr *clusterManager) latestProvisionForDeployment(ctx context.Context, cd *hivev1.ClusterDeployment) (*hivev1.ClusterProvision, error) { + provisionList := &hivev1.ClusterProvisionList{} + if err := hr.hiveClientset.List( + ctx, + provisionList, + client.InNamespace(cd.Namespace), + client.MatchingLabels(map[string]string{"hive.openshift.io/cluster-deployment-name": cd.Name}), + ); err != nil { + hr.log.WithError(err).Warn("could not list provisions for clusterdeployment") + return nil, err + } + if len(provisionList.Items) == 0 { + return nil, nil + } + provisions := make([]*hivev1.ClusterProvision, len(provisionList.Items)) + for i := range provisionList.Items { + provisions[i] = &provisionList.Items[i] + } + sort.Slice(provisions, func(i, j int) bool { return provisions[i].Spec.Attempt > provisions[j].Spec.Attempt }) + return provisions[0], nil +} diff --git a/pkg/hive/manager_test.go b/pkg/hive/manager_test.go index 0de2aa46d4c..387b81c3f11 100644 --- a/pkg/hive/manager_test.go +++ b/pkg/hive/manager_test.go @@ -5,6 +5,7 @@ package hive import ( "context" + "net/http" "reflect" "testing" @@ -181,9 +182,10 @@ func TestIsClusterInstallationComplete(t *testing.T) { for _, tt := range []struct { name string - cd kruntime.Object + cd *hivev1.ClusterDeployment + cp *hivev1.ClusterProvision wantResult bool - wantErr string + wantErr error }{ { name: "is installed", @@ -228,7 +230,7 @@ func TestIsClusterInstallationComplete(t *testing.T) { wantResult: false, }, { - name: "has failed provisioning", + name: "has failed provisioning - no Reason", cd: &hivev1.ClusterDeployment{ ObjectMeta: metav1.ObjectMeta{ Name: ClusterDeploymentName, @@ -243,22 +245,35 @@ func TestIsClusterInstallationComplete(t *testing.T) { }, }, }, - wantErr: "clusterdeployment has failed: ProvisionFailed == True", + wantErr: &api.CloudError{ + StatusCode: http.StatusInternalServerError, + CloudErrorBody: &api.CloudErrorBody{ + Code: api.CloudErrorCodeInternalServerError, + Message: "Deployment failed.", + }, + }, wantResult: false, }, + // TODO: move test cases for handleProvisionFailed here } { t.Run(tt.name, func(t *testing.T) { fakeClientBuilder := fake.NewClientBuilder() if tt.cd != nil { fakeClientBuilder = fakeClientBuilder.WithRuntimeObjects(tt.cd) } + if tt.cp != nil { + fakeClientBuilder = fakeClientBuilder.WithRuntimeObjects(tt.cp) + } c := clusterManager{ hiveClientset: fakeClientBuilder.Build(), log: logrus.NewEntry(logrus.StandardLogger()), } result, err := c.IsClusterInstallationComplete(context.Background(), doc) - utilerror.AssertErrorMessage(t, err, tt.wantErr) + + if diff := cmp.Diff(tt.wantErr, err); diff != "" { + t.Error(diff) + } if tt.wantResult != result { t.Error(result) @@ -440,3 +455,132 @@ func TestGetClusterDeployment(t *testing.T) { }) } } + +func TestHandleProvisionFailed(t *testing.T) { + fakeNamespace := "aro-00000000-0000-0000-0000-00000000000" + genericErr := &api.CloudError{ + StatusCode: http.StatusInternalServerError, + CloudErrorBody: &api.CloudErrorBody{ + Code: api.CloudErrorCodeInternalServerError, + Message: "Deployment failed.", + }, + } + + for _, tt := range []struct { + name string + reason string + installLog string + wantErr error + }{ + { + name: "No Reason provided returns generic error", + reason: "", + wantErr: genericErr, + }, + { + name: "Known Reason not relevant to ARO returns generic error", + reason: "AWSInsufficientCapacity", + wantErr: genericErr, + }, + { + name: "Reason: UnknownError returns generic error", + reason: ProvisionFailedReasonUnknownError, + wantErr: genericErr, + }, + { + name: "Reason: InvalidTemplateDeployment extracts error from logs", + reason: ProvisionFailedReasonInvalidTemplateDeployment, + installLog: `level=info msg=running in local development mode + level=info msg=creating development InstanceMetadata + level=info msg=InstanceMetadata: running on AzurePublicCloud + level=info msg=running step [Action github.com/Azure/ARO-RP/pkg/installer.(*manager).Manifests.func1] + level=info msg=running step [Action github.com/Azure/ARO-RP/pkg/installer.(*manager).Manifests.func2] + level=info msg=resolving graph + level=info msg=running step [Action github.com/Azure/ARO-RP/pkg/installer.(*manager).Manifests.func3] + level=info msg=checking if graph exists + level=info msg=save graph + Generates the Ignition Config asset + + level=info msg=running in local development mode + level=info msg=creating development InstanceMetadata + level=info msg=InstanceMetadata: running on AzurePublicCloud + level=info msg=running step [AuthorizationRefreshingAction [Action github.com/Azure/ARO-RP/pkg/installer.(*manager).deployResourceTemplate-fm]] + level=info msg=load persisted graph + level=info msg=deploying resources template + level=error msg=step [AuthorizationRefreshingAction [Action github.com/Azure/ARO-RP/pkg/installer.(*manager).deployResourceTemplate-fm]] encountered error: 400: DeploymentFailed: : Deployment failed. Details: : : {"code": "InvalidTemplateDeployment","message": "The template deployment failed with multiple errors. Please see details for more information.","target": null,"details": [{"code": "RequestDisallowedByPolicy","message": "Resource 'aro-test-aaaaa-bootstrap' was disallowed by policy.","target": "aro-test-aaaaa-bootstrap"},{"code": "RequestDisallowedByPolicy","message": "Resource 'aro-test-aaaaa-master-0' was disallowed by policy.","target": "aro-test-aaaaa-master-0"},{"code": "RequestDisallowedByPolicy","message": "Resource 'aro-test-aaaaa-master-1' was disallowed by policy.","target": "aro-test-aaaaa-master-1"},{"code": "RequestDisallowedByPolicy","message": "Resource 'aro-test-aaaaa-master-2' was disallowed by policy.","target": "aro-test-aaaaa-master-2"}]} + level=error msg=400: DeploymentFailed: : Deployment failed. Details: : : {"code": "InvalidTemplateDeployment","message": "The template deployment failed with multiple errors. Please see details for more information.","target": null,"details": [{"code": "RequestDisallowedByPolicy","message": "Resource 'aro-test-aaaaa-bootstrap' was disallowed by policy.","target": "aro-test-aaaaa-bootstrap"},{"code": "RequestDisallowedByPolicy","message": "Resource 'aro-test-aaaaa-master-0' was disallowed by policy.","target": "aro-test-aaaaa-master-0"},{"code": "RequestDisallowedByPolicy","message": "Resource 'aro-test-aaaaa-master-1' was disallowed by policy.","target": "aro-test-aaaaa-master-1"},{"code": "RequestDisallowedByPolicy","message": "Resource 'aro-test-aaaaa-master-2' was disallowed by policy.","target": "aro-test-aaaaa-master-2"}]}`, + wantErr: &api.CloudError{ + StatusCode: http.StatusBadRequest, + CloudErrorBody: &api.CloudErrorBody{ + Code: api.CloudErrorCodeDeploymentFailed, + Message: "The deployment failed. Please see details for more information.", + Details: []api.CloudErrorBody{ + { + Code: api.CloudErrorCodeRequestDisallowedByPolicy, + Message: "Resource 'aro-test-aaaaa-bootstrap' was disallowed by policy.", + Target: "aro-test-aaaaa-bootstrap", + }, + { + Code: api.CloudErrorCodeRequestDisallowedByPolicy, + Message: "Resource 'aro-test-aaaaa-master-0' was disallowed by policy.", + Target: "aro-test-aaaaa-master-0", + }, + { + Code: api.CloudErrorCodeRequestDisallowedByPolicy, + Message: "Resource 'aro-test-aaaaa-master-1' was disallowed by policy.", + Target: "aro-test-aaaaa-master-1", + }, + { + Code: api.CloudErrorCodeRequestDisallowedByPolicy, + Message: "Resource 'aro-test-aaaaa-master-2' was disallowed by policy.", + Target: "aro-test-aaaaa-master-2", + }, + }, + }, + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + cond := hivev1.ClusterDeploymentCondition{ + Type: hivev1.ProvisionFailedCondition, + Status: corev1.ConditionTrue, + Reason: tt.reason, + } + hcd := &hivev1.ClusterDeployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: ClusterDeploymentName, + Namespace: fakeNamespace, + }, + Status: hivev1.ClusterDeploymentStatus{ + Conditions: []hivev1.ClusterDeploymentCondition{cond}, + }, + } + hcp := &hivev1.ClusterProvision{ + ObjectMeta: metav1.ObjectMeta{ + Name: ClusterDeploymentName + "-0-bbbbb", + Namespace: fakeNamespace, + Labels: map[string]string{ + "hive.openshift.io/cluster-deployment-name": ClusterDeploymentName, + }, + }, + Spec: hivev1.ClusterProvisionSpec{ + InstallLog: &tt.installLog, + }, + } + + fakeClientBuilder := fake.NewClientBuilder(). + WithRuntimeObjects(hcd, hcp) + + c := clusterManager{ + hiveClientset: fakeClientBuilder.Build(), + log: logrus.NewEntry(logrus.StandardLogger()), + } + + err := c.handleProvisionFailed(context.Background(), hcd, cond) + + if diff := cmp.Diff(tt.wantErr, err); diff != "" { + t.Error(diff) + } + }) + } +} From 035e8c8fb8ced6a00857453929085c665f5774f5 Mon Sep 17 00:00:00 2001 From: swetha Date: Thu, 6 Jul 2023 13:55:18 -0700 Subject: [PATCH 2/3] Refactored tests --- pkg/hive/manager_test.go | 312 +++++++++++++++++---------------------- 1 file changed, 136 insertions(+), 176 deletions(-) diff --git a/pkg/hive/manager_test.go b/pkg/hive/manager_test.go index 387b81c3f11..632701e658a 100644 --- a/pkg/hive/manager_test.go +++ b/pkg/hive/manager_test.go @@ -180,6 +180,43 @@ func TestIsClusterInstallationComplete(t *testing.T) { }, } + genericErr := &api.CloudError{ + StatusCode: http.StatusInternalServerError, + CloudErrorBody: &api.CloudErrorBody{ + Code: api.CloudErrorCodeInternalServerError, + Message: "Deployment failed.", + }, + } + + makeClusterDeployment := func(installed bool, provisionFailedCond hivev1.ClusterDeploymentCondition) *hivev1.ClusterDeployment { + return &hivev1.ClusterDeployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: ClusterDeploymentName, + Namespace: fakeNamespace, + }, + Spec: hivev1.ClusterDeploymentSpec{ + Installed: installed, + }, + Status: hivev1.ClusterDeploymentStatus{ + Conditions: []hivev1.ClusterDeploymentCondition{provisionFailedCond}, + }, + } + } + makeClusterProvision := func(installLog string) *hivev1.ClusterProvision { + return &hivev1.ClusterProvision{ + ObjectMeta: metav1.ObjectMeta{ + Name: ClusterDeploymentName + "-0-bbbbb", + Namespace: fakeNamespace, + Labels: map[string]string{ + "hive.openshift.io/cluster-deployment-name": ClusterDeploymentName, + }, + }, + Spec: hivev1.ClusterProvisionSpec{ + InstallLog: &installLog, + }, + } + } + for _, tt := range []struct { name string cd *hivev1.ClusterDeployment @@ -189,72 +226,124 @@ func TestIsClusterInstallationComplete(t *testing.T) { }{ { name: "is installed", - cd: &hivev1.ClusterDeployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: ClusterDeploymentName, - Namespace: fakeNamespace, - }, - Spec: hivev1.ClusterDeploymentSpec{ - Installed: true, + cd: makeClusterDeployment( + true, + hivev1.ClusterDeploymentCondition{ + Type: hivev1.ProvisionFailedCondition, + Status: corev1.ConditionFalse, }, - Status: hivev1.ClusterDeploymentStatus{ - Conditions: []hivev1.ClusterDeploymentCondition{ - { - Type: hivev1.ProvisionFailedCondition, - Status: corev1.ConditionFalse, - }, - }, - }, - }, + ), wantResult: true, }, { name: "is not installed yet", - cd: &hivev1.ClusterDeployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: ClusterDeploymentName, - Namespace: fakeNamespace, + cd: makeClusterDeployment( + false, + hivev1.ClusterDeploymentCondition{ + Type: hivev1.ProvisionFailedCondition, + Status: corev1.ConditionFalse, }, - Spec: hivev1.ClusterDeploymentSpec{ - Installed: false, + ), + wantResult: false, + }, + { + name: "has failed provisioning - no Reason", + cd: makeClusterDeployment( + false, + hivev1.ClusterDeploymentCondition{ + Type: hivev1.ProvisionFailedCondition, + Status: corev1.ConditionTrue, }, - Status: hivev1.ClusterDeploymentStatus{ - Conditions: []hivev1.ClusterDeploymentCondition{ - { - Type: hivev1.ProvisionFailedCondition, - Status: corev1.ConditionFalse, - }, - }, + ), + wantErr: genericErr, + wantResult: false, + }, + { + name: "has failed provisioning - Known Reason not relevant to ARO", + cd: makeClusterDeployment( + false, + hivev1.ClusterDeploymentCondition{ + Type: hivev1.ProvisionFailedCondition, + Status: corev1.ConditionTrue, + Reason: "AWSInsufficientCapacity", }, - }, + ), + wantErr: genericErr, wantResult: false, }, { - name: "has failed provisioning - no Reason", - cd: &hivev1.ClusterDeployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: ClusterDeploymentName, - Namespace: fakeNamespace, + name: "has failed provisioning - UnknownError", + cd: makeClusterDeployment( + false, + hivev1.ClusterDeploymentCondition{ + Type: hivev1.ProvisionFailedCondition, + Status: corev1.ConditionTrue, + Reason: ProvisionFailedReasonUnknownError, }, - Status: hivev1.ClusterDeploymentStatus{ - Conditions: []hivev1.ClusterDeploymentCondition{ - { - Type: hivev1.ProvisionFailedCondition, - Status: corev1.ConditionTrue, - }, - }, + ), + wantErr: genericErr, + wantResult: false, + }, + { + name: "has failed provisioning - InvalidTemplateDeployment", + cd: makeClusterDeployment( + false, + hivev1.ClusterDeploymentCondition{ + Type: hivev1.ProvisionFailedCondition, + Status: corev1.ConditionTrue, + Reason: ProvisionFailedReasonInvalidTemplateDeployment, }, - }, + ), + cp: makeClusterProvision(`level=info msg=running in local development mode + level=info msg=creating development InstanceMetadata + level=info msg=InstanceMetadata: running on AzurePublicCloud + level=info msg=running step [Action github.com/Azure/ARO-RP/pkg/installer.(*manager).Manifests.func1] + level=info msg=running step [Action github.com/Azure/ARO-RP/pkg/installer.(*manager).Manifests.func2] + level=info msg=resolving graph + level=info msg=running step [Action github.com/Azure/ARO-RP/pkg/installer.(*manager).Manifests.func3] + level=info msg=checking if graph exists + level=info msg=save graph + Generates the Ignition Config asset + + level=info msg=running in local development mode + level=info msg=creating development InstanceMetadata + level=info msg=InstanceMetadata: running on AzurePublicCloud + level=info msg=running step [AuthorizationRefreshingAction [Action github.com/Azure/ARO-RP/pkg/installer.(*manager).deployResourceTemplate-fm]] + level=info msg=load persisted graph + level=info msg=deploying resources template + level=error msg=step [AuthorizationRefreshingAction [Action github.com/Azure/ARO-RP/pkg/installer.(*manager).deployResourceTemplate-fm]] encountered error: 400: DeploymentFailed: : Deployment failed. Details: : : {"code": "InvalidTemplateDeployment","message": "The template deployment failed with multiple errors. Please see details for more information.","target": null,"details": [{"code": "RequestDisallowedByPolicy","message": "Resource 'aro-test-aaaaa-bootstrap' was disallowed by policy.","target": "aro-test-aaaaa-bootstrap"},{"code": "RequestDisallowedByPolicy","message": "Resource 'aro-test-aaaaa-master-0' was disallowed by policy.","target": "aro-test-aaaaa-master-0"},{"code": "RequestDisallowedByPolicy","message": "Resource 'aro-test-aaaaa-master-1' was disallowed by policy.","target": "aro-test-aaaaa-master-1"},{"code": "RequestDisallowedByPolicy","message": "Resource 'aro-test-aaaaa-master-2' was disallowed by policy.","target": "aro-test-aaaaa-master-2"}]} + level=error msg=400: DeploymentFailed: : Deployment failed. Details: : : {"code": "InvalidTemplateDeployment","message": "The template deployment failed with multiple errors. Please see details for more information.","target": null,"details": [{"code": "RequestDisallowedByPolicy","message": "Resource 'aro-test-aaaaa-bootstrap' was disallowed by policy.","target": "aro-test-aaaaa-bootstrap"},{"code": "RequestDisallowedByPolicy","message": "Resource 'aro-test-aaaaa-master-0' was disallowed by policy.","target": "aro-test-aaaaa-master-0"},{"code": "RequestDisallowedByPolicy","message": "Resource 'aro-test-aaaaa-master-1' was disallowed by policy.","target": "aro-test-aaaaa-master-1"},{"code": "RequestDisallowedByPolicy","message": "Resource 'aro-test-aaaaa-master-2' was disallowed by policy.","target": "aro-test-aaaaa-master-2"}]}`), wantErr: &api.CloudError{ - StatusCode: http.StatusInternalServerError, + StatusCode: http.StatusBadRequest, CloudErrorBody: &api.CloudErrorBody{ - Code: api.CloudErrorCodeInternalServerError, - Message: "Deployment failed.", + Code: api.CloudErrorCodeDeploymentFailed, + Message: "The deployment failed. Please see details for more information.", + Details: []api.CloudErrorBody{ + { + Code: api.CloudErrorCodeRequestDisallowedByPolicy, + Message: "Resource 'aro-test-aaaaa-bootstrap' was disallowed by policy.", + Target: "aro-test-aaaaa-bootstrap", + }, + { + Code: api.CloudErrorCodeRequestDisallowedByPolicy, + Message: "Resource 'aro-test-aaaaa-master-0' was disallowed by policy.", + Target: "aro-test-aaaaa-master-0", + }, + { + Code: api.CloudErrorCodeRequestDisallowedByPolicy, + Message: "Resource 'aro-test-aaaaa-master-1' was disallowed by policy.", + Target: "aro-test-aaaaa-master-1", + }, + { + Code: api.CloudErrorCodeRequestDisallowedByPolicy, + Message: "Resource 'aro-test-aaaaa-master-2' was disallowed by policy.", + Target: "aro-test-aaaaa-master-2", + }, + }, }, }, wantResult: false, }, - // TODO: move test cases for handleProvisionFailed here } { t.Run(tt.name, func(t *testing.T) { fakeClientBuilder := fake.NewClientBuilder() @@ -455,132 +544,3 @@ func TestGetClusterDeployment(t *testing.T) { }) } } - -func TestHandleProvisionFailed(t *testing.T) { - fakeNamespace := "aro-00000000-0000-0000-0000-00000000000" - genericErr := &api.CloudError{ - StatusCode: http.StatusInternalServerError, - CloudErrorBody: &api.CloudErrorBody{ - Code: api.CloudErrorCodeInternalServerError, - Message: "Deployment failed.", - }, - } - - for _, tt := range []struct { - name string - reason string - installLog string - wantErr error - }{ - { - name: "No Reason provided returns generic error", - reason: "", - wantErr: genericErr, - }, - { - name: "Known Reason not relevant to ARO returns generic error", - reason: "AWSInsufficientCapacity", - wantErr: genericErr, - }, - { - name: "Reason: UnknownError returns generic error", - reason: ProvisionFailedReasonUnknownError, - wantErr: genericErr, - }, - { - name: "Reason: InvalidTemplateDeployment extracts error from logs", - reason: ProvisionFailedReasonInvalidTemplateDeployment, - installLog: `level=info msg=running in local development mode - level=info msg=creating development InstanceMetadata - level=info msg=InstanceMetadata: running on AzurePublicCloud - level=info msg=running step [Action github.com/Azure/ARO-RP/pkg/installer.(*manager).Manifests.func1] - level=info msg=running step [Action github.com/Azure/ARO-RP/pkg/installer.(*manager).Manifests.func2] - level=info msg=resolving graph - level=info msg=running step [Action github.com/Azure/ARO-RP/pkg/installer.(*manager).Manifests.func3] - level=info msg=checking if graph exists - level=info msg=save graph - Generates the Ignition Config asset - - level=info msg=running in local development mode - level=info msg=creating development InstanceMetadata - level=info msg=InstanceMetadata: running on AzurePublicCloud - level=info msg=running step [AuthorizationRefreshingAction [Action github.com/Azure/ARO-RP/pkg/installer.(*manager).deployResourceTemplate-fm]] - level=info msg=load persisted graph - level=info msg=deploying resources template - level=error msg=step [AuthorizationRefreshingAction [Action github.com/Azure/ARO-RP/pkg/installer.(*manager).deployResourceTemplate-fm]] encountered error: 400: DeploymentFailed: : Deployment failed. Details: : : {"code": "InvalidTemplateDeployment","message": "The template deployment failed with multiple errors. Please see details for more information.","target": null,"details": [{"code": "RequestDisallowedByPolicy","message": "Resource 'aro-test-aaaaa-bootstrap' was disallowed by policy.","target": "aro-test-aaaaa-bootstrap"},{"code": "RequestDisallowedByPolicy","message": "Resource 'aro-test-aaaaa-master-0' was disallowed by policy.","target": "aro-test-aaaaa-master-0"},{"code": "RequestDisallowedByPolicy","message": "Resource 'aro-test-aaaaa-master-1' was disallowed by policy.","target": "aro-test-aaaaa-master-1"},{"code": "RequestDisallowedByPolicy","message": "Resource 'aro-test-aaaaa-master-2' was disallowed by policy.","target": "aro-test-aaaaa-master-2"}]} - level=error msg=400: DeploymentFailed: : Deployment failed. Details: : : {"code": "InvalidTemplateDeployment","message": "The template deployment failed with multiple errors. Please see details for more information.","target": null,"details": [{"code": "RequestDisallowedByPolicy","message": "Resource 'aro-test-aaaaa-bootstrap' was disallowed by policy.","target": "aro-test-aaaaa-bootstrap"},{"code": "RequestDisallowedByPolicy","message": "Resource 'aro-test-aaaaa-master-0' was disallowed by policy.","target": "aro-test-aaaaa-master-0"},{"code": "RequestDisallowedByPolicy","message": "Resource 'aro-test-aaaaa-master-1' was disallowed by policy.","target": "aro-test-aaaaa-master-1"},{"code": "RequestDisallowedByPolicy","message": "Resource 'aro-test-aaaaa-master-2' was disallowed by policy.","target": "aro-test-aaaaa-master-2"}]}`, - wantErr: &api.CloudError{ - StatusCode: http.StatusBadRequest, - CloudErrorBody: &api.CloudErrorBody{ - Code: api.CloudErrorCodeDeploymentFailed, - Message: "The deployment failed. Please see details for more information.", - Details: []api.CloudErrorBody{ - { - Code: api.CloudErrorCodeRequestDisallowedByPolicy, - Message: "Resource 'aro-test-aaaaa-bootstrap' was disallowed by policy.", - Target: "aro-test-aaaaa-bootstrap", - }, - { - Code: api.CloudErrorCodeRequestDisallowedByPolicy, - Message: "Resource 'aro-test-aaaaa-master-0' was disallowed by policy.", - Target: "aro-test-aaaaa-master-0", - }, - { - Code: api.CloudErrorCodeRequestDisallowedByPolicy, - Message: "Resource 'aro-test-aaaaa-master-1' was disallowed by policy.", - Target: "aro-test-aaaaa-master-1", - }, - { - Code: api.CloudErrorCodeRequestDisallowedByPolicy, - Message: "Resource 'aro-test-aaaaa-master-2' was disallowed by policy.", - Target: "aro-test-aaaaa-master-2", - }, - }, - }, - }, - }, - } { - t.Run(tt.name, func(t *testing.T) { - cond := hivev1.ClusterDeploymentCondition{ - Type: hivev1.ProvisionFailedCondition, - Status: corev1.ConditionTrue, - Reason: tt.reason, - } - hcd := &hivev1.ClusterDeployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: ClusterDeploymentName, - Namespace: fakeNamespace, - }, - Status: hivev1.ClusterDeploymentStatus{ - Conditions: []hivev1.ClusterDeploymentCondition{cond}, - }, - } - hcp := &hivev1.ClusterProvision{ - ObjectMeta: metav1.ObjectMeta{ - Name: ClusterDeploymentName + "-0-bbbbb", - Namespace: fakeNamespace, - Labels: map[string]string{ - "hive.openshift.io/cluster-deployment-name": ClusterDeploymentName, - }, - }, - Spec: hivev1.ClusterProvisionSpec{ - InstallLog: &tt.installLog, - }, - } - - fakeClientBuilder := fake.NewClientBuilder(). - WithRuntimeObjects(hcd, hcp) - - c := clusterManager{ - hiveClientset: fakeClientBuilder.Build(), - log: logrus.NewEntry(logrus.StandardLogger()), - } - - err := c.handleProvisionFailed(context.Background(), hcd, cond) - - if diff := cmp.Diff(tt.wantErr, err); diff != "" { - t.Error(diff) - } - }) - } -} From 3145dbd41a6f4542dde4aabab68b6491ea1d50c0 Mon Sep 17 00:00:00 2001 From: swetha Date: Fri, 28 Jul 2023 18:03:59 -0700 Subject: [PATCH 3/3] EncryptionAtHostIsNotValid_error_message --- pkg/hive/const.go | 5 +++-- pkg/hive/manager.go | 31 +++++++++++++++++++++++++++++++ pkg/hive/manager_test.go | 27 +++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 2 deletions(-) diff --git a/pkg/hive/const.go b/pkg/hive/const.go index 3c4057bf7de..57a8e99ec7b 100644 --- a/pkg/hive/const.go +++ b/pkg/hive/const.go @@ -4,6 +4,7 @@ package hive // Licensed under the Apache License 2.0. const ( - ProvisionFailedReasonInvalidTemplateDeployment = "AzureInvalidTemplateDeployment" - ProvisionFailedReasonUnknownError = "UnknownError" + ProvisionFailedReasonInvalidTemplateDeployment = "AzureInvalidTemplateDeployment" + ProvisionFailedReasonEncryptionAtHostIsNotValid = "EncryptionAtHostIsNotValid" + ProvisionFailedReasonUnknownError = "UnknownError" ) diff --git a/pkg/hive/manager.go b/pkg/hive/manager.go index 5e27d57a5d0..afa505eeacf 100644 --- a/pkg/hive/manager.go +++ b/pkg/hive/manager.go @@ -282,6 +282,37 @@ func (hr *clusterManager) handleProvisionFailed(ctx context.Context, cd *hivev1. } return cloudErr + + case ProvisionFailedReasonEncryptionAtHostIsNotValid: + latestProvision, err := hr.latestProvisionForDeployment(ctx, cd) + if err != nil { + return err + } + installLog := *latestProvision.Spec.InstallLog + installLog = strings.TrimSpace(installLog) + installLogLines := strings.Split(installLog, "\n") + lastLine := installLogLines[len(installLogLines)-1] + + //regex := regexp.MustCompile(`"The property 'securityProfile.encryptionAtHost' is not valid"`) + //match := regex.FindString(installLog) + regex := regexp.MustCompile(`(\{.*\})`) + responseJson := regex.FindStringSubmatch(lastLine)[1] + + response := &mgmtfeatures.ErrorResponse{} + if err := json.Unmarshal([]byte(responseJson), response); err != nil { + return err + } + + cloudErr := &api.CloudError{ + StatusCode: http.StatusBadRequest, + CloudErrorBody: &api.CloudErrorBody{ + Code: api.CloudErrorCodeInvalidParameter, + Message: "Microsoft.Compute/EncryptionAtHost feature is not enabled for this subscription.", + }, + } + + return cloudErr + default: return &api.CloudError{ StatusCode: http.StatusInternalServerError, diff --git a/pkg/hive/manager_test.go b/pkg/hive/manager_test.go index 632701e658a..74da64d128f 100644 --- a/pkg/hive/manager_test.go +++ b/pkg/hive/manager_test.go @@ -344,6 +344,33 @@ func TestIsClusterInstallationComplete(t *testing.T) { }, wantResult: false, }, + { + name: "has failed provisioning - EncryptionAtHostIsNotValid", + cd: makeClusterDeployment( + false, + hivev1.ClusterDeploymentCondition{ + Type: hivev1.ProvisionFailedCondition, + Status: corev1.ConditionTrue, + Reason: ProvisionFailedReasonEncryptionAtHostIsNotValid, + }, + ), + cp: makeClusterProvision(`level=info msg=running in local development mode + level=info msg=creating development InstanceMetadata + level=info msg=InstanceMetadata: running on AzurePublicCloud + level=info msg=running step [AuthorizationRetryingAction github.com/openshift/ARO-Installer/pkg/installer.(*manager).deployResourceTemplate-fm] + level=info msg=load persisted graph + level=info msg=deploying resources template + level=error msg=step [AuthorizationRetryingAction github.com/openshift/ARO-Installer/pkg/installer.(*manager).deployResourceTemplate-fm] encountered error: 400: DeploymentFailed: : Deployment failed. Details: : : {"code":"DeploymentFailed","message":"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.","target":null,"details":[{"code":"BadRequest","message":"{\r\n \"error\": {\r\n \"code\": \"InvalidParameter\",\r\n \"message\": \"The property 'securityProfile.encryptionAtHost' is not valid because the 'Microsoft.Compute/EncryptionAtHost' feature is not enabled for this subscription.\",\r\n \"target\": \"securityProfile.encryptionAtHost\"\r\n }\r\n}"},{"code":"BadRequest","message":"{\r\n \"error\": {\r\n \"code\": \"InvalidParameter\",\r\n \"message\": \"The property 'securityProfile.encryptionAtHost' is not valid because the 'Microsoft.Compute/EncryptionAtHost' feature is not enabled for this subscription.\",\r\n \"target\": \"securityProfile.encryptionAtHost\"\r\n }\r\n}"},{"code":"BadRequest","message":"{\r\n \"error\": {\r\n \"code\": \"InvalidParameter\",\r\n \"message\": \"The property 'securityProfile.encryptionAtHost' is not valid because the 'Microsoft.Compute/EncryptionAtHost' feature is not enabled for this subscription.\",\r\n \"target\": \"securityProfile.encryptionAtHost\"\r\n }\r\n}"},{"code":"BadRequest","message":"{\r\n \"error\": {\r\n \"code\": \"InvalidParameter\",\r\n \"message\": \"The property 'securityProfile.encryptionAtHost' is not valid because the 'Microsoft.Compute/EncryptionAtHost' feature is not enabled for this subscription.\",\r\n \"target\": \"securityProfile.encryptionAtHost\"\r\n }\r\n}"}],"innererror":null,"additionalInfo":null} + level=error msg=400: DeploymentFailed: : Deployment failed. Details: : : {"code":"DeploymentFailed","message":"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.","target":null,"details":[{"code":"BadRequest","message":"{\r\n \"error\": {\r\n \"code\": \"InvalidParameter\",\r\n \"message\": \"The property 'securityProfile.encryptionAtHost' is not valid because the 'Microsoft.Compute/EncryptionAtHost' feature is not enabled for this subscription.\",\r\n \"target\": \"securityProfile.encryptionAtHost\"\r\n }\r\n}"},{"code":"BadRequest","message":"{\r\n \"error\": {\r\n \"code\": \"InvalidParameter\",\r\n \"message\": \"The property 'securityProfile.encryptionAtHost' is not valid because the 'Microsoft.Compute/EncryptionAtHost' feature is not enabled for this subscription.\",\r\n \"target\": \"securityProfile.encryptionAtHost\"\r\n }\r\n}"},{"code":"BadRequest","message":"{\r\n \"error\": {\r\n \"code\": \"InvalidParameter\",\r\n \"message\": \"The property 'securityProfile.encryptionAtHost' is not valid because the 'Microsoft.Compute/EncryptionAtHost' feature is not enabled for this subscription.\",\r\n \"target\": \"securityProfile.encryptionAtHost\"\r\n }\r\n}"},{"code":"BadRequest","message":"{\r\n \"error\": {\r\n \"code\": \"InvalidParameter\",\r\n \"message\": \"The property 'securityProfile.encryptionAtHost' is not valid because the 'Microsoft.Compute/EncryptionAtHost' feature is not enabled for this subscription.\",\r\n \"target\": \"securityProfile.encryptionAtHost\"\r\n }\r\n}"}],"innererror":null,"additionalInfo":null}`), + wantErr: &api.CloudError{ + StatusCode: http.StatusBadRequest, + CloudErrorBody: &api.CloudErrorBody{ + Code: api.CloudErrorCodeInvalidParameter, + Message: "Microsoft.Compute/EncryptionAtHost feature is not enabled for this subscription.", + }, + }, + wantResult: false, + }, } { t.Run(tt.name, func(t *testing.T) { fakeClientBuilder := fake.NewClientBuilder()