From d209c8a0ab47e708c76f30c20baa85017ff27bc8 Mon Sep 17 00:00:00 2001 From: LiniSusan Date: Tue, 28 Nov 2023 23:23:22 +0530 Subject: [PATCH] ListAccountSAS throttling error capturing --- pkg/api/error.go | 1 + pkg/util/steps/runner.go | 13 +++++++++++++ pkg/util/storage/manager.go | 18 ++++++++++++++++++ 3 files changed, 32 insertions(+) diff --git a/pkg/api/error.go b/pkg/api/error.go index ea481074ee6..9a855139a00 100644 --- a/pkg/api/error.go +++ b/pkg/api/error.go @@ -102,6 +102,7 @@ const ( CloudErrorCodeInUseSubnetCannotBeDeleted = "InUseSubnetCannotBeDeleted" CloudErrorCodeScopeLocked = "ScopeLocked" CloudErrorCodeRequestDisallowedByPolicy = "RequestDisallowedByPolicy" + CloudErrorCodeThrottlingLimitExceeded = "ThrottlingLimitExceeded" ) // NewCloudError returns a new CloudError diff --git a/pkg/util/steps/runner.go b/pkg/util/steps/runner.go index c2666dda07c..8cabb1f7e57 100644 --- a/pkg/util/steps/runner.go +++ b/pkg/util/steps/runner.go @@ -5,6 +5,8 @@ package steps import ( "context" + "fmt" + "net/http" "reflect" "runtime" "strings" @@ -13,6 +15,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/sirupsen/logrus" + "github.com/Azure/ARO-RP/pkg/api" msgraph_errors "github.com/Azure/ARO-RP/pkg/util/graph/graphsdk/models/odataerrors" ) @@ -54,6 +57,16 @@ func Run(ctx context.Context, log *logrus.Entry, pollInterval time.Duration, ste if err != nil { log.Errorf("step %s encountered error: %s", step, err.Error()) + if strings.Contains(err.Error(), "ThrottlingLimitExceeded") || strings.Contains(err.Error(), "TooManyRequests") { + correlation_id := fmt.Sprintf("%v", log.Data["correlation_id"]) + message := fmt.Sprintf("Requests are being throttled due to Azure Storage limits being exceeded. Please visit https://learn.microsoft.com/en-us/azure/openshift/troubleshoot#exceeding-azure-storage-limits for more details. CorrelationId: " + correlation_id) + err = api.NewCloudError( + http.StatusTooManyRequests, + api.CloudErrorCodeThrottlingLimitExceeded, + "", + message) + return nil, err + } if oDataError, ok := err.(msgraph_errors.ODataErrorable); ok { spew.Fdump(log.Writer(), oDataError.GetErrorEscaped()) } diff --git a/pkg/util/storage/manager.go b/pkg/util/storage/manager.go index 54bdcde82c5..907988b6e71 100644 --- a/pkg/util/storage/manager.go +++ b/pkg/util/storage/manager.go @@ -5,7 +5,9 @@ package storage import ( "context" + "net/http" "net/url" + "strings" "time" mgmtstorage "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-06-01/storage" @@ -13,6 +15,7 @@ import ( "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/date" + "github.com/Azure/ARO-RP/pkg/api" "github.com/Azure/ARO-RP/pkg/env" "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/storage" ) @@ -43,7 +46,22 @@ func (m *manager) BlobService(ctx context.Context, resourceGroup, account string SharedAccessStartTime: &date.Time{Time: t}, SharedAccessExpiryTime: &date.Time{Time: t.Add(24 * time.Hour)}, }) + if err != nil { + if strings.Contains(err.Error(), "TooManyRequests") { + err = &api.CloudError{ + StatusCode: http.StatusTooManyRequests, + CloudErrorBody: &api.CloudErrorBody{ + Code: api.CloudErrorCodeThrottlingLimitExceeded, + Message: "ThrottlingLimitExceeded", + Details: []api.CloudErrorBody{ + { + Message: string("Requests are being throttled due to Azure Storage limits being exceeded. Please visit https://learn.microsoft.com/en-us/azure/openshift/troubleshoot#exceeding-azure-storage-limits for more details."), + }, + }, + }, + } + } return nil, err }