ListAccountSAS throttling error capturing

Azure · Dec 5, 2023 · d209c8a · d209c8a
1 parent 14d5b29
commit d209c8a
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 0 deletions.
diff --git a/pkg/api/error.go b/pkg/api/error.go
@@ -102,6 +102,7 @@ const (
 	CloudErrorCodeInUseSubnetCannotBeDeleted         = "InUseSubnetCannotBeDeleted"
 	CloudErrorCodeScopeLocked                        = "ScopeLocked"
 	CloudErrorCodeRequestDisallowedByPolicy          = "RequestDisallowedByPolicy"
+	CloudErrorCodeThrottlingLimitExceeded            = "ThrottlingLimitExceeded"
 )
 
 // NewCloudError returns a new CloudError

diff --git a/pkg/util/steps/runner.go b/pkg/util/steps/runner.go
@@ -5,6 +5,8 @@ package steps
 
 import (
 	"context"
+	"fmt"
+	"net/http"
 	"reflect"
 	"runtime"
 	"strings"
@@ -13,6 +15,7 @@ import (
 	"github.com/davecgh/go-spew/spew"
 	"github.com/sirupsen/logrus"
 
+	"github.com/Azure/ARO-RP/pkg/api"
 	msgraph_errors "github.com/Azure/ARO-RP/pkg/util/graph/graphsdk/models/odataerrors"
 )
 
@@ -54,6 +57,16 @@ func Run(ctx context.Context, log *logrus.Entry, pollInterval time.Duration, ste
 
 		if err != nil {
 			log.Errorf("step %s encountered error: %s", step, err.Error())
+			if strings.Contains(err.Error(), "ThrottlingLimitExceeded") || strings.Contains(err.Error(), "TooManyRequests") {
+				correlation_id := fmt.Sprintf("%v", log.Data["correlation_id"])
+				message := fmt.Sprintf("Requests are being throttled due to Azure Storage limits being exceeded. Please visit https://learn.microsoft.com/en-us/azure/openshift/troubleshoot#exceeding-azure-storage-limits for more details. CorrelationId: " + correlation_id)
+				err = api.NewCloudError(
+					http.StatusTooManyRequests,
+					api.CloudErrorCodeThrottlingLimitExceeded,
+					"",
+					message)
+				return nil, err
+			}
 			if oDataError, ok := err.(msgraph_errors.ODataErrorable); ok {
 				spew.Fdump(log.Writer(), oDataError.GetErrorEscaped())
 			}

diff --git a/pkg/util/storage/manager.go b/pkg/util/storage/manager.go
@@ -5,14 +5,17 @@ package storage
 
 import (
 	"context"
+	"net/http"
 	"net/url"
+	"strings"
 	"time"
 
 	mgmtstorage "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-06-01/storage"
 	azstorage "github.com/Azure/azure-sdk-for-go/storage"
 	"github.com/Azure/go-autorest/autorest"
 	"github.com/Azure/go-autorest/autorest/date"
 
+	"github.com/Azure/ARO-RP/pkg/api"
 	"github.com/Azure/ARO-RP/pkg/env"
 	"github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/storage"
 )
@@ -43,7 +46,22 @@ func (m *manager) BlobService(ctx context.Context, resourceGroup, account string
 		SharedAccessStartTime:  &date.Time{Time: t},
 		SharedAccessExpiryTime: &date.Time{Time: t.Add(24 * time.Hour)},
 	})
+
 	if err != nil {
+		if strings.Contains(err.Error(), "TooManyRequests") {
+			err = &api.CloudError{
+				StatusCode: http.StatusTooManyRequests,
+				CloudErrorBody: &api.CloudErrorBody{
+					Code:    api.CloudErrorCodeThrottlingLimitExceeded,
+					Message: "ThrottlingLimitExceeded",
+					Details: []api.CloudErrorBody{
+						{
+							Message: string("Requests are being throttled due to Azure Storage limits being exceeded. Please visit https://learn.microsoft.com/en-us/azure/openshift/troubleshoot#exceeding-azure-storage-limits for more details."),
+						},
+					},
+				},
+			}
+		}
 		return nil, err
 	}