diff --git a/pkg/util/loadbalancer/loadbalancer.go b/pkg/util/loadbalancer/loadbalancer.go
index d18b862a0c0..5bfabff0666 100644
--- a/pkg/util/loadbalancer/loadbalancer.go
+++ b/pkg/util/loadbalancer/loadbalancer.go
@@ -15,6 +15,10 @@ import (
 )
 
 func RemoveFrontendIPConfiguration(lb *mgmtnetwork.LoadBalancer, resourceID string) error {
+	if lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations == nil {
+		return fmt.Errorf("FrontendIPConfigurations in nil!")
+	}
+
 	newFrontendIPConfig := make([]mgmtnetwork.FrontendIPConfiguration, 0, len(*lb.FrontendIPConfigurations))
 	for _, fipConfig := range *lb.LoadBalancerPropertiesFormat.FrontendIPConfigurations {
 		if strings.EqualFold(*fipConfig.ID, resourceID) {
diff --git a/pkg/util/loadbalancer/loadbalancer_test.go b/pkg/util/loadbalancer/loadbalancer_test.go
index c6deda3a61b..a449bb0f7c6 100644
--- a/pkg/util/loadbalancer/loadbalancer_test.go
+++ b/pkg/util/loadbalancer/loadbalancer_test.go
@@ -125,6 +125,17 @@ func TestRemoveLoadBalancerFrontendIPConfiguration(t *testing.T) {
 			expectedLB:    originalLB,
 			expectedErr:   fmt.Sprintf("frontend IP Configuration %s has external references, remove the external references prior to removing the frontend IP configuration", *publicIngressFIPConfigID),
 		},
+		{
+			name:          "removal of frontend ip config fails when frontend ip config doesn't exist",
+			fipResourceID: *publicIngressFIPConfigID,
+			currentLB: mgmtnetwork.LoadBalancer{
+				LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{},
+			},
+			expectedLB: mgmtnetwork.LoadBalancer{
+				LoadBalancerPropertiesFormat: &mgmtnetwork.LoadBalancerPropertiesFormat{},
+			},
+			expectedErr: fmt.Sprintf("there are no frontend configurations to remove"),
+		},
 	} {
 		t.Run(tt.name, func(t *testing.T) {
 			err := RemoveFrontendIPConfiguration(&tt.currentLB, tt.fipResourceID)