Skip to content

Commit 3eb8f81

Browse files
dagnirdagnir
authored
Retries fixes (#6920)
This commit fixes two issues with the original retries implementation: - Honor the 'max_retries' property in the profile file if present - Treat 'LimitExceededException' as a throttling exception
1 parent 4de251e commit 3eb8f81

6 files changed

Lines changed: 310 additions & 8 deletions

File tree

‎core/aws-core/src/main/java/software/amazon/awssdk/awscore/retry/AwsRetryStrategy.java‎

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,11 @@
1818
import software.amazon.awssdk.annotations.SdkPublicApi;
1919
import software.amazon.awssdk.awscore.exception.AwsServiceException;
2020
import software.amazon.awssdk.awscore.internal.AwsErrorCode;
21+
import software.amazon.awssdk.core.exception.SdkException;
2122
import software.amazon.awssdk.core.internal.retry.RetryPolicyAdapter;
2223
import software.amazon.awssdk.core.internal.retry.SdkDefaultRetryStrategy;
2324
import software.amazon.awssdk.core.retry.RetryMode;
25+
import software.amazon.awssdk.core.retry.RetryUtils;
2426
import software.amazon.awssdk.retries.AdaptiveRetryStrategy;
2527
import software.amazon.awssdk.retries.DefaultRetryStrategy;
2628
import software.amazon.awssdk.retries.LegacyRetryStrategy;
@@ -135,7 +137,7 @@ public static StandardRetryStrategy standardRetryStrategy() {
135137
*/
136138
public static StandardRetryStrategy standardRetryStrategy(boolean newRetries2026Enabled) {
137139
StandardRetryStrategy.Builder builder = SdkDefaultRetryStrategy.standardRetryStrategyBuilder(newRetries2026Enabled);
138-
return configure(builder).build();
140+
return configure(builder, newRetries2026Enabled).build();
139141
}
140142

141143
/**
@@ -167,7 +169,7 @@ public static AdaptiveRetryStrategy adaptiveRetryStrategy() {
167169
*/
168170
public static AdaptiveRetryStrategy adaptiveRetryStrategy(boolean newRetries2026Enabled) {
169171
AdaptiveRetryStrategy.Builder builder = SdkDefaultRetryStrategy.adaptiveRetryStrategyBuilder(newRetries2026Enabled);
170-
return configure(builder)
172+
return configure(builder, newRetries2026Enabled)
171173
.build();
172174
}
173175

@@ -179,7 +181,22 @@ public static AdaptiveRetryStrategy adaptiveRetryStrategy(boolean newRetries2026
179181
* @return The given builder
180182
*/
181183
public static <T extends RetryStrategy.Builder<T, ?>> T configure(T builder) {
184+
return configure(builder, false);
185+
}
186+
187+
/**
188+
* Configures a retry strategy using its builder to add AWS-specific retry exceptions.
189+
*
190+
* @param builder The builder to add the AWS-specific retry exceptions
191+
* @param <T> The type of the builder extending {@link RetryStrategy.Builder}
192+
* @return The given builder
193+
*/
194+
private static <T extends RetryStrategy.Builder<T, ?>> T configure(T builder, boolean newRetries2026Enabled) {
182195
builder.retryOnException(AwsRetryStrategy::retryOnAwsRetryableErrors);
196+
if (newRetries2026Enabled) {
197+
builder.retryOnException(AwsRetryStrategy::isLimitExceededErrorCode);
198+
builder.treatAsThrottling(AwsRetryStrategy::treatAsThrottlingV21);
199+
}
183200
markDefaultsAdded(builder);
184201
return builder;
185202
}
@@ -205,6 +222,25 @@ private static boolean retryOnAwsRetryableErrors(Throwable ex) {
205222
return false;
206223
}
207224

225+
/**
226+
* Additionally, check for LimitExceededException as it was not previously treated as a throttling exception.
227+
*/
228+
private static boolean treatAsThrottlingV21(Throwable ex) {
229+
if (!(ex instanceof SdkException)) {
230+
return false;
231+
}
232+
233+
SdkException sdkException = (SdkException) ex;
234+
235+
return RetryUtils.isThrottlingException(sdkException)
236+
|| isLimitExceededErrorCode(sdkException);
237+
}
238+
239+
private static boolean isLimitExceededErrorCode(Throwable ex) {
240+
return ex instanceof AwsServiceException
241+
&& "LimitExceededException".equals(((AwsServiceException) ex).awsErrorDetails().errorCode());
242+
}
243+
208244
/**
209245
* Returns a {@link RetryStrategy} that implements the legacy {@link RetryMode#ADAPTIVE} mode.
210246
*

‎core/aws-core/src/test/java/software/amazon/awssdk/awscore/retry/AwsRetryStrategyTest.java‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.awscore.retry;
17+
18+
import static org.assertj.core.api.Assertions.assertThat;
19+
import static org.assertj.core.api.Assertions.assertThatThrownBy;
20+
21+
import com.google.common.base.Supplier;
22+
import java.time.Duration;
23+
import org.junit.jupiter.params.ParameterizedTest;
24+
import org.junit.jupiter.params.provider.CsvSource;
25+
import software.amazon.awssdk.awscore.exception.AwsErrorDetails;
26+
import software.amazon.awssdk.awscore.exception.AwsServiceException;
27+
import software.amazon.awssdk.retries.StandardRetryStrategy;
28+
import software.amazon.awssdk.retries.api.AcquireInitialTokenRequest;
29+
import software.amazon.awssdk.retries.api.RefreshRetryTokenRequest;
30+
import software.amazon.awssdk.retries.api.RetryToken;
31+
import software.amazon.awssdk.retries.api.TokenAcquisitionFailedException;
32+
import software.amazon.awssdk.retries.internal.DefaultRetryToken;
33+
34+
public class AwsRetryStrategyTest {
35+
36+
@ParameterizedTest
37+
@CsvSource({"true", "false"})
38+
void standardRetryStrategy_limitExceededException_retryBehaviorCorrect(boolean newRetries2026Enabled) {
39+
StandardRetryStrategy strategy = AwsRetryStrategy.standardRetryStrategy(newRetries2026Enabled);
40+
41+
RetryToken token = strategy.acquireInitialToken(AcquireInitialTokenRequest.create("test")).token();
42+
RefreshRetryTokenRequest refresh = RefreshRetryTokenRequest.builder()
43+
.failure(createTestException("LimitExceededException"))
44+
.token(token)
45+
.build();
46+
47+
if (newRetries2026Enabled) {
48+
assertThat(strategy.refreshRetryToken(refresh).delay()).isGreaterThanOrEqualTo(Duration.ZERO);
49+
} else {
50+
assertThatThrownBy(() -> strategy.refreshRetryToken(refresh))
51+
.isInstanceOf(TokenAcquisitionFailedException.class)
52+
.matches(e -> {
53+
TokenAcquisitionFailedException acquireException = (TokenAcquisitionFailedException) e;
54+
DefaultRetryToken exceptionToken = (DefaultRetryToken) acquireException.token();
55+
return exceptionToken.state() == DefaultRetryToken.TokenState.NON_RETRYABLE_EXCEPTION;
56+
});
57+
}
58+
}
59+
60+
@ParameterizedTest
61+
@CsvSource({"Throttling",
62+
"ThrottlingException",
63+
"ThrottledException",
64+
"RequestThrottledException",
65+
"TooManyRequestsException",
66+
"ProvisionedThroughputExceededException",
67+
"TransactionInProgressException",
68+
"RequestLimitExceeded",
69+
"BandwidthLimitExceeded",
70+
"LimitExceededException",
71+
"RequestThrottled",
72+
"SlowDown",
73+
"PriorRequestNotComplete",
74+
"EC2ThrottledException"})
75+
void standardRetryStrategy_retry21_throttlingBehaviorCorrect(String errorCode) {
76+
AwsServiceException exception = createTestException(errorCode);
77+
78+
for (int i = 0; i < 128; ++i) {
79+
StandardRetryStrategy strategy = AwsRetryStrategy.standardRetryStrategy(true);
80+
81+
RetryToken token = strategy.acquireInitialToken(AcquireInitialTokenRequest.create("test")).token();
82+
RefreshRetryTokenRequest refresh = RefreshRetryTokenRequest.builder()
83+
.token(token)
84+
.failure(exception)
85+
.build();
86+
Duration delay = strategy.refreshRetryToken(refresh).delay();
87+
88+
assertThat(delay).isBetween(Duration.ZERO, Duration.ofMillis(1000));
89+
}
90+
}
91+
92+
private static AwsServiceException createTestException(String errorCode) {
93+
AwsErrorDetails details = AwsErrorDetails.builder()
94+
.errorCode(errorCode)
95+
.build();
96+
return AwsServiceException.builder().awsErrorDetails(details).build();
97+
}
98+
}

‎core/profiles/src/main/java/software/amazon/awssdk/profiles/ProfileProperty.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,12 @@ public final class ProfileProperty {
111111
*/
112112
public static final String RETRY_MODE = "retry_mode";
113113

114+
/**
115+
* How many HTTP requests an SDK should make for a single SDK operation invocation before giving up. See the JavaDocs for
116+
* {@code SdkSystemSetting.AWS_MAX_ATTEMPTS} and {@code RetryStrategy.maxAttempts()} for more information.
117+
*/
118+
public static final String MAX_ATTEMPTS = "max_attempts";
119+
114120
/**
115121
* The "defaults mode" to be used for clients created using the currently-configured profile. Defaults mode determins how SDK
116122
* default configuration should be resolved. See the {@code DefaultsMode} class JavaDoc for more

‎core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/retry/MaxAttemptsResolver.java‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.core.internal.retry;
17+
18+
import java.util.Optional;
19+
import java.util.function.Supplier;
20+
import software.amazon.awssdk.annotations.SdkInternalApi;
21+
import software.amazon.awssdk.core.SdkSystemSetting;
22+
import software.amazon.awssdk.profiles.ProfileFile;
23+
import software.amazon.awssdk.profiles.ProfileFileSystemSetting;
24+
import software.amazon.awssdk.profiles.ProfileProperty;
25+
import software.amazon.awssdk.utils.OptionalUtils;
26+
27+
/**
28+
* Resolves the retry max attempts from {@link SdkSystemSetting#AWS_MAX_ATTEMPTS} and {@link ProfileProperty#MAX_ATTEMPTS}.
29+
*/
30+
@SdkInternalApi
31+
public class MaxAttemptsResolver {
32+
private Supplier<ProfileFile> profileFile;
33+
private String profileName;
34+
35+
/**
36+
* Configure the profile file that should be used when determining the max attempts. The supplier is only consulted
37+
* if a higher-priority determinant (e.g. environment variables) does not find the setting.
38+
*/
39+
public MaxAttemptsResolver profileFile(Supplier<ProfileFile> profileFile) {
40+
this.profileFile = profileFile;
41+
return this;
42+
}
43+
44+
/**
45+
* Configure the profile file name should be used when determining the max attempts.
46+
*/
47+
public MaxAttemptsResolver profileName(String profileName) {
48+
this.profileName = profileName;
49+
return this;
50+
}
51+
52+
/**
53+
* Resolve the max attempts based on the configured values. If not configured, returns {@code null}.
54+
*/
55+
public Integer resolve() {
56+
return OptionalUtils.firstPresent(fromSystemSettings(), () -> fromProfileFile(profileFile, profileName))
57+
.orElse(null);
58+
}
59+
60+
61+
private static Optional<Integer> fromSystemSettings() {
62+
return SdkSystemSetting.AWS_MAX_ATTEMPTS.getIntegerValue();
63+
}
64+
65+
private static Optional<Integer> fromProfileFile(Supplier<ProfileFile> profileFile, String profileName) {
66+
profileFile = profileFile != null ? profileFile : ProfileFile::defaultProfileFile;
67+
profileName = profileName != null ? profileName : ProfileFileSystemSetting.AWS_PROFILE.getStringValueOrThrow();
68+
return profileFile.get()
69+
.profile(profileName)
70+
.flatMap(p -> p.property(ProfileProperty.MAX_ATTEMPTS))
71+
.map(Integer::parseInt);
72+
}
73+
}

‎core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/retry/SdkDefaultRetryStrategy.java‎

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -70,9 +70,19 @@ public static RetryStrategy defaultRetryStrategy() {
7070
* @return the appropriate retry strategy for the retry mode with AWS-specific conditions added.
7171
*/
7272
public static RetryStrategy forRetryMode(RetryMode mode) {
73+
return forRetryMode(mode, false);
74+
}
75+
76+
/**
77+
* Retrieve the appropriate retry strategy for the retry mode with AWS-specific conditions added.
78+
*
79+
* @param mode The retry mode for which we want the retry strategy
80+
* @return the appropriate retry strategy for the retry mode with AWS-specific conditions added.
81+
*/
82+
public static RetryStrategy forRetryMode(RetryMode mode, boolean newRetries2026Enabled) {
7383
switch (mode) {
7484
case STANDARD:
75-
return standardRetryStrategy();
85+
return standardRetryStrategy(newRetries2026Enabled);
7686
case ADAPTIVE:
7787
return legacyAdaptiveRetryStrategy();
7888
case ADAPTIVE_V2:
@@ -115,6 +125,10 @@ public static StandardRetryStrategy standardRetryStrategy() {
115125
return standardRetryStrategyBuilder().build();
116126
}
117127

128+
public static StandardRetryStrategy standardRetryStrategy(boolean newRetries2026Enabled) {
129+
return standardRetryStrategyBuilder(newRetries2026Enabled).build();
130+
}
131+
118132
/**
119133
* Returns a {@link LegacyRetryStrategy} with generic SDK retry conditions.
120134
*
@@ -149,7 +163,7 @@ public static StandardRetryStrategy.Builder standardRetryStrategyBuilder() {
149163
*/
150164
public static StandardRetryStrategy.Builder standardRetryStrategyBuilder(boolean newRetries2026Enabled) {
151165
StandardRetryStrategy.Builder builder = DefaultRetryStrategy.standardStrategyBuilder(newRetries2026Enabled);
152-
return configure(builder);
166+
return configure(builder, newRetries2026Enabled);
153167
}
154168

155169

@@ -179,7 +193,7 @@ public static AdaptiveRetryStrategy.Builder adaptiveRetryStrategyBuilder() {
179193
*/
180194
public static AdaptiveRetryStrategy.Builder adaptiveRetryStrategyBuilder(boolean newRetries2026Enabled) {
181195
AdaptiveRetryStrategy.Builder builder = DefaultRetryStrategy.adaptiveStrategyBuilder(newRetries2026Enabled);
182-
return configure(builder);
196+
return configure(builder, newRetries2026Enabled);
183197
}
184198

185199
/**
@@ -190,13 +204,17 @@ public static AdaptiveRetryStrategy.Builder adaptiveRetryStrategyBuilder(boolean
190204
* @return The given builder
191205
*/
192206
public static <T extends RetryStrategy.Builder<T, ?>> T configure(T builder) {
207+
return configure(builder, false);
208+
}
209+
210+
private static <T extends RetryStrategy.Builder<T, ?>> T configure(T builder, boolean newRetries2026Enabled) {
193211
builder.retryOnException(SdkDefaultRetryStrategy::retryOnRetryableException)
194212
.retryOnException(SdkDefaultRetryStrategy::retryOnStatusCodes)
195213
.retryOnException(SdkDefaultRetryStrategy::retryOnClockSkewException)
196214
.retryOnException(SdkDefaultRetryStrategy::retryOnThrottlingCondition);
197215
SdkDefaultRetrySetting.RETRYABLE_EXCEPTIONS.forEach(builder::retryOnExceptionOrCauseInstanceOf);
198216
builder.treatAsThrottling(SdkDefaultRetryStrategy::treatAsThrottling);
199-
Integer maxAttempts = SdkSystemSetting.AWS_MAX_ATTEMPTS.getIntegerValue().orElse(null);
217+
Integer maxAttempts = resolveMaxAttempts(newRetries2026Enabled);
200218
if (maxAttempts != null) {
201219
builder.maxAttempts(maxAttempts);
202220
}
@@ -281,5 +299,13 @@ private static void markDefaultsAdded(RetryStrategy.Builder<?, ?> builder) {
281299
}
282300
}
283301

302+
static Integer resolveMaxAttempts(boolean newRetries2026Enabled) {
303+
if (newRetries2026Enabled) {
304+
return new MaxAttemptsResolver().resolve();
305+
}
306+
307+
// pre 2.1 changes, we never looked at the profile file
308+
return SdkSystemSetting.AWS_MAX_ATTEMPTS.getIntegerValue().orElse(null);
309+
}
284310
}
285311

0 commit comments

Comments
 (0)