From 6fdebd676dc041456bad45f3b3e363db203203db Mon Sep 17 00:00:00 2001 From: Fuat Basik Date: Tue, 11 Jun 2024 14:08:39 +0100 Subject: [PATCH] Adding new logic to send user-agent in S3 Requests With this commit, we will send s3connectorframework user-agent in every S3 request to help with observability. This commit also introduces ObjectClientConfiguration to let uses to prepend user-agent with custom values. --- .../s3/ObjectClientConfiguration.java | 29 ++++++++++++ .../connector/s3/S3SdkObjectClient.java | 44 ++++++++++++++++--- .../com/amazon/connector/s3/UserAgent.java | 33 ++++++++++++++ .../connector/s3/S3SdkObjectClientTest.java | 7 +++ .../amazon/connector/s3/UserAgentTest.java | 37 ++++++++++++++++ 5 files changed, 143 insertions(+), 7 deletions(-) create mode 100644 object-client/src/main/java/com/amazon/connector/s3/ObjectClientConfiguration.java create mode 100644 object-client/src/main/java/com/amazon/connector/s3/UserAgent.java create mode 100644 object-client/src/test/java/com/amazon/connector/s3/UserAgentTest.java diff --git a/object-client/src/main/java/com/amazon/connector/s3/ObjectClientConfiguration.java b/object-client/src/main/java/com/amazon/connector/s3/ObjectClientConfiguration.java new file mode 100644 index 00000000..47dc741a --- /dev/null +++ b/object-client/src/main/java/com/amazon/connector/s3/ObjectClientConfiguration.java @@ -0,0 +1,29 @@ +package com.amazon.connector.s3; + +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; + +/** Configuration for {@link ObjectClient} */ +@Getter +@Builder +@EqualsAndHashCode +public class ObjectClientConfiguration { + private static final String DEFAULT_USER_AGENT_PREFIX = null; + + /** User Agent Prefix. {@link ObjectClientConfiguration#DEFAULT_USER_AGENT_PREFIX} by default. */ + @Builder.Default private String userAgentPrefix = DEFAULT_USER_AGENT_PREFIX; + + public static final ObjectClientConfiguration DEFAULT = + ObjectClientConfiguration.builder().build(); + + /** + * Construct {@link ObjectClientConfiguration} + * + * @param userAgentPrefix Prefix to prepend to ObjectClient's userAgent. + */ + @Builder + private ObjectClientConfiguration(String userAgentPrefix) { + this.userAgentPrefix = userAgentPrefix; + } +} diff --git a/object-client/src/main/java/com/amazon/connector/s3/S3SdkObjectClient.java b/object-client/src/main/java/com/amazon/connector/s3/S3SdkObjectClient.java index c2350c90..e7039d1b 100644 --- a/object-client/src/main/java/com/amazon/connector/s3/S3SdkObjectClient.java +++ b/object-client/src/main/java/com/amazon/connector/s3/S3SdkObjectClient.java @@ -15,15 +15,37 @@ /** Object client, based on AWS SDK v2 */ public class S3SdkObjectClient implements ObjectClient, AutoCloseable { + + public static final String HEADER_USER_AGENT = "User-Agent"; + private S3AsyncClient s3AsyncClient = null; + private ObjectClientConfiguration objectClientConfiguration = null; + private final UserAgent userAgent; /** - * Create an instance of a S3 client for interaction with Amazon S3 compatible object stores. + * Create an instance of a S3 client, with default configuration, for interaction with Amazon S3 + * compatible object stores. * * @param s3AsyncClient Underlying client to be used for making requests to S3. */ public S3SdkObjectClient(@NonNull S3AsyncClient s3AsyncClient) { + this(s3AsyncClient, ObjectClientConfiguration.DEFAULT); + } + + /** + * Create an instance of a S3 client, for interaction with Amazon S3 compatible object stores. + * + * @param s3AsyncClient Underlying client to be used for making requests to S3. + * @param objectClientConfiguration Configuration for object client. + */ + public S3SdkObjectClient( + @NonNull S3AsyncClient s3AsyncClient, + @NonNull ObjectClientConfiguration objectClientConfiguration) { + this.s3AsyncClient = s3AsyncClient; + this.objectClientConfiguration = objectClientConfiguration; + this.userAgent = new UserAgent(); + this.userAgent.prepend(objectClientConfiguration.getUserAgentPrefix()); } @Override @@ -39,12 +61,17 @@ public void close() { */ @Override public CompletableFuture headObject(HeadRequest headRequest) { + HeadObjectRequest.Builder builder = + HeadObjectRequest.builder().bucket(headRequest.getBucket()).key(headRequest.getKey()); + + // Add User-Agent header to the request. + builder.overrideConfiguration( + AwsRequestOverrideConfiguration.builder() + .putHeader(HEADER_USER_AGENT, this.userAgent.getUserAgent()) + .build()); + return s3AsyncClient - .headObject( - HeadObjectRequest.builder() - .bucket(headRequest.getBucket()) - .key(headRequest.getKey()) - .build()) + .headObject(builder.build()) .thenApply( headObjectResponse -> ObjectMetadata.builder().contentLength(headObjectResponse.contentLength()).build()); @@ -71,7 +98,10 @@ public CompletableFuture getObject(GetRequest getRequest) { // Temporarily adding range of data requested as a Referrer header to allow for easy analysis // of access logs. This is similar to what the Auditing feature in S3A does. builder.overrideConfiguration( - AwsRequestOverrideConfiguration.builder().putHeader("Referer", range).build()); + AwsRequestOverrideConfiguration.builder() + .putHeader("Referer", range) + .putHeader(HEADER_USER_AGENT, this.userAgent.getUserAgent()) + .build()); } return s3AsyncClient diff --git a/object-client/src/main/java/com/amazon/connector/s3/UserAgent.java b/object-client/src/main/java/com/amazon/connector/s3/UserAgent.java new file mode 100644 index 00000000..93aafc80 --- /dev/null +++ b/object-client/src/main/java/com/amazon/connector/s3/UserAgent.java @@ -0,0 +1,33 @@ +package com.amazon.connector.s3; + +import java.util.Objects; +import lombok.Getter; + +/** User-Agent to be used by ObjectClients */ +@Getter +public final class UserAgent { + // Hard-coded user-agent string + private static final String UA_STRING = "s3connectorframework"; + // TODO: Get VersionInfo and append it to UA. We need to understand how to create (if we want a) + // global version (for InputStream and ObjectClient). + private static final String VERSION_INFO = null; + /** + * Disallowed characters in the user agent token: @see RFC 7230 + */ + private static final String UA_DENYLIST_REGEX = "[() ,/:;<=>?@\\[\\]{}\\\\]"; + + private String userAgent = UA_STRING + ":" + VERSION_INFO; + + /** + * @param userAgentPrefix to prepend the default user-agent string + */ + public void prepend(String userAgentPrefix) { + if (Objects.nonNull(userAgentPrefix)) + this.userAgent = sanitizeInput(userAgentPrefix) + "/" + this.userAgent; + } + + private static String sanitizeInput(String input) { + return input == null ? "unknown" : input.replaceAll(UA_DENYLIST_REGEX, "_"); + } +} diff --git a/object-client/src/test/java/com/amazon/connector/s3/S3SdkObjectClientTest.java b/object-client/src/test/java/com/amazon/connector/s3/S3SdkObjectClientTest.java index a97e49f0..4306e3af 100644 --- a/object-client/src/test/java/com/amazon/connector/s3/S3SdkObjectClientTest.java +++ b/object-client/src/test/java/com/amazon/connector/s3/S3SdkObjectClientTest.java @@ -53,6 +53,13 @@ void testConstructorWithWrappedClient() { assertNotNull(client); } + @Test + void testConstructorWithConfiguration() { + ObjectClientConfiguration configuration = ObjectClientConfiguration.DEFAULT; + S3SdkObjectClient client = new S3SdkObjectClient(s3AsyncClient, configuration); + assertNotNull(client); + } + @Test void testConstructorThrowsOnNullArgument() { assertThrows( diff --git a/object-client/src/test/java/com/amazon/connector/s3/UserAgentTest.java b/object-client/src/test/java/com/amazon/connector/s3/UserAgentTest.java new file mode 100644 index 00000000..c24e25ba --- /dev/null +++ b/object-client/src/test/java/com/amazon/connector/s3/UserAgentTest.java @@ -0,0 +1,37 @@ +package com.amazon.connector.s3; + +import static org.junit.jupiter.api.Assertions.*; + +import org.junit.jupiter.api.Test; + +public class UserAgentTest { + + @Test + void testDefaultUserAgent() { + UserAgent agent = new UserAgent(); + assertNotNull(agent.getUserAgent()); + } + + @Test + void testPrependUserAgent() { + UserAgent agent = new UserAgent(); + agent.prepend("unit_test"); + assertTrue(agent.getUserAgent().startsWith("unit_test")); + } + + @Test + void testNullPrependIsNoop() { + UserAgent agent = new UserAgent(); + String pre = agent.getUserAgent(); + agent.prepend(null); + String pos = agent.getUserAgent(); + assertEquals(pre, pos); + } + + @Test + void testInvalidInputIsSanitised() { + UserAgent agent = new UserAgent(); + agent.prepend("unit/test"); + assertTrue(agent.getUserAgent().startsWith("unit_test")); + } +}