From 9769751dc9e0aeddd561ace88976a42fc466a1e3 Mon Sep 17 00:00:00 2001 From: Bo Blodgett Date: Mon, 1 Jul 2024 14:33:26 -0700 Subject: [PATCH] Updated support for bidi characters in resource paths. --- .../CrtIntegrationTests/V4aSignerTests.cs | 3 - .../Amazon.Runtime/AmazonServiceClient.cs | 32 +----- .../Amazon.Runtime/Internal/DefaultRequest.cs | 21 ---- .../Core/Amazon.Runtime/Internal/IRequest.cs | 11 -- sdk/src/Core/Amazon.Util/AWSSDKUtils.cs | 103 ------------------ sdk/test/IntegrationTests/Tests/General.cs | 30 ----- .../S3/IntegrationTests/PutObjectTests.cs | 47 ++++++++ 7 files changed, 49 insertions(+), 198 deletions(-) diff --git a/extensions/test/CrtIntegrationTests/V4aSignerTests.cs b/extensions/test/CrtIntegrationTests/V4aSignerTests.cs index e3b4d7470125..7110cbdd6533 100644 --- a/extensions/test/CrtIntegrationTests/V4aSignerTests.cs +++ b/extensions/test/CrtIntegrationTests/V4aSignerTests.cs @@ -131,7 +131,6 @@ internal static IRequest BuildHeaderRequestToSign(string resourcePath) IRequest request = mock.Object; request.HttpMethod = "POST"; request.ResourcePath = resourcePath; - request.MarshallerVersion = 2; request.Content = Encoding.ASCII.GetBytes("Param1=value1"); request.Endpoint = new Uri("https://" + SigningTestHost + "/"); @@ -220,7 +219,6 @@ internal static IRequest BuildQueryParamRequestToSign() var request = mock.Object; request.HttpMethod = "GET"; request.ResourcePath = "/"; - request.MarshallerVersion = 2; request.Endpoint = new Uri("https://" + SigningTestHost + "/"); return request; @@ -276,7 +274,6 @@ internal IRequest BuildMockChunkedRequest() mock.SetupGet(x => x.SubResources).Returns(new Dictionary()); var request = mock.Object; - request.MarshallerVersion = 2; request.HttpMethod = "PUT"; request.ResourcePath = "/examplebucket/chunkObject.txt"; request.Endpoint = new Uri("https://s3.amazonaws.com/"); diff --git a/sdk/src/Core/Amazon.Runtime/AmazonServiceClient.cs b/sdk/src/Core/Amazon.Runtime/AmazonServiceClient.cs index 69e798bf0b2a..d34b3dbcdfcf 100644 --- a/sdk/src/Core/Amazon.Runtime/AmazonServiceClient.cs +++ b/sdk/src/Core/Amazon.Runtime/AmazonServiceClient.cs @@ -494,19 +494,7 @@ public static Uri ComposeUrl(IRequest internalRequest, bool skipEncodingValidPat if (resourcePath.StartsWith("/", StringComparison.Ordinal)) resourcePath = resourcePath.Substring(1); - // Microsoft added support for unicode bidi control characters to the Uri class in .NET 4.7.2 - // https://github.com/microsoft/dotnet/blob/master/Documentation/compatibility/uri-unicode-bidirectional-characters.md - // However, we only want to support it on .NET Core 3.1 and higher due to not having to deal with .NET Standard support matrix. -#if BCL || NETSTANDARD20 - if (AWSSDKUtils.HasBidiControlCharacters(resourcePath) || - (internalRequest.PathResources?.Any(v => AWSSDKUtils.HasBidiControlCharacters(v.Value)) == true)) - { - resourcePath = string.Join("/", AWSSDKUtils.SplitResourcePathIntoSegments(resourcePath, internalRequest.PathResources).ToArray()); - throw new AmazonClientException(string.Format(CultureInfo.InvariantCulture, - "Target resource path [{0}] has bidirectional characters, which are not supported" + - "by System.Uri and thus cannot be handled by the .NET SDK.", resourcePath)); - } -#endif // Since S3 is the only service that is single encoded, we send the URL unencoded for special characters + // Since S3 is the only service that is single encoded, we send the URL unencoded for special characters // to match the previous behavior compatible with the SigV2 backend. if (internalRequest.SignatureVersion == SignatureVersion.SigV2 && String.Equals(internalRequest.ServiceName, "AmazonS3")) { @@ -547,23 +535,7 @@ public static Uri ComposeUrl(IRequest internalRequest, bool skipEncodingValidPat sb.AppendFormat("{0}{1}", delim, queryString); } - var parameterizedPath = string.Empty; - if(internalRequest.MarshallerVersion >= 2) - { - parameterizedPath = string.Concat(resourcePath, sb); - } - else - { - if (AWSSDKUtils.HasBidiControlCharacters(resourcePath)) - throw new AmazonClientException(string.Format(CultureInfo.InvariantCulture, - "Target resource path [{0}] has bidirectional characters, which are not supported" + - "by System.Uri and thus cannot be handled by the .NET SDK.", resourcePath)); - -#pragma warning disable CS0612,CS0618 // Type or member is obsolete - parameterizedPath = string.Concat(AWSSDKUtils.ProtectEncodedSlashUrlEncode(resourcePath, skipEncodingValidPathChars), sb); -#pragma warning restore CS0612,CS0618 // Type or member is obsolete - } - + var parameterizedPath = string.Concat(resourcePath, sb); var hasSlash = url.AbsoluteUri.EndsWith("/", StringComparison.Ordinal) || parameterizedPath.StartsWith("/", StringComparison.Ordinal); var strUri = hasSlash diff --git a/sdk/src/Core/Amazon.Runtime/Internal/DefaultRequest.cs b/sdk/src/Core/Amazon.Runtime/Internal/DefaultRequest.cs index 94fac1655e0b..80eb41c2b7cd 100644 --- a/sdk/src/Core/Amazon.Runtime/Internal/DefaultRequest.cs +++ b/sdk/src/Core/Amazon.Runtime/Internal/DefaultRequest.cs @@ -55,7 +55,6 @@ public class DefaultRequest : IRequest string canonicalResource; RegionEndpoint alternateRegion; long originalStreamLength; - int marshallerVersion = 2; //2 is the default version and must be used whenever a version is not specified in the marshaller. /// /// Constructs a new DefaultRequest with the specified service name and the @@ -256,26 +255,6 @@ public void AddPathResource(string key, string value) PathResources.Add(key, value); } - /// - /// Gets and Sets the version number for the marshaller used to create this request. The version number - /// is used to support backward compatible changes that would otherwise be breaking changes when a - /// newer core is used with an older service assembly. - /// Versions: - /// 1 - Legacy version (no longer supported) - /// 2 - Default version. Support for path segments - /// - public int MarshallerVersion - { - get - { - return this.marshallerVersion; - } - set - { - this.marshallerVersion = value; - } - } - public string CanonicalResource { get diff --git a/sdk/src/Core/Amazon.Runtime/Internal/IRequest.cs b/sdk/src/Core/Amazon.Runtime/Internal/IRequest.cs index 2ba9082ee22b..241f0ef6195a 100644 --- a/sdk/src/Core/Amazon.Runtime/Internal/IRequest.cs +++ b/sdk/src/Core/Amazon.Runtime/Internal/IRequest.cs @@ -144,17 +144,6 @@ IDictionary PathResources /// Value of the entry void AddPathResource(string key, string value); - /// - /// Gets and Sets the version number for the marshaller used to create this request. The version number - /// is used to support backward compatible changes that would otherwise be breaking changes when a - /// newer core is used with an older service assembly. - /// - int MarshallerVersion - { - get; - set; - } - /// /// Gets and Sets the content for this request. /// diff --git a/sdk/src/Core/Amazon.Util/AWSSDKUtils.cs b/sdk/src/Core/Amazon.Util/AWSSDKUtils.cs index 9e45563d1ee6..21277cca627d 100644 --- a/sdk/src/Core/Amazon.Util/AWSSDKUtils.cs +++ b/sdk/src/Core/Amazon.Util/AWSSDKUtils.cs @@ -88,8 +88,6 @@ public static partial class AWSSDKUtils internal const string S3Accelerate = "s3-accelerate"; internal const string S3Control = "s3-control"; - private const int DefaultMarshallerVersion = 2; - private static readonly string _userAgent = InternalSDKUtils.BuildUserAgentString(string.Empty, string.Empty); #endregion @@ -324,107 +322,6 @@ internal static string GetParametersAsString(ParameterCollection parameterCollec return result.Remove(result.Length - 1); } - /// - /// Returns the canonicalized resource path for the service endpoint with single URL encoded path segments. - /// - /// Endpoint URL for the request - /// Resource path for the request - /// - /// If resourcePath begins or ends with slash, the resulting canonicalized - /// path will follow suit. - /// - /// Canonicalized resource path for the endpoint - [Obsolete("Use CanonicalizeResourcePathV2 instead")] - public static string CanonicalizeResourcePath(Uri endpoint, string resourcePath) - { - // This overload is kept for backward compatibility in existing code bases. - return CanonicalizeResourcePath(endpoint, resourcePath, false, null, DefaultMarshallerVersion); - } - - /// - /// Returns the canonicalized resource path for the service endpoint - /// - /// Endpoint URL for the request - /// Resource path for the request - /// If true pre URL encode path segments if necessary. - /// S3 is currently the only service that does not expect pre URL encoded segments. - /// - /// If resourcePath begins or ends with slash, the resulting canonicalized - /// path will follow suit. - /// - /// Canonicalized resource path for the endpoint - [Obsolete("Use CanonicalizeResourcePathV2 instead")] - public static string CanonicalizeResourcePath(Uri endpoint, string resourcePath, bool detectPreEncode) - { - // This overload is kept for backward compatibility in existing code bases. - return CanonicalizeResourcePath(endpoint, resourcePath, detectPreEncode, null, DefaultMarshallerVersion); - } - - /// - /// Returns the canonicalized resource path for the service endpoint - /// - /// Endpoint URL for the request - /// Resource path for the request - /// If true pre URL encode path segments if necessary. - /// S3 is currently the only service that does not expect pre URL encoded segments. - /// Dictionary of key/value parameters containing the values for the ResourcePath key replacements - /// The version of the marshaller that constructed the request object. - /// - /// If resourcePath begins or ends with slash, the resulting canonicalized - /// path will follow suit. - /// - /// Canonicalized resource path for the endpoint - [Obsolete("Use CanonicalizeResourcePathV2 instead")] - public static string CanonicalizeResourcePath(Uri endpoint, string resourcePath, bool detectPreEncode, IDictionary pathResources, int marshallerVersion) - { - if (endpoint != null) - { - var path = endpoint.AbsolutePath; - if (string.IsNullOrEmpty(path) || string.Equals(path, Slash, StringComparison.Ordinal)) - path = string.Empty; - - if (!string.IsNullOrEmpty(resourcePath) && resourcePath.StartsWith(Slash, StringComparison.Ordinal)) - resourcePath = resourcePath.Substring(1); - - if (!string.IsNullOrEmpty(resourcePath)) - path = path + Slash + resourcePath; - - resourcePath = path; - } - - if (string.IsNullOrEmpty(resourcePath)) - return Slash; - - IEnumerable encodedSegments = AWSSDKUtils.SplitResourcePathIntoSegments(resourcePath, pathResources); - - var pathWasPreEncoded = false; - if (detectPreEncode) - { - if (endpoint == null) - throw new ArgumentNullException(nameof(endpoint), "A non-null endpoint is necessary to decide whether or not to pre URL encode."); - - // S3 is a special case. For S3 skip the pre encode. - // For everything else URL pre encode the resource path segments. - if (!S3Uri.IsS3Uri(endpoint)) - { - encodedSegments = encodedSegments.Select(segment => UrlEncode(segment, true).Replace(Slash, EncodedSlash)); - - pathWasPreEncoded = true; - } - } -#pragma warning disable 0618 - var canonicalizedResourcePath = AWSSDKUtils.JoinResourcePathSegments(encodedSegments, false); -#pragma warning restore 0618 - // Get the logger each time (it's cached) because we shouldn't store it in a static variable. - Logger.GetLogger(typeof(AWSSDKUtils)).DebugFormat("{0} encoded {1}{2} for canonicalization: {3}", - pathWasPreEncoded ? "Double" : "Single", - resourcePath, - endpoint == null ? "" : " with endpoint " + endpoint.AbsoluteUri, - canonicalizedResourcePath); - - return canonicalizedResourcePath; - } - /// /// Returns the canonicalized resource path for the service endpoint. /// diff --git a/sdk/test/IntegrationTests/Tests/General.cs b/sdk/test/IntegrationTests/Tests/General.cs index 095aa68708b4..d5288e88f06b 100644 --- a/sdk/test/IntegrationTests/Tests/General.cs +++ b/sdk/test/IntegrationTests/Tests/General.cs @@ -564,36 +564,6 @@ public void TestLargeRetryCount() } } - [TestMethod] - [TestCategory("General")] - public void TestBidiCharsInUri() - { - var bidiChar = '\u200E'; - using(var client = TestBase.CreateClient()) - { - var part1 = "test"; - var part2 = "key"; - var bidiKey = part1 + bidiChar + part2; - - // verify character is in the string - Assert.IsTrue(bidiKey.IndexOf(bidiChar) > 0); - Assert.IsTrue(AWSSDKUtils.HasBidiControlCharacters(bidiKey)); - - // verify character is dropped by the Uri class - Uri uri = new Uri(new Uri("http://www.amazon.com/"), bidiKey); - Assert.IsTrue(uri.AbsoluteUri.IndexOf(bidiChar) < 0); - Assert.IsFalse(AWSSDKUtils.HasBidiControlCharacters(uri.AbsoluteUri)); - Assert.IsTrue(uri.AbsoluteUri.IndexOf(part1 + part2) > 0); - - // verify that trying to use key throws the appropriate exception - var e = AssertExtensions.ExpectException(() => client.GetObject("fake-bucket", bidiKey)); - Assert.IsNotNull(e); - Assert.IsTrue(e.Message.Contains("[" + bidiKey + "]")); - Assert.IsTrue(e.Message.Contains("cannot be handled by the .NET SDK")); - Assert.IsTrue(AWSSDKUtils.HasBidiControlCharacters(e.Message)); - } - } - [TestMethod] [TestCategory("General")] public void TestClientDispose() diff --git a/sdk/test/Services/S3/IntegrationTests/PutObjectTests.cs b/sdk/test/Services/S3/IntegrationTests/PutObjectTests.cs index 0f3afe142951..bc2244822587 100644 --- a/sdk/test/Services/S3/IntegrationTests/PutObjectTests.cs +++ b/sdk/test/Services/S3/IntegrationTests/PutObjectTests.cs @@ -100,6 +100,53 @@ public void TestPutAndGetWithInvalidExpires() #pragma warning restore CS0618 // Type or member is obsolete } + [TestMethod] + [TestCategory("S3")] + public void TestPutAndGetWithBidiCharacters() + { + var bidiChar = '\u200E'; + var encodedBidiChar = "%E2%80%8E"; + var content = "TestPutAndGetWithBidiCharacters"; + var bidiKey = UtilityMethods.GenerateName($"TestPutAndGetWithBidi{bidiChar}Characters"); + + // Verify character is in the string + Assert.IsTrue(bidiKey.IndexOf(bidiChar) > 0); + Assert.IsTrue(AWSSDKUtils.HasBidiControlCharacters(bidiKey)); + + // Verify character is encoded by the Uri class + Uri uri = new Uri(new Uri("http://www.amazon.com/"), bidiKey); + Assert.IsTrue(uri.AbsoluteUri.Contains(encodedBidiChar)); + Assert.IsFalse(AWSSDKUtils.HasBidiControlCharacters(uri.AbsoluteUri)); + Assert.IsTrue(uri.AbsoluteUri.Contains($"TestPutAndGetWithBidi{encodedBidiChar}Characters")); + + // Verify the bidi key can be used to put an object + var putObjectRequest = new PutObjectRequest + { + BucketName = bucketName, + Key = bidiKey, + ContentBody = content + }; + + Client.PutObject(putObjectRequest); + + // Verify the bidi key object can be read + var response = Client.GetObject(new GetObjectRequest + { + BucketName = bucketName, + Key = bidiKey, + }); + + // Read S3 bucket response content + var responseBody = string.Empty; + using (var reader = new StreamReader(response.ResponseStream)) + { + responseBody = reader.ReadToEnd(); + } + + // Verify the correct response was read + Assert.IsTrue(content == responseBody); + } + [TestMethod] [TestCategory("S3")] public void TestStorageClass()