Skip to content

Commit

Permalink
Updated support for bidi characters in resource paths.
Browse files Browse the repository at this point in the history
  • Loading branch information
@boblodgett
boblodgett committed Jul 1, 2024
1 parent 99ba6f1 commit 9769751
Show file tree
Hide file tree
Showing 7 changed files with 49 additions and 198 deletions.
3 changes: 0 additions & 3 deletions extensions/test/CrtIntegrationTests/V4aSignerTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,6 @@ internal static IRequest BuildHeaderRequestToSign(string resourcePath)
IRequest request = mock.Object;
request.HttpMethod = "POST";
request.ResourcePath = resourcePath;
request.MarshallerVersion = 2;
request.Content = Encoding.ASCII.GetBytes("Param1=value1");
request.Endpoint = new Uri("https://" + SigningTestHost + "/");

Expand Down Expand Up @@ -220,7 +219,6 @@ internal static IRequest BuildQueryParamRequestToSign()
var request = mock.Object;
request.HttpMethod = "GET";
request.ResourcePath = "/";
request.MarshallerVersion = 2;
request.Endpoint = new Uri("https://" + SigningTestHost + "/");

return request;
Expand Down Expand Up @@ -276,7 +274,6 @@ internal IRequest BuildMockChunkedRequest()
mock.SetupGet(x => x.SubResources).Returns(new Dictionary<string, string>());

var request = mock.Object;
request.MarshallerVersion = 2;
request.HttpMethod = "PUT";
request.ResourcePath = "/examplebucket/chunkObject.txt";
request.Endpoint = new Uri("https://s3.amazonaws.com/");
Expand Down
32 changes: 2 additions & 30 deletions sdk/src/Core/Amazon.Runtime/AmazonServiceClient.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -494,19 +494,7 @@ public static Uri ComposeUrl(IRequest internalRequest, bool skipEncodingValidPat
if (resourcePath.StartsWith("/", StringComparison.Ordinal))
resourcePath = resourcePath.Substring(1);

// Microsoft added support for unicode bidi control characters to the Uri class in .NET 4.7.2
// https://github.com/microsoft/dotnet/blob/master/Documentation/compatibility/uri-unicode-bidirectional-characters.md
// However, we only want to support it on .NET Core 3.1 and higher due to not having to deal with .NET Standard support matrix.
#if BCL || NETSTANDARD20
if (AWSSDKUtils.HasBidiControlCharacters(resourcePath) ||
(internalRequest.PathResources?.Any(v => AWSSDKUtils.HasBidiControlCharacters(v.Value)) == true))
{
resourcePath = string.Join("/", AWSSDKUtils.SplitResourcePathIntoSegments(resourcePath, internalRequest.PathResources).ToArray());
throw new AmazonClientException(string.Format(CultureInfo.InvariantCulture,
"Target resource path [{0}] has bidirectional characters, which are not supported" +
"by System.Uri and thus cannot be handled by the .NET SDK.", resourcePath));
}
#endif // Since S3 is the only service that is single encoded, we send the URL unencoded for special characters
// Since S3 is the only service that is single encoded, we send the URL unencoded for special characters
// to match the previous behavior compatible with the SigV2 backend.
if (internalRequest.SignatureVersion == SignatureVersion.SigV2 && String.Equals(internalRequest.ServiceName, "AmazonS3"))
{
Expand Down Expand Up @@ -547,23 +535,7 @@ public static Uri ComposeUrl(IRequest internalRequest, bool skipEncodingValidPat
sb.AppendFormat("{0}{1}", delim, queryString);
}

var parameterizedPath = string.Empty;
if(internalRequest.MarshallerVersion >= 2)
{
parameterizedPath = string.Concat(resourcePath, sb);
}
else
{
if (AWSSDKUtils.HasBidiControlCharacters(resourcePath))
throw new AmazonClientException(string.Format(CultureInfo.InvariantCulture,
"Target resource path [{0}] has bidirectional characters, which are not supported" +
"by System.Uri and thus cannot be handled by the .NET SDK.", resourcePath));

#pragma warning disable CS0612,CS0618 // Type or member is obsolete
parameterizedPath = string.Concat(AWSSDKUtils.ProtectEncodedSlashUrlEncode(resourcePath, skipEncodingValidPathChars), sb);
#pragma warning restore CS0612,CS0618 // Type or member is obsolete
}

var parameterizedPath = string.Concat(resourcePath, sb);
var hasSlash = url.AbsoluteUri.EndsWith("/", StringComparison.Ordinal) || parameterizedPath.StartsWith("/", StringComparison.Ordinal);

var strUri = hasSlash
Expand Down
21 changes: 0 additions & 21 deletions sdk/src/Core/Amazon.Runtime/Internal/DefaultRequest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,6 @@ public class DefaultRequest : IRequest
string canonicalResource;
RegionEndpoint alternateRegion;
long originalStreamLength;
int marshallerVersion = 2; //2 is the default version and must be used whenever a version is not specified in the marshaller.

/// <summary>
/// Constructs a new DefaultRequest with the specified service name and the
Expand Down Expand Up @@ -256,26 +255,6 @@ public void AddPathResource(string key, string value)
PathResources.Add(key, value);
}

/// <summary>
/// Gets and Sets the version number for the marshaller used to create this request. The version number
/// is used to support backward compatible changes that would otherwise be breaking changes when a
/// newer core is used with an older service assembly.
/// Versions:
/// 1 - Legacy version (no longer supported)
/// 2 - Default version. Support for path segments
/// </summary>
public int MarshallerVersion
{
get
{
return this.marshallerVersion;
}
set
{
this.marshallerVersion = value;
}
}

public string CanonicalResource
{
get
Expand Down
11 changes: 0 additions & 11 deletions sdk/src/Core/Amazon.Runtime/Internal/IRequest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,17 +144,6 @@ IDictionary<string, string> PathResources
/// <param name="value">Value of the entry</param>
void AddPathResource(string key, string value);

/// <summary>
/// Gets and Sets the version number for the marshaller used to create this request. The version number
/// is used to support backward compatible changes that would otherwise be breaking changes when a
/// newer core is used with an older service assembly.
/// </summary>
int MarshallerVersion
{
get;
set;
}

/// <summary>
/// Gets and Sets the content for this request.
/// </summary>
Expand Down
103 changes: 0 additions & 103 deletions sdk/src/Core/Amazon.Util/AWSSDKUtils.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,8 +88,6 @@ public static partial class AWSSDKUtils
internal const string S3Accelerate = "s3-accelerate";
internal const string S3Control = "s3-control";

private const int DefaultMarshallerVersion = 2;

private static readonly string _userAgent = InternalSDKUtils.BuildUserAgentString(string.Empty, string.Empty);

#endregion
Expand Down Expand Up @@ -324,107 +322,6 @@ internal static string GetParametersAsString(ParameterCollection parameterCollec
return result.Remove(result.Length - 1);
}

/// <summary>
/// Returns the canonicalized resource path for the service endpoint with single URL encoded path segments.
/// </summary>
/// <param name="endpoint">Endpoint URL for the request</param>
/// <param name="resourcePath">Resource path for the request</param>
/// <remarks>
/// If resourcePath begins or ends with slash, the resulting canonicalized
/// path will follow suit.
/// </remarks>
/// <returns>Canonicalized resource path for the endpoint</returns>
[Obsolete("Use CanonicalizeResourcePathV2 instead")]
public static string CanonicalizeResourcePath(Uri endpoint, string resourcePath)
{
// This overload is kept for backward compatibility in existing code bases.
return CanonicalizeResourcePath(endpoint, resourcePath, false, null, DefaultMarshallerVersion);
}

/// <summary>
/// Returns the canonicalized resource path for the service endpoint
/// </summary>
/// <param name="endpoint">Endpoint URL for the request</param>
/// <param name="resourcePath">Resource path for the request</param>
/// <param name="detectPreEncode">If true pre URL encode path segments if necessary.
/// S3 is currently the only service that does not expect pre URL encoded segments.</param>
/// <remarks>
/// If resourcePath begins or ends with slash, the resulting canonicalized
/// path will follow suit.
/// </remarks>
/// <returns>Canonicalized resource path for the endpoint</returns>
[Obsolete("Use CanonicalizeResourcePathV2 instead")]
public static string CanonicalizeResourcePath(Uri endpoint, string resourcePath, bool detectPreEncode)
{
// This overload is kept for backward compatibility in existing code bases.
return CanonicalizeResourcePath(endpoint, resourcePath, detectPreEncode, null, DefaultMarshallerVersion);
}

/// <summary>
/// Returns the canonicalized resource path for the service endpoint
/// </summary>
/// <param name="endpoint">Endpoint URL for the request</param>
/// <param name="resourcePath">Resource path for the request</param>
/// <param name="detectPreEncode">If true pre URL encode path segments if necessary.
/// S3 is currently the only service that does not expect pre URL encoded segments.</param>
/// <param name="pathResources">Dictionary of key/value parameters containing the values for the ResourcePath key replacements</param>
/// <param name="marshallerVersion">The version of the marshaller that constructed the request object.</param>
/// <remarks>
/// If resourcePath begins or ends with slash, the resulting canonicalized
/// path will follow suit.
/// </remarks>
/// <returns>Canonicalized resource path for the endpoint</returns>
[Obsolete("Use CanonicalizeResourcePathV2 instead")]
public static string CanonicalizeResourcePath(Uri endpoint, string resourcePath, bool detectPreEncode, IDictionary<string, string> pathResources, int marshallerVersion)
{
if (endpoint != null)
{
var path = endpoint.AbsolutePath;
if (string.IsNullOrEmpty(path) || string.Equals(path, Slash, StringComparison.Ordinal))
path = string.Empty;

if (!string.IsNullOrEmpty(resourcePath) && resourcePath.StartsWith(Slash, StringComparison.Ordinal))
resourcePath = resourcePath.Substring(1);

if (!string.IsNullOrEmpty(resourcePath))
path = path + Slash + resourcePath;

resourcePath = path;
}

if (string.IsNullOrEmpty(resourcePath))
return Slash;

IEnumerable<string> encodedSegments = AWSSDKUtils.SplitResourcePathIntoSegments(resourcePath, pathResources);

var pathWasPreEncoded = false;
if (detectPreEncode)
{
if (endpoint == null)
throw new ArgumentNullException(nameof(endpoint), "A non-null endpoint is necessary to decide whether or not to pre URL encode.");

// S3 is a special case. For S3 skip the pre encode.
// For everything else URL pre encode the resource path segments.
if (!S3Uri.IsS3Uri(endpoint))
{
encodedSegments = encodedSegments.Select(segment => UrlEncode(segment, true).Replace(Slash, EncodedSlash));

pathWasPreEncoded = true;
}
}
#pragma warning disable 0618
var canonicalizedResourcePath = AWSSDKUtils.JoinResourcePathSegments(encodedSegments, false);
#pragma warning restore 0618
// Get the logger each time (it's cached) because we shouldn't store it in a static variable.
Logger.GetLogger(typeof(AWSSDKUtils)).DebugFormat("{0} encoded {1}{2} for canonicalization: {3}",
pathWasPreEncoded ? "Double" : "Single",
resourcePath,
endpoint == null ? "" : " with endpoint " + endpoint.AbsoluteUri,
canonicalizedResourcePath);

return canonicalizedResourcePath;
}

/// <summary>
/// Returns the canonicalized resource path for the service endpoint.
/// </summary>
Expand Down
30 changes: 0 additions & 30 deletions sdk/test/IntegrationTests/Tests/General.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -564,36 +564,6 @@ public void TestLargeRetryCount()
}
}

[TestMethod]
[TestCategory("General")]
public void TestBidiCharsInUri()
{
var bidiChar = '\u200E';
using(var client = TestBase<AmazonS3Client>.CreateClient())
{
var part1 = "test";
var part2 = "key";
var bidiKey = part1 + bidiChar + part2;

// verify character is in the string
Assert.IsTrue(bidiKey.IndexOf(bidiChar) > 0);
Assert.IsTrue(AWSSDKUtils.HasBidiControlCharacters(bidiKey));

// verify character is dropped by the Uri class
Uri uri = new Uri(new Uri("http://www.amazon.com/"), bidiKey);
Assert.IsTrue(uri.AbsoluteUri.IndexOf(bidiChar) < 0);
Assert.IsFalse(AWSSDKUtils.HasBidiControlCharacters(uri.AbsoluteUri));
Assert.IsTrue(uri.AbsoluteUri.IndexOf(part1 + part2) > 0);

// verify that trying to use key throws the appropriate exception
var e = AssertExtensions.ExpectException<AmazonClientException>(() => client.GetObject("fake-bucket", bidiKey));
Assert.IsNotNull(e);
Assert.IsTrue(e.Message.Contains("[" + bidiKey + "]"));
Assert.IsTrue(e.Message.Contains("cannot be handled by the .NET SDK"));
Assert.IsTrue(AWSSDKUtils.HasBidiControlCharacters(e.Message));
}
}

[TestMethod]
[TestCategory("General")]
public void TestClientDispose()
Expand Down
47 changes: 47 additions & 0 deletions sdk/test/Services/S3/IntegrationTests/PutObjectTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,53 @@ public void TestPutAndGetWithInvalidExpires()
#pragma warning restore CS0618 // Type or member is obsolete
}

[TestMethod]
[TestCategory("S3")]
public void TestPutAndGetWithBidiCharacters()
{
var bidiChar = '\u200E';
var encodedBidiChar = "%E2%80%8E";
var content = "TestPutAndGetWithBidiCharacters";
var bidiKey = UtilityMethods.GenerateName($"TestPutAndGetWithBidi{bidiChar}Characters");

// Verify character is in the string
Assert.IsTrue(bidiKey.IndexOf(bidiChar) > 0);
Assert.IsTrue(AWSSDKUtils.HasBidiControlCharacters(bidiKey));

// Verify character is encoded by the Uri class
Uri uri = new Uri(new Uri("http://www.amazon.com/"), bidiKey);
Assert.IsTrue(uri.AbsoluteUri.Contains(encodedBidiChar));
Assert.IsFalse(AWSSDKUtils.HasBidiControlCharacters(uri.AbsoluteUri));
Assert.IsTrue(uri.AbsoluteUri.Contains($"TestPutAndGetWithBidi{encodedBidiChar}Characters"));

// Verify the bidi key can be used to put an object
var putObjectRequest = new PutObjectRequest
{
BucketName = bucketName,
Key = bidiKey,
ContentBody = content
};

Client.PutObject(putObjectRequest);

// Verify the bidi key object can be read
var response = Client.GetObject(new GetObjectRequest
{
BucketName = bucketName,
Key = bidiKey,
});

// Read S3 bucket response content
var responseBody = string.Empty;
using (var reader = new StreamReader(response.ResponseStream))
{
responseBody = reader.ReadToEnd();
}

// Verify the correct response was read
Assert.IsTrue(content == responseBody);
}

[TestMethod]
[TestCategory("S3")]
public void TestStorageClass()
Expand Down

0 comments on commit 9769751

Please sign in to comment.