diff --git a/.github/workflows/DownloadObject_build_and_test_on_main.yml b/.github/workflows/DownloadObject_build_and_test_on_main.yml index e1caf61..a572afc 100644 --- a/.github/workflows/DownloadObject_build_and_test_on_main.yml +++ b/.github/workflows/DownloadObject_build_and_test_on_main.yml @@ -13,6 +13,7 @@ jobs: uses: FrendsPlatform/FrendsTasks/.github/workflows/build_main.yml@main with: workdir: Frends.AmazonS3.DownloadObject + strict_analyzers: true env_var_name_1: HiQ_AWSS3Test_AccessKey env_var_name_2: HiQ_AWSS3Test_BucketName env_var_name_3: HiQ_AWSS3Test_SecretAccessKey @@ -20,4 +21,4 @@ jobs: badge_service_api_key: ${{ secrets.BADGE_SERVICE_API_KEY }} env_var_value_1: ${{ secrets.HIQ_AWSS3TEST_ACCESSKEY }} env_var_value_2: ${{ secrets.HIQ_AWSS3TEST_BUCKETNAME }} - env_var_value_3: ${{ secrets.HIQ_AWSS3TEST_SECRETACCESSKEY }} \ No newline at end of file + env_var_value_3: ${{ secrets.HIQ_AWSS3TEST_SECRETACCESSKEY }} diff --git a/.github/workflows/DownloadObject_build_and_test_on_push.yml b/.github/workflows/DownloadObject_build_and_test_on_push.yml index 1235603..ba83f9c 100644 --- a/.github/workflows/DownloadObject_build_and_test_on_push.yml +++ b/.github/workflows/DownloadObject_build_and_test_on_push.yml @@ -13,6 +13,7 @@ jobs: uses: FrendsPlatform/FrendsTasks/.github/workflows/build_test.yml@main with: workdir: Frends.AmazonS3.DownloadObject + strict_analyzers: true env_var_name_1: HiQ_AWSS3Test_AccessKey env_var_name_2: HiQ_AWSS3Test_BucketName env_var_name_3: HiQ_AWSS3Test_SecretAccessKey @@ -21,4 +22,4 @@ jobs: test_feed_api_key: ${{ secrets.TASKS_TEST_FEED_API_KEY }} env_var_value_1: ${{ secrets.HIQ_AWSS3TEST_ACCESSKEY }} env_var_value_2: ${{ secrets.HIQ_AWSS3TEST_BUCKETNAME }} - env_var_value_3: ${{ secrets.HIQ_AWSS3TEST_SECRETACCESSKEY }} \ No newline at end of file + env_var_value_3: ${{ secrets.HIQ_AWSS3TEST_SECRETACCESSKEY }} diff --git a/.github/workflows/DownloadObject_release.yml b/.github/workflows/DownloadObject_release.yml index 5d4067d..45ab3d4 100644 --- a/.github/workflows/DownloadObject_release.yml +++ b/.github/workflows/DownloadObject_release.yml @@ -8,6 +8,7 @@ jobs: uses: FrendsPlatform/FrendsTasks/.github/workflows/release.yml@main with: workdir: Frends.AmazonS3.DownloadObject + strict_analyzers: true env_var_name_1: HiQ_AWSS3Test_AccessKey env_var_name_2: HiQ_AWSS3Test_BucketName env_var_name_3: HiQ_AWSS3Test_SecretAccessKey @@ -15,4 +16,4 @@ jobs: feed_api_key: ${{ secrets.TASKS_FEED_API_KEY }} env_var_value_1: ${{ secrets.HIQ_AWSS3TEST_ACCESSKEY }} env_var_value_2: ${{ secrets.HIQ_AWSS3TEST_BUCKETNAME }} - env_var_value_3: ${{ secrets.HIQ_AWSS3TEST_SECRETACCESSKEY }} \ No newline at end of file + env_var_value_3: ${{ secrets.HIQ_AWSS3TEST_SECRETACCESSKEY }} diff --git a/Frends.AmazonS3.DownloadObject/CHANGELOG.md b/Frends.AmazonS3.DownloadObject/CHANGELOG.md index 055741f..29cb073 100644 --- a/Frends.AmazonS3.DownloadObject/CHANGELOG.md +++ b/Frends.AmazonS3.DownloadObject/CHANGELOG.md @@ -1,4 +1,57 @@ -# Changelog +# Changelog + +## [3.0.0] - 2025-07-17 + +## [Breaking] Major refactoring - restructured parameters into Input, Connection, and Options classes with improved error handling + +### Breaking Changes +- **Parameter Structure Reorganization**: Task parameters have been restructured into three main classes: + - **Input**: Contains core task parameters (BucketName, SourceDirectory, SearchPattern, DownloadFromCurrentDirectoryOnly, TargetDirectory) + - **Connection**: Contains connection-related parameters (AwsCredentials, PreSignedUrl) + - **Options**: Contains optional configuration parameters (FileLockedRetries, DeleteSourceObject, ThrowErrorIfNoMatch, ActionOnExistingFile, ThrowErrorOnFailure, ErrorMessageOnFailure) + +### Parameter Changes +- **Moved to Input tab**: + - `Connection.BucketName` → `Input.BucketName` + - `Connection.S3Directory` → `Input.SourceDirectory` (renamed) + - `Connection.SearchPattern` → `Input.SearchPattern` + - `Connection.DownloadFromCurrentDirectoryOnly` → `Input.DownloadFromCurrentDirectoryOnly` + - `Connection.DestinationDirectory` → `Input.TargetDirectory` (renamed) + +- **Moved to Options tab**: + - `Connection.FileLockedRetries` → `Options.FileLockedRetries` + - `Connection.DeleteSourceObject` → `Options.DeleteSourceObject` + - `Connection.ThrowErrorIfNoMatch` → `Options.ThrowErrorIfNoMatch` + - `Connection.DestinationFileExistsAction` → `Options.ActionOnExistingFile` (renamed) + +- **Updated in Connection tab**: + - `Connection.AWSCredentials` → `Connection.AwsCredentials` (renamed) + - `Connection.PreSignedURL` → `Connection.PreSignedUrl` (renamed) + +### New Features +- **Enhanced Error Handling**: Added comprehensive error handling with new Options properties: + - `ThrowErrorOnFailure` (bool, default: false) - Controls whether errors throw exceptions + - `ErrorMessageOnFailure` (string, default: "") - Custom error message for failures +- **Improved Result Structure**: + - Renamed `Data` property to `Objects` in Result class + - Added structured `Error` property with `Message` and `AdditionalInfo` fields +- **Error Handler Helper**: New static ErrorHandler class for consistent error processing + +### Migration Guide +To upgrade to the new version: +1. **Input tab**: Select the same values for BucketName, SourceDirectory (previously S3Directory), SearchPattern, DownloadFromCurrentDirectoryOnly, and TargetDirectory (previously DestinationDirectory) as they were in the Connection tab +2. **Options tab**: Configure FileLockedRetries, DeleteSourceObject, ThrowErrorIfNoMatch, and ActionOnExistingFile (previously DestinationFileExistsAction) with the same values as they were in the Connection tab +3. **Connection tab**: Update references to AwsCredentials (previously AWSCredentials) and PreSignedUrl (previously PreSignedURL) +4. **Result handling**: Update code that references `result.Data` to use `result.Objects` instead +5. **Error handling**: Consider setting `ThrowErrorOnFailure` to true and providing custom `ErrorMessageOnFailure` if needed + +### Technical Improvements +- Separated concerns by organizing parameters into logical groups (Input, Connection, Options) +- Implemented consistent error handling pattern following Frends task guidelines +- Added proper error reporting with structured Error objects +- Improved naming consistency (AWS → Aws, URL → Url) +- Enhanced maintainability and testability with better class structure + ## [2.2.0] - 2024-12-11 ### Updated diff --git a/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject.Tests/Frends.AmazonS3.DownloadObject.Tests.csproj b/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject.Tests/Frends.AmazonS3.DownloadObject.Tests.csproj index fb5850c..9bd9c2e 100644 --- a/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject.Tests/Frends.AmazonS3.DownloadObject.Tests.csproj +++ b/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject.Tests/Frends.AmazonS3.DownloadObject.Tests.csproj @@ -1,7 +1,7 @@  - net6.0 + net8.0 enable false diff --git a/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject.Tests/UnitTests.cs b/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject.Tests/UnitTests.cs index 3b7b84e..042e51d 100644 --- a/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject.Tests/UnitTests.cs +++ b/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject.Tests/UnitTests.cs @@ -8,7 +8,6 @@ using System.IO; using System.Linq; using System.Threading.Tasks; -using Xunit.Sdk; namespace Frends.AmazonS3.DownloadObject.Tests; @@ -21,26 +20,38 @@ public class UnitTests private readonly string _dir = Path.Combine(Environment.CurrentDirectory); // .\Frends.AmazonS3.DownloadObject\Frends.AmazonS3.DownloadObject.Test\bin\Debug\net6.0\ private Connection _connection = new(); + private Input _input = new(); + private Options _options = new(); [TestInitialize] public async Task Initialize() { _connection = new Connection() { - AuthenticationMethod = AuthenticationMethods.AWSCredentials, + AuthenticationMethod = AuthenticationMethods.AwsCredentials, AwsAccessKeyId = _accessKey, AwsSecretAccessKey = _secretAccessKey, - BucketName = _bucketName, Region = Region.EuCentral1, - S3Directory = "DownloadTest/", - SearchPattern = "*", + PreSignedUrl = null, + }; + + _options = new Options() + { DeleteSourceObject = true, - DownloadFromCurrentDirectoryOnly = true, ThrowErrorIfNoMatch = true, - DestinationDirectory = @$"{_dir}\Download", - DestinationFileExistsAction = DestinationFileExistsActions.Overwrite, + ActionOnExistingFile = DestinationFileExistsActions.Overwrite, FileLockedRetries = 0, - PreSignedURL = null, + ThrowErrorOnFailure = false, + ErrorMessageOnFailure = "" + }; + + _input = new Input() + { + BucketName = _bucketName, + SourceDirectory = "DownloadTest/", + SearchPattern = "*", + DownloadFromCurrentDirectoryOnly = true, + TargetDirectory = @$"{_dir}\Download", }; await CreateTestFiles(); @@ -58,15 +69,20 @@ public async Task PreSignedURL_DownloadFile_Test() { var setS3Key = $"DownloadTest/Testfile.txt"; - var connection = _connection; - connection.AuthenticationMethod = AuthenticationMethods.PreSignedURL; - connection.PreSignedURL = CreatePreSignedURL(setS3Key); + var connection = new Connection + { + AuthenticationMethod = AuthenticationMethods.PreSignedUrl, + PreSignedUrl = CreatePreSignedURL(setS3Key), + AwsAccessKeyId = _connection.AwsAccessKeyId, + AwsSecretAccessKey = _connection.AwsSecretAccessKey, + Region = _connection.Region + }; - var result = await AmazonS3.DownloadObject(connection, default); - Assert.IsNotNull(result.Data); + var result = await AmazonS3.DownloadObject(_input, connection, _options, default); + Assert.IsNotNull(result.Objects); Assert.IsTrue(result.Success); - Assert.AreEqual(1, result.Data.Count); - Assert.IsTrue(result.Data.Any(x => x.ObjectName != null)); + Assert.AreEqual(1, result.Objects.Count); + Assert.IsTrue(result.Objects.Any(x => x.ObjectName != null)); Assert.IsTrue(File.Exists(@$"{_dir}\Download\Testfile.txt")); } @@ -75,15 +91,20 @@ public async Task PreSignedURL_NoDestinationFilename_Test() { var setS3Key = $"DownloadTest/Testfile.txt"; - var connection = _connection; - connection.AuthenticationMethod = AuthenticationMethods.PreSignedURL; - connection.PreSignedURL = CreatePreSignedURL(setS3Key); + var connection = new Connection + { + AuthenticationMethod = AuthenticationMethods.PreSignedUrl, + PreSignedUrl = CreatePreSignedURL(setS3Key), + AwsAccessKeyId = _connection.AwsAccessKeyId, + AwsSecretAccessKey = _connection.AwsSecretAccessKey, + Region = _connection.Region + }; - var result = await AmazonS3.DownloadObject(connection, default); - Assert.IsNotNull(result.Data); + var result = await AmazonS3.DownloadObject(_input, connection, _options, default); + Assert.IsNotNull(result.Objects); Assert.IsTrue(result.Success); - Assert.AreEqual(1, result.Data.Count); - Assert.IsTrue(result.Data.Any(x => x.ObjectName != null)); + Assert.AreEqual(1, result.Objects.Count); + Assert.IsTrue(result.Objects.Any(x => x.ObjectName != null)); Assert.IsTrue(File.Exists(@$"{_dir}\Download\Testfile.txt")); } @@ -92,16 +113,21 @@ public async Task PreSignedURL_Overwrite_Exists_Test() { var setS3Key = $"DownloadTest/Overwrite.txt"; - var connection = _connection; - connection.AuthenticationMethod = AuthenticationMethods.PreSignedURL; - connection.PreSignedURL = CreatePreSignedURL(setS3Key); + var connection = new Connection + { + AuthenticationMethod = AuthenticationMethods.PreSignedUrl, + PreSignedUrl = CreatePreSignedURL(setS3Key), + AwsAccessKeyId = _connection.AwsAccessKeyId, + AwsSecretAccessKey = _connection.AwsSecretAccessKey, + Region = _connection.Region + }; - var result = await AmazonS3.DownloadObject(connection, default); - Assert.IsNotNull(result.Data); + var result = await AmazonS3.DownloadObject(_input, connection, _options, default); + Assert.IsNotNull(result.Objects); Assert.IsTrue(result.Success); - Assert.AreEqual(1, result.Data.Count); - Assert.IsTrue(result.Data.Any(x => x.ObjectName != null)); - Assert.IsTrue(result.Data.Any(x => x.Overwritten.Equals(true))); + Assert.AreEqual(1, result.Objects.Count); + Assert.IsTrue(result.Objects.Any(x => x.ObjectName != null)); + Assert.IsTrue(result.Objects.Any(x => x.Overwritten.Equals(true))); Assert.IsTrue(CompareFiles()); } @@ -110,17 +136,31 @@ public async Task PreSignedURL_Info_Exists_Test() { var setS3Key = $"DownloadTest/Overwrite.txt"; - var connection = _connection; - connection.AuthenticationMethod = AuthenticationMethods.PreSignedURL; - connection.PreSignedURL = CreatePreSignedURL(setS3Key); - connection.DestinationFileExistsAction = DestinationFileExistsActions.Info; + var connection = new Connection + { + AuthenticationMethod = AuthenticationMethods.PreSignedUrl, + PreSignedUrl = CreatePreSignedURL(setS3Key), + AwsAccessKeyId = _connection.AwsAccessKeyId, + AwsSecretAccessKey = _connection.AwsSecretAccessKey, + Region = _connection.Region + }; + + var options = new Options + { + ActionOnExistingFile = DestinationFileExistsActions.Info, + DeleteSourceObject = _options.DeleteSourceObject, + ThrowErrorIfNoMatch = _options.ThrowErrorIfNoMatch, + FileLockedRetries = _options.FileLockedRetries, + ThrowErrorOnFailure = _options.ThrowErrorOnFailure, + ErrorMessageOnFailure = _options.ErrorMessageOnFailure + }; - var result = await AmazonS3.DownloadObject(connection, default); - Assert.IsNotNull(result.Data); + var result = await AmazonS3.DownloadObject(_input, connection, options, default); + Assert.IsNotNull(result.Objects); Assert.IsTrue(result.Success); - Assert.AreEqual(1, result.Data.Count); - Assert.IsTrue(result.Data.Any(x => x.ObjectName != null)); - Assert.IsTrue(result.Data.Any(x => x.Info.Contains("Object skipped because file already exists in destination"))); + Assert.AreEqual(1, result.Objects.Count); + Assert.IsTrue(result.Objects.Any(x => x.ObjectName != null)); + Assert.IsTrue(result.Objects.Any(x => x.Info.Contains("Object skipped because file already exists in destination"))); Assert.IsTrue(File.Exists(@$"{_dir}\Download\Overwrite.txt")); Assert.IsFalse(CompareFiles()); } @@ -131,27 +171,48 @@ public async Task PreSignedURL_Error_Exists_Test() var setS3Key = $"DownloadTest/Testfile.txt"; File.WriteAllText(@$"{_dir}\Download\Testfile.txt", "I exist"); - var connection = _connection; - connection.AuthenticationMethod = AuthenticationMethods.PreSignedURL; - connection.PreSignedURL = CreatePreSignedURL(setS3Key); - connection.DestinationFileExistsAction = DestinationFileExistsActions.Error; + var connection = new Connection + { + AuthenticationMethod = AuthenticationMethods.PreSignedUrl, + PreSignedUrl = CreatePreSignedURL(setS3Key), + AwsAccessKeyId = _connection.AwsAccessKeyId, + AwsSecretAccessKey = _connection.AwsSecretAccessKey, + Region = _connection.Region + }; - var ex = await Assert.ThrowsExceptionAsync(async () => await AmazonS3.DownloadObject(connection, default)); - Assert.IsTrue(ex.Message.Contains("already exists")); + var options = new Options + { + ActionOnExistingFile = DestinationFileExistsActions.Error, + DeleteSourceObject = _options.DeleteSourceObject, + ThrowErrorIfNoMatch = _options.ThrowErrorIfNoMatch, + FileLockedRetries = _options.FileLockedRetries, + ThrowErrorOnFailure = _options.ThrowErrorOnFailure, + ErrorMessageOnFailure = _options.ErrorMessageOnFailure + }; + + var result = await AmazonS3.DownloadObject(_input, connection, options, default); + Assert.IsFalse(result.Success); + Assert.IsNotNull(result.Error); + Assert.IsTrue(result.Error.Message.Contains("already exists")); Assert.IsTrue(File.Exists(@$"{_dir}\Download\Testfile.txt")); } [TestMethod] public async Task PreSignedURL_MissingURL_Test() { - var setS3Key = $"DownloadTest/Testfile.txt"; - - var connection = _connection; - connection.AuthenticationMethod = AuthenticationMethods.PreSignedURL; - connection.PreSignedURL = ""; + var connection = new Connection + { + AuthenticationMethod = AuthenticationMethods.PreSignedUrl, + PreSignedUrl = "", + AwsAccessKeyId = _connection.AwsAccessKeyId, + AwsSecretAccessKey = _connection.AwsSecretAccessKey, + Region = _connection.Region + }; - var ex = await Assert.ThrowsExceptionAsync(async () => await AmazonS3.DownloadObject(connection, default)); - Assert.IsTrue(ex.Message.Contains("AWS pre-signed URL required.")); + var result = await AmazonS3.DownloadObject(_input, connection, _options, default); + Assert.IsFalse(result.Success); + Assert.IsNotNull(result.Error); + Assert.IsTrue(result.Error.Message.Contains("AWS pre-signed URL required.")); } [TestMethod] @@ -159,13 +220,28 @@ public async Task PreSignedURL_MissingDestinationDirectory_Test() { var setS3Key = $"DownloadTest/Testfile.txt"; - var connection = _connection; - connection.AuthenticationMethod = AuthenticationMethods.PreSignedURL; - connection.PreSignedURL = CreatePreSignedURL(setS3Key); - connection.DestinationDirectory = ""; + var connection = new Connection + { + AuthenticationMethod = AuthenticationMethods.PreSignedUrl, + PreSignedUrl = CreatePreSignedURL(setS3Key), + AwsAccessKeyId = _connection.AwsAccessKeyId, + AwsSecretAccessKey = _connection.AwsSecretAccessKey, + Region = _connection.Region + }; + + var input = new Input + { + BucketName = _input.BucketName, + SourceDirectory = _input.SourceDirectory, + SearchPattern = _input.SearchPattern, + DownloadFromCurrentDirectoryOnly = _input.DownloadFromCurrentDirectoryOnly, + TargetDirectory = "" + }; - var ex = await Assert.ThrowsExceptionAsync(async () => await AmazonS3.DownloadObject(connection, default)); - Assert.IsTrue(ex.Message.Contains("Path cannot be the empty")); + var result = await AmazonS3.DownloadObject(input, connection, _options, default); + Assert.IsFalse(result.Success); + Assert.IsNotNull(result.Error); + Assert.IsTrue(result.Error.Message.Contains("Path cannot be the empty")); } [TestMethod] @@ -177,16 +253,39 @@ public async Task AWSCreds_DownloadFiles_TestAllDestinationFileExistsActions_Tes { Directory.Delete($@"{_dir}\Download", true); - var connection = _connection; - connection.DestinationFileExistsAction = action; - connection.DeleteSourceObject = false; - connection.DownloadFromCurrentDirectoryOnly = false; + var connection = new Connection + { + AuthenticationMethod = _connection.AuthenticationMethod, + AwsAccessKeyId = _connection.AwsAccessKeyId, + AwsSecretAccessKey = _connection.AwsSecretAccessKey, + Region = _connection.Region, + PreSignedUrl = _connection.PreSignedUrl + }; + + var options = new Options + { + ActionOnExistingFile = action, + DeleteSourceObject = false, + ThrowErrorIfNoMatch = _options.ThrowErrorIfNoMatch, + FileLockedRetries = _options.FileLockedRetries, + ThrowErrorOnFailure = _options.ThrowErrorOnFailure, + ErrorMessageOnFailure = _options.ErrorMessageOnFailure + }; + + var input = new Input + { + BucketName = _input.BucketName, + SourceDirectory = _input.SourceDirectory, + SearchPattern = _input.SearchPattern, + DownloadFromCurrentDirectoryOnly = false, + TargetDirectory = _input.TargetDirectory + }; - var result = await AmazonS3.DownloadObject(connection, default); - Assert.IsNotNull(result.Data, $"method: {action}"); + var result = await AmazonS3.DownloadObject(input, connection, options, default); + Assert.IsNotNull(result.Objects, $"method: {action}"); Assert.IsTrue(result.Success, $"method: {action}"); - Assert.AreEqual(4, result.Data.Count, $"method: {action}"); - Assert.IsTrue(result.Data.Any(x => x.ObjectName != null), $"method: {action}"); + Assert.AreEqual(4, result.Objects.Count, $"method: {action}"); + Assert.IsTrue(result.Objects.Any(x => x.ObjectName != null), $"method: {action}"); Assert.IsTrue(File.Exists(@$"{_dir}\Download\DownloadFromCurrentDirectoryOnly.txt"), $"method: {action}"); } } @@ -194,15 +293,39 @@ public async Task AWSCreds_DownloadFiles_TestAllDestinationFileExistsActions_Tes [TestMethod] public async Task AWSCreds_DownloadFiles_Info_Exists_Test() { - var connection = _connection; - connection.DestinationFileExistsAction = DestinationFileExistsActions.Info; - connection.DownloadFromCurrentDirectoryOnly = false; + var connection = new Connection + { + AuthenticationMethod = _connection.AuthenticationMethod, + AwsAccessKeyId = _connection.AwsAccessKeyId, + AwsSecretAccessKey = _connection.AwsSecretAccessKey, + Region = _connection.Region, + PreSignedUrl = _connection.PreSignedUrl + }; + + var options = new Options + { + ActionOnExistingFile = DestinationFileExistsActions.Info, + DeleteSourceObject = _options.DeleteSourceObject, + ThrowErrorIfNoMatch = _options.ThrowErrorIfNoMatch, + FileLockedRetries = _options.FileLockedRetries, + ThrowErrorOnFailure = _options.ThrowErrorOnFailure, + ErrorMessageOnFailure = _options.ErrorMessageOnFailure + }; + + var input = new Input + { + BucketName = _input.BucketName, + SourceDirectory = _input.SourceDirectory, + SearchPattern = _input.SearchPattern, + DownloadFromCurrentDirectoryOnly = false, + TargetDirectory = _input.TargetDirectory + }; - var result = await AmazonS3.DownloadObject(connection, default); - Assert.IsNotNull(result.Data); + var result = await AmazonS3.DownloadObject(input, connection, options, default); + Assert.IsNotNull(result.Objects); Assert.IsTrue(result.Success); - Assert.AreEqual(4, result.Data.Count); - Assert.IsTrue(result.Data.Any(x => x.ObjectName != null)); + Assert.AreEqual(4, result.Objects.Count); + Assert.IsTrue(result.Objects.Any(x => x.ObjectName != null)); Assert.IsTrue(File.Exists(@$"{_dir}\Download\Overwrite.txt")); Assert.IsTrue(File.Exists(@$"{_dir}\Download\Testfile.txt")); Assert.IsFalse(CompareFiles()); @@ -212,11 +335,29 @@ public async Task AWSCreds_DownloadFiles_Info_Exists_Test() [TestMethod] public async Task AWSCreds_DownloadFiles_Error_Exists_Test() { - var connection = _connection; - connection.DestinationFileExistsAction = DestinationFileExistsActions.Error; + var connection = new Connection + { + AuthenticationMethod = _connection.AuthenticationMethod, + AwsAccessKeyId = _connection.AwsAccessKeyId, + AwsSecretAccessKey = _connection.AwsSecretAccessKey, + Region = _connection.Region, + PreSignedUrl = _connection.PreSignedUrl + }; - var ex = await Assert.ThrowsExceptionAsync(async () => await AmazonS3.DownloadObject(connection, default)); - Assert.IsTrue(ex.Message.Contains("already exists")); + var options = new Options + { + ActionOnExistingFile = DestinationFileExistsActions.Error, + DeleteSourceObject = _options.DeleteSourceObject, + ThrowErrorIfNoMatch = _options.ThrowErrorIfNoMatch, + FileLockedRetries = _options.FileLockedRetries, + ThrowErrorOnFailure = _options.ThrowErrorOnFailure, + ErrorMessageOnFailure = _options.ErrorMessageOnFailure + }; + + var result = await AmazonS3.DownloadObject(_input, connection, options, default); + Assert.IsFalse(result.Success); + Assert.IsNotNull(result.Error); + Assert.IsTrue(result.Error.Message.Contains("already exists")); Assert.IsTrue(File.Exists(@$"{_dir}\Download\Overwrite.txt")); Assert.IsFalse(CompareFiles()); } @@ -225,14 +366,30 @@ public async Task AWSCreds_DownloadFiles_Error_Exists_Test() public async Task AWSCreds_DeleteSource_Test() { Directory.Delete($@"{_dir}\Download", true); - var connection = _connection; - connection.DeleteSourceObject = true; + var connection = new Connection + { + AuthenticationMethod = _connection.AuthenticationMethod, + AwsAccessKeyId = _connection.AwsAccessKeyId, + AwsSecretAccessKey = _connection.AwsSecretAccessKey, + Region = _connection.Region, + PreSignedUrl = _connection.PreSignedUrl + }; + + var options = new Options + { + DeleteSourceObject = true, + ActionOnExistingFile = _options.ActionOnExistingFile, + ThrowErrorIfNoMatch = _options.ThrowErrorIfNoMatch, + FileLockedRetries = _options.FileLockedRetries, + ThrowErrorOnFailure = _options.ThrowErrorOnFailure, + ErrorMessageOnFailure = _options.ErrorMessageOnFailure + }; - var result = await AmazonS3.DownloadObject(connection, default); - Assert.IsNotNull(result.Data); + var result = await AmazonS3.DownloadObject(_input, connection, options, default); + Assert.IsNotNull(result.Objects); Assert.IsTrue(result.Success); - Assert.AreEqual(3, result.Data.Count); - Assert.IsTrue(result.Data.Any(x => x.ObjectName != null)); + Assert.AreEqual(3, result.Objects.Count); + Assert.IsTrue(result.Objects.Any(x => x.ObjectName != null)); Assert.IsTrue(File.Exists(@$"{_dir}\Download\Overwrite.txt")); Assert.IsTrue(File.Exists(@$"{_dir}\Download\Testfile.txt")); Assert.IsFalse(await FileExistsInS3("DownloadTest/testikansio/")); @@ -242,14 +399,29 @@ public async Task AWSCreds_DeleteSource_Test() public async Task AWSCreds_Pattern_Test() { Directory.Delete($@"{_dir}\Download", true); - var connection = _connection; - connection.SearchPattern = "Testfi*"; + var connection = new Connection + { + AuthenticationMethod = _connection.AuthenticationMethod, + AwsAccessKeyId = _connection.AwsAccessKeyId, + AwsSecretAccessKey = _connection.AwsSecretAccessKey, + Region = _connection.Region, + PreSignedUrl = _connection.PreSignedUrl + }; - var result = await AmazonS3.DownloadObject(connection, default); - Assert.IsNotNull(result.Data); + var input = new Input + { + BucketName = _input.BucketName, + SourceDirectory = _input.SourceDirectory, + SearchPattern = "Testfi*", + DownloadFromCurrentDirectoryOnly = _input.DownloadFromCurrentDirectoryOnly, + TargetDirectory = _input.TargetDirectory + }; + + var result = await AmazonS3.DownloadObject(input, connection, _options, default); + Assert.IsNotNull(result.Objects); Assert.IsTrue(result.Success); - Assert.AreEqual(1, result.Data.Count); - Assert.IsTrue(result.Data.Any(x => x.ObjectName != null)); + Assert.AreEqual(1, result.Objects.Count); + Assert.IsTrue(result.Objects.Any(x => x.ObjectName != null)); Assert.IsTrue(File.Exists(@$"{_dir}\Download\Testfile.txt")); Assert.IsFalse(File.Exists(@$"{_dir}\Download\Overwrite.txt")); Assert.IsFalse(File.Exists(@$"{_dir}\Download\DownloadFromCurrentDirectoryOnly.txt")); @@ -259,14 +431,29 @@ public async Task AWSCreds_Pattern_Test() public async Task AWSCreds_DownloadFromCurrentDirectoryOnly_Test() { Directory.Delete($@"{_dir}\Download", true); - var connection = _connection; - connection.DownloadFromCurrentDirectoryOnly = true; + var connection = new Connection + { + AuthenticationMethod = _connection.AuthenticationMethod, + AwsAccessKeyId = _connection.AwsAccessKeyId, + AwsSecretAccessKey = _connection.AwsSecretAccessKey, + Region = _connection.Region, + PreSignedUrl = _connection.PreSignedUrl + }; + + var input = new Input + { + BucketName = _input.BucketName, + SourceDirectory = _input.SourceDirectory, + SearchPattern = _input.SearchPattern, + DownloadFromCurrentDirectoryOnly = true, + TargetDirectory = _input.TargetDirectory + }; - var result = await AmazonS3.DownloadObject(connection, default); - Assert.IsNotNull(result.Data); + var result = await AmazonS3.DownloadObject(input, connection, _options, default); + Assert.IsNotNull(result.Objects); Assert.IsTrue(result.Success); - Assert.AreEqual(3, result.Data.Count); - Assert.IsTrue(result.Data.Any(x => x.ObjectName != null)); + Assert.AreEqual(3, result.Objects.Count); + Assert.IsTrue(result.Objects.Any(x => x.ObjectName != null)); Assert.IsTrue(File.Exists(@$"{_dir}\Download\Overwrite.txt")); Assert.IsTrue(File.Exists(@$"{_dir}\Download\Testfile.txt")); Assert.IsFalse(File.Exists(@$"{_dir}\Download\DownloadFromCurrentDirectoryOnly.txt")); @@ -275,11 +462,127 @@ public async Task AWSCreds_DownloadFromCurrentDirectoryOnly_Test() [TestMethod] public async Task AWSCreds_ThrowErrorIfNoMatch_Test() { - var connection = _connection; - connection.SearchPattern = "nofile"; + var connection = new Connection + { + AuthenticationMethod = _connection.AuthenticationMethod, + AwsAccessKeyId = _connection.AwsAccessKeyId, + AwsSecretAccessKey = _connection.AwsSecretAccessKey, + Region = _connection.Region, + PreSignedUrl = _connection.PreSignedUrl + }; + + var input = new Input + { + BucketName = _input.BucketName, + SourceDirectory = _input.SourceDirectory, + SearchPattern = "nofile", + DownloadFromCurrentDirectoryOnly = _input.DownloadFromCurrentDirectoryOnly, + TargetDirectory = _input.TargetDirectory + }; + + var result = await AmazonS3.DownloadObject(input, connection, _options, default); + Assert.IsFalse(result.Success); + Assert.IsNotNull(result.Error); + Assert.IsTrue(result.Error.Message.Contains("No matches found with search pattern")); + } + + [TestMethod] + public async Task ErrorHandler_ThrowErrorOnFailure_True_WithCustomMessage_Test() + { + var connection = new Connection + { + AuthenticationMethod = _connection.AuthenticationMethod, + AwsAccessKeyId = "invalid", + AwsSecretAccessKey = _connection.AwsSecretAccessKey, + Region = _connection.Region, + PreSignedUrl = _connection.PreSignedUrl + }; + + var options = new Options + { + ThrowErrorOnFailure = true, + ErrorMessageOnFailure = "Custom error message for testing", + ActionOnExistingFile = _options.ActionOnExistingFile, + DeleteSourceObject = _options.DeleteSourceObject, + ThrowErrorIfNoMatch = _options.ThrowErrorIfNoMatch, + FileLockedRetries = _options.FileLockedRetries + }; + + try + { + await AmazonS3.DownloadObject(_input, connection, options, default); + Assert.Fail("Expected exception was not thrown"); + } + catch (Exception ex) + { + Assert.AreEqual("Custom error message for testing", ex.Message); + } + } + + [TestMethod] + public async Task ErrorHandler_ThrowErrorOnFailure_True_WithOriginalMessage_Test() + { + var connection = new Connection + { + AuthenticationMethod = _connection.AuthenticationMethod, + AwsAccessKeyId = "invalid", + AwsSecretAccessKey = _connection.AwsSecretAccessKey, + Region = _connection.Region, + PreSignedUrl = _connection.PreSignedUrl + }; + + var options = new Options + { + ThrowErrorOnFailure = true, + ErrorMessageOnFailure = "", + ActionOnExistingFile = _options.ActionOnExistingFile, + DeleteSourceObject = _options.DeleteSourceObject, + ThrowErrorIfNoMatch = _options.ThrowErrorIfNoMatch, + FileLockedRetries = _options.FileLockedRetries + }; + + try + { + await AmazonS3.DownloadObject(_input, connection, options, default); + Assert.Fail("Expected exception was not thrown"); + } + catch (Exception ex) + { + Assert.IsTrue(ex.Message.Contains("The AWS Access Key Id you provided does not exist") || + ex.Message.Contains("InvalidAccessKeyId") || + ex.Message.Contains("credentials")); + } + } + + [TestMethod] + public async Task ErrorHandler_ThrowErrorOnFailure_False_Test() + { + var connection = new Connection + { + AuthenticationMethod = _connection.AuthenticationMethod, + AwsAccessKeyId = "invalid", + AwsSecretAccessKey = _connection.AwsSecretAccessKey, + Region = _connection.Region, + PreSignedUrl = _connection.PreSignedUrl + }; + + var options = new Options + { + ThrowErrorOnFailure = false, + ErrorMessageOnFailure = _options.ErrorMessageOnFailure, + ActionOnExistingFile = _options.ActionOnExistingFile, + DeleteSourceObject = _options.DeleteSourceObject, + ThrowErrorIfNoMatch = _options.ThrowErrorIfNoMatch, + FileLockedRetries = _options.FileLockedRetries + }; - var ex = await Assert.ThrowsExceptionAsync(async () => await AmazonS3.DownloadObject(connection, default)); - Assert.IsTrue(ex.Message.Contains("No matches found with search pattern")); + var result = await AmazonS3.DownloadObject(_input, connection, options, default); + Assert.IsFalse(result.Success); + Assert.IsNotNull(result.Error); + Assert.IsTrue(result.Error.Message.Contains("The AWS Access Key Id you provided does not exist") || + result.Error.Message.Contains("InvalidAccessKeyId") || + result.Error.Message.Contains("credentials")); + Assert.IsNotNull(result.Error.AdditionalInfo); } private string CreatePreSignedURL(string key) @@ -354,4 +657,4 @@ private async Task FileExistsInS3(string key) client.Dispose(); return (response != null && response.S3Objects != null && response.S3Objects.Count > 0); } -} \ No newline at end of file +} diff --git a/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject/Definitions/Enums.cs b/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject/Definitions/Enums.cs index 8cacf1c..a8a1dfe 100644 --- a/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject/Definitions/Enums.cs +++ b/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject/Definitions/Enums.cs @@ -40,12 +40,12 @@ public enum AuthenticationMethods /// /// AwsAccessKeyId+AwsSecretAccessKey. /// - AWSCredentials, + AwsCredentials, /// /// Pre-signed URL. /// - PreSignedURL + PreSignedUrl } /// @@ -67,4 +67,4 @@ public enum DestinationFileExistsActions /// Stop the process. /// Error -} \ No newline at end of file +} diff --git a/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject/Definitions/Error.cs b/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject/Definitions/Error.cs new file mode 100644 index 0000000..cd5a01b --- /dev/null +++ b/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject/Definitions/Error.cs @@ -0,0 +1,39 @@ +namespace Frends.AmazonS3.DownloadObject.Definitions; + +/// +/// Error information. +/// +public class Error +{ + /// + /// Error message. + /// + /// An error occurred during processing + public string Message { get; set; } + + /// + /// Additional error information. + /// + /// { "ErrorCode": 500, "Details": "Connection timeout" } + public dynamic AdditionalInfo { get; set; } + + /// + /// Initializes a new instance of the Error class. + /// + /// Error message + /// Additional error information + public Error(string message, dynamic additionalInfo = null) + { + Message = message; + AdditionalInfo = additionalInfo; + } + + /// + /// Parameterless constructor for Error class. + /// + public Error() + { + Message = string.Empty; + AdditionalInfo = null; + } +} diff --git a/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject/Definitions/Input.cs b/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject/Definitions/Input.cs index 359ba77..f0efe0c 100644 --- a/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject/Definitions/Input.cs +++ b/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject/Definitions/Input.cs @@ -1,74 +1,30 @@ -using System.ComponentModel; +using System.ComponentModel; using System.ComponentModel.DataAnnotations; namespace Frends.AmazonS3.DownloadObject.Definitions; /// -/// Connection parameters. +/// Input parameters. /// -public class Connection +public class Input { - /// - /// Authentication method to use when connecting to AWS S3 bucket. - /// - /// AWSCredentials - [DefaultValue(AuthenticationMethods.AWSCredentials)] - public AuthenticationMethods AuthenticationMethod { get; set; } - - /// - /// A pre-signed URL allows you to grant temporary access to users who don't have permission to directly run AWS operations in your account. - /// - /// "https://bucket.s3.region.amazonaws.com/object/file.txt?X... - [PasswordPropertyText] - [UIHint(nameof(AuthenticationMethod), "", AuthenticationMethods.PreSignedURL)] - [DefaultValue(null)] - public string PreSignedURL { get; set; } - - /// - /// AWS Access Key ID. - /// - /// AKIAQWERTY7NJ5Q7NZ6Q - [PasswordPropertyText] - [UIHint(nameof(AuthenticationMethod), "", AuthenticationMethods.AWSCredentials)] - [DefaultValue(null)] - public string AwsAccessKeyId { get; set; } - - /// - /// AWS Secret Access Key. - /// - /// TVh5hgd3uGY/2CqH - [PasswordPropertyText] - [UIHint(nameof(AuthenticationMethod), "", AuthenticationMethods.AWSCredentials)] - [DefaultValue(null)] - public string AwsSecretAccessKey { get; set; } - /// /// AWS S3 bucket's name. /// /// Bucket - [UIHint(nameof(AuthenticationMethod), "", AuthenticationMethods.AWSCredentials)] public string BucketName { get; set; } - /// - /// AWS S3 bucket's region. - /// - /// EuCentral1 - [UIHint(nameof(AuthenticationMethod), "", AuthenticationMethods.AWSCredentials)] - public Region Region { get; set; } - /// /// Downloads all objects with this prefix. /// /// directory/ - [UIHint(nameof(AuthenticationMethod), "", AuthenticationMethods.AWSCredentials)] [DisplayFormat(DataFormatString = "Text")] - public string S3Directory { get; set; } + public string SourceDirectory { get; set; } /// /// String pattern to search objects. /// /// *.*, *file?.txt - [UIHint(nameof(AuthenticationMethod), "", AuthenticationMethods.AWSCredentials)] [DisplayFormat(DataFormatString = "Text")] [DefaultValue("*")] public string SearchPattern { get; set; } @@ -77,47 +33,63 @@ public class Connection /// Set to true to download objects from current directory only. /// /// false - [UIHint(nameof(AuthenticationMethod), "", AuthenticationMethods.AWSCredentials)] [DefaultValue(true)] public bool DownloadFromCurrentDirectoryOnly { get; set; } /// - /// Delete S3 source object after download. - /// Subfolders will also be deleted if they are part of the object's key and there are no objects left. - /// Create subfolders manually to make sure they won't be deleted. + /// Destination directory where to create folders and files. /// - /// false - [UIHint(nameof(AuthenticationMethod), "", AuthenticationMethods.AWSCredentials)] - [DefaultValue(false)] - public bool DeleteSourceObject { get; set; } + /// c:\temp, \\network\folder + [DisplayFormat(DataFormatString = "Text")] + public string TargetDirectory { get; set; } +} +/// +/// Connection parameters. +/// +public class Connection +{ /// - /// Throw an error if there are no objects in the path matching the search pattern. + /// Authentication method to use when connecting to AWS S3 bucket. /// - /// false - [UIHint(nameof(AuthenticationMethod), "", AuthenticationMethods.AWSCredentials)] - [DefaultValue(true)] - public bool ThrowErrorIfNoMatch { get; set; } + /// AwsCredentials + [DefaultValue(AuthenticationMethods.AwsCredentials)] + public AuthenticationMethods AuthenticationMethod { get; set; } /// - /// Actions if destination file already exists. + /// A pre-signed URL allows you to grant temporary access to users who don't have permission to directly run AWS operations in your account. /// - /// Info - [DefaultValue(DestinationFileExistsActions.Info)] - public DestinationFileExistsActions DestinationFileExistsAction { get; set; } + /// "https://bucket.s3.region.amazonaws.com/object/file.txt?X... + [PasswordPropertyText] + [UIHint(nameof(AuthenticationMethod), "", AuthenticationMethods.PreSignedUrl)] + [DefaultValue(null)] + public string PreSignedUrl { get; set; } /// - /// Destination directory where to create folders and files. + /// AWS Access Key ID. /// - /// c:\temp, \\network\folder - [DisplayFormat(DataFormatString = "Text")] - public string DestinationDirectory { get; set; } + /// AKIAQWERTY7NJ5Q7NZ6Q + [PasswordPropertyText] + [UIHint(nameof(AuthenticationMethod), "", AuthenticationMethods.AwsCredentials)] + [DefaultValue(null)] + public string AwsAccessKeyId { get; set; } + + /// + /// AWS Secret Access Key. + /// + /// TVh5hgd3uGY/2CqH + [PasswordPropertyText] + [UIHint(nameof(AuthenticationMethod), "", AuthenticationMethods.AwsCredentials)] + [DefaultValue(null)] + public string AwsSecretAccessKey { get; set; } + /// - /// For how long will this Task try to write to a locked file. - /// Value in seconds. + /// AWS S3 bucket's region. /// - /// 10 - [DefaultValue(10)] - public int FileLockedRetries { get; set; } -} \ No newline at end of file + /// EuCentral1 + [UIHint(nameof(AuthenticationMethod), "", AuthenticationMethods.AwsCredentials)] + public Region Region { get; set; } + + +} diff --git a/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject/Definitions/Options.cs b/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject/Definitions/Options.cs new file mode 100644 index 0000000..4ee6132 --- /dev/null +++ b/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject/Definitions/Options.cs @@ -0,0 +1,54 @@ +using System.ComponentModel; + +namespace Frends.AmazonS3.DownloadObject.Definitions; + +/// +/// Options parameters. +/// +public class Options +{ + /// + /// Delete S3 source object after download. + /// Subfolders will also be deleted if they are part of the object's key and there are no objects left. + /// Create subfolders manually to make sure they won't be deleted. + /// + /// false + [DefaultValue(false)] + public bool DeleteSourceObject { get; set; } + + /// + /// Throw an error if there are no objects in the path matching the search pattern. + /// + /// false + [DefaultValue(true)] + public bool ThrowErrorIfNoMatch { get; set; } + + /// + /// Actions if destination file already exists. + /// + /// Info + [DefaultValue(DestinationFileExistsActions.Info)] + public DestinationFileExistsActions ActionOnExistingFile { get; set; } + + /// + /// For how long will this Task try to write to a locked file. + /// Value in seconds. + /// + /// 10 + [DefaultValue(10)] + public int FileLockedRetries { get; set; } + + /// + /// Throw an error on failure. + /// + /// false + [DefaultValue(false)] + public bool ThrowErrorOnFailure { get; set; } + + /// + /// Error message to display on failure. + /// + /// + [DefaultValue("")] + public string ErrorMessageOnFailure { get; set; } +} diff --git a/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject/Definitions/Result.cs b/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject/Definitions/Result.cs index 689dd43..05a58b4 100644 --- a/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject/Definitions/Result.cs +++ b/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject/Definitions/Result.cs @@ -17,11 +17,18 @@ public class Result /// List of downloaded objects. /// /// { "File.txt", "C:\temp\File.txt", true, false, "Additional information" } - public List Data { get; private set; } + public List Objects { get; private set; } - internal Result(bool success, List data) + /// + /// Error information if operation failed. + /// + /// { "An error occurred", { "ErrorCode": 500 } } + public Error Error { get; private set; } + + internal Result(bool success, List objects, Error error = null) { Success = success; - Data = data; + Objects = objects; + Error = error; } -} \ No newline at end of file +} diff --git a/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject/DownloadObject.cs b/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject/DownloadObject.cs index 648f2a2..c387b00 100644 --- a/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject/DownloadObject.cs +++ b/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject/DownloadObject.cs @@ -2,6 +2,7 @@ using Amazon.S3; using Amazon.S3.Model; using Frends.AmazonS3.DownloadObject.Definitions; +using Frends.AmazonS3.DownloadObject.Helpers; using System; using System.Collections.Generic; using System.ComponentModel; @@ -26,20 +27,22 @@ public class AmazonS3 /// Download objects from AWS S3. /// [Documentation](https://tasks.frends.com/tasks/frends-tasks/Frends.AmazonS3.DownloadObject) /// - /// Connection parameters + /// Input parameters + /// Connection parameters + /// Options parameters /// Token generated by Frends to stop this task. - /// Object { bool Success, List { string ObjectName, string FullPath, string Overwritten, bool SourceDeleted, string Info } } - public static async Task DownloadObject([PropertyTab] Connection input, CancellationToken cancellationToken) + /// Object { bool Success, List Objects { string ObjectName, string FullPath, string Overwritten, bool SourceDeleted, string Info }, Error Error { string Message, dynamic AdditionalInfo } } + public static async Task DownloadObject([PropertyTab] Input input, [PropertyTab] Connection connection, [PropertyTab] Options options, CancellationToken cancellationToken) { var result = new List(); try { - if (input.AuthenticationMethod is AuthenticationMethods.AWSCredentials) + if (connection.AuthenticationMethod is AuthenticationMethods.AwsCredentials) { var mask = new Regex(input.SearchPattern.Replace(".", "[.]").Replace("*", ".*").Replace("?", ".")); - var targetPath = input.S3Directory + input.SearchPattern; - using (AmazonS3Client client = new(input.AwsAccessKeyId, input.AwsSecretAccessKey, RegionSelection(input.Region))) + var targetPath = input.SourceDirectory + input.SearchPattern; + using (AmazonS3Client client = new(connection.AwsAccessKeyId, connection.AwsSecretAccessKey, RegionSelection(connection.Region))) { var clientRequest = new ListObjectsV2Request { @@ -48,7 +51,7 @@ public static async Task DownloadObject([PropertyTab] Connection input, Encoding = null, FetchOwner = false, MaxKeys = 1000, - Prefix = string.IsNullOrWhiteSpace(input.S3Directory) ? null : input.S3Directory, + Prefix = string.IsNullOrWhiteSpace(input.SourceDirectory) ? null : input.SourceDirectory, StartAfter = null }; @@ -56,45 +59,45 @@ public static async Task DownloadObject([PropertyTab] Connection input, foreach (var fileObject in allObjectsResponse.S3Objects) { cancellationToken.ThrowIfCancellationRequested(); - if (mask.IsMatch(fileObject.Key.Split('/').Last()) && (targetPath.Split('/').Length == fileObject.Key.Split('/').Length || !input.DownloadFromCurrentDirectoryOnly) && !fileObject.Key.EndsWith("/") && fileObject.Key.StartsWith(input.S3Directory)) + if (mask.IsMatch(fileObject.Key.Split('/').Last()) && (targetPath.Split('/').Length == fileObject.Key.Split('/').Length || !input.DownloadFromCurrentDirectoryOnly) && !fileObject.Key.EndsWith("/") && fileObject.Key.StartsWith(input.SourceDirectory)) { - var path = Path.Combine(input.DestinationDirectory, fileObject.Key.Split('/').Last()); + var path = Path.Combine(input.TargetDirectory, fileObject.Key.Split('/').Last()); var request = new GetObjectRequest { BucketName = input.BucketName, Key = fileObject.Key }; var response = await client.GetObjectAsync(request, cancellationToken); - result.Add(await WriteToFile(response, null, client, fileObject, input, path, cancellationToken)); + result.Add(await WriteToFile(response, null, client, fileObject, options, path, cancellationToken)); } } } } else { - if (string.IsNullOrWhiteSpace(input.PreSignedURL)) + if (string.IsNullOrWhiteSpace(connection.PreSignedUrl)) throw new Exception("AWS pre-signed URL required."); - var responseStream = await Client.GetStreamAsync(input.PreSignedURL, cancellationToken); - var nameFromURI = Regex.Match(input.PreSignedURL, @"[^\/]+(?=\?)"); + var responseStream = await Client.GetStreamAsync(connection.PreSignedUrl, cancellationToken); + var nameFromURI = Regex.Match(connection.PreSignedUrl, @"[^\/]+(?=\?)"); var fileName = nameFromURI.Value; - var path = Path.Combine(input.DestinationDirectory, fileName); + var path = Path.Combine(input.TargetDirectory, fileName); if (responseStream != null) - result.Add(await WriteToFile(null, responseStream, null, null, input, path, cancellationToken)); + result.Add(await WriteToFile(null, responseStream, null, null, options, path, cancellationToken)); responseStream.Dispose(); } - if (result.Count == 0 && input.ThrowErrorIfNoMatch) + if (result.Count == 0 && options.ThrowErrorIfNoMatch) throw new Exception("No matches found with search pattern"); return new Result(true, result); } catch (Exception ex) { - throw new Exception(ex.Message); + return ErrorHandler.Handle(ex, options, result); } } - private static async Task WriteToFile(GetObjectResponse getObjectResponse, Stream preSignedStream, AmazonS3Client amazonS3Client, S3Object fileObject, Connection input, string fullPath, CancellationToken cancellationToken) + private static async Task WriteToFile(GetObjectResponse getObjectResponse, Stream preSignedStream, AmazonS3Client amazonS3Client, S3Object fileObject, Options options, string fullPath, CancellationToken cancellationToken) { var file = fileObject != null ? fileObject.Key.Split('/').Last() : Path.GetFileName(fullPath); var sourceDeleted = false; @@ -103,10 +106,10 @@ private static async Task WriteToFile(GetObjectResponse getO { if (File.Exists(fullPath)) { - switch (input.DestinationFileExistsAction) + switch (options.ActionOnExistingFile) { case DestinationFileExistsActions.Overwrite: - if (!FileLocked(input.FileLockedRetries, fullPath, cancellationToken)) + if (!FileLocked(options.FileLockedRetries, fullPath, cancellationToken)) File.Delete(fullPath); break; case DestinationFileExistsActions.Info: @@ -130,10 +133,10 @@ private static async Task WriteToFile(GetObjectResponse getO else throw new Exception("Write failed because the stream is empty."); - if (input.DeleteSourceObject && getObjectResponse != null) - sourceDeleted = await DeleteSourceFile(amazonS3Client, input.BucketName, fileObject.Key, cancellationToken); + if (options.DeleteSourceObject && getObjectResponse != null) + sourceDeleted = await DeleteSourceFile(amazonS3Client, getObjectResponse.BucketName, fileObject.Key, cancellationToken); - return new SingleResultObject(file, fullPath, input.DestinationFileExistsAction is DestinationFileExistsActions.Overwrite, sourceDeleted, null); + return new SingleResultObject(file, fullPath, options.ActionOnExistingFile is DestinationFileExistsActions.Overwrite, sourceDeleted, null); } catch (Exception ex) { @@ -155,7 +158,7 @@ private static bool FileLocked(int fileLockedRetries, string fullPath, Cancellat Thread.Sleep(1000); } - throw new Exception($"FileLocked error: {fullPath} was locked. Max Connection.FileLockedRetries = {fileLockedRetries} exceeded."); + throw new Exception($"FileLocked error: {fullPath} was locked. Max Options.FileLockedRetries = {fileLockedRetries} exceeded."); } catch (Exception ex) { @@ -214,4 +217,4 @@ private static RegionEndpoint RegionSelection(Region region) _ => RegionEndpoint.EUWest1, }; } -} \ No newline at end of file +} diff --git a/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject.csproj b/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject.csproj index ea9c7fc..a2e64fb 100644 --- a/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject.csproj +++ b/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject.csproj @@ -1,31 +1,31 @@ - - + + - net6.0 - 2.2.0 - Frends - Frends - Frends - Frends - Frends - MIT - true - Frends Task for downloading objects from an Amazon S3 - https://frends.com/ - https://github.com/FrendsPlatform/Frends.AmazonS3 + net8.0 + 3.0.0 + Frends + Frends + Frends + Frends + Frends + MIT + true + Frends Task for downloading objects from an Amazon S3 + https://frends.com/ + https://github.com/FrendsPlatform/Frends.AmazonS3 - - - PreserveNewest - + + PreserveNewest + - - - - + + + + + \ No newline at end of file diff --git a/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject/Helpers/ErrorHandler.cs b/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject/Helpers/ErrorHandler.cs new file mode 100644 index 0000000..4657b5b --- /dev/null +++ b/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject/Helpers/ErrorHandler.cs @@ -0,0 +1,32 @@ +using Frends.AmazonS3.DownloadObject.Definitions; +using System; +using System.Collections.Generic; + +namespace Frends.AmazonS3.DownloadObject.Helpers; + +/// +/// Static class for handling errors in the AmazonS3 DownloadObject task. +/// +public static class ErrorHandler +{ + /// + /// Handles exceptions based on the provided options. + /// + /// The exception to handle + /// Options containing error handling configuration + /// The current result list + /// Result object with error information + public static Result Handle(Exception exception, Options options, List result) + { + if (options.ThrowErrorOnFailure) + { + var errorMessage = !string.IsNullOrWhiteSpace(options.ErrorMessageOnFailure) + ? options.ErrorMessageOnFailure + : exception.Message; + throw new Exception(errorMessage); + } + + var error = new Error(exception.Message, new { StackTrace = exception.StackTrace, InnerException = exception.InnerException?.Message }); + return new Result(false, result, error); + } +} diff --git a/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject/migration.json b/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject/migration.json new file mode 100644 index 0000000..3ed683e --- /dev/null +++ b/Frends.AmazonS3.DownloadObject/Frends.AmazonS3.DownloadObject/migration.json @@ -0,0 +1,78 @@ +[ + { + "Task": "Frends.AmazonS3.DownloadObject", + "Migrations": [ + { + "Version": "3.0.0", + "Description": "Major refactoring - restructured parameters into Input, Connection, and Options classes with improved error handling", + "Migration": [ + { + "Type": "Copy", + "Source": "Connection.BucketName", + "Target": "Input.BucketName" + }, + { + "Type": "Copy", + "Source": "Connection.S3Directory", + "Target": "Input.SourceDirectory" + }, + { + "Type": "Copy", + "Source": "Connection.SearchPattern", + "Target": "Input.SearchPattern" + }, + { + "Type": "Copy", + "Source": "Connection.DownloadFromCurrentDirectoryOnly", + "Target": "Input.DownloadFromCurrentDirectoryOnly" + }, + { + "Type": "Copy", + "Source": "Connection.DestinationDirectory", + "Target": "Input.TargetDirectory" + }, + { + "Type": "Copy", + "Source": "Connection.FileLockedRetries", + "Target": "Options.FileLockedRetries" + }, + { + "Type": "Copy", + "Source": "Connection.DeleteSourceObject", + "Target": "Options.DeleteSourceObject" + }, + { + "Type": "Copy", + "Source": "Connection.ThrowErrorIfNoMatch", + "Target": "Options.ThrowErrorIfNoMatch" + }, + { + "Type": "Copy", + "Source": "Connection.DestinationFileExistsAction", + "Target": "Options.ActionOnExistingFile" + }, + { + "Type": "Copy", + "Source": "Connection.AWSCredentials", + "Target": "Connection.AwsCredentials" + }, + { + "Type": "Copy", + "Source": "Connection.PreSignedURL", + "Target": "Connection.PreSignedUrl" + }, + { + "Type": "Set", + "Target": "Options.ThrowErrorOnFailure", + "Value": "false" + }, + { + "Type": "Set", + "Target": "Options.ErrorMessageOnFailure", + "Value": "\"\"" + } + ] + } + ] + } +]