diff --git a/docs/rfcs/017-asset-modified-cleanup.md b/docs/rfcs/017-asset-modified-cleanup.md index 8750acbd4..315f28d47 100644 --- a/docs/rfcs/017-asset-modified-cleanup.md +++ b/docs/rfcs/017-asset-modified-cleanup.md @@ -106,6 +106,8 @@ This is that the delivery channel stays the same, but the id of the policy has c - iiif-img changed - `info.json` needs removed + - If it moves to a `use-original` policy, the derivative asset can be removed + - If it moves away from `use-original`, then the `/original` asset can also be removed, provided there isn't a `file` channel - Thumbs changed - Thumbs need to be removed that are no longer required. - `s.json` and asset application metadata should be updated - `s.json` should be updated by the reingest @@ -113,7 +115,7 @@ This is that the delivery channel stays the same, but the id of the policy has c - iiif-av changed - Old transcode derivative removed if the file extension is no longer required - File changed - - The asset at origin should be removed if there's an asset on the `/original` path - should only be removed if `iiif-img` is not using it + - do nothing #### Roles changed @@ -126,7 +128,7 @@ The policy data being updated can be found from the date that the delivery chann - iiif-img changed - `info.json` needs removed - If it moves to a `use-original` policy, the derivative asset can be removed - - If it moves awa from `use-original`, then the `/original` asset can also be removed, provided there isn't a `file` channel + - If it moves away from `use-original`, then the `/original` asset can also be removed, provided there isn't a `file` channel - Thumbs changed - Thumbs need to be removed that are no longer required. - `s.json` and asset application metadata should be updated - `s.json` should be updated by the reingest @@ -134,7 +136,7 @@ The policy data being updated can be found from the date that the delivery chann - iiif-av changed - Old transcode derivative removed if the file extension is no longer required - File changed - - The asset at origin should be removed if there's an asset on the `/original` path - should only be removed if `iiif-img` is not using it + - do nothing ## General comments diff --git a/src/protagonist/API.Client/DlcsClient.cs b/src/protagonist/API.Client/DlcsClient.cs index 1592cf518..1116d96b8 100644 --- a/src/protagonist/API.Client/DlcsClient.cs +++ b/src/protagonist/API.Client/DlcsClient.cs @@ -118,7 +118,7 @@ public async Task> GetSpaceImages(int page, int pageSize, var response = await httpClient.GetAsync(url); return await response.ReadAsHydraResponseAsync(jsonSerializerSettings); } - catch (Exception ex) + catch (Exception) { logger.LogError("Failed to deserialize storage for space {SpaceStorage}", spaceId); return null; @@ -196,7 +196,7 @@ public async Task DeletePortalUser(string portalUserId) var url = $"customers/{currentUser.GetCustomerId()}/portalUsers/{portalUserId}"; try { - var response = await httpClient.DeleteAsync(url); + await httpClient.DeleteAsync(url); return true; } catch (Exception ex) @@ -226,7 +226,7 @@ public async Task DeleteImage(int spaceId, string imageId) var uri = $"customers/{currentUser.GetCustomerId()}/spaces/{spaceId}/images/{imageId}"; try { - var response = await httpClient.DeleteAsync(uri); + await httpClient.DeleteAsync(uri); return true; } catch (Exception ex) @@ -317,7 +317,49 @@ public async Task GetQueue() var queue = await response.ReadAsHydraResponseAsync(jsonSerializerSettings); return queue; } + + public async Task> GetNamedQueries(bool includeGlobal) + { + var url = $"customers/{currentUser.GetCustomerId()}/namedQueries"; + var response = await httpClient.GetAsync(url); + var namedQueries = await response.ReadAsHydraResponseAsync>(jsonSerializerSettings); + + return includeGlobal + ? namedQueries.Members + : namedQueries.Members.Where(nq => nq.Global == false); + } + + public async Task DeleteNamedQuery(string namedQueryId) + { + var url = $"customers/{currentUser.GetCustomerId()}/namedQueries/{namedQueryId}"; + try + { + await httpClient.DeleteAsync(url); + return true; + } + catch (Exception ex) + { + logger.LogError(ex, "Error deleting named query '{NamedQueryId}'", namedQueryId); + return false; + } + } + public async Task UpdateNamedQuery(string namedQueryId, string template) + { + var url = $"customers/{currentUser.GetCustomerId()}/namedQueries/{namedQueryId}"; + var response = await httpClient.PutAsync(url, ApiBody(new NamedQuery(){ Template = template })); + var updatedNamedQuery = await response.ReadAsHydraResponseAsync(jsonSerializerSettings); + return updatedNamedQuery; + } + + public async Task CreateNamedQuery(NamedQuery newNamedQuery) + { + var url = $"customers/{currentUser.GetCustomerId()}/namedQueries"; + var response = await httpClient.PostAsync(url, ApiBody(newNamedQuery)); + var namedQuery = await response.ReadAsHydraResponseAsync(jsonSerializerSettings); + return namedQuery; + } + private HttpContent ApiBody(JsonLdBase apiObject) { var jsonString = JsonConvert.SerializeObject(apiObject, jsonSerializerSettings); diff --git a/src/protagonist/API.Client/IDlcsClient.cs b/src/protagonist/API.Client/IDlcsClient.cs index 6e3567d0e..edaf4dd97 100644 --- a/src/protagonist/API.Client/IDlcsClient.cs +++ b/src/protagonist/API.Client/IDlcsClient.cs @@ -34,4 +34,8 @@ Task> GetSpaceImages(int page, int pageSize, int spaceId, Task TestBatch(int batchId); Task> GetBatchImages(int batchId, int page, int pageSize); Task GetQueue(); + Task> GetNamedQueries(bool includeGlobal); + Task DeleteNamedQuery(string namedQueryId); + Task UpdateNamedQuery(string namedQueryId, string template); + Task CreateNamedQuery(NamedQuery newNamedQuery); } \ No newline at end of file diff --git a/src/protagonist/API.Tests/Infrastructure/Messaging/AssetModificationRecordTests.cs b/src/protagonist/API.Tests/Infrastructure/Messaging/AssetModificationRecordTests.cs index 44188b552..2229d7e85 100644 --- a/src/protagonist/API.Tests/Infrastructure/Messaging/AssetModificationRecordTests.cs +++ b/src/protagonist/API.Tests/Infrastructure/Messaging/AssetModificationRecordTests.cs @@ -37,19 +37,22 @@ public void Create_SetsCorrectFields() notification.Before.Should().BeNull(); } - [Fact] - public void Update_SetsCorrectFields() + [Theory] + [InlineData(true)] + [InlineData(false)] + public void Update_SetsCorrectFields(bool engineNotified) { // Arrange var before = new Asset { Id = new AssetId(1, 2, "foo") }; var after = new Asset { Id = new AssetId(1, 2, "foo"), MaxUnauthorised = 10 }; // Act - var notification = AssetModificationRecord.Update(before, after); + var notification = AssetModificationRecord.Update(before, after, engineNotified); // Assert notification.ChangeType.Should().Be(ChangeType.Update); notification.Before.Should().Be(before); notification.After.Should().Be(after); + notification.EngineNotified.Should().Be(engineNotified); } } \ No newline at end of file diff --git a/src/protagonist/API.Tests/Infrastructure/Messaging/AssetNotificationSenderTests.cs b/src/protagonist/API.Tests/Infrastructure/Messaging/AssetNotificationSenderTests.cs index f851b685b..ee63a4f5d 100644 --- a/src/protagonist/API.Tests/Infrastructure/Messaging/AssetNotificationSenderTests.cs +++ b/src/protagonist/API.Tests/Infrastructure/Messaging/AssetNotificationSenderTests.cs @@ -1,7 +1,6 @@ using System.Collections.Generic; using System.Linq; using System.Threading; -using System.Threading.Tasks; using API.Infrastructure.Messaging; using DLCS.AWS.SNS; using DLCS.Core.Types; @@ -33,7 +32,7 @@ public async Task SendAssetModifiedMessage_Single_SendsNotification_IfUpdate() { // Arrange var assetModifiedRecord = - AssetModificationRecord.Update(new Asset(new AssetId(1, 2, "foo")), new Asset(new AssetId(1, 2, "bar"))); + AssetModificationRecord.Update(new Asset(new AssetId(1, 2, "foo")), new Asset(new AssetId(1, 2, "bar")), true); // Act await sut.SendAssetModifiedMessage(assetModifiedRecord, CancellationToken.None); @@ -76,7 +75,7 @@ public async Task SendAssetModifiedMessage_Single_SendsNotification_IfDelete() A.CallTo(() => topicPublisher.PublishToAssetModifiedTopic( A>.That.Matches(n => - n.Single().ChangeType == ChangeType.Delete && n.Single().MessageContents.Contains(customerName)), + n.Single().Attributes.Values.Contains(ChangeType.Delete.ToString()) && n.Single().MessageContents.Contains(customerName)), A._)).MustHaveHappened(); } @@ -102,7 +101,7 @@ public async Task SendAssetModifiedMessage_Multiple_SendsNotification_IfDelete() topicPublisher.PublishToAssetModifiedTopic( A>.That.Matches(n => n.Count == 2 && n.All(m => - m.ChangeType == ChangeType.Delete && m.MessageContents.Contains(customerName))), + n.First().Attributes.Values.Contains(ChangeType.Delete.ToString()) && m.MessageContents.Contains(customerName))), A._)).MustHaveHappened(); } } diff --git a/src/protagonist/API.Tests/Integration/CustomerQueueTests.cs b/src/protagonist/API.Tests/Integration/CustomerQueueTests.cs index a0e50f472..c4d7577cf 100644 --- a/src/protagonist/API.Tests/Integration/CustomerQueueTests.cs +++ b/src/protagonist/API.Tests/Integration/CustomerQueueTests.cs @@ -558,6 +558,30 @@ public async Task Post_CreateBatch_201_IfLegacyModeEnabled() response.StatusCode.Should().Be(HttpStatusCode.Created); } + [Fact] + public async Task Post_CreateBatch_201_IfLegacyModeEnabled_MembersEmpty() + { + // Arrange + var hydraImageBody = @"{ + ""@context"": ""http://www.w3.org/ns/hydra/context.jsonld"", + ""@type"": ""Collection"", + ""member"": [] +}"; + + var content = new StringContent(hydraImageBody, Encoding.UTF8, "application/json"); + var path = $"/customers/{LegacyModeHelpers.LegacyCustomer}/queue"; + + // Act + var response = await httpClient.AsCustomer(LegacyModeHelpers.LegacyCustomer).PostAsync(path, content); + + // Assert + response.StatusCode.Should().Be(HttpStatusCode.Created); + + var model = await response.ReadAsHydraResponseAsync(); + var dbBatch = dbContext.Batches.Single(a => a.Id == model.Id.GetLastPathElementAsInt()); + dbBatch.Count.Should().Be(0); + } + [Fact] public async Task Post_CreateBatch_201_IfLegacyModeEnabledWithAtIdFieldSet() { @@ -698,6 +722,26 @@ public async Task Post_CreateBatch_400_WithInvalidIdsSet() response.StatusCode.Should().Be(HttpStatusCode.BadRequest); } + [Fact] + public async Task Post_CreateBatch_400_MembersEmpty() + { + // Arrange + var hydraImageBody = @"{ + ""@context"": ""http://www.w3.org/ns/hydra/context.jsonld"", + ""@type"": ""Collection"", + ""member"": [] +}"; + + var content = new StringContent(hydraImageBody, Encoding.UTF8, "application/json"); + const string path = "/customers/99/queue"; + + // Act + var response = await httpClient.AsCustomer(99).PostAsync(path, content); + + // Assert + response.StatusCode.Should().Be(HttpStatusCode.BadRequest); + } + [Fact] public async Task Post_CreateBatch_400_IfSpaceNotFound() { diff --git a/src/protagonist/API.Tests/Integration/CustomerResourcesTests.cs b/src/protagonist/API.Tests/Integration/CustomerResourcesTests.cs index 4ad297929..ac48b8570 100644 --- a/src/protagonist/API.Tests/Integration/CustomerResourcesTests.cs +++ b/src/protagonist/API.Tests/Integration/CustomerResourcesTests.cs @@ -54,7 +54,7 @@ public async Task Delete_PDF_Returns400_IfUnableToFind() } [Fact] - public async Task Delete_PDF_Returns400_IfArgsIncorrect() + public async Task Delete_PDF_Returns200_IfLessArgsThanQuery() { // Arrange var path = "/customers/99/resources/pdf/cust-resource?args=too-little"; @@ -62,6 +62,19 @@ public async Task Delete_PDF_Returns400_IfArgsIncorrect() // Act var response = await httpClient.AsCustomer().DeleteAsync(path); + // Assert + response.StatusCode.Should().Be(HttpStatusCode.OK); + } + + [Fact] + public async Task Delete_PDF_Returns400_IfNoArgs() + { + // Arrange + var path = "/customers/99/resources/pdf/cust-resource"; + + // Act + var response = await httpClient.AsCustomer().DeleteAsync(path); + // Assert response.StatusCode.Should().Be(HttpStatusCode.BadRequest); } diff --git a/src/protagonist/API/Converters/LegacyModeConverter.cs b/src/protagonist/API/Converters/LegacyModeConverter.cs index 2542d0047..4c71e7a74 100644 --- a/src/protagonist/API/Converters/LegacyModeConverter.cs +++ b/src/protagonist/API/Converters/LegacyModeConverter.cs @@ -4,6 +4,7 @@ using DLCS.HydraModel; using DLCS.Model.Assets; using Hydra; +using Microsoft.Extensions.Logging; using AssetFamily = DLCS.HydraModel.AssetFamily; namespace API.Converters; @@ -14,13 +15,16 @@ namespace API.Converters; public static class LegacyModeConverter { private const string DefaultMediaType = "image/unknown"; + + internal static void LogLegacyUsage(this ILogger logger, string message, params object?[] args) + => logger.LogWarning("LEGACY USE:" + message, args); /// /// Converts from legacy format to new format /// /// The image to convert should be emulated and translated into delivery channels /// A converted image - public static T VerifyAndConvertToModernFormat(T image) + public static T VerifyAndConvertToModernFormat(T image, ILogger? logger = null) where T : Image { if (image.Origin.IsNullOrEmpty()) @@ -30,6 +34,7 @@ public static T VerifyAndConvertToModernFormat(T image) if (image.MediaType.IsNullOrEmpty()) { + logger?.LogLegacyUsage("Null or empty media type"); var contentType = image.Origin?.Split('.').Last() ?? string.Empty; image.MediaType = MIMEHelper.GetContentTypeForExtension(contentType) ?? DefaultMediaType; @@ -43,6 +48,7 @@ public static T VerifyAndConvertToModernFormat(T image) if (image.MaxUnauthorised is null or 0 && image.Roles.IsNullOrEmpty()) { + logger?.LogLegacyUsage("MaxUnauthorised"); image.MaxUnauthorised = -1; } diff --git a/src/protagonist/API/Features/Image/Ingest/AssetProcessor.cs b/src/protagonist/API/Features/Image/Ingest/AssetProcessor.cs index e5d5a0305..cbf08eebc 100644 --- a/src/protagonist/API/Features/Image/Ingest/AssetProcessor.cs +++ b/src/protagonist/API/Features/Image/Ingest/AssetProcessor.cs @@ -44,16 +44,16 @@ public AssetProcessor( /// /// Optional delegate for modifying asset prior to saving /// Current cancellation token + /// Whether the request is for the priority queue or not public async Task Process(AssetBeforeProcessing assetBeforeProcessing, bool mustExist, bool alwaysReingest, bool isBatchUpdate, Func? requiresReingestPreSave = null, CancellationToken cancellationToken = default) { - Asset? existingAsset; try { - existingAsset = await assetRepository.GetAsset(assetBeforeProcessing.Asset.Id, true); + var assetFromDatabase = await assetRepository.GetAsset(assetBeforeProcessing.Asset.Id, true, true); - if (existingAsset == null) + if (assetFromDatabase == null) { if (mustExist) { @@ -102,9 +102,10 @@ public async Task Process(AssetBeforeProcessing assetBeforeP ) }; } - + + var existingAsset = assetFromDatabase?.Clone(); var assetPreparationResult = - AssetPreparer.PrepareAssetForUpsert(existingAsset, assetBeforeProcessing.Asset, false, isBatchUpdate, + AssetPreparer.PrepareAssetForUpsert(assetFromDatabase, assetBeforeProcessing.Asset, false, isBatchUpdate, settings.RestrictedAssetIdCharacters); if (!assetPreparationResult.Success) @@ -115,11 +116,11 @@ public async Task Process(AssetBeforeProcessing assetBeforeP WriteResult.FailedValidation) }; } - + var updatedAsset = assetPreparationResult.UpdatedAsset!; // this is from Database var requiresEngineNotification = assetPreparationResult.RequiresReingest || alwaysReingest; - var deliveryChannelChanged = await deliveryChannelProcessor.ProcessImageDeliveryChannels(existingAsset, + var deliveryChannelChanged = await deliveryChannelProcessor.ProcessImageDeliveryChannels(assetFromDatabase, updatedAsset, assetBeforeProcessing.DeliveryChannelsBeforeProcessing); if (deliveryChannelChanged) { @@ -140,14 +141,14 @@ public async Task Process(AssetBeforeProcessing assetBeforeP updatedAsset.MarkAsFinished(); } - var assetAfterSave = await assetRepository.Save(updatedAsset, existingAsset != null, cancellationToken); + var assetAfterSave = await assetRepository.Save(updatedAsset, assetFromDatabase != null, cancellationToken); return new ProcessAssetResult { ExistingAsset = existingAsset, RequiresEngineNotification = requiresEngineNotification, Result = ModifyEntityResult.Success(assetAfterSave, - existingAsset == null ? WriteResult.Created : WriteResult.Updated) + assetFromDatabase == null ? WriteResult.Created : WriteResult.Updated) }; } catch (APIException apiEx) @@ -175,4 +176,4 @@ public class ProcessAssetResult public bool RequiresEngineNotification { get; set; } public bool IsSuccess => Result.IsSuccess; -} \ No newline at end of file +} diff --git a/src/protagonist/API/Features/Image/Ingest/DeliveryChannelProcessor.cs b/src/protagonist/API/Features/Image/Ingest/DeliveryChannelProcessor.cs index da8198c90..eb32c4648 100644 --- a/src/protagonist/API/Features/Image/Ingest/DeliveryChannelProcessor.cs +++ b/src/protagonist/API/Features/Image/Ingest/DeliveryChannelProcessor.cs @@ -45,12 +45,9 @@ public async Task ProcessImageDeliveryChannels(Asset? existingAsset, Asset deliveryChannelsBeforeProcessing, existingAsset != null); return deliveryChannelChanged; } - catch (InvalidOperationException) + catch (InvalidOperationException ioEx) { - throw new APIException("Failed to match delivery channel policy") - { - StatusCode = 400 - }; + throw new BadRequestException("Failed to match delivery channel policy", ioEx); } } @@ -160,7 +157,7 @@ private async Task SetImageDeliveryChannels(Asset asset, DeliveryChannelsB private async Task GetDeliveryChannelPolicy(Asset asset, DeliveryChannelsBeforeProcessing deliveryChannel) { DeliveryChannelPolicy deliveryChannelPolicy; - if (deliveryChannel.Policy.IsNullOrEmpty()) + if (string.IsNullOrEmpty(deliveryChannel.Policy)) { deliveryChannelPolicy = await defaultDeliveryChannelRepository.MatchDeliveryChannelPolicyForChannel( asset.MediaType!, asset.Space, asset.Customer, deliveryChannel.Channel); @@ -201,11 +198,9 @@ await defaultDeliveryChannelRepository.MatchedDeliveryChannels(asset.MediaType!, if (matchedDeliveryChannels.Any(x => x.Channel == AssetDeliveryChannels.None) && matchedDeliveryChannels.Count != 1) { - throw new APIException("An asset can only be automatically assigned a delivery channel of type 'None' when it is the only one available. " + - "Please check your default delivery channel configuration.") - { - StatusCode = 400 - }; + throw new BadRequestException( + "An asset can only be automatically assigned a delivery channel of type 'None' when it is the only one available. " + + "Please check your default delivery channel configuration."); } foreach (var deliveryChannel in matchedDeliveryChannels) diff --git a/src/protagonist/API/Features/Image/Requests/CreateOrUpdateImage.cs b/src/protagonist/API/Features/Image/Requests/CreateOrUpdateImage.cs index 5b06bb5ed..0b94f0e39 100644 --- a/src/protagonist/API/Features/Image/Requests/CreateOrUpdateImage.cs +++ b/src/protagonist/API/Features/Image/Requests/CreateOrUpdateImage.cs @@ -132,7 +132,7 @@ public async Task> Handle(CreateOrUpdateImage request, var assetModificationRecord = existingAsset == null ? AssetModificationRecord.Create(assetAfterSave) - : AssetModificationRecord.Update(existingAsset, assetAfterSave); + : AssetModificationRecord.Update(existingAsset, assetAfterSave, processAssetResult.RequiresEngineNotification); await assetNotificationSender.SendAssetModifiedMessage(assetModificationRecord, cancellationToken); diff --git a/src/protagonist/API/Features/Image/Requests/ReingestAsset.cs b/src/protagonist/API/Features/Image/Requests/ReingestAsset.cs index 201b37cc6..09c350e44 100644 --- a/src/protagonist/API/Features/Image/Requests/ReingestAsset.cs +++ b/src/protagonist/API/Features/Image/Requests/ReingestAsset.cs @@ -52,7 +52,7 @@ public async Task> Handle(ReingestAsset request, Cance var asset = await MarkAssetAsIngesting(cancellationToken, existingAsset!); - await assetNotificationSender.SendAssetModifiedMessage(AssetModificationRecord.Update(existingAsset!, asset), + await assetNotificationSender.SendAssetModifiedMessage(AssetModificationRecord.Update(existingAsset!, asset, true), cancellationToken); var statusCode = await ingestNotificationSender.SendImmediateIngestAssetRequest(asset, cancellationToken); diff --git a/src/protagonist/API/Features/Queues/CustomerQueueController.cs b/src/protagonist/API/Features/Queues/CustomerQueueController.cs index dda627229..89e588165 100644 --- a/src/protagonist/API/Features/Queues/CustomerQueueController.cs +++ b/src/protagonist/API/Features/Queues/CustomerQueueController.cs @@ -1,5 +1,4 @@ using System.Collections.Generic; -using System.Net; using API.Converters; using API.Exceptions; using API.Features.Image; @@ -8,6 +7,7 @@ using API.Features.Queues.Validation; using API.Infrastructure; using API.Settings; +using DLCS.Core.Collections; using DLCS.Core.Strings; using DLCS.HydraModel; using DLCS.Model.Assets; @@ -16,6 +16,7 @@ using MediatR; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Batch = DLCS.HydraModel.Batch; @@ -28,8 +29,12 @@ namespace API.Features.Queues; [ApiController] public class CustomerQueueController : HydraController { - public CustomerQueueController(IOptions settings, IMediator mediator) : base(settings.Value, mediator) + private readonly ILogger logger; + + public CustomerQueueController(IOptions settings, IMediator mediator, + ILogger logger) : base(settings.Value, mediator) { + this.logger = logger; } /// @@ -331,6 +336,15 @@ private async Task CreateBatchInternal(int customerId, HydraColle try { UpdateMembers(customerId, images.Members); + + if (images.Members.IsEmpty() && Settings.LegacyModeEnabledForCustomer(customerId)) + { + logger.LogLegacyUsage("Empty batch received for customer {CustomerId}", customerId); + return await HandleUpsert(new CreateEmptyBatch(customerId), + batch => batch.ToHydra(GetUrlRoots().BaseUrl), + errorTitle: "Create batch failed", + cancellationToken: cancellationToken); + } } catch (APIException apiEx) { diff --git a/src/protagonist/API/Features/Queues/Requests/CreateBatchOfImages.cs b/src/protagonist/API/Features/Queues/Requests/CreateBatchOfImages.cs index e9d02821f..8b10e0dab 100644 --- a/src/protagonist/API/Features/Queues/Requests/CreateBatchOfImages.cs +++ b/src/protagonist/API/Features/Queues/Requests/CreateBatchOfImages.cs @@ -2,6 +2,7 @@ using System.Data; using API.Features.Image; using API.Features.Image.Ingest; +using API.Infrastructure.Messaging; using API.Infrastructure.Requests; using DLCS.Core; using DLCS.Model.Assets; @@ -37,6 +38,7 @@ public class CreateBatchOfImagesHandler : IRequestHandler logger; public CreateBatchOfImagesHandler( @@ -44,12 +46,14 @@ public CreateBatchOfImagesHandler( IBatchRepository batchRepository, AssetProcessor assetProcessor, IIngestNotificationSender ingestNotificationSender, + IAssetNotificationSender assetNotificationSender, ILogger logger) { this.dlcsContext = dlcsContext; this.batchRepository = batchRepository; this.assetProcessor = assetProcessor; this.ingestNotificationSender = ingestNotificationSender; + this.assetNotificationSender = assetNotificationSender; this.logger = logger; } @@ -84,7 +88,9 @@ public async Task> Handle(CreateBatchOfImages request, var batch = await batchRepository.CreateBatch(request.CustomerId, request.AssetsBeforeProcessing.Select(a => a.Asset).ToList(), cancellationToken); - var assetNotificationList = new List(request.AssetsBeforeProcessing.Count); + var engineNotificationList = new List(request.AssetsBeforeProcessing.Count); + var assetModifiedNotificationList = new List(); + try { using var logScope = logger.BeginScope("Processing batch {BatchId}", batch.Id); @@ -106,9 +112,15 @@ await assetProcessor.Process(assetBeforeProcessing, false, true, true, var savedAsset = processAssetResult.Result.Entity!; + var existingAsset = processAssetResult.ExistingAsset; + var assetModificationRecord = existingAsset == null + ? AssetModificationRecord.Create(savedAsset) + : AssetModificationRecord.Update(existingAsset, savedAsset, processAssetResult.RequiresEngineNotification); + assetModifiedNotificationList.Add(assetModificationRecord); + if (processAssetResult.RequiresEngineNotification) { - assetNotificationList.Add(savedAsset); + engineNotificationList.Add(savedAsset); } else { @@ -119,6 +131,8 @@ await assetProcessor.Process(assetBeforeProcessing, false, true, true, } } + await assetNotificationSender.SendAssetModifiedMessage(assetModifiedNotificationList, cancellationToken); + if (batch.Completed > 0) { await dlcsContext.SaveChangesAsync(cancellationToken); @@ -151,7 +165,7 @@ await assetProcessor.Process(assetBeforeProcessing, false, true, true, { // Raise notifications logger.LogDebug("Batch {BatchId} created - sending engine notifications", batch.Id); - await ingestNotificationSender.SendIngestAssetsRequest(assetNotificationList, request.IsPriority, + await ingestNotificationSender.SendIngestAssetsRequest(engineNotificationList, request.IsPriority, cancellationToken); } diff --git a/src/protagonist/API/Features/Queues/Requests/CreateEmptyBatch.cs b/src/protagonist/API/Features/Queues/Requests/CreateEmptyBatch.cs new file mode 100644 index 000000000..bf66ff479 --- /dev/null +++ b/src/protagonist/API/Features/Queues/Requests/CreateEmptyBatch.cs @@ -0,0 +1,35 @@ +using API.Infrastructure.Requests; +using DLCS.Core; +using DLCS.Model.Assets; +using MediatR; + +namespace API.Features.Queues.Requests; + +/// +/// Handler that creates an empty batch of 0 images +/// +public class CreateEmptyBatch : IRequest> +{ + public int CustomerId { get; } + + public CreateEmptyBatch(int customerId) + { + CustomerId = customerId; + } +} + +public class CreateEmptyBatchHandler : IRequestHandler> +{ + private readonly IBatchRepository batchRepository; + + public CreateEmptyBatchHandler(IBatchRepository batchRepository) + { + this.batchRepository = batchRepository; + } + + public async Task> Handle(CreateEmptyBatch request, CancellationToken cancellationToken) + { + var batch = await batchRepository.CreateBatch(request.CustomerId, Array.Empty(), cancellationToken); + return ModifyEntityResult.Success(batch, WriteResult.Created); + } +} \ No newline at end of file diff --git a/src/protagonist/API/Infrastructure/Messaging/AssetModificationRecord.cs b/src/protagonist/API/Infrastructure/Messaging/AssetModificationRecord.cs index 478167b8c..2a0d43ff8 100644 --- a/src/protagonist/API/Infrastructure/Messaging/AssetModificationRecord.cs +++ b/src/protagonist/API/Infrastructure/Messaging/AssetModificationRecord.cs @@ -13,22 +13,27 @@ public class AssetModificationRecord public Asset? Before { get; } public Asset? After { get; } + public bool EngineNotified { get; } + public ImageCacheType? DeleteFrom { get; } - - private AssetModificationRecord(ChangeType changeType, Asset? before, Asset? after, ImageCacheType? deleteFrom) + + private AssetModificationRecord(ChangeType changeType, Asset? before, Asset? after, ImageCacheType? deleteFrom, bool assetModifiedEngineNotified) { ChangeType = changeType; Before = before; After = after; DeleteFrom = deleteFrom; + EngineNotified = assetModifiedEngineNotified; } - public static AssetModificationRecord Delete(Asset before, ImageCacheType deleteFrom) - => new(ChangeType.Delete, before.ThrowIfNull(nameof(before)), null, deleteFrom.ThrowIfNull(nameof(deleteFrom))); - - public static AssetModificationRecord Update(Asset before, Asset after) - => new(ChangeType.Update, before.ThrowIfNull(nameof(before)), after.ThrowIfNull(nameof(after)), null); + public static AssetModificationRecord Delete(Asset before, ImageCacheType deleteFrom) + => new(ChangeType.Delete, before.ThrowIfNull(nameof(before)), null, deleteFrom.ThrowIfNull(nameof(deleteFrom)), + false); + + public static AssetModificationRecord Update(Asset before, Asset after, bool assetModifiedEngineNotified) + => new(ChangeType.Update, before.ThrowIfNull(nameof(before)), after.ThrowIfNull(nameof(after)), null, + assetModifiedEngineNotified); - public static AssetModificationRecord Create(Asset after) - => new(ChangeType.Create, null, after.ThrowIfNull(nameof(after)), null); + public static AssetModificationRecord Create(Asset after) + => new(ChangeType.Create, null, after.ThrowIfNull(nameof(after)), null, false); } \ No newline at end of file diff --git a/src/protagonist/API/Infrastructure/Messaging/AssetNotificationSender.cs b/src/protagonist/API/Infrastructure/Messaging/AssetNotificationSender.cs index f7fa568a1..7cfaff11a 100644 --- a/src/protagonist/API/Infrastructure/Messaging/AssetNotificationSender.cs +++ b/src/protagonist/API/Infrastructure/Messaging/AssetNotificationSender.cs @@ -43,25 +43,28 @@ public Task SendAssetModifiedMessage(AssetModificationRecord notification, public async Task SendAssetModifiedMessage(IReadOnlyCollection notifications, CancellationToken cancellationToken = default) { - // Iterate through AssetModifiedMessage objects and build list(s) of changes - var changes = new Dictionary>() - { - [ChangeType.Create] = new(), - [ChangeType.Update] = new(), - [ChangeType.Delete] = new(), - }; + var changes = new List(); foreach (var notification in notifications) { var serialisedNotification = await GetSerialisedNotification(notification); if (serialisedNotification.HasText()) { - changes[notification.ChangeType].Add(serialisedNotification); + var attributes = new Dictionary() + { + { "messageType", notification.ChangeType.ToString() } + }; + + if (notification.EngineNotified) + { + attributes.Add("engineNotified", "True"); + } + + changes.Add(new AssetModifiedNotification(serialisedNotification!, attributes)); } } - // Send notifications generated in above method - await SendAssetModifiedRequest(changes, cancellationToken); + await topicPublisher.PublishToAssetModifiedTopic(changes, cancellationToken); } private async Task GetSerialisedNotification(AssetModificationRecord notification) @@ -131,16 +134,4 @@ private async Task GetCustomerPathElement(int customer) customerPathElements[customer] = customerPathElement; return customerPathElement; } - - private async Task SendAssetModifiedRequest(Dictionary> change, CancellationToken cancellationToken) - { - if (change.IsNullOrEmpty()) return true; - - var toSend = change - .SelectMany(kvp => kvp.Value - .Select(v => new AssetModifiedNotification(v, kvp.Key))) - .ToList(); - - return await topicPublisher.PublishToAssetModifiedTopic(toSend, cancellationToken); - } } \ No newline at end of file diff --git a/src/protagonist/API/Infrastructure/ServiceCollectionX.cs b/src/protagonist/API/Infrastructure/ServiceCollectionX.cs index 42ddb9dc1..224096ade 100644 --- a/src/protagonist/API/Infrastructure/ServiceCollectionX.cs +++ b/src/protagonist/API/Infrastructure/ServiceCollectionX.cs @@ -73,7 +73,7 @@ public static IServiceCollection AddAws(this IServiceCollection services, .AddSingleton() .AddSingleton() .AddSingleton() - .AddSingleton() + .AddScoped() .AddSingleton() .AddSingleton() .AddSingleton() diff --git a/src/protagonist/API/Startup.cs b/src/protagonist/API/Startup.cs index ad4a1a181..f0f17c2ba 100644 --- a/src/protagonist/API/Startup.cs +++ b/src/protagonist/API/Startup.cs @@ -73,7 +73,7 @@ public void ConfigureServices(IServiceCollection services) .AddCaching(cacheSettings) .AddDataAccess(configuration) .AddScoped() - .AddSingleton() + .AddScoped() .AddScoped() .AddScoped() .AddTransient() diff --git a/src/protagonist/CleanupHandler/AssetUpdatedHandler.cs b/src/protagonist/CleanupHandler/AssetUpdatedHandler.cs new file mode 100644 index 000000000..c4130a251 --- /dev/null +++ b/src/protagonist/CleanupHandler/AssetUpdatedHandler.cs @@ -0,0 +1,408 @@ +using System.Diagnostics.CodeAnalysis; +using System.IO.Enumeration; +using CleanupHandler.Infrastructure; +using CleanupHandler.Repository; +using DLCS.AWS.ElasticTranscoder; +using DLCS.AWS.S3; +using DLCS.AWS.S3.Models; +using DLCS.AWS.SQS; +using DLCS.Core.Collections; +using DLCS.Model.Assets; +using DLCS.Model.Assets.Metadata; +using DLCS.Model.Messaging; +using DLCS.Model.Policies; +using DLCS.Repository.Messaging; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NuGet.Packaging; + +namespace CleanupHandler; + +public class AssetUpdatedHandler : IMessageHandler +{ + private readonly CleanupHandlerSettings handlerSettings; + private readonly IStorageKeyGenerator storageKeyGenerator; + private readonly IBucketWriter bucketWriter; + private readonly IBucketReader bucketReader; + private readonly IAssetApplicationMetadataRepository assetMetadataRepository; + private readonly IThumbRepository thumbRepository; + private readonly ILogger logger; + private readonly IEngineClient engineClient; + private readonly ICleanupHandlerAssetRepository cleanupHandlerAssetRepository; + + + public AssetUpdatedHandler( + IStorageKeyGenerator storageKeyGenerator, + IBucketWriter bucketWriter, + IBucketReader bucketReader, + IAssetApplicationMetadataRepository assetMetadataRepository, + IThumbRepository thumbRepository, + IOptions handlerSettings, + IEngineClient engineClient, + ICleanupHandlerAssetRepository cleanupHandlerAssetRepository, + ILogger logger) + { + this.storageKeyGenerator = storageKeyGenerator; + this.bucketWriter = bucketWriter; + this.bucketReader = bucketReader; + this.handlerSettings = handlerSettings.Value; + this.assetMetadataRepository = assetMetadataRepository; + this.thumbRepository = thumbRepository; + this.engineClient = engineClient; + this.cleanupHandlerAssetRepository = cleanupHandlerAssetRepository; + this.logger = logger; + } + + public async Task HandleMessage(QueueMessage message, CancellationToken cancellationToken = default) + { + AssetUpdatedNotificationRequest? request; + try + { + request = message.GetMessageContents(); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to deserialize asset {@Message}", message); + return false; + } + + if (request?.AssetBeforeUpdate?.Id == null) return false; + + var assetBefore = request.AssetBeforeUpdate; + + var assetAfter = await cleanupHandlerAssetRepository.RetrieveAssetWithDeliveryChannels(assetBefore.Id); + + if (assetAfter == null) + { + logger.LogInformation("Asset {AssetId} was not found in the database for use in after calculation", + assetBefore.Id); + return false; + } + + if (NoCleanupRequired(message, assetAfter, assetBefore)) return true; + if (AssetStillIngesting(assetAfter, assetBefore)) return false; + + var (modifiedOrAdded, removed) = GetChangeSets(assetAfter, assetBefore); + + if (handlerSettings.AssetModifiedSettings.DryRun) + { + logger.LogInformation("Dry run enabled. Asset {AssetId} will log deletions, but not remove them", + assetBefore.Id); + } + + (HashSet objectsToRemove, HashSet foldersToRemove) s3Objects; + s3Objects.objectsToRemove = new HashSet(); + s3Objects.foldersToRemove = new HashSet(); + + if (removed.Any()) + { + foreach (var deliveryChannel in removed) + { + await CleanupRemoved(deliveryChannel, assetAfter, s3Objects); + } + } + + if (modifiedOrAdded.Any()) + { + try + { + await CleanupModified(modifiedOrAdded, assetBefore, assetAfter, s3Objects); + } + catch (Exception ex) + { + logger.LogError(ex, "Error cleaning modified delivery channels"); + return false; + } + } + + if (!Equals(assetAfter.Roles ?? string.Empty, assetBefore.Roles ?? string.Empty)) + { + CleanupRolesChanged(assetAfter, s3Objects.foldersToRemove); + } + + if (s3Objects.objectsToRemove.Count > 0) + { + await RemoveObjectsFromBucket(s3Objects.objectsToRemove); + } + + if (s3Objects.foldersToRemove.Count > 0) + { + await RemoveFolderInBucket(s3Objects.foldersToRemove); + } + + return true; + } + + private static (List modifiedOrAdded, List removed) GetChangeSets( + Asset? assetAfter, Asset assetBefore) + { + var modifiedOrAdded = + assetAfter!.ImageDeliveryChannels.Where(after => + assetBefore.ImageDeliveryChannels.All(before => + before.DeliveryChannelPolicyId != after.DeliveryChannelPolicyId || + before.DeliveryChannelPolicy.Modified != after.DeliveryChannelPolicy.Modified)).ToList(); + var removed = assetBefore.ImageDeliveryChannels.Where(before => + assetAfter.ImageDeliveryChannels.All(after => after.Channel != before.Channel)).ToList(); + return (modifiedOrAdded, removed); + } + + private static bool AssetStillIngesting(Asset assetAfter, Asset assetBefore) + { + return assetAfter.Ingesting == true && assetBefore.Finished > assetAfter.Finished; + } + + private static bool NoCleanupRequired([NotNullWhen(true)] QueueMessage message, Asset? assetAfter, Asset assetBefore) + { + return !message.MessageAttributes.Keys.Contains("engineNotified") && + (assetBefore.Roles ?? string.Empty) == (assetAfter.Roles ?? string.Empty); + } + + private void CleanupRolesChanged(Asset assetAfter, HashSet foldersToRemove) + { + var infoJsonRoot = storageKeyGenerator.GetInfoJsonRoot(assetAfter.Id); + foldersToRemove.Add(infoJsonRoot); + } + + private async Task CleanupModified(List modifiedOrAdded, Asset assetBefore, Asset assetAfter, + (HashSet objectsToRemove, HashSet foldersToRemove) s3Objects) + { + foreach (var deliveryChannel in modifiedOrAdded) + { + if (assetBefore.ImageDeliveryChannels.Any(x => x.Channel == deliveryChannel.Channel)) // checks for updated rather than added + { + await CleanupChangedPolicy(deliveryChannel, assetAfter, s3Objects.objectsToRemove); + } + } + } + + private async Task CleanupRemoved(ImageDeliveryChannel deliveryChannelRemoved, Asset assetAfter, + (HashSet objectsToRemove, HashSet foldersToRemove) s3Objects) + { + switch (deliveryChannelRemoved.Channel) + { + case AssetDeliveryChannels.Image: + CleanupRemovedImageDeliveryChannel(assetAfter, s3Objects.objectsToRemove); + break; + case AssetDeliveryChannels.Thumbnails: + await CleanupRemovedThumbnailDeliveryChannel(assetAfter, s3Objects); + break; + case AssetDeliveryChannels.Timebased: + await CleanupRemovedTimebasedDeliveryChannel(assetAfter, s3Objects.objectsToRemove); + break; + case AssetDeliveryChannels.File: + CleanupFileDeliveryChannel(assetAfter, s3Objects.objectsToRemove); + break; + default: + logger.LogDebug("policy {PolicyName} does not require any changes for asset {AssetId}", + deliveryChannelRemoved.DeliveryChannelPolicy.Name, assetAfter.Id); + break; + } + } + + private async Task CleanupChangedPolicy(ImageDeliveryChannel deliveryChannelModified, Asset assetAfter, + HashSet objectsToRemove) + { + switch (deliveryChannelModified.Channel) + { + case AssetDeliveryChannels.Image: + CleanupChangedImageDeliveryChannel(deliveryChannelModified, assetAfter, objectsToRemove); + break; + case AssetDeliveryChannels.Thumbnails: + await CleanupChangedThumbnailDeliveryChannel(assetAfter, objectsToRemove); + break; + case AssetDeliveryChannels.Timebased: + await CleanupChangedTimebasedDeliveryChannel(deliveryChannelModified, assetAfter, objectsToRemove); + break; + default: + logger.LogDebug("policy {PolicyName} does not require any changes for asset {AssetId}", + deliveryChannelModified.DeliveryChannelPolicy.Name, assetAfter.Id); + break; + } + } + + private async Task CleanupChangedTimebasedDeliveryChannel(ImageDeliveryChannel imageDeliveryChannel, + Asset assetAfter, HashSet objectsToRemove) + { + var presetList = imageDeliveryChannel.DeliveryChannelPolicy.AsTimebasedPresets(); + var keys = new List(); + var extensions = new List(); + var mediaPath = RetrieveMediaPath(assetAfter); + + var presetDictionary = await engineClient.GetAvPresets(); + + if (presetDictionary.IsNullOrEmpty()) + { + logger.LogWarning( + "retrieved no timebased presets from engine, {AssetId} will not be cleaned up for the timebased channel", + assetAfter.Id); + throw new ArgumentNullException(nameof(presetDictionary), "Failed to retrieve any preset values"); + } + + foreach (var presetIdentifier in presetList ?? new List()) + { + if (presetDictionary.TryGetValue(presetIdentifier, out var transcoderPreset)) + { + var timebasedFolder = storageKeyGenerator.GetStorageLocationRoot(assetAfter.Id); + + var keysFromAws = await bucketReader.GetMatchingKeys(timebasedFolder); + + keys.AddRange(keysFromAws); + extensions.Add(transcoderPreset.Extension); + } + } + + List assetsToDelete = keys.Where(k => + !extensions.Contains(k.Split('.').Last()) && k.Contains(mediaPath)) + .Select(k => new ObjectInBucket(handlerSettings.AWS.S3.StorageBucket, k)).ToList(); + + objectsToRemove.AddRange(assetsToDelete); + } + + private async Task CleanupChangedThumbnailDeliveryChannel(Asset assetAfter, HashSet objectsToRemove) + { + var thumbsToDelete = await ThumbsToBeDeleted(assetAfter); + objectsToRemove.AddRange(thumbsToDelete); + } + + private void CleanupChangedImageDeliveryChannel(ImageDeliveryChannel modifiedDeliveryChannel, Asset assetAfter, + HashSet objectsToRemove) + { + List bucketObjectsToBeRemoved = new(); + + if (modifiedDeliveryChannel.DeliveryChannelPolicyId == KnownDeliveryChannelPolicies.ImageUseOriginal) + { + bucketObjectsToBeRemoved.Add(storageKeyGenerator.GetStorageLocation(assetAfter.Id)); + } + else + { + if (assetAfter.DoesNotHaveDeliveryChannel(AssetDeliveryChannels.File)) + { + bucketObjectsToBeRemoved.Add(storageKeyGenerator.GetStoredOriginalLocation(assetAfter.Id)); + } + } + + objectsToRemove.AddRange(bucketObjectsToBeRemoved); + } + + private void CleanupFileDeliveryChannel(Asset assetAfter, HashSet objectsToRemove) + { + if (assetAfter.ImageDeliveryChannels.Any(i => i.DeliveryChannelPolicyId == KnownDeliveryChannelPolicies.ImageUseOriginal)) return; + List bucketObjectsTobeRemoved = new() + { + storageKeyGenerator.GetStoredOriginalLocation(assetAfter.Id) + }; + + objectsToRemove.AddRange(bucketObjectsTobeRemoved); + } + + private async Task CleanupRemovedTimebasedDeliveryChannel(Asset assetAfter, HashSet objectsToRemove) + { + List bucketObjectsTobeRemoved = new() + { + storageKeyGenerator.GetTimebasedMetadataLocation(assetAfter.Id), + }; + + var timebasedFolder = storageKeyGenerator.GetStorageLocationRoot(assetAfter.Id); + var keys = await bucketReader.GetMatchingKeys(timebasedFolder); + var path = RetrieveMediaPath(assetAfter); + + foreach (var key in keys) + { + if (key.Contains(path)) + { + bucketObjectsTobeRemoved.Add(new ObjectInBucket(handlerSettings.AWS.S3.StorageBucket, key)); + } + } + + objectsToRemove.AddRange(bucketObjectsTobeRemoved); + } + + private async Task CleanupRemovedThumbnailDeliveryChannel(Asset assetAfter, (HashSet objectsToRemove, HashSet foldersToRemove) s3Objects) + { + if (assetAfter.DoesNotHaveDeliveryChannel(AssetDeliveryChannels.Image)) + { + s3Objects.foldersToRemove.Add(storageKeyGenerator.GetThumbnailsRoot(assetAfter.Id)); + + if (!handlerSettings.AssetModifiedSettings.DryRun) + { + await assetMetadataRepository.DeleteAssetApplicationMetadata(assetAfter.Id, + AssetApplicationMetadataTypes.ThumbSizes); + } + } + else + { + var thumbsToDelete = await ThumbsToBeDeleted(assetAfter); + s3Objects.objectsToRemove.AddRange(thumbsToDelete); + } + } + + private async Task> ThumbsToBeDeleted(Asset assetAfter) + { + var thumbSizes = await thumbRepository.GetAllSizes(assetAfter.Id) ?? new List(); + var thumbsBucketKeys = await bucketReader.GetMatchingKeys(storageKeyGenerator.GetThumbnailsRoot(assetAfter.Id)); + + var thumbsBucketSizes = GetThumbSizesFromKeys(thumbsBucketKeys); + var convertedThumbSizes = thumbSizes.Select(s => Math.Max(s[0], s[1]).ToString()); + + var thumbsToDelete = thumbsBucketSizes.Where(t => !convertedThumbSizes.Contains(t.size)) + .Select(t => new ObjectInBucket(handlerSettings.AWS.S3.ThumbsBucket, t.path)).ToList(); + + return thumbsToDelete; + } + + private List<(string size, string path)> GetThumbSizesFromKeys(string[] thumbsBucketKeys) + { + var filteredFilenames = thumbsBucketKeys.Where(t => FileSystemName.MatchesSimpleExpression("*.jpg", t)); + + var thumbBucketSizes = filteredFilenames + .Select(f => (f.Split("/").Last().Split('.').First(), f)).ToList(); + + return thumbBucketSizes; + } + + private void CleanupRemovedImageDeliveryChannel(Asset assetAfter, HashSet objectsToRemove) + { + List bucketObjectsTobeRemoved = new() + { + storageKeyGenerator.GetStorageLocation(assetAfter.Id) + }; + + if (assetAfter.DoesNotHaveDeliveryChannel(AssetDeliveryChannels.File)) + { + bucketObjectsTobeRemoved.Add(storageKeyGenerator.GetStoredOriginalLocation(assetAfter.Id)); + } + + objectsToRemove.AddRange(bucketObjectsTobeRemoved); + } + + private async Task RemoveObjectsFromBucket(HashSet bucketObjectsTobeRemoved) + { + logger.LogInformation("locations to potentially be removed: {Objects}", bucketObjectsTobeRemoved); + + if (handlerSettings.AssetModifiedSettings.DryRun) return; + + await bucketWriter.DeleteFromBucket(bucketObjectsTobeRemoved.ToArray()); + } + + private async Task RemoveFolderInBucket(HashSet bucketFoldersToBeRemoved) + { + logger.LogInformation("bucket folders to potentially be removed: {Objects}", bucketFoldersToBeRemoved); + + if (handlerSettings.AssetModifiedSettings.DryRun) return; + + foreach (var bucketFolderToBeRemoved in bucketFoldersToBeRemoved) + { + await bucketWriter.DeleteFolder(bucketFolderToBeRemoved, true); + } + } + + private static string RetrieveMediaPath(Asset asset) + { + var template = TranscoderTemplates.GetDestinationTemplate(asset.MediaType!); + var path = template + .Replace("{jobId}/", "") + .Replace("{asset}", S3StorageKeyGenerator.GetStorageKey(asset.Id)) + .Replace(".{extension}", ""); + return path; + } +} \ No newline at end of file diff --git a/src/protagonist/CleanupHandler/CleanupHandler.csproj b/src/protagonist/CleanupHandler/CleanupHandler.csproj index 78e0d8e0f..83d328f11 100644 --- a/src/protagonist/CleanupHandler/CleanupHandler.csproj +++ b/src/protagonist/CleanupHandler/CleanupHandler.csproj @@ -9,15 +9,20 @@ + + + + + diff --git a/src/protagonist/CleanupHandler/Infrastructure/AssetQueueType.cs b/src/protagonist/CleanupHandler/Infrastructure/AssetQueueType.cs new file mode 100644 index 000000000..84745f836 --- /dev/null +++ b/src/protagonist/CleanupHandler/Infrastructure/AssetQueueType.cs @@ -0,0 +1,7 @@ +namespace CleanupHandler.Infrastructure; + +public enum AssetQueueType +{ + Delete, + Update +} \ No newline at end of file diff --git a/src/protagonist/CleanupHandler/Infrastructure/DeleteQueueMonitor.cs b/src/protagonist/CleanupHandler/Infrastructure/CleanupHandlerQueueMonitor.cs similarity index 51% rename from src/protagonist/CleanupHandler/Infrastructure/DeleteQueueMonitor.cs rename to src/protagonist/CleanupHandler/Infrastructure/CleanupHandlerQueueMonitor.cs index 36d00f212..a6ef3c519 100644 --- a/src/protagonist/CleanupHandler/Infrastructure/DeleteQueueMonitor.cs +++ b/src/protagonist/CleanupHandler/Infrastructure/CleanupHandlerQueueMonitor.cs @@ -7,16 +7,16 @@ namespace CleanupHandler.Infrastructure; /// -/// Background worker that monitors SQS queue for delete notifications +/// Background worker that monitors SQS queue for cleanup notifications /// -public class DeleteQueueMonitor : BackgroundService +public class CleanupHandlerQueueMonitor : BackgroundService { private readonly IHostApplicationLifetime hostApplicationLifetime; private readonly IOptions awsSettings; private readonly SqsListenerManager sqsListenerManager; - private readonly ILogger logger; + private readonly ILogger logger; - public DeleteQueueMonitor(SqsListenerManager sqsListenerManager, ILogger logger, + public CleanupHandlerQueueMonitor(SqsListenerManager sqsListenerManager, ILogger logger, IHostApplicationLifetime hostApplicationLifetime, IOptions awsSettings) { this.sqsListenerManager = sqsListenerManager; @@ -27,15 +27,27 @@ public DeleteQueueMonitor(SqsListenerManager sqsListenerManager, ILogger + { + sqsListenerManager.AddQueueListener(awsSettings.Value.SQS.DeleteNotificationQueueName, AssetQueueType.Delete), + sqsListenerManager.AddQueueListener(awsSettings.Value.SQS.UpdateNotificationQueueName, AssetQueueType.Update), + }; + + await Task.WhenAll(startTasks); + + sqsListenerManager.StartListening(); + + var configuredQueues = sqsListenerManager.GetConfiguredQueues(); + logger.LogInformation("Configured {QueueCount} queues", configuredQueues.Count); - await sqsListenerManager.SetupDefaultQueue(awsSettings.Value.SQS.DeleteNotificationQueueName); hostApplicationLifetime.ApplicationStopping.Register(OnStopping); } public override Task StopAsync(CancellationToken cancellationToken) { - logger.LogWarning("Stopping DeleteQueueMonitor"); + logger.LogWarning("Stopping CleanupHandlerQueueMonitor"); return Task.CompletedTask; } diff --git a/src/protagonist/CleanupHandler/Infrastructure/CleanupHandlerSettings.cs b/src/protagonist/CleanupHandler/Infrastructure/CleanupHandlerSettings.cs index 7cf6da086..df6cce89a 100644 --- a/src/protagonist/CleanupHandler/Infrastructure/CleanupHandlerSettings.cs +++ b/src/protagonist/CleanupHandler/Infrastructure/CleanupHandlerSettings.cs @@ -1,4 +1,5 @@ using DLCS.AWS.Settings; +using DLCS.Core.Settings; namespace CleanupHandler.Infrastructure; @@ -13,4 +14,22 @@ public class CleanupHandlerSettings /// AWS config /// public AWSSettings AWS { get; set; } + + /// + /// Asset modified settings + /// + public AssetModifiedSettings AssetModifiedSettings { get; set; } +} + +public class AssetModifiedSettings +{ + /// + /// Whether the removal of data will actually be removed, or only logged + /// + public bool DryRun { get; set; } = false; + + /// + /// Root URL of the engine + /// + public Uri EngineRoot { get; set; } } \ No newline at end of file diff --git a/src/protagonist/CleanupHandler/Infrastructure/ServiceCollectionX.cs b/src/protagonist/CleanupHandler/Infrastructure/ServiceCollectionX.cs index 8e9b7d515..2710769c6 100644 --- a/src/protagonist/CleanupHandler/Infrastructure/ServiceCollectionX.cs +++ b/src/protagonist/CleanupHandler/Infrastructure/ServiceCollectionX.cs @@ -1,8 +1,16 @@ -using DLCS.AWS.Cloudfront; +using CleanupHandler.Repository; +using DLCS.AWS.Cloudfront; using DLCS.AWS.Configuration; using DLCS.AWS.S3; using DLCS.AWS.SQS; +using DLCS.Core.Caching; using DLCS.Core.FileSystem; +using DLCS.Model.Assets; +using DLCS.Model.Assets.Metadata; +using DLCS.Repository; +using DLCS.Repository.Assets; +using DLCS.Repository.Messaging; +using DLCS.Web.Handlers; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; @@ -19,7 +27,10 @@ public static IServiceCollection AddAws(this IServiceCollection services, { services .AddSingleton() + .AddSingleton() .AddSingleton() + .AddSingleton() + .AddSingleton() .AddSingleton() .AddTransient(typeof(SqsListener<>)) .AddSingleton() @@ -28,7 +39,7 @@ public static IServiceCollection AddAws(this IServiceCollection services, .WithAmazonS3() .WithAmazonCloudfront() .WithAmazonSQS(); - + return services; } @@ -37,8 +48,48 @@ public static IServiceCollection AddAws(this IServiceCollection services, /// public static IServiceCollection AddQueueMonitoring(this IServiceCollection services) => services + .AddScoped>(provider => messageType => messageType switch + { + AssetQueueType.Delete => provider.GetRequiredService(), + AssetQueueType.Update => provider.GetRequiredService(), + _ => throw new ArgumentOutOfRangeException(nameof(messageType), messageType, null) + }) .AddScoped() - .AddDefaultQueueHandler() + .AddScoped() .AddSingleton() - .AddHostedService(); -} \ No newline at end of file + .AddHostedService(); + + /// + /// Add all data access dependencies, including repositories and DLCS context + /// + public static IServiceCollection AddDataAccess(this IServiceCollection services, IConfiguration configuration, + CleanupHandlerSettings cleanupHandlerSettings) + { + services + .AddScoped() + .AddSingleton() + .AddTransient() + .AddScoped() + .AddDlcsContext(configuration); + + services.AddHttpClient(client => + { + client.BaseAddress = cleanupHandlerSettings.AssetModifiedSettings.EngineRoot; + }) + .AddHttpMessageHandler(); + + return services; + } + + /// + /// Add required caching dependencies + /// + public static IServiceCollection AddCaching(this IServiceCollection services, CacheSettings cacheSettings) + => services + .AddMemoryCache(memoryCacheOptions => + { + memoryCacheOptions.SizeLimit = cacheSettings.MemoryCacheSizeLimit; + memoryCacheOptions.CompactionPercentage = cacheSettings.MemoryCacheCompactionPercentage; + }) + .AddLazyCache(); +} diff --git a/src/protagonist/CleanupHandler/Program.cs b/src/protagonist/CleanupHandler/Program.cs index a12d4074c..3f7c32842 100644 --- a/src/protagonist/CleanupHandler/Program.cs +++ b/src/protagonist/CleanupHandler/Program.cs @@ -1,5 +1,7 @@ using CleanupHandler.Infrastructure; using DLCS.AWS.SSM; +using DLCS.Core.Caching; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Serilog; @@ -36,6 +38,8 @@ public static IHostBuilder CreateHostBuilder(string[] args) => services .Configure(hostContext.Configuration) .AddAws(hostContext.Configuration, hostContext.HostingEnvironment) + .AddDataAccess(hostContext.Configuration, hostContext.Configuration.Get()) + .AddCaching(hostContext.Configuration.GetSection("Caching").Get()) .AddQueueMonitoring(); }) .UseSerilog((hostingContext, loggerConfiguration) diff --git a/src/protagonist/CleanupHandler/Repository/CleanupHandlerAssetRepository.cs b/src/protagonist/CleanupHandler/Repository/CleanupHandlerAssetRepository.cs new file mode 100644 index 000000000..4e6984ad5 --- /dev/null +++ b/src/protagonist/CleanupHandler/Repository/CleanupHandlerAssetRepository.cs @@ -0,0 +1,22 @@ +using DLCS.Core.Types; +using DLCS.Model.Assets; +using DLCS.Repository; +using DLCS.Repository.Assets; +using Microsoft.EntityFrameworkCore; + +namespace CleanupHandler.Repository; + +public class CleanupHandlerAssetRepository : ICleanupHandlerAssetRepository +{ + private readonly DlcsContext dbContext; + + public CleanupHandlerAssetRepository(DlcsContext dbContext) + { + this.dbContext = dbContext; + } + + public async Task RetrieveAssetWithDeliveryChannels(AssetId assetId) + { + return await dbContext.Images.IncludeDeliveryChannelsWithPolicy().SingleOrDefaultAsync(x => x.Id == assetId); + } +} \ No newline at end of file diff --git a/src/protagonist/CleanupHandler/Repository/ICleanupHandlerAssetRepository.cs b/src/protagonist/CleanupHandler/Repository/ICleanupHandlerAssetRepository.cs new file mode 100644 index 000000000..c529ebb29 --- /dev/null +++ b/src/protagonist/CleanupHandler/Repository/ICleanupHandlerAssetRepository.cs @@ -0,0 +1,14 @@ +using DLCS.Core.Types; +using DLCS.Model.Assets; + +namespace CleanupHandler.Repository; + +public interface ICleanupHandlerAssetRepository +{ + /// + /// Retrieves an asset from the database with attached delivery channel policies + /// + /// The asset id to retrieve details for + /// an asset + Task RetrieveAssetWithDeliveryChannels(AssetId assetId); +} \ No newline at end of file diff --git a/src/protagonist/CleanupHandler/appsettings-Development-Example.json b/src/protagonist/CleanupHandler/appsettings-Development-Example.json index 53b88542b..1a72fe240 100644 --- a/src/protagonist/CleanupHandler/appsettings-Development-Example.json +++ b/src/protagonist/CleanupHandler/appsettings-Development-Example.json @@ -6,7 +6,8 @@ "StorageBucket": "storage-bucket" }, "SQS": { - "DeleteNotificationQueueName": "dlcsspinup-delete-notification" + "DeleteNotificationQueueName": "dlcsspinup-delete-notification", + "UpdateNotificationQueueName": "dlcsspinup-update-notification" } }, "ImageFolderTemplate": "/nas/{customer}/{space}/{image-dir}/{image}.jp2" diff --git a/src/protagonist/CleanupHandler/appsettings.json b/src/protagonist/CleanupHandler/appsettings.json index f15f78c74..bead06786 100644 --- a/src/protagonist/CleanupHandler/appsettings.json +++ b/src/protagonist/CleanupHandler/appsettings.json @@ -18,5 +18,14 @@ "Properties": { "ApplicationName": "CleanupHandler" } + }, + "Caching": { + "TimeToLive": { + "Memory": { + "ShortTtlSecs": 60, + "DefaultTtlSecs": 600, + "LongTtlSecs": 1800 + } + } } } \ No newline at end of file diff --git a/src/protagonist/CleanupHandlerTests/AssetUpdatedHandlerTests.cs b/src/protagonist/CleanupHandlerTests/AssetUpdatedHandlerTests.cs new file mode 100644 index 000000000..df5236114 --- /dev/null +++ b/src/protagonist/CleanupHandlerTests/AssetUpdatedHandlerTests.cs @@ -0,0 +1,1226 @@ +using System.Text.Json; +using System.Text.Json.Nodes; +using CleanupHandler; +using CleanupHandler.Infrastructure; +using CleanupHandler.Repository; +using DLCS.AWS.ElasticTranscoder.Models; +using DLCS.AWS.S3; +using DLCS.AWS.S3.Models; +using DLCS.AWS.Settings; +using DLCS.AWS.SQS; +using DLCS.Core.Types; +using DLCS.Model.Assets; +using DLCS.Model.Assets.Metadata; +using DLCS.Model.Messaging; +using DLCS.Model.PathElements; +using DLCS.Model.Policies; +using DLCS.Repository.Messaging; +using FakeItEasy; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Test.Helpers.Integration; + +namespace DeleteHandlerTests; + +public class AssetUpdatedHandlerTests +{ + private readonly CleanupHandlerSettings handlerSettings; + private readonly IBucketWriter bucketWriter; + private readonly IBucketReader bucketReader; + private readonly IStorageKeyGenerator storageKeyGenerator; + private readonly IAssetApplicationMetadataRepository assetMetadataRepository; + private readonly IEngineClient engineClient; + private readonly IThumbRepository thumbRepository; + private readonly ICleanupHandlerAssetRepository cleanupHandlerAssetRepository; + private readonly JsonSerializerOptions settings = new(JsonSerializerDefaults.Web); + + private readonly ImageDeliveryChannel imageDeliveryChannelUseOriginalImage = new() + { + Id = 567587, + Channel = AssetDeliveryChannels.Image, + DeliveryChannelPolicy = new DeliveryChannelPolicy() + { + Id = KnownDeliveryChannelPolicies.ImageUseOriginal, + Channel = AssetDeliveryChannels.Image, + Modified = DateTime.MinValue, + Created = DateTime.MinValue + }, + DeliveryChannelPolicyId = KnownDeliveryChannelPolicies.ImageUseOriginal + }; + + private readonly ImageDeliveryChannel imageDeliveryChannelDefaultImage = new() + { + Id = 58465678, + Channel = AssetDeliveryChannels.Image, + DeliveryChannelPolicy = new DeliveryChannelPolicy() + { + Id = KnownDeliveryChannelPolicies.ImageDefault, + Channel = AssetDeliveryChannels.Image, + Modified = DateTime.MinValue, + Created = DateTime.MinValue + }, + DeliveryChannelPolicyId = KnownDeliveryChannelPolicies.ImageDefault + }; + + private readonly ImageDeliveryChannel imageDeliveryChannelFile = new() + { + Id = 56785678, + Channel = AssetDeliveryChannels.File, + DeliveryChannelPolicy = new DeliveryChannelPolicy() + { + Id = KnownDeliveryChannelPolicies.FileNone, + Channel = AssetDeliveryChannels.File, + Modified = DateTime.MinValue, + Created = DateTime.MinValue + }, + DeliveryChannelPolicyId = KnownDeliveryChannelPolicies.FileNone + }; + + private readonly ImageDeliveryChannel imageDeliveryChannelThumbnail = new() + { + Channel = AssetDeliveryChannels.Thumbnails, + Id = 34256, + DeliveryChannelPolicy = new DeliveryChannelPolicy() + { + Channel = AssetDeliveryChannels.Thumbnails, + Id = KnownDeliveryChannelPolicies.ThumbsDefault, + Created = DateTime.MinValue, + Modified = DateTime.MinValue + }, + DeliveryChannelPolicyId = KnownDeliveryChannelPolicies.ThumbsDefault + }; + + private readonly ImageDeliveryChannel imageDeliveryChannelTimebased = new() + { + Channel = AssetDeliveryChannels.Timebased, + Id = 356367, + DeliveryChannelPolicy = new DeliveryChannelPolicy() + { + Channel = AssetDeliveryChannels.Timebased, + Created = DateTime.MinValue, + Modified = DateTime.MinValue + }, + DeliveryChannelPolicyId = KnownDeliveryChannelPolicies.AvDefaultVideo + }; + + public AssetUpdatedHandlerTests() + { + handlerSettings = new CleanupHandlerSettings + { + AWS = new AWSSettings + { + S3 = new S3Settings + { + StorageBucket = LocalStackFixture.StorageBucketName, + ThumbsBucket = LocalStackFixture.ThumbsBucketName, + OriginBucket = LocalStackFixture.OriginBucketName + } + }, + ImageFolderTemplate = "/nas/{customer}/{space}/{image-dir}/{image}.jp2", + AssetModifiedSettings = new AssetModifiedSettings() + { + DryRun = false + } + }; + storageKeyGenerator = new S3StorageKeyGenerator(Options.Create(handlerSettings.AWS)); + bucketWriter = A.Fake(); + bucketReader = A.Fake(); + engineClient = A.Fake(); + assetMetadataRepository = A.Fake(); + thumbRepository = A.Fake(); + cleanupHandlerAssetRepository = A.Fake(); + + A.CallTo(() => thumbRepository.GetAllSizes(A._)).Returns(new List() + { + new[] + { + 50, 100 + }, + new[] + { + 100, 200 + }, + new[] + { + 200, 400 + }, + new[] + { + 516, 1024 + } + }); + } + + private AssetUpdatedHandler GetSut() + => new(storageKeyGenerator, bucketWriter, bucketReader, assetMetadataRepository, thumbRepository, + Options.Create(handlerSettings), engineClient, cleanupHandlerAssetRepository, + new NullLogger()); + + [Fact] + public async Task Handle_ReturnsFalse_IfInvalidFormat() + { + // Arrange + var queueMessage = new QueueMessage + { + Body = new JsonObject { ["not-id"] = "foo" } + }; + + // Act + var sut = GetSut(); + var response = await sut.HandleMessage(queueMessage); + + // Assert + response.Should().BeFalse(); + A.CallTo(() => bucketWriter.DeleteFromBucket(A._)).MustNotHaveHappened(); + A.CallTo(() => bucketWriter.DeleteFolder(A._, A._)).MustNotHaveHappened(); + } + + // removed + + [Fact] + public async Task Handle_DeletesOriginal_WhenFileChannelRemoved() + { + // Arrange + var requestDetails = CreateMinimalRequestDetails(new List() { imageDeliveryChannelFile }, + new List(), + string.Empty, string.Empty); + + A.CallTo(() => cleanupHandlerAssetRepository.RetrieveAssetWithDeliveryChannels(A._)) + .Returns(requestDetails.assetAfter); + + // Act + var sut = GetSut(); + var response = await sut.HandleMessage(requestDetails.queueMessage); + + // Assert + response.Should().BeTrue(); + A.CallTo(() => + bucketWriter.DeleteFromBucket(A.That.Matches(o => o[0].Key == "1/99/foo/original"))) + .MustHaveHappened(); + A.CallTo(() => bucketWriter.DeleteFolder(A._, A._)).MustNotHaveHappened(); + } + + [Fact] + public async Task Handle_DoesNotDeleteOriginal_WhenFileChannelRemovedWithUseOriginalPolicy() + { + // Arrange + var imageDeliveryChannelsBefore = new List + { + imageDeliveryChannelFile, + imageDeliveryChannelUseOriginalImage + }; + + var requestDetails = CreateMinimalRequestDetails(imageDeliveryChannelsBefore, + new List() { imageDeliveryChannelUseOriginalImage }, + string.Empty, string.Empty); + + A.CallTo(() => cleanupHandlerAssetRepository.RetrieveAssetWithDeliveryChannels(A._)) + .Returns(requestDetails.assetAfter); + + // Act + var sut = GetSut(); + var response = await sut.HandleMessage(requestDetails.queueMessage); + + // Assert + response.Should().BeTrue(); + A.CallTo(() => bucketWriter.DeleteFromBucket(A._)).MustNotHaveHappened(); + A.CallTo(() => bucketWriter.DeleteFolder(A._, A._)).MustNotHaveHappened(); + } + + [Fact] + public async Task Handle_DeletesTimebasedAssets_WhenTimebasedChannelRemoved() + { + // Arrange + var requestDetails = CreateMinimalRequestDetails( + new List() { imageDeliveryChannelTimebased }, new List(), + string.Empty, string.Empty, "video/mp3"); + + A.CallTo(() => cleanupHandlerAssetRepository.RetrieveAssetWithDeliveryChannels(A._)) + .Returns(requestDetails.assetAfter); + + A.CallTo(() => bucketReader.GetMatchingKeys(A._)) + .Returns(new []{ "1/99/foo/full/full/max/max/0/default.mp4", "1/99/foo/some/other/key" }); + + // Act + var sut = GetSut(); + var response = await sut.HandleMessage(requestDetails.queueMessage); + + // Assert + response.Should().BeTrue(); + A.CallTo(() => + bucketWriter.DeleteFromBucket(A.That.Matches(o => o[0].Key == "1/99/foo/metadata"))) + .MustHaveHappened(); + A.CallTo(() => + bucketWriter.DeleteFromBucket( + A.That.Matches(o => o[1].Key == "1/99/foo/full/full/max/max/0/default.mp4"))) + .MustHaveHappened(); + A.CallTo(() => + bucketWriter.DeleteFromBucket( + A.That.Matches(o => o.Any(x => x.Key == "1/99/foo/some/other/key")))) + .MustNotHaveHappened(); + A.CallTo(() => bucketWriter.DeleteFolder(A._, A._)).MustNotHaveHappened(); + } + + [Fact] + public async Task Handle_DeletesThumbnailAssets_WhenThumbnailChannelRemoved() + { + // Arrange + var requestDetails = CreateMinimalRequestDetails( + new List() { imageDeliveryChannelThumbnail }, new List(), + string.Empty, string.Empty); + + A.CallTo(() => cleanupHandlerAssetRepository.RetrieveAssetWithDeliveryChannels(A._)) + .Returns(requestDetails.assetAfter); + A.CallTo(() => + assetMetadataRepository.DeleteAssetApplicationMetadata(A._, A._, + A._)) + .Returns(true); + A.CallTo(() => bucketReader.GetMatchingKeys(A._)) + .Returns(new[] + { + "1/99/foo/stuff/100.jpg", "1/99/foo/stuff/200.jpg", "1/99/foo/stuff/400.jpg", "1/99/foo/stuff/1024.jpg" + }); + + // Act + var sut = GetSut(); + var response = await sut.HandleMessage(requestDetails.queueMessage); + + // Assert + response.Should().BeTrue(); + A.CallTo(() => assetMetadataRepository.DeleteAssetApplicationMetadata(A._, A._, A._)).MustHaveHappened(); + A.CallTo(() => + bucketWriter.DeleteFolder( + A.That.Matches(o => + o.Key == "1/99/foo/" && o.Bucket == handlerSettings.AWS.S3.ThumbsBucket), + A.That.Matches(r => r == true))).MustHaveHappened(); + } + + [Fact] + public async Task Handle_DeletesSomeThumbnailAssets_WhenThumbnailChannelRemovedWithImageChannel() + { + // Arrange + var imageDeliveryChannelsBefore = new List + { + imageDeliveryChannelThumbnail, + imageDeliveryChannelUseOriginalImage + }; + + var requestDetails = CreateMinimalRequestDetails(imageDeliveryChannelsBefore, + new List() { imageDeliveryChannelUseOriginalImage }, + string.Empty, string.Empty); + + A.CallTo(() => cleanupHandlerAssetRepository.RetrieveAssetWithDeliveryChannels(A._)) + .Returns(requestDetails.assetAfter); + A.CallTo(() => + assetMetadataRepository.DeleteAssetApplicationMetadata(A._, A._, + A._)) + .Returns(true); + A.CallTo(() => bucketReader.GetMatchingKeys(A._)) + .Returns(new[] + { + "1/99/foo/stuff/100.jpg", "1/99/foo/stuff/200.jpg", "1/99/foo/stuff/400.jpg", "1/99/foo/stuff/1024.jpg", + "1/99/foo/stuff/2048.jpg" + }); + + // Act + var sut = GetSut(); + var response = await sut.HandleMessage(requestDetails.queueMessage); + + // Assert + response.Should().BeTrue(); + A.CallTo(() => + bucketWriter.DeleteFromBucket( + A.That.Matches(o => + o[0].Key == "1/99/foo/stuff/2048.jpg" && + o[0].Bucket == handlerSettings.AWS.S3.ThumbsBucket))) + .MustHaveHappened(); + A.CallTo(() => bucketWriter.DeleteFolder(A._, A._)).MustNotHaveHappened(); + } + + [Fact] + public async Task Handle_DeletesSomePortraitThumbnailAssets_WhenThumbnailChannelRemovedWithImageChannel() + { + // Arrange + var imageDeliveryChannelsBefore = new List + { + imageDeliveryChannelThumbnail, + imageDeliveryChannelUseOriginalImage + }; + + var requestDetails = CreateMinimalRequestDetails(imageDeliveryChannelsBefore, + new List() { imageDeliveryChannelUseOriginalImage }, + string.Empty, string.Empty); + + A.CallTo(() => cleanupHandlerAssetRepository.RetrieveAssetWithDeliveryChannels(A._)) + .Returns(requestDetails.assetAfter); + A.CallTo(() => + assetMetadataRepository.DeleteAssetApplicationMetadata(A._, A._, + A._)) + .Returns(true); + A.CallTo(() => bucketReader.GetMatchingKeys(A._)) + .Returns(new[] + { + "1/99/foo/stuff/100.jpg", "1/99/foo/stuff/200.jpg", "1/99/foo/stuff/400.jpg", "1/99/foo/stuff/1024.jpg", + "1/99/foo/stuff/2048.jpg" + }); + + A.CallTo(() => thumbRepository.GetAllSizes(A._)).Returns(new List() + { + new[] + { + 100, 50 + + }, + new[] + { + 200, 100 + + }, + new[] + { + 400, 200 + }, + new[] + { + 1024, 516 + } + }); + + // Act + var sut = GetSut(); + var response = await sut.HandleMessage(requestDetails.queueMessage); + + // Assert + response.Should().BeTrue(); + A.CallTo(() => + bucketWriter.DeleteFromBucket( + A.That.Matches(o => + o[0].Key == "1/99/foo/stuff/2048.jpg" && + o[0].Bucket == handlerSettings.AWS.S3.ThumbsBucket))) + .MustHaveHappened(); + A.CallTo(() => bucketWriter.DeleteFolder(A._, A._)).MustNotHaveHappened(); + } + + [Fact] + public async Task Handle_DeletesValidPaths_WhenImageChannelRemoved() + { + // Arrange + var requestDetails = CreateMinimalRequestDetails( + new List() { imageDeliveryChannelUseOriginalImage }, new List(), + string.Empty, string.Empty); + + A.CallTo(() => cleanupHandlerAssetRepository.RetrieveAssetWithDeliveryChannels(A._)) + .Returns(requestDetails.assetAfter); + + // Act + var sut = GetSut(); + var response = await sut.HandleMessage(requestDetails.queueMessage); + + // Assert + response.Should().BeTrue(); + A.CallTo(() => + bucketWriter.DeleteFromBucket(A.That.Matches(o => o[0].Key == "1/99/foo"))) + .MustHaveHappened(); + A.CallTo(() => + bucketWriter.DeleteFromBucket( + A.That.Matches(o => o[1].Key == "1/99/foo/original"))) + .MustHaveHappened(); + A.CallTo(() => bucketWriter.DeleteFolder(A._, A._)).MustNotHaveHappened(); + } + + [Fact] + public async Task Handle_DoesNotDeleteOriginal_WhenImageChannelRemovedWithFileLeft() + { + // Arrange + var imageDeliveryChannelsBefore = new List + { + imageDeliveryChannelUseOriginalImage, + imageDeliveryChannelFile + }; + + var requestDetails = CreateMinimalRequestDetails(imageDeliveryChannelsBefore, new List() { imageDeliveryChannelFile }, + string.Empty, string.Empty); + + A.CallTo(() => cleanupHandlerAssetRepository.RetrieveAssetWithDeliveryChannels(A._)) + .Returns(requestDetails.assetAfter); + + // Act + var sut = GetSut(); + var response = await sut.HandleMessage(requestDetails.queueMessage); + + // Assert + response.Should().BeTrue(); + A.CallTo(() => + bucketWriter.DeleteFromBucket(A.That.Matches(o => o[0].Key == "1/99/foo"))) + .MustHaveHappened(); + A.CallTo(() => + bucketWriter.DeleteFromBucket( + A.That.Matches(o => o.Any(o => o.Key == "1/99/foo/original")))) + .MustNotHaveHappened(); + A.CallTo(() => bucketWriter.DeleteFolder(A._, A._)).MustNotHaveHappened(); + } + + // modified + + [Fact] + public async Task Handle_DoesNotRemoveAnything_WhenFileChannelModified() + { + // Arrange + var fileDeliveryChannelAfter = new ImageDeliveryChannel() + { + Id = 34512245, + Channel = AssetDeliveryChannels.File, + DeliveryChannelPolicy = new DeliveryChannelPolicy() + { + Id = 34534, + Channel = AssetDeliveryChannels.File, + Modified = DateTime.MinValue, + Created = DateTime.MinValue, + Name = "some-delivery-channel" + }, + DeliveryChannelPolicyId = 34534 + }; + + var requestDetails = CreateMinimalRequestDetails(new List { imageDeliveryChannelFile }, + new List() { fileDeliveryChannelAfter }, + string.Empty, string.Empty); + + A.CallTo(() => cleanupHandlerAssetRepository.RetrieveAssetWithDeliveryChannels(A._)) + .Returns(requestDetails.assetAfter); + + // Act + var sut = GetSut(); + var response = await sut.HandleMessage(requestDetails.queueMessage); + + // Assert + response.Should().BeTrue(); + A.CallTo(() => bucketWriter.DeleteFromBucket(A._)).MustNotHaveHappened(); + A.CallTo(() => bucketWriter.DeleteFolder(A._, A._)).MustNotHaveHappened(); + } + + [Fact] + public async Task Handle_DeletesUnneededTimebasedAssets_WhenTimebasedChannelModified() + { + // Arrange + var imageDeliveryChannelAfter = new ImageDeliveryChannel() + { + Channel = AssetDeliveryChannels.Timebased, + Id = 152445, + DeliveryChannelPolicy = new DeliveryChannelPolicy() + { + Id = 8239, + Channel = AssetDeliveryChannels.Timebased, + Created = DateTime.MinValue, + Modified = DateTime.MinValue, + PolicyData = "[\"webm-policy\", \"oga-policy\"]" + }, + DeliveryChannelPolicyId = 8239 + }; + + var requestDetails = CreateMinimalRequestDetails( + new List() { imageDeliveryChannelTimebased }, + new List() { imageDeliveryChannelAfter }, + string.Empty, string.Empty, "video/*"); + + A.CallTo(() => cleanupHandlerAssetRepository.RetrieveAssetWithDeliveryChannels(A._)) + .Returns(requestDetails.assetAfter); + + A.CallTo(() => engineClient.GetAvPresets(A._)).Returns(new Dictionary() + { + { "webm-policy", new ("", "some-webm-preset", "oga") }, + { "oga-policy", new ("", "some-oga-preset", "webm") } + }); + + A.CallTo(() => bucketReader.GetMatchingKeys(A._)) + .Returns(new []{ "1/99/foo/full/full/max/max/0/default.mp4", "1/99/foo/some/other/key", "1/99/foo/full/full/max/max/0/default.webm", "1/99/foo/full/full/max/max/0/default.oga" }); + + // Act + var sut = GetSut(); + var response = await sut.HandleMessage(requestDetails.queueMessage); + + // Assert + response.Should().BeTrue(); + A.CallTo(() => + bucketWriter.DeleteFromBucket(A.That.Matches(o => o[0].Key == "1/99/foo/full/full/max/max/0/default.mp4"))) + .MustHaveHappened(); + A.CallTo(() => + bucketWriter.DeleteFromBucket( + A.That.Matches(o => o.Any(x => x.Key == "1/99/foo/full/full/max/max/0/default.webm")))) + .MustNotHaveHappened(); + A.CallTo(() => + bucketWriter.DeleteFromBucket( + A.That.Matches(o => o.Any(x => x.Key == "1/99/foo/some/other/key")))) + .MustNotHaveHappened(); + A.CallTo(() => + bucketWriter.DeleteFromBucket( + A.That.Matches(o => o.Any(x => x.Key == "1/99/foo/full/full/max/max/0/default.oga")))) + .MustNotHaveHappened(); + A.CallTo(() => bucketWriter.DeleteFolder(A._, A._)).MustNotHaveHappened(); + } + + [Fact] + public async Task Handle_DoesNothing_WhenTimebasedChannelUpdatedWithInvalidPreset() + { + // Arrange + var imageDeliveryChannelAfter = new ImageDeliveryChannel() + { + Channel = AssetDeliveryChannels.Timebased, + Id = 23456, + DeliveryChannelPolicy = new DeliveryChannelPolicy() + { + Id = 8239, + Channel = AssetDeliveryChannels.Timebased, + Created = DateTime.MinValue, + Modified = DateTime.MinValue, + PolicyData = "[\"policy-not-found\"]" + }, + DeliveryChannelPolicyId = 8239 + }; + + A.CallTo(() => engineClient.GetAvPresets(A._)).Returns(new Dictionary() + { + { "webm-policy", new ("", "some-webm-preset", "oga") }, + { "oga-policy", new ("", "some-oga-preset", "webm") } + }); + + var requestDetails = CreateMinimalRequestDetails( + new List() { imageDeliveryChannelTimebased }, + new List() { imageDeliveryChannelAfter }, + string.Empty, string.Empty, "video/*"); + + A.CallTo(() => cleanupHandlerAssetRepository.RetrieveAssetWithDeliveryChannels(A._)) + .Returns(requestDetails.assetAfter); + + // Act + var sut = GetSut(); + var response = await sut.HandleMessage(requestDetails.queueMessage); + + // Assert + response.Should().BeTrue(); + A.CallTo(() => bucketWriter.DeleteFromBucket(A._)).MustNotHaveHappened(); + A.CallTo(() => bucketWriter.DeleteFolder(A._, A._)).MustNotHaveHappened(); + } + + [Fact] + public async Task Handle_ReturnsFalse_WhenTimebasedChannelUpdatedWithNoAvPresets() + { + // Arrange + var imageDeliveryChannelAfter = new ImageDeliveryChannel() + { + Channel = AssetDeliveryChannels.Timebased, + Id = 23456, + DeliveryChannelPolicy = new DeliveryChannelPolicy() + { + Id = 8239, + Channel = AssetDeliveryChannels.Timebased, + Created = DateTime.MinValue, + Modified = DateTime.MinValue, + PolicyData = "[\"policy-not-found\"]" + }, + DeliveryChannelPolicyId = 8239 + }; + + var requestDetails = CreateMinimalRequestDetails( + new List() { imageDeliveryChannelTimebased }, + new List() { imageDeliveryChannelAfter }, + string.Empty, string.Empty, "video/*"); + + A.CallTo(() => cleanupHandlerAssetRepository.RetrieveAssetWithDeliveryChannels(A._)) + .Returns(requestDetails.assetAfter); + + // Act + var sut = GetSut(); + var response = await sut.HandleMessage(requestDetails.queueMessage); + + // Assert + response.Should().BeFalse(); + A.CallTo(() => bucketWriter.DeleteFromBucket(A._)).MustNotHaveHappened(); + A.CallTo(() => bucketWriter.DeleteFolder(A._, A._)).MustNotHaveHappened(); + } + + [Fact] + public async Task Handle_DoesNothing_WhenTimebasedChannelModfiedWithInvalidPresetDetails() + { + // Arrange + var imageDeliveryChannelAfter = new ImageDeliveryChannel() + { + Channel = AssetDeliveryChannels.Timebased, + Id = 345634, + DeliveryChannelPolicy = new DeliveryChannelPolicy() + { + Channel = AssetDeliveryChannels.Timebased, + Id = 8239, + Created = DateTime.MinValue, + Modified = DateTime.MinValue, + PolicyData = "[\"some-policy\"]" + }, + DeliveryChannelPolicyId = 8239 + }; + + var requestDetails = CreateMinimalRequestDetails( + new List() { imageDeliveryChannelTimebased }, + new List() { imageDeliveryChannelAfter }, + string.Empty, string.Empty, "video/*"); + + A.CallTo(() => cleanupHandlerAssetRepository.RetrieveAssetWithDeliveryChannels(A._)) + .Returns(requestDetails.assetAfter); + + A.CallTo(() => engineClient.GetAvPresets(A._)).Returns(new Dictionary() + { + { "some-policy", new ("", "some-transcode-preset", "") } + }); + + // Act + var sut = GetSut(); + var response = await sut.HandleMessage(requestDetails.queueMessage); + + // Assert + response.Should().BeTrue(); + A.CallTo(() => bucketWriter.DeleteFromBucket(A._)).MustNotHaveHappened(); + A.CallTo(() => bucketWriter.DeleteFolder(A._, A._)).MustNotHaveHappened(); + } + + [Fact] + public async Task Handle_DeletesSomeThumbnailAssets_WhenThumbnailChannelModifed() + { + // Arrange + var imageDeliveryChannelsAfter = new List + { + new () + { + Channel = AssetDeliveryChannels.Thumbnails, + Id = 42356, + DeliveryChannelPolicy = new DeliveryChannelPolicy() + { + Id = 35467, + Channel = AssetDeliveryChannels.Thumbnails, + Created = DateTime.MinValue, + Modified = DateTime.MinValue + }, + DeliveryChannelPolicyId = 35467 + } + }; + + var requestDetails = CreateMinimalRequestDetails( + new List { imageDeliveryChannelThumbnail }, imageDeliveryChannelsAfter, + string.Empty, string.Empty); + + A.CallTo(() => cleanupHandlerAssetRepository.RetrieveAssetWithDeliveryChannels(A._)) + .Returns(requestDetails.assetAfter); + A.CallTo(() => + assetMetadataRepository.DeleteAssetApplicationMetadata(A._, A._, + A._)) + .Returns(true); + A.CallTo(() => bucketReader.GetMatchingKeys(A._)) + .Returns(new[] + { + "1/99/foo/stuff/100.jpg", "1/99/foo/stuff/200.jpg", "1/99/foo/stuff/400.jpg", "1/99/foo/stuff/1024.jpg", + "1/99/foo/stuff/2048.jpg" , "1/99/full/100,200/0/default.jpg" + }); + + // Act + var sut = GetSut(); + var response = await sut.HandleMessage(requestDetails.queueMessage); + + // Assert + response.Should().BeTrue(); + A.CallTo(() => + bucketWriter.DeleteFromBucket( + A.That.Matches(o => + o[0].Key == "1/99/foo/stuff/2048.jpg" && + o[0].Bucket == handlerSettings.AWS.S3.ThumbsBucket))) + .MustHaveHappened(); + A.CallTo(() => + bucketWriter.DeleteFromBucket( + A.That.Matches(o => + o[1].Key == "1/99/full/100,200/0/default.jpg" && + o[1].Bucket == handlerSettings.AWS.S3.ThumbsBucket))) + .MustHaveHappened(); + A.CallTo(() => + bucketWriter.DeleteFromBucket( + A.That.Matches(o => o.Any(x => x.Key == "1/99/foo/stuff/200.jpg")))) + .MustNotHaveHappened(); + A.CallTo(() => bucketWriter.DeleteFolder(A._, A._)).MustNotHaveHappened(); + } + + [Fact] + public async Task Handle_DeletesValidPaths_WhenImageChannelUpdatedToDefault() + { + // Arrange + var requestDetails = CreateMinimalRequestDetails( + new List { imageDeliveryChannelUseOriginalImage }, new List { imageDeliveryChannelDefaultImage }, + string.Empty, string.Empty); + + A.CallTo(() => cleanupHandlerAssetRepository.RetrieveAssetWithDeliveryChannels(A._)) + .Returns(requestDetails.assetAfter); + + // Act + var sut = GetSut(); + var response = await sut.HandleMessage(requestDetails.queueMessage); + + // Assert + response.Should().BeTrue(); + A.CallTo(() => + bucketWriter.DeleteFromBucket(A.That.Matches(o => o[0].Key == "1/99/foo/original"))) + .MustHaveHappened(); + A.CallTo(() => bucketWriter.DeleteFolder(A._, A._)).MustNotHaveHappened(); + } + + [Fact] + public async Task Handle_DeletesValidPaths_WhenImageChannelUpdatedToUseOriginal() + { + // Arrange + var requestDetails = CreateMinimalRequestDetails( + new List { imageDeliveryChannelDefaultImage }, new List { imageDeliveryChannelUseOriginalImage }, + string.Empty, string.Empty); + + A.CallTo(() => cleanupHandlerAssetRepository.RetrieveAssetWithDeliveryChannels(A._)) + .Returns(requestDetails.assetAfter); + + // Act + var sut = GetSut(); + var response = await sut.HandleMessage(requestDetails.queueMessage); + + // Assert + response.Should().BeTrue(); + A.CallTo(() => + bucketWriter.DeleteFromBucket(A.That.Matches(o => o[0].Key == "1/99/foo"))) + .MustHaveHappened(); + A.CallTo(() => bucketWriter.DeleteFolder(A._, A._)).MustNotHaveHappened(); + } + + [Fact] + public async Task Handle_DoesNotDeleteAnything_WhenImageChannelUpdatedToUseDefaultWithFileChannel() + { + // Arrange + var requestDetails = CreateMinimalRequestDetails( + new List { imageDeliveryChannelUseOriginalImage }, + new List { imageDeliveryChannelDefaultImage, imageDeliveryChannelFile }, + string.Empty, string.Empty); + + A.CallTo(() => cleanupHandlerAssetRepository.RetrieveAssetWithDeliveryChannels(A._)) + .Returns(requestDetails.assetAfter); + + // Act + var sut = GetSut(); + var response = await sut.HandleMessage(requestDetails.queueMessage); + + // Assert + response.Should().BeTrue(); + A.CallTo(() => + bucketWriter.DeleteFromBucket(A._)).MustNotHaveHappened(); + A.CallTo(() => bucketWriter.DeleteFolder(A._, A._)).MustNotHaveHappened(); + } + + // updated + + [Fact] + public async Task Handle_DoesNotRemoveAnything_WhenFilePolicyUpdated() + { + // Arrange + var fileDeliveryChannelAfter = new ImageDeliveryChannel() + { + Id = 56785678, + Channel = AssetDeliveryChannels.File, + DeliveryChannelPolicy = new DeliveryChannelPolicy() + { + Id = KnownDeliveryChannelPolicies.FileNone, + Channel = AssetDeliveryChannels.File, + Modified = DateTime.MaxValue, + Created = DateTime.MaxValue, + Name = "some-delivery-channel" + }, + DeliveryChannelPolicyId = KnownDeliveryChannelPolicies.FileNone + }; + + var requestDetails = CreateMinimalRequestDetails(new List { imageDeliveryChannelFile }, + new List() { fileDeliveryChannelAfter }, + string.Empty, string.Empty); + + A.CallTo(() => cleanupHandlerAssetRepository.RetrieveAssetWithDeliveryChannels(A._)) + .Returns(requestDetails.assetAfter); + + // Act + var sut = GetSut(); + var response = await sut.HandleMessage(requestDetails.queueMessage); + + // Assert + response.Should().BeTrue(); + A.CallTo(() => bucketWriter.DeleteFromBucket(A._)).MustNotHaveHappened(); + A.CallTo(() => bucketWriter.DeleteFolder(A._, A._)).MustNotHaveHappened(); + } + + [Fact] + public async Task Handle_DeletesUnneededTimebasedAssets_WhenTimebasedChannelUpdated() + { + // Arrange + var imageDeliveryChannelAfter = new ImageDeliveryChannel() + { + Channel = AssetDeliveryChannels.Timebased, + Id = 356367, + DeliveryChannelPolicy = new DeliveryChannelPolicy() + { + Id = KnownDeliveryChannelPolicies.AvDefaultVideo, + Channel = AssetDeliveryChannels.Timebased, + Created = DateTime.MinValue, + Modified = DateTime.MaxValue, + PolicyData = "[\"webm-policy\", \"oga-policy\"]" + }, + DeliveryChannelPolicyId = KnownDeliveryChannelPolicies.AvDefaultVideo + }; + + var requestDetails = CreateMinimalRequestDetails( + new List() { imageDeliveryChannelTimebased }, + new List() { imageDeliveryChannelAfter }, + string.Empty, string.Empty, "video/*"); + + A.CallTo(() => cleanupHandlerAssetRepository.RetrieveAssetWithDeliveryChannels(A._)) + .Returns(requestDetails.assetAfter); + + A.CallTo(() => engineClient.GetAvPresets(A._)).Returns(new Dictionary() + { + { "webm-policy", new ("", "some-webm-preset", "oga") }, + { "oga-policy", new ("", "some-oga-preset", "webm") } + }); + + A.CallTo(() => bucketReader.GetMatchingKeys(A._)) + .Returns(new []{ "1/99/foo/full/full/max/max/0/default.mp4", "1/99/foo/some/other/key", "1/99/foo/full/full/max/max/0/default.webm", "1/99/foo/full/full/max/max/0/default.oga" }); + + // Act + var sut = GetSut(); + var response = await sut.HandleMessage(requestDetails.queueMessage); + + // Assert + response.Should().BeTrue(); + A.CallTo(() => + bucketWriter.DeleteFromBucket(A.That.Matches(o => o[0].Key == "1/99/foo/full/full/max/max/0/default.mp4"))) + .MustHaveHappened(); + A.CallTo(() => + bucketWriter.DeleteFromBucket( + A.That.Matches(o => o.Any(x => x.Key == "1/99/foo/full/full/max/max/0/default.webm")))) + .MustNotHaveHappened(); + A.CallTo(() => + bucketWriter.DeleteFromBucket( + A.That.Matches(o => o.Any(x => x.Key == "1/99/foo/some/other/key")))) + .MustNotHaveHappened(); + A.CallTo(() => + bucketWriter.DeleteFromBucket( + A.That.Matches(o => o.Any(x => x.Key == "1/99/foo/full/full/max/max/0/default.oga")))) + .MustNotHaveHappened(); + A.CallTo(() => bucketWriter.DeleteFolder(A._, A._)).MustNotHaveHappened(); + } + + [Fact] + public async Task Handle_DoesNothing_WhenTimebasedPolicyUpdatedWithInvalidPreset() + { + // Arrange + var imageDeliveryChannelAfter = new ImageDeliveryChannel() + { + Channel = AssetDeliveryChannels.Timebased, + Id = 356367, + DeliveryChannelPolicy = new DeliveryChannelPolicy() + { + Id = KnownDeliveryChannelPolicies.AvDefaultVideo, + Channel = AssetDeliveryChannels.Timebased, + Created = DateTime.MinValue, + Modified = DateTime.MaxValue, + PolicyData = "[\"policy-not-found\"]" + }, + DeliveryChannelPolicyId = KnownDeliveryChannelPolicies.AvDefaultVideo + }; + + var requestDetails = CreateMinimalRequestDetails( + new List() { imageDeliveryChannelTimebased }, + new List() { imageDeliveryChannelAfter }, + string.Empty, string.Empty, "video/*"); + + A.CallTo(() => cleanupHandlerAssetRepository.RetrieveAssetWithDeliveryChannels(A._)) + .Returns(requestDetails.assetAfter); + + A.CallTo(() => engineClient.GetAvPresets(A._)).Returns(new Dictionary() + { + { "some-policy", new ("some-transcode-preset", "", "") } + }); + + // Act + var sut = GetSut(); + var response = await sut.HandleMessage(requestDetails.queueMessage); + + // Assert + response.Should().BeTrue(); + A.CallTo(() => bucketWriter.DeleteFromBucket(A._)).MustNotHaveHappened(); + A.CallTo(() => bucketWriter.DeleteFolder(A._, A._)).MustNotHaveHappened(); + } + + [Fact] + public async Task Handle_DoesNothing_WhenTimebasedPolicyUpdatedWithInvalidPresetDetails() + { + // Arrange + var imageDeliveryChannelAfter = new ImageDeliveryChannel() + { + Channel = AssetDeliveryChannels.Timebased, + Id = 356367, + DeliveryChannelPolicy = new DeliveryChannelPolicy() + { + Channel = AssetDeliveryChannels.Timebased, + Id = KnownDeliveryChannelPolicies.AvDefaultVideo, + Created = DateTime.MinValue, + Modified = DateTime.MaxValue, + PolicyData = "[\"some-policy\"]" + }, + DeliveryChannelPolicyId = KnownDeliveryChannelPolicies.AvDefaultVideo + }; + + var requestDetails = CreateMinimalRequestDetails( + new List() { imageDeliveryChannelTimebased }, + new List() { imageDeliveryChannelAfter }, + string.Empty, string.Empty, "video/*"); + + A.CallTo(() => cleanupHandlerAssetRepository.RetrieveAssetWithDeliveryChannels(A._)) + .Returns(requestDetails.assetAfter); + + A.CallTo(() => engineClient.GetAvPresets(A._)).Returns(new Dictionary() + { + { "some-policy", new ("", "some-transcode-preset", "") } + }); + + // Act + var sut = GetSut(); + var response = await sut.HandleMessage(requestDetails.queueMessage); + + // Assert + response.Should().BeTrue(); + A.CallTo(() => bucketWriter.DeleteFromBucket(A._)).MustNotHaveHappened(); + A.CallTo(() => bucketWriter.DeleteFolder(A._, A._)).MustNotHaveHappened(); + } + + [Fact] + public async Task Handle_DeletesSomeThumbnailAssets_WhenThumbnailPolicyUpdated() + { + // Arrange + var imageDeliveryChannelsAfter = new List + { + new () + { + Channel = AssetDeliveryChannels.Thumbnails, + Id = 356367, + DeliveryChannelPolicy = new DeliveryChannelPolicy + { + Id = KnownDeliveryChannelPolicies.AvDefaultVideo, + Channel = AssetDeliveryChannels.Thumbnails, + Created = DateTime.MinValue, + Modified = DateTime.MaxValue + }, + DeliveryChannelPolicyId = KnownDeliveryChannelPolicies.AvDefaultVideo + } + }; + + var requestDetails = CreateMinimalRequestDetails( + new List { imageDeliveryChannelThumbnail }, imageDeliveryChannelsAfter, + string.Empty, string.Empty); + + A.CallTo(() => cleanupHandlerAssetRepository.RetrieveAssetWithDeliveryChannels(A._)) + .Returns(requestDetails.assetAfter); + A.CallTo(() => + assetMetadataRepository.DeleteAssetApplicationMetadata(A._, A._, + A._)) + .Returns(true); + A.CallTo(() => bucketReader.GetMatchingKeys(A._)) + .Returns(new[] + { + "1/99/foo/stuff/100.jpg", "1/99/foo/stuff/200.jpg", "1/99/foo/stuff/400.jpg", "1/99/foo/stuff/1024.jpg", + "1/99/foo/stuff/2048.jpg", "1/99/full/100,200/0/default.jpg" + }); + + // Act + var sut = GetSut(); + var response = await sut.HandleMessage(requestDetails.queueMessage); + + // Assert + response.Should().BeTrue(); + A.CallTo(() => + bucketWriter.DeleteFromBucket( + A.That.Matches(o => + o[0].Key == "1/99/foo/stuff/2048.jpg" && + o[0].Bucket == handlerSettings.AWS.S3.ThumbsBucket))) + .MustHaveHappened(); + A.CallTo(() => + bucketWriter.DeleteFromBucket( + A.That.Matches(o => + o[1].Key == "1/99/full/100,200/0/default.jpg" && + o[1].Bucket == handlerSettings.AWS.S3.ThumbsBucket))) + .MustHaveHappened(); + A.CallTo(() => + bucketWriter.DeleteFromBucket( + A.That.Matches(o => o.Any(x => x.Key == "1/99/foo/stuff/200.jpg")))) + .MustNotHaveHappened(); + A.CallTo(() => bucketWriter.DeleteFolder(A._, A._)).MustNotHaveHappened(); + } + + [Fact] + public async Task Handle_DeletesValidPaths_WhenImageChannelIsUpdatedDefaultImagePolicy() + { + // Arrange + var imageDeliveryChannelDefaultUpdated = new ImageDeliveryChannel() + { + Id = 58465678, + Channel = AssetDeliveryChannels.Image, + DeliveryChannelPolicy = new DeliveryChannelPolicy() + { + Id = KnownDeliveryChannelPolicies.ImageDefault, + Channel = AssetDeliveryChannels.Image, + Created = DateTime.MinValue, + Modified = DateTime.MaxValue + }, + DeliveryChannelPolicyId = KnownDeliveryChannelPolicies.ImageDefault + }; + + var requestDetails = CreateMinimalRequestDetails( + new List { imageDeliveryChannelDefaultImage }, new List { imageDeliveryChannelDefaultUpdated }, + string.Empty, string.Empty); + + A.CallTo(() => cleanupHandlerAssetRepository.RetrieveAssetWithDeliveryChannels(A._)) + .Returns(requestDetails.assetAfter); + + // Act + var sut = GetSut(); + var response = await sut.HandleMessage(requestDetails.queueMessage); + + // Assert + response.Should().BeTrue(); + A.CallTo(() => + bucketWriter.DeleteFromBucket(A.That.Matches(o => o[0].Key == "1/99/foo/original"))) + .MustHaveHappened(); + A.CallTo(() => bucketWriter.DeleteFolder(A._, A._)).MustNotHaveHappened(); + } + + [Fact] + public async Task Handle_DeletesValidPaths_WhenImageChannelHasUpdatedUseOrginalPolicy() + { + // Arrange + var imageDeliveryChannelUseOriginalUpdated = new ImageDeliveryChannel() + { + Id = 567587, + Channel = AssetDeliveryChannels.Image, + DeliveryChannelPolicy = new DeliveryChannelPolicy() + { + Id = KnownDeliveryChannelPolicies.ImageUseOriginal, + Channel = AssetDeliveryChannels.Image, + Created = DateTime.MinValue, + Modified = DateTime.MaxValue + }, + DeliveryChannelPolicyId = KnownDeliveryChannelPolicies.ImageUseOriginal + }; + + var requestDetails = CreateMinimalRequestDetails( + new List { imageDeliveryChannelUseOriginalImage }, new List { imageDeliveryChannelUseOriginalUpdated }, + string.Empty, string.Empty); + + A.CallTo(() => cleanupHandlerAssetRepository.RetrieveAssetWithDeliveryChannels(A._)) + .Returns(requestDetails.assetAfter); + + // Act + var sut = GetSut(); + var response = await sut.HandleMessage(requestDetails.queueMessage); + + // Assert + response.Should().BeTrue(); + A.CallTo(() => + bucketWriter.DeleteFromBucket(A.That.Matches(o => o[0].Key == "1/99/foo"))) + .MustHaveHappened(); + A.CallTo(() => bucketWriter.DeleteFolder(A._, A._)).MustNotHaveHappened(); + } + + // roles + + [Theory] + [InlineData("", "new role")] + [InlineData(null, "new role")] + [InlineData("old role", null)] + public async Task Handle_DeletesInfoJson_WhenRolesChanged(string? rolesBefore, string? rolesAfter) + { + // Arrange + var requestDetails = CreateMinimalRequestDetails( + new List() { imageDeliveryChannelUseOriginalImage }, new List() { imageDeliveryChannelUseOriginalImage }, + string.Empty, "new role"); + + A.CallTo(() => cleanupHandlerAssetRepository.RetrieveAssetWithDeliveryChannels(A._)) + .Returns(requestDetails.assetAfter); + + // Act + var sut = GetSut(); + var response = await sut.HandleMessage(requestDetails.queueMessage); + + // Assert + response.Should().BeTrue(); + A.CallTo(() => + bucketWriter.DeleteFromBucket(A._)).MustNotHaveHappened(); + A.CallTo(() => bucketWriter.DeleteFolder(A.That.Matches(o => o.Key == "1/99/foo/info/"), A._)).MustHaveHappened(); + } + + [Theory] + [InlineData("", null)] + [InlineData(null, "")] + [InlineData(null, null)] + public async Task Handle_DeletesInfoJson_WhenRolesChangedBothNullOrEmpty(string? rolesBefore, string? rolesAfter) + { + // Arrange + var requestDetails = CreateMinimalRequestDetails( + new List() { imageDeliveryChannelUseOriginalImage }, new List() { imageDeliveryChannelUseOriginalImage }, + rolesBefore, rolesAfter); + + A.CallTo(() => cleanupHandlerAssetRepository.RetrieveAssetWithDeliveryChannels(A._)) + .Returns(requestDetails.assetAfter); + + // Act + var sut = GetSut(); + var response = await sut.HandleMessage(requestDetails.queueMessage); + + // Assert + response.Should().BeTrue(); + A.CallTo(() => + bucketWriter.DeleteFromBucket(A._)).MustNotHaveHappened(); + A.CallTo(() => bucketWriter.DeleteFolder(A.That.Matches(o => o.Key == "1/99/foo/info/"), A._)).MustNotHaveHappened(); + } + + // helper functions + + private (QueueMessage queueMessage, Asset assetAfter) CreateMinimalRequestDetails(List imageDeliveryChannelsBefore, + List imageDeliveryChannelsAfter, string? rolesBefore, string? rolesAfter, string mediaType = "image/jpg") + { + var assetBefore = new Asset() + { + Id = new AssetId(1, 99, "foo"), + ImageDeliveryChannels = imageDeliveryChannelsBefore, + Roles = rolesBefore, + MediaType = mediaType + }; + + var assetAfter = new Asset() + { + Id = new AssetId(1, 99, "foo"), + ImageDeliveryChannels = imageDeliveryChannelsAfter, + Roles = rolesAfter, + MediaType = mediaType + }; + + var cleanupRequest = new AssetUpdatedNotificationRequest() + { + AssetBeforeUpdate = assetBefore, + CustomerPathElement = new CustomerPathElement(99, "stuff"), + AssetAfterUpdate = assetAfter + }; + + var serialized = JsonSerializer.Serialize(cleanupRequest, settings); + + var queueMessage = new QueueMessage + { + Body = JsonNode.Parse(serialized)!.AsObject(), + MessageAttributes = new Dictionary() + { + { "engineNotified", "True" } + } + }; + return (queueMessage, assetAfter); + } +} \ No newline at end of file diff --git a/src/protagonist/DLCS.AWS.Tests/ElasticTranscoder/ElasticTranscoderWrapperTests.cs b/src/protagonist/DLCS.AWS.Tests/ElasticTranscoder/ElasticTranscoderWrapperTests.cs index 4809fade9..539d13763 100644 --- a/src/protagonist/DLCS.AWS.Tests/ElasticTranscoder/ElasticTranscoderWrapperTests.cs +++ b/src/protagonist/DLCS.AWS.Tests/ElasticTranscoder/ElasticTranscoderWrapperTests.cs @@ -25,6 +25,23 @@ public ElasticTranscoderWrapperTests() bucketReader = A.Fake(); storageKeyGenerator = A.Fake(); + A.CallTo(() => elasticTranscoder.ListPresetsAsync(A._, A._)).Returns(new ListPresetsResponse() + { + Presets = new List() + { + new() + { + Id = "some-preset", + Name = "some-preset-name" + }, + new() + { + Id = "some-preset2", + Name = "some-preset-2-name" + } + } + }); + var cacheSettings = Options.Create(new CacheSettings()); sut = new ElasticTranscoderWrapper(elasticTranscoder, new MockCachingService(), bucketWriter, bucketReader, storageKeyGenerator, cacheSettings, new NullLogger()); @@ -59,4 +76,37 @@ public async Task CreateJob_CreatesETJob_WithCorrectInput() // Assert createRequest.Input.Should().BeEquivalentTo(expectedInput); } + + [Fact] + public async Task GetPresetIdLookup_ReturnsPresets_WhenCalled() + { + // Arrange and Act + var presets = await sut.GetPresetIdLookup(default); + + // Assert + presets.Count.Should().Be(2); + presets.Should().ContainKey("some-preset-name"); + presets.Should().NotContainKey("random-preset"); + } + + [Fact] + public async Task GetPresetDetails_ReturnsPresetDetails_WhenCalled() + { + // Arrange and Act + var preset = await sut.GetPresetDetails("some-preset-name", default); + + // Assert + preset.Should().NotBeNull(); + preset.Id.Should().Be("some-preset"); + } + + [Fact] + public async Task GetPresetDetails_ReturnsNull_WhenCalledWithInvalidPreset() + { + // Arrange and Act + var preset = await sut.GetPresetDetails("incorrect-name", default); + + // Assert + preset.Should().BeNull(); + } } \ No newline at end of file diff --git a/src/protagonist/DLCS.AWS.Tests/SNS/TopicPublisherTests.cs b/src/protagonist/DLCS.AWS.Tests/SNS/TopicPublisherTests.cs index f1c32ea52..69b49f0da 100644 --- a/src/protagonist/DLCS.AWS.Tests/SNS/TopicPublisherTests.cs +++ b/src/protagonist/DLCS.AWS.Tests/SNS/TopicPublisherTests.cs @@ -31,7 +31,7 @@ public TopicPublisherTests() public async Task PublishToAssetModifiedTopicBatch_SuccessfullyPublishesSingleMessage_IfSingleItemInBatch() { // Arrange - var notification = new AssetModifiedNotification("message", ChangeType.Delete); + var notification = new AssetModifiedNotification("message", GetAttributes(ChangeType.Delete, false)); // Act await sut.PublishToAssetModifiedTopic(new[] { notification }); @@ -52,7 +52,7 @@ public async Task PublishToAssetModifiedTopicBatch_SuccessfullyPublishesSingleMe public async Task PublishToAssetModifiedTopicBatch_SingleItemInBatch_ReturnsSuccessDependentOnStatusCode(HttpStatusCode statusCode, bool expected) { // Arrange - var notification = new AssetModifiedNotification("message", ChangeType.Delete); + var notification = new AssetModifiedNotification("message", GetAttributes(ChangeType.Delete, false)); A.CallTo(() => snsClient.PublishAsync(A._, A._)) .Returns(new PublishResponse { HttpStatusCode = statusCode }); @@ -67,8 +67,8 @@ public async Task PublishToAssetModifiedTopicBatch_SingleItemInBatch_ReturnsSucc public async Task PublishToAssetModifiedTopicBatch_SuccessfullyPublishesSingleBatch() { // Arrange - var notification = new AssetModifiedNotification("message", ChangeType.Delete); - var notification2 = new AssetModifiedNotification("message", ChangeType.Delete); + var notification = new AssetModifiedNotification("message", GetAttributes(ChangeType.Delete, false)); + var notification2 = new AssetModifiedNotification("message", GetAttributes(ChangeType.Delete, false)); // Act await sut.PublishToAssetModifiedTopic(new[] { notification, notification2 }); @@ -91,7 +91,7 @@ public async Task PublishToAssetModifiedTopicBatch_SuccessfullyPublishesMultiple var notifications = new List(15); for (int x = 0; x < 15; x++) { - notifications.Add(new AssetModifiedNotification(x < 10 ? "message" : "next", ChangeType.Delete)); + notifications.Add(new AssetModifiedNotification(x < 10 ? "message" : "next", GetAttributes(ChangeType.Delete, false))); } // Act @@ -123,7 +123,7 @@ public async Task PublishToAssetModifiedTopicBatch_ReturnsTrue_IfAllBatchesSucce var notifications = new List(15); for (int x = 0; x < 15; x++) { - notifications.Add(new AssetModifiedNotification("message", ChangeType.Delete)); + notifications.Add(new AssetModifiedNotification("message", GetAttributes(ChangeType.Delete, false))); } A.CallTo(() => snsClient.PublishBatchAsync(A._, A._)) @@ -143,7 +143,7 @@ public async Task PublishToAssetModifiedTopicBatch_ReturnsFalse_IfAnyBatchFails( var notifications = new List(15); for (int x = 0; x < 15; x++) { - notifications.Add(new AssetModifiedNotification("message", ChangeType.Delete)); + notifications.Add(new AssetModifiedNotification("message", GetAttributes(ChangeType.Delete, false))); } A.CallTo(() => snsClient.PublishBatchAsync(A._, A._)) @@ -157,4 +157,58 @@ public async Task PublishToAssetModifiedTopicBatch_ReturnsFalse_IfAnyBatchFails( // Assert response.Should().BeFalse(); } + + [Fact] + public async Task PublishToAssetModifiedTopicBatch_SuccessfullyPublishesSingleMessageWithEngineNotified_IfEngineNotifiedTrue() + { + // Arrange + var notification = new AssetModifiedNotification("message", GetAttributes(ChangeType.Update, true)); + + // Act + await sut.PublishToAssetModifiedTopic(new[] { notification }); + + // Assert + A.CallTo(() => + snsClient.PublishAsync( + A.That.Matches(r => + r.Message == "message" && r.MessageAttributes["messageType"].StringValue == "Update" && + r.MessageAttributes["engineNotified"].StringValue == "True"), + A._)).MustHaveHappened(); + } + + [Fact] + public async Task PublishToAssetModifiedTopicBatch_SuccessfullyPublishesSingleBatchWithEngineNotified() + { + // Arrange + var notification = new AssetModifiedNotification("message", GetAttributes(ChangeType.Update, true)); + var notification2 = new AssetModifiedNotification("message", GetAttributes(ChangeType.Update, true)); + + // Act + await sut.PublishToAssetModifiedTopic(new[] { notification, notification2 }); + + // Assert + A.CallTo(() => + snsClient.PublishBatchAsync( + A.That.Matches(b => b.PublishBatchRequestEntries.All(r => + r.Message == "message" && + r.MessageAttributes["messageType"].StringValue == + "Update"&& + r.MessageAttributes["engineNotified"].StringValue == "True") && + b.PublishBatchRequestEntries.Count == 2), + A._)).MustHaveHappened(); + } + + private Dictionary GetAttributes(ChangeType changeType, bool engineNotified) + { + var attributes = new Dictionary() + { + { "messageType", changeType.ToString() } + }; + if (engineNotified) + { + attributes.Add("engineNotified", "True"); + } + + return attributes; + } } \ No newline at end of file diff --git a/src/protagonist/DLCS.AWS/ElasticTranscoder/ElasticTranscoderWrapper.cs b/src/protagonist/DLCS.AWS/ElasticTranscoder/ElasticTranscoderWrapper.cs index 770b33966..d648f0d52 100644 --- a/src/protagonist/DLCS.AWS/ElasticTranscoder/ElasticTranscoderWrapper.cs +++ b/src/protagonist/DLCS.AWS/ElasticTranscoder/ElasticTranscoderWrapper.cs @@ -1,6 +1,7 @@ using System.Xml.Linq; using Amazon.ElasticTranscoder; using Amazon.ElasticTranscoder.Model; +using DLCS.AWS.ElasticTranscoder.Models; using DLCS.AWS.ElasticTranscoder.Models.Job; using DLCS.AWS.S3; using DLCS.AWS.S3.Models; @@ -45,25 +46,38 @@ public ElasticTranscoderWrapper(IAmazonElasticTranscoder elasticTranscoder, this.storageKeyGenerator = storageKeyGenerator; } - public Task> GetPresetIdLookup(CancellationToken token) + public async Task> GetPresetIdLookup(CancellationToken token) { - const string presetLookupKey = "MediaTranscode:Presets"; + var presets = await RetrievePresets(token); + + var presetsDictionary = presets.ToDictionary(pair => pair.Name, pair => pair); + + return presetsDictionary; + } + public async Task GetPresetDetails(string name, CancellationToken token) + { + var presets = await RetrievePresets(token); + + return presets.FirstOrDefault(p => p.Name == name); + } + + private Task> RetrievePresets(CancellationToken token) + { + const string presetLookupKey = "MediaTranscode:Presets"; + return cache.GetOrAddAsync(presetLookupKey, async entry => { - var presets = new Dictionary(); + var presets = new List(); var response = new ListPresetsResponse(); - + do { - var request = new ListPresetsRequest {PageToken = response.NextPageToken}; + var request = new ListPresetsRequest { PageToken = response.NextPageToken }; response = await elasticTranscoder.ListPresetsAsync(request, token); - foreach (var preset in response.Presets) - { - presets.Add(preset.Name, preset.Id); - } - + presets.AddRange(response.Presets.Select(r => new TranscoderPreset(r.Id, r.Name, r.Container)) + .ToList()); } while (response.NextPageToken != null); if (presets.Count == 0) diff --git a/src/protagonist/DLCS.AWS/ElasticTranscoder/IElasticTranscoderWrapper.cs b/src/protagonist/DLCS.AWS/ElasticTranscoder/IElasticTranscoderWrapper.cs index f673c41f5..a70367b47 100644 --- a/src/protagonist/DLCS.AWS/ElasticTranscoder/IElasticTranscoderWrapper.cs +++ b/src/protagonist/DLCS.AWS/ElasticTranscoder/IElasticTranscoderWrapper.cs @@ -1,4 +1,5 @@ using Amazon.ElasticTranscoder.Model; +using DLCS.AWS.ElasticTranscoder.Models; using DLCS.AWS.ElasticTranscoder.Models.Job; using DLCS.Core.Types; @@ -14,7 +15,15 @@ public interface IElasticTranscoderWrapper /// /// CancellationToken /// Dictionary of ElasticTranscoder presets - Task> GetPresetIdLookup(CancellationToken token = default); + Task> GetPresetIdLookup(CancellationToken token = default); + + /// + /// Gets details of a preset based on the name + /// + /// The name of the preset + /// Cancellation token + /// Details of a single preset + public Task GetPresetDetails(string name, CancellationToken token = default); /// /// Get ElasticTranscoder Pipeline Id from name. diff --git a/src/protagonist/DLCS.AWS/ElasticTranscoder/Models/TranscoderPreset.cs b/src/protagonist/DLCS.AWS/ElasticTranscoder/Models/TranscoderPreset.cs new file mode 100644 index 000000000..2c3b46409 --- /dev/null +++ b/src/protagonist/DLCS.AWS/ElasticTranscoder/Models/TranscoderPreset.cs @@ -0,0 +1,3 @@ +namespace DLCS.AWS.ElasticTranscoder.Models; + +public record TranscoderPreset(string Id, string Name, string Extension); \ No newline at end of file diff --git a/src/protagonist/DLCS.AWS/ElasticTranscoder/TranscoderTemplates.cs b/src/protagonist/DLCS.AWS/ElasticTranscoder/TranscoderTemplates.cs index ba1b60084..cf52d8cf5 100644 --- a/src/protagonist/DLCS.AWS/ElasticTranscoder/TranscoderTemplates.cs +++ b/src/protagonist/DLCS.AWS/ElasticTranscoder/TranscoderTemplates.cs @@ -37,7 +37,7 @@ public static class TranscoderTemplates public static string GetFinalDestinationKey(string outputKey) => outputKey.Substring(outputKey.IndexOf("/", StringComparison.Ordinal) + 1); - private static string GetDestinationTemplate(string mediaType) + public static string GetDestinationTemplate(string mediaType) { // audio: {customer}/{space}/{image}/full/max/default.{extension} (mediatype like audio/) // video: {customer}/{space}/{image}/full/full/max/max/0/default.{extension} (mediatype like video/) diff --git a/src/protagonist/DLCS.AWS/SNS/ITopicPublisher.cs b/src/protagonist/DLCS.AWS/SNS/ITopicPublisher.cs index 60df6190e..8bc59efa2 100644 --- a/src/protagonist/DLCS.AWS/SNS/ITopicPublisher.cs +++ b/src/protagonist/DLCS.AWS/SNS/ITopicPublisher.cs @@ -17,4 +17,4 @@ public Task PublishToAssetModifiedTopic(IReadOnlyList /// Represents the contents + type of change for Asset modified notification /// -public record AssetModifiedNotification(string MessageContents, ChangeType ChangeType); \ No newline at end of file +public record AssetModifiedNotification(string MessageContents, Dictionary Attributes); \ No newline at end of file diff --git a/src/protagonist/DLCS.AWS/SNS/TopicPublisher.cs b/src/protagonist/DLCS.AWS/SNS/TopicPublisher.cs index 677e52c02..20db04da9 100644 --- a/src/protagonist/DLCS.AWS/SNS/TopicPublisher.cs +++ b/src/protagonist/DLCS.AWS/SNS/TopicPublisher.cs @@ -57,7 +57,7 @@ private async Task PublishToAssetModifiedTopic(AssetModifiedNotification m { TopicArn = snsSettings.AssetModifiedNotificationTopicArn, Message = message.MessageContents, - MessageAttributes = GetMessageAttributes(message.ChangeType) + MessageAttributes = GetMessageAttributes(message.Attributes) }; try @@ -83,7 +83,7 @@ private async Task PublishBatch(AssetModifiedNotification[] chunk, Guid ba TopicArn = snsSettings.AssetModifiedNotificationTopicArn, PublishBatchRequestEntries = chunk.Select(m => new PublishBatchRequestEntry { - MessageAttributes = GetMessageAttributes(m.ChangeType), + MessageAttributes = GetMessageAttributes(m.Attributes), Message = m.MessageContents, Id = $"{batchIdPrefix}_{batchNumber}_{batchCount++}", }).ToList() @@ -99,16 +99,19 @@ private async Task PublishBatch(AssetModifiedNotification[] chunk, Guid ba } } - private static Dictionary GetMessageAttributes(ChangeType changeType) + private static Dictionary GetMessageAttributes(Dictionary attributes) { - var attributeValue = new MessageAttributeValue + var messageAttributes = new Dictionary(); + foreach (var attribute in attributes) { - StringValue = changeType.ToString(), - DataType = "String" - }; - return new Dictionary - { - { "messageType", attributeValue } - }; + messageAttributes.Add(attribute.Key, + new MessageAttributeValue() + { + DataType = "String", + StringValue = attribute.Value + }); + } + + return messageAttributes; } } \ No newline at end of file diff --git a/src/protagonist/DLCS.AWS/SQS/Models/QueueMessage.cs b/src/protagonist/DLCS.AWS/SQS/Models/QueueMessage.cs index bea57abe5..c632c1155 100644 --- a/src/protagonist/DLCS.AWS/SQS/Models/QueueMessage.cs +++ b/src/protagonist/DLCS.AWS/SQS/Models/QueueMessage.cs @@ -13,6 +13,11 @@ public class QueueMessage /// public JsonObject Body { get; set; } + /// + /// Any attributes associated with message + /// + public Dictionary MessageAttributes { get; set; } + /// /// Any attributes associated with message /// diff --git a/src/protagonist/DLCS.AWS/SQS/SqsListener.cs b/src/protagonist/DLCS.AWS/SQS/SqsListener.cs index 5336a1f2d..e03b15108 100644 --- a/src/protagonist/DLCS.AWS/SQS/SqsListener.cs +++ b/src/protagonist/DLCS.AWS/SQS/SqsListener.cs @@ -142,6 +142,7 @@ private Task GetMessagesFromQueue(string queueUrl, Cance QueueUrl = queueUrl, WaitTimeSeconds = options.SQS.WaitTimeSecs, MaxNumberOfMessages = options.SQS.MaxNumberOfMessages, + MessageAttributeNames = new List{ "All" } }, cancellationToken); private async Task HandleMessage(Message message, CancellationToken cancellationToken) @@ -153,8 +154,12 @@ private async Task HandleMessage(Message message, CancellationToken cancel logger.LogTrace("Handling message {Message} from {Queue}", message.MessageId, QueueName); } + var messageAttributes = message.MessageAttributes + .ToDictionary(pair => pair.Key, pair => pair.Value.StringValue); + var queueMessage = new QueueMessage { + MessageAttributes = messageAttributes, Attributes = message.Attributes, Body = JsonNode.Parse(message.Body)!.AsObject(), MessageId = message.MessageId, diff --git a/src/protagonist/DLCS.AWS/Settings/SQSSettings.cs b/src/protagonist/DLCS.AWS/Settings/SQSSettings.cs index cf1b584e0..72301463d 100644 --- a/src/protagonist/DLCS.AWS/Settings/SQSSettings.cs +++ b/src/protagonist/DLCS.AWS/Settings/SQSSettings.cs @@ -39,6 +39,11 @@ public class SQSSettings /// public string? DeleteNotificationQueueName { get; set; } + /// + /// Name of queue for handling notifications that assets have been updated + /// + public string? UpdateNotificationQueueName { get; set; } + /// /// The duration (in seconds) for which the call waits for a message to arrive in the queue before returning /// diff --git a/src/protagonist/DLCS.Core.Tests/Collections/CollectionXTests.cs b/src/protagonist/DLCS.Core.Tests/Collections/CollectionXTests.cs index 0319e6e7a..7979325d6 100644 --- a/src/protagonist/DLCS.Core.Tests/Collections/CollectionXTests.cs +++ b/src/protagonist/DLCS.Core.Tests/Collections/CollectionXTests.cs @@ -54,6 +54,30 @@ public void IsNullOrEmpty_List_False_IfHasValues() coll.IsNullOrEmpty().Should().BeFalse(); } + + [Fact] + public void IsEmpty_List_False_IfNull() + { + List coll = null; + + coll.IsEmpty().Should().BeFalse(); + } + + [Fact] + public void IsEmpty_List_True_IfEmpty() + { + var coll = new List(); + + coll.IsEmpty().Should().BeTrue(); + } + + [Fact] + public void IsEmpty_List_False_IfHasValues() + { + var coll = new List {2}; + + coll.IsEmpty().Should().BeFalse(); + } [Fact] public void AsList_ReturnsExpected() diff --git a/src/protagonist/DLCS.Core/Collections/CollectionX.cs b/src/protagonist/DLCS.Core/Collections/CollectionX.cs index 74c897f3e..c64126e3b 100644 --- a/src/protagonist/DLCS.Core/Collections/CollectionX.cs +++ b/src/protagonist/DLCS.Core/Collections/CollectionX.cs @@ -20,6 +20,13 @@ public static bool IsNullOrEmpty([NotNullWhen(false)] this IEnumerable? co /// true if null or empty, else false public static bool IsNullOrEmpty([NotNullWhen(false)] this IList? collection) => collection == null || collection.Count == 0; + + /// + /// Check if IList is empty - this explicitly checks for Empty only, list could still be null + /// + /// true if empty, else false + public static bool IsEmpty(this IList? collection) + => collection?.Count == 0; /// /// Check if list contains single specified item diff --git a/src/protagonist/DLCS.Core/Settings/ImageServer.cs b/src/protagonist/DLCS.Core/Settings/ImageServer.cs new file mode 100644 index 000000000..494f79281 --- /dev/null +++ b/src/protagonist/DLCS.Core/Settings/ImageServer.cs @@ -0,0 +1,17 @@ +namespace DLCS.Core.Settings; + +/// +/// Enum representing image server used for serving image requests +/// +public enum ImageServer +{ + /// + /// Cantaloupe image server + /// + Cantaloupe, + + /// + /// IIP Image Server + /// + IIPImage +} \ No newline at end of file diff --git a/src/protagonist/DLCS.Model.Tests/Assets/AssetTests.cs b/src/protagonist/DLCS.Model.Tests/Assets/AssetTests.cs index 583525aeb..20e7ae6fc 100644 --- a/src/protagonist/DLCS.Model.Tests/Assets/AssetTests.cs +++ b/src/protagonist/DLCS.Model.Tests/Assets/AssetTests.cs @@ -1,5 +1,9 @@ -using DLCS.Core.Types; +using System.Collections.Generic; +using System.Linq; +using DLCS.Core.Types; using DLCS.Model.Assets; +using DLCS.Model.Assets.Metadata; +using DLCS.Model.Policies; using FluentAssertions; using Xunit; @@ -76,4 +80,63 @@ public void Tags_Convert_From_List() var expected = "a,b,c"; asset.Tags.Should().Be(expected); } + + [Fact] + public void Clone_ClonesObject_From_List() + { + // Arrange + var asset = new Asset + { + Reference1 = "someReference", + Reference2 = "ref2", + ImageDeliveryChannels =new List() + { + new() + { + DeliveryChannelPolicyId = KnownDeliveryChannelPolicies.ImageDefault, + Channel = AssetDeliveryChannels.Image, + DeliveryChannelPolicy = new DeliveryChannelPolicy() + { + Id = KnownDeliveryChannelPolicies.ImageDefault, + Channel = AssetDeliveryChannels.Image + } + } + }, + AssetApplicationMetadata = new List() + { + new() + { + MetadataType = "someType" + } + } + }; + + // Act + var secondAsset = asset.Clone(); + + secondAsset.Reference1 = "someReference updated"; + secondAsset.ImageDeliveryChannels.ToList()[0].DeliveryChannelPolicyId = + KnownDeliveryChannelPolicies.ImageUseOriginal; + secondAsset.ImageDeliveryChannels.Add(new ImageDeliveryChannel + { + DeliveryChannelPolicyId = KnownDeliveryChannelPolicies.AvDefaultVideo, + Channel = AssetDeliveryChannels.Timebased, + DeliveryChannelPolicy = new DeliveryChannelPolicy() + { + Id = KnownDeliveryChannelPolicies.AvDefaultVideo, + Channel = AssetDeliveryChannels.Timebased + } + }); + secondAsset.AssetApplicationMetadata!.ToList()[0].MetadataType = "someType 2"; + + // Assert + secondAsset.Reference1.Should().NotBe(asset.Reference1); + secondAsset.Reference2.Should().Be(asset.Reference2); + secondAsset.ImageDeliveryChannels.ToList()[0].DeliveryChannelPolicyId.Should() + .NotBe(asset.ImageDeliveryChannels.ToList()[0].DeliveryChannelPolicyId); + secondAsset.ImageDeliveryChannels.Count.Should().Be(2); + asset.ImageDeliveryChannels.Count.Should().Be(1); + secondAsset.AssetApplicationMetadata.ToList()[0].MetadataType.Should() + .NotBe(asset.AssetApplicationMetadata.ToList()[0].MetadataType); + } } \ No newline at end of file diff --git a/src/protagonist/DLCS.Model.Tests/Assets/InfoJsonBuilderTests.cs b/src/protagonist/DLCS.Model.Tests/Assets/InfoJsonBuilderTests.cs index 662ac0ac1..58bf4c382 100644 --- a/src/protagonist/DLCS.Model.Tests/Assets/InfoJsonBuilderTests.cs +++ b/src/protagonist/DLCS.Model.Tests/Assets/InfoJsonBuilderTests.cs @@ -5,7 +5,7 @@ using Xunit; namespace DLCS.Model.Tests.Assets; - + public class InfoJsonBuilderTests { [Fact] @@ -15,6 +15,7 @@ public void GetImageApi2_1Level0_ReturnsExpected() var expected = @"{ ""@context"": ""http://iiif.io/api/image/2/context.json"", ""@id"": ""https://test.example.com/iiif-img/2/1/jackal"", + ""@type"": ""iiif:Image"", ""profile"": [ ""http://iiif.io/api/image/2/level0.json"", { @@ -48,6 +49,7 @@ public void GetImageApi2_1Level1_ReturnsExpected() var expected = @"{ ""@context"": ""http://iiif.io/api/image/2/context.json"", ""@id"": ""https://test.example.com/iiif-img/2/1/jackal"", + ""@type"": ""iiif:Image"", ""profile"": [ ""http://iiif.io/api/image/2/level1.json"", { diff --git a/src/protagonist/DLCS.Model.Tests/Policies/DeliveryChannelPolicyXTests.cs b/src/protagonist/DLCS.Model.Tests/Policies/DeliveryChannelPolicyXTests.cs index b4b69c2e7..632f7ab9e 100644 --- a/src/protagonist/DLCS.Model.Tests/Policies/DeliveryChannelPolicyXTests.cs +++ b/src/protagonist/DLCS.Model.Tests/Policies/DeliveryChannelPolicyXTests.cs @@ -34,6 +34,29 @@ public void ThumbsDataAsSizeParameters_ReturnsSizeParameters() actual.Should().BeEquivalentTo(expected); } + [Fact] + public void AsTimebasedPresets_Throws_IfPolicyNotTimebased() + { + var policy = new DeliveryChannelPolicy { Channel = "iiif-img" }; + + Action action = () => policy.AsTimebasedPresets(); + action.Should().ThrowExactly(); + } + + [Fact] + public void AsTimebasedPresets_ReturnsExpected() + { + // Arrange + var expected = new List { "foo", "bar", "foobar" }; + var policy = new DeliveryChannelPolicy { Channel = "iiif-av", PolicyData = "[\"foo\",\"bar\",\"foobar\"]" }; + + // Act + var actual = policy.AsTimebasedPresets(); + + // Assert + actual.Should().BeEquivalentTo(expected); + } + [Fact] public void PolicyDataAs_ReturnsExpected() { diff --git a/src/protagonist/DLCS.Model/Assets/Asset.cs b/src/protagonist/DLCS.Model/Assets/Asset.cs index 11cdb54f9..f89c5f153 100644 --- a/src/protagonist/DLCS.Model/Assets/Asset.cs +++ b/src/protagonist/DLCS.Model/Assets/Asset.cs @@ -11,7 +11,7 @@ namespace DLCS.Model.Assets; /// /// Represents an Asset that is stored in the DLCS database. /// -public class Asset +public class Asset : ICloneable { public AssetId Id { get; set; } public int Customer { get; set; } @@ -116,4 +116,23 @@ public Asset(AssetId assetId) Customer = assetId.Customer; Space = assetId.Space; } + + public Asset Clone() + { + var asset = (Asset)MemberwiseClone(); + + var deliveryChannels = asset.ImageDeliveryChannels.Select(d => (ImageDeliveryChannel)d.Clone()).ToList(); + asset.ImageDeliveryChannels = deliveryChannels; + + if (asset.AssetApplicationMetadata != null) + { + var assetApplicationMetadata = + asset.AssetApplicationMetadata.Select(a => (AssetApplicationMetadata)a.Clone()).ToList(); + asset.AssetApplicationMetadata = assetApplicationMetadata; + } + + return asset; + } + + object ICloneable.Clone() { return Clone(); } } \ No newline at end of file diff --git a/src/protagonist/DLCS.Model/Assets/AssetDeliveryChannels.cs b/src/protagonist/DLCS.Model/Assets/AssetDeliveryChannels.cs index 1dd7036cc..3f219fb98 100644 --- a/src/protagonist/DLCS.Model/Assets/AssetDeliveryChannels.cs +++ b/src/protagonist/DLCS.Model/Assets/AssetDeliveryChannels.cs @@ -51,6 +51,12 @@ public static bool HasSingleDeliveryChannel(this Asset asset, string deliveryCha asset.ImageDeliveryChannels.Count == 1 && asset.HasDeliveryChannel(deliveryChannel); + /// + /// Checks if asset does not have a specified deliveryChannel + /// + public static bool DoesNotHaveDeliveryChannel(this Asset asset, string deliveryChannel) + => !asset.HasDeliveryChannel(deliveryChannel); + /// /// Checks if string is a valid delivery channel /// diff --git a/src/protagonist/DLCS.Model/Assets/ImageDeliveryChannel.cs b/src/protagonist/DLCS.Model/Assets/ImageDeliveryChannel.cs index b4c8b0826..e71899ef3 100644 --- a/src/protagonist/DLCS.Model/Assets/ImageDeliveryChannel.cs +++ b/src/protagonist/DLCS.Model/Assets/ImageDeliveryChannel.cs @@ -1,10 +1,11 @@ #nullable disable +using System; using DLCS.Core.Types; using DLCS.Model.Policies; namespace DLCS.Model.Assets; -public class ImageDeliveryChannel +public class ImageDeliveryChannel : ICloneable { /// /// Unique identifier @@ -27,4 +28,8 @@ public class ImageDeliveryChannel /// The delivery channel policy id for the attached delivery channel policy /// public int DeliveryChannelPolicyId { get; set; } + + public ImageDeliveryChannel Clone() => (ImageDeliveryChannel)MemberwiseClone(); + + object ICloneable.Clone() => Clone(); } \ No newline at end of file diff --git a/src/protagonist/DLCS.Model/Assets/InfoJsonBuilder.cs b/src/protagonist/DLCS.Model/Assets/InfoJsonBuilder.cs index 19d8721ce..07bec1f5b 100644 --- a/src/protagonist/DLCS.Model/Assets/InfoJsonBuilder.cs +++ b/src/protagonist/DLCS.Model/Assets/InfoJsonBuilder.cs @@ -25,7 +25,7 @@ public static ImageService2 GetImageApi2_1Level0(string serviceEndpoint, List /// The image id for the attached asset @@ -31,4 +31,8 @@ public class AssetApplicationMetadata /// When the metadata was last modified /// public DateTime Modified { get; set; } + + public AssetApplicationMetadata Clone() => (AssetApplicationMetadata)MemberwiseClone(); + + object ICloneable.Clone() => Clone(); } \ No newline at end of file diff --git a/src/protagonist/DLCS.Model/Assets/Metadata/IAssetApplicationMetadataRepository.cs b/src/protagonist/DLCS.Model/Assets/Metadata/IAssetApplicationMetadataRepository.cs index 9b4c2f56d..aa29db259 100644 --- a/src/protagonist/DLCS.Model/Assets/Metadata/IAssetApplicationMetadataRepository.cs +++ b/src/protagonist/DLCS.Model/Assets/Metadata/IAssetApplicationMetadataRepository.cs @@ -17,4 +17,14 @@ public interface IAssetApplicationMetadataRepository public Task UpsertApplicationMetadata( AssetId assetId, string metadataType, string metadataValue, CancellationToken cancellationToken = default); + + /// + /// Deletes asset application metadata from the table + /// + /// The asset id associated with the metadata + /// The type of metadata to delete + /// A cancellation token + /// A boolean value based on successful deletion + public Task DeleteAssetApplicationMetadata(AssetId assetId, string metadataType, + CancellationToken cancellationToken = default); } \ No newline at end of file diff --git a/src/protagonist/DLCS.Model/DLCS.Model.csproj b/src/protagonist/DLCS.Model/DLCS.Model.csproj index f9dffa14c..8f423dac8 100644 --- a/src/protagonist/DLCS.Model/DLCS.Model.csproj +++ b/src/protagonist/DLCS.Model/DLCS.Model.csproj @@ -11,7 +11,7 @@ - + diff --git a/src/protagonist/DLCS.Model/IIIF/Constants.cs b/src/protagonist/DLCS.Model/IIIF/Constants.cs new file mode 100644 index 000000000..54045d91e --- /dev/null +++ b/src/protagonist/DLCS.Model/IIIF/Constants.cs @@ -0,0 +1,9 @@ +namespace DLCS.Model.IIIF; + +public static class Constants +{ + /// + /// @type value for ImageService2 for rendering as IIIF Image or Presentation v2 + /// + public const string ImageService2Type = "iiif:Image"; +} \ No newline at end of file diff --git a/src/protagonist/DLCS.Model/Policies/DeliveryChannelPolicyX.cs b/src/protagonist/DLCS.Model/Policies/DeliveryChannelPolicyX.cs index f602de776..73ad0eea1 100644 --- a/src/protagonist/DLCS.Model/Policies/DeliveryChannelPolicyX.cs +++ b/src/protagonist/DLCS.Model/Policies/DeliveryChannelPolicyX.cs @@ -29,6 +29,24 @@ public static List ThumbsDataAsSizeParameters(this DeliveryChanne .ToList(); } + /// + /// Get timebased PolicyData as a list of strings + /// + /// Current + /// Collection of strings representing timebased policies + /// Thrown if specified policy is not for thumbs channel + public static List AsTimebasedPresets(this DeliveryChannelPolicy deliveryChannelPolicy) + { + if (deliveryChannelPolicy.Channel != AssetDeliveryChannels.Timebased) + { + throw new InvalidOperationException("Policy is not for timebased channel"); + } + + var timeBasedPresets = deliveryChannelPolicy.PolicyDataAs>(); + + return timeBasedPresets.ThrowIfNull(nameof(timeBasedPresets)); + } + /// /// Deserialise PolicyData as specified type /// diff --git a/src/protagonist/DLCS.Repository.Tests/Assets/AssetApplicationMetadataRepositoryTests.cs b/src/protagonist/DLCS.Repository.Tests/Assets/AssetApplicationMetadataRepositoryTests.cs index d3073e4ed..c2fd227ba 100644 --- a/src/protagonist/DLCS.Repository.Tests/Assets/AssetApplicationMetadataRepositoryTests.cs +++ b/src/protagonist/DLCS.Repository.Tests/Assets/AssetApplicationMetadataRepositoryTests.cs @@ -3,6 +3,7 @@ using DLCS.Model.Assets.Metadata; using DLCS.Repository.Assets; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging.Abstractions; using Test.Helpers.Integration; namespace DLCS.Repository.Tests.Assets; @@ -23,7 +24,7 @@ public AssetApplicationMetadataRepositoryTests(DlcsDatabaseFixture dbFixture) optionsBuilder.UseNpgsql(dbFixture.ConnectionString); contextForTests = new DlcsContext(optionsBuilder.Options); - sut = new AssetApplicationMetadataRepository(contextForTests); + sut = new AssetApplicationMetadataRepository(contextForTests, new NullLogger()); dbFixture.CleanUp(); dbContext.Images.AddTestAsset(AssetId.FromString("99/1/1"), ref1: "foobar"); @@ -85,4 +86,45 @@ public async Task UpsertApplicationMetadata_ThrowsException_WhenCalledWithInvali // Assert await action.Should().ThrowAsync(); } + + [Fact] + public async Task DeleteAssetApplicationMetadata_DeletesMetadata_WhenCalled() + { + // Arrange + var assetId = AssetId.FromString("99/1/1"); + + var assetApplicationMetadata = new AssetApplicationMetadata() + { + AssetId = assetId, + MetadataType = AssetApplicationMetadataTypes.ThumbSizes, + MetadataValue = "{\"a\": [], \"o\": [[75, 100], [150, 200], [300, 400], [769, 1024]]}", + Created = DateTime.UtcNow, + Modified = DateTime.UtcNow + }; + await dbContext.AssetApplicationMetadata.AddAsync(assetApplicationMetadata); + await dbContext.SaveChangesAsync(); + + // Act + var metadata = await sut.DeleteAssetApplicationMetadata(assetId, AssetApplicationMetadataTypes.ThumbSizes); + + var metaDataFromDatabase = await dbContext.AssetApplicationMetadata.FirstOrDefaultAsync(x => + x.AssetId == assetId && x.MetadataType == AssetApplicationMetadataTypes.ThumbSizes); + + // Assert + metadata.Should().BeTrue(); + metaDataFromDatabase.Should().BeNull(); + } + + [Fact] + public async Task DeleteAssetApplicationMetadata_DoesNotDeleteMetadata_WhenCalledWithNonexistentMetadata() + { + // Arrange + var assetId = AssetId.FromString("99/1/1"); + + // Act + var metadata = await sut.DeleteAssetApplicationMetadata(assetId, AssetApplicationMetadataTypes.ThumbSizes); + + // Assert + metadata.Should().BeFalse(); + } } \ No newline at end of file diff --git a/src/protagonist/DLCS.Repository.Tests/Assets/AssetQueryXTests.cs b/src/protagonist/DLCS.Repository.Tests/Assets/AssetQueryXTests.cs new file mode 100644 index 000000000..e216fa693 --- /dev/null +++ b/src/protagonist/DLCS.Repository.Tests/Assets/AssetQueryXTests.cs @@ -0,0 +1,56 @@ +using System.Collections.Generic; +using System.Linq; +using DLCS.Model.Assets; +using DLCS.Model.Policies; +using DLCS.Repository.Assets; +using Microsoft.EntityFrameworkCore; +using Test.Helpers.Data; +using Test.Helpers.Integration; + +namespace DLCS.Repository.Tests.Assets; + +[Trait("Category", "Database")] +[Collection(DatabaseCollection.CollectionName)] +public class AssetQueryXTests +{ + private readonly DlcsContext dbContext; + + public AssetQueryXTests(DlcsDatabaseFixture dbFixture) + { + dbContext = dbFixture.DbContext; + dbFixture.CleanUp(); + } + + [Fact] + public async Task IncludeDeliveryChannelsWithPolicy_ReturnsDeliveryChannels_ByOrderOfChannel() + { + var assetId = AssetIdGenerator.GetAssetId(); + await dbContext.ImageDeliveryChannels.AddRangeAsync( + new() + { + ImageId = assetId, Channel = "gamma", + DeliveryChannelPolicyId = KnownDeliveryChannelPolicies.ImageDefault + }, + new() + { + ImageId = assetId, Channel = "alpha", + DeliveryChannelPolicyId = KnownDeliveryChannelPolicies.ImageDefault + }, + new() + { + ImageId = assetId, Channel = "beta", + DeliveryChannelPolicyId = KnownDeliveryChannelPolicies.ImageDefault + }); + await dbContext.Images.AddTestAsset(assetId); + await dbContext.SaveChangesAsync(); + + // Act + var result = await dbContext.Images + .Where(i => i.Id == assetId) + .IncludeDeliveryChannelsWithPolicy() + .ToListAsync(); + + // Assert + result.Single().ImageDeliveryChannels.Should().BeInAscendingOrder(idc => idc.Channel); + } +} \ No newline at end of file diff --git a/src/protagonist/DLCS.Repository.Tests/Messaging/EngineClientTests.cs b/src/protagonist/DLCS.Repository.Tests/Messaging/EngineClientTests.cs index 096f2d317..efb64c22d 100644 --- a/src/protagonist/DLCS.Repository.Tests/Messaging/EngineClientTests.cs +++ b/src/protagonist/DLCS.Repository.Tests/Messaging/EngineClientTests.cs @@ -1,16 +1,21 @@ using System; +using System.Collections.Generic; using System.Net; using System.Net.Http; using System.Text.Json; using System.Text.Json.Serialization; using System.Threading; +using DLCS.AWS.ElasticTranscoder.Models; using DLCS.AWS.SQS; +using DLCS.Core.Caching; using DLCS.Core.Types; using DLCS.Model.Assets; using DLCS.Model.Messaging; using DLCS.Repository.Messaging; using FakeItEasy; +using LazyCache.Mocks; using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; using Test.Helpers.Http; namespace DLCS.Repository.Tests.Messaging; @@ -33,8 +38,9 @@ public EngineClientTests() queueLookup = A.Fake(); queueSender = A.Fake(); - - sut = new EngineClient(queueLookup, queueSender, httpClient, new NullLogger()); + + sut = new EngineClient(queueLookup, queueSender, httpClient, new MockCachingService(), Options.Create(new CacheSettings()), + new NullLogger()); } [Fact] @@ -139,4 +145,45 @@ public async Task GetAllowedAvOptions_ReturnsNull_IfEngineAvPolicyEndpointUnreac message.Method.Should().Be(HttpMethod.Get); returnedAvPolicyOptions.Should().BeNull(); } + + [Fact] + public async Task GetAvPresets_RetrievesAllowedAvPresets() + { + // Arrange + HttpRequestMessage message = null; + httpHandler.RegisterCallback(r => message = r); + + var response = JsonSerializer.Serialize(new Dictionary() + { + { "webm-policy", new("webm-policy", "some-webm-preset", "oga") }, + { "oga-policy", new("oga-policy", "some-oga-preset", "webm") } + }); + + httpHandler.GetResponseMessage(response, HttpStatusCode.OK); + + // Act + var returnedAvPresets = await sut.GetAvPresets(); + + // Assert + httpHandler.CallsMade.Should().ContainSingle().Which.Should().Be("http://engine.dlcs/av-presets"); + message.Method.Should().Be(HttpMethod.Get); + returnedAvPresets!.Count.Should().Be(2); + returnedAvPresets!.Keys.Should().BeEquivalentTo("webm-policy", "oga-policy"); + returnedAvPresets!.Values.Should().Contain(new TranscoderPreset("webm-policy", "some-webm-preset", "oga")); + } + + [Fact] + public async Task GetAvPresets_ReturnsEmpty_IfEngineAvPolicyEndpointThrowsError() + { + // Arrange + httpHandler.RegisterCallback(r => throw new Exception("error")); + httpHandler.GetResponseMessage("Not found", HttpStatusCode.NotFound); + + // Act + var returnedAvPresets = await sut.GetAvPresets(); + + // Assert + httpHandler.CallsMade.Should().ContainSingle().Which.Should().Be("http://engine.dlcs/av-presets"); + returnedAvPresets.Should().BeEmpty(); + } } \ No newline at end of file diff --git a/src/protagonist/DLCS.Repository.Tests/NamedQueries/Parsing/IIIFNamedQueryParserTests.cs b/src/protagonist/DLCS.Repository.Tests/NamedQueries/Parsing/IIIFNamedQueryParserTests.cs index 6f85b33b6..c5b750ad8 100644 --- a/src/protagonist/DLCS.Repository.Tests/NamedQueries/Parsing/IIIFNamedQueryParserTests.cs +++ b/src/protagonist/DLCS.Repository.Tests/NamedQueries/Parsing/IIIFNamedQueryParserTests.cs @@ -32,11 +32,25 @@ public void GenerateParsedNamedQueryFromRequest_Throws_IfTemplateEmptyOrWhiteSpa .WithMessage("Value cannot be null. (Parameter 'namedQueryTemplate')"); } + [Fact] + public void GenerateParsedNamedQueryFromRequest_ReturnFaultyNQ_IfNoParametersPassed() + { + // Act + var result = + sut.GenerateParsedNamedQueryFromRequest(Customer, "", "s1=p1&space=p2", "my-query"); + + // Assert + result.IsFaulty.Should().BeTrue(); + result.ErrorMessage.Should().StartWith("Named query must have at least 1 argument"); + } + [Theory] - [InlineData("space=p1", "")] - [InlineData("space=p1&s1=p2", "1")] + [InlineData("s1=p1&space=p2", "1")] + [InlineData("s1=p1&n1=p2", "1")] + [InlineData("s1=p1&n1=&n2=p2", "1/2")] [InlineData("space=p1&s1=p2&#=1", "")] - public void GenerateParsedNamedQueryFromRequest_ReturnsFaultParsedNQ_IfTooFewParamsPassed(string template, + [InlineData("space=p1&s1=p2&#=1", "10")] + public void GenerateParsedNamedQueryFromRequest_ReturnsNQ_IfLessQueriesPassedThanParameters(string template, string args) { // Act @@ -44,8 +58,7 @@ public void GenerateParsedNamedQueryFromRequest_ReturnsFaultParsedNQ_IfTooFewPar sut.GenerateParsedNamedQueryFromRequest(Customer, args, template, "my-query"); // Assert - result.IsFaulty.Should().BeTrue(); - result.ErrorMessage.Should().StartWith("Not enough query arguments to satisfy template element parameter"); + result.IsFaulty.Should().BeFalse(); } [Theory] diff --git a/src/protagonist/DLCS.Repository.Tests/NamedQueries/Parsing/PdfNamedQueryParserTests.cs b/src/protagonist/DLCS.Repository.Tests/NamedQueries/Parsing/PdfNamedQueryParserTests.cs index fbb5251b7..e3488efc9 100644 --- a/src/protagonist/DLCS.Repository.Tests/NamedQueries/Parsing/PdfNamedQueryParserTests.cs +++ b/src/protagonist/DLCS.Repository.Tests/NamedQueries/Parsing/PdfNamedQueryParserTests.cs @@ -34,11 +34,25 @@ public void GenerateParsedNamedQueryFromRequest_Throws_IfTemplateEmptyOrWhiteSpa .WithMessage("Value cannot be null. (Parameter 'namedQueryTemplate')"); } + [Fact] + public void GenerateParsedNamedQueryFromRequest_ReturnFaultyNQ_IfNoParametersPassed() + { + // Act + var result = + sut.GenerateParsedNamedQueryFromRequest(99, "", "s1=p1&space=p2", "my-query"); + + // Assert + result.IsFaulty.Should().BeTrue(); + result.ErrorMessage.Should().StartWith("Named query must have at least 1 argument"); + } + [Theory] - [InlineData("space=p1", "")] - [InlineData("space=p1&s1=p2", "1")] + [InlineData("s1=p1&space=p2", "1")] + [InlineData("s1=p1&n1=p2", "1")] + [InlineData("s1=p1&n1=&n2=p2", "1/2")] [InlineData("space=p1&s1=p2&#=1", "")] - public void GenerateParsedNamedQueryFromRequest_ReturnsFaultParsedNQ_IfTooFewParamsPassed(string template, + [InlineData("space=p1&s1=p2&#=1", "10")] + public void GenerateParsedNamedQueryFromRequest_ReturnsNQ_IfLessQueriesPassedThanParameters(string template, string args) { // Act @@ -46,8 +60,7 @@ public void GenerateParsedNamedQueryFromRequest_ReturnsFaultParsedNQ_IfTooFewPar sut.GenerateParsedNamedQueryFromRequest(99, args, template, "my-query"); // Assert - result.IsFaulty.Should().BeTrue(); - result.ErrorMessage.Should().StartWith("Not enough query arguments to satisfy template element parameter"); + result.IsFaulty.Should().BeFalse(); } [Theory] diff --git a/src/protagonist/DLCS.Repository/Assets/AssetApplicationMetadataRepository.cs b/src/protagonist/DLCS.Repository/Assets/AssetApplicationMetadataRepository.cs index 7b5f64f9f..1e0e92de6 100644 --- a/src/protagonist/DLCS.Repository/Assets/AssetApplicationMetadataRepository.cs +++ b/src/protagonist/DLCS.Repository/Assets/AssetApplicationMetadataRepository.cs @@ -4,16 +4,20 @@ using DLCS.Core.Types; using DLCS.Model.Assets.Metadata; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using Z.EntityFramework.Plus; namespace DLCS.Repository.Assets; public class AssetApplicationMetadataRepository : IAssetApplicationMetadataRepository { private readonly DlcsContext dlcsContext; + private readonly ILogger logger; - public AssetApplicationMetadataRepository(DlcsContext dlcsContext) + public AssetApplicationMetadataRepository(DlcsContext dlcsContext, ILogger logger) { this.dlcsContext = dlcsContext; + this.logger = logger; } /// @@ -49,4 +53,22 @@ public async Task UpsertApplicationMetadata(AssetId as await dlcsContext.SaveChangesAsync(cancellationToken); return assetApplicationMetadata; } + + public async Task DeleteAssetApplicationMetadata(AssetId assetId, string metadataType, + CancellationToken cancellationToken = default) + { + var assetApplicationMetadata = await dlcsContext.AssetApplicationMetadata + .SingleOrDefaultAsync(i => i.AssetId == assetId && i.MetadataType == metadataType, cancellationToken); + + if (assetApplicationMetadata == null) + { + logger.LogDebug("Attempt to delete non-existent asset metadata {MetadataType} for {AssetId}", metadataType, + assetId); + return false; + } + + dlcsContext.AssetApplicationMetadata.Remove(assetApplicationMetadata); + await dlcsContext.SaveChangesAsync(cancellationToken); + return true; + } } \ No newline at end of file diff --git a/src/protagonist/DLCS.Repository/Assets/AssetQueryX.cs b/src/protagonist/DLCS.Repository/Assets/AssetQueryX.cs index 7bf97f454..f2d5986b8 100644 --- a/src/protagonist/DLCS.Repository/Assets/AssetQueryX.cs +++ b/src/protagonist/DLCS.Repository/Assets/AssetQueryX.cs @@ -26,7 +26,7 @@ public static IQueryable AsOrderedAssetQuery(this IQueryable asset /// The orderBy field can be the API version of property or the full property version. /// Defaults to "Created" field ordering if no field specified. /// - public static IQueryable AsOrderedAssetQuery(this IQueryable assetQuery, string? orderBy, + private static IQueryable AsOrderedAssetQuery(this IQueryable assetQuery, string? orderBy, bool descending = false) { var field = GetPropertyName(orderBy); @@ -57,7 +57,6 @@ private static string GetPropertyName(string? orderBy) }; } - // Create an Expression from the PropertyName. // I think Split(".") handles nested properties maybe - seems unnecessary but from an SO post // "x" means nothing when creating the Parameter, it's just used for debug messages @@ -115,11 +114,12 @@ public static IQueryable ApplyAssetFilter(this IQueryable queryabl return filtered; } - + /// /// Include asset delivery channels and their associated policies. /// public static IQueryable IncludeDeliveryChannelsWithPolicy(this IQueryable assetQuery) - => assetQuery.Include(a => a.ImageDeliveryChannels) + => assetQuery + .Include(a => a.ImageDeliveryChannels.OrderBy(idc => idc.Channel)) .ThenInclude(dc => dc.DeliveryChannelPolicy); } \ No newline at end of file diff --git a/src/protagonist/DLCS.Repository/Assets/DapperAssetRepository.cs b/src/protagonist/DLCS.Repository/Assets/DapperAssetRepository.cs index 26abf3b94..4be176488 100644 --- a/src/protagonist/DLCS.Repository/Assets/DapperAssetRepository.cs +++ b/src/protagonist/DLCS.Repository/Assets/DapperAssetRepository.cs @@ -22,7 +22,7 @@ public DapperAssetRepository( Configuration = configuration; this.assetCachingHelper = assetCachingHelper; } - + public async Task GetImageLocation(AssetId assetId) => await this.QuerySingleOrDefaultAsync(ImageLocationSql, new {Id = assetId.ToString()}); diff --git a/src/protagonist/DLCS.Repository/DLCS.Repository.csproj b/src/protagonist/DLCS.Repository/DLCS.Repository.csproj index 8f13fc325..b850efeae 100644 --- a/src/protagonist/DLCS.Repository/DLCS.Repository.csproj +++ b/src/protagonist/DLCS.Repository/DLCS.Repository.csproj @@ -8,7 +8,7 @@ - + all diff --git a/src/protagonist/DLCS.Repository/Messaging/EngineClient.cs b/src/protagonist/DLCS.Repository/Messaging/EngineClient.cs index 9aa2feae5..33dbaae4d 100644 --- a/src/protagonist/DLCS.Repository/Messaging/EngineClient.cs +++ b/src/protagonist/DLCS.Repository/Messaging/EngineClient.cs @@ -9,10 +9,14 @@ using System.Text.Json.Serialization; using System.Threading; using System.Threading.Tasks; +using DLCS.AWS.ElasticTranscoder.Models; using DLCS.AWS.SQS; +using DLCS.Core.Caching; using DLCS.Model.Assets; using DLCS.Model.Messaging; +using LazyCache; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; namespace DLCS.Repository.Messaging; @@ -24,22 +28,31 @@ public class EngineClient : IEngineClient private readonly IQueueLookup queueLookup; private readonly IQueueSender queueSender; private readonly HttpClient httpClient; + private readonly CacheSettings cacheSettings; + private readonly IAppCache appCache; private readonly ILogger logger; private static readonly JsonSerializerOptions SerializerOptions = new (JsonSerializerDefaults.Web) { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, }; + + private static readonly IReadOnlyDictionary NullPresetDictionary = + new Dictionary(); public EngineClient( IQueueLookup queueLookup, IQueueSender queueSender, HttpClient httpClient, + IAppCache appCache, + IOptions cacheOptions, ILogger logger) { this.queueLookup = queueLookup; this.queueSender = queueSender; this.httpClient = httpClient; + this.appCache = appCache; + cacheSettings = cacheOptions.Value; this.logger = logger; } @@ -146,6 +159,26 @@ public async Task AsynchronousIngestBatch(IReadOnlyCollection assets } } + public async Task?> GetAvPresets(CancellationToken cancellationToken = default) + { + const string key = "avPresetList"; + return await appCache.GetOrAddAsync(key, async entry => + { + try + { + var response = await httpClient.GetAsync("av-presets", cancellationToken); + return await response.Content.ReadFromJsonAsync>( + cancellationToken: cancellationToken) ?? new Dictionary(); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to retrieve allowed iiif-av policy options from Engine"); + entry.AbsoluteExpirationRelativeToNow = TimeSpan.FromSeconds(cacheSettings.GetTtl(CacheDuration.Short)); + return NullPresetDictionary; + } + }, cacheSettings.GetMemoryCacheOptions(CacheDuration.Long)); + } + private string GetJsonString(Asset asset) { var ingestAssetRequest = new IngestAssetRequest(asset.Id, DateTime.UtcNow); diff --git a/src/protagonist/DLCS.Repository/Messaging/IEngineClient.cs b/src/protagonist/DLCS.Repository/Messaging/IEngineClient.cs index 7df9a8ee0..dd90d59e4 100644 --- a/src/protagonist/DLCS.Repository/Messaging/IEngineClient.cs +++ b/src/protagonist/DLCS.Repository/Messaging/IEngineClient.cs @@ -2,6 +2,7 @@ using System.Net; using System.Threading; using System.Threading.Tasks; +using DLCS.AWS.ElasticTranscoder.Models; using DLCS.Model.Assets; using DLCS.Model.Messaging; @@ -42,4 +43,11 @@ Task AsynchronousIngestBatch(IReadOnlyCollection assets, /// /// Current cancellation token Task?> GetAllowedAvPolicyOptions(CancellationToken cancellationToken = default); + + /// + /// Retrieves av presets + /// + /// a cancellation token + /// A dictionary of identifiers and presets + Task?> GetAvPresets(CancellationToken cancellationToken = default); } \ No newline at end of file diff --git a/src/protagonist/DLCS.Repository/NamedQueries/Parsing/BaseNamedQueryParser.cs b/src/protagonist/DLCS.Repository/NamedQueries/Parsing/BaseNamedQueryParser.cs index 3f507844f..c15bc56dd 100644 --- a/src/protagonist/DLCS.Repository/NamedQueries/Parsing/BaseNamedQueryParser.cs +++ b/src/protagonist/DLCS.Repository/NamedQueries/Parsing/BaseNamedQueryParser.cs @@ -35,7 +35,7 @@ public abstract class BaseNamedQueryParser : INamedQueryParser protected const string String3 = "s3"; protected const string AssetOrdering = "assetOrder"; protected const string PathReplacement = "%2F"; - + public BaseNamedQueryParser(ILogger logger) { Logger = logger; @@ -91,7 +91,7 @@ private static List GetQueryArgsList(string? namedQueryArgs, string[] te private T GenerateParsedNamedQuery(int customerId, string[] templatePairing, List queryArgs) { var assetQuery = GenerateParsedQueryObject(customerId); - + // Iterate through all of the pairs and generate the NQ model try { @@ -107,7 +107,8 @@ private T GenerateParsedNamedQuery(int customerId, string[] templatePairing, Lis assetQuery.AssetOrdering = GetAssetOrderingFromTemplateElement(elements[1]); break; case Space: - assetQuery.Space = int.Parse(GetQueryArgumentFromTemplateElement(queryArgs, elements[1])); + assetQuery.Space = + (int?)ConvertToLongQueryArg(GetQueryArgumentFromTemplateElement(queryArgs, elements[1])); break; case SpaceName: assetQuery.SpaceName = GetQueryArgumentFromTemplateElement(queryArgs, elements[1]); @@ -123,15 +124,15 @@ private T GenerateParsedNamedQuery(int customerId, string[] templatePairing, Lis break; case Number1: assetQuery.Number1 = - long.Parse(GetQueryArgumentFromTemplateElement(queryArgs, elements[1])); + ConvertToLongQueryArg(GetQueryArgumentFromTemplateElement(queryArgs, elements[1])); break; case Number2: - assetQuery.Number2 = - long.Parse(GetQueryArgumentFromTemplateElement(queryArgs, elements[1])); + assetQuery.Number2 = + ConvertToLongQueryArg(GetQueryArgumentFromTemplateElement(queryArgs, elements[1])); break; case Number3: - assetQuery.Number3 = - long.Parse(GetQueryArgumentFromTemplateElement(queryArgs, elements[1])); + assetQuery.Number3 = + ConvertToLongQueryArg(GetQueryArgumentFromTemplateElement(queryArgs, elements[1])); break; } @@ -147,6 +148,16 @@ private T GenerateParsedNamedQuery(int customerId, string[] templatePairing, Lis return assetQuery; } + private long? ConvertToLongQueryArg(string? argToConvert) + { + if (argToConvert.IsNullOrEmpty()) + { + return null; + } + + return long.Parse(argToConvert); + } + /// /// Adds handling for any custom key/value pairs, in addition to the core s1, s2, p1 etc /// @@ -162,7 +173,7 @@ private T GenerateParsedNamedQuery(int customerId, string[] templatePairing, Lis /// Could use Activator.CreateInstance this avoids using reflection protected abstract T GenerateParsedQueryObject(int customerId); - protected string GetQueryArgumentFromTemplateElement(List args, string element) + protected string? GetQueryArgumentFromTemplateElement(List args, string element) { // Arg will be in format p1, p2, p3 etc. Get the index, then extract that element from args list if (!element.StartsWith(ParameterPrefix) || element.Length <= 1) @@ -170,6 +181,11 @@ protected string GetQueryArgumentFromTemplateElement(List args, string e // default to just return the element as a literal return element; } + + if (args.Count == 0) + { + throw new ArgumentException("Named query must have at least 1 argument"); + } if (int.TryParse(element[1..], out int argNumber)) { @@ -178,8 +194,8 @@ protected string GetQueryArgumentFromTemplateElement(List args, string e return args[argNumber - 1].Replace(PathReplacement, "/", StringComparison.OrdinalIgnoreCase); } - throw new ArgumentOutOfRangeException(element, - "Not enough query arguments to satisfy template element parameter"); + // parameter out of range of supplied arguments, assumed to be an optional param to the NQ + return null; } throw new ArgumentException($"Could not parse template element parameter '{element}'", element); diff --git a/src/protagonist/DLCS.Web/DLCS.Web.csproj b/src/protagonist/DLCS.Web/DLCS.Web.csproj index 2f84c51ca..e7205ab18 100644 --- a/src/protagonist/DLCS.Web/DLCS.Web.csproj +++ b/src/protagonist/DLCS.Web/DLCS.Web.csproj @@ -12,7 +12,7 @@ - + diff --git a/src/protagonist/Engine.Tests/Ingest/Image/ImageServer/Clients/CantaloupeThumbsClientTests.cs b/src/protagonist/Engine.Tests/Ingest/Image/ImageServer/Clients/CantaloupeThumbsClientTests.cs index 27fc69718..85a3c8d37 100644 --- a/src/protagonist/Engine.Tests/Ingest/Image/ImageServer/Clients/CantaloupeThumbsClientTests.cs +++ b/src/protagonist/Engine.Tests/Ingest/Image/ImageServer/Clients/CantaloupeThumbsClientTests.cs @@ -6,9 +6,14 @@ using Engine.Ingest.Image; using Engine.Ingest.Image.ImageServer.Clients; using Engine.Ingest.Image.ImageServer.Measuring; +using Engine.Settings; using FakeItEasy; using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Net.Http.Headers; +using Test.Helpers.Data; using Test.Helpers.Http; +using Test.Helpers.Settings; +using CookieHeaderValue = System.Net.Http.Headers.CookieHeaderValue; namespace Engine.Tests.Ingest.Image.ImageServer.Clients; @@ -17,6 +22,7 @@ public class CantaloupeThumbsClientTests private readonly ControllableHttpMessageHandler httpHandler; private readonly CantaloupeThumbsClient sut; private readonly IImageMeasurer imageMeasurer; + private readonly HttpClient httpClient; private readonly List defaultThumbs = new() { @@ -33,9 +39,17 @@ public CantaloupeThumbsClientTests() A.CallTo(() => imageMeasurer.MeasureImage(A._, A._)).Returns(new ImageOnDisk()); - var httpClient = new HttpClient(httpHandler); + httpClient = new HttpClient(httpHandler); httpClient.BaseAddress = new Uri("http://image-processor/"); - sut = new CantaloupeThumbsClient(httpClient, fileSystem, imageMeasurer, new NullLogger()); + + var engineSettings = new EngineSettings + { + ImageIngest = new ImageIngestSettings() + }; + var optionsMonitor = OptionsHelpers.GetOptionsMonitor(engineSettings); + + + sut = new CantaloupeThumbsClient(httpClient, fileSystem, imageMeasurer, optionsMonitor, new NullLogger()); } [Fact] @@ -299,6 +313,110 @@ public async Task GenerateThumbnails_ReturnsThumbForSuccessfulResponse_AfterFirs "Landscape images - invalid sizes altered", }, }; + + [Fact] + public async Task GenerateThumbnails_UpdatesHandlerWithCookies() + { + // Arrange + var assetId = AssetIdGenerator.GetAssetId(); + var context = IngestionContextFactory.GetIngestionContext(assetId: assetId.ToString()); + + var response = new HttpResponseMessage(HttpStatusCode.OK); + response.Headers.Add(HeaderNames.SetCookie, new List() + { + "AWSALB=_remove_; Path=/", + "AWSALBCORS=_remove_; Path=/" + }); + httpHandler.SetResponse(response); + List cookieHeaders = new(); + + context.Asset.Width = 2000; + context.Asset.Height = 2000; + + context.WithLocation(new ImageLocation + { + S3 = "//some/location/with/s3" + }); + + await sut.GenerateThumbnails(context, defaultThumbs, ThumbsRoot); + + httpHandler.RegisterCallback(message => cookieHeaders = message.Headers.GetCookies().ToList()); + httpHandler.GetResponseMessage("{ \"engine\": \"hello\" }", HttpStatusCode.OK); + + // Act + await sut.GenerateThumbnails(context, defaultThumbs, ThumbsRoot); + + // Assert + cookieHeaders.Count.Should().Be(2); + cookieHeaders[0].Cookies[0].Name.Should().Be("AWSALB"); + cookieHeaders[0].Cookies[0].Value.Should().Be("_remove_"); + cookieHeaders[1].Cookies[0].Name.Should().Be("AWSALBCORS"); + } + + [Fact] + public async Task GenerateThumbnails_DoesNotUpdateHandlerWithCookiesWhenUnrecognised() + { + // Arrange + var assetId = AssetIdGenerator.GetAssetId(); + var context = IngestionContextFactory.GetIngestionContext(assetId: assetId.ToString()); + + var response = new HttpResponseMessage(HttpStatusCode.OK); + response.Headers.Add(HeaderNames.SetCookie, new List() + { + "SOMECOOKIE=_remove_; Path=/", + "SOMECOOKIE2=_remove_; Path=/" + }); + httpHandler.SetResponse(response); + List cookieHeaders = new(); + + context.Asset.Width = 2000; + context.Asset.Height = 2000; + + context.WithLocation(new ImageLocation + { + S3 = "//some/location/with/s3" + }); + + await sut.GenerateThumbnails(context, defaultThumbs, ThumbsRoot); + + httpHandler.RegisterCallback(message => cookieHeaders = message.Headers.GetCookies().ToList()); + httpHandler.GetResponseMessage("{ \"engine\": \"hello\" }", HttpStatusCode.OK); + + // Act + await sut.GenerateThumbnails(context, defaultThumbs, ThumbsRoot); + + // Assert + cookieHeaders.Count.Should().Be(0); + } + + [Fact] + public async Task GenerateThumbnails_UpdatesHandlerWithNoCookiesSet() + { + // Arrange + var assetId = new AssetId(2, 1, nameof(GenerateThumbnails_ReturnsThumbForSuccessfulResponse)); + var context = IngestionContextFactory.GetIngestionContext(assetId: assetId.ToString()); + + httpHandler.SetResponse(new HttpResponseMessage(HttpStatusCode.OK)); + context.Asset.Width = 2000; + context.Asset.Height = 2000; + + context.WithLocation(new ImageLocation + { + S3 = "//some/location/with/s3" + }); + + await sut.GenerateThumbnails(context, defaultThumbs, ThumbsRoot); + + List cookieHeaders = new(); + httpHandler.RegisterCallback(message => cookieHeaders = message.Headers.GetCookies().ToList()); + httpHandler.GetResponseMessage("{ \"engine\": \"hello\" }", HttpStatusCode.OK); + + // Act + await sut.GenerateThumbnails(context, defaultThumbs, ThumbsRoot); + + // Assert + cookieHeaders.Count.Should().Be(0); + } public class ImageOnDiskResults { diff --git a/src/protagonist/Engine.Tests/Ingest/IngestControllerTests.cs b/src/protagonist/Engine.Tests/Ingest/IngestControllerTests.cs index 33907cb6a..cec36e8b3 100644 --- a/src/protagonist/Engine.Tests/Ingest/IngestControllerTests.cs +++ b/src/protagonist/Engine.Tests/Ingest/IngestControllerTests.cs @@ -1,4 +1,6 @@ -using Engine.Ingest; +using DLCS.AWS.ElasticTranscoder; +using DLCS.AWS.ElasticTranscoder.Models; +using Engine.Ingest; using Engine.Settings; using FakeItEasy; using Microsoft.AspNetCore.Mvc; @@ -10,10 +12,12 @@ public class IngestControllerTests { private IngestController sut; private IAssetIngester ingester; + private IElasticTranscoderWrapper elasticTranscoderWrapper; public IngestControllerTests() { - ingester = A.Fake(); + ingester = A.Fake(); + elasticTranscoderWrapper = A.Fake(); var engineSettings = new EngineSettings { TimebasedIngest = new TimebasedIngestSettings() @@ -26,11 +30,18 @@ public IngestControllerTests() } }; - sut = new IngestController(ingester, Options.Create(engineSettings)); + A.CallTo(() => elasticTranscoderWrapper.GetPresetIdLookup(A._)).Returns( + new Dictionary() + { + { "An amazon policy", new TranscoderPreset("some-id", "An amazon policy", ".ext") }, + { "An amazon policy 2", new TranscoderPreset("some-id-2", "An amazon policy 2", ".ext2") } + }); + + sut = new IngestController(ingester, elasticTranscoderWrapper, Options.Create(engineSettings)); } [Fact] - public void ReturnAllowedAvOptions_ReturnsAvOptions_WhenCalled() + public void GetAllowedAvOptions_ReturnsAvOptions_WhenCalled() { // Arrange and Act var avReturn = sut.GetAllowedAvOptions(); @@ -46,7 +57,7 @@ public void ReturnAllowedAvOptions_ReturnsAvOptions_WhenCalled() } [Fact] - public void ReturnAllowedAvOptions_ReturnsEmptyList_WhenCalledWithDefaultSettings() + public void GetAllowedAvOptions_ReturnsEmptyList_WhenCalledWithDefaultSettings() { // Arrange and var engineSettings = new EngineSettings() @@ -54,7 +65,7 @@ public void ReturnAllowedAvOptions_ReturnsEmptyList_WhenCalledWithDefaultSetting TimebasedIngest = new TimebasedIngestSettings() }; - var ingestController = new IngestController(ingester, Options.Create(engineSettings)); + var ingestController = new IngestController(ingester, elasticTranscoderWrapper, Options.Create(engineSettings)); // Act var avReturn = ingestController.GetAllowedAvOptions(); @@ -66,4 +77,44 @@ public void ReturnAllowedAvOptions_ReturnsEmptyList_WhenCalledWithDefaultSetting options.StatusCode.Should().Be(200); avOptions.Count.Should().Be(0); } + + [Fact] + public async Task GetAllowedAvPresetOptions_ReturnsAvOptions_WhenCalled() + { + // Arrange and Act + var avReturn = await sut.GetAllowedAvPresetOptions(); + + var options = avReturn as OkObjectResult; + var avOptions = options.Value as Dictionary; + + // Assert + options.StatusCode.Should().Be(200); + avOptions.Count.Should().Be(2); + avOptions.Keys.Should().Contain("somePolicy"); + avOptions.Keys.Should().Contain("somePolicy"); + avOptions.Values.Any(x => x.Name == "An amazon policy").Should().BeTrue(); + avOptions.Values.Any(x => x.Name == "An amazon policy 2").Should().BeTrue(); + } + + [Fact] + public async Task GetAllowedAvPresetOptions_ReturnsEmptyList_WhenCalledWithDefaultSettings() + { + // Arrange and + var engineSettings = new EngineSettings() + { + TimebasedIngest = new TimebasedIngestSettings() + }; + + var ingestController = new IngestController(ingester, elasticTranscoderWrapper, Options.Create(engineSettings)); + + // Act + var avReturn = await ingestController.GetAllowedAvPresetOptions(); + + var options = avReturn as OkObjectResult; + var avOptions = options.Value as Dictionary; + + // Assert + options.StatusCode.Should().Be(200); + avOptions.Count.Should().Be(0); + } } \ No newline at end of file diff --git a/src/protagonist/Engine.Tests/Ingest/Timebased/Transcode/ElasticTranscoderTests.cs b/src/protagonist/Engine.Tests/Ingest/Timebased/Transcode/ElasticTranscoderTests.cs index 64639086a..480f1095a 100644 --- a/src/protagonist/Engine.Tests/Ingest/Timebased/Transcode/ElasticTranscoderTests.cs +++ b/src/protagonist/Engine.Tests/Ingest/Timebased/Transcode/ElasticTranscoderTests.cs @@ -1,6 +1,7 @@ using System.Net; using Amazon.ElasticTranscoder.Model; using DLCS.AWS.ElasticTranscoder; +using DLCS.AWS.ElasticTranscoder.Models; using DLCS.Core.Types; using DLCS.Model.Assets; using DLCS.Model.Policies; @@ -157,11 +158,11 @@ public async Task InitiateTranscodeOperation_MakesCreateJobRequest() .Returns("1234567890123-abcdef"); A.CallTo(() => elasticTranscoderWrapper.GetPresetIdLookup(A._)) - .Returns(new Dictionary() + .Returns(new Dictionary() { - ["Standard WebM"] = "1111111111111-aaaaaa", - ["Standard mp4"] = "1111111111111-aaaaab", - ["auto-preset"] = "9999999999999-bbbbbb" + ["Standard WebM"] = new ("1111111111111-aaaaaa", "Standard WebM", ""), + ["Standard mp4"] = new ("1111111111111-aaaaab", "Standard mp4", ""), + ["auto-preset"] = new ("9999999999999-bbbbbb", "auto-preset", "") }); Dictionary? metadata = null; @@ -224,10 +225,10 @@ public async Task InitiateTranscodeOperation_ReturnsFalseAndSetsError_IfErrorSta .Returns("1234567890123-abcdef"); A.CallTo(() => elasticTranscoderWrapper.GetPresetIdLookup(A._)) - .Returns(new Dictionary() + .Returns(new Dictionary() { - ["Standard mp4"] = "1111111111111-aaaaab", - ["auto-preset"] = "9999999999999-bbbbbb" + ["Standard mp4"] = new ("1111111111111-aaaaab", "Standard mp4", ""), + ["auto-preset"] = new ("9999999999999-bbbbbb", "auto-preset", "") }); A.CallTo(() => elasticTranscoderWrapper.CreateJob(A._, A._, @@ -277,10 +278,10 @@ public async Task InitiateTranscodeOperation_ReturnsTrue_IfSuccessStatusCodeFrom .Returns(elasticTranscoderJobId); A.CallTo(() => elasticTranscoderWrapper.GetPresetIdLookup(A._)) - .Returns(new Dictionary + .Returns(new Dictionary { - ["Standard mp4"] = "1111111111111-aaaaab", - ["auto-preset"] = "9999999999999-bbbbbb" + ["Standard mp4"] = new ("1111111111111-aaaaab", "Standard mp4", ""), + ["auto-preset"] = new ("9999999999999-bbbbbb", "auto-preset", "") }); A.CallTo(() => elasticTranscoderWrapper.CreateJob(A._, A._, diff --git a/src/protagonist/Engine.Tests/Integration/TimebasedIngestTests.cs b/src/protagonist/Engine.Tests/Integration/TimebasedIngestTests.cs index 3bc402209..e70c42d5b 100644 --- a/src/protagonist/Engine.Tests/Integration/TimebasedIngestTests.cs +++ b/src/protagonist/Engine.Tests/Integration/TimebasedIngestTests.cs @@ -3,6 +3,7 @@ using System.Text.Json; using Amazon.ElasticTranscoder.Model; using DLCS.AWS.ElasticTranscoder; +using DLCS.AWS.ElasticTranscoder.Models; using DLCS.AWS.S3; using DLCS.Core.FileSystem; using DLCS.Core.Types; @@ -71,10 +72,10 @@ public TimebasedIngestTests(ProtagonistAppFactory appFactory, EngineFix A.CallTo(() => ElasticTranscoderWrapper.GetPipelineId("protagonist-pipeline", A._)) .Returns("pipeline-id-1234"); A.CallTo(() => ElasticTranscoderWrapper.GetPresetIdLookup(A._)) - .Returns(new Dictionary + .Returns(new Dictionary { - ["System preset: Generic 720p"] = "123-123", - ["System preset: Audio MP3 - 128k"] = "456-456" + ["System preset: Generic 720p"] = new ("123-123", "System preset: Generic 720p", ""), + ["System preset: Audio MP3 - 128k"] = new ("456-456", "System preset: Audio MP3 - 128k", "") }); } diff --git a/src/protagonist/Engine/Engine.csproj b/src/protagonist/Engine/Engine.csproj index b5dd83659..09779ca0d 100644 --- a/src/protagonist/Engine/Engine.csproj +++ b/src/protagonist/Engine/Engine.csproj @@ -7,7 +7,7 @@ - + diff --git a/src/protagonist/Engine/Infrastructure/ServiceCollectionX.cs b/src/protagonist/Engine/Infrastructure/ServiceCollectionX.cs index c8edfe3e8..a8413f4ab 100644 --- a/src/protagonist/Engine/Infrastructure/ServiceCollectionX.cs +++ b/src/protagonist/Engine/Infrastructure/ServiceCollectionX.cs @@ -102,18 +102,19 @@ public static IServiceCollection AddAssetIngestion(this IServiceCollection servi services.AddTransient(); services.AddScoped() .AddScoped(); - + services.AddHttpClient(client => { client.BaseAddress = engineSettings.ImageIngest.ImageProcessorUrl; client.Timeout = TimeSpan.FromMilliseconds(engineSettings.ImageIngest.ImageProcessorTimeoutMs); }).AddHttpMessageHandler(); - + services.AddHttpClient(client => - { - client.BaseAddress = engineSettings.ImageIngest.ThumbsProcessorUrl; - client.Timeout = TimeSpan.FromMilliseconds(engineSettings.ImageIngest.ImageProcessorTimeoutMs); - }).AddHttpMessageHandler(); + { + client.BaseAddress = engineSettings.ImageIngest.ThumbsProcessorUrl; + client.Timeout = TimeSpan.FromMilliseconds(engineSettings.ImageIngest.ImageProcessorTimeoutMs); + }).ConfigurePrimaryHttpMessageHandler(() => new HttpClientHandler() { UseCookies = false }) + .AddHttpMessageHandler(); services.AddHttpClient(client => { diff --git a/src/protagonist/Engine/Ingest/Image/ImageServer/Clients/CantaloupeThumbsClient.cs b/src/protagonist/Engine/Ingest/Image/ImageServer/Clients/CantaloupeThumbsClient.cs index fb85599ef..967450862 100644 --- a/src/protagonist/Engine/Ingest/Image/ImageServer/Clients/CantaloupeThumbsClient.cs +++ b/src/protagonist/Engine/Ingest/Image/ImageServer/Clients/CantaloupeThumbsClient.cs @@ -3,8 +3,11 @@ using DLCS.Core.FileSystem; using DLCS.Core.Types; using Engine.Ingest.Image.ImageServer.Measuring; +using Engine.Settings; using IIIF; using IIIF.ImageApi; +using Microsoft.Extensions.Options; +using Microsoft.Net.Http.Headers; namespace Engine.Ingest.Image.ImageServer.Clients; @@ -17,17 +20,21 @@ public class CantaloupeThumbsClient : IThumbsClient private readonly IFileSystem fileSystem; private readonly IImageMeasurer imageMeasurer; private readonly ILogger logger; - + private List loadBalancerCookies = new(); + private readonly EngineSettings engineSettings; + public CantaloupeThumbsClient( HttpClient cantaloupeClient, IFileSystem fileSystem, IImageMeasurer imageMeasurer, + IOptionsMonitor engineOptionsMonitor, ILogger logger) { this.cantaloupeClient = cantaloupeClient; this.fileSystem = fileSystem; this.imageMeasurer = imageMeasurer; this.logger = logger; + engineSettings = engineOptionsMonitor.CurrentValue; } public async Task> GenerateThumbnails(IngestionContext context, @@ -63,9 +70,11 @@ public async Task> GenerateThumbnails(IngestionContext context string convertedS3Location, string size, AssetId assetId, int count, List thumbsResponse, Size imageSize, bool shouldRetry, CancellationToken cancellationToken) { - using var response = - await cantaloupeClient.GetAsync( - $"iiif/3/{convertedS3Location}/full/{size}/0/default.jpg", cancellationToken); + var request = CreateCantaloupeRequestMessage(convertedS3Location, size); + + using var response = await cantaloupeClient.SendAsync(request, cancellationToken); + + AttemptToAddStickinessCookie(response); if (response.StatusCode == HttpStatusCode.BadRequest) { @@ -84,6 +93,35 @@ await cantaloupeClient.GetAsync( throw new HttpException(response.StatusCode, "failed to retrieve data from the thumbs processor"); } + private HttpRequestMessage CreateCantaloupeRequestMessage(string convertedS3Location, string size) + { + var request = new HttpRequestMessage(HttpMethod.Get, $"iiif/3/{convertedS3Location}/full/{size}/0/default.jpg"); + + if (loadBalancerCookies.Any()) + { + request.Headers.Add(HeaderNames.Cookie, loadBalancerCookies); + } + + return request; + } + + private void AttemptToAddStickinessCookie(HttpResponseMessage response) + { + var hasCookie = response.Headers.TryGetValues(HeaderNames.SetCookie, out var cookies); + if (hasCookie) + { + loadBalancerCookies = new List(); + foreach (var cookie in cookies!) + { + if (engineSettings.ImageIngest!.LoadBalancerStickinessCookieNames.Any(c => + cookie.Split(';').Any(h => h.Trim(' ').StartsWith($"{c}=")))) + { + loadBalancerCookies.Add(cookie); + } + } + } + } + private async Task SaveImageToDisk(HttpResponseMessage response, string size, string thumbFolder, int count, bool shouldRetry, string convertedS3Location, AssetId assetId, List thumbsResponse, Size imageSize, CancellationToken cancellationToken) diff --git a/src/protagonist/Engine/Ingest/IngestController.cs b/src/protagonist/Engine/Ingest/IngestController.cs index 8319d3a69..3262522dc 100644 --- a/src/protagonist/Engine/Ingest/IngestController.cs +++ b/src/protagonist/Engine/Ingest/IngestController.cs @@ -1,5 +1,6 @@ using System.Net; using System.Text.Json; +using DLCS.AWS.ElasticTranscoder; using DLCS.Model.Messaging; using Engine.Settings; using Microsoft.AspNetCore.Mvc; @@ -12,11 +13,14 @@ public class IngestController : Controller { private readonly IAssetIngester ingester; private static readonly JsonSerializerOptions JsonSerializerOptions = new(JsonSerializerDefaults.Web); + private readonly IElasticTranscoderWrapper elasticTranscoderWrapper; private TimebasedIngestSettings timebasedIngestSettings; - public IngestController(IAssetIngester ingester, IOptions engineSettings) + public IngestController(IAssetIngester ingester, IElasticTranscoderWrapper elasticTranscoderWrapper, + IOptions engineSettings) { this.ingester = ingester; + this.elasticTranscoderWrapper = elasticTranscoderWrapper; timebasedIngestSettings = engineSettings.Value.TimebasedIngest; } @@ -46,6 +50,24 @@ public IActionResult GetAllowedAvOptions() { return Ok(timebasedIngestSettings.DeliveryChannelMappings.Keys.ToList()); } + + /// + /// Retrieve av option presets + /// + [HttpGet] + [Route("av-presets")] + public async Task GetAllowedAvPresetOptions() + { + var presets = await elasticTranscoderWrapper.GetPresetIdLookup(); + + var allowedPresets = + presets.Where(x => timebasedIngestSettings.DeliveryChannelMappings.Values.Contains(x.Key)) + .ToDictionary( + pair => timebasedIngestSettings.DeliveryChannelMappings.First(x => x.Value == pair.Key) + .Key, pair => pair.Value); + + return Ok(allowedPresets); + } private IActionResult ConvertToStatusCode(object message, IngestResultStatus result) => result switch diff --git a/src/protagonist/Engine/Ingest/Timebased/Transcode/ElasticTranscoder.cs b/src/protagonist/Engine/Ingest/Timebased/Transcode/ElasticTranscoder.cs index 6ae7cd542..7afa18c44 100644 --- a/src/protagonist/Engine/Ingest/Timebased/Transcode/ElasticTranscoder.cs +++ b/src/protagonist/Engine/Ingest/Timebased/Transcode/ElasticTranscoder.cs @@ -1,6 +1,7 @@ using System.Text.Json; using Amazon.ElasticTranscoder.Model; using DLCS.AWS.ElasticTranscoder; +using DLCS.AWS.ElasticTranscoder.Models; using DLCS.Core.Guard; using DLCS.Model.Assets; using Engine.Ingest.Timebased.Models; @@ -75,7 +76,7 @@ public async Task InitiateTranscodeOperation(IngestionContext context, Dic } private List GetJobOutputs(IngestionContext context, string jobId, - TimebasedIngestSettings settings, Dictionary presets) + TimebasedIngestSettings settings, Dictionary presets) { var asset = context.Asset; var assetId = context.AssetId; @@ -99,7 +100,7 @@ private List GetJobOutputs(IngestionContext context, string job var destinationPath = TranscoderTemplates.ProcessPreset( mediaType, assetId, jobId, parsedTimeBasedPolicy.Extension); - if (!presets.TryGetValue(mappedPresetName, out var presetId)) + if (!presets.TryGetValue(mappedPresetName, out var transcoderPreset)) { logger.LogWarning("Mapping for preset '{PresetName}' not found!", mappedPresetName); continue; @@ -107,7 +108,7 @@ private List GetJobOutputs(IngestionContext context, string job outputs.Add(new CreateJobOutput { - PresetId = presetId, + PresetId = transcoderPreset.Id, Key = destinationPath, }); diff --git a/src/protagonist/Engine/Settings/EngineSettings.cs b/src/protagonist/Engine/Settings/EngineSettings.cs index 9e93c4035..b7452316d 100644 --- a/src/protagonist/Engine/Settings/EngineSettings.cs +++ b/src/protagonist/Engine/Settings/EngineSettings.cs @@ -114,6 +114,15 @@ public class ImageIngestSettings /// public List DefaultThumbs { get; set; } = new(); + /// + /// A set of cookie names used by the load balancer to indicate stickiness + /// + public List LoadBalancerStickinessCookieNames { get; set; } = new() + { + "AWSALB", + "AWSALBCORS" + }; + /// /// Get the root folder, if forImageProcessor will ensure that it is compatible with needs of image-processor /// sidecar. diff --git a/src/protagonist/Orchestrator.Tests/Features/Images/ImageRequestHandlerTests.cs b/src/protagonist/Orchestrator.Tests/Features/Images/ImageRequestHandlerTests.cs index d7bbb63d3..696145fba 100644 --- a/src/protagonist/Orchestrator.Tests/Features/Images/ImageRequestHandlerTests.cs +++ b/src/protagonist/Orchestrator.Tests/Features/Images/ImageRequestHandlerTests.cs @@ -3,6 +3,7 @@ using System.Net; using System.Text.Json.Nodes; using System.Threading; +using DLCS.Core.Settings; using DLCS.Core.Types; using DLCS.Model.Assets.CustomHeaders; using DLCS.Model.PathElements; diff --git a/src/protagonist/Orchestrator.Tests/Integration/ImageHandlingTests.cs b/src/protagonist/Orchestrator.Tests/Integration/ImageHandlingTests.cs index ffd8565b3..81e8f5148 100644 --- a/src/protagonist/Orchestrator.Tests/Integration/ImageHandlingTests.cs +++ b/src/protagonist/Orchestrator.Tests/Integration/ImageHandlingTests.cs @@ -26,6 +26,7 @@ using Orchestrator.Infrastructure.IIIF; using Orchestrator.Tests.Integration.Infrastructure; using Test.Helpers; +using Test.Helpers.Data; using Test.Helpers.Integration; using Yarp.ReverseProxy.Forwarder; using Version = IIIF.ImageApi.Version; @@ -261,7 +262,7 @@ await amazonS3.PutObjectAsync(new PutObjectRequest public async Task GetInfoJsonV2_Correct_ViaDirectPath_NotInS3() { // Arrange - var id = AssetId.FromString($"99/1/{nameof(GetInfoJsonV2_Correct_ViaDirectPath_NotInS3)}"); + var id = AssetIdGenerator.GetAssetId(); await dbFixture.DbContext.Images.AddTestAsset(id, imageDeliveryChannels: deliveryChannelsForImage); await amazonS3.PutObjectAsync(new PutObjectRequest @@ -282,6 +283,7 @@ await amazonS3.PutObjectAsync(new PutObjectRequest jsonResponse.Id.Should().Be($"http://localhost/iiif-img/v2/{id}"); jsonResponse.Context.ToString().Should().Be("http://iiif.io/api/image/2/context.json"); jsonResponse.Sizes.Should().BeEquivalentTo(expectedSizes); + jsonResponse.Type.Should().Be("iiif:Image", "ImageService2 is default on class, overridden when read"); // With correct headers/status response.StatusCode.Should().Be(HttpStatusCode.OK); diff --git a/src/protagonist/Orchestrator.Tests/Integration/ManifestHandlingTests.cs b/src/protagonist/Orchestrator.Tests/Integration/ManifestHandlingTests.cs index a68ff7546..729e02479 100644 --- a/src/protagonist/Orchestrator.Tests/Integration/ManifestHandlingTests.cs +++ b/src/protagonist/Orchestrator.Tests/Integration/ManifestHandlingTests.cs @@ -237,6 +237,29 @@ public async Task Get_ManifestForImage_ReturnsManifest() response.Headers.CacheControl.MaxAge.Should().BeGreaterThan(TimeSpan.FromSeconds(2)); } + [Fact] + public async Task Get_ManifestForImage_V2_ReturnsManifest_WithoutIIIFImage3Services() + { + // Arrange + var id = AssetIdGenerator.GetAssetId(); + await dbFixture.DbContext.Images.AddTestAsset(id, origin: "testorigin", imageDeliveryChannels: imageDeliveryChannels); + await dbFixture.DbContext.SaveChangesAsync(); + + var path = $"iiif-manifest/v2/{id}"; + + // Act + var response = await httpClient.GetAsync(path); + + // Assert + var json = await response.Content.ReadAsStringAsync(); + json.Should().NotContain("ImageService3"); + + response.StatusCode.Should().Be(HttpStatusCode.OK); + response.Headers.Should().ContainKey("x-asset-id").WhoseValue.Should().ContainSingle(id.ToString()); + response.Headers.CacheControl.Public.Should().BeTrue(); + response.Headers.CacheControl.MaxAge.Should().BeGreaterThan(TimeSpan.FromSeconds(2)); + } + [Fact] public async Task Get_V2ManifestForImage_ReturnsManifest_FromMetadata() { diff --git a/src/protagonist/Orchestrator.Tests/Integration/NamedQueryTests.cs b/src/protagonist/Orchestrator.Tests/Integration/NamedQueryTests.cs index 083ae50cb..196d8d5af 100644 --- a/src/protagonist/Orchestrator.Tests/Integration/NamedQueryTests.cs +++ b/src/protagonist/Orchestrator.Tests/Integration/NamedQueryTests.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Generic; using System.Linq; using System.Net; using System.Net.Http; @@ -117,10 +116,23 @@ public async Task Get_Returns404_IfCustomerNotFound(string path) } [Theory] - [InlineData("iiif-resource/99/test-named-query/too-little-params")] - [InlineData("iiif-resource/v2/99/test-named-query/too-little-params")] - [InlineData("iiif-resource/v3/99/test-named-query/too-little-params")] - public async Task Get_Returns400_IfNamedQueryParametersIncorrect(string path) + [InlineData("iiif-resource/99/test-named-query/my-ref")] + [InlineData("iiif-resource/v2/99/test-named-query/my-ref")] + [InlineData("iiif-resource/v3/99/test-named-query/my-ref")] + public async Task Get_Returns200_IfNamedQueryParametersLessThanMax(string path) + { + // Act + var response = await httpClient.GetAsync(path); + + // Assert + response.StatusCode.Should().Be(HttpStatusCode.OK); + } + + [Theory] + [InlineData("iiif-resource/99/test-named-query")] + [InlineData("iiif-resource/v2/99/test-named-query")] + [InlineData("iiif-resource/v3/99/test-named-query")] + public async Task Get_Returns400_IfNoNamedQueryParameters(string path) { // Act var response = await httpClient.GetAsync(path); @@ -195,6 +207,25 @@ public async Task Get_ReturnsV2Manifest_WithCorrectId_IgnoringQueryParam() jsonResponse["@id"].ToString().Should().Be($"http://localhost/{path}"); } + [Fact] + public async Task Get_ReturnsV2Manifest_WithoutImageService3Services() + { + // Arrange + const string path = "iiif-resource/v2/99/test-named-query/my-ref/1"; + const string iiif2 = "application/ld+json; profile=\"http://iiif.io/api/presentation/2/context.json\""; + + // Act + var response = await httpClient.GetAsync($"{path}?foo=bar"); + + // Assert + response.StatusCode.Should().Be(HttpStatusCode.OK); + response.Headers.Vary.Should().Contain("Accept"); + response.Content.Headers.ContentType.ToString().Should().Be(iiif2); + + var json = await response.Content.ReadAsStringAsync(); + json.Should().NotContain("ImageService3"); + } + [Fact] public async Task Get_ReturnsV3ManifestWithCorrectCount_ViaConneg() { diff --git a/src/protagonist/Orchestrator.Tests/Integration/PdfTests.cs b/src/protagonist/Orchestrator.Tests/Integration/PdfTests.cs index 727fc5ffd..e960f801c 100644 --- a/src/protagonist/Orchestrator.Tests/Integration/PdfTests.cs +++ b/src/protagonist/Orchestrator.Tests/Integration/PdfTests.cs @@ -63,6 +63,8 @@ public PdfTests(ProtagonistAppFactory factory, StorageFixture orchestra maxUnauthorised: 10, roles: "clickthrough"); dbFixture.DbContext.Images.AddTestAsset(AssetId.FromString("99/1/not-for-delivery"), num1: 6, ref1: "my-ref", notForDelivery: true); + dbFixture.DbContext.Images.AddTestAsset(AssetId.FromString("99/1/limited-projection"), num1: 2, + ref1: "limited-ref"); dbFixture.DbContext.SaveChanges(); } @@ -110,10 +112,10 @@ public async Task GetPdf_Returns404_IfNQNotFound() } [Fact] - public async Task GetPdf_Returns400_IfParametersIncorrect() + public async Task GetPdf_Returns400_IfNoParameters() { // Arrange - const string path = "pdf/99/test-pdf/too-little-params"; + const string path = "pdf/99/test-pdf"; // Act var response = await httpClient.GetAsync(path); @@ -150,6 +152,28 @@ await AddPdfControlFile("99/pdf/test-pdf/my-ref/1/1/tester.json", response.StatusCode.Should().Be(HttpStatusCode.Accepted); response.Headers.Should().ContainKey("Retry-After"); } + + [Fact] + public async Task GetPdf_Returns200_IfParametersLessThanMax() + { + // Arrange + var fakePdfContent = nameof(GetPdf_Returns200_IfParametersLessThanMax); + const string path = "pdf/99/test-pdf/limited-ref"; + const string pdfStorageKey = "99/pdf/test-pdf/limited-ref/tester"; + await AddPdfControlFile("99/pdf/test-pdf/limited-ref/tester", + new ControlFile { Created = DateTime.UtcNow, InProcess = false }); + pdfCreator.AddCallbackFor(pdfStorageKey, (query, assets) => + { + AddPdf(pdfStorageKey, fakePdfContent).Wait(); + return true; + }); + + // Act + var response = await httpClient.GetAsync(path); + + // Assert + response.StatusCode.Should().Be(HttpStatusCode.OK); + } [Fact] public async Task GetPdf_Returns200_WithExistingPdf_IfPdfControlFileAndPdfExist() @@ -398,10 +422,10 @@ public async Task GetPdfControlFile_Returns404_IfNQNotFound() } [Fact] - public async Task GetPdfControlFile_Returns404_IfParametersIncorrect() + public async Task GetPdfControlFile_Returns404_IfNoParameters() { // Arrange - const string path = "pdf-control/99/test-pdf/too-little-params"; + const string path = "pdf-control/99/test-pdf"; // Act var response = await httpClient.GetAsync(path); @@ -410,11 +434,12 @@ public async Task GetPdfControlFile_Returns404_IfParametersIncorrect() response.StatusCode.Should().Be(HttpStatusCode.NotFound); } - [Fact] - public async Task GetPdfControlFile_Returns200_WithEmptyControlFile_IfNQValidButNoControlFile() + [Theory] + [InlineData("pdf-control/99/test-pdf/any-ref/1/2")] + [InlineData("pdf-control/99/test-pdf/any-ref")] + public async Task GetPdfControlFile_Returns200_WithEmptyControlFile_IfNQValidButNoControlFile(string path) { // Arrange - const string path = "pdf-control/99/test-pdf/any-ref/1/2"; var pdfControlFile = new PdfControlFile { Created = DateTime.MinValue, InProcess = false, Exists = false, Key = string.Empty, ItemCount = 0, diff --git a/src/protagonist/Orchestrator.Tests/Integration/ZipTests.cs b/src/protagonist/Orchestrator.Tests/Integration/ZipTests.cs index 468db6ef1..72e6b2c89 100644 --- a/src/protagonist/Orchestrator.Tests/Integration/ZipTests.cs +++ b/src/protagonist/Orchestrator.Tests/Integration/ZipTests.cs @@ -57,6 +57,8 @@ public ZipTests(ProtagonistAppFactory factory, StorageFixture orchestra dbFixture.DbContext.Images.AddTestAsset(AssetId.FromString("99/1/matching-zip-5"), num1: 5, ref1: "my-ref"); dbFixture.DbContext.Images.AddTestAsset(AssetId.FromString("99/1/not-for-delivery"), num1: 6, ref1: "my-ref", notForDelivery: true); + dbFixture.DbContext.Images.AddTestAsset(AssetId.FromString("99/1/limited-parameter-zip-1"), num1: 2, + ref1: "limited-ref"); dbFixture.DbContext.SaveChanges(); } @@ -103,19 +105,6 @@ public async Task GetZip_Returns404_IfNQNotFound() response.StatusCode.Should().Be(HttpStatusCode.NotFound); } - [Fact] - public async Task GetZip_Returns400_IfParametersIncorrect() - { - // Arrange - const string path = "zip/99/test-zip/too-little-params"; - - // Act - var response = await httpClient.GetAsync(path); - - // Assert - response.StatusCode.Should().Be(HttpStatusCode.BadRequest); - } - [Fact] public async Task GetZip_Returns404_IfNoMatchingRecordsFound() { @@ -163,6 +152,23 @@ await AddControlFile("99/zip/test-zip/my-ref/1/1/tester.zip.json", (await response.Content.ReadAsStringAsync()).Should().Be(fakeContent); response.Content.Headers.ContentType.Should().Be(new MediaTypeHeaderValue("application/zip")); } + + [Fact] + public async Task GetZip_Returns200_IfLessParametersThanTotal() + { + // Arrange + var fakeContent = nameof(GetZip_Returns200_IfLessParametersThanTotal); + const string path = "zip/99/test-zip/limited-ref"; + await AddControlFile("99/zip/test-zip/limited-ref/tester.zip.json", + new ControlFile { Created = DateTime.UtcNow, InProcess = false }); + await AddZipArchive("99/zip/test-zip/limited-ref/tester.zip", fakeContent); + + // Act + var response = await httpClient.GetAsync(path); + + // Assert + response.StatusCode.Should().Be(HttpStatusCode.OK); + } [Fact] public async Task GetZip_Returns200_WithNewlyCreatedZip_IfControlFileExistsButZipDoesnt() @@ -327,24 +333,12 @@ public async Task GetZipControlFile_Returns404_IfNQNotFound() response.StatusCode.Should().Be(HttpStatusCode.NotFound); } - [Fact] - public async Task GetZipControlFile_Returns404_IfParametersIncorrect() - { - // Arrange - const string path = "zip-control/99/test-zip/too-little-params"; - - // Act - var response = await httpClient.GetAsync(path); - - // Assert - response.StatusCode.Should().Be(HttpStatusCode.NotFound); - } - - [Fact] - public async Task GetZipControlFile_Returns200_WithEmptyControlFile_IfNQValidButNoControlFile() + [Theory] + [InlineData("zip-control/99/test-zip/any-ref/1/2")] + [InlineData("zip-control/99/test-zip/any-ref")] + public async Task GetZipControlFile_Returns200_WithEmptyControlFile_IfNQValidButNoControlFile(string path) { // Arrange - const string path = "zip-control/99/test-zip/any-ref/1/2"; var controlFileJson = JsonConvert.SerializeObject(ControlFile.Empty); // Act diff --git a/src/protagonist/Orchestrator.Tests/Orchestrator.Tests.csproj b/src/protagonist/Orchestrator.Tests/Orchestrator.Tests.csproj index 7e3705f9b..1306bb515 100644 --- a/src/protagonist/Orchestrator.Tests/Orchestrator.Tests.csproj +++ b/src/protagonist/Orchestrator.Tests/Orchestrator.Tests.csproj @@ -7,7 +7,7 @@ - + diff --git a/src/protagonist/Orchestrator.Tests/Settings/OrchestratorSettingsTests.cs b/src/protagonist/Orchestrator.Tests/Settings/OrchestratorSettingsTests.cs index 53dfd494c..8a8fdaead 100644 --- a/src/protagonist/Orchestrator.Tests/Settings/OrchestratorSettingsTests.cs +++ b/src/protagonist/Orchestrator.Tests/Settings/OrchestratorSettingsTests.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using DLCS.Core.Settings; using DLCS.Core.Types; using IIIF.ImageApi; using Orchestrator.Settings; diff --git a/src/protagonist/Orchestrator/Features/Images/ImageServer/InfoJsonService.cs b/src/protagonist/Orchestrator/Features/Images/ImageServer/InfoJsonService.cs index ba68d1ea7..941a0ff95 100644 --- a/src/protagonist/Orchestrator/Features/Images/ImageServer/InfoJsonService.cs +++ b/src/protagonist/Orchestrator/Features/Images/ImageServer/InfoJsonService.cs @@ -57,8 +57,9 @@ public InfoJsonService( JsonLdBase deserialisedInfoJson = version == Version.V2 ? infoJson.FromJsonStream() : infoJson.FromJsonStream(); + logger.LogTrace("Found info.json version {Version} for {AssetId}", version, orchestrationImage.AssetId); - return new InfoJsonResponse(deserialisedInfoJson, false); + return GetInfoJsonResponse(deserialisedInfoJson, false); } // If not found, build new copy @@ -69,7 +70,17 @@ public InfoJsonService( if (infoJsonResponse == null) return null; await StoreInfoJson(infoJsonKey, infoJsonResponse, cancellationToken); - return new InfoJsonResponse(infoJsonResponse, true); + return GetInfoJsonResponse(infoJsonResponse, true); + } + + private InfoJsonResponse GetInfoJsonResponse(JsonLdBase infoJsonResponse, bool wasOrchestrated) + { + if (infoJsonResponse is ImageService2 imageService2) + { + imageService2.Type = DLCS.Model.IIIF.Constants.ImageService2Type; + } + + return new InfoJsonResponse(infoJsonResponse, wasOrchestrated); } private ObjectInBucket GetInfoJsonKey(OrchestrationImage asset, Version version) diff --git a/src/protagonist/Orchestrator/Infrastructure/IIIF/IIIFCanvasFactory.cs b/src/protagonist/Orchestrator/Infrastructure/IIIF/IIIFCanvasFactory.cs index c1eed0191..393ccc4e0 100644 --- a/src/protagonist/Orchestrator/Infrastructure/IIIF/IIIFCanvasFactory.cs +++ b/src/protagonist/Orchestrator/Infrastructure/IIIF/IIIFCanvasFactory.cs @@ -85,7 +85,7 @@ public IIIFCanvasFactory( Format = "image/jpeg", Width = thumbnailSizes.MaxDerivativeSize.Width, Height = thumbnailSizes.MaxDerivativeSize.Height, - Service = GetImageServices(asset, customerPathElement, authProbeServices) + Service = GetImageServices(asset, customerPathElement, false, authProbeServices) } : null, }.AsListOf() @@ -98,7 +98,7 @@ public IIIFCanvasFactory( { Id = GetFullQualifiedThumbPath(asset, customerPathElement, thumbnailSizes.OpenThumbnails), Format = "image/jpeg", - Service = GetImageServiceForThumbnail(asset, customerPathElement, + Service = GetImageServiceForThumbnail(asset, customerPathElement, false, thumbnailSizes.OpenThumbnails) }.AsListOf(); } @@ -147,7 +147,7 @@ public IIIFCanvasFactory( thumbnailSizes.MaxDerivativeSize, false), Width = thumbnailSizes.MaxDerivativeSize.Width, Height = thumbnailSizes.MaxDerivativeSize.Height, - Service = GetImageServices(asset, customerPathElement, null) + Service = GetImageServices(asset, customerPathElement, true, null) } : null, }.AsList() @@ -158,7 +158,8 @@ public IIIFCanvasFactory( canvas.Thumbnail = new IIIF2.Thumbnail { Id = GetFullQualifiedThumbPath(asset, customerPathElement, thumbnailSizes.OpenThumbnails), - Service = GetImageServiceForThumbnail(asset, customerPathElement, thumbnailSizes.OpenThumbnails) + Service = GetImageServiceForThumbnail(asset, customerPathElement, true, + thumbnailSizes.OpenThumbnails) }.AsList(); } @@ -183,21 +184,28 @@ public IIIFCanvasFactory( }; } - private List GetImageServiceForThumbnail(Asset asset, CustomerPathElement customerPathElement, - List thumbnailSizes) + private List GetImageServiceForThumbnail(Asset asset, CustomerPathElement customerPathElement, + bool forPresentation2, List thumbnailSizes) { - var services = new List(2); + var services = new List(); if (orchestratorSettings.ImageServerConfig.VersionPathTemplates.ContainsKey(ImageApi.Version.V2)) { - services.Add(new ImageService2 + var imageService = new ImageService2 { Id = GetFullyQualifiedId(asset, customerPathElement, true, ImageApi.Version.V2), Profile = ImageService2.Level0Profile, Sizes = thumbnailSizes, Context = ImageService2.Image2Context, - }); + }; + + if (forPresentation2) imageService.Type = null; // '@Type' is not used in Presentation2 + + services.Add(imageService); } + // NOTE - we never include ImageService3 on Presentation2 manifests + if (forPresentation2) return services; + if (orchestratorSettings.ImageServerConfig.VersionPathTemplates.ContainsKey(ImageApi.Version.V3)) { services.Add(new ImageService3 @@ -259,15 +267,16 @@ private string GetFullyQualifiedId(Asset asset, CustomerPathElement customerPath return assetPathGenerator.GetFullPathForRequest(imageRequest, true, false); } - private List GetImageServices(Asset asset, CustomerPathElement customerPathElement, + private List GetImageServices(Asset asset, CustomerPathElement customerPathElement, bool forPresentation2, Dictionary? authProbeServices) { var noAuthServices = authProbeServices.IsNullOrEmpty(); + var versionPathTemplates = orchestratorSettings.ImageServerConfig.VersionPathTemplates; - var services = new List(2); - if (orchestratorSettings.ImageServerConfig.VersionPathTemplates.ContainsKey(ImageApi.Version.V2)) + var services = new List(); + if (versionPathTemplates.ContainsKey(ImageApi.Version.V2)) { - services.Add(new ImageService2 + var imageService = new ImageService2 { Id = GetFullyQualifiedId(asset, customerPathElement, false, ImageApi.Version.V2), Profile = ImageService2.Level2Profile, @@ -275,10 +284,17 @@ private List GetImageServices(Asset asset, CustomerPathElement custome Width = asset.Width ?? 0, Height = asset.Height ?? 0, Service = TryGetAuthServices(), - }); + }; + + if (forPresentation2) imageService.Type = null; // '@Type' is not used in Presentation2 + + services.Add(imageService); } - if (orchestratorSettings.ImageServerConfig.VersionPathTemplates.ContainsKey(ImageApi.Version.V3)) + // NOTE - we never include ImageService3 on Presentation2 manifests + if (forPresentation2) return services; + + if (versionPathTemplates.ContainsKey(ImageApi.Version.V3)) { services.Add(new ImageService3 { diff --git a/src/protagonist/Orchestrator/Infrastructure/ReverseProxy/DownstreamDestinationSelector.cs b/src/protagonist/Orchestrator/Infrastructure/ReverseProxy/DownstreamDestinationSelector.cs index d821f7696..86b264011 100644 --- a/src/protagonist/Orchestrator/Infrastructure/ReverseProxy/DownstreamDestinationSelector.cs +++ b/src/protagonist/Orchestrator/Infrastructure/ReverseProxy/DownstreamDestinationSelector.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using DLCS.Core.Collections; +using DLCS.Core.Settings; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; diff --git a/src/protagonist/Orchestrator/Settings/OrchestratorSettings.cs b/src/protagonist/Orchestrator/Settings/OrchestratorSettings.cs index bc023a4f5..38795f908 100644 --- a/src/protagonist/Orchestrator/Settings/OrchestratorSettings.cs +++ b/src/protagonist/Orchestrator/Settings/OrchestratorSettings.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Linq; using DLCS.Core.Caching; +using DLCS.Core.Settings; using DLCS.Web.Response; namespace Orchestrator.Settings; @@ -272,22 +273,6 @@ public class CustomerOverride public List PdfRolesWhitelist { get; set; } = new(); } -/// -/// Enum representing image server used for serving image requests -/// -public enum ImageServer -{ - /// - /// Cantaloupe image server - /// - Cantaloupe, - - /// - /// IIP Image Server - /// - IIPImage -} - /// /// Represents redirect configuration for redirecting ImageServer requests /// diff --git a/src/protagonist/Orchestrator/Settings/OrchestratorSettingsX.cs b/src/protagonist/Orchestrator/Settings/OrchestratorSettingsX.cs index c98e693b6..8cd634652 100644 --- a/src/protagonist/Orchestrator/Settings/OrchestratorSettingsX.cs +++ b/src/protagonist/Orchestrator/Settings/OrchestratorSettingsX.cs @@ -1,4 +1,5 @@ -using DLCS.Core.Types; +using DLCS.Core.Settings; +using DLCS.Core.Types; using DLCS.Model.Templates; namespace Orchestrator.Settings; diff --git a/src/protagonist/Portal.Tests/Portal.Tests.csproj b/src/protagonist/Portal.Tests/Portal.Tests.csproj index b5a415982..5e3974e3b 100644 --- a/src/protagonist/Portal.Tests/Portal.Tests.csproj +++ b/src/protagonist/Portal.Tests/Portal.Tests.csproj @@ -8,7 +8,7 @@ - + diff --git a/src/protagonist/Portal/Features/NamedQueries/NamedQueryController.cs b/src/protagonist/Portal/Features/NamedQueries/NamedQueryController.cs new file mode 100644 index 000000000..9c9ebb56e --- /dev/null +++ b/src/protagonist/Portal/Features/NamedQueries/NamedQueryController.cs @@ -0,0 +1,54 @@ +using System; +using System.Threading.Tasks; +using API.Client; +using MediatR; +using Microsoft.AspNetCore.Mvc; +using Portal.Features.NamedQueries.Requests; + +namespace Portal.Features.NamedQueries; + +[Route("[controller]/[action]")] +public class NamedQueryController : Controller +{ + private readonly IMediator mediator; + + public NamedQueryController(IMediator mediator) + { + this.mediator = mediator; + } + + [HttpPost] + public async Task Delete([FromForm] string namedQueryId) + { + await mediator.Send(new DeleteNamedQuery(){ NamedQueryId = namedQueryId }); + return RedirectToPage("/NamedQueries/Index"); + } + + [HttpPost] + public async Task Update([FromForm] string namedQueryId, [FromForm] string template) + { + try + { + await mediator.Send(new UpdateNamedQuery(){ NamedQueryId = namedQueryId, Template = template }); + return Ok(); + } + catch (DlcsException dlcsEx) + { + return BadRequest(dlcsEx.Message); + } + } + + [HttpPost] + public async Task Create(string queryName, string queryTemplate) + { + try + { + await mediator.Send(new CreateNamedQuery() { Name = queryName, Template = queryTemplate }); + return Ok(); + } + catch (DlcsException dlcsEx) + { + return BadRequest(dlcsEx.Message); + } + } +} \ No newline at end of file diff --git a/src/protagonist/Portal/Features/NamedQueries/Requests/CreateNamedQuery.cs b/src/protagonist/Portal/Features/NamedQueries/Requests/CreateNamedQuery.cs new file mode 100644 index 000000000..12c7b049b --- /dev/null +++ b/src/protagonist/Portal/Features/NamedQueries/Requests/CreateNamedQuery.cs @@ -0,0 +1,32 @@ +using System.Threading; +using System.Threading.Tasks; +using API.Client; +using DLCS.HydraModel; +using MediatR; + +namespace Portal.Features.NamedQueries.Requests; + +/// +/// Create a new named query belonging to the current customer +/// +public class CreateNamedQuery: IRequest +{ + public string Name { get; set; } + + public string Template { get; set; } +} + +public class CreateNamedQueryHandler : IRequestHandler +{ + private readonly IDlcsClient dlcsClient; + + public CreateNamedQueryHandler(IDlcsClient dlcsClient) + { + this.dlcsClient = dlcsClient; + } + + public async Task Handle(CreateNamedQuery request, CancellationToken cancellationToken) + { + return await dlcsClient.CreateNamedQuery(new NamedQuery(){ Name = request.Name, Template = request.Template }); + } +} \ No newline at end of file diff --git a/src/protagonist/Portal/Features/NamedQueries/Requests/DeleteNamedQuery.cs b/src/protagonist/Portal/Features/NamedQueries/Requests/DeleteNamedQuery.cs new file mode 100644 index 000000000..c17c02937 --- /dev/null +++ b/src/protagonist/Portal/Features/NamedQueries/Requests/DeleteNamedQuery.cs @@ -0,0 +1,29 @@ +using System.Threading; +using System.Threading.Tasks; +using API.Client; +using MediatR; + +namespace Portal.Features.NamedQueries.Requests; + +/// +/// Delete a specified named query belonging to the current customer +/// +public class DeleteNamedQuery: IRequest +{ + public string NamedQueryId { get; set; } +} + +public class DeleteNamedQueryHandler : IRequestHandler +{ + private readonly IDlcsClient dlcsClient; + + public DeleteNamedQueryHandler(IDlcsClient dlcsClient) + { + this.dlcsClient = dlcsClient; + } + + public async Task Handle(DeleteNamedQuery request, CancellationToken cancellationToken) + { + return await dlcsClient.DeleteNamedQuery(request.NamedQueryId); + } +} \ No newline at end of file diff --git a/src/protagonist/Portal/Features/NamedQueries/Requests/GetCustomerNamedQueries.cs b/src/protagonist/Portal/Features/NamedQueries/Requests/GetCustomerNamedQueries.cs new file mode 100644 index 000000000..b5ea5f306 --- /dev/null +++ b/src/protagonist/Portal/Features/NamedQueries/Requests/GetCustomerNamedQueries.cs @@ -0,0 +1,31 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using API.Client; +using DLCS.HydraModel; +using MediatR; + +namespace Portal.Features.NamedQueries.Requests; + +/// +/// Retrieves all named queries belonging to the current customer +/// +public class GetCustomerNamedQueries: IRequest> +{ +} + +public class GetCustomerNamedQueriesHandler : IRequestHandler> +{ + private readonly IDlcsClient dlcsClient; + + public GetCustomerNamedQueriesHandler(IDlcsClient dlcsClient) + { + this.dlcsClient = dlcsClient; + } + + public async Task> Handle(GetCustomerNamedQueries request, CancellationToken cancellationToken) + { + var namedQueries = await dlcsClient.GetNamedQueries(false); + return namedQueries; + } +} \ No newline at end of file diff --git a/src/protagonist/Portal/Features/NamedQueries/Requests/UpdateNamedQuery.cs b/src/protagonist/Portal/Features/NamedQueries/Requests/UpdateNamedQuery.cs new file mode 100644 index 000000000..a24e5508f --- /dev/null +++ b/src/protagonist/Portal/Features/NamedQueries/Requests/UpdateNamedQuery.cs @@ -0,0 +1,32 @@ +using System.Threading; +using System.Threading.Tasks; +using API.Client; +using DLCS.HydraModel; +using MediatR; + +namespace Portal.Features.NamedQueries.Requests; + +/// +/// Update a specified named query belonging to the current customer +/// +public class UpdateNamedQuery: IRequest +{ + public string NamedQueryId { get; set; } + + public string Template { get; set; } +} + +public class UpdateNamedQueryHandler : IRequestHandler +{ + private readonly IDlcsClient dlcsClient; + + public UpdateNamedQueryHandler(IDlcsClient dlcsClient) + { + this.dlcsClient = dlcsClient; + } + + public async Task Handle(UpdateNamedQuery request, CancellationToken cancellationToken) + { + return await dlcsClient.UpdateNamedQuery(request.NamedQueryId, request.Template); + } +} \ No newline at end of file diff --git a/src/protagonist/Portal/Features/Spaces/Requests/GetImage.cs b/src/protagonist/Portal/Features/Spaces/Requests/GetImage.cs index 03a03eda5..0747da5ec 100644 --- a/src/protagonist/Portal/Features/Spaces/Requests/GetImage.cs +++ b/src/protagonist/Portal/Features/Spaces/Requests/GetImage.cs @@ -6,11 +6,13 @@ using System.Threading.Tasks; using API.Client; using DLCS.Core.Collections; +using DLCS.Core.Settings; using DLCS.HydraModel; using DLCS.Web.Auth; using IIIF.ImageApi.V3; using MediatR; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; namespace Portal.Features.Spaces.Requests; @@ -31,14 +33,17 @@ public class GetImageHandler : IRequestHandler { private readonly ILogger logger; private readonly IDlcsClient dlcsClient; + private readonly DlcsSettings dlcsSettings; private readonly HttpClient httpClient; private readonly string customerId; - public GetImageHandler(IDlcsClient dlcsClient, HttpClient httpClient, ILogger logger, ClaimsPrincipal currentUser) + public GetImageHandler(IDlcsClient dlcsClient, HttpClient httpClient, ILogger logger, + ClaimsPrincipal currentUser, IOptions dlcsSettings) { this.logger = logger; this.dlcsClient = dlcsClient; this.httpClient = httpClient; + this.dlcsSettings = dlcsSettings.Value; customerId = (currentUser.GetCustomerId() ?? -1).ToString(); } @@ -67,11 +72,9 @@ public GetImageHandler(IDlcsClient dlcsClient, HttpClient httpClient, ILogger GetImageThumbnailService(Image image) { - if (image.ThumbnailImageService.IsNullOrEmpty()) return null; - try { - var response = await httpClient.GetAsync($"{image.ThumbnailImageService}/info.json"); + var response = await httpClient.GetAsync($"{dlcsSettings.ResourceRoot}thumbs/v3/{customerId}/{image.Space}/{image.ModelId}/info.json"); if (!response.IsSuccessStatusCode) return null; return await response.Content.ReadFromJsonAsync(); } diff --git a/src/protagonist/Portal/Pages/Images/Index.cshtml.cs b/src/protagonist/Portal/Pages/Images/Index.cshtml.cs index d9fb7fe56..1a86ab93a 100644 --- a/src/protagonist/Portal/Pages/Images/Index.cshtml.cs +++ b/src/protagonist/Portal/Pages/Images/Index.cshtml.cs @@ -57,7 +57,7 @@ public async Task OnGetAsync(int space, string image) public string CreateSrc(Size size) { - return $"{Image.ThumbnailImageService}/full/{size.Width},{size.Height}/0/default.jpg"; + return $"{dlcsSettings.ResourceRoot}thumbs/v3/{Customer}/{Image.Space}/{Image.ModelId}/full/{size.Width},{size.Height}/0/default.jpg"; } public string CreateUniversalViewerUrl(string singleAssetManifest) diff --git a/src/protagonist/Portal/Pages/Index.cshtml b/src/protagonist/Portal/Pages/Index.cshtml index e6f0b1ad1..faeac3d7e 100644 --- a/src/protagonist/Portal/Pages/Index.cshtml +++ b/src/protagonist/Portal/Pages/Index.cshtml @@ -57,6 +57,15 @@

View your queue

+
+ +
+

+ Named Queries +

+

View and manage named queries

+
+
diff --git a/src/protagonist/Portal/Pages/NamedQueries/Index.cshtml b/src/protagonist/Portal/Pages/NamedQueries/Index.cshtml new file mode 100644 index 000000000..514e5d672 --- /dev/null +++ b/src/protagonist/Portal/Pages/NamedQueries/Index.cshtml @@ -0,0 +1,190 @@ +@page +@model Portal.Pages.NamedQueries.IndexModel + +@{ + ViewData["Title"] = "Named Queries"; +} + +
+
+ @if (!Model.NamedQueries.Any()) + { + + } + else + { + + + + + + + + + + @foreach (var namedQuery in Model.NamedQueries) + { + var namedQueryId = namedQuery.Id.Split('/').Last(); + + + + + + } + + + + + + +
NameTemplateAction
@namedQuery.Name@namedQuery.Template + + +
+ +
+ } +
+
+ +
+ +
+ +
+ +
+ +
+ +
+ +@section Scripts +{ + +} \ No newline at end of file diff --git a/src/protagonist/Portal/Pages/NamedQueries/Index.cshtml.cs b/src/protagonist/Portal/Pages/NamedQueries/Index.cshtml.cs new file mode 100644 index 000000000..92b8d2874 --- /dev/null +++ b/src/protagonist/Portal/Pages/NamedQueries/Index.cshtml.cs @@ -0,0 +1,28 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using DLCS.HydraModel; +using MediatR; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc.RazorPages; +using Portal.Features.NamedQueries.Requests; + +namespace Portal.Pages.NamedQueries; + +public class IndexModel : PageModel +{ + private readonly IMediator mediator; + + [BindProperty] + public IEnumerable NamedQueries { get; set; } = Enumerable.Empty(); + + public IndexModel(IMediator mediator) + { + this.mediator = mediator; + } + + public async Task OnGetAsync([FromQuery] bool? success = null) + { + NamedQueries = await mediator.Send(new GetCustomerNamedQueries()); + } +} \ No newline at end of file diff --git a/src/protagonist/Portal/Pages/Shared/_ControlPanelLayout.cshtml b/src/protagonist/Portal/Pages/Shared/_ControlPanelLayout.cshtml index a57938f94..08e7cad90 100644 --- a/src/protagonist/Portal/Pages/Shared/_ControlPanelLayout.cshtml +++ b/src/protagonist/Portal/Pages/Shared/_ControlPanelLayout.cshtml @@ -66,6 +66,9 @@ Upload } +
  • + Named Queries +
  • Users
    • diff --git a/src/protagonist/Portal/Portal.csproj b/src/protagonist/Portal/Portal.csproj index 238008543..9b53c010a 100644 --- a/src/protagonist/Portal/Portal.csproj +++ b/src/protagonist/Portal/Portal.csproj @@ -10,7 +10,7 @@ - + diff --git a/src/protagonist/Test.Helpers/Http/ControllableHttpMessageHandler.cs b/src/protagonist/Test.Helpers/Http/ControllableHttpMessageHandler.cs index 89e0c7fde..942f58870 100644 --- a/src/protagonist/Test.Helpers/Http/ControllableHttpMessageHandler.cs +++ b/src/protagonist/Test.Helpers/Http/ControllableHttpMessageHandler.cs @@ -35,7 +35,7 @@ public HttpResponseMessage GetResponseMessage(string content, HttpStatusCode htt }; return response; } - + /// /// Set a pre-canned response /// diff --git a/src/protagonist/Thumbs/Thumbs.csproj b/src/protagonist/Thumbs/Thumbs.csproj index 47a9ec15c..98cab05fb 100644 --- a/src/protagonist/Thumbs/Thumbs.csproj +++ b/src/protagonist/Thumbs/Thumbs.csproj @@ -10,7 +10,7 @@ - +