diff --git a/src/apps/Odin.Hosting/Controllers/Base/Drive/V1DriveUploadControllerBase.cs b/src/apps/Odin.Hosting/Controllers/Base/Drive/V1DriveUploadControllerBase.cs index 34bda72f59..051426531e 100644 --- a/src/apps/Odin.Hosting/Controllers/Base/Drive/V1DriveUploadControllerBase.cs +++ b/src/apps/Odin.Hosting/Controllers/Base/Drive/V1DriveUploadControllerBase.cs @@ -64,7 +64,7 @@ private async Task ReceiveNewFileStreamInternal(Guid? driveId) { try { - await driveUploadService.CleanupTempFiles(WebOdinContext); + await driveUploadService.CleanupStagingFiles(WebOdinContext); } catch (Exception e) { @@ -190,7 +190,7 @@ await updateWriter.AddThumbnail(thumbnailUploadKey, contentTypeFromMultipartSect { try { - await updateWriter.CleanupTempFiles(WebOdinContext); + await updateWriter.CleanupStagingFiles(WebOdinContext); } catch (Exception e) { @@ -317,7 +317,7 @@ protected async Task ReceivePayloadStream() { try { - await writer.CleanupTempFiles(WebOdinContext); + await writer.CleanupStagingFiles(WebOdinContext); } catch (Exception e) { diff --git a/src/apps/Odin.Hosting/Controllers/Base/Transit/PeerSenderControllerBase.cs b/src/apps/Odin.Hosting/Controllers/Base/Transit/PeerSenderControllerBase.cs index adc0ce11fe..d59e0a0cfa 100644 --- a/src/apps/Odin.Hosting/Controllers/Base/Transit/PeerSenderControllerBase.cs +++ b/src/apps/Odin.Hosting/Controllers/Base/Transit/PeerSenderControllerBase.cs @@ -55,7 +55,7 @@ public async Task SendFile() { try { - await fileSystemWriter.CleanupTempFiles(WebOdinContext); + await fileSystemWriter.CleanupStagingFiles(WebOdinContext); } catch (Exception e) { diff --git a/src/apps/Odin.Hosting/Controllers/OwnerToken/Drive/OwnerDriveStorageController.cs b/src/apps/Odin.Hosting/Controllers/OwnerToken/Drive/OwnerDriveStorageController.cs index b58d0664b5..de24838d45 100644 --- a/src/apps/Odin.Hosting/Controllers/OwnerToken/Drive/OwnerDriveStorageController.cs +++ b/src/apps/Odin.Hosting/Controllers/OwnerToken/Drive/OwnerDriveStorageController.cs @@ -29,36 +29,55 @@ public class OwnerDriveStorageController( private readonly ILogger _logger = logger; /// - /// Indicates if the specified TempFile exists. This is used for testing only + /// Indicates if the specified upload temp file exists. This is used for testing only /// [SwaggerOperation(Tags = [ControllerConstants.OwnerDrive])] - [HttpGet("temp-file-exists")] - public async Task TempFileExists([FromQuery] Guid fileId, + [HttpGet("upload-file-exists")] + public async Task UploadFileExists([FromQuery] Guid fileId, [FromQuery] Guid alias, [FromQuery] Guid type, - [FromQuery] TempStorageType storageType, [FromQuery] string extension) { - var tempFile = new TempFile() + var file = MapToInternalFile(new ExternalFileIdentifier() { - File = MapToInternalFile(new ExternalFileIdentifier() + FileId = fileId, + TargetDrive = new TargetDrive() { - FileId = fileId, - TargetDrive = new TargetDrive() - { - Alias = alias, - Type = type - } - }), - StorageType = storageType - }; + Alias = alias, + Type = type + } + }); - var result = await this.GetHttpFileSystemResolver() + return await this.GetHttpFileSystemResolver() .ResolveFileSystem() .Storage - .TempFileExists(tempFile, extension, WebOdinContext); + .UploadFileExists(file, extension, WebOdinContext); + } - return result; + /// + /// Indicates if the specified inbox temp file exists. This is used for testing only + /// + [SwaggerOperation(Tags = [ControllerConstants.OwnerDrive])] + [HttpGet("inbox-file-exists")] + public async Task InboxFileExists([FromQuery] Guid fileId, + [FromQuery] Guid alias, + [FromQuery] Guid type, + [FromQuery] string extension) + { + var file = MapToInternalFile(new ExternalFileIdentifier() + { + FileId = fileId, + TargetDrive = new TargetDrive() + { + Alias = alias, + Type = type + } + }); + + return await this.GetHttpFileSystemResolver() + .ResolveFileSystem() + .Storage + .InboxFileExists(file, extension, WebOdinContext); } /// diff --git a/src/apps/Odin.Hosting/Controllers/PeerIncoming/Drive/PeerIncomingDriveUpdateController.cs b/src/apps/Odin.Hosting/Controllers/PeerIncoming/Drive/PeerIncomingDriveUpdateController.cs index cc9633c038..f8461b2319 100644 --- a/src/apps/Odin.Hosting/Controllers/PeerIncoming/Drive/PeerIncomingDriveUpdateController.cs +++ b/src/apps/Odin.Hosting/Controllers/PeerIncoming/Drive/PeerIncomingDriveUpdateController.cs @@ -130,7 +130,7 @@ public async Task ReceiveIncomingUpdate() { if (null != _fileUpdateService) { - await _fileUpdateService.CleanupTempFiles(uploadedPayloads, WebOdinContext); + await _fileUpdateService.CleanupStagingFiles(uploadedPayloads, WebOdinContext); } } catch (Exception e) diff --git a/src/apps/Odin.Hosting/Controllers/PeerIncoming/Drive/PeerIncomingDriveUploadController.cs b/src/apps/Odin.Hosting/Controllers/PeerIncoming/Drive/PeerIncomingDriveUploadController.cs index 451cc025eb..896fdaa7f6 100644 --- a/src/apps/Odin.Hosting/Controllers/PeerIncoming/Drive/PeerIncomingDriveUploadController.cs +++ b/src/apps/Odin.Hosting/Controllers/PeerIncoming/Drive/PeerIncomingDriveUploadController.cs @@ -169,7 +169,7 @@ public async Task ReceiveIncomingTransfer() { if (null != _incomingTransferService) { - await _incomingTransferService.CleanupTempFiles(uploadedPayloads, WebOdinContext); + await _incomingTransferService.CleanupStagingFiles(uploadedPayloads, WebOdinContext); } } catch (Exception e) diff --git a/src/apps/Odin.Hosting/TenantServices.cs b/src/apps/Odin.Hosting/TenantServices.cs index 69c33d67c6..7b85f42b98 100644 --- a/src/apps/Odin.Hosting/TenantServices.cs +++ b/src/apps/Odin.Hosting/TenantServices.cs @@ -205,6 +205,7 @@ internal static ContainerBuilder ConfigureTenantServices( cb.RegisterType().InstancePerLifetimeScope(); cb.RegisterType().InstancePerLifetimeScope(); + cb.RegisterType().InstancePerLifetimeScope(); // cb.RegisterType().InstancePerLifetimeScope(); cb.RegisterType().As().InstancePerLifetimeScope(); diff --git a/src/services/Odin.Services/AppNotifications/ClientNotifications/FileAddedNotification.cs b/src/services/Odin.Services/AppNotifications/ClientNotifications/FileAddedNotification.cs index 70d154d17a..caab5dd9c9 100644 --- a/src/services/Odin.Services/AppNotifications/ClientNotifications/FileAddedNotification.cs +++ b/src/services/Odin.Services/AppNotifications/ClientNotifications/FileAddedNotification.cs @@ -13,7 +13,7 @@ public class FileAddedNotification : IClientNotification public OdinId Sender { get; set; } - public ExternalFileIdentifier TempFile { get; set; } + public ExternalFileIdentifier File { get; set; } public string GetClientData() { diff --git a/src/services/Odin.Services/Background/BackgroundServices/Tenant/TempFolderCleanUpBackgroundService.cs b/src/services/Odin.Services/Background/BackgroundServices/Tenant/TempFolderCleanUpBackgroundService.cs index 6b86221236..3560d2b1a4 100644 --- a/src/services/Odin.Services/Background/BackgroundServices/Tenant/TempFolderCleanUpBackgroundService.cs +++ b/src/services/Odin.Services/Background/BackgroundServices/Tenant/TempFolderCleanUpBackgroundService.cs @@ -3,7 +3,6 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using Odin.Core.Exceptions; using Odin.Services.Base; using Odin.Services.Drives.FileSystem.Base; using Directory = System.IO.Directory; @@ -29,16 +28,28 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) try { - TempFolderCleanUp.Execute( + UploadFolderCleanUp.Execute( logger, - tenantContext.TenantPathManager.TempPath, + tenantContext.TenantPathManager.UploadDrivesPath, uploadAgeThreshold, + stoppingToken); + } + catch (Exception e) + { + logger.LogError(e, "UploadFolderCleanUp: {message}", e.Message); + } + + try + { + InboxFolderCleanUp.Execute( + logger, + tenantContext.TenantPathManager.InboxDrivesPath, inboxAgeThreshold, stoppingToken); } catch (Exception e) { - logger.LogError(e, "TempFolderCleanUpBackgroundService: {message}", e.Message); + logger.LogError(e, "InboxFolderCleanUp: {message}", e.Message); } var interval = TimeSpan.FromHours(24); @@ -50,39 +61,100 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) // -public static class TempFolderCleanUp +public static class UploadFolderCleanUp { public static void Execute( ILogger logger, - string tempFolder, - TimeSpan uploadAgeThreshold, - TimeSpan inboxAgeThreshold, + string uploadDrivesPath, + TimeSpan ageThreshold, CancellationToken stoppingToken = default) { - ArgumentException.ThrowIfNullOrWhiteSpace(tempFolder, nameof(tempFolder)); + if (!Directory.Exists(uploadDrivesPath)) + { + return; + } - if (uploadAgeThreshold <= TimeSpan.Zero) + var drives = Directory.GetDirectories(uploadDrivesPath); + foreach (var drive in drives) { - throw new ArgumentException("Upload age threshold must be positive", nameof(uploadAgeThreshold)); + if (stoppingToken.IsCancellationRequested) + { + return; + } + + var uploadsPath = Path.Combine(drive, TenantPathManager.UploadFolder); + CleanUp(logger, uploadsPath, ageThreshold, stoppingToken); } + } - if (inboxAgeThreshold <= TimeSpan.Zero) + // + + private static void CleanUp(ILogger logger, string folder, TimeSpan threshold, CancellationToken stoppingToken) + { + if (!Directory.Exists(folder)) { - throw new ArgumentException("Inbox age threshold must be positive", nameof(inboxAgeThreshold)); + return; } - if (!Directory.Exists(tempFolder)) + logger.LogDebug("{service} is scanning {folder}", nameof(UploadFolderCleanUp), folder); + + try { - throw new OdinSystemException($"Temp folder {tempFolder} does not exist"); + var subDirectories = Directory.GetDirectories(folder); + if (subDirectories.Length > 0) + { + logger.LogError("Illegal subdirectories detected in {Folder}", folder); + } + + var cutoffTime = DateTime.UtcNow.Subtract(threshold); + var files = Directory.GetFiles(folder); + + foreach (var file in files) + { + if (stoppingToken.IsCancellationRequested) + { + return; + } + + try + { + var fileInfo = new FileInfo(file); + if (fileInfo.LastWriteTimeUtc < cutoffTime) + { + logger.LogDebug("UploadFolderCleanUp: deleting file {file}", file); + File.Delete(file); + } + } + catch (Exception e) + { + logger.LogError(e, "UploadFolderCleanUp({file}): {message}", file, e.Message); + } + } } + catch (Exception e) + { + logger.LogError(e, "UploadFolderCleanUp({folder}): {message}", folder, e.Message); + } + } +} - var drivesFolder = Path.Combine(tempFolder, TenantPathManager.DrivesFolder); - if (!Directory.Exists(drivesFolder)) +// + +// NOTE: This class is likely dead code in the future — inbox files are not expected to be cleaned up +public static class InboxFolderCleanUp +{ + public static void Execute( + ILogger logger, + string inboxDrivesPath, + TimeSpan ageThreshold, + CancellationToken stoppingToken = default) + { + if (!Directory.Exists(inboxDrivesPath)) { return; } - var drives = Directory.GetDirectories(drivesFolder); + var drives = Directory.GetDirectories(inboxDrivesPath); foreach (var drive in drives) { if (stoppingToken.IsCancellationRequested) @@ -90,17 +162,8 @@ public static void Execute( return; } - if (!stoppingToken.IsCancellationRequested) - { - var uploadsPath = Path.Combine(drive, TenantPathManager.UploadFolder); - CleanUp(logger, uploadsPath, uploadAgeThreshold, stoppingToken); - } - - if (!stoppingToken.IsCancellationRequested) - { - var inboxPath = Path.Combine(drive, TenantPathManager.InboxFolder); - CleanUp(logger, inboxPath, inboxAgeThreshold, stoppingToken); - } + var inboxPath = Path.Combine(drive, TenantPathManager.InboxFolder); + CleanUp(logger, inboxPath, ageThreshold, stoppingToken); } } @@ -113,7 +176,7 @@ private static void CleanUp(ILogger logger, string folder, TimeSpan threshold, C return; } - logger.LogDebug("{service} is scanning {folder}", nameof(TempFolderCleanUp), folder); + logger.LogDebug("{service} is scanning {folder}", nameof(InboxFolderCleanUp), folder); try { @@ -138,20 +201,19 @@ private static void CleanUp(ILogger logger, string folder, TimeSpan threshold, C var fileInfo = new FileInfo(file); if (fileInfo.LastWriteTimeUtc < cutoffTime) { - logger.LogDebug("TempFolderCleanUp: deleting file {file}", file); + logger.LogDebug("InboxFolderCleanUp: deleting file {file}", file); File.Delete(file); } } catch (Exception e) { - logger.LogError(e, "TempFolderCleanUp({file}): {message}", file, e.Message); + logger.LogError(e, "InboxFolderCleanUp({file}): {message}", file, e.Message); } } } catch (Exception e) { - logger.LogError(e, "TempFolderCleanUp({folder}): {message}", folder, e.Message); + logger.LogError(e, "InboxFolderCleanUp({folder}): {message}", folder, e.Message); } } } - diff --git a/src/services/Odin.Services/DataSubscription/ReceivingHost/FeedDistributionPerimeterService.cs b/src/services/Odin.Services/DataSubscription/ReceivingHost/FeedDistributionPerimeterService.cs index 7e0cf078d7..7d9672baf7 100644 --- a/src/services/Odin.Services/DataSubscription/ReceivingHost/FeedDistributionPerimeterService.cs +++ b/src/services/Odin.Services/DataSubscription/ReceivingHost/FeedDistributionPerimeterService.cs @@ -220,14 +220,10 @@ private async Task RouteFeedRequestToInboxAsync(UpdateFeed logger.LogDebug("Found feed drive id {id}", feedDriveId); // Write to temp file - var tempFile = new TempFile() - { - File = await fileSystem.Storage.CreateInternalFileId(feedDriveId, odinContext), - StorageType = TempStorageType.Inbox - }; + var file = await fileSystem.Storage.CreateInternalFileId(feedDriveId, odinContext); var stream = OdinSystemSerializer.Serialize(request.FileMetadata).ToUtf8ByteArray().ToMemoryStream(); - await fileSystem.Storage.WriteTempStream(tempFile, MultipartHostTransferParts.Metadata.ToString().ToLower(), stream, + await fileSystem.Storage.WriteInboxStream(file, MultipartHostTransferParts.Metadata.ToString().ToLower(), stream, odinContext); await stream.DisposeAsync(); @@ -239,8 +235,8 @@ await fileSystem.Storage.WriteTempStream(tempFile, MultipartHostTransferParts.Me Sender = odinContext.GetCallerOdinIdOrFail(), InstructionType = TransferInstructionType.SaveFile, - FileId = tempFile.File.FileId, - DriveId = tempFile.File.DriveId, + FileId = file.FileId, + DriveId = file.DriveId, GlobalTransitId = request.FileMetadata.GlobalTransitId.GetValueOrDefault(), FileSystemType = FileSystemType.Standard, //comments are never distributed diff --git a/src/services/Odin.Services/Drives/DriveCore/Storage/Defragmenter.cs b/src/services/Odin.Services/Drives/DriveCore/Storage/Defragmenter.cs index 86f28db64e..6cb76f3816 100644 --- a/src/services/Odin.Services/Drives/DriveCore/Storage/Defragmenter.cs +++ b/src/services/Odin.Services/Drives/DriveCore/Storage/Defragmenter.cs @@ -232,7 +232,7 @@ ENABLE WHEN WE CREATE A DIRECTORY AND NIBBLES ON DRIVE CREATION public async Task VerifyDriveDirectoriesTemp(bool cleanup) { - var rootpath = _tenantPathManager.TempDrivesPath; + var rootpath = _tenantPathManager.UploadDrivesPath; if (Directory.Exists(rootpath)) { diff --git a/src/services/Odin.Services/Drives/DriveCore/Storage/InboxStorageManager.cs b/src/services/Odin.Services/Drives/DriveCore/Storage/InboxStorageManager.cs new file mode 100644 index 0000000000..b7baa21084 --- /dev/null +++ b/src/services/Odin.Services/Drives/DriveCore/Storage/InboxStorageManager.cs @@ -0,0 +1,107 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Odin.Services.Base; +using Odin.Services.Drives.FileSystem.Base; + +namespace Odin.Services.Drives.DriveCore.Storage +{ + /// + /// Temporary storage for a given drive. Used to stage incoming file parts from peer transfers (inbox). + /// + public class InboxStorageManager( + FileReaderWriter fileReaderWriter, + ILogger logger, + TenantContext tenantContext) + { + private readonly TenantPathManager _tenantPathManager = tenantContext.TenantPathManager; + + public Task InboxFileExists(InternalDriveFileId file, string extension) + { + string path = _tenantPathManager.GetDriveInboxFilePath(file.DriveId, file.FileId, extension); + return Task.FromResult(fileReaderWriter.FileExists(path)); + } + + /// + /// Gets all bytes for the specified file + /// + public async Task GetAllInboxFileBytes(InternalDriveFileId file, string extension) + { + string path = _tenantPathManager.GetDriveInboxFilePath(file.DriveId, file.FileId, extension); + return await fileReaderWriter.GetAllFileBytesAsync(path); + } + + /// + /// Writes a stream for a given file and part to the configured provider. + /// + public async Task WriteInboxStream(InternalDriveFileId file, string extension, Stream stream) + { + fileReaderWriter.CreateDirectory(_tenantPathManager.GetDriveInboxPath(file.DriveId)); + string path = _tenantPathManager.GetDriveInboxFilePath(file.DriveId, file.FileId, extension); + return await fileReaderWriter.WriteStreamAsync(path, stream); + } + + public Task CleanupInboxFiles(InternalDriveFileId file, List descriptors) + { + logger.LogDebug("CleanupInboxFiles called - file: {file}", file); + + CleanupInboxFilesInternal(file, descriptors); + + //TODO: the extensions should be centralized + string[] additionalFiles = + [ + _tenantPathManager.GetDriveInboxFilePath(file.DriveId, file.FileId, TenantPathManager.MetadataExtension), + _tenantPathManager.GetDriveInboxFilePath(file.DriveId, file.FileId, TenantPathManager.TransferInstructionSetExtension) + ]; + + foreach (var f in additionalFiles) + { + logger.LogDebug("CleanupInboxFiles Deleting additional File: {file}", f); + } + + // clean up the transfer header and metadata since we keep those in the inbox + fileReaderWriter.DeleteFiles(additionalFiles); + + return Task.CompletedTask; + } + + private void CleanupInboxFilesInternal(InternalDriveFileId file, List descriptors) + { + try + { + if (!descriptors?.Any() ?? false) + { + return; + } + + var targetFiles = new List(); + + descriptors!.ForEach(descriptor => + { + var payloadExtension = TenantPathManager.GetBasePayloadFileNameAndExtension(descriptor.Key, descriptor.Uid); + string payloadDirectoryAndFilename = _tenantPathManager.GetDriveInboxFilePath(file.DriveId, file.FileId, payloadExtension); + targetFiles.Add(payloadDirectoryAndFilename); + + descriptor.Thumbnails?.ForEach(thumb => + { + var thumbnailExtension = TenantPathManager.GetThumbnailFileNameAndExtension(descriptor.Key, + descriptor.Uid, + thumb.PixelWidth, + thumb.PixelHeight); + string thumbnailDirectoryAndFilename = _tenantPathManager.GetDriveInboxFilePath(file.DriveId, file.FileId, thumbnailExtension); + targetFiles.Add(thumbnailDirectoryAndFilename); + }); + }); + + fileReaderWriter.DeleteFiles(targetFiles); + } + catch (Exception e) + { + logger.LogError(e, "Failure while cleaning up inbox files"); + } + } + } +} diff --git a/src/services/Odin.Services/Drives/DriveCore/Storage/UploadStorageManager.cs b/src/services/Odin.Services/Drives/DriveCore/Storage/UploadStorageManager.cs index 9e45081fcd..3d54b586a2 100644 --- a/src/services/Odin.Services/Drives/DriveCore/Storage/UploadStorageManager.cs +++ b/src/services/Odin.Services/Drives/DriveCore/Storage/UploadStorageManager.cs @@ -6,107 +6,54 @@ using Microsoft.Extensions.Logging; using Odin.Services.Base; using Odin.Services.Drives.FileSystem.Base; -using Odin.Services.Drives.Management; namespace Odin.Services.Drives.DriveCore.Storage { /// - /// Temporary storage for a given driven. Used to stage incoming file parts from uploads and transfers. + /// Temporary storage for a given drive. Used to stage incoming file parts from uploads. /// public class UploadStorageManager( FileReaderWriter fileReaderWriter, - IDriveManager driveManager, ILogger logger, TenantContext tenantContext) { private readonly TenantPathManager _tenantPathManager = tenantContext.TenantPathManager; - public async Task TempFileExists(TempFile tempFile, string extension) + public Task UploadFileExists(InternalDriveFileId file, string extension) { - string path = await GetTempFilenameAndPathInternal(tempFile, extension); - return fileReaderWriter.FileExists(path); + string path = _tenantPathManager.GetDriveUploadFilePath(file.DriveId, file.FileId, extension); + return Task.FromResult(fileReaderWriter.FileExists(path)); } /// - /// Gets a stream of data for the specified file + /// Gets all bytes for the specified file /// - public async Task GetAllFileBytes(TempFile tempFile, string extension) + public async Task GetAllUploadFileBytes(InternalDriveFileId file, string extension) { - string path = await GetTempFilenameAndPathInternal(tempFile, extension); - - logger.LogDebug("Getting temp file bytes for [{path}]", path); - var bytes = await fileReaderWriter.GetAllFileBytesAsync(path); - logger.LogDebug("Got {count} bytes from {path}", bytes.Length, path); - return bytes; + string path = _tenantPathManager.GetDriveUploadFilePath(file.DriveId, file.FileId, extension); + return await fileReaderWriter.GetAllFileBytesAsync(path); } /// /// Writes a stream for a given file and part to the configured provider. /// - public async Task WriteStream(TempFile tempFile, string extension, Stream stream) + public async Task WriteUploadStream(InternalDriveFileId file, string extension, Stream stream) { - string path = await GetTempFilenameAndPathInternal(tempFile, extension, true); - logger.LogDebug("Writing temp file: {filePath}", path); - var bytesWritten = await fileReaderWriter.WriteStreamAsync(path, stream); - if (bytesWritten == 0) - { - // Sanity #1 - logger.LogDebug("I didn't write anything to {filePath}", path); - } - else if (!File.Exists(path)) - { - // Sanity #2 - logger.LogError("I wrote {count} bytes, but file is not there {filePath}", bytesWritten, path); - } - - logger.LogDebug("Wrote {count} bytes to {filePath}", bytesWritten, path); - - return bytesWritten; - } - - public async Task CleanupInboxFiles(TempFile tempFile, List descriptors) - { - if (tempFile.StorageType != TempStorageType.Inbox) - { - logger.LogDebug("{method} ignoring call to cleanup {type} files", nameof(CleanupInboxFiles), tempFile.StorageType); - return; - } - - logger.LogDebug("CleanupInboxTemporaryFiles called - tempFile: {t}", tempFile); - - await CleanupTempFilesInternal(tempFile, descriptors); - - //TODO: the extensions should be centralized - string[] additionalFiles = - [ - await GetTempFilenameAndPathInternal(tempFile, TenantPathManager.MetadataExtension), - await GetTempFilenameAndPathInternal(tempFile, TenantPathManager.TransferInstructionSetExtension) - ]; - - foreach (var file in additionalFiles) - { - logger.LogDebug("CleanupInboxTemporaryFiles Deleting additional File: {file}", file); - } - - // clean up the transfer header and metadata since we keep those in the inbox - fileReaderWriter.DeleteFiles(additionalFiles); + fileReaderWriter.CreateDirectory(_tenantPathManager.GetDriveUploadPath(file.DriveId)); + string path = _tenantPathManager.GetDriveUploadFilePath(file.DriveId, file.FileId, extension); + return await fileReaderWriter.WriteStreamAsync(path, stream); } /// - /// Deletes all files matching regardless of extension + /// Deletes all files matching regardless of extension /// - public async Task CleanupUploadedTempFiles(TempFile tempFile, List descriptors) + public Task CleanupUploadFiles(InternalDriveFileId file, List descriptors) { - if (tempFile.StorageType != TempStorageType.Upload) - { - logger.LogDebug("{method} ignoring call to cleanup {type} files", nameof(CleanupUploadedTempFiles), tempFile.StorageType); - return; - } - - await CleanupTempFilesInternal(tempFile, descriptors); + CleanupUploadFilesInternal(file, descriptors); + return Task.CompletedTask; } - private async Task CleanupTempFilesInternal(TempFile tempFile, List descriptors) + private void CleanupUploadFilesInternal(InternalDriveFileId file, List descriptors) { try { @@ -115,17 +62,12 @@ private async Task CleanupTempFilesInternal(TempFile tempFile, List(); - // add in transfer history and metadata - - descriptors!.ForEach(descriptor => { var payloadExtension = TenantPathManager.GetBasePayloadFileNameAndExtension(descriptor.Key, descriptor.Uid); - string payloadDirectoryAndFilename = GetTempFilenameAndPathInternal(drive, tempFile, payloadExtension); + string payloadDirectoryAndFilename = _tenantPathManager.GetDriveUploadFilePath(file.DriveId, file.FileId, payloadExtension); targetFiles.Add(payloadDirectoryAndFilename); descriptor.Thumbnails?.ForEach(thumb => @@ -134,7 +76,7 @@ private async Task CleanupTempFilesInternal(TempFile tempFile, List - /// Gets the physical path of the specified file - /// - public async Task GetPath(TempFile tempFile, string extension) - { - string path = await GetTempFilenameAndPathInternal(tempFile, extension); - return path; - } - - private string GetUploadOrInboxFileDirectory(StorageDrive drive, TempFile tempFile, bool ensureExists = false) - { - string path; - - if (tempFile.StorageType == TempStorageType.Upload) - path = drive.GetDriveUploadPath(); - else - path = drive.GetDriveInboxPath(); - - if (ensureExists) - { - Directory.CreateDirectory(path); + logger.LogError(e, "Failure while cleaning up upload files"); } - - return path; - } - - private async Task GetTempFilenameAndPathInternal(TempFile tempFile, string extension, bool ensureExists = false) - { - var drive = await driveManager.GetDriveAsync(tempFile.File.DriveId); - return GetTempFilenameAndPathInternal(drive, tempFile, extension, ensureExists); - } - - private string GetTempFilenameAndPathInternal(StorageDrive drive, TempFile tempFile, string extension, bool ensureExists = false) - { - var fileId = tempFile.File.FileId; - - string dir = GetUploadOrInboxFileDirectory(drive, tempFile, ensureExists); - var r = Path.Combine(dir, TenantPathManager.GetFilename(fileId, extension)); - - return r; } } -} \ No newline at end of file +} diff --git a/src/services/Odin.Services/Drives/FileSystem/Base/DriveStorageServiceBase.cs b/src/services/Odin.Services/Drives/FileSystem/Base/DriveStorageServiceBase.cs index 3c023bbb7e..f839eda939 100644 --- a/src/services/Odin.Services/Drives/FileSystem/Base/DriveStorageServiceBase.cs +++ b/src/services/Odin.Services/Drives/FileSystem/Base/DriveStorageServiceBase.cs @@ -34,6 +34,8 @@ public abstract class DriveStorageServiceBase( IDriveManager driveManager, LongTermStorageManager longTermStorageManager, UploadStorageManager uploadStorageManager, + InboxStorageManager inboxStorageManager, + FileReaderWriter fileReaderWriter, IdentityDatabase db) : RequirePermissionsBase { private readonly ILogger _logger = loggerFactory.CreateLogger(); @@ -212,31 +214,42 @@ await TryPublishAsync(new DriveFileAddedNotification } } - public async Task WriteTempStream(TempFile tempFile, string extension, Stream stream, IOdinContext odinContext) + public async Task WriteUploadStream(InternalDriveFileId file, string extension, Stream stream, IOdinContext odinContext) { - await AssertDriveIsNotArchived(tempFile.File.DriveId, odinContext); + await AssertDriveIsNotArchived(file.DriveId, odinContext); + await AssertCanWriteToDrive(file.DriveId, odinContext); + return await uploadStorageManager.WriteUploadStream(file, extension, stream); + } - await AssertCanWriteToDrive(tempFile.File.DriveId, odinContext); - return await uploadStorageManager.WriteStream(tempFile, extension, stream); + public async Task WriteInboxStream(InternalDriveFileId file, string extension, Stream stream, IOdinContext odinContext) + { + await AssertDriveIsNotArchived(file.DriveId, odinContext); + await AssertCanWriteToDrive(file.DriveId, odinContext); + return await inboxStorageManager.WriteInboxStream(file, extension, stream); } /// /// Reads the whole file so be sure this is only used on small'ish files; ones you're ok with loaded fully into server-memory /// - /// - public async Task GetAllFileBytesFromTempFile(TempFile tempFile, string extension, IOdinContext odinContext) + public async Task GetAllFileBytesFromInboxFile(InternalDriveFileId file, string extension, IOdinContext odinContext) { - await AssertDriveIsNotArchived(tempFile.File.DriveId, odinContext); - await AssertCanReadDriveAsync(tempFile.File.DriveId, odinContext); - var bytes = await uploadStorageManager.GetAllFileBytes(tempFile, extension); - return bytes; + await AssertDriveIsNotArchived(file.DriveId, odinContext); + await AssertCanReadDriveAsync(file.DriveId, odinContext); + return await inboxStorageManager.GetAllInboxFileBytes(file, extension); } - public async Task TempFileExists(TempFile tempFile, string extension, IOdinContext odinContext) + public async Task UploadFileExists(InternalDriveFileId file, string extension, IOdinContext odinContext) { - await AssertDriveIsNotArchived(tempFile.File.DriveId, odinContext); + await AssertDriveIsNotArchived(file.DriveId, odinContext); odinContext.Caller.AssertCallerIsOwner(); - return await uploadStorageManager.TempFileExists(tempFile, extension); + return await uploadStorageManager.UploadFileExists(file, extension); + } + + public async Task InboxFileExists(InternalDriveFileId file, string extension, IOdinContext odinContext) + { + await AssertDriveIsNotArchived(file.DriveId, odinContext); + odinContext.Caller.AssertCallerIsOwner(); + return await inboxStorageManager.InboxFileExists(file, extension); } /// @@ -251,12 +264,13 @@ public async Task HasOrphanPayloads(InternalDriveFileId file, IOdinContext return false; } - public async Task GetAllFileBytesFromTempFileForWriting(TempFile tempFile, string extension, - IOdinContext odinContext) + public async Task GetAllFileBytesFromTempFileForWriting(InternalDriveFileId file, string extension, + string sourceFolderPath, IOdinContext odinContext) { - await AssertDriveIsNotArchived(tempFile.File.DriveId, odinContext); - await AssertCanWriteToDrive(tempFile.File.DriveId, odinContext); - return await uploadStorageManager.GetAllFileBytes(tempFile, extension); + await AssertDriveIsNotArchived(file.DriveId, odinContext); + await AssertCanWriteToDrive(file.DriveId, odinContext); + var path = Path.Combine(sourceFolderPath, TenantPathManager.GetFilename(file.FileId, extension)); + return await fileReaderWriter.GetAllFileBytesAsync(path); } public async Task<(Stream stream, ThumbnailDescriptor thumbnail)> GetThumbnailPayloadStreamAsync(InternalDriveFileId file, @@ -518,24 +532,26 @@ await TryPublishAsync(new DriveFileDeletedNotification } public async Task<(bool success, List payloads)> CommitNewFile( - TempFile originFile, KeyHeader keyHeader, + InternalDriveFileId originFile, KeyHeader keyHeader, FileMetadata newMetadata, ServerMetadata serverMetadata, - bool? ignorePayload, IOdinContext odinContext, Guid? useThisVersionTag = null, WriteSecondDatabaseRowBase markComplete = null) + bool? ignorePayload, IOdinContext odinContext, Guid? useThisVersionTag = null, WriteSecondDatabaseRowBase markComplete = null, + string sourceFolderPath = null) { - await AssertDriveIsNotArchived(originFile.File.DriveId, odinContext); - await AssertCanWriteToDrive(originFile.File.DriveId, odinContext); - var drive = await DriveManager.GetDriveAsync(originFile.File.DriveId); + await AssertDriveIsNotArchived(originFile.DriveId, odinContext); + await AssertCanWriteToDrive(originFile.DriveId, odinContext); + var drive = await DriveManager.GetDriveAsync(originFile.DriveId); ignorePayload = ignorePayload.GetValueOrDefault(false) || newMetadata.PayloadsAreRemote; - var targetFile = originFile.File; + var targetFile = originFile; newMetadata.File = targetFile; // this is a new file so we can use the same fileId from the temp file serverMetadata.FileSystemType = GetFileSystemType(); // First copy and prepare everything we need if (!ignorePayload.GetValueOrDefault(false)) { - await CopyPayloadsAndThumbnailsToLongTermStorage(originFile, targetFile, newMetadata.Payloads ?? [], drive); + var src = sourceFolderPath ?? drive.GetDriveUploadPath(); + await CopyPayloadsAndThumbnailsToLongTermStorage(originFile, targetFile, newMetadata.Payloads ?? [], drive, src); } // set the version tag null on a new file sine it will be handled by the @@ -573,7 +589,7 @@ await TryPublishAsync(new DriveFileDeletedNotification { await TryPublishAsync(new DriveFileAddedNotification { - File = originFile.File, + File = originFile, ServerFileHeader = serverHeader, OdinContext = odinContext, }); @@ -592,9 +608,10 @@ await TryPublishAsync(new DriveFileAddedNotification } public async Task<(bool success, List payloads)> OverwriteFile( - TempFile originFile, InternalDriveFileId targetFile, + InternalDriveFileId originFile, InternalDriveFileId targetFile, KeyHeader keyHeader, FileMetadata newMetadata, - ServerMetadata serverMetadata, bool? ignorePayload, IOdinContext odinContext, WriteSecondDatabaseRowBase markComplete) + ServerMetadata serverMetadata, bool? ignorePayload, IOdinContext odinContext, WriteSecondDatabaseRowBase markComplete, + string sourceFolderPath = null) { await AssertDriveIsNotArchived(targetFile.DriveId, odinContext); await AssertCanWriteToDrive(targetFile.DriveId, odinContext); @@ -637,14 +654,15 @@ await TryPublishAsync(new DriveFileAddedNotification if (!ignorePayload.GetValueOrDefault(false)) { - await CopyPayloadsAndThumbnailsToLongTermStorage(originFile, targetFile, payloads, drive); + var src = sourceFolderPath ?? drive.GetDriveUploadPath(); + await CopyPayloadsAndThumbnailsToLongTermStorage(originFile, targetFile, payloads, drive, src); } bool success = false; var serverHeader = new ServerFileHeader() { - EncryptedKeyHeader = await EncryptKeyHeader(originFile.File.DriveId, keyHeader, odinContext), + EncryptedKeyHeader = await EncryptKeyHeader(originFile.DriveId, keyHeader, odinContext), FileMetadata = newMetadata, ServerMetadata = serverMetadata }; @@ -705,7 +723,7 @@ await TryPublishAsync(new DriveFileChangedNotification } public async Task UpdatePayloads( - TempFile originFile, + InternalDriveFileId originFile, InternalDriveFileId targetFile, List incomingPayloads, IOdinContext odinContext, @@ -735,7 +753,7 @@ public async Task UpdatePayloads( try { //Note: we do not delete existing payloads. this feature adds or overwrites existing ones - await CopyPayloadsAndThumbnailsToLongTermStorage(originFile, targetFile, incomingPayloads, drive); + await CopyPayloadsAndThumbnailsToLongTermStorage(originFile, targetFile, incomingPayloads, drive, drive.GetDriveUploadPath()); // get all the existing payloads that are not in the incoming list, we'll keep these var payloadsToKeep = existingServerHeader.FileMetadata.Payloads.Where( @@ -946,11 +964,12 @@ public async Task UpdateActiveFileHeader(InternalDriveFileId targetFile, ServerF } - public async Task<(bool success, List uploadedPayloads)> UpdateBatchAsync(TempFile originFile, + public async Task<(bool success, List uploadedPayloads)> UpdateBatchAsync(InternalDriveFileId originFile, InternalDriveFileId targetFile, BatchUpdateManifest manifest, IOdinContext odinContext, - WriteSecondDatabaseRowBase markComplete) + WriteSecondDatabaseRowBase markComplete, + string sourceFolderPath = null) { bool success = false; @@ -965,7 +984,7 @@ public async Task UpdateActiveFileHeader(InternalDriveFileId targetFile, ServerF } // First prepare by copying everything needed - var (header, copiedPayloads, zombies) = await UpdateBatchCopyFilesAsync(originFile, targetFile, manifest, odinContext); + var (header, copiedPayloads, zombies) = await UpdateBatchCopyFilesAsync(originFile, targetFile, manifest, odinContext, sourceFolderPath); try { await AssertPayloadsExistOnFileSystemAsync(header); @@ -1043,9 +1062,9 @@ private async Task TryPublishAsync(TNotification notification, Ca private async Task<(ServerFileHeader success, List copiedPayloads, List zombies)> - UpdateBatchCopyFilesAsync(TempFile originFile, + UpdateBatchCopyFilesAsync(InternalDriveFileId originFile, InternalDriveFileId targetFile, BatchUpdateManifest manifest, - IOdinContext odinContext) + IOdinContext odinContext, string sourceFolderPath = null) { List copiedPayloads = new(); @@ -1104,7 +1123,8 @@ async Task> ProcessAppendOrOverwrite(StorageDrive storag } // Copy all payload from the temp folder to the long term folder - await CopyPayloadsAndThumbnailsToLongTermStorage(originFile, targetFile, copiedPayloads, storageDrive); + var src = sourceFolderPath ?? storageDrive.GetDriveUploadPath(); + await CopyPayloadsAndThumbnailsToLongTermStorage(originFile, targetFile, copiedPayloads, storageDrive, src); return zombies; } @@ -1266,25 +1286,23 @@ await TryPublishAsync(new DriveFileChangedNotification }; } - // TODO I think this should be in the upload manager - public async Task CleanupUploadTemporaryFiles(TempFile tempFile, List descriptors, + public async Task CleanupUploadTemporaryFiles(InternalDriveFileId file, List descriptors, IOdinContext odinContext) { - await AssertDriveIsNotArchived(tempFile.File.DriveId, odinContext); - if (await CanWriteToDrive(tempFile.File.DriveId, odinContext)) + await AssertDriveIsNotArchived(file.DriveId, odinContext); + if (await CanWriteToDrive(file.DriveId, odinContext)) { - await uploadStorageManager.CleanupUploadedTempFiles(tempFile, descriptors); + await uploadStorageManager.CleanupUploadFiles(file, descriptors); } } - // TODO I think this should be in the inbox manager - public async Task CleanupInboxTemporaryFiles(TempFile tempFile, List descriptors, IOdinContext odinContext, - string[] additionalFiles = null) + public async Task CleanupInboxTemporaryFiles(InternalDriveFileId file, List descriptors, + IOdinContext odinContext) { - await AssertDriveIsNotArchived(tempFile.File.DriveId, odinContext); - if (await CanWriteToDrive(tempFile.File.DriveId, odinContext)) + await AssertDriveIsNotArchived(file.DriveId, odinContext); + if (await CanWriteToDrive(file.DriveId, odinContext)) { - await uploadStorageManager.CleanupInboxFiles(tempFile, descriptors); + await inboxStorageManager.CleanupInboxFiles(file, descriptors); } } @@ -1564,14 +1582,14 @@ private async Task OverwriteMetadataInternal(byte[] keyHeaderIv, ServerFileHeade /// Copies payloads and thumbs to long term storage /// /// List of all files copied (directory and filename) - private async Task CopyPayloadsAndThumbnailsToLongTermStorage(TempFile originFile, InternalDriveFileId targetFile, - List descriptors, StorageDrive drive) + private async Task CopyPayloadsAndThumbnailsToLongTermStorage(InternalDriveFileId originFile, InternalDriveFileId targetFile, + List descriptors, StorageDrive drive, string sourceFolderPath) { try { foreach (var descriptor in descriptors) { - await CopyPayloadAndThumbnailsToLongTermStorage(originFile, targetFile, drive, descriptor); + await CopyPayloadAndThumbnailsToLongTermStorage(originFile, targetFile, drive, descriptor, sourceFolderPath); } } catch @@ -1584,26 +1602,23 @@ private async Task CopyPayloadsAndThumbnailsToLongTermStorage(TempFile originFil /// /// Copies payload and thumbs to long term storage /// - /// List of all files copied (directory and filename) - private async Task CopyPayloadAndThumbnailsToLongTermStorage(TempFile originFile, InternalDriveFileId targetFile, - StorageDrive drive, - PayloadDescriptor descriptor) + private async Task CopyPayloadAndThumbnailsToLongTermStorage(InternalDriveFileId originFile, InternalDriveFileId targetFile, + StorageDrive drive, PayloadDescriptor descriptor, string sourceFolderPath) { var payloadExtension = TenantPathManager.GetBasePayloadFileNameAndExtension(descriptor.Key, descriptor.Uid); - var sourceFilePath = await uploadStorageManager.GetPath(originFile, payloadExtension); + var sourceFilePath = Path.Combine(sourceFolderPath, TenantPathManager.GetFilename(originFile.FileId, payloadExtension)); await longTermStorageManager.CopyPayloadToLongTermAsync( drive, targetFile.FileId, descriptor, sourceFilePath); - foreach (var thumb in descriptor.Thumbnails ?? []) { var thumbExt = TenantPathManager.GetThumbnailFileNameAndExtension( descriptor.Key, descriptor.Uid, thumb.PixelWidth, thumb.PixelHeight); - var sourceThumbnail = await uploadStorageManager.GetPath(originFile, thumbExt); + var sourceThumbnail = Path.Combine(sourceFolderPath, TenantPathManager.GetFilename(originFile.FileId, thumbExt)); await longTermStorageManager.CopyThumbnailToLongTermAsync(drive, targetFile.FileId, sourceThumbnail, descriptor, thumb); } diff --git a/src/services/Odin.Services/Drives/FileSystem/Base/TenantPathManager.cs b/src/services/Odin.Services/Drives/FileSystem/Base/TenantPathManager.cs index f6dce26cfb..4f9b2d03f8 100644 --- a/src/services/Odin.Services/Drives/FileSystem/Base/TenantPathManager.cs +++ b/src/services/Odin.Services/Drives/FileSystem/Base/TenantPathManager.cs @@ -57,8 +57,10 @@ public class TenantPathManager public readonly string RegistrationPath; // e.g. /data/tenants/registrations// public readonly string HeadersPath; // e.g. /data/tenants/registrations//headers - public readonly string TempPath; // e.g. /data/tenants/registrations//temp - public readonly string TempDrivesPath; // e.g. /data/tenants/registrations//temp/drives + public readonly string UploadPath; // e.g. /data/tenants/registrations//temp + public readonly string UploadDrivesPath; // e.g. /data/tenants/registrations//temp/drives + public readonly string InboxPath; // e.g. /data/tenants/registrations//temp (same dir for now) + public readonly string InboxDrivesPath; // e.g. /data/tenants/registrations//temp/drives (same dir for now) public readonly string PayloadsPath; // e.g. /data/tenants/payloads// public readonly string PayloadsDrivesPath; // e.g. /data/tenants/payloads//drives OR /drives if S3 is enabled @@ -87,8 +89,10 @@ public TenantPathManager(OdinConfiguration config, Guid tenantId) RegistrationPath = Path.Combine(RootRegistrationsPath, tenant); HeadersPath = Path.Combine(RegistrationPath, HeadersFolder); - TempPath = Path.Combine(RegistrationPath, TempFolder); - TempDrivesPath = Path.Combine(TempPath, DrivesFolder); + UploadPath = Path.Combine(RegistrationPath, TempFolder); + UploadDrivesPath = Path.Combine(UploadPath, DrivesFolder); + InboxPath = Path.Combine(RegistrationPath, TempFolder); + InboxDrivesPath = Path.Combine(InboxPath, DrivesFolder); PayloadsPath = Path.Combine(RootPayloadsPath, tenant); PayloadsDrivesPath = Path.Combine(PayloadsPath, DrivesFolder); @@ -114,13 +118,25 @@ public static string GetFilename(Guid fileId, string extension) // e.g. /data/tenants/registrations//temp/drives//inbox public string GetDriveInboxPath(Guid driveId) { - return Path.Combine(TempDrivesPath, GuidToPathSafeString(driveId), InboxFolder); + return Path.Combine(InboxDrivesPath, GuidToPathSafeString(driveId), InboxFolder); } // e.g. /data/tenants/registrations//temp/drives//uploads public string GetDriveUploadPath(Guid driveId) { - return Path.Combine(TempDrivesPath, GuidToPathSafeString(driveId), UploadFolder); + return Path.Combine(UploadDrivesPath, GuidToPathSafeString(driveId), UploadFolder); + } + + // e.g. /data/tenants/registrations//temp/drives//inbox/. + public string GetDriveInboxFilePath(Guid driveId, Guid fileId, string extension) + { + return Path.Combine(GetDriveInboxPath(driveId), GetFilename(fileId, extension)); + } + + // e.g. /data/tenants/registrations//temp/drives//uploads/. + public string GetDriveUploadFilePath(Guid driveId, Guid fileId, string extension) + { + return Path.Combine(GetDriveUploadPath(driveId), GetFilename(fileId, extension)); } // e.g. /data/tenants/payloads//drives//files @@ -251,7 +267,7 @@ public static string GetThumbnailFileExtensionStarStar(string payloadKey, UnixTi public void CreateDirectories() { Directory.CreateDirectory(HeadersPath); - Directory.CreateDirectory(TempPath); + Directory.CreateDirectory(UploadPath); if (!S3PayloadsEnabled) { diff --git a/src/services/Odin.Services/Drives/FileSystem/Base/Update/FileSystemUpdateWriterBase.cs b/src/services/Odin.Services/Drives/FileSystem/Base/Update/FileSystemUpdateWriterBase.cs index 2418329924..8a4a323a43 100644 --- a/src/services/Odin.Services/Drives/FileSystem/Base/Update/FileSystemUpdateWriterBase.cs +++ b/src/services/Odin.Services/Drives/FileSystem/Base/Update/FileSystemUpdateWriterBase.cs @@ -29,7 +29,7 @@ namespace Odin.Services.Drives.FileSystem.Base.Update; /// public abstract class FileSystemUpdateWriterBase { - private readonly IDriveManager _driveManager; + protected readonly IDriveManager _driveManager; private readonly PeerOutgoingTransferService _peerOutgoingTransferService; private readonly ILogger _logger; @@ -159,8 +159,7 @@ public virtual async Task AddPayload(string key, string contentTypeFromMultipart var extension = TenantPathManager.GetBasePayloadFileNameAndExtension(key, descriptor.PayloadUid); - var bytesWritten = await FileSystem.Storage.WriteTempStream(Package.InternalFile - .AsTempFileUpload(), extension, data, odinContext); + var bytesWritten = await FileSystem.Storage.WriteUploadStream(Package.InternalFile, extension, data, odinContext); if (bytesWritten != data.Length) { @@ -209,7 +208,7 @@ public virtual async Task AddThumbnail(string thumbnailUploadKey, string overrid result.ThumbnailDescriptor.PixelHeight ); - var bytesWritten = await FileSystem.Storage.WriteTempStream(Package.InternalFile.AsTempFileUpload(), extension, data, odinContext); + var bytesWritten = await FileSystem.Storage.WriteUploadStream(Package.InternalFile, extension, data, odinContext); if (bytesWritten != data.Length) { @@ -282,8 +281,9 @@ public async Task FinalizeFileUpdate(IOdinContext odinContext) throw new OdinClientException("AllowDistribution must be true when UpdateLocale is Peer"); } - await FileSystem.Storage.CommitNewFile(Package.InternalFile.AsTempFileUpload(), keyHeader, metadata, serverMetadata, false, - odinContext); + var peerDrive = await _driveManager.GetDriveAsync(Package.InternalFile.DriveId); + await FileSystem.Storage.CommitNewFile(Package.InternalFile, keyHeader, metadata, serverMetadata, false, + odinContext, sourceFolderPath: peerDrive.GetDriveUploadPath()); var recipientStatus = await ProcessTransitInstructions(Package, Package.InstructionSet.File, keyHeader, odinContext); @@ -336,9 +336,10 @@ protected virtual async Task ProcessExistingFileUploadAsync(FileUpdatePackage pa ServerMetadata = serverMetadata }; + var drive = await _driveManager.GetDriveAsync(package.InternalFile.DriveId); // TODO what if success is false? - var (success, payloads) = await FileSystem.Storage.UpdateBatchAsync(package.InternalFile.AsTempFileUpload(), package.InternalFile, - manifest, odinContext, null); + var (success, payloads) = await FileSystem.Storage.UpdateBatchAsync(package.InternalFile, package.InternalFile, + manifest, odinContext, null, sourceFolderPath: drive.GetDriveUploadPath()); if (success == false) throw new OdinClientException("No, I couldn't do it, success is false"); @@ -487,12 +488,12 @@ private async Task ValidateUploadCore(FileUpdatePackage package, KeyHeader keyHe metadata.Validate(odinContext.Tenant); } - public async Task CleanupTempFiles(IOdinContext odinContext) + public async Task CleanupStagingFiles(IOdinContext odinContext) { if (Package?.Payloads?.Any() ?? false) { await FileSystem.Storage.CleanupUploadTemporaryFiles( - Package.InternalFile.AsTempFileUpload(), + Package.InternalFile, Package.GetFinalPayloadDescriptors(), odinContext); } diff --git a/src/services/Odin.Services/Drives/FileSystem/Base/Upload/Attachments/PayloadOnlyPackage.cs b/src/services/Odin.Services/Drives/FileSystem/Base/Upload/Attachments/PayloadOnlyPackage.cs index 5a0e62b21f..4eaddad4d0 100644 --- a/src/services/Odin.Services/Drives/FileSystem/Base/Upload/Attachments/PayloadOnlyPackage.cs +++ b/src/services/Odin.Services/Drives/FileSystem/Base/Upload/Attachments/PayloadOnlyPackage.cs @@ -14,7 +14,7 @@ public class PayloadOnlyPackage /// public PayloadOnlyPackage(InternalDriveFileId internalFile, UploadPayloadInstructionSet instructionSet) { - this.TempFile = internalFile with { FileId = Guid.NewGuid() }; + this.UploadFile = internalFile with { FileId = Guid.NewGuid() }; this.InternalFile = internalFile; this.InstructionSet = instructionSet; this.Payloads = new List(); @@ -27,7 +27,7 @@ public PayloadOnlyPackage(InternalDriveFileId internalFile, UploadPayloadInstruc /// The temporary file to which incoming payloads are written. This is /// not the same as the target file to which the payloads will be attach /// - public InternalDriveFileId TempFile { get; init; } + public InternalDriveFileId UploadFile { get; init; } /// /// The file to which the payloads will be attached diff --git a/src/services/Odin.Services/Drives/FileSystem/Base/Upload/Attachments/PayloadStreamWriterBase.cs b/src/services/Odin.Services/Drives/FileSystem/Base/Upload/Attachments/PayloadStreamWriterBase.cs index 61fd9c2b00..1948bf6b9f 100644 --- a/src/services/Odin.Services/Drives/FileSystem/Base/Upload/Attachments/PayloadStreamWriterBase.cs +++ b/src/services/Odin.Services/Drives/FileSystem/Base/Upload/Attachments/PayloadStreamWriterBase.cs @@ -66,7 +66,7 @@ public virtual async Task AddPayload(string key, string contentTypeFromMultipart } var extension = TenantPathManager.GetBasePayloadFileNameAndExtension(key, descriptor.PayloadUid); - var bytesWritten = await FileSystem.Storage.WriteTempStream(_package.TempFile.AsTempFileUpload(), extension, data, odinContext); + var bytesWritten = await FileSystem.Storage.WriteUploadStream(_package.UploadFile, extension, data, odinContext); if (bytesWritten > 0) { _package.Payloads.Add(descriptor.PackagePayloadDescriptor(bytesWritten, contentTypeFromMultipartSection)); @@ -112,7 +112,7 @@ public virtual async Task AddThumbnail(string thumbnailUploadKey, string content result.ThumbnailDescriptor.PixelWidth, result.ThumbnailDescriptor.PixelHeight); - var bytesWritten = await FileSystem.Storage.WriteTempStream(_package.TempFile.AsTempFileUpload(), extenstion, data, odinContext); + var bytesWritten = await FileSystem.Storage.WriteUploadStream(_package.UploadFile, extenstion, data, odinContext); _package.Thumbnails.Add(new PackageThumbnailDescriptor() { @@ -154,12 +154,12 @@ public async Task FinalizeUpload(IOdinContext odinContext) }; } - public async Task CleanupTempFiles(IOdinContext odinContext) + public async Task CleanupStagingFiles(IOdinContext odinContext) { - if (_package?.TempFile != null) + if (_package?.UploadFile != null) { var uploadedPayloads = _package.GetFinalPayloadDescriptors(); - await FileSystem.Storage.CleanupUploadTemporaryFiles(_package.TempFile.AsTempFileUpload(), uploadedPayloads, odinContext); + await FileSystem.Storage.CleanupUploadTemporaryFiles(_package.UploadFile, uploadedPayloads, odinContext); } } diff --git a/src/services/Odin.Services/Drives/FileSystem/Base/Upload/FileSystemStreamWriterBase.cs b/src/services/Odin.Services/Drives/FileSystem/Base/Upload/FileSystemStreamWriterBase.cs index 7e17459c2f..48f1e9c8f3 100644 --- a/src/services/Odin.Services/Drives/FileSystem/Base/Upload/FileSystemStreamWriterBase.cs +++ b/src/services/Odin.Services/Drives/FileSystem/Base/Upload/FileSystemStreamWriterBase.cs @@ -30,7 +30,7 @@ public abstract class FileSystemStreamWriterBase private readonly TenantContext _tenantContext; - private readonly IDriveManager _driveManager; + protected readonly IDriveManager _driveManager; private readonly PeerOutgoingTransferService _peerOutgoingTransferService; private readonly ILogger _logger; @@ -115,7 +115,7 @@ public virtual async Task AddPayload(string key, string contentTypeFromMultipart } var extension = TenantPathManager.GetBasePayloadFileNameAndExtension(key, descriptor.PayloadUid); - var bytesWritten = await FileSystem.Storage.WriteTempStream(Package.InternalFile.AsTempFileUpload(), extension, data, odinContext); + var bytesWritten = await FileSystem.Storage.WriteUploadStream(Package.InternalFile, extension, data, odinContext); if (bytesWritten != data.Length) { @@ -163,7 +163,7 @@ public virtual async Task AddThumbnail(string thumbnailUploadKey, string overrid result.ThumbnailDescriptor.PixelHeight ); - var bytesWritten = await FileSystem.Storage.WriteTempStream(Package.InternalFile.AsTempFileUpload(), extension, data, odinContext); + var bytesWritten = await FileSystem.Storage.WriteUploadStream(Package.InternalFile, extension, data, odinContext); if (bytesWritten != data.Length) { @@ -391,13 +391,13 @@ await _peerOutgoingTransferService.SendPeerPushNotification( return recipientStatus; } - public async Task CleanupTempFiles(IOdinContext odinContext) + public async Task CleanupStagingFiles(IOdinContext odinContext) { if (Package?.InternalFile != null) { // use the descriptors from the package as they would have been uploaded to the upload folder var descriptors = Package.GetFinalPayloadDescriptors(); - await FileSystem.Storage.CleanupUploadTemporaryFiles(Package.InternalFile.AsTempFileUpload(), descriptors, odinContext); + await FileSystem.Storage.CleanupUploadTemporaryFiles(Package.InternalFile, descriptors, odinContext); } } diff --git a/src/services/Odin.Services/Drives/FileSystem/Comment/Attachments/CommentStreamWriter.cs b/src/services/Odin.Services/Drives/FileSystem/Comment/Attachments/CommentStreamWriter.cs index 16ed8afc76..31bf4f8dfb 100644 --- a/src/services/Odin.Services/Drives/FileSystem/Comment/Attachments/CommentStreamWriter.cs +++ b/src/services/Odin.Services/Drives/FileSystem/Comment/Attachments/CommentStreamWriter.cs @@ -24,8 +24,7 @@ protected override Task ValidatePayloads(PayloadOnlyPackage package, ServerFileH protected override async Task UpdatePayloads(PayloadOnlyPackage package, ServerFileHeader header, IOdinContext odinContext) { return await FileSystem.Storage.UpdatePayloads( - // package.InternalFile, - package.TempFile.AsTempFileUpload(), + package.UploadFile, targetFile: package.InternalFile, incomingPayloads: package.GetFinalPayloadDescriptors(), odinContext, diff --git a/src/services/Odin.Services/Drives/FileSystem/Comment/CommentFileStorageService.cs b/src/services/Odin.Services/Drives/FileSystem/Comment/CommentFileStorageService.cs index ee10d619f1..784c70d448 100644 --- a/src/services/Odin.Services/Drives/FileSystem/Comment/CommentFileStorageService.cs +++ b/src/services/Odin.Services/Drives/FileSystem/Comment/CommentFileStorageService.cs @@ -20,6 +20,8 @@ public class CommentFileStorageService( IDriveManager driveManager, LongTermStorageManager longTermStorageManager, UploadStorageManager uploadStorageManager, + InboxStorageManager inboxStorageManager, + FileReaderWriter fileReaderWriter, // OrphanTestUtil orphanTestUtil, IdentityDatabase db ) @@ -30,6 +32,8 @@ IdentityDatabase db driveManager, longTermStorageManager, uploadStorageManager, + inboxStorageManager, + fileReaderWriter, // orphanTestUtil, db) { diff --git a/src/services/Odin.Services/Drives/FileSystem/Comment/CommentStreamWriter.cs b/src/services/Odin.Services/Drives/FileSystem/Comment/CommentStreamWriter.cs index ee8a790601..4fae966c14 100644 --- a/src/services/Odin.Services/Drives/FileSystem/Comment/CommentStreamWriter.cs +++ b/src/services/Odin.Services/Drives/FileSystem/Comment/CommentStreamWriter.cs @@ -70,8 +70,9 @@ protected override async Task ProcessNewFileUpload(FileUploadPackage package, Ke // this point, we have validated the ReferenceToFile already exists // - await FileSystem.Storage.CommitNewFile(package.InternalFile.AsTempFileUpload(), keyHeader, metadata, serverMetadata, false, - odinContext); + var drive = await _driveManager.GetDriveAsync(package.InternalFile.DriveId); + await FileSystem.Storage.CommitNewFile(package.InternalFile, keyHeader, metadata, serverMetadata, false, + odinContext, sourceFolderPath: drive.GetDriveUploadPath()); } protected override async Task ProcessExistingFileUpload(FileUploadPackage package, KeyHeader keyHeader, FileMetadata metadata, @@ -95,14 +96,17 @@ await FileSystem.Storage.OverwriteMetadata( if (package.InstructionSet.StorageOptions.StorageIntent == StorageIntent.NewFileOrOverwrite) { + var drive = await _driveManager.GetDriveAsync(package.InternalFile.DriveId); await FileSystem.Storage.OverwriteFile( - originFile: package.InternalFile.AsTempFileUpload(), + originFile: package.InternalFile, targetFile: targetFile, keyHeader: keyHeader, newMetadata: metadata, serverMetadata: serverMetadata, ignorePayload: false, - odinContext: odinContext, markComplete: null); + odinContext: odinContext, + markComplete: null, + sourceFolderPath: drive.GetDriveUploadPath()); return; } diff --git a/src/services/Odin.Services/Drives/FileSystem/Standard/Attachments/StandardFileStreamWriter.cs b/src/services/Odin.Services/Drives/FileSystem/Standard/Attachments/StandardFileStreamWriter.cs index a40daeabcd..947757141b 100644 --- a/src/services/Odin.Services/Drives/FileSystem/Standard/Attachments/StandardFileStreamWriter.cs +++ b/src/services/Odin.Services/Drives/FileSystem/Standard/Attachments/StandardFileStreamWriter.cs @@ -22,8 +22,7 @@ protected override Task ValidatePayloads(PayloadOnlyPackage package, ServerFileH protected override async Task UpdatePayloads(PayloadOnlyPackage package, ServerFileHeader header,IOdinContext odinContext) { return await FileSystem.Storage.UpdatePayloads( - // package.InternalFile, - package.TempFile.AsTempFileUpload(), + package.UploadFile, targetFile: package.InternalFile, incomingPayloads: package.GetFinalPayloadDescriptors(), odinContext, diff --git a/src/services/Odin.Services/Drives/FileSystem/Standard/StandardFileDriveStorageService.cs b/src/services/Odin.Services/Drives/FileSystem/Standard/StandardFileDriveStorageService.cs index 5db65c5e25..5db97ca892 100644 --- a/src/services/Odin.Services/Drives/FileSystem/Standard/StandardFileDriveStorageService.cs +++ b/src/services/Odin.Services/Drives/FileSystem/Standard/StandardFileDriveStorageService.cs @@ -20,6 +20,8 @@ public class StandardFileDriveStorageService( IDriveManager driveManager, LongTermStorageManager longTermStorageManager, UploadStorageManager uploadStorageManager, + InboxStorageManager inboxStorageManager, + FileReaderWriter fileReaderWriter, //OrphanTestUtil orphanTestUtil, IdentityDatabase db) : DriveStorageServiceBase( @@ -29,6 +31,8 @@ public class StandardFileDriveStorageService( driveManager, longTermStorageManager, uploadStorageManager, + inboxStorageManager, + fileReaderWriter, // orphanTestUtil, db) { diff --git a/src/services/Odin.Services/Drives/FileSystem/Standard/StandardFileStreamWriter.cs b/src/services/Odin.Services/Drives/FileSystem/Standard/StandardFileStreamWriter.cs index 7de53b0e78..91a4329570 100644 --- a/src/services/Odin.Services/Drives/FileSystem/Standard/StandardFileStreamWriter.cs +++ b/src/services/Odin.Services/Drives/FileSystem/Standard/StandardFileStreamWriter.cs @@ -59,8 +59,9 @@ protected override async Task ProcessNewFileUpload(FileUploadPackage package, Ke ServerMetadata serverMetadata, IOdinContext odinContext) { - await FileSystem.Storage.CommitNewFile(package.InternalFile.AsTempFileUpload(), keyHeader, metadata, serverMetadata, false, - odinContext); + var drive = await _driveManager.GetDriveAsync(package.InternalFile.DriveId); + await FileSystem.Storage.CommitNewFile(package.InternalFile, keyHeader, metadata, serverMetadata, false, + odinContext, sourceFolderPath: drive.GetDriveUploadPath()); } protected override async Task ProcessExistingFileUpload(FileUploadPackage package, KeyHeader keyHeader, FileMetadata metadata, @@ -80,15 +81,17 @@ await FileSystem.Storage.OverwriteMetadata( if (package.InstructionSet.StorageOptions.StorageIntent == StorageIntent.NewFileOrOverwrite) { + var drive = await _driveManager.GetDriveAsync(package.InternalFile.DriveId); await FileSystem.Storage.OverwriteFile( - originFile: package.InternalFile.AsTempFileUpload(), + originFile: package.InternalFile, targetFile: package.InternalFile, keyHeader: keyHeader, newMetadata: metadata, serverMetadata: serverMetadata, ignorePayload: false, odinContext: odinContext, - markComplete: null); + markComplete: null, + sourceFolderPath: drive.GetDriveUploadPath()); return; } diff --git a/src/services/Odin.Services/Drives/InternalDriveFileId.cs b/src/services/Odin.Services/Drives/InternalDriveFileId.cs index 5b2184a099..b57083a5ed 100644 --- a/src/services/Odin.Services/Drives/InternalDriveFileId.cs +++ b/src/services/Odin.Services/Drives/InternalDriveFileId.cs @@ -47,15 +47,6 @@ public void Validate() } - public TempFile AsTempFileUpload() - { - return new TempFile() - { - File = this, - StorageType = TempStorageType.Upload - }; - } - public static bool operator ==(InternalDriveFileId d1, InternalDriveFileId d2) { return d1.DriveId == d2.DriveId && d1.FileId == d2.FileId; diff --git a/src/services/Odin.Services/Drives/TempFile.cs b/src/services/Odin.Services/Drives/TempFile.cs deleted file mode 100644 index 3798c14029..0000000000 --- a/src/services/Odin.Services/Drives/TempFile.cs +++ /dev/null @@ -1,13 +0,0 @@ -#nullable enable -namespace Odin.Services.Drives; - -public struct TempFile -{ - public InternalDriveFileId File { get; set; } - public TempStorageType StorageType { get; set; } - - public override string ToString() - { - return $"File: {File} StorageType: {StorageType}"; - } -} \ No newline at end of file diff --git a/src/services/Odin.Services/Drives/TempStorageType.cs b/src/services/Odin.Services/Drives/TempStorageType.cs deleted file mode 100644 index 5d1511f60f..0000000000 --- a/src/services/Odin.Services/Drives/TempStorageType.cs +++ /dev/null @@ -1,8 +0,0 @@ -#nullable enable -namespace Odin.Services.Drives; - -public enum TempStorageType -{ - Upload = 1, - Inbox = 2 -} \ No newline at end of file diff --git a/src/services/Odin.Services/Peer/Incoming/Drive/Transfer/FileUpdate/PeerDriveIncomingFileUpdateService.cs b/src/services/Odin.Services/Peer/Incoming/Drive/Transfer/FileUpdate/PeerDriveIncomingFileUpdateService.cs index 75e3d1ba64..e7d200bedd 100644 --- a/src/services/Odin.Services/Peer/Incoming/Drive/Transfer/FileUpdate/PeerDriveIncomingFileUpdateService.cs +++ b/src/services/Odin.Services/Peer/Incoming/Drive/Transfer/FileUpdate/PeerDriveIncomingFileUpdateService.cs @@ -35,7 +35,8 @@ public class PeerDriveIncomingFileUpdateService( TransitInboxBoxStorage transitInboxBoxStorage) { private EncryptedRecipientFileUpdateInstructionSet _updateInstructionSet; - private TempFile _tempFile; + private InternalDriveFileId _file; + private bool _isDirectWrite; private readonly Dictionary> _uploadedKeys = new(StringComparer.InvariantCultureIgnoreCase); @@ -44,14 +45,10 @@ public async Task InitializeIncomingTransfer(EncryptedRecipientFileUpdateInstruc IOdinContext odinContext) { var driveId = transferInstructionSet.Request.File.TargetDrive.Alias; - var canDirectWrite = await CanDirectWriteFile(driveId, fileMetadata, transferInstructionSet.FileSystemType, odinContext); + _isDirectWrite = await CanDirectWriteFile(driveId, fileMetadata, transferInstructionSet.FileSystemType, odinContext); // Notice here: we always create a new fileId when receiving a new file. - _tempFile = new TempFile() - { - File = await fileSystem.Storage.CreateInternalFileId(driveId, odinContext), - StorageType = canDirectWrite ? TempStorageType.Upload : TempStorageType.Inbox - }; + _file = await fileSystem.Storage.CreateInternalFileId(driveId, odinContext); _updateInstructionSet = transferInstructionSet; await WriteInstructionsAndMetadataToDisk(fileMetadata, odinContext); @@ -60,7 +57,10 @@ public async Task InitializeIncomingTransfer(EncryptedRecipientFileUpdateInstruc public async Task AcceptPayload(string key, string fileExtension, Stream data, IOdinContext odinContext) { _uploadedKeys.TryAdd(key, new List()); - await fileSystem.Storage.WriteTempStream(_tempFile, fileExtension, data, odinContext); + if (_isDirectWrite) + await fileSystem.Storage.WriteUploadStream(_file, fileExtension, data, odinContext); + else + await fileSystem.Storage.WriteInboxStream(_file, fileExtension, data, odinContext); } public async Task AcceptThumbnail(string payloadKey, string thumbnailKey, string fileExtension, Stream data, @@ -75,7 +75,10 @@ public async Task AcceptThumbnail(string payloadKey, string thumbnailKey, string thumbnailKeys.Add(thumbnailKey); _uploadedKeys[payloadKey] = thumbnailKeys; - await fileSystem.Storage.WriteTempStream(_tempFile, fileExtension, data, odinContext); + if (_isDirectWrite) + await fileSystem.Storage.WriteUploadStream(_file, fileExtension, data, odinContext); + else + await fileSystem.Storage.WriteInboxStream(_file, fileExtension, data, odinContext); } public async Task FinalizeTransfer(FileMetadata fileMetadata, IOdinContext odinContext) @@ -120,9 +123,10 @@ public async Task FinalizeTransfer(FileMetadata fileMetada throw new OdinSystemException("Unhandled Routing"); } - public async Task CleanupTempFiles(List descriptors, IOdinContext odinContext) + public async Task CleanupStagingFiles(List descriptors, IOdinContext odinContext) { - await fileSystem.Storage.CleanupUploadTemporaryFiles(this._tempFile, descriptors, odinContext); + if (_isDirectWrite) + await fileSystem.Storage.CleanupUploadTemporaryFiles(_file, descriptors, odinContext); } // @@ -130,7 +134,7 @@ public async Task CleanupTempFiles(List descriptors, IOdinCon private async Task FinalizeTransferInternal(FileMetadata fileMetadata, IOdinContext odinContext) { //S0001, S1000, S2000 - can the sender write the content to the target drive? - await fileSystem.Storage.AssertCanWriteToDrive(_tempFile.File.DriveId, odinContext); + await fileSystem.Storage.AssertCanWriteToDrive(_file.DriveId, odinContext); var directWriteSuccess = await TryDirectWriteFileAsync(fileMetadata, odinContext); @@ -140,17 +144,17 @@ private async Task FinalizeTransferInternal(FileMetadata fileM } logger.LogDebug("TryDirectWrite failed for file ({file}) - falling back to inbox. Writing metadata to inbox", - _tempFile); + _file); try { - _tempFile.StorageType = TempStorageType.Inbox; + _isDirectWrite = false; await WriteInstructionsAndMetadataToDisk(fileMetadata, odinContext); } catch (Exception e) { logger.LogError(e, "After TryDirectWriteFailed, we also failed to ensure " + - "metadata and instructions are available to the inbox. file: {tempFile}", _tempFile.File); + "metadata and instructions are available to the inbox. file: {file}", _file); } //S1220 @@ -159,20 +163,25 @@ private async Task FinalizeTransferInternal(FileMetadata fileM private async Task WriteInstructionsAndMetadataToDisk(FileMetadata fileMetadata, IOdinContext odinContext) { - logger.LogDebug("Writing metadata as {tempFile}", _tempFile); + logger.LogDebug("Writing metadata for file {file} (isDirectWrite: {isDirectWrite})", _file, _isDirectWrite); // Write the instruction set to disk await using var stream = new MemoryStream(OdinSystemSerializer.Serialize(_updateInstructionSet).ToUtf8ByteArray()); - await fileSystem.Storage.WriteTempStream(_tempFile, TenantPathManager.TransferInstructionSetExtension, stream, - odinContext); + if (_isDirectWrite) + await fileSystem.Storage.WriteUploadStream(_file, TenantPathManager.TransferInstructionSetExtension, stream, odinContext); + else + await fileSystem.Storage.WriteInboxStream(_file, TenantPathManager.TransferInstructionSetExtension, stream, odinContext); var metadataStream = new MemoryStream(Encoding.UTF8.GetBytes(OdinSystemSerializer.Serialize(fileMetadata))); - await fileSystem.Storage.WriteTempStream(_tempFile, TenantPathManager.MetadataExtension, metadataStream, odinContext); + if (_isDirectWrite) + await fileSystem.Storage.WriteUploadStream(_file, TenantPathManager.MetadataExtension, metadataStream, odinContext); + else + await fileSystem.Storage.WriteInboxStream(_file, TenantPathManager.MetadataExtension, metadataStream, odinContext); } private async Task TryDirectWriteFileAsync(FileMetadata metadata, IOdinContext odinContext) { - if (!await CanDirectWriteFile(_tempFile.File.DriveId, metadata, _updateInstructionSet.FileSystemType, odinContext)) + if (!await CanDirectWriteFile(_file.DriveId, metadata, _updateInstructionSet.FileSystemType, odinContext)) { return false; } @@ -180,11 +189,14 @@ private async Task TryDirectWriteFileAsync(FileMetadata metadata, IOdinCon PeerFileUpdateWriter updateWriter = new PeerFileUpdateWriter(logger, fileSystemResolver, driveManager); var sender = odinContext.GetCallerOdinIdOrFail(); var decryptedKeyHeader = DecryptKeyHeaderWithSharedSecret(_updateInstructionSet.EncryptedKeyHeader, odinContext); + var drive = await driveManager.GetDriveAsync(_file.DriveId); + var sourceFolderPath = _isDirectWrite ? drive.GetDriveUploadPath() : drive.GetDriveInboxPath(); if (metadata.IsEncrypted == false) { //S1110 - Write to disk and send notifications - await updateWriter.UpsertFileAsync(_tempFile, decryptedKeyHeader, sender, _updateInstructionSet, odinContext, null); + await updateWriter.UpsertFileAsync(_file, decryptedKeyHeader, sender, _updateInstructionSet, odinContext, null, + sourceFolderPath: sourceFolderPath); return true; } @@ -192,13 +204,14 @@ private async Task TryDirectWriteFileAsync(FileMetadata metadata, IOdinCon if (metadata.IsEncrypted) { // Next determine if we can direct write the file - var hasStorageKey = odinContext.PermissionsContext.TryGetDriveStorageKey(_tempFile.File.DriveId, out _); + var hasStorageKey = odinContext.PermissionsContext.TryGetDriveStorageKey(_file.DriveId, out _); //S1200 if (hasStorageKey) { //S1205 - await updateWriter.UpsertFileAsync(_tempFile, decryptedKeyHeader, sender, _updateInstructionSet, odinContext, null); + await updateWriter.UpsertFileAsync(_file, decryptedKeyHeader, sender, _updateInstructionSet, odinContext, null, + sourceFolderPath: sourceFolderPath); return true; } @@ -232,8 +245,8 @@ private async Task RouteToInboxAsync(IOdinContext odinContext) Priority = 500, InstructionType = TransferInstructionType.UpdateFile, - DriveId = _tempFile.File.DriveId, - FileId = _tempFile.File.FileId, + DriveId = _file.DriveId, + FileId = _file.FileId, FileSystemType = _updateInstructionSet.FileSystemType, Data = OdinSystemSerializer.Serialize(_updateInstructionSet).ToUtf8ByteArray(), diff --git a/src/services/Odin.Services/Peer/Incoming/Drive/Transfer/FileUpdate/PeerFileUpdateWriter.cs b/src/services/Odin.Services/Peer/Incoming/Drive/Transfer/FileUpdate/PeerFileUpdateWriter.cs index 2f5ab411a4..2fa10364eb 100644 --- a/src/services/Odin.Services/Peer/Incoming/Drive/Transfer/FileUpdate/PeerFileUpdateWriter.cs +++ b/src/services/Odin.Services/Peer/Incoming/Drive/Transfer/FileUpdate/PeerFileUpdateWriter.cs @@ -25,25 +25,26 @@ namespace Odin.Services.Peer.Incoming.Drive.Transfer.FileUpdate /// public class PeerFileUpdateWriter(ILogger logger, FileSystemResolver fileSystemResolver, IDriveManager driveManager) { - public async Task<(bool success, List payloads)> UpsertFileAsync(TempFile tempFile, + public async Task<(bool success, List payloads)> UpsertFileAsync(InternalDriveFileId file, KeyHeader decryptedKeyHeader, OdinId sender, EncryptedRecipientFileUpdateInstructionSet instructionSet, IOdinContext odinContext, - WriteSecondDatabaseRowBase markComplete) + WriteSecondDatabaseRowBase markComplete, + string sourceFolderPath) { bool success = false; List payloads = []; var fileSystemType = instructionSet.FileSystemType; var fs = fileSystemResolver.ResolveFileSystem(fileSystemType); - logger.LogDebug("PeerFileUpdateWriter - UpsertFileAsync called tempFile: {file}", tempFile); + logger.LogDebug("PeerFileUpdateWriter - UpsertFileAsync called file: {file}", file); - var incomingMetadata = await LoadMetadataFromTemp(tempFile, fs, odinContext); + var incomingMetadata = await LoadMetadataFromTemp(file, fs, sourceFolderPath, odinContext); // Validations var (targetFile, existingHeader) = await GetTargetFileHeader(instructionSet.Request.File, fs, odinContext); - var (targetAcl, isCollaborationChannel) = await DetermineAclAsync(tempFile, + var (targetAcl, isCollaborationChannel) = await DetermineAclAsync(file, instructionSet, fileSystemType, incomingMetadata, @@ -61,8 +62,8 @@ public class PeerFileUpdateWriter(ILogger logger, FileSystemResolver fileSystemR if (null == existingHeader) { - logger.LogDebug("PeerFileUpdateWriter WriteNewFile - temp file: {file}. Payload count: {pc}", - tempFile, + logger.LogDebug("PeerFileUpdateWriter WriteNewFile - file: {file}. Payload count: {pc}", + file, incomingMetadata.Payloads?.Count); // // we must create a new file @@ -71,14 +72,15 @@ public class PeerFileUpdateWriter(ILogger logger, FileSystemResolver fileSystemR await PerformanceCounter.MeasureExecutionTime("PeerFileUpdateWriter WriteNewFile", async () => { - (success, payloads) = await fs.Storage.CommitNewFile(tempFile, + (success, payloads) = await fs.Storage.CommitNewFile(file, decryptedKeyHeader, incomingMetadata, serverMetadata, ignorePayload: false, odinContext, - useThisVersionTag: instructionSet.Request.NewVersionTag, - markComplete); + useThisVersionTag: instructionSet.Request.NewVersionTag, + markComplete, + sourceFolderPath: sourceFolderPath); }); logger.LogDebug("PeerFileUpdateWriter WriteNewFile - success: {success} committed payload count {pc}", @@ -91,8 +93,8 @@ await PerformanceCounter.MeasureExecutionTime("PeerFileUpdateWriter WriteNewFile //Update existing file await PerformanceCounter.MeasureExecutionTime("PeerFileUpdateWriter UpdateExistingFile", async () => { - logger.LogDebug("PeerFileUpdateWriter UpdateExistingFile - temp file: {file}. Payload count: {pc}. GTID: {gtid}", - tempFile, + logger.LogDebug("PeerFileUpdateWriter UpdateExistingFile - file: {file}. Payload count: {pc}. GTID: {gtid}", + file, incomingMetadata.Payloads?.Count, incomingMetadata.GlobalTransitId); @@ -120,7 +122,7 @@ await PerformanceCounter.MeasureExecutionTime("PeerFileUpdateWriter UpdateExisti ServerMetadata = serverMetadata }; - (success, payloads) = await fs.Storage.UpdateBatchAsync(tempFile, targetFile.Value, manifest, odinContext, markComplete); + (success, payloads) = await fs.Storage.UpdateBatchAsync(file, targetFile.Value, manifest, odinContext, markComplete, sourceFolderPath: sourceFolderPath); logger.LogDebug("PeerFileUpdateWriter UpdateExistingFile - success: {success} committed payload count {pc}", success, @@ -132,21 +134,22 @@ await PerformanceCounter.MeasureExecutionTime("PeerFileUpdateWriter UpdateExisti } private async Task LoadMetadataFromTemp( - TempFile tempFile, + InternalDriveFileId file, IDriveFileSystem fs, + string sourceFolderPath, IOdinContext odinContext) { FileMetadata incomingMetadata = default; var metadataMs = await PerformanceCounter.MeasureExecutionTime("PeerFileUpdateWriter HandleFile ReadTempFile", async () => { - var bytes = await fs.Storage.GetAllFileBytesFromTempFile(tempFile, MultipartHostTransferParts.Metadata.ToString().ToLower(), - odinContext); + var extension = MultipartHostTransferParts.Metadata.ToString().ToLower(); + var bytes = await fs.Storage.GetAllFileBytesFromTempFileForWriting(file, extension, sourceFolderPath, odinContext); if (bytes == null) { // this is bad error. - logger.LogError("Cannot find the metadata file (File:{file} on DriveId:{driveID}) was not found ", tempFile.File.FileId, - tempFile.File.DriveId); + logger.LogError("Cannot find the metadata file (File:{file} on DriveId:{driveID}) was not found ", file.FileId, + file.DriveId); throw new OdinFileWriteException("Missing temp file while processing inbox"); } @@ -155,8 +158,8 @@ private async Task LoadMetadataFromTemp( incomingMetadata = OdinSystemSerializer.Deserialize(json); if (null == incomingMetadata) { - logger.LogError("Metadata file (File:{file} on DriveId:{driveID}) could not be deserialized ", tempFile.File.FileId, - tempFile.File.DriveId); + logger.LogError("Metadata file (File:{file} on DriveId:{driveID}) could not be deserialized ", file.FileId, + file.DriveId); throw new OdinFileWriteException("Metadata could not be deserialized"); } }); @@ -171,7 +174,7 @@ private async Task LoadMetadataFromTemp( return incomingMetadata; } - private async Task<(AccessControlList acl, bool isCollabChannel)> DetermineAclAsync(TempFile tempFile, + private async Task<(AccessControlList acl, bool isCollabChannel)> DetermineAclAsync(InternalDriveFileId file, EncryptedRecipientFileUpdateInstructionSet instructionSet, FileSystemType fileSystemType, FileMetadata metadata, @@ -184,7 +187,7 @@ private async Task LoadMetadataFromTemp( RequiredSecurityGroup = SecurityGroupType.Owner }; - var drive = await driveManager.GetDriveAsync(tempFile.File.DriveId); + var drive = await driveManager.GetDriveAsync(file.DriveId); var isCollaborationChannel = drive.IsCollaborationDrive(); //TODO: this might be a hacky place to put this but let's let it cook. It might better be put into the comment storage diff --git a/src/services/Odin.Services/Peer/Incoming/Drive/Transfer/InboxStorage/IncomingTransferStateItem.cs b/src/services/Odin.Services/Peer/Incoming/Drive/Transfer/InboxStorage/IncomingTransferStateItem.cs index dfee57a35d..74b9c5869e 100644 --- a/src/services/Odin.Services/Peer/Incoming/Drive/Transfer/InboxStorage/IncomingTransferStateItem.cs +++ b/src/services/Odin.Services/Peer/Incoming/Drive/Transfer/InboxStorage/IncomingTransferStateItem.cs @@ -1,14 +1,18 @@ -using Odin.Core; using Odin.Services.Drives; -using Odin.Services.Drives.DriveCore.Storage; using Odin.Services.Peer.Encryption; namespace Odin.Services.Peer.Incoming.Drive.Transfer.InboxStorage { - public class IncomingTransferStateItem(TempFile tempFile, EncryptedRecipientTransferInstructionSet transferInstructionSet) + public class IncomingTransferStateItem(InternalDriveFileId file, bool isDirectWrite, EncryptedRecipientTransferInstructionSet transferInstructionSet) { public EncryptedRecipientTransferInstructionSet TransferInstructionSet { get; init; } = transferInstructionSet; - - public TempFile TempFile { get; init; } = tempFile; + + public InternalDriveFileId File { get; init; } = file; + + /// + /// True if the file was written to upload (temp) storage for direct write processing. + /// False if the file was written to inbox storage. + /// + public bool IsDirectWrite { get; init; } = isDirectWrite; } -} \ No newline at end of file +} diff --git a/src/services/Odin.Services/Peer/Incoming/Drive/Transfer/PeerDriveIncomingTransferService.cs b/src/services/Odin.Services/Peer/Incoming/Drive/Transfer/PeerDriveIncomingTransferService.cs index 2a2a82e075..03bbec7dcd 100644 --- a/src/services/Odin.Services/Peer/Incoming/Drive/Transfer/PeerDriveIncomingTransferService.cs +++ b/src/services/Odin.Services/Peer/Incoming/Drive/Transfer/PeerDriveIncomingTransferService.cs @@ -59,20 +59,19 @@ public async Task InitializeIncomingTransfer(EncryptedRecipientTransferInstructi var canDirectWrite = await CanDirectWriteFile(driveId, metadata, transferInstructionSet, odinContext); // Notice here: we always create a new fileId when receiving a new file. - var file = new TempFile() - { - File = await fileSystem.Storage.CreateInternalFileId(driveId, odinContext), - StorageType = canDirectWrite ? TempStorageType.Upload : TempStorageType.Inbox - }; + var file = await fileSystem.Storage.CreateInternalFileId(driveId, odinContext); - _transferState = new IncomingTransferStateItem(file, transferInstructionSet); - await WriteInstructionsAndMetadataToInbox(file, metadata, transferInstructionSet, odinContext); + _transferState = new IncomingTransferStateItem(file, canDirectWrite, transferInstructionSet); + await WriteInstructionsAndMetadataToStorage(file, canDirectWrite, metadata, transferInstructionSet, odinContext); } public async Task AcceptPayload(string key, string fileExtension, Stream data, IOdinContext odinContext) { _uploadedKeys.TryAdd(key, new List()); - await fileSystem.Storage.WriteTempStream(_transferState.TempFile, fileExtension, data, odinContext); + if (_transferState.IsDirectWrite) + await fileSystem.Storage.WriteUploadStream(_transferState.File, fileExtension, data, odinContext); + else + await fileSystem.Storage.WriteInboxStream(_transferState.File, fileExtension, data, odinContext); } public async Task AcceptThumbnail(string payloadKey, string thumbnailKey, string fileExtension, Stream data, @@ -87,7 +86,10 @@ public async Task AcceptThumbnail(string payloadKey, string thumbnailKey, string thumbnailKeys.Add(thumbnailKey); _uploadedKeys[payloadKey] = thumbnailKeys; - await fileSystem.Storage.WriteTempStream(_transferState.TempFile, fileExtension, data, odinContext); + if (_transferState.IsDirectWrite) + await fileSystem.Storage.WriteUploadStream(_transferState.File, fileExtension, data, odinContext); + else + await fileSystem.Storage.WriteInboxStream(_transferState.File, fileExtension, data, odinContext); } public async Task FinalizeTransfer(FileMetadata fileMetadata, IOdinContext odinContext) @@ -281,12 +283,12 @@ await mediator.Publish(new InboxItemReceivedNotification }; } - public async Task CleanupTempFiles(List descriptors, IOdinContext odinContext) + public async Task CleanupStagingFiles(List descriptors, IOdinContext odinContext) { - if (_transferState?.TempFile != null) + if (_transferState?.File != null && _transferState.IsDirectWrite) { // use the descriptors from the package as they would have been uploaded to the upload folder - await fileSystem.Storage.CleanupUploadTemporaryFiles(_transferState.TempFile, descriptors, odinContext); + await fileSystem.Storage.CleanupUploadTemporaryFiles(_transferState.File, descriptors, odinContext); } } @@ -296,7 +298,7 @@ private async Task FinalizeTransferInternal(IncomingTransferSt IOdinContext odinContext) { //S0001, S1000, S2000 - can the sender write the content to the target drive? - await fileSystem.Storage.AssertCanWriteToDrive(stateItem.TempFile.File.DriveId, odinContext); + await fileSystem.Storage.AssertCanWriteToDrive(stateItem.File.DriveId, odinContext); var directWriteSuccess = await TryDirectWriteFile(stateItem, fileMetadata, odinContext); @@ -306,43 +308,47 @@ private async Task FinalizeTransferInternal(IncomingTransferSt } logger.LogDebug("TryDirectWrite failed for file ({file}) - falling back to inbox. Writing metadata to inbox", - stateItem.TempFile); + stateItem.File); - var tempFile = _transferState.TempFile with { StorageType = TempStorageType.Inbox }; var instructionSet = _transferState.TransferInstructionSet; try { - await WriteInstructionsAndMetadataToInbox(tempFile, fileMetadata, instructionSet, odinContext); + await WriteInstructionsAndMetadataToStorage(stateItem.File, isDirectWrite: false, fileMetadata, instructionSet, odinContext); } catch (Exception e) { logger.LogError(e, "After TryDirectWriteFailed, we also failed to ensure " + - "metadata and instructions are available to the inbox. file: {tempFile}", tempFile); + "metadata and instructions are available to the inbox. file: {file}", stateItem.File); } //S1220 return await RouteToInboxAsync(stateItem, odinContext); } - private async Task WriteInstructionsAndMetadataToInbox(TempFile tempFile, FileMetadata fileMetadata, + private async Task WriteInstructionsAndMetadataToStorage(InternalDriveFileId file, bool isDirectWrite, + FileMetadata fileMetadata, EncryptedRecipientTransferInstructionSet instructionSet, IOdinContext odinContext) { - logger.LogDebug("Writing metadata as {tempFile}", tempFile); + logger.LogDebug("Writing metadata for file {file} (isDirectWrite: {isDirectWrite})", file, isDirectWrite); await using var stream = new MemoryStream(OdinSystemSerializer.Serialize(instructionSet).ToUtf8ByteArray()); - await fileSystem.Storage.WriteTempStream(tempFile, TenantPathManager.TransferInstructionSetExtension, stream, - odinContext); + if (isDirectWrite) + await fileSystem.Storage.WriteUploadStream(file, TenantPathManager.TransferInstructionSetExtension, stream, odinContext); + else + await fileSystem.Storage.WriteInboxStream(file, TenantPathManager.TransferInstructionSetExtension, stream, odinContext); var metadataStream = new MemoryStream(Encoding.UTF8.GetBytes(OdinSystemSerializer.Serialize(fileMetadata))); - await fileSystem.Storage.WriteTempStream(tempFile, TenantPathManager.MetadataExtension, metadataStream, - odinContext); + if (isDirectWrite) + await fileSystem.Storage.WriteUploadStream(file, TenantPathManager.MetadataExtension, metadataStream, odinContext); + else + await fileSystem.Storage.WriteInboxStream(file, TenantPathManager.MetadataExtension, metadataStream, odinContext); } private async Task TryDirectWriteFile(IncomingTransferStateItem stateItem, FileMetadata metadata, IOdinContext odinContext) { - if (!await CanDirectWriteFile(stateItem.TempFile.File.DriveId, metadata, stateItem.TransferInstructionSet, odinContext)) + if (!await CanDirectWriteFile(stateItem.File.DriveId, metadata, stateItem.TransferInstructionSet, odinContext)) { return false; } @@ -353,12 +359,13 @@ private async Task TryDirectWriteFile(IncomingTransferStateItem stateItem, var sender = odinContext.GetCallerOdinIdOrFail(); var decryptedKeyHeader = DecryptKeyHeaderWithSharedSecret(stateItem.TransferInstructionSet.SharedSecretEncryptedKeyHeader, odinContext); + var drive = await driveManager.GetDriveAsync(stateItem.File.DriveId); if (metadata.IsEncrypted == false) { //S1110 - Write to disk and send notifications - await writer.HandleFile(stateItem.TempFile, fileSystem, decryptedKeyHeader, sender, stateItem.TransferInstructionSet, - odinContext); + await writer.HandleFile(stateItem.File, fileSystem, decryptedKeyHeader, sender, stateItem.TransferInstructionSet, + odinContext, sourceFolderPath: stateItem.IsDirectWrite ? drive.GetDriveUploadPath() : drive.GetDriveInboxPath()); return true; } @@ -367,19 +374,19 @@ await writer.HandleFile(stateItem.TempFile, fileSystem, decryptedKeyHeader, send if (metadata.IsEncrypted) { // Next determine if we can direct write the file - var hasStorageKey = odinContext.PermissionsContext.TryGetDriveStorageKey(stateItem.TempFile.File.DriveId, out _); + var hasStorageKey = odinContext.PermissionsContext.TryGetDriveStorageKey(stateItem.File.DriveId, out _); //S1200 if (hasStorageKey) { //S1205 - await writer.HandleFile(stateItem.TempFile, fileSystem, decryptedKeyHeader, sender, stateItem.TransferInstructionSet, - odinContext); + await writer.HandleFile(stateItem.File, fileSystem, decryptedKeyHeader, sender, stateItem.TransferInstructionSet, + odinContext, sourceFolderPath: stateItem.IsDirectWrite ? drive.GetDriveUploadPath() : drive.GetDriveInboxPath()); return true; } logger.LogDebug("Caller can direct-write to drive [{drive}] but does not have storage " + - "key for encrypted file", stateItem.TempFile.File.DriveId); + "key for encrypted file", stateItem.File.DriveId); //S2210 - comments cannot fall back to inbox if (stateItem.TransferInstructionSet.FileSystemType == FileSystemType.Comment) @@ -410,8 +417,8 @@ private async Task RouteToInboxAsync(IncomingTransferStateItem Sender = odinContext.GetCallerOdinIdOrFail(), InstructionType = TransferInstructionType.SaveFile, - DriveId = stateItem.TempFile.File.DriveId, - FileId = stateItem.TempFile.File.FileId, + DriveId = stateItem.File.DriveId, + FileId = stateItem.File.FileId, TransferInstructionSet = stateItem.TransferInstructionSet, FileSystemType = stateItem.TransferInstructionSet.FileSystemType, diff --git a/src/services/Odin.Services/Peer/Incoming/Drive/Transfer/PeerFileWriter.cs b/src/services/Odin.Services/Peer/Incoming/Drive/Transfer/PeerFileWriter.cs index 0c68065aa3..dd9495b01a 100644 --- a/src/services/Odin.Services/Peer/Incoming/Drive/Transfer/PeerFileWriter.cs +++ b/src/services/Odin.Services/Peer/Incoming/Drive/Transfer/PeerFileWriter.cs @@ -30,29 +30,32 @@ namespace Odin.Services.Peer.Incoming.Drive.Transfer /// public class PeerFileWriter(ILogger logger, FileSystemResolver fileSystemResolver, IDriveManager driveManager, FeedWriter feedWriter) { - public async Task<(bool success, List payloads)> HandleFile(TempFile tempFile, + public async Task<(bool success, List payloads)> HandleFile(InternalDriveFileId file, IDriveFileSystem fs, KeyHeader decryptedKeyHeader, OdinId sender, EncryptedRecipientTransferInstructionSet encryptedRecipientTransferInstructionSet, IOdinContext odinContext, bool driveOriginWasCollaborative = false, + string sourceFolderPath = null, WriteSecondDatabaseRowBase markComplete = null) { var fileSystemType = encryptedRecipientTransferInstructionSet.FileSystemType; var transferFileType = encryptedRecipientTransferInstructionSet.TransferFileType; + var drive = await driveManager.GetDriveAsync(file.DriveId); + FileMetadata metadata = null; var metadataMs = await PerformanceCounter.MeasureExecutionTime("PeerFileWriter HandleFile ReadTempFile", async () => { - var bytes = await fs.Storage.GetAllFileBytesFromTempFileForWriting(tempFile, - MultipartHostTransferParts.Metadata.ToString().ToLower(), odinContext); + var bytes = await fs.Storage.GetAllFileBytesFromTempFileForWriting(file, + MultipartHostTransferParts.Metadata.ToString().ToLower(), sourceFolderPath, odinContext); if (bytes == null) { // this is bad error. - logger.LogError("Cannot find the metadata file (File:{file} on DriveId:{driveID}) was not found ", tempFile.File.FileId, - tempFile.File.DriveId); + logger.LogError("Cannot find the metadata file (File:{file} on DriveId:{driveID}) was not found ", file.FileId, + file.DriveId); throw new OdinFileWriteException("Missing temp file while processing inbox"); } @@ -60,12 +63,11 @@ public class PeerFileWriter(ILogger logger, FileSystemResolver fileSystemResolve metadata = OdinSystemSerializer.Deserialize(json); - // var theDrive = await driveManager.GetDrive(tempFile.DriveId); if (null == metadata) { logger.LogError("Metadata file (File:{file} on DriveId:{driveID}) could not be deserialized ", - tempFile.File.FileId, - tempFile.File.DriveId); + file.FileId, + file.DriveId); throw new OdinFileWriteException("Metadata could not be deserialized"); } }); @@ -79,7 +81,6 @@ public class PeerFileWriter(ILogger logger, FileSystemResolver fileSystemResolve RequiredSecurityGroup = SecurityGroupType.Owner }; - var drive = await driveManager.GetDriveAsync(tempFile.File.DriveId); var isCollaborationChannel = drive.IsCollaborationDrive(); //TODO: this might be a hacky place to put this but let's let it cook. It might better be put into the comment storage @@ -112,12 +113,12 @@ public class PeerFileWriter(ILogger logger, FileSystemResolver fileSystemResolve switch (transferFileType) { case TransferFileType.Normal: - return await StoreNormalFileLongTermAsync(fs, tempFile, decryptedKeyHeader, metadata, serverMetadata, - encryptedRecipientTransferInstructionSet, odinContext, markComplete); + return await StoreNormalFileLongTermAsync(fs, file, decryptedKeyHeader, metadata, serverMetadata, + encryptedRecipientTransferInstructionSet, odinContext, sourceFolderPath, markComplete); case TransferFileType.EncryptedFileForFeed: case TransferFileType.EncryptedFileForFeedViaTransit: - return await StoreEncryptedFeedFile(fs, tempFile, decryptedKeyHeader, metadata, + return await StoreEncryptedFeedFile(fs, file, decryptedKeyHeader, metadata, driveOriginWasCollaborative, odinContext, markComplete); @@ -226,18 +227,18 @@ private async Task ResetAclForComment(FileMetadata metadata, return targetAcl; } - private async Task<(bool success, List payloads)> WriteNewFile(IDriveFileSystem fs, TempFile tempFile, + private async Task<(bool success, List payloads)> WriteNewFile(IDriveFileSystem fs, InternalDriveFileId file, KeyHeader keyHeader, FileMetadata metadata, ServerMetadata serverMetadata, bool ignorePayloads, IOdinContext odinContext, - WriteSecondDatabaseRowBase markComplete) + string sourceFolderPath, WriteSecondDatabaseRowBase markComplete) { bool success = false; List payloads = []; var ms = await PerformanceCounter.MeasureExecutionTime("PeerFileWriter WriteNewFile", async () => { metadata.TransitCreated = UnixTimeUtc.Now().milliseconds; - (success, payloads) = await fs.Storage.CommitNewFile(tempFile, keyHeader, metadata, - serverMetadata, ignorePayloads, odinContext, markComplete: markComplete); + (success, payloads) = await fs.Storage.CommitNewFile(file, keyHeader, metadata, + serverMetadata, ignorePayloads, odinContext, markComplete: markComplete, sourceFolderPath: sourceFolderPath); }); logger.LogDebug("Handle file->CommitNewFile: {ms} ms", ms); @@ -246,10 +247,10 @@ private async Task ResetAclForComment(FileMetadata metadata, private async Task<(bool success, List payloads)> UpdateExistingFile(IDriveFileSystem fs, - TempFile tempSourceFile, InternalDriveFileId targetFile, + InternalDriveFileId sourceFile, InternalDriveFileId targetFile, KeyHeader keyHeader, FileMetadata metadata, ServerMetadata serverMetadata, bool ignorePayloads, IOdinContext odinContext, - WriteSecondDatabaseRowBase markComplete) + string sourceFolderPath, WriteSecondDatabaseRowBase markComplete) { bool success = false; List payloads = []; @@ -260,9 +261,9 @@ await PerformanceCounter.MeasureExecutionTime("PeerFileWriter UpdateExistingFile metadata.TransitUpdated = UnixTimeUtc.Now().milliseconds; //note: we also update the key header because it might have been changed by the sender - (success, payloads) = await fs.Storage.OverwriteFile(tempSourceFile, targetFile, keyHeader, metadata, serverMetadata, + (success, payloads) = await fs.Storage.OverwriteFile(sourceFile, targetFile, keyHeader, metadata, serverMetadata, ignorePayloads, - odinContext, markComplete); + odinContext, markComplete, sourceFolderPath: sourceFolderPath); }); return (success, payloads); @@ -272,13 +273,13 @@ await PerformanceCounter.MeasureExecutionTime("PeerFileWriter UpdateExistingFile /// Stores a long-term file or overwrites an existing long-term file if a global transit id was set /// private async Task<(bool success, List payloads)> StoreNormalFileLongTermAsync(IDriveFileSystem fs, - TempFile tempFile, KeyHeader keyHeader, + InternalDriveFileId file, KeyHeader keyHeader, FileMetadata newMetadata, ServerMetadata serverMetadata, EncryptedRecipientTransferInstructionSet encryptedRecipientTransferInstructionSet, IOdinContext odinContext, - WriteSecondDatabaseRowBase markComplete) + string sourceFolderPath, WriteSecondDatabaseRowBase markComplete) { var ignorePayloads = newMetadata.PayloadsAreRemote; - var targetDriveId = tempFile.File.DriveId; + var targetDriveId = file.DriveId; if (newMetadata.GlobalTransitId.HasValue == false) { @@ -289,7 +290,7 @@ await PerformanceCounter.MeasureExecutionTime("PeerFileWriter UpdateExistingFile // If we can, then the gtid is the winner and decides the matching file // - SharedSecretEncryptedFileHeader header = await GetFileByGlobalTransitId(fs, tempFile.File.DriveId, + SharedSecretEncryptedFileHeader header = await GetFileByGlobalTransitId(fs, file.DriveId, newMetadata.GlobalTransitId.GetValueOrDefault(), odinContext); // If there is no file matching the gtid, let's check if the UID might point to one @@ -301,7 +302,7 @@ await PerformanceCounter.MeasureExecutionTime("PeerFileWriter UpdateExistingFile if (header == null) { // Neither gtid not uid points to an exiting file, so it's a new file - return await WriteNewFile(fs, tempFile, keyHeader, newMetadata, serverMetadata, ignorePayloads, odinContext, markComplete); + return await WriteNewFile(fs, file, keyHeader, newMetadata, serverMetadata, ignorePayloads, odinContext, sourceFolderPath, markComplete); } header.AssertFileIsActive(); @@ -321,12 +322,12 @@ await PerformanceCounter.MeasureExecutionTime("PeerFileWriter UpdateExistingFile }; //note: we also update the key header because it might have been changed by the sender - return await UpdateExistingFile(fs, tempFile, targetFile, keyHeader, newMetadata, serverMetadata, ignorePayloads, odinContext, - markComplete); + return await UpdateExistingFile(fs, file, targetFile, keyHeader, newMetadata, serverMetadata, ignorePayloads, odinContext, + sourceFolderPath, markComplete); } - private async Task<(bool success, List payloads)> StoreEncryptedFeedFile(IDriveFileSystem fs, TempFile tempFile, + private async Task<(bool success, List payloads)> StoreEncryptedFeedFile(IDriveFileSystem fs, InternalDriveFileId file, KeyHeader keyHeader, FileMetadata newMetadata, bool driveOriginWasCollaborative, IOdinContext odinContext, WriteSecondDatabaseRowBase markComplete) @@ -341,7 +342,7 @@ await PerformanceCounter.MeasureExecutionTime("PeerFileWriter UpdateExistingFile throw new OdinClientException("Must have a global transit id to write to the feed drive.", OdinClientErrorCode.InvalidFile); } - var header = await GetFileByGlobalTransitId(fs, tempFile.File.DriveId, newMetadata.GlobalTransitId.GetValueOrDefault(), + var header = await GetFileByGlobalTransitId(fs, file.DriveId, newMetadata.GlobalTransitId.GetValueOrDefault(), odinContext); @@ -415,10 +416,10 @@ private async Task GetFileByGlobalTransitId(IDr return existingFile; } - public async Task CleanupInboxFiles(TempFile tempFile, List payloads, IOdinContext odinContext) + public async Task CleanupInboxFiles(InternalDriveFileId file, List payloads, IOdinContext odinContext) { var fs = fileSystemResolver.ResolveFileSystem(FileSystemType.Standard); - await fs.Storage.CleanupInboxTemporaryFiles(tempFile, payloads, odinContext); + await fs.Storage.CleanupInboxTemporaryFiles(file, payloads, odinContext); } } } \ No newline at end of file diff --git a/src/services/Odin.Services/Peer/Incoming/Drive/Transfer/PeerInboxProcessor.cs b/src/services/Odin.Services/Peer/Incoming/Drive/Transfer/PeerInboxProcessor.cs index 274a6e0b23..816f664914 100644 --- a/src/services/Odin.Services/Peer/Incoming/Drive/Transfer/PeerInboxProcessor.cs +++ b/src/services/Odin.Services/Peer/Incoming/Drive/Transfer/PeerInboxProcessor.cs @@ -91,18 +91,14 @@ public async Task ProcessInboxAsync(TargetDrive targetDrive, IOdinC logger.LogDebug("Processing Inbox -> Getting Pending Items returned: {itemCount}", items.Count); logger.LogDebug("Processing Inbox (no call to CUOWA) item with marker/popStamp [{marker}]", inboxItem.Marker); - var tempFile = new TempFile() + var file = new InternalDriveFileId() { - File = new InternalDriveFileId() - { - DriveId = inboxItem.DriveId, - FileId = inboxItem.FileId - }, - StorageType = TempStorageType.Inbox + DriveId = inboxItem.DriveId, + FileId = inboxItem.FileId }; PeerFileWriter writer = new PeerFileWriter(logger, fileSystemResolver, driveManager, feedWriter); - var markComplete = new MarkInboxComplete(transitInboxBoxStorage, tempFile.File, inboxItem.Marker); + var markComplete = new MarkInboxComplete(transitInboxBoxStorage, file, inboxItem.Marker); var payloads = new List(); var success = InboxReturnTypes.DeleteFromInbox; @@ -110,27 +106,27 @@ public async Task ProcessInboxAsync(TargetDrive targetDrive, IOdinC { // This function will have marked the inbox item as complete if successful // Otherwise if it returns false, it's a failure - (success, payloads) = await ProcessInboxItemAsync(tempFile, inboxItem, writer, odinContext, markComplete); + (success, payloads) = await ProcessInboxItemAsync(file, inboxItem, writer, odinContext, markComplete); - logger.LogDebug("Item with tempFile ({file}) Processed. success: {s}", tempFile.File.FileId, success); + logger.LogDebug("Item with file ({fileId}) Processed. success: {s}", file.FileId, success); } finally { if (success == InboxReturnTypes.TryAgainLater) { - int n = await transitInboxBoxStorage.MarkFailureAsync(tempFile.File, inboxItem.Marker); + int n = await transitInboxBoxStorage.MarkFailureAsync(file, inboxItem.Marker); if (n != 1) logger.LogError("Inbox: Unable to MarkFailureAsync for TryAgainLater."); } else if (success == InboxReturnTypes.DeleteFromInbox) { - int n = await transitInboxBoxStorage.MarkCompleteAsync(tempFile.File, + int n = await transitInboxBoxStorage.MarkCompleteAsync(file, inboxItem.Marker); // markComplete removes in from the Inbox if (n == 1) { var fs = fileSystemResolver.ResolveFileSystem(inboxItem.FileSystemType); - await fs.Storage.CleanupInboxTemporaryFiles(tempFile, payloads, odinContext); + await fs.Storage.CleanupInboxTemporaryFiles(file, payloads, odinContext); } else { @@ -140,7 +136,7 @@ public async Task ProcessInboxAsync(TargetDrive targetDrive, IOdinC else if (success == InboxReturnTypes.HasBeenMarkedComplete) { var fs = fileSystemResolver.ResolveFileSystem(inboxItem.FileSystemType); - await fs.Storage.CleanupInboxTemporaryFiles(tempFile, payloads, odinContext); + await fs.Storage.CleanupInboxTemporaryFiles(file, payloads, odinContext); } } } @@ -156,7 +152,7 @@ public async Task ProcessInboxAsync(TargetDrive targetDrive, IOdinC /// success return false: the item is failed and we should retry later /// This function should never throw an exception, only return true / false /// - private async Task<(InboxReturnTypes, List payloads)> ProcessInboxItemAsync(TempFile tempFile, + private async Task<(InboxReturnTypes, List payloads)> ProcessInboxItemAsync(InternalDriveFileId file, TransferInboxItem inboxItem, PeerFileWriter writer, IOdinContext odinContext, WriteSecondDatabaseRowBase markComplete) { @@ -171,7 +167,7 @@ public async Task ProcessInboxAsync(TargetDrive targetDrive, IOdinC if (inboxItem.InstructionType == TransferInstructionType.UpdateFile) { - var (success, payloadDescriptors) = await HandleUpdateFileAsync(tempFile, inboxItem, odinContext, markComplete); + var (success, payloadDescriptors) = await HandleUpdateFileAsync(file, inboxItem, odinContext, markComplete); return (success ? InboxReturnTypes.HasBeenMarkedComplete : InboxReturnTypes.TryAgainLater, payloadDescriptors); } @@ -188,21 +184,21 @@ public async Task ProcessInboxAsync(TargetDrive targetDrive, IOdinC if (inboxItem.TransferFileType == TransferFileType.EncryptedFileForFeedViaTransit) { //this was a file sent over transit (fully encrypted for connected identities but targeting the feed drive) - var (success, payloadDescriptors) = await ProcessFeedItemViaTransit(inboxItem, odinContext, writer, tempFile, fs, + var (success, payloadDescriptors) = await ProcessFeedItemViaTransit(inboxItem, odinContext, writer, file, fs, markComplete); return (success ? InboxReturnTypes.HasBeenMarkedComplete : InboxReturnTypes.TryAgainLater, payloadDescriptors); } if (inboxItem.TransferFileType == TransferFileType.EncryptedFileForFeed) //older path { - var (success, payloadDescriptors) = await ProcessEccEncryptedFeedInboxItem(inboxItem, writer, tempFile, fs, + var (success, payloadDescriptors) = await ProcessEccEncryptedFeedInboxItem(inboxItem, writer, file, fs, odinContext, markComplete); return (success ? InboxReturnTypes.HasBeenMarkedComplete : InboxReturnTypes.TryAgainLater, payloadDescriptors); } if (inboxItem.TransferFileType == TransferFileType.Normal) { - var (success, payloadDescriptors) = await ProcessNormalFileSaveOperation(inboxItem, odinContext, writer, tempFile, + var (success, payloadDescriptors) = await ProcessNormalFileSaveOperation(inboxItem, odinContext, writer, file, fs, markComplete); return (success ? InboxReturnTypes.HasBeenMarkedComplete : InboxReturnTypes.TryAgainLater, payloadDescriptors); } @@ -271,13 +267,13 @@ public async Task ProcessInboxAsync(TargetDrive targetDrive, IOdinC "Processing Inbox -> UniqueId Conflict: " + "\nSender: {sender}. " + "\nInbox InstructionType: {instructionType}. " + - "\nTemp File:{f}. " + + "\nFile:{f}. " + "\nInbox item gtid: {gtid} (gtid as hex x'{gtidHex}'). " + "\nPopStamp (hex): {marker} for drive (hex): {driveId} " + "\nAction: Marking Complete", inboxItem.Sender, inboxItem.InstructionType, - tempFile, + file, inboxItem.GlobalTransitId, Convert.ToHexString(inboxItem.GlobalTransitId.ToByteArray()), Utilities.BytesToHexString(inboxItem.Marker.ToByteArray()), @@ -292,13 +288,13 @@ public async Task ProcessInboxAsync(TargetDrive targetDrive, IOdinC "Processing Inbox -> Security Exception: " + "\nSender: {sender}. " + "\nInbox InstructionType: {instructionType}. " + - "\nTemp File:{f}. " + + "\nFile:{f}. " + "\nInbox item gtid: {gtid} (gtid as hex x'{gtidHex}'). " + "\nPopStamp (hex): {marker} for drive (hex): {driveId} " + "\nAction: Marking Complete", inboxItem.Sender, inboxItem.InstructionType, - tempFile, + file, inboxItem.GlobalTransitId, Convert.ToHexString(inboxItem.GlobalTransitId.ToByteArray()), Utilities.BytesToHexString(inboxItem.Marker.ToByteArray()), @@ -314,7 +310,7 @@ public async Task ProcessInboxAsync(TargetDrive targetDrive, IOdinC "Inbox item gtid: {gtid} (gtid as hex x'{gtidHex}'). " + "PopStamp (hex): {marker} for drive (hex): {driveId} Action: Marking Complete", inboxItem.InstructionType, - tempFile, + file, SequentialGuid.ToUnixTimeUtc(inboxItem.FileId).ToDateTime(), inboxItem.GlobalTransitId, Convert.ToHexString(inboxItem.GlobalTransitId.ToByteArray()), @@ -329,17 +325,18 @@ public async Task ProcessInboxAsync(TargetDrive targetDrive, IOdinC private async Task<(bool success, List payloads)> ProcessNormalFileSaveOperation(TransferInboxItem inboxItem, IOdinContext odinContext, PeerFileWriter writer, - TempFile tempFile, IDriveFileSystem fs, WriteSecondDatabaseRowBase markComplete) + InternalDriveFileId file, IDriveFileSystem fs, WriteSecondDatabaseRowBase markComplete) { logger.LogDebug("Processing Inbox -> HandleFile with gtid: {gtid}", inboxItem.GlobalTransitId); var success = false; List payloads = []; var decryptedKeyHeader = await DecryptedKeyHeaderAsync(inboxItem.Sender, inboxItem.SharedSecretEncryptedKeyHeader, odinContext); + var drive = await driveManager.GetDriveAsync(file.DriveId); var handleFileMs = await Benchmark.MillisecondsAsync(async () => { - (success, payloads) = await writer.HandleFile(tempFile, fs, decryptedKeyHeader, inboxItem.Sender, + (success, payloads) = await writer.HandleFile(file, fs, decryptedKeyHeader, inboxItem.Sender, inboxItem.TransferInstructionSet, - odinContext, markComplete: markComplete); + odinContext, sourceFolderPath: drive.GetDriveInboxPath(), markComplete: markComplete); }); logger.LogDebug("Processing Inbox -> HandleFile Complete. gtid: {gtid} Took {ms} ms", inboxItem.GlobalTransitId, @@ -349,18 +346,19 @@ public async Task ProcessInboxAsync(TargetDrive targetDrive, IOdinC private async Task<(bool success, List payloads)> ProcessFeedItemViaTransit(TransferInboxItem inboxItem, IOdinContext odinContext, PeerFileWriter writer, - TempFile tempFile, IDriveFileSystem fs, WriteSecondDatabaseRowBase markComplete) + InternalDriveFileId file, IDriveFileSystem fs, WriteSecondDatabaseRowBase markComplete) { logger.LogDebug("ProcessFeedItemViaTransit -> HandleFile with gtid: {gtid}", inboxItem.GlobalTransitId); bool success = false; List payloads = []; var decryptedKeyHeader = await DecryptedKeyHeaderAsync(inboxItem.Sender, inboxItem.SharedSecretEncryptedKeyHeader, odinContext); + var drive = await driveManager.GetDriveAsync(file.DriveId); var handleFileMs = await Benchmark.MillisecondsAsync(async () => { - (success, payloads) = await writer.HandleFile(tempFile, fs, decryptedKeyHeader, inboxItem.Sender, + (success, payloads) = await writer.HandleFile(file, fs, decryptedKeyHeader, inboxItem.Sender, inboxItem.TransferInstructionSet, - odinContext, false, markComplete: markComplete); + odinContext, sourceFolderPath: drive.GetDriveInboxPath(), markComplete: markComplete); }); logger.LogDebug("ProcessFeedItemViaTransit -> HandleFile Complete. gtid: {gtid} Took {ms} ms", inboxItem.GlobalTransitId, @@ -368,7 +366,7 @@ public async Task ProcessInboxAsync(TargetDrive targetDrive, IOdinC return (success, payloads); } - private async Task<(bool success, List payloadDescriptors)> HandleUpdateFileAsync(TempFile tempFile, + private async Task<(bool success, List payloadDescriptors)> HandleUpdateFileAsync(InternalDriveFileId file, TransferInboxItem inboxItem, IOdinContext odinContext, WriteSecondDatabaseRowBase markComplete) { var writer = new PeerFileUpdateWriter(logger, fileSystemResolver, driveManager); @@ -380,8 +378,9 @@ public async Task ProcessInboxAsync(TargetDrive targetDrive, IOdinC inboxItem.Sender, updateInstructionSet.EncryptedKeyHeader, odinContext); logger.LogDebug("PeerFileUpdateWriter called. Sender: {sender} FileId: {file}", inboxItem.Sender, inboxItem.FileId); - return await writer.UpsertFileAsync(tempFile, decryptedKeyHeader, inboxItem.Sender, updateInstructionSet, odinContext, - markComplete); + var drive = await driveManager.GetDriveAsync(file.DriveId); + return await writer.UpsertFileAsync(file, decryptedKeyHeader, inboxItem.Sender, updateInstructionSet, odinContext, + markComplete, sourceFolderPath: drive.GetDriveInboxPath()); } private async Task HandleReaction(TransferInboxItem inboxItem, IDriveFileSystem fs, IOdinContext odinContext, @@ -430,7 +429,7 @@ private T DecryptUsingSharedSecret(SharedSecretEncryptedTransitPayload payloa private async Task<(bool success, List payloads)> ProcessEccEncryptedFeedInboxItem(TransferInboxItem inboxItem, PeerFileWriter writer, - TempFile tempFile, + InternalDriveFileId file, IDriveFileSystem fs, IOdinContext odinContext, WriteSecondDatabaseRowBase markComplete) @@ -446,11 +445,12 @@ private T DecryptUsingSharedSecret(SharedSecretEncryptedTransitPayload payloa var feedPayload = OdinSystemSerializer.Deserialize(decryptedBytes.ToStringFromUtf8Bytes()); var decryptedKeyHeader = KeyHeader.FromCombinedBytes(feedPayload.KeyHeaderBytes); + var drive = await driveManager.GetDriveAsync(file.DriveId); var handleFileMs = await Benchmark.MillisecondsAsync(async () => { - (success, payloads) = await writer.HandleFile(tempFile, fs, decryptedKeyHeader, inboxItem.Sender, + (success, payloads) = await writer.HandleFile(file, fs, decryptedKeyHeader, inboxItem.Sender, inboxItem.TransferInstructionSet, - odinContext, feedPayload.DriveOriginWasCollaborative, markComplete); + odinContext, feedPayload.DriveOriginWasCollaborative, sourceFolderPath: drive.GetDriveInboxPath(), markComplete); }); logger.LogDebug("Processing Feed Inbox Item -> HandleFile Complete. Took {ms} ms", handleFileMs); diff --git a/tests/apps/Odin.Hosting.Tests/DefraggerTest.cs b/tests/apps/Odin.Hosting.Tests/DefraggerTest.cs index edbf660bc3..28e790ade3 100644 --- a/tests/apps/Odin.Hosting.Tests/DefraggerTest.cs +++ b/tests/apps/Odin.Hosting.Tests/DefraggerTest.cs @@ -108,7 +108,7 @@ public async Task RemoveInvalidFolderTest() var driveCount = drives.Count; - Directory.CreateDirectory(ownerClient.Configuration..TempDrivesPath); + Directory.CreateDirectory(ownerClient.Configuration..UploadDrivesPath); foreach (var drive in drives) { diff --git a/tests/apps/Odin.Hosting.Tests/OwnerApi/ApiClient/Drive/DriveApiClientRedux.cs b/tests/apps/Odin.Hosting.Tests/OwnerApi/ApiClient/Drive/DriveApiClientRedux.cs index 109087b432..ad3576bb9e 100644 --- a/tests/apps/Odin.Hosting.Tests/OwnerApi/ApiClient/Drive/DriveApiClientRedux.cs +++ b/tests/apps/Odin.Hosting.Tests/OwnerApi/ApiClient/Drive/DriveApiClientRedux.cs @@ -510,6 +510,22 @@ public async Task> GetFileHeader(Ex } } + public async Task> UploadFileExists(ExternalFileIdentifier file, string extension, + FileSystemType fileSystemType = FileSystemType.Standard) + { + var client = _ownerApi.CreateOwnerApiHttpClient(_identity, out var sharedSecret, fileSystemType); + var svc = RefitCreator.RestServiceFor(client, sharedSecret); + return await svc.UploadFileExists(file.FileId, file.TargetDrive.Alias, file.TargetDrive.Type, extension); + } + + public async Task> InboxFileExists(ExternalFileIdentifier file, string extension, + FileSystemType fileSystemType = FileSystemType.Standard) + { + var client = _ownerApi.CreateOwnerApiHttpClient(_identity, out var sharedSecret, fileSystemType); + var svc = RefitCreator.RestServiceFor(client, sharedSecret); + return await svc.InboxFileExists(file.FileId, file.TargetDrive.Alias, file.TargetDrive.Type, extension); + } + public async Task> GetPayload(ExternalFileIdentifier file, string key, FileChunk chunk = null, FileSystemType fileSystemType = FileSystemType.Standard) { diff --git a/tests/apps/Odin.Hosting.Tests/OwnerApi/ApiClient/Drive/IDriveTestHttpClientForOwner.cs b/tests/apps/Odin.Hosting.Tests/OwnerApi/ApiClient/Drive/IDriveTestHttpClientForOwner.cs index c799ff3d81..ff8ed8264c 100644 --- a/tests/apps/Odin.Hosting.Tests/OwnerApi/ApiClient/Drive/IDriveTestHttpClientForOwner.cs +++ b/tests/apps/Odin.Hosting.Tests/OwnerApi/ApiClient/Drive/IDriveTestHttpClientForOwner.cs @@ -64,6 +64,12 @@ public interface IDriveTestHttpClientForOwner [Get(RootStorageEndpoint + "/header")] Task> GetFileHeader(Guid fileId, Guid alias, Guid type); + [Get(RootStorageEndpoint + "/upload-file-exists")] + Task> UploadFileExists(Guid fileId, Guid alias, Guid type, string extension); + + [Get(RootStorageEndpoint + "/inbox-file-exists")] + Task> InboxFileExists(Guid fileId, Guid alias, Guid type, string extension); + [Post(RootQueryEndpoint + "/modified")] Task> GetModified(QueryModifiedRequest request); diff --git a/tests/apps/Odin.Hosting.Tests/_Universal/ApiClient/Drive/IUniversalDriveHttpClientApi.cs b/tests/apps/Odin.Hosting.Tests/_Universal/ApiClient/Drive/IUniversalDriveHttpClientApi.cs index 60efb9575b..3363eb3f37 100644 --- a/tests/apps/Odin.Hosting.Tests/_Universal/ApiClient/Drive/IUniversalDriveHttpClientApi.cs +++ b/tests/apps/Odin.Hosting.Tests/_Universal/ApiClient/Drive/IUniversalDriveHttpClientApi.cs @@ -76,9 +76,11 @@ public interface IUniversalDriveHttpClientApi [Get(RootStorageEndpoint + "/header")] Task> GetFileHeader(Guid fileId, Guid alias, Guid type); - [Get(RootStorageEndpoint + "/temp-file-exists")] - Task> TempFileExists(Guid fileId, Guid alias, Guid type, TempStorageType storageType, - string extension); + [Get(RootStorageEndpoint + "/upload-file-exists")] + Task> UploadFileExists(Guid fileId, Guid alias, Guid type, string extension); + + [Get(RootStorageEndpoint + "/inbox-file-exists")] + Task> InboxFileExists(Guid fileId, Guid alias, Guid type, string extension); [Get(RootStorageEndpoint + "/has-orphan-payloads")] Task> HasOrphanPayloads(Guid fileId, Guid alias, Guid type); diff --git a/tests/apps/Odin.Hosting.Tests/_Universal/ApiClient/Drive/UniversalDriveApiClient.cs b/tests/apps/Odin.Hosting.Tests/_Universal/ApiClient/Drive/UniversalDriveApiClient.cs index 07cb721f3a..9dfef8626e 100644 --- a/tests/apps/Odin.Hosting.Tests/_Universal/ApiClient/Drive/UniversalDriveApiClient.cs +++ b/tests/apps/Odin.Hosting.Tests/_Universal/ApiClient/Drive/UniversalDriveApiClient.cs @@ -799,17 +799,30 @@ public async Task> GetThumbnail(ExternalFileIdentifier } - public async Task> TempFileExists(ExternalFileIdentifier file, TempStorageType storageType, string extension, + public async Task> UploadFileExists(ExternalFileIdentifier file, string extension, FileSystemType fileSystemType = FileSystemType.Standard) { var client = factory.CreateHttpClient(identity, out var sharedSecret, fileSystemType); var svc = RefitCreator.RestServiceFor(client, sharedSecret); - return await svc.TempFileExists( + return await svc.UploadFileExists( + file.FileId, + file.TargetDrive.Alias, + file.TargetDrive.Type, + extension + ); + } + + public async Task> InboxFileExists(ExternalFileIdentifier file, string extension, + FileSystemType fileSystemType = FileSystemType.Standard) + { + var client = factory.CreateHttpClient(identity, out var sharedSecret, fileSystemType); + var svc = RefitCreator.RestServiceFor(client, sharedSecret); + + return await svc.InboxFileExists( file.FileId, file.TargetDrive.Alias, file.TargetDrive.Type, - storageType, extension ); } diff --git a/tests/apps/Odin.Hosting.Tests/_Universal/DriveTests/FileCleanup/DriveFileUploadTempFilesAreRemovedTests.cs b/tests/apps/Odin.Hosting.Tests/_Universal/DriveTests/FileCleanup/DriveFileUploadTempFilesAreRemovedTests.cs index 1e3ad9a36f..23c5258082 100644 --- a/tests/apps/Odin.Hosting.Tests/_Universal/DriveTests/FileCleanup/DriveFileUploadTempFilesAreRemovedTests.cs +++ b/tests/apps/Odin.Hosting.Tests/_Universal/DriveTests/FileCleanup/DriveFileUploadTempFilesAreRemovedTests.cs @@ -144,21 +144,21 @@ public async Task CanUploadFileWith2PayloadsAnd2ThumbnailsAndTempFilesAreDeleted foreach (var descriptor in header.FileMetadata.Payloads) { var payloadExtension = TenantPathManager.GetBasePayloadFileNameAndExtension(descriptor.Key, descriptor.Uid); - var payloadTempFileExistsResponse = await ownerApiClient.DriveRedux.TempFileExists( - uploadResult.File, TempStorageType.Upload, payloadExtension); + var payloadStagingFileExistsResponse = await ownerApiClient.DriveRedux.UploadFileExists( + uploadResult.File, payloadExtension); - ClassicAssert.IsTrue(payloadTempFileExistsResponse.IsSuccessStatusCode); - ClassicAssert.IsFalse(payloadTempFileExistsResponse.Content); + ClassicAssert.IsTrue(payloadStagingFileExistsResponse.IsSuccessStatusCode); + ClassicAssert.IsFalse(payloadStagingFileExistsResponse.Content); foreach (var thumbnail in descriptor.Thumbnails) { var thumbnailExtension = TenantPathManager.GetThumbnailFileNameAndExtension(descriptor.Key, descriptor.Uid, thumbnail.PixelWidth, thumbnail.PixelHeight); - var thumbnailTempFileExistsResponse = await ownerApiClient.DriveRedux.TempFileExists( - uploadResult.File, TempStorageType.Upload, thumbnailExtension); + var thumbnailStagingFileExistsResponse = await ownerApiClient.DriveRedux.UploadFileExists( + uploadResult.File, thumbnailExtension); - ClassicAssert.IsTrue(thumbnailTempFileExistsResponse.IsSuccessStatusCode); - ClassicAssert.IsFalse(thumbnailTempFileExistsResponse.Content); + ClassicAssert.IsTrue(thumbnailStagingFileExistsResponse.IsSuccessStatusCode); + ClassicAssert.IsFalse(thumbnailStagingFileExistsResponse.Content); } } } @@ -280,22 +280,22 @@ public async Task CanUpdateFilePayloadsAndThumbnailsAndOrphansAreDeleted(IApiCli foreach (var descriptor in getUpdatedHeaderResponse.Content!.FileMetadata.Payloads) { var payloadExtension = TenantPathManager.GetBasePayloadFileNameAndExtension(descriptor.Key, descriptor.Uid); - var payloadTempFileExistsResponse = await ownerApiClient.DriveRedux.TempFileExists( - uploadResult.File, TempStorageType.Upload, payloadExtension); + var payloadStagingFileExistsResponse = await ownerApiClient.DriveRedux.UploadFileExists( + uploadResult.File, payloadExtension); - ClassicAssert.IsTrue(payloadTempFileExistsResponse.IsSuccessStatusCode); - ClassicAssert.IsFalse(payloadTempFileExistsResponse.Content); + ClassicAssert.IsTrue(payloadStagingFileExistsResponse.IsSuccessStatusCode); + ClassicAssert.IsFalse(payloadStagingFileExistsResponse.Content); foreach (var thumbnail in descriptor.Thumbnails) { - var thumbnailExtension = TenantPathManager.GetThumbnailFileNameAndExtension(descriptor.Key, + var thumbnailExtension = TenantPathManager.GetThumbnailFileNameAndExtension(descriptor.Key, descriptor.Uid, thumbnail.PixelWidth, thumbnail.PixelHeight); - - var thumbnailTempFileExistsResponse = await ownerApiClient.DriveRedux.TempFileExists( - uploadResult.File, TempStorageType.Upload, thumbnailExtension); - ClassicAssert.IsTrue(thumbnailTempFileExistsResponse.IsSuccessStatusCode); - ClassicAssert.IsFalse(thumbnailTempFileExistsResponse.Content); + var thumbnailStagingFileExistsResponse = await ownerApiClient.DriveRedux.UploadFileExists( + uploadResult.File, thumbnailExtension); + + ClassicAssert.IsTrue(thumbnailStagingFileExistsResponse.IsSuccessStatusCode); + ClassicAssert.IsFalse(thumbnailStagingFileExistsResponse.Content); } } diff --git a/tests/services/Odin.Services.Tests/BackgroundServices/Services/Tenant/TempFolderCleanUpBackgroundServiceTests.cs b/tests/services/Odin.Services.Tests/BackgroundServices/Services/Tenant/TempFolderCleanUpBackgroundServiceTests.cs index 5fcaa912e6..53b64b83eb 100644 --- a/tests/services/Odin.Services.Tests/BackgroundServices/Services/Tenant/TempFolderCleanUpBackgroundServiceTests.cs +++ b/tests/services/Odin.Services.Tests/BackgroundServices/Services/Tenant/TempFolderCleanUpBackgroundServiceTests.cs @@ -4,7 +4,6 @@ using Microsoft.Extensions.Logging; using Moq; using NUnit.Framework; -using Odin.Core.Exceptions; using Odin.Services.Background.BackgroundServices.Tenant; namespace Odin.Services.Tests.BackgroundServices.Services.Tenant; @@ -12,7 +11,7 @@ namespace Odin.Services.Tests.BackgroundServices.Services.Tenant; [TestFixture] public class TempFolderCleanUpBackgroundServiceTests { - private string _testTempRoot = ""; + private string _testDrivesRoot = ""; private Mock _loggerMock = new (); private TimeSpan _uploadAgeThreshold; private TimeSpan _inboxAgeThreshold; @@ -20,11 +19,10 @@ public class TempFolderCleanUpBackgroundServiceTests [SetUp] public void Setup() { - // Create a unique temp folder for testing - _testTempRoot = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); - Directory.CreateDirectory(_testTempRoot); + // Create a unique drives folder for testing (mirrors UploadDrivesPath / InboxDrivesPath) + _testDrivesRoot = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString(), "drives"); + Directory.CreateDirectory(_testDrivesRoot); - // Define thresholds for testing _uploadAgeThreshold = TimeSpan.FromHours(1); _inboxAgeThreshold = TimeSpan.FromHours(2); } @@ -32,85 +30,77 @@ public void Setup() [TearDown] public void TearDown() { - // Clean up test directory after test - if (Directory.Exists(_testTempRoot)) + var root = Path.GetDirectoryName(_testDrivesRoot)!; + if (Directory.Exists(root)) { try { - Directory.Delete(_testTempRoot, true); + Directory.Delete(root, true); } catch (IOException) { - // If we can't delete it immediately, schedule it for deletion on process exit - Thread.Sleep(100); // Give system a moment to release file handles - try - { - Directory.Delete(_testTempRoot, true); - } - catch - { - Console.WriteLine($"Warning: Could not delete temp directory {_testTempRoot}"); - } + Thread.Sleep(100); + try { Directory.Delete(root, true); } + catch { Console.WriteLine($"Warning: Could not delete temp directory {root}"); } } } } [Test] - public void Execute_DeletesOldFiles_PreservesRecentFiles() + public void UploadFolderCleanUp_DeletesOldFiles_PreservesRecentFiles() { // Arrange SetupTestFolderStructure(); - string drive1UploadsPath = Path.Combine(_testTempRoot, "drives", "drive1", "uploads"); - string drive1InboxPath = Path.Combine(_testTempRoot, "drives", "drive1", "inbox"); + string drive1UploadsPath = Path.Combine(_testDrivesRoot, "drive1", "uploads"); - // Create old files that should be deleted - string oldUploadFile = Path.Combine(drive1UploadsPath, "old_upload.txt"); - string oldInboxFile = Path.Combine(drive1InboxPath, "old_inbox.txt"); + string oldFile = Path.Combine(drive1UploadsPath, "old_upload.txt"); + string newFile = Path.Combine(drive1UploadsPath, "new_upload.txt"); - // Create new files that should be preserved - string newUploadFile = Path.Combine(drive1UploadsPath, "new_upload.txt"); - string newInboxFile = Path.Combine(drive1InboxPath, "new_inbox.txt"); - - CreateFileWithTimestamp(oldUploadFile, DateTime.UtcNow.Subtract(_uploadAgeThreshold).Subtract(TimeSpan.FromMinutes(30))); - CreateFileWithTimestamp(oldInboxFile, DateTime.UtcNow.Subtract(_inboxAgeThreshold).Subtract(TimeSpan.FromMinutes(30))); - CreateFileWithTimestamp(newUploadFile, DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(10))); - CreateFileWithTimestamp(newInboxFile, DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(10))); + CreateFileWithTimestamp(oldFile, DateTime.UtcNow.Subtract(_uploadAgeThreshold).Subtract(TimeSpan.FromMinutes(30))); + CreateFileWithTimestamp(newFile, DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(10))); // Act - TempFolderCleanUp.Execute( - _loggerMock.Object, - _testTempRoot, - _uploadAgeThreshold, - _inboxAgeThreshold - ); + UploadFolderCleanUp.Execute(_loggerMock.Object, _testDrivesRoot, _uploadAgeThreshold); // Assert - Assert.That(File.Exists(oldUploadFile), Is.False, "Old upload file should be deleted"); - Assert.That(File.Exists(oldInboxFile), Is.False, "Old inbox file should be deleted"); - Assert.That(File.Exists(newUploadFile), Is.True,"Recent upload file should be preserved"); - Assert.That(File.Exists(newInboxFile), Is.True,"Recent inbox file should be preserved"); + Assert.That(File.Exists(oldFile), Is.False, "Old upload file should be deleted"); + Assert.That(File.Exists(newFile), Is.True, "Recent upload file should be preserved"); } [Test] - public void Execute_HandlesIllegalSubdirectories() + public void InboxFolderCleanUp_DeletesOldFiles_PreservesRecentFiles() { // Arrange SetupTestFolderStructure(); - string drive1UploadsPath = Path.Combine(_testTempRoot, "drives", "drive1", "uploads"); + string drive1InboxPath = Path.Combine(_testDrivesRoot, "drive1", "inbox"); + + string oldFile = Path.Combine(drive1InboxPath, "old_inbox.txt"); + string newFile = Path.Combine(drive1InboxPath, "new_inbox.txt"); - // Create illegal subdirectory - string illegalSubdir = Path.Combine(drive1UploadsPath, "illegal_subdir"); - Directory.CreateDirectory(illegalSubdir); + CreateFileWithTimestamp(oldFile, DateTime.UtcNow.Subtract(_inboxAgeThreshold).Subtract(TimeSpan.FromMinutes(30))); + CreateFileWithTimestamp(newFile, DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(10))); // Act - TempFolderCleanUp.Execute( - _loggerMock.Object, - _testTempRoot, - _uploadAgeThreshold, - _inboxAgeThreshold - ); + InboxFolderCleanUp.Execute(_loggerMock.Object, _testDrivesRoot, _inboxAgeThreshold); + + // Assert + Assert.That(File.Exists(oldFile), Is.False, "Old inbox file should be deleted"); + Assert.That(File.Exists(newFile), Is.True, "Recent inbox file should be preserved"); + } + + [Test] + public void UploadFolderCleanUp_HandlesIllegalSubdirectories() + { + // Arrange + SetupTestFolderStructure(); + + string drive1UploadsPath = Path.Combine(_testDrivesRoot, "drive1", "uploads"); + Directory.CreateDirectory(Path.Combine(drive1UploadsPath, "illegal_subdir")); + + // Act + UploadFolderCleanUp.Execute(_loggerMock.Object, _testDrivesRoot, _uploadAgeThreshold); // Assert _loggerMock.Verify( @@ -126,85 +116,54 @@ public void Execute_HandlesIllegalSubdirectories() } [Test] - public void Execute_RespectsStoppingToken() + public void UploadFolderCleanUp_RespectsStoppingToken() { // Arrange SetupTestFolderStructure(); - string drive1UploadsPath = Path.Combine(_testTempRoot, "drives", "drive1", "uploads"); - string oldUploadFile = Path.Combine(drive1UploadsPath, "old_upload.txt"); - CreateFileWithTimestamp(oldUploadFile, DateTime.UtcNow.Subtract(_uploadAgeThreshold).Subtract(TimeSpan.FromMinutes(30))); + string drive1UploadsPath = Path.Combine(_testDrivesRoot, "drive1", "uploads"); + string oldFile = Path.Combine(drive1UploadsPath, "old_upload.txt"); + CreateFileWithTimestamp(oldFile, DateTime.UtcNow.Subtract(_uploadAgeThreshold).Subtract(TimeSpan.FromMinutes(30))); using var cts = new CancellationTokenSource(); - cts.Cancel(); // Cancel token immediately + cts.Cancel(); // Act - TempFolderCleanUp.Execute( - _loggerMock.Object, - _testTempRoot, - _uploadAgeThreshold, - _inboxAgeThreshold, - cts.Token - ); + UploadFolderCleanUp.Execute(_loggerMock.Object, _testDrivesRoot, _uploadAgeThreshold, cts.Token); // Assert - Assert.That(File.Exists(oldUploadFile), Is.True, "File should not be deleted when token is cancelled"); + Assert.That(File.Exists(oldFile), Is.True, "File should not be deleted when token is cancelled"); } [Test] - public void Execute_ThrowsException_WhenTempFolderDoesNotExist() + public void UploadFolderCleanUp_ReturnsSilently_WhenDrivesPathDoesNotExist() { // Arrange - string nonExistentFolder = Path.Combine(_testTempRoot, "non_existent_folder"); + var nonExistentPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); - // Act & Assert - var ex = Assert.Throws(() => TempFolderCleanUp.Execute( - _loggerMock.Object, - nonExistentFolder, - _uploadAgeThreshold, - _inboxAgeThreshold - )); - - Assert.That(ex?.Message, Does.Contain("does not exist")); + // Act & Assert — should not throw + Assert.DoesNotThrow(() => + UploadFolderCleanUp.Execute(_loggerMock.Object, nonExistentPath, _uploadAgeThreshold)); } [Test] - public void Execute_ThrowsException_WhenThresholdsAreNegative() + public void InboxFolderCleanUp_ReturnsSilently_WhenDrivesPathDoesNotExist() { - // Act & Assert - Negative upload threshold - var ex1 = Assert.Throws(() => TempFolderCleanUp.Execute( - _loggerMock.Object, - _testTempRoot, - TimeSpan.FromSeconds(-1), - _inboxAgeThreshold - )); - - Assert.That(ex1?.Message, Does.Contain("Upload age threshold")); - - // Act & Assert - Negative inbox threshold - var ex2 = Assert.Throws(() => TempFolderCleanUp.Execute( - _loggerMock.Object, - _testTempRoot, - _uploadAgeThreshold, - TimeSpan.FromSeconds(-1) - )); - - Assert.That(ex2?.Message, Does.Contain("Inbox age threshold")); + // Arrange + var nonExistentPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + + // Act & Assert — should not throw + Assert.DoesNotThrow(() => + InboxFolderCleanUp.Execute(_loggerMock.Object, nonExistentPath, _inboxAgeThreshold)); } private void SetupTestFolderStructure() { - // Create the base folders structure - string drivesFolder = Path.Combine(_testTempRoot, "drives"); - Directory.CreateDirectory(drivesFolder); - - // Create two drive folders - string drive1 = Path.Combine(drivesFolder, "drive1"); - string drive2 = Path.Combine(drivesFolder, "drive2"); + string drive1 = Path.Combine(_testDrivesRoot, "drive1"); + string drive2 = Path.Combine(_testDrivesRoot, "drive2"); Directory.CreateDirectory(drive1); Directory.CreateDirectory(drive2); - // Create uploads and inbox folders for each drive Directory.CreateDirectory(Path.Combine(drive1, "uploads")); Directory.CreateDirectory(Path.Combine(drive1, "inbox")); Directory.CreateDirectory(Path.Combine(drive2, "uploads")); @@ -218,4 +177,3 @@ private void CreateFileWithTimestamp(string filePath, DateTime timestamp) File.SetLastWriteTimeUtc(filePath, timestamp); } } - diff --git a/tests/services/Odin.Services.Tests/Drives/FileSystem/Base/TenantPathManagerTest.cs b/tests/services/Odin.Services.Tests/Drives/FileSystem/Base/TenantPathManagerTest.cs index 064c503056..3e0f44eef0 100644 --- a/tests/services/Odin.Services.Tests/Drives/FileSystem/Base/TenantPathManagerTest.cs +++ b/tests/services/Odin.Services.Tests/Drives/FileSystem/Base/TenantPathManagerTest.cs @@ -75,13 +75,26 @@ public void AllBasePathsShouldBeCorrect() tenantId.ToString(), "headers"))); - Assert.That(tenantPathManager.TempPath, Is.EqualTo(Path.Combine( + Assert.That(tenantPathManager.UploadPath, Is.EqualTo(Path.Combine( _config.Host.TenantDataRootPath, "registrations", tenantId.ToString(), "temp"))); - Assert.That(tenantPathManager.TempDrivesPath, Is.EqualTo(Path.Combine( + Assert.That(tenantPathManager.UploadDrivesPath, Is.EqualTo(Path.Combine( + _config.Host.TenantDataRootPath, + "registrations", + tenantId.ToString(), + "temp", + "drives"))); + + Assert.That(tenantPathManager.InboxPath, Is.EqualTo(Path.Combine( + _config.Host.TenantDataRootPath, + "registrations", + tenantId.ToString(), + "temp"))); + + Assert.That(tenantPathManager.InboxDrivesPath, Is.EqualTo(Path.Combine( _config.Host.TenantDataRootPath, "registrations", tenantId.ToString(),