diff --git a/mobile/apps/photos/lib/core/errors.dart b/mobile/apps/photos/lib/core/errors.dart index d6165065cc6..c9d3f09bec4 100644 --- a/mobile/apps/photos/lib/core/errors.dart +++ b/mobile/apps/photos/lib/core/errors.dart @@ -28,6 +28,8 @@ class InvalidFileError extends ArgumentError { } } +class SkippedQueuedFileError extends Error {} + class SubscriptionAlreadyClaimedError extends Error {} class WiFiUnavailableError extends Error {} diff --git a/mobile/apps/photos/lib/db/device_files_db.dart b/mobile/apps/photos/lib/db/device_files_db.dart index 0e56b1a73b3..c0fc4a6654d 100644 --- a/mobile/apps/photos/lib/db/device_files_db.dart +++ b/mobile/apps/photos/lib/db/device_files_db.dart @@ -279,6 +279,20 @@ extension DeviceFiles on FilesDB { return result; } + Future> getSelectedDevicePathIds() async { + final db = await sqliteAsyncDB; + final rows = await db.getAll( + ''' + SELECT id FROM device_collections where should_backup = $_sqlBoolTrue; + ''', + ); + final Set result = {}; + for (final row in rows) { + result.add(row['id'] as String); + } + return result; + } + Future updateDevicePathSyncStatus( Map syncStatus, ) async { diff --git a/mobile/apps/photos/lib/db/files_db.dart b/mobile/apps/photos/lib/db/files_db.dart index 2790418a647..0d99c9c8f64 100644 --- a/mobile/apps/photos/lib/db/files_db.dart +++ b/mobile/apps/photos/lib/db/files_db.dart @@ -60,6 +60,7 @@ class FilesDB with SqlDbBase { static const columnThumbnailDecryptionHeader = 'thumbnail_decryption_header'; static const columnMetadataDecryptionHeader = 'metadata_decryption_header'; static const columnFileSize = 'file_size'; + static const columnQueueSource = 'queue_source'; // MMD -> Magic Metadata static const columnMMdEncodedJson = 'mmd_encoded_json'; @@ -90,6 +91,7 @@ class FilesDB with SqlDbBase { ...updateIndexes(), ...createEntityDataTable(), ...addAddedTime(), + ...addQueueSourceColumn(), ]; static const List _columnNames = [ @@ -98,6 +100,7 @@ class FilesDB with SqlDbBase { columnUploadedFileID, columnOwnerID, columnCollectionID, + columnQueueSource, columnTitle, columnDeviceFolder, columnLatitude, @@ -416,6 +419,14 @@ class FilesDB with SqlDbBase { ]; } + static List addQueueSourceColumn() { + return [ + ''' + ALTER TABLE $filesTable ADD COLUMN $columnQueueSource TEXT; + ''', + ]; + } + Future clearTable() async { final db = await instance.sqliteAsyncDB; await db.execute('DELETE FROM $filesTable'); @@ -1073,20 +1084,75 @@ class FilesDB with SqlDbBase { // corresponding file entries are not already mapped to some other collection Future setCollectionIDForUnMappedLocalFiles( int collectionID, - Set localIDs, + Set localIDs, { + String? queueSource, + } ) async { + if (localIDs.isEmpty) { + return; + } final db = await instance.sqliteAsyncDB; - final inParam = localIDs.map((id) => "'$id'").join(','); + final localIDList = localIDs.toList(); + final placeholders = List.filled(localIDList.length, '?').join(','); + if (queueSource == null) { + await db.execute( + ''' + UPDATE $filesTable + SET $columnCollectionID = $collectionID + WHERE $columnLocalID IN ($placeholders) AND ($columnCollectionID IS NULL OR + $columnCollectionID = -1); + ''', + localIDList, + ); + return; + } await db.execute( ''' UPDATE $filesTable - SET $columnCollectionID = $collectionID - WHERE $columnLocalID IN ($inParam) AND ($columnCollectionID IS NULL OR + SET $columnCollectionID = ?, + $columnQueueSource = ? + WHERE $columnLocalID IN ($placeholders) AND ($columnCollectionID IS NULL OR $columnCollectionID = -1); ''', + [collectionID, queueSource, ...localIDList], ); } + Future cleanupQueuedEntry({ + required String localID, + required int collectionID, + required String queueSource, + }) async { + final db = await instance.sqliteAsyncDB; + await db.writeTransaction((tx) async { + final hasUnmapped = await tx.getAll( + 'SELECT 1 FROM $filesTable WHERE $columnLocalID = ? ' + 'AND ($columnCollectionID IS NULL OR $columnCollectionID = -1) ' + 'AND ($columnUploadedFileID IS NULL OR $columnUploadedFileID = -1) ' + 'LIMIT 1', + [localID], + ); + + if (hasUnmapped.isNotEmpty) { + await tx.execute( + 'DELETE FROM $filesTable WHERE $columnLocalID = ? ' + 'AND $columnCollectionID = ? AND $columnQueueSource = ? ' + 'AND ($columnUploadedFileID IS NULL OR $columnUploadedFileID = -1)', + [localID, collectionID, queueSource], + ); + } else { + await tx.execute( + 'UPDATE $filesTable SET $columnCollectionID = -1, ' + '$columnQueueSource = NULL ' + 'WHERE $columnLocalID = ? AND $columnCollectionID = ? ' + 'AND $columnQueueSource = ? ' + 'AND ($columnUploadedFileID IS NULL OR $columnUploadedFileID = -1)', + [localID, collectionID, queueSource], + ); + } + }); + } + Future markFilesForReUpload( int ownerID, String localID, @@ -1877,6 +1943,7 @@ class FilesDB with SqlDbBase { file.uploadedFileID ?? -1, file.ownerID, file.collectionID ?? -1, + file.queueSource, file.title, file.deviceFolder, latitude, @@ -1960,6 +2027,7 @@ class FilesDB with SqlDbBase { file.hash = row[columnHash]; file.metadataVersion = row[columnMetadataVersion] ?? 0; file.fileSize = row[columnFileSize]; + file.queueSource = row[columnQueueSource]; file.mMdVersion = row[columnMMdVersion] ?? 0; file.mMdEncodedJson = row[columnMMdEncodedJson] ?? '{}'; diff --git a/mobile/apps/photos/lib/models/file/file.dart b/mobile/apps/photos/lib/models/file/file.dart index 2ac284927ea..7499b3796c4 100644 --- a/mobile/apps/photos/lib/models/file/file.dart +++ b/mobile/apps/photos/lib/models/file/file.dart @@ -41,6 +41,7 @@ class EnteFile { String? thumbnailDecryptionHeader; String? metadataDecryptionHeader; int? fileSize; + String? queueSource; String? mMdEncodedJson; int mMdVersion = 0; @@ -387,6 +388,7 @@ class EnteFile { String? thumbnailDecryptionHeader, String? metadataDecryptionHeader, int? fileSize, + String? queueSource, String? mMdEncodedJson, int? mMdVersion, MagicMetadata? magicMetadata, @@ -421,6 +423,7 @@ class EnteFile { ..metadataDecryptionHeader = metadataDecryptionHeader ?? this.metadataDecryptionHeader ..fileSize = fileSize ?? this.fileSize + ..queueSource = queueSource ?? this.queueSource ..mMdEncodedJson = mMdEncodedJson ?? this.mMdEncodedJson ..mMdVersion = mMdVersion ?? this.mMdVersion ..magicMetadata = magicMetadata ?? this.magicMetadata diff --git a/mobile/apps/photos/lib/services/sync/remote_sync_service.dart b/mobile/apps/photos/lib/services/sync/remote_sync_service.dart index 351ab78bf92..2cc4ad39fc3 100644 --- a/mobile/apps/photos/lib/services/sync/remote_sync_service.dart +++ b/mobile/apps/photos/lib/services/sync/remote_sync_service.dart @@ -17,6 +17,8 @@ import 'package:photos/events/files_updated_event.dart'; import 'package:photos/events/force_reload_home_gallery_event.dart'; import 'package:photos/events/local_photos_updated_event.dart'; import 'package:photos/events/sync_status_update_event.dart'; +import 'package:photos/events/account_configured_event.dart'; +import 'package:photos/events/user_logged_out_event.dart'; import "package:photos/main.dart" show isProcessBg; import 'package:photos/models/device_collection.dart'; import "package:photos/models/file/extensions/file_props.dart"; @@ -48,6 +50,7 @@ class RemoteSyncService { LocalFileUpdateService.instance; int _completedUploads = 0; int _ignoredUploads = 0; + Set? _selectedDevicePathIdsCache; late SharedPreferences _prefs; Completer? _existingSync; bool _isExistingSyncSilent = false; @@ -93,6 +96,16 @@ class RemoteSyncService { } } }); + + if (flagService.queueSourceEnabled) { + Bus.instance.on().listen((event) { + _clearSelectedPathIdsCache(); + }); + + Bus.instance.on().listen((event) { + _clearSelectedPathIdsCache(); + }); + } } Future sync({bool silently = false}) async { @@ -359,6 +372,10 @@ class RemoteSyncService { _logger.info("Syncing device collections to be uploaded"); final int ownerID = _config.getUserID()!; + if (flagService.queueSourceEnabled) { + await _ensureSelectedPathIdsCache(); + } + final deviceCollections = await _db.getDeviceCollections(); deviceCollections.removeWhere((element) => !element.shouldBackup); // Sort by count to ensure that photos in iOS are first inserted in @@ -415,6 +432,8 @@ class RemoteSyncService { await _db.setCollectionIDForUnMappedLocalFiles( collectionID, localIDsToSync, + queueSource: + flagService.queueSourceEnabled ? deviceCollection.id : null, ); // mark IDs as already synced if corresponding entry is present in @@ -451,6 +470,9 @@ class RemoteSyncService { existingFile.collectionID = collectionID; existingFile.uploadedFileID = null; existingFile.ownerID = null; + if (flagService.queueSourceEnabled) { + existingFile.queueSource = deviceCollection.id; + } newFilesToInsert.add(existingFile); fileFoundForLocalIDs.add(localID); } @@ -478,6 +500,9 @@ class RemoteSyncService { final Set oldCollectionIDsForAutoSync = await _db.getDeviceSyncCollectionIDs(); await _db.updateDevicePathSyncStatus(syncStatusUpdate); + if (flagService.queueSourceEnabled) { + await _refreshSelectedPathIdsCacheIfChanged(syncStatusUpdate); + } final Set newCollectionIDsForAutoSync = await _db.getDeviceSyncCollectionIDs(); SyncService.instance.onDeviceCollectionSet(newCollectionIDsForAutoSync); @@ -493,6 +518,45 @@ class RemoteSyncService { Bus.instance.fire(BackupFoldersUpdatedEvent()); } + Future ensureSelectedPathIdsCache() async { + await _ensureSelectedPathIdsCache(); + } + + Future _ensureSelectedPathIdsCache() async { + if (_selectedDevicePathIdsCache != null) { + return; + } + _selectedDevicePathIdsCache = await _db.getSelectedDevicePathIds(); + } + + void _clearSelectedPathIdsCache() { + _selectedDevicePathIdsCache = null; + } + + Future _refreshSelectedPathIdsCacheIfChanged( + Map syncStatusUpdate, + ) async { + if (syncStatusUpdate.isEmpty) { + return; + } + await _ensureSelectedPathIdsCache(); + final cache = _selectedDevicePathIdsCache!; + final hasChange = syncStatusUpdate.entries.any( + (entry) => cache.contains(entry.key) != entry.value, + ); + if (!hasChange) { + return; + } + _selectedDevicePathIdsCache = await _db.getSelectedDevicePathIds(); + } + + bool isDevicePathSelected(String pathId) { + if (!flagService.queueSourceEnabled) { + return true; + } + return _selectedDevicePathIdsCache?.contains(pathId) ?? true; + } + Future removeFilesQueuedForUpload(List collectionIDs) async { /* For each collection, perform following action @@ -757,6 +821,9 @@ class RemoteSyncService { if (error is InvalidFileError) { _ignoredUploads++; _logger.warning("Invalid file error", error); + } else if (error is SkippedQueuedFileError) { + _ignoredUploads++; + _logger.info("Skipped queued file due to queue source"); } else { throw error; } diff --git a/mobile/apps/photos/lib/ui/actions/collection/collection_file_actions.dart b/mobile/apps/photos/lib/ui/actions/collection/collection_file_actions.dart index bc32c07600d..ee53cbc3670 100644 --- a/mobile/apps/photos/lib/ui/actions/collection/collection_file_actions.dart +++ b/mobile/apps/photos/lib/ui/actions/collection/collection_file_actions.dart @@ -10,6 +10,7 @@ import "package:photos/generated/l10n.dart"; import 'package:photos/models/collection/collection.dart'; import 'package:photos/models/file/file.dart'; import 'package:photos/models/selected_files.dart'; +import "package:photos/service_locator.dart"; import "package:photos/services/collections_service.dart"; import 'package:photos/services/favorites_service.dart'; import "package:photos/services/hidden_service.dart"; @@ -144,8 +145,12 @@ extension CollectionFileActions on CollectionActions { files.add(uploadedFile); } } else { + final bool queueSourceEnabled = flagService.queueSourceEnabled; for (final file in filesPendingUpload) { file.collectionID = collection.id; + if (queueSourceEnabled) { + file.queueSource = 'manual'; + } } // filesPendingUpload might be getting ignored during auto-upload // because the user deleted these files from ente in the past. @@ -266,8 +271,12 @@ extension CollectionFileActions on CollectionActions { files.add(uploadedFile); } } else { + final bool queueSourceEnabled = flagService.queueSourceEnabled; for (final file in filesPendingUpload) { file.collectionID = collectionID; + if (queueSourceEnabled) { + file.queueSource = 'manual'; + } } // filesPendingUpload might be getting ignored during auto-upload // because the user deleted these files from ente in the past. diff --git a/mobile/apps/photos/lib/utils/file_uploader.dart b/mobile/apps/photos/lib/utils/file_uploader.dart index 7b9e3afb6ff..c372c2f364b 100644 --- a/mobile/apps/photos/lib/utils/file_uploader.dart +++ b/mobile/apps/photos/lib/utils/file_uploader.dart @@ -37,6 +37,7 @@ import "package:photos/service_locator.dart"; import "package:photos/services/account/user_service.dart"; import 'package:photos/services/collections_service.dart'; import 'package:photos/services/sync/local_sync_service.dart'; +import 'package:photos/services/sync/remote_sync_service.dart'; import 'package:photos/services/sync/sync_service.dart'; import "package:photos/utils/exif_util.dart"; import "package:photos/utils/file_key.dart"; @@ -141,6 +142,9 @@ class FileUploader { UploadLocksDB.instance, flagService, ); + if (flagService.queueSourceEnabled) { + await RemoteSyncService.instance.ensureSelectedPathIdsCache(); + } if (currentTime - (_prefs.getInt(_lastStaleFileCleanupTime) ?? 0) > tempDirCleanUpInterval) { await removeStaleFiles(); @@ -298,6 +302,12 @@ class FileUploader { ?.value; } if (pendingEntry != null) { + if (flagService.queueSourceEnabled && + _shouldSkipQueuedItem(pendingEntry)) { + _handleSkippedQueuedItem(pendingEntry); + _pollQueue(); + return; + } pendingEntry.status = UploadStatus.inProgress; _allBackups[pendingEntry.file.localID!] = _allBackups[pendingEntry.file.localID]! @@ -311,6 +321,44 @@ class FileUploader { } } + bool _shouldSkipQueuedItem(FileUploadItem item) { + final file = item.file; + final qs = file.queueSource; + if (qs == null || qs == 'manual') { + return false; + } + + final bool isSelected = RemoteSyncService.instance.isDevicePathSelected(qs); + if (!isSelected) { + return true; + } + + final int? onlyNewSince = backupPreferenceService.onlyNewSinceEpoch; + return onlyNewSince != null && (file.creationTime ?? 0) < onlyNewSince; + } + + void _handleSkippedQueuedItem(FileUploadItem item) { + final file = item.file; + final qs = file.queueSource; + if (file.localID == null || qs == null) { + return; + } + unawaited( + FilesDB.instance.cleanupQueuedEntry( + localID: file.localID!, + collectionID: item.collectionID, + queueSource: qs, + ), + ); + _queue.remove(file.localID!); + _allBackups.remove(file.localID!); + if (_totalCountInUploadSession > 0) { + _totalCountInUploadSession--; + } + item.completer.completeError(SkippedQueuedFileError()); + Bus.instance.fire(BackupUpdatedEvent(_allBackups)); + } + Future _encryptAndUploadFileToCollection( EnteFile file, int collectionID, { diff --git a/mobile/apps/photos/plugins/ente_feature_flag/lib/src/service.dart b/mobile/apps/photos/plugins/ente_feature_flag/lib/src/service.dart index b04e9287c4c..e3d173b28fb 100644 --- a/mobile/apps/photos/plugins/ente_feature_flag/lib/src/service.dart +++ b/mobile/apps/photos/plugins/ente_feature_flag/lib/src/service.dart @@ -69,6 +69,8 @@ class FlagService { bool get enableUploadV2 => ((flags.serverApiFlag & _uploadV2Flag) != 0); + bool get queueSourceEnabled => internalUser; + bool get enableVectorDb => hasGrantedMLConsent; String get castUrl => flags.castUrl;