[mob][photos] Fix backups out of sync

mobile/apps/photos/lib/core/errors.dart

@@ -28,6 +28,8 @@ class InvalidFileError extends ArgumentError {
   }
 }
 
+class SkippedQueuedFileError extends Error {}
+
 class SubscriptionAlreadyClaimedError extends Error {}
 
 class WiFiUnavailableError extends Error {}

mobile/apps/photos/lib/db/device_files_db.dart

@@ -279,6 +279,20 @@ extension DeviceFiles on FilesDB {
     return result;
   }
 
+  Future<Set<String>> getSelectedDevicePathIds() async {
+    final db = await sqliteAsyncDB;
+    final rows = await db.getAll(
+      '''
+      SELECT id FROM device_collections where should_backup = $_sqlBoolTrue;
+      ''',
+    );
+    final Set<String> result = <String>{};
+    for (final row in rows) {
+      result.add(row['id'] as String);
+    }
+    return result;
+  }
+
   Future<void> updateDevicePathSyncStatus(
     Map<String, bool> syncStatus,
   ) async {

mobile/apps/photos/lib/db/files_db.dart

@@ -60,6 +60,7 @@ class FilesDB with SqlDbBase {
   static const columnThumbnailDecryptionHeader = 'thumbnail_decryption_header';
   static const columnMetadataDecryptionHeader = 'metadata_decryption_header';
   static const columnFileSize = 'file_size';
+  static const columnQueueSource = 'queue_source';
 
   // MMD -> Magic Metadata
   static const columnMMdEncodedJson = 'mmd_encoded_json';
@@ -90,6 +91,7 @@ class FilesDB with SqlDbBase {
     ...updateIndexes(),
     ...createEntityDataTable(),
     ...addAddedTime(),
+    ...addQueueSourceColumn(),
   ];
 
   static const List<String> _columnNames = [
@@ -98,6 +100,7 @@ class FilesDB with SqlDbBase {
     columnUploadedFileID,
     columnOwnerID,
     columnCollectionID,
+    columnQueueSource,
     columnTitle,
     columnDeviceFolder,
     columnLatitude,
@@ -416,6 +419,14 @@ class FilesDB with SqlDbBase {
     ];
   }
 
+  static List<String> addQueueSourceColumn() {
+    return [
+      '''
+        ALTER TABLE $filesTable ADD COLUMN $columnQueueSource TEXT;
+      ''',
+    ];
+  }
+
   Future<void> clearTable() async {
     final db = await instance.sqliteAsyncDB;
     await db.execute('DELETE FROM $filesTable');
@@ -1073,20 +1084,75 @@ class FilesDB with SqlDbBase {
   // corresponding file entries are not already mapped to some other collection
   Future<void> setCollectionIDForUnMappedLocalFiles(
     int collectionID,
-    Set<String> localIDs,
+    Set<String> localIDs, {
+    String? queueSource,
+  }
   ) async {
+    if (localIDs.isEmpty) {
+      return;
+    }
     final db = await instance.sqliteAsyncDB;
-    final inParam = localIDs.map((id) => "'$id'").join(',');
+    final localIDList = localIDs.toList();
+    final placeholders = List.filled(localIDList.length, '?').join(',');
+    if (queueSource == null) {
+      await db.execute(
+        '''
+        UPDATE $filesTable
+        SET $columnCollectionID = $collectionID
+        WHERE $columnLocalID IN ($placeholders) AND ($columnCollectionID IS NULL OR 
+        $columnCollectionID = -1);
+      ''',
+        localIDList,
+      );
+      return;
+    }
     await db.execute(
       '''
       UPDATE $filesTable
-      SET $columnCollectionID = $collectionID
-      WHERE $columnLocalID IN ($inParam) AND ($columnCollectionID IS NULL OR 
+      SET $columnCollectionID = ?,
+          $columnQueueSource = ?
+      WHERE $columnLocalID IN ($placeholders) AND ($columnCollectionID IS NULL OR 
       $columnCollectionID = -1);
     ''',
+      [collectionID, queueSource, ...localIDList],
     );
   }
 
+  Future<void> cleanupQueuedEntry({
+    required String localID,
+    required int collectionID,
+    required String queueSource,
+  }) async {
+    final db = await instance.sqliteAsyncDB;
+    await db.writeTransaction((tx) async {
+      final hasUnmapped = await tx.get(
+        'SELECT 1 FROM $filesTable WHERE $columnLocalID = ? '
+        'AND ($columnCollectionID IS NULL OR $columnCollectionID = -1) '
+        'AND ($columnUploadedFileID IS NULL OR $columnUploadedFileID = -1) '
+        'LIMIT 1',
+        [localID],
+      );
+
+      if (hasUnmapped != null) {
+        await tx.execute(
+          'DELETE FROM $filesTable WHERE $columnLocalID = ? '
+          'AND $columnCollectionID = ? AND $columnQueueSource = ? '
+          'AND ($columnUploadedFileID IS NULL OR $columnUploadedFileID = -1)',
+          [localID, collectionID, queueSource],
+        );
+      } else {
+        await tx.execute(
+          'UPDATE $filesTable SET $columnCollectionID = -1, '
+          '$columnQueueSource = NULL '
+          'WHERE $columnLocalID = ? AND $columnCollectionID = ? '
+          'AND $columnQueueSource = ? '
+          'AND ($columnUploadedFileID IS NULL OR $columnUploadedFileID = -1)',
+          [localID, collectionID, queueSource],
+        );
+      }
+    });
+  }
+
   Future<void> markFilesForReUpload(
     int ownerID,
     String localID,
@@ -1877,6 +1943,7 @@ class FilesDB with SqlDbBase {
       file.uploadedFileID ?? -1,
       file.ownerID,
       file.collectionID ?? -1,
+      file.queueSource,
       file.title,
       file.deviceFolder,
       latitude,
@@ -1960,6 +2027,7 @@ class FilesDB with SqlDbBase {
     file.hash = row[columnHash];
     file.metadataVersion = row[columnMetadataVersion] ?? 0;
     file.fileSize = row[columnFileSize];
+    file.queueSource = row[columnQueueSource];
 
     file.mMdVersion = row[columnMMdVersion] ?? 0;
     file.mMdEncodedJson = row[columnMMdEncodedJson] ?? '{}';

mobile/apps/photos/lib/models/file/file.dart

@@ -41,6 +41,7 @@ class EnteFile {
   String? thumbnailDecryptionHeader;
   String? metadataDecryptionHeader;
   int? fileSize;
+  String? queueSource;
 
   String? mMdEncodedJson;
   int mMdVersion = 0;
@@ -387,6 +388,7 @@ class EnteFile {
     String? thumbnailDecryptionHeader,
     String? metadataDecryptionHeader,
     int? fileSize,
+    String? queueSource,
     String? mMdEncodedJson,
     int? mMdVersion,
     MagicMetadata? magicMetadata,
@@ -421,6 +423,7 @@ class EnteFile {
       ..metadataDecryptionHeader =
           metadataDecryptionHeader ?? this.metadataDecryptionHeader
       ..fileSize = fileSize ?? this.fileSize
+      ..queueSource = queueSource ?? this.queueSource
       ..mMdEncodedJson = mMdEncodedJson ?? this.mMdEncodedJson
       ..mMdVersion = mMdVersion ?? this.mMdVersion
       ..magicMetadata = magicMetadata ?? this.magicMetadata

mobile/apps/photos/lib/services/sync/remote_sync_service.dart

@@ -48,6 +48,7 @@ class RemoteSyncService {
       LocalFileUpdateService.instance;
   int _completedUploads = 0;
   int _ignoredUploads = 0;
+  Set<String>? _selectedDevicePathIdsCache;
   late SharedPreferences _prefs;
   Completer<void>? _existingSync;
   bool _isExistingSyncSilent = false;
@@ -359,6 +360,10 @@ class RemoteSyncService {
     _logger.info("Syncing device collections to be uploaded");
     final int ownerID = _config.getUserID()!;
 
+    if (flagService.queueSourceEnabled) {
+      await _ensureSelectedPathIdsCache();
+    }
+
     final deviceCollections = await _db.getDeviceCollections();
     deviceCollections.removeWhere((element) => !element.shouldBackup);
     // Sort by count to ensure that photos in iOS are first inserted in
@@ -415,6 +420,8 @@ class RemoteSyncService {
       await _db.setCollectionIDForUnMappedLocalFiles(
         collectionID,
         localIDsToSync,
+        queueSource:
+            flagService.queueSourceEnabled ? deviceCollection.id : null,
       );
 
       // mark IDs as already synced if corresponding entry is present in
@@ -451,6 +458,9 @@ class RemoteSyncService {
             existingFile.collectionID = collectionID;
             existingFile.uploadedFileID = null;
             existingFile.ownerID = null;
+            if (flagService.queueSourceEnabled) {
+              existingFile.queueSource = deviceCollection.id;
+            }
             newFilesToInsert.add(existingFile);
             fileFoundForLocalIDs.add(localID);
           }
@@ -478,6 +488,9 @@ class RemoteSyncService {
     final Set<int> oldCollectionIDsForAutoSync =
         await _db.getDeviceSyncCollectionIDs();
     await _db.updateDevicePathSyncStatus(syncStatusUpdate);
+    if (flagService.queueSourceEnabled) {
+      await _refreshSelectedPathIdsCacheIfChanged(syncStatusUpdate);
+    }
     final Set<int> newCollectionIDsForAutoSync =
         await _db.getDeviceSyncCollectionIDs();
     SyncService.instance.onDeviceCollectionSet(newCollectionIDsForAutoSync);
@@ -493,6 +506,40 @@ class RemoteSyncService {
     Bus.instance.fire(BackupFoldersUpdatedEvent());
   }
 
+  Future<void> _ensureSelectedPathIdsCache() async {
+    if (_selectedDevicePathIdsCache != null) {
+      return;
+    }
+    _selectedDevicePathIdsCache = await _db.getSelectedDevicePathIds();
+  }
+
+  Future<void> _refreshSelectedPathIdsCacheIfChanged(
+    Map<String, bool> syncStatusUpdate,
+  ) async {
+    if (syncStatusUpdate.isEmpty) {
+      return;
+    }
+    await _ensureSelectedPathIdsCache();
+    final cache = _selectedDevicePathIdsCache!;
+    final hasChange = syncStatusUpdate.entries.any(
+      (entry) => cache.contains(entry.key) != entry.value,
+    );
+    if (!hasChange) {
+      return;
+    }
+    _selectedDevicePathIdsCache = await _db.getSelectedDevicePathIds();
+  }
+
+  bool isDevicePathSelected(String pathId) {
+    if (!flagService.queueSourceEnabled) {
+      return true;
+    }
+    if (_selectedDevicePathIdsCache == null) {
+      unawaited(_ensureSelectedPathIdsCache());
+    }
+    return _selectedDevicePathIdsCache?.contains(pathId) ?? true;
+  }
+
   Future<void> removeFilesQueuedForUpload(List<int> collectionIDs) async {
     /*
       For each collection, perform following action
@@ -757,6 +804,9 @@ class RemoteSyncService {
     if (error is InvalidFileError) {
       _ignoredUploads++;
       _logger.warning("Invalid file error", error);
+    } else if (error is SkippedQueuedFileError) {
+      _ignoredUploads++;
+      _logger.info("Skipped queued file due to queue source");
     } else {
       throw error;
     }

mobile/apps/photos/lib/ui/actions/collection/collection_file_actions.dart

@@ -10,6 +10,7 @@ import "package:photos/generated/l10n.dart";
 import 'package:photos/models/collection/collection.dart';
 import 'package:photos/models/file/file.dart';
 import 'package:photos/models/selected_files.dart';
+import "package:photos/service_locator.dart";
 import "package:photos/services/collections_service.dart";
 import 'package:photos/services/favorites_service.dart';
 import "package:photos/services/hidden_service.dart";
@@ -144,8 +145,12 @@ extension CollectionFileActions on CollectionActions {
               files.add(uploadedFile);
             }
           } else {
+            final bool queueSourceEnabled = flagService.queueSourceEnabled;
             for (final file in filesPendingUpload) {
               file.collectionID = collection.id;
+              if (queueSourceEnabled) {
+                file.queueSource = 'manual';
+              }
             }
             // filesPendingUpload might be getting ignored during auto-upload
             // because the user deleted these files from ente in the past.
@@ -266,8 +271,12 @@ extension CollectionFileActions on CollectionActions {
             files.add(uploadedFile);
           }
         } else {
+          final bool queueSourceEnabled = flagService.queueSourceEnabled;
           for (final file in filesPendingUpload) {
             file.collectionID = collectionID;
+            if (queueSourceEnabled) {
+              file.queueSource = 'manual';
+            }
           }
           // filesPendingUpload might be getting ignored during auto-upload
           // because the user deleted these files from ente in the past.

mobile/apps/photos/lib/utils/file_uploader.dart

@@ -37,6 +37,7 @@ import "package:photos/service_locator.dart";
 import "package:photos/services/account/user_service.dart";
 import 'package:photos/services/collections_service.dart';
 import 'package:photos/services/sync/local_sync_service.dart';
+import 'package:photos/services/sync/remote_sync_service.dart';
 import 'package:photos/services/sync/sync_service.dart';
 import "package:photos/utils/exif_util.dart";
 import "package:photos/utils/file_key.dart";
@@ -298,6 +299,12 @@ class FileUploader {
             ?.value;
       }
       if (pendingEntry != null) {
+        if (flagService.queueSourceEnabled &&
+            _shouldSkipQueuedItem(pendingEntry)) {
+          _handleSkippedQueuedItem(pendingEntry);
+          _pollQueue();
+          return;
+        }
         pendingEntry.status = UploadStatus.inProgress;
         _allBackups[pendingEntry.file.localID!] =
             _allBackups[pendingEntry.file.localID]!
@@ -311,6 +318,44 @@ class FileUploader {
     }
   }
 
+  bool _shouldSkipQueuedItem(FileUploadItem item) {
+    final file = item.file;
+    final qs = file.queueSource;
+    if (qs == null || qs == 'manual') {
+      return false;
+    }
+
+    final bool isSelected = RemoteSyncService.instance.isDevicePathSelected(qs);
+    if (!isSelected) {
+      return true;
+    }
+
+    final int? onlyNewSince = backupPreferenceService.onlyNewSinceEpoch;
+    return onlyNewSince != null && (file.creationTime ?? 0) < onlyNewSince;
+  }
+
+  void _handleSkippedQueuedItem(FileUploadItem item) {
+    final file = item.file;
+    final qs = file.queueSource;
+    if (file.localID == null || qs == null) {
+      return;
+    }
+    unawaited(
+      FilesDB.instance.cleanupQueuedEntry(
+        localID: file.localID!,
+        collectionID: item.collectionID,
+        queueSource: qs,
+      ),
+    );
+    _queue.remove(file.localID!);
+    _allBackups.remove(file.localID!);
+    if (_totalCountInUploadSession > 0) {
+      _totalCountInUploadSession--;
+    }
+    item.completer.completeError(SkippedQueuedFileError());
+    Bus.instance.fire(BackupUpdatedEvent(_allBackups));
+  }
+
   Future<EnteFile?> _encryptAndUploadFileToCollection(
     EnteFile file,
     int collectionID, {

mobile/apps/photos/plugins/ente_feature_flag/lib/src/service.dart

@@ -69,6 +69,8 @@ class FlagService {
 
   bool get enableUploadV2 => ((flags.serverApiFlag & _uploadV2Flag) != 0);
 
+  bool get queueSourceEnabled => internalUser;
+
   bool get enableVectorDb => hasGrantedMLConsent;
 
   String get castUrl => flags.castUrl;