From ccc8aa94c82bb3d20d69ab9665fa598da39fd9bd Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 18 Jun 2024 15:24:34 -0700 Subject: [PATCH 1/5] feat(mobile): render asset on device by default --- mobile/lib/services/album.service.dart | 69 +------------------------- mobile/lib/services/hash.service.dart | 6 +-- mobile/lib/services/sync.service.dart | 32 ++++-------- 3 files changed, 14 insertions(+), 93 deletions(-) diff --git a/mobile/lib/services/album.service.dart b/mobile/lib/services/album.service.dart index c2494680c7da5..dbb70fed7e80b 100644 --- a/mobile/lib/services/album.service.dart +++ b/mobile/lib/services/album.service.dart @@ -1,12 +1,8 @@ import 'dart:async'; -import 'dart:collection'; -import 'dart:io'; -import 'package:collection/collection.dart'; import 'package:flutter/foundation.dart'; import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/models/albums/album_add_asset_response.model.dart'; -import 'package:immich_mobile/entities/backup_album.entity.dart'; import 'package:immich_mobile/services/backup.service.dart'; import 'package:immich_mobile/entities/album.entity.dart'; import 'package:immich_mobile/entities/asset.entity.dart'; @@ -62,60 +58,14 @@ class AlbumService { final Stopwatch sw = Stopwatch()..start(); bool changes = false; try { - final List excludedIds = - await _backupService.excludedAlbumsQuery().idProperty().findAll(); - final List selectedIds = - await _backupService.selectedAlbumsQuery().idProperty().findAll(); - if (selectedIds.isEmpty) { - final numLocal = await _db.albums.where().localIdIsNotNull().count(); - if (numLocal > 0) { - _syncService.removeAllLocalAlbumsAndAssets(); - } - return false; - } final List onDevice = await PhotoManager.getAssetPathList( hasAll: true, filterOption: FilterOptionGroup(containsPathModified: true), ); _log.info("Found ${onDevice.length} device albums"); - Set? excludedAssets; - if (excludedIds.isNotEmpty) { - if (Platform.isIOS) { - // iOS and Android device album working principle differ significantly - // on iOS, an asset can be in multiple albums - // on Android, an asset can only be in exactly one album (folder!) at the same time - // thus, on Android, excluding an album can be done by ignoring that album - // however, on iOS, it it necessary to load the assets from all excluded - // albums and check every asset from any selected album against the set - // of excluded assets - excludedAssets = await _loadExcludedAssetIds(onDevice, excludedIds); - _log.info("Found ${excludedAssets.length} assets to exclude"); - } - // remove all excluded albums - onDevice.removeWhere((e) => excludedIds.contains(e.id)); - _log.info( - "Ignoring ${excludedIds.length} excluded albums resulting in ${onDevice.length} device albums", - ); - } - final hasAll = selectedIds - .map((id) => onDevice.firstWhereOrNull((a) => a.id == id)) - .whereNotNull() - .any((a) => a.isAll); - if (hasAll) { - if (Platform.isAndroid) { - // remove the virtual "Recent" album and keep and individual albums - // on Android, the virtual "Recent" `lastModified` value is always null - onDevice.removeWhere((e) => e.isAll); - _log.info("'Recents' is selected, keeping all individual albums"); - } - } else { - // keep only the explicitly selected albums - onDevice.removeWhere((e) => !selectedIds.contains(e.id)); - _log.info("'Recents' is not selected, keeping only selected albums"); - } - changes = - await _syncService.syncLocalAlbumAssetsToDb(onDevice, excludedAssets); + + changes = await _syncService.syncLocalAlbumAssetsToDb(onDevice); _log.info("Syncing completed. Changes: $changes"); } finally { _localCompleter.complete(changes); @@ -124,21 +74,6 @@ class AlbumService { return changes; } - Future> _loadExcludedAssetIds( - List albums, - List excludedAlbumIds, - ) async { - final Set result = HashSet(); - for (AssetPathEntity a in albums) { - if (excludedAlbumIds.contains(a.id)) { - final List assets = - await a.getAssetListRange(start: 0, end: 0x7fffffffffffffff); - result.addAll(assets.map((e) => e.id)); - } - } - return result; - } - /// Checks remote albums (owned if `isShared` is false) for changes, /// updates the local database and returns `true` if there were any changes Future refreshRemoteAlbums({required bool isShared}) async { diff --git a/mobile/lib/services/hash.service.dart b/mobile/lib/services/hash.service.dart index ffc81a3445acf..3071a89c0039d 100644 --- a/mobile/lib/services/hash.service.dart +++ b/mobile/lib/services/hash.service.dart @@ -24,13 +24,9 @@ class HashService { AssetPathEntity album, { int start = 0, int end = 0x7fffffffffffffff, - Set? excludedAssets, }) async { final entities = await album.getAssetListRange(start: start, end: end); - final filtered = excludedAssets == null - ? entities - : entities.where((e) => !excludedAssets.contains(e.id)).toList(); - return _hashAssets(filtered); + return _hashAssets(entities); } /// Converts a list of [AssetEntity]s to [Asset]s including only those diff --git a/mobile/lib/services/sync.service.dart b/mobile/lib/services/sync.service.dart index 8ec56e925f7f8..2120dda589150 100644 --- a/mobile/lib/services/sync.service.dart +++ b/mobile/lib/services/sync.service.dart @@ -68,10 +68,9 @@ class SyncService { /// Syncs all device albums and their assets to the database /// Returns `true` if there were any changes Future syncLocalAlbumAssetsToDb( - List onDevice, [ - Set? excludedAssets, - ]) => - _lock.run(() => _syncLocalAlbumAssetsToDb(onDevice, excludedAssets)); + List onDevice, + ) => + _lock.run(() => _syncLocalAlbumAssetsToDb(onDevice)); /// returns all Asset IDs that are not contained in the existing list List sharedAssetsToRemove( @@ -492,9 +491,8 @@ class SyncService { /// Syncs all device albums and their assets to the database /// Returns `true` if there were any changes Future _syncLocalAlbumAssetsToDb( - List onDevice, [ - Set? excludedAssets, - ]) async { + List onDevice, + ) async { onDevice.sort((a, b) => a.id.compareTo(b.id)); final inDb = await _db.albums.where().localIdIsNotNull().sortByLocalId().findAll(); @@ -510,10 +508,8 @@ class SyncService { album, deleteCandidates, existing, - excludedAssets, ), - onlyFirst: (AssetPathEntity ape) => - _addAlbumFromDevice(ape, existing, excludedAssets), + onlyFirst: (AssetPathEntity ape) => _addAlbumFromDevice(ape, existing), onlySecond: (Album a) => _removeAlbumFromDb(a, deleteCandidates), ); _log.fine( @@ -545,16 +541,13 @@ class SyncService { Album album, List deleteCandidates, List existing, [ - Set? excludedAssets, bool forceRefresh = false, ]) async { if (!forceRefresh && !await _hasAssetPathEntityChanged(ape, album)) { _log.fine("Local album ${ape.name} has not changed. Skipping sync."); return false; } - if (!forceRefresh && - excludedAssets == null && - await _syncDeviceAlbumFast(ape, album)) { + if (!forceRefresh && await _syncDeviceAlbumFast(ape, album)) { return true; } @@ -566,8 +559,7 @@ class SyncService { .findAll(); assert(inDb.isSorted(Asset.compareByChecksum), "inDb not sorted!"); final int assetCountOnDevice = await ape.assetCountAsync; - final List onDevice = - await _hashService.getHashedAssets(ape, excludedAssets: excludedAssets); + final List onDevice = await _hashService.getHashedAssets(ape); _removeDuplicates(onDevice); // _removeDuplicates sorts `onDevice` by checksum final (toAdd, toUpdate, toDelete) = _diffAssets(onDevice, inDb); @@ -678,13 +670,11 @@ class SyncService { /// assets already existing in the database to the list of `existing` assets Future _addAlbumFromDevice( AssetPathEntity ape, - List existing, [ - Set? excludedAssets, - ]) async { + List existing, + ) async { _log.info("Syncing a new local album to DB: ${ape.name}"); final Album a = Album.local(ape); - final assets = - await _hashService.getHashedAssets(ape, excludedAssets: excludedAssets); + final assets = await _hashService.getHashedAssets(ape); _removeDuplicates(assets); final (existingInDb, updated) = await _linkWithExistingFromDb(assets); _log.info( From 41ab21dacd0036f84fcf6a2f8aefbd97c7210068 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 18 Jun 2024 15:45:52 -0700 Subject: [PATCH 2/5] remove unused service --- mobile/lib/services/album.service.dart | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mobile/lib/services/album.service.dart b/mobile/lib/services/album.service.dart index dbb70fed7e80b..b367318553d84 100644 --- a/mobile/lib/services/album.service.dart +++ b/mobile/lib/services/album.service.dart @@ -3,7 +3,6 @@ import 'dart:async'; import 'package:flutter/foundation.dart'; import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/models/albums/album_add_asset_response.model.dart'; -import 'package:immich_mobile/services/backup.service.dart'; import 'package:immich_mobile/entities/album.entity.dart'; import 'package:immich_mobile/entities/asset.entity.dart'; import 'package:immich_mobile/entities/store.entity.dart'; @@ -24,7 +23,6 @@ final albumServiceProvider = Provider( ref.watch(userServiceProvider), ref.watch(syncServiceProvider), ref.watch(dbProvider), - ref.watch(backupServiceProvider), ), ); @@ -33,7 +31,6 @@ class AlbumService { final UserService _userService; final SyncService _syncService; final Isar _db; - final BackupService _backupService; final Logger _log = Logger('AlbumService'); Completer _localCompleter = Completer()..complete(false); Completer _remoteCompleter = Completer()..complete(false); @@ -43,7 +40,6 @@ class AlbumService { this._userService, this._syncService, this._db, - this._backupService, ); /// Checks all selected device albums for changes of albums and their assets From 7be6ba235b3f2a3681a00d5ec024afd08ff197bb Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 20 Jun 2024 14:16:20 -0700 Subject: [PATCH 3/5] feat(mobile): render asset on device by default - with stream --- mobile/lib/main.dart | 1 + mobile/lib/services/hash.service.dart | 46 +++++-- mobile/lib/services/sync.service.dart | 177 +++++++++++++------------- 3 files changed, 127 insertions(+), 97 deletions(-) diff --git a/mobile/lib/main.dart b/mobile/lib/main.dart index 2a320fbddbe4a..3e32b8e9a61d4 100644 --- a/mobile/lib/main.dart +++ b/mobile/lib/main.dart @@ -81,6 +81,7 @@ Future initApp() async { PlatformDispatcher.instance.onError = (error, stack) { log.severe('PlatformDispatcher - Catch all', error, stack); + debugPrint("PlatformDispatcher - Catch all: $error $stack"); return true; }; diff --git a/mobile/lib/services/hash.service.dart b/mobile/lib/services/hash.service.dart index 3071a89c0039d..e79837340f549 100644 --- a/mobile/lib/services/hash.service.dart +++ b/mobile/lib/services/hash.service.dart @@ -1,5 +1,6 @@ import 'dart:io'; +import 'package:collection/collection.dart'; import 'package:flutter/foundation.dart'; import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/services/background.service.dart'; @@ -20,20 +21,23 @@ class HashService { final _log = Logger('HashService'); /// Returns all assets that were successfully hashed - Future> getHashedAssets( + Stream> getHashedAssets( AssetPathEntity album, { int start = 0, int end = 0x7fffffffffffffff, - }) async { + }) async* { final entities = await album.getAssetListRange(start: start, end: end); - return _hashAssets(entities); + // yield _hashAssets(entities); + await for (final assets in _hashAssets(entities)) { + yield assets; + } } /// Converts a list of [AssetEntity]s to [Asset]s including only those /// that were successfully hashed. Hashes are looked up in a DB table /// [AndroidDeviceAsset] / [IOSDeviceAsset] by local id. Only missing /// entries are newly hashed and added to the DB table. - Future> _hashAssets(List assetEntities) async { + Stream> _hashAssets(List assetEntities) async* { const int batchFileCount = 128; const int batchDataSize = 1024 * 1024 * 1024; // 1GB @@ -45,6 +49,7 @@ class HashService { final List toHash = []; int bytes = 0; + int batchCount = 0; for (int i = 0; i < assetEntities.length; i++) { if (hashes[i] != null) { @@ -73,16 +78,33 @@ class HashService { toAdd.add(deviceAsset); hashes[i] = deviceAsset; if (toHash.length == batchFileCount || bytes >= batchDataSize) { - await _processBatch(toHash, toAdd); + await for (final batch in _processBatch(toHash, toAdd)) { + debugPrint( + "batch length ${batch.length} - assetEntities length ${assetEntities.length}", + ); + yield _mapAllHashedAssets( + assetEntities.slice( + batchCount, + batchCount + batch.length, + ), + batch, + ); + + batchCount += batch.length; + } toAdd.clear(); toHash.clear(); bytes = 0; } } if (toHash.isNotEmpty) { - await _processBatch(toHash, toAdd); + await for (final batch in _processBatch(toHash, toAdd)) { + debugPrint( + "To hash is not empty with length ${toHash.length}, PROCESS TO ADD ${toAdd.length}, ASSET ENTITIEES LENGTH ${assetEntities.length} . batch length ${batch.length}", + ); + yield _mapAllHashedAssets(assetEntities, batch); + } } - return _mapAllHashedAssets(assetEntities, hashes); } /// Lookup hashes of assets by their local ID @@ -93,10 +115,10 @@ class HashService { /// Processes a batch of files and saves any successfully hashed /// values to the DB table. - Future _processBatch( + Stream> _processBatch( final List toHash, final List toAdd, - ) async { + ) async* { final hashes = await _hashFiles(toHash); bool anyNull = false; for (int j = 0; j < hashes.length; j++) { @@ -116,6 +138,7 @@ class HashService { : _db.iOSDeviceAssets.putAll(validHashes.cast()), ); _log.fine("Hashed ${validHashes.length}/${toHash.length} assets"); + yield validHashes; } /// Hashes the given files and returns a list of the same length @@ -134,8 +157,11 @@ class HashService { List assets, List hashes, ) { + debugPrint( + "[_mapAllHashedAssets] Mapping all hashed assets, assets length ${assets.length}, hashes length ${hashes.length}", + ); final List result = []; - for (int i = 0; i < assets.length; i++) { + for (int i = 0; i < hashes.length; i++) { if (hashes[i] != null && hashes[i]!.hash.isNotEmpty) { result.add(Asset.local(assets[i], hashes[i]!.hash)); } diff --git a/mobile/lib/services/sync.service.dart b/mobile/lib/services/sync.service.dart index 2120dda589150..4e4d0ef756775 100644 --- a/mobile/lib/services/sync.service.dart +++ b/mobile/lib/services/sync.service.dart @@ -559,61 +559,62 @@ class SyncService { .findAll(); assert(inDb.isSorted(Asset.compareByChecksum), "inDb not sorted!"); final int assetCountOnDevice = await ape.assetCountAsync; - final List onDevice = await _hashService.getHashedAssets(ape); - _removeDuplicates(onDevice); - // _removeDuplicates sorts `onDevice` by checksum - final (toAdd, toUpdate, toDelete) = _diffAssets(onDevice, inDb); - if (toAdd.isEmpty && - toUpdate.isEmpty && - toDelete.isEmpty && - album.name == ape.name && - ape.lastModified != null && - album.modifiedAt.isAtSameMomentAs(ape.lastModified!)) { - // changes only affeted excluded albums + await for (final onDevice in _hashService.getHashedAssets(ape)) { + _removeDuplicates(onDevice); + // _removeDuplicates sorts `onDevice` by checksum + final (toAdd, toUpdate, toDelete) = _diffAssets(onDevice, inDb); + if (toAdd.isEmpty && + toUpdate.isEmpty && + toDelete.isEmpty && + album.name == ape.name && + ape.lastModified != null && + album.modifiedAt.isAtSameMomentAs(ape.lastModified!)) { + // changes only affeted excluded albums + _log.fine( + "Only excluded assets in local album ${ape.name} changed. Stopping sync.", + ); + if (assetCountOnDevice != + _db.eTags.getByIdSync(ape.eTagKeyAssetCount)?.assetCount) { + await _db.writeTxn( + () => _db.eTags.put( + ETag(id: ape.eTagKeyAssetCount, assetCount: assetCountOnDevice), + ), + ); + } + return false; + } + _log.fine( + "Syncing local album ${ape.name}. ${toAdd.length} assets to add, ${toUpdate.length} to update, ${toDelete.length} to delete", + ); + final (existingInDb, updated) = await _linkWithExistingFromDb(toAdd); _log.fine( - "Only excluded assets in local album ${ape.name} changed. Stopping sync.", + "Linking assets to add with existing from db. ${existingInDb.length} existing, ${updated.length} to update", ); - if (assetCountOnDevice != - _db.eTags.getByIdSync(ape.eTagKeyAssetCount)?.assetCount) { - await _db.writeTxn( - () => _db.eTags.put( + deleteCandidates.addAll(toDelete); + existing.addAll(existingInDb); + album.name = ape.name; + album.modifiedAt = ape.lastModified ?? DateTime.now(); + if (album.thumbnail.value != null && + toDelete.contains(album.thumbnail.value)) { + album.thumbnail.value = null; + } + try { + await _db.writeTxn(() async { + await _db.assets.putAll(updated); + await _db.assets.putAll(toUpdate); + await album.assets + .update(link: existingInDb + updated, unlink: toDelete); + await _db.albums.put(album); + album.thumbnail.value ??= await album.assets.filter().findFirst(); + await album.thumbnail.save(); + await _db.eTags.put( ETag(id: ape.eTagKeyAssetCount, assetCount: assetCountOnDevice), - ), - ); + ); + }); + _log.info("Synced changes of local album ${ape.name} to DB"); + } on IsarError catch (e) { + _log.severe("Failed to update synced album ${ape.name} in DB", e); } - return false; - } - _log.fine( - "Syncing local album ${ape.name}. ${toAdd.length} assets to add, ${toUpdate.length} to update, ${toDelete.length} to delete", - ); - final (existingInDb, updated) = await _linkWithExistingFromDb(toAdd); - _log.fine( - "Linking assets to add with existing from db. ${existingInDb.length} existing, ${updated.length} to update", - ); - deleteCandidates.addAll(toDelete); - existing.addAll(existingInDb); - album.name = ape.name; - album.modifiedAt = ape.lastModified ?? DateTime.now(); - if (album.thumbnail.value != null && - toDelete.contains(album.thumbnail.value)) { - album.thumbnail.value = null; - } - try { - await _db.writeTxn(() async { - await _db.assets.putAll(updated); - await _db.assets.putAll(toUpdate); - await album.assets - .update(link: existingInDb + updated, unlink: toDelete); - await _db.albums.put(album); - album.thumbnail.value ??= await album.assets.filter().findFirst(); - await album.thumbnail.save(); - await _db.eTags.put( - ETag(id: ape.eTagKeyAssetCount, assetCount: assetCountOnDevice), - ); - }); - _log.info("Synced changes of local album ${ape.name} to DB"); - } on IsarError catch (e) { - _log.severe("Failed to update synced album ${ape.name} in DB", e); } return true; @@ -641,26 +642,27 @@ class SyncService { if (modified == null) { return false; } - final List newAssets = await _hashService.getHashedAssets(modified); - if (totalOnDevice != lastKnownTotal + newAssets.length) { - return false; - } - album.modifiedAt = ape.lastModified ?? DateTime.now(); - _removeDuplicates(newAssets); - final (existingInDb, updated) = await _linkWithExistingFromDb(newAssets); - try { - await _db.writeTxn(() async { - await _db.assets.putAll(updated); - await album.assets.update(link: existingInDb + updated); - await _db.albums.put(album); - await _db.eTags - .put(ETag(id: ape.eTagKeyAssetCount, assetCount: totalOnDevice)); - }); - _log.info("Fast synced local album ${ape.name} to DB"); - } on IsarError catch (e) { - _log.severe("Failed to fast sync local album ${ape.name} to DB", e); - return false; + await for (final newAssets in _hashService.getHashedAssets(modified)) { + if (totalOnDevice != lastKnownTotal + newAssets.length) { + return false; + } + album.modifiedAt = ape.lastModified ?? DateTime.now(); + _removeDuplicates(newAssets); + final (existingInDb, updated) = await _linkWithExistingFromDb(newAssets); + try { + await _db.writeTxn(() async { + await _db.assets.putAll(updated); + await album.assets.update(link: existingInDb + updated); + await _db.albums.put(album); + await _db.eTags + .put(ETag(id: ape.eTagKeyAssetCount, assetCount: totalOnDevice)); + }); + _log.info("Fast synced local album ${ape.name} to DB"); + } on IsarError catch (e) { + _log.severe("Failed to fast sync local album ${ape.name} to DB", e); + return false; + } } return true; @@ -674,23 +676,24 @@ class SyncService { ) async { _log.info("Syncing a new local album to DB: ${ape.name}"); final Album a = Album.local(ape); - final assets = await _hashService.getHashedAssets(ape); - _removeDuplicates(assets); - final (existingInDb, updated) = await _linkWithExistingFromDb(assets); - _log.info( - "${existingInDb.length} assets already existed in DB, to upsert ${updated.length}", - ); - await upsertAssetsWithExif(updated); - existing.addAll(existingInDb); - a.assets.addAll(existingInDb); - a.assets.addAll(updated); - final thumb = existingInDb.firstOrNull ?? updated.firstOrNull; - a.thumbnail.value = thumb; - try { - await _db.writeTxn(() => _db.albums.store(a)); - _log.info("Added a new local album to DB: ${ape.name}"); - } on IsarError catch (e) { - _log.severe("Failed to add new local album ${ape.name} to DB", e); + await for (final assets in _hashService.getHashedAssets(ape)) { + _removeDuplicates(assets); + final (existingInDb, updated) = await _linkWithExistingFromDb(assets); + _log.info( + "${existingInDb.length} assets already existed in DB, to upsert ${updated.length}", + ); + await upsertAssetsWithExif(updated); + existing.addAll(existingInDb); + a.assets.addAll(existingInDb); + a.assets.addAll(updated); + final thumb = existingInDb.firstOrNull ?? updated.firstOrNull; + a.thumbnail.value = thumb; + try { + await _db.writeTxn(() => _db.albums.store(a)); + _log.info("Added a new local album to DB: ${ape.name}"); + } on IsarError catch (e) { + _log.severe("Failed to add new local album ${ape.name} to DB", e); + } } } From 17af8838eaaebda16765ee4648ecd98c70a82491 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 20 Jun 2024 15:00:43 -0700 Subject: [PATCH 4/5] simplify local assets --- mobile/lib/services/hash.service.dart | 53 ++++++--------------------- 1 file changed, 12 insertions(+), 41 deletions(-) diff --git a/mobile/lib/services/hash.service.dart b/mobile/lib/services/hash.service.dart index e79837340f549..59741531c128f 100644 --- a/mobile/lib/services/hash.service.dart +++ b/mobile/lib/services/hash.service.dart @@ -1,6 +1,5 @@ import 'dart:io'; -import 'package:collection/collection.dart'; import 'package:flutter/foundation.dart'; import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/services/background.service.dart'; @@ -46,10 +45,9 @@ class HashService { .toList(); final List hashes = await _lookupHashes(ids); final List toAdd = []; - final List toHash = []; + final List> toHash = []; int bytes = 0; - int batchCount = 0; for (int i = 0; i < assetEntities.length; i++) { if (hashes[i] != null) { @@ -71,7 +69,7 @@ class HashService { continue; } bytes += await file.length(); - toHash.add(file.path); + toHash.add({file.path: assetEntities[i]}); final deviceAsset = Platform.isAndroid ? AndroidDeviceAsset(id: ids[i] as int, hash: const []) : IOSDeviceAsset(id: ids[i] as String, hash: const []); @@ -79,18 +77,7 @@ class HashService { hashes[i] = deviceAsset; if (toHash.length == batchFileCount || bytes >= batchDataSize) { await for (final batch in _processBatch(toHash, toAdd)) { - debugPrint( - "batch length ${batch.length} - assetEntities length ${assetEntities.length}", - ); - yield _mapAllHashedAssets( - assetEntities.slice( - batchCount, - batchCount + batch.length, - ), - batch, - ); - - batchCount += batch.length; + yield batch; } toAdd.clear(); toHash.clear(); @@ -99,10 +86,7 @@ class HashService { } if (toHash.isNotEmpty) { await for (final batch in _processBatch(toHash, toAdd)) { - debugPrint( - "To hash is not empty with length ${toHash.length}, PROCESS TO ADD ${toAdd.length}, ASSET ENTITIEES LENGTH ${assetEntities.length} . batch length ${batch.length}", - ); - yield _mapAllHashedAssets(assetEntities, batch); + yield batch; } } } @@ -115,30 +99,34 @@ class HashService { /// Processes a batch of files and saves any successfully hashed /// values to the DB table. - Stream> _processBatch( - final List toHash, + Stream> _processBatch( + final List> toHash, final List toAdd, ) async* { - final hashes = await _hashFiles(toHash); + final List validLocalAssets = []; + final hashes = await _hashFiles(toHash.map((e) => e.keys.first).toList()); bool anyNull = false; for (int j = 0; j < hashes.length; j++) { if (hashes[j]?.length == 20) { toAdd[j].hash = hashes[j]!; + validLocalAssets.add(Asset.local(toHash[j].values.first, hashes[j]!)); } else { _log.warning("Failed to hash file ${toHash[j]}, skipping"); anyNull = true; } } + final validHashes = anyNull ? toAdd.where((e) => e.hash.length == 20).toList(growable: false) : toAdd; + await _db.writeTxn( () => Platform.isAndroid ? _db.androidDeviceAssets.putAll(validHashes.cast()) : _db.iOSDeviceAssets.putAll(validHashes.cast()), ); _log.fine("Hashed ${validHashes.length}/${toHash.length} assets"); - yield validHashes; + yield validLocalAssets; } /// Hashes the given files and returns a list of the same length @@ -151,23 +139,6 @@ class HashService { } return hashes; } - - /// Converts [AssetEntity]s that were successfully hashed to [Asset]s - List _mapAllHashedAssets( - List assets, - List hashes, - ) { - debugPrint( - "[_mapAllHashedAssets] Mapping all hashed assets, assets length ${assets.length}, hashes length ${hashes.length}", - ); - final List result = []; - for (int i = 0; i < hashes.length; i++) { - if (hashes[i] != null && hashes[i]!.hash.isNotEmpty) { - result.add(Asset.local(assets[i], hashes[i]!.hash)); - } - } - return result; - } } final hashServiceProvider = Provider( From fe067aa9526ab5f75fb31d3c4d21413d14133c13 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 21 Jun 2024 14:34:27 -0700 Subject: [PATCH 5/5] sort by local id --- mobile/lib/entities/asset.entity.dart | 11 +++++++++++ mobile/lib/services/hash.service.dart | 3 ++- mobile/lib/services/sync.service.dart | 12 +++++++++--- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/mobile/lib/entities/asset.entity.dart b/mobile/lib/entities/asset.entity.dart index 3f8c1fa74cbe7..43b9149daedd9 100644 --- a/mobile/lib/entities/asset.entity.dart +++ b/mobile/lib/entities/asset.entity.dart @@ -413,6 +413,17 @@ class Asset { static int compareById(Asset a, Asset b) => a.id.compareTo(b.id); + static int compareByLocalId(Asset a, Asset b) { + if (a.localId == null && b.localId == null) { + return 0; + } else if (a.localId == null) { + return 1; + } else if (b.localId == null) { + return -1; + } + return a.localId!.compareTo(b.localId!); + } + static int compareByChecksum(Asset a, Asset b) => a.checksum.compareTo(b.checksum); diff --git a/mobile/lib/services/hash.service.dart b/mobile/lib/services/hash.service.dart index 59741531c128f..a0af880983f5f 100644 --- a/mobile/lib/services/hash.service.dart +++ b/mobile/lib/services/hash.service.dart @@ -1,5 +1,6 @@ import 'dart:io'; +import 'package:collection/collection.dart'; import 'package:flutter/foundation.dart'; import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/services/background.service.dart'; @@ -26,7 +27,7 @@ class HashService { int end = 0x7fffffffffffffff, }) async* { final entities = await album.getAssetListRange(start: start, end: end); - // yield _hashAssets(entities); + entities.sortBy((e) => e.id); await for (final assets in _hashAssets(entities)) { yield assets; } diff --git a/mobile/lib/services/sync.service.dart b/mobile/lib/services/sync.service.dart index 4e4d0ef756775..8b19426c48fd6 100644 --- a/mobile/lib/services/sync.service.dart +++ b/mobile/lib/services/sync.service.dart @@ -555,14 +555,20 @@ class SyncService { final inDb = await album.assets .filter() .ownerIdEqualTo(Store.get(StoreKey.currentUser).isarId) - .sortByChecksum() + .sortByLocalId() .findAll(); - assert(inDb.isSorted(Asset.compareByChecksum), "inDb not sorted!"); + assert(inDb.isSorted(Asset.compareByLocalId), "inDb not sorted!"); final int assetCountOnDevice = await ape.assetCountAsync; + await for (final onDevice in _hashService.getHashedAssets(ape)) { _removeDuplicates(onDevice); // _removeDuplicates sorts `onDevice` by checksum - final (toAdd, toUpdate, toDelete) = _diffAssets(onDevice, inDb); + final (toAdd, toUpdate, toDelete) = _diffAssets( + onDevice, + inDb, + compare: Asset.compareByLocalId, + ); + if (toAdd.isEmpty && toUpdate.isEmpty && toDelete.isEmpty &&