1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2025-04-21 07:26:25 +02:00

feat(mobile): use efficient sync ()

* feat(mobile): use efficient sync

review feedback

* adapt to changed  server endpoints

* formatting

* fix memory lane bug

* fix: bad merge

* fix call not returning correct number of asset

---------

Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
This commit is contained in:
Fynn Petersen-Frey 2024-05-14 17:35:37 +02:00 committed by GitHub
parent acc611a3d9
commit 116043b2b4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 185 additions and 125 deletions

View file

@ -33,7 +33,6 @@ class PhotosPage extends HookConsumerWidget {
() {
ref.read(websocketProvider.notifier).connect();
Future(() => ref.read(assetProvider.notifier).getAllAsset());
ref.read(assetProvider.notifier).getPartnerAssets();
ref.read(albumProvider.notifier).getAllAlbums();
ref.read(sharedAlbumProvider.notifier).getAllSharedAlbums();
ref.read(serverInfoProvider.notifier).getServerInfo();
@ -85,9 +84,6 @@ class PhotosPage extends HookConsumerWidget {
Future<void> refreshAssets() async {
final fullRefresh = refreshCount.value > 0;
await ref.read(assetProvider.notifier).getAllAsset(clear: fullRefresh);
if (timelineUsers.length > 1) {
await ref.read(assetProvider.notifier).getPartnerAssets();
}
if (fullRefresh) {
// refresh was forced: user requested another refresh within 2 seconds
refreshCount.value = 0;

View file

@ -22,7 +22,7 @@ class PartnerDetailPage extends HookConsumerWidget {
useEffect(
() {
ref.read(assetProvider.notifier).getPartnerAssets(partner);
ref.read(assetProvider.notifier).getAllAsset();
return null;
},
[],
@ -78,8 +78,7 @@ class PartnerDetailPage extends HookConsumerWidget {
),
body: MultiselectGrid(
renderListProvider: assetsProvider(partner.isarId),
onRefresh: () =>
ref.read(assetProvider.notifier).getPartnerAssets(partner),
onRefresh: () => ref.read(assetProvider.notifier).getAllAsset(),
deleteEnabled: false,
favoriteEnabled: false,
),

View file

@ -56,11 +56,10 @@ class AppLifeCycleNotifier extends StateNotifier<AppLifeCycleEnum> {
switch (_ref.read(tabProvider)) {
case TabEnum.home:
_ref.read(assetProvider.notifier).getAllAsset();
_ref.read(assetProvider.notifier).getPartnerAssets();
case TabEnum.search:
// nothing to do
case TabEnum.sharing:
_ref.read(assetProvider.notifier).getPartnerAssets();
_ref.read(assetProvider.notifier).getAllAsset();
_ref.read(sharedAlbumProvider.notifier).getAllSharedAlbums();
case TabEnum.library:
_ref.read(albumProvider.notifier).getAllAlbums();

View file

@ -1,9 +1,9 @@
import 'package:flutter/material.dart';
import 'package:hooks_riverpod/hooks_riverpod.dart';
import 'package:immich_mobile/providers/memory.provider.dart';
import 'package:immich_mobile/services/album.service.dart';
import 'package:immich_mobile/entities/exif_info.entity.dart';
import 'package:immich_mobile/entities/store.entity.dart';
import 'package:immich_mobile/entities/user.entity.dart';
import 'package:immich_mobile/providers/db.provider.dart';
import 'package:immich_mobile/providers/user.provider.dart';
import 'package:immich_mobile/services/asset.service.dart';
@ -23,10 +23,10 @@ class AssetNotifier extends StateNotifier<bool> {
final UserService _userService;
final SyncService _syncService;
final Isar _db;
final StateNotifierProviderRef _ref;
final log = Logger('AssetNotifier');
bool _getAllAssetInProgress = false;
bool _deleteInProgress = false;
bool _getPartnerAssetsInProgress = false;
AssetNotifier(
this._assetService,
@ -34,6 +34,7 @@ class AssetNotifier extends StateNotifier<bool> {
this._userService,
this._syncService,
this._db,
this._ref,
) : super(false);
Future<void> getAllAsset({bool clear = false}) async {
@ -49,9 +50,15 @@ class AssetNotifier extends StateNotifier<bool> {
await clearAssetsAndAlbums(_db);
log.info("Manual refresh requested, cleared assets and albums from db");
}
final bool changedUsers = await _userService.refreshUsers();
final bool newRemote = await _assetService.refreshRemoteAssets();
final bool newLocal = await _albumService.refreshDeviceAlbums();
debugPrint("newRemote: $newRemote, newLocal: $newLocal");
debugPrint(
"changedUsers: $changedUsers, newRemote: $newRemote, newLocal: $newLocal",
);
if (newRemote) {
_ref.invalidate(memoryFutureProvider);
}
log.info("Load assets: ${stopwatch.elapsedMilliseconds}ms");
} finally {
@ -60,27 +67,6 @@ class AssetNotifier extends StateNotifier<bool> {
}
}
Future<void> getPartnerAssets([User? partner]) async {
if (_getPartnerAssetsInProgress) return;
try {
final stopwatch = Stopwatch()..start();
_getPartnerAssetsInProgress = true;
if (partner == null) {
await _userService.refreshUsers();
final List<User> partners =
await _db.users.filter().isPartnerSharedWithEqualTo(true).findAll();
for (User u in partners) {
await _assetService.refreshRemoteAssets(u);
}
} else {
await _assetService.refreshRemoteAssets(partner);
}
log.info("Load partner assets: ${stopwatch.elapsedMilliseconds}ms");
} finally {
_getPartnerAssetsInProgress = false;
}
}
Future<void> clearAllAsset() {
return clearAssetsAndAlbums(_db);
}
@ -321,6 +307,7 @@ final assetProvider = StateNotifierProvider<AssetNotifier, bool>((ref) {
ref.watch(userServiceProvider),
ref.watch(syncServiceProvider),
ref.watch(dbProvider),
ref,
);
});

View file

@ -43,7 +43,7 @@ class TabNavigationObserver extends AutoRouterObserver {
if (route.name == 'SharingRoute') {
ref.read(sharedAlbumProvider.notifier).getAllSharedAlbums();
ref.read(assetProvider.notifier).getPartnerAssets();
Future(() => ref.read(assetProvider.notifier).getAllAsset());
}
if (route.name == 'LibraryRoute') {

View file

@ -23,6 +23,7 @@ class ApiService {
late PersonApi personApi;
late AuditApi auditApi;
late SharedLinkApi sharedLinkApi;
late SyncApi syncApi;
late SystemConfigApi systemConfigApi;
late ActivityApi activityApi;
late DownloadApi downloadApi;
@ -53,6 +54,7 @@ class ApiService {
personApi = PersonApi(_apiClient);
auditApi = AuditApi(_apiClient);
sharedLinkApi = SharedLinkApi(_apiClient);
syncApi = SyncApi(_apiClient);
systemConfigApi = SystemConfigApi(_apiClient);
activityApi = ActivityApi(_apiClient);
downloadApi = DownloadApi(_apiClient);

View file

@ -5,13 +5,14 @@ import 'dart:async';
import 'package:flutter/material.dart';
import 'package:hooks_riverpod/hooks_riverpod.dart';
import 'package:immich_mobile/entities/asset.entity.dart';
import 'package:immich_mobile/entities/etag.entity.dart';
import 'package:immich_mobile/entities/exif_info.entity.dart';
import 'package:immich_mobile/entities/store.entity.dart';
import 'package:immich_mobile/entities/user.entity.dart';
import 'package:immich_mobile/providers/api.provider.dart';
import 'package:immich_mobile/providers/db.provider.dart';
import 'package:immich_mobile/services/api.service.dart';
import 'package:immich_mobile/services/sync.service.dart';
import 'package:immich_mobile/services/user.service.dart';
import 'package:isar/isar.dart';
import 'package:logging/logging.dart';
import 'package:maplibre_gl/maplibre_gl.dart';
@ -21,6 +22,7 @@ final assetServiceProvider = Provider(
(ref) => AssetService(
ref.watch(apiServiceProvider),
ref.watch(syncServiceProvider),
ref.watch(userServiceProvider),
ref.watch(dbProvider),
),
);
@ -28,24 +30,33 @@ final assetServiceProvider = Provider(
class AssetService {
final ApiService _apiService;
final SyncService _syncService;
final UserService _userService;
final log = Logger('AssetService');
final Isar _db;
AssetService(
this._apiService,
this._syncService,
this._userService,
this._db,
);
/// Checks the server for updated assets and updates the local database if
/// required. Returns `true` if there were any changes.
Future<bool> refreshRemoteAssets([User? user]) async {
user ??= Store.get<User>(StoreKey.currentUser);
Future<bool> refreshRemoteAssets() async {
final syncedUserIds = await _db.eTags.where().idProperty().findAll();
final List<User> syncedUsers = syncedUserIds.isEmpty
? []
: await _db.users
.where()
.anyOf(syncedUserIds, (q, id) => q.idEqualTo(id))
.findAll();
final Stopwatch sw = Stopwatch()..start();
final bool changes = await _syncService.syncRemoteAssetsToDb(
user,
_getRemoteAssetChanges,
_getRemoteAssets,
users: syncedUsers,
getChangedAssets: _getRemoteAssetChanges,
loadAssets: _getRemoteAssets,
refreshUsers: _userService.getUsersFromServer,
);
debugPrint("refreshRemoteAssets full took ${sw.elapsedMilliseconds}ms");
return changes;
@ -53,14 +64,15 @@ class AssetService {
/// Returns `(null, null)` if changes are invalid -> requires full sync
Future<(List<Asset>? toUpsert, List<String>? toDelete)>
_getRemoteAssetChanges(User user, DateTime since) async {
final deleted = await _apiService.auditApi
.getAuditDeletes(since, EntityType.ASSET, userId: user.id);
if (deleted == null || deleted.needsFullSync) return (null, null);
final assetDto = await _apiService.assetApi
.getAllAssets(userId: user.id, updatedAfter: since);
if (assetDto == null) return (null, null);
return (assetDto.map(Asset.remote).toList(), deleted.ids);
_getRemoteAssetChanges(List<User> users, DateTime since) async {
final dto = AssetDeltaSyncDto(
updatedAfter: since,
userIds: users.map((e) => e.id).toList(),
);
final changes = await _apiService.syncApi.getDeltaSync(dto);
return changes == null || changes.needsFullSync
? (null, null)
: (changes.upserted.map(Asset.remote).toList(), changes.deleted);
}
/// Returns the list of people of the given asset id.
@ -85,38 +97,32 @@ class AssetService {
}
/// Returns `null` if the server state did not change, else list of assets
Future<List<Asset>?> _getRemoteAssets(User user) async {
Future<List<Asset>?> _getRemoteAssets(User user, DateTime until) async {
const int chunkSize = 10000;
try {
final DateTime now = DateTime.now().toUtc();
final List<Asset> allAssets = [];
for (int i = 0;; i += chunkSize) {
final List<AssetResponseDto>? assets =
await _apiService.assetApi.getAllAssets(
DateTime? lastCreationDate;
String? lastId;
// will break on error or once all assets are loaded
while (true) {
final dto = AssetFullSyncDto(
limit: chunkSize,
updatedUntil: until,
lastId: lastId,
lastCreationDate: lastCreationDate,
userId: user.id,
// updatedBefore is important! without it we could
// a) get the same Asset multiple times in different versions (when
// the asset is modified while the chunks are loaded from the server)
// b) miss assets when new assets are inserted in between the calls
updatedBefore: now,
skip: i,
take: chunkSize,
);
if (assets == null) {
return null;
}
final List<AssetResponseDto>? assets =
await _apiService.syncApi.getFullSyncForUser(dto);
if (assets == null) return null;
allAssets.addAll(assets.map(Asset.remote));
if (assets.length < chunkSize) {
break;
}
if (assets.isEmpty) break;
lastCreationDate = assets.last.fileCreatedAt;
lastId = assets.last.id;
}
return allAssets;
} catch (error, stack) {
log.severe(
'Error while getting remote assets',
error,
stack,
);
log.severe('Error while getting remote assets', error, stack);
return null;
}
}

View file

@ -37,12 +37,16 @@ class MemoryService {
List<Memory> memories = [];
for (final MemoryLaneResponseDto(:title, :assets) in data) {
memories.add(
Memory(
title: title,
assets: await _db.assets.getAllByRemoteId(assets.map((e) => e.id)),
),
);
final dbAssets =
await _db.assets.getAllByRemoteId(assets.map((e) => e.id));
if (dbAssets.isNotEmpty) {
memories.add(
Memory(
title: title,
assets: dbAssets,
),
);
}
}
return memories.isNotEmpty ? memories : null;

View file

@ -40,18 +40,20 @@ class SyncService {
/// Syncs remote assets owned by the logged-in user to the DB
/// Returns `true` if there were any changes
Future<bool> syncRemoteAssetsToDb(
User user,
Future<(List<Asset>? toUpsert, List<String>? toDelete)> Function(
User user,
Future<bool> syncRemoteAssetsToDb({
required List<User> users,
required Future<(List<Asset>? toUpsert, List<String>? toDelete)> Function(
List<User> users,
DateTime since,
) getChangedAssets,
FutureOr<List<Asset>?> Function(User user) loadAssets,
) =>
required FutureOr<List<Asset>?> Function(User user, DateTime until)
loadAssets,
required FutureOr<List<User>?> Function() refreshUsers,
}) =>
_lock.run(
() async =>
await _syncRemoteAssetChanges(user, getChangedAssets) ??
await _syncRemoteAssetsFull(user, loadAssets),
await _syncRemoteAssetChanges(users, getChangedAssets) ??
await _syncRemoteAssetsFull(refreshUsers, loadAssets),
);
/// Syncs remote albums to the database
@ -111,7 +113,8 @@ class SyncService {
both: (User a, User b) {
if (!a.updatedAt.isAtSameMomentAs(b.updatedAt) ||
a.isPartnerSharedBy != b.isPartnerSharedBy ||
a.isPartnerSharedWith != b.isPartnerSharedWith) {
a.isPartnerSharedWith != b.isPartnerSharedWith ||
a.inTimeline != b.inTimeline) {
toUpsert.add(a);
return true;
}
@ -149,17 +152,22 @@ class SyncService {
/// Efficiently syncs assets via changes. Returns `null` when a full sync is required.
Future<bool?> _syncRemoteAssetChanges(
User user,
List<User> users,
Future<(List<Asset>? toUpsert, List<String>? toDelete)> Function(
User user,
List<User> users,
DateTime since,
) getChangedAssets,
) async {
final DateTime? since = _db.eTags.getByIdSync(user.id)?.time?.toUtc();
final currentUser = Store.get(StoreKey.currentUser);
final DateTime? since =
_db.eTags.getSync(currentUser.isarId)?.time?.toUtc();
if (since == null) return null;
final DateTime now = DateTime.now();
final (toUpsert, toDelete) = await getChangedAssets(user, since);
if (toUpsert == null || toDelete == null) return null;
final (toUpsert, toDelete) = await getChangedAssets(users, since);
if (toUpsert == null || toDelete == null) {
await _clearUserAssetsETag(users);
return null;
}
try {
if (toDelete.isNotEmpty) {
await handleRemoteAssetRemoval(toDelete);
@ -169,7 +177,7 @@ class SyncService {
await upsertAssetsWithExif(updated);
}
if (toUpsert.isNotEmpty || toDelete.isNotEmpty) {
await _updateUserAssetsETag(user, now);
await _updateUserAssetsETag(users, now);
return true;
}
return false;
@ -203,11 +211,34 @@ class SyncService {
/// Syncs assets by loading and comparing all assets from the server.
Future<bool> _syncRemoteAssetsFull(
FutureOr<List<User>?> Function() refreshUsers,
FutureOr<List<Asset>?> Function(User user, DateTime until) loadAssets,
) async {
final serverUsers = await refreshUsers();
if (serverUsers == null) {
_log.warning("_syncRemoteAssetsFull aborted because user refresh failed");
return false;
}
await _syncUsersFromServer(serverUsers);
final List<User> users = await _db.users
.filter()
.isPartnerSharedWithEqualTo(true)
.or()
.isarIdEqualTo(Store.get(StoreKey.currentUser).isarId)
.findAll();
bool changes = false;
for (User u in users) {
changes |= await _syncRemoteAssetsForUser(u, loadAssets);
}
return changes;
}
Future<bool> _syncRemoteAssetsForUser(
User user,
FutureOr<List<Asset>?> Function(User user) loadAssets,
FutureOr<List<Asset>?> Function(User user, DateTime until) loadAssets,
) async {
final DateTime now = DateTime.now().toUtc();
final List<Asset>? remote = await loadAssets(user);
final List<Asset>? remote = await loadAssets(user, now);
if (remote == null) {
return false;
}
@ -225,7 +256,7 @@ class SyncService {
final (toAdd, toUpdate, toRemove) = _diffAssets(remote, inDb, remote: true);
if (toAdd.isEmpty && toUpdate.isEmpty && toRemove.isEmpty) {
await _updateUserAssetsETag(user, now);
await _updateUserAssetsETag([user], now);
return false;
}
final idsToDelete = toRemove.map((e) => e.id).toList();
@ -235,12 +266,19 @@ class SyncService {
} on IsarError catch (e) {
_log.severe("Failed to sync remote assets to db", e);
}
await _updateUserAssetsETag(user, now);
await _updateUserAssetsETag([user], now);
return true;
}
Future<void> _updateUserAssetsETag(User user, DateTime time) =>
_db.writeTxn(() => _db.eTags.put(ETag(id: user.id, time: time)));
Future<void> _updateUserAssetsETag(List<User> users, DateTime time) {
final etags = users.map((u) => ETag(id: u.id, time: time)).toList();
return _db.writeTxn(() => _db.eTags.putAll(etags));
}
Future<void> _clearUserAssetsETag(List<User> users) {
final ids = users.map((u) => u.id).toList();
return _db.writeTxn(() => _db.eTags.deleteAllById(ids));
}
/// Syncs remote albums to the database
/// returns `true` if there were any changes

View file

@ -70,7 +70,7 @@ class UserService {
}
}
Future<bool> refreshUsers() async {
Future<List<User>?> getUsersFromServer() async {
final List<User>? users = await _getAllUsers(isAll: true);
final List<User>? sharedBy =
await _partnerService.getPartners(PartnerDirection.sharedBy);
@ -79,7 +79,7 @@ class UserService {
if (users == null || sharedBy == null || sharedWith == null) {
_log.warning("Failed to refresh users");
return false;
return null;
}
users.sortBy((u) => u.id);
@ -108,6 +108,12 @@ class UserService {
onlySecond: (_) {},
);
return users;
}
Future<bool> refreshUsers() async {
final users = await getUsersFromServer();
if (users == null) return false;
return _syncService.syncUsersFromServer(users);
}
}

View file

@ -4,17 +4,12 @@ import 'package:immich_mobile/entities/store.entity.dart';
import 'package:immich_mobile/utils/db.dart';
import 'package:isar/isar.dart';
const int targetVersion = 6;
Future<void> migrateDatabaseIfNeeded(Isar db) async {
final int version = Store.get(StoreKey.version, 1);
switch (version) {
case 1:
await _migrateTo(db, 2);
case 2:
await _migrateTo(db, 3);
case 3:
await _migrateTo(db, 4);
case 4:
await _migrateTo(db, 5);
if (version < targetVersion) {
_migrateTo(db, targetVersion);
}
}

View file

@ -75,8 +75,12 @@ void main() {
makeAsset(checksum: "c", remoteId: "1-1"),
];
expect(db.assets.countSync(), 5);
final bool c1 =
await s.syncRemoteAssetsToDb(owner, _failDiff, (u) => remoteAssets);
final bool c1 = await s.syncRemoteAssetsToDb(
users: [owner],
getChangedAssets: _failDiff,
loadAssets: (u, d) => remoteAssets,
refreshUsers: () => [owner],
);
expect(c1, isFalse);
expect(db.assets.countSync(), 5);
});
@ -92,8 +96,12 @@ void main() {
makeAsset(checksum: "g", remoteId: "3-1"),
];
expect(db.assets.countSync(), 5);
final bool c1 =
await s.syncRemoteAssetsToDb(owner, _failDiff, (u) => remoteAssets);
final bool c1 = await s.syncRemoteAssetsToDb(
users: [owner],
getChangedAssets: _failDiff,
loadAssets: (u, d) => remoteAssets,
refreshUsers: () => [owner],
);
expect(c1, isTrue);
expect(db.assets.countSync(), 7);
});
@ -109,23 +117,39 @@ void main() {
makeAsset(checksum: "j", remoteId: "2-1d"),
];
expect(db.assets.countSync(), 5);
final bool c1 =
await s.syncRemoteAssetsToDb(owner, _failDiff, (u) => remoteAssets);
final bool c1 = await s.syncRemoteAssetsToDb(
users: [owner],
getChangedAssets: _failDiff,
loadAssets: (u, d) => remoteAssets,
refreshUsers: () => [owner],
);
expect(c1, isTrue);
expect(db.assets.countSync(), 8);
final bool c2 =
await s.syncRemoteAssetsToDb(owner, _failDiff, (u) => remoteAssets);
final bool c2 = await s.syncRemoteAssetsToDb(
users: [owner],
getChangedAssets: _failDiff,
loadAssets: (u, d) => remoteAssets,
refreshUsers: () => [owner],
);
expect(c2, isFalse);
expect(db.assets.countSync(), 8);
remoteAssets.removeAt(4);
final bool c3 =
await s.syncRemoteAssetsToDb(owner, _failDiff, (u) => remoteAssets);
final bool c3 = await s.syncRemoteAssetsToDb(
users: [owner],
getChangedAssets: _failDiff,
loadAssets: (u, d) => remoteAssets,
refreshUsers: () => [owner],
);
expect(c3, isTrue);
expect(db.assets.countSync(), 7);
remoteAssets.add(makeAsset(checksum: "k", remoteId: "2-1e"));
remoteAssets.add(makeAsset(checksum: "l", remoteId: "2-2"));
final bool c4 =
await s.syncRemoteAssetsToDb(owner, _failDiff, (u) => remoteAssets);
final bool c4 = await s.syncRemoteAssetsToDb(
users: [owner],
getChangedAssets: _failDiff,
loadAssets: (u, d) => remoteAssets,
refreshUsers: () => [owner],
);
expect(c4, isTrue);
expect(db.assets.countSync(), 9);
});
@ -140,9 +164,10 @@ void main() {
toUpsert[0].isFavorite = true;
final List<String> toDelete = ["2-1", "1-1"];
final bool c = await s.syncRemoteAssetsToDb(
owner,
(user, since) async => (toUpsert, toDelete),
(user) => throw Exception(),
users: [owner],
getChangedAssets: (user, since) async => (toUpsert, toDelete),
loadAssets: (user, date) => throw Exception(),
refreshUsers: () => throw Exception(),
);
expect(c, isTrue);
expect(db.assets.countSync(), 6);
@ -150,5 +175,8 @@ void main() {
});
}
Future<(List<Asset>?, List<String>?)> _failDiff(User user, DateTime time) =>
Future<(List<Asset>?, List<String>?)> _failDiff(
List<User> user,
DateTime time,
) =>
Future.value((null, null));