From 911c35a7f14c34c093804435024bc276c7ff0d99 Mon Sep 17 00:00:00 2001 From: Fynn Petersen-Frey Date: Thu, 9 Feb 2023 18:32:08 +0100 Subject: [PATCH] refactor(mobile): add Isar DB & Store class (#1574) * refactor(mobile): add Isar DB & Store class new Store: globally accessible key-value store like Hive (but based on Isar) replace first few places of Hive usage with the new Store * reduce max. DB size to prevent errors on older iOS devices --------- Co-authored-by: Alex --- .../test_utils/general_helper.dart | 8 +- mobile/lib/main.dart | 27 ++++- .../providers/authentication.provider.dart | 6 +- mobile/lib/shared/models/store.dart | 96 ++++++++++++++++++ mobile/lib/shared/models/store.g.dart | Bin 0 -> 16260 bytes .../lib/shared/providers/asset.provider.dart | 9 +- mobile/lib/shared/providers/db.provider.dart | 5 + mobile/lib/shared/services/asset.service.dart | 19 ++-- mobile/lib/utils/migration.dart | 24 +++++ mobile/pubspec.lock | 42 ++++++++ mobile/pubspec.yaml | 4 + 11 files changed, 222 insertions(+), 18 deletions(-) create mode 100644 mobile/lib/shared/models/store.dart create mode 100644 mobile/lib/shared/models/store.g.dart create mode 100644 mobile/lib/shared/providers/db.provider.dart create mode 100644 mobile/lib/utils/migration.dart diff --git a/mobile/integration_test/test_utils/general_helper.dart b/mobile/integration_test/test_utils/general_helper.dart index 0ce776ce97..24b98c9617 100644 --- a/mobile/integration_test/test_utils/general_helper.dart +++ b/mobile/integration_test/test_utils/general_helper.dart @@ -1,7 +1,9 @@ import 'package:easy_localization/easy_localization.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:hive/hive.dart'; +import 'package:immich_mobile/shared/models/store.dart'; import 'package:integration_test/integration_test.dart'; +import 'package:isar/isar.dart'; // ignore: depend_on_referenced_packages import 'package:meta/meta.dart'; import 'package:immich_mobile/main.dart' as app; @@ -34,8 +36,12 @@ class ImmichTestHelper { // Clear all data from Hive await Hive.deleteFromDisk(); await app.openBoxes(); + // Clear all data from Isar (reuse existing instance if available) + final db = Isar.getInstance() ?? await app.loadDb(); + await Store.clear(); + await db.writeTxn(() => db.clear()); // Load main Widget - await tester.pumpWidget(app.getMainWidget()); + await tester.pumpWidget(app.getMainWidget(db)); // Post run tasks await tester.pumpAndSettle(); await EasyLocalization.ensureInitialized(); diff --git a/mobile/lib/main.dart b/mobile/lib/main.dart index 262612e44d..fe96a5d274 100644 --- a/mobile/lib/main.dart +++ b/mobile/lib/main.dart @@ -17,8 +17,10 @@ import 'package:immich_mobile/modules/login/providers/authentication.provider.da import 'package:immich_mobile/routing/router.dart'; import 'package:immich_mobile/routing/tab_navigation_observer.dart'; import 'package:immich_mobile/shared/models/immich_logger_message.model.dart'; +import 'package:immich_mobile/shared/models/store.dart'; import 'package:immich_mobile/shared/providers/app_state.provider.dart'; import 'package:immich_mobile/shared/providers/asset.provider.dart'; +import 'package:immich_mobile/shared/providers/db.provider.dart'; import 'package:immich_mobile/shared/providers/release_info.provider.dart'; import 'package:immich_mobile/shared/providers/server_info.provider.dart'; import 'package:immich_mobile/shared/providers/websocket.provider.dart'; @@ -26,11 +28,16 @@ import 'package:immich_mobile/shared/services/immich_logger.service.dart'; import 'package:immich_mobile/shared/views/immich_loading_overlay.dart'; import 'package:immich_mobile/shared/views/version_announcement_overlay.dart'; import 'package:immich_mobile/utils/immich_app_theme.dart'; +import 'package:immich_mobile/utils/migration.dart'; +import 'package:isar/isar.dart'; +import 'package:path_provider/path_provider.dart'; import 'constants/hive_box.dart'; void main() async { await initApp(); - runApp(getMainWidget()); + final db = await loadDb(); + await migrateHiveToStoreIfNecessary(); + runApp(getMainWidget(db)); } Future openBoxes() async { @@ -70,13 +77,27 @@ Future initApp() async { ImmichLogger().init(); } -Widget getMainWidget() { +Future loadDb() async { + final dir = await getApplicationDocumentsDirectory(); + Isar db = await Isar.open( + [StoreValueSchema], + directory: dir.path, + maxSizeMiB: 256, + ); + Store.init(db); + return db; +} + +Widget getMainWidget(Isar db) { return EasyLocalization( supportedLocales: locales, path: translationsPath, useFallbackTranslations: true, fallbackLocale: locales.first, - child: const ProviderScope(child: ImmichApp()), + child: ProviderScope( + overrides: [dbProvider.overrideWithValue(db)], + child: const ImmichApp(), + ), ); } diff --git a/mobile/lib/modules/login/providers/authentication.provider.dart b/mobile/lib/modules/login/providers/authentication.provider.dart index f5f8481c59..88b49a4b10 100644 --- a/mobile/lib/modules/login/providers/authentication.provider.dart +++ b/mobile/lib/modules/login/providers/authentication.provider.dart @@ -4,6 +4,7 @@ import 'package:hive/hive.dart'; import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/constants/hive_box.dart'; import 'package:immich_mobile/modules/album/services/album_cache.service.dart'; +import 'package:immich_mobile/shared/models/store.dart'; import 'package:immich_mobile/shared/services/asset_cache.service.dart'; import 'package:immich_mobile/modules/login/models/authentication_state.model.dart'; import 'package:immich_mobile/modules/login/models/hive_saved_login_info.model.dart'; @@ -94,7 +95,8 @@ class AuthenticationNotifier extends StateNotifier { await Future.wait([ _apiService.authenticationApi.logout(), Hive.box(userInfoBox).delete(accessTokenKey), - Hive.box(userInfoBox).delete(assetEtagKey), + Store.delete(StoreKey.assetETag), + Store.delete(StoreKey.userRemoteId), _assetCacheService.invalidate(), _albumCacheService.invalidate(), _sharedAlbumCacheService.invalidate(), @@ -153,7 +155,7 @@ class AuthenticationNotifier extends StateNotifier { var deviceInfo = await _deviceInfoService.getDeviceInfo(); userInfoHiveBox.put(deviceIdKey, deviceInfo["deviceId"]); userInfoHiveBox.put(accessTokenKey, accessToken); - userInfoHiveBox.put(userIdKey, userResponseDto.id); + Store.put(StoreKey.userRemoteId, userResponseDto.id); state = state.copyWith( isAuthenticated: true, diff --git a/mobile/lib/shared/models/store.dart b/mobile/lib/shared/models/store.dart new file mode 100644 index 0000000000..537ac5443b --- /dev/null +++ b/mobile/lib/shared/models/store.dart @@ -0,0 +1,96 @@ +import 'package:isar/isar.dart'; +import 'dart:convert'; + +part 'store.g.dart'; + +/// Key-value store for individual items enumerated in StoreKey. +/// Supports String, int and JSON-serializable Objects +/// Can be used concurrently from multiple isolates +class Store { + static late final Isar _db; + static final List _cache = List.filled(StoreKey.values.length, null); + + /// Initializes the store (call exactly once per app start) + static void init(Isar db) { + _db = db; + _populateCache(); + _db.storeValues.where().build().watch().listen(_onChangeListener); + } + + /// clears all values from this store (cache and DB), only for testing! + static Future clear() { + _cache.fillRange(0, _cache.length, null); + return _db.writeTxn(() => _db.storeValues.clear()); + } + + /// Returns the stored value for the given key, or the default value if null + static T? get(StoreKey key, [T? defaultValue]) => + _cache[key._id] ?? defaultValue; + + /// Stores the value synchronously in the cache and asynchronously in the DB + static Future put(StoreKey key, T value) { + _cache[key._id] = value; + return _db.writeTxn(() => _db.storeValues.put(StoreValue._of(value, key))); + } + + /// Removes the value synchronously from the cache and asynchronously from the DB + static Future delete(StoreKey key) { + _cache[key._id] = null; + return _db.writeTxn(() => _db.storeValues.delete(key._id)); + } + + /// Fills the cache with the values from the DB + static _populateCache() { + for (StoreKey key in StoreKey.values) { + final StoreValue? value = _db.storeValues.getSync(key._id); + if (value != null) { + _cache[key._id] = value._extract(key); + } + } + } + + /// updates the state if a value is updated in any isolate + static void _onChangeListener(List? data) { + if (data != null) { + for (StoreValue value in data) { + _cache[value.id] = value._extract(StoreKey.values[value.id]); + } + } + } +} + +/// Internal class for `Store`, do not use elsewhere. +@Collection(inheritance: false) +class StoreValue { + StoreValue(this.id, {this.intValue, this.strValue}); + Id id; + int? intValue; + String? strValue; + + T? _extract(StoreKey key) => key._isInt + ? intValue + : (key._fromJson != null + ? key._fromJson!(json.decode(strValue!)) + : strValue); + static StoreValue _of(dynamic value, StoreKey key) => StoreValue( + key._id, + intValue: key._isInt ? value : null, + strValue: key._isInt + ? null + : (key._fromJson == null ? value : json.encode(value.toJson())), + ); +} + +/// Key for each possible value in the `Store`. +/// Defines the data type (int, String, JSON) for each value +enum StoreKey { + userRemoteId(0), + assetETag(1), + ; + + // ignore: unused_element + const StoreKey(this._id, [this._isInt = false, this._fromJson]); + final int _id; + final bool _isInt; + final Function(dynamic)? _fromJson; +} diff --git a/mobile/lib/shared/models/store.g.dart b/mobile/lib/shared/models/store.g.dart new file mode 100644 index 0000000000000000000000000000000000000000..6370573a68a6fd900d6d94b9dd64115f539d978d GIT binary patch literal 16260 zcmds8ZExE~68^4Vu?HL=xm6WAxd4e`>k!9oD!{Sb+HFu20YQ^1iwR9C%avW-!TL{6FWFz$0z6P)5$^q z=Rev0pX`_S#|MqZ$QP2O1J=xBDtITvWAk03@!|#hVQGIfT0ir}hcu3PAfq(-$P+Gn zNL;cY8ZJn$G4>_zMwbbu^wS`UIhE^2Z;%R)in=TukK!osCHHhynH2R2IE8l?m@v6a|QNgU0{DqbI7v@L%2!`|HjLtfP_j#GLB;?!-H?A z_?kwEC%7Do#Jh>)(DSnlXkKB=e1?N4@ng&;@MHAwuT$XkHS*3o zRsW3b^_Uz+StrOD+S+>8xNS6)by(5$3=_Zd*&Zuo>Y^3|5>2{HG>hiVHj1OL%ieCk zeDn70%Qvt8_4eh?%U7>=wy6d*8F3*ao^_SR44naoQmWZyr@DGlNMQJ*;Z$#Wup;wb zG%Wqe$q2d-7U%d^hq$Dxw)BsH!;6On?bz|GpuCr@s0h-OF`}T$E&Y>cGP;7JokjnF zQlOQ!+@VuJ4Kpqx_)(?RnJuLzA+K+AP?T4i5sW3NPE)F*=23WTj2DI4XyE4G$}VaK z@U(AaBt0FtbO`EdyX3_nRUfu-rU#bxLmD1cM|y;EttzAjzLb72EOmabil`Ftj7Z;Q z%~zf6&THrgayDRcSC$JUk+C2lOE1167|}>aQ6_iMv4@a9$T%Dig`fP}vy}*#8d=?SH z47*W3i=w4R>8PJ0IA6=4_1E^7wnfD{Ev(6mSl!`F#ofHiPSM2zsiTu)V~1A`y=lJv z5bUPu&>Vl`euy~=^|R<4wW+mki8dfj>5&eTAt44s z)+&awJjVinv+b@zC+Bc$8ai8B%+J`VCco;wGo^bCR;2+#KJdq}v?Iu&NN*H94uwcX ztNDBKElF>Z90W$G9rk4ZEauWwpx;uG1j8Kqsi4;?$Y9v6=Uy^WCkmCFR8%5%QaIcP zHoM^j`6dc_y~@BUUOq7PUteltfDWfE5>w>vToBfFaof0xL*$a~CYAw)Jz$}P6bq7K zzQ2ySn0y{$Eq0a)scei@i7MS6M={*bS&FnBVQu!S>ef?{^l=^D4-m3UFF%>|!<4d;oDgA@bf^oEOW9ulc&Ji^+~Evljk zuw3ii&zgb+#e|Pjxs*9muGXBmv6NGnj&;l6SDvddlz)QqkIFXWBn90gK(uS**siQII z&p0$P>mSl+IY`W}*!*D3iGGfRl5E2P9_Al-Y zV$3zm+*%cNoZ6L8t<13d(no1d6le^fCP=wY)?4yCW9Off+O#%7)qS#ks|g89y?fL9 z&rysZss?}bYDf3^kS3wp^qD!w-|1iEc{5A!K)q-#D>_spwkppZ?9HiVMh>!FFVnV= z*;ydF;hmmdkh}WlKD;r?4Dqv~OAm_;H-;*eEFN`UU_&WsvowxZl)3|$8%uN03>UFB z(s|$FlW&+!)&plAcQ+HuJpOJxkhQT|TNw;hk*bcNYl_nkarjQOzSrqh{O;=1RFK-R z{Lw-DPR|)aIH<8uNHJSE{Y>rLtSYSP_st6Kok$DK*a@PR@!m&P@urHPcD*xW3G+U*@QzY90ia#i^-$}fH&R|p**q&YLT?fQD{mN;0 z1CgCO5w8{2Qc_l^s&u6Ts(SwaO3Zh`e)j3v9-M4kdbWzGqg8tSSxI`fQZ_U_TlH21 z0xmS@1N~BZK2t}Db-9O5kydq_V@GJ8p&#A~ykpwBw%i2Y=%H0;Hd)O>!F{^a8t z7O&zw!j(%d61zm)0Hkq4*46Vtpl@)QDDVZYoL1(tjsRi^YeQT=JG>f6+z>JWdeLL~ z%3koDbEG-nC7St~b@eZ#PYX!xdj4_A=zdV99RQ!Go4$$$3mgDXlb|mDtx^D8YTBR7 zCPDX^`RLkq%aq8z*^645yXfIt&esIEXlSo&>B~CvUDB!78+U7i)$1NY%0ky@%E}|( zs=fN&FxC~hzVkscmr0`g`5bVge=#6 ztgfrEVWDr0^$^}>I03dUV-$XNajLBMir~&q5;RGc47&CA=M$$i=vMF5fWBayQa6qd z@HG^+WM{miZ+p!WsER(Jk%>sPY-h4U_9t%QpEqOL(R!><*_zb8vOd8}lT|}a$}xKB zow%~?Uzzf>e06F*^O { _getAllAssetInProgress = true; bool isCacheValid = await _assetCacheService.isValid(); stopwatch.start(); - final Box box = Hive.box(userInfoBox); if (isCacheValid && state.allAssets.isEmpty) { final List? cachedData = await _assetCacheService.get(); if (cachedData == null) { @@ -122,7 +122,7 @@ class AssetNotifier extends StateNotifier { } final localTask = _assetService.getLocalAssets(urgent: !isCacheValid); final remoteTask = _assetService.getRemoteAssets( - etag: isCacheValid ? box.get(assetEtagKey) : null, + etag: isCacheValid ? Store.get(StoreKey.assetETag) : null, ); int remoteBegin = state.allAssets.indexWhere((a) => a.isRemote); @@ -151,7 +151,7 @@ class AssetNotifier extends StateNotifier { log.info("Combining assets: ${stopwatch.elapsedMilliseconds}ms"); - box.put(assetEtagKey, remoteResult.second); + Store.put(StoreKey.assetETag, remoteResult.second); } finally { _getAllAssetInProgress = false; } @@ -279,8 +279,7 @@ class AssetNotifier extends StateNotifier { final index = state.allAssets.indexWhere((a) => asset.id == a.id); if (index > 0) { - state.allAssets.removeAt(index); - state.allAssets.insert(index, Asset.remote(newAsset)); + state.allAssets[index] = newAsset; _updateAssetsState(state.allAssets); } diff --git a/mobile/lib/shared/providers/db.provider.dart b/mobile/lib/shared/providers/db.provider.dart new file mode 100644 index 0000000000..e03e037f36 --- /dev/null +++ b/mobile/lib/shared/providers/db.provider.dart @@ -0,0 +1,5 @@ +import 'package:hooks_riverpod/hooks_riverpod.dart'; +import 'package:isar/isar.dart'; + +// overwritten in main.dart due to async loading +final dbProvider = Provider((_) => throw UnimplementedError()); diff --git a/mobile/lib/shared/services/asset.service.dart b/mobile/lib/shared/services/asset.service.dart index 0cc04936fb..91e3a015db 100644 --- a/mobile/lib/shared/services/asset.service.dart +++ b/mobile/lib/shared/services/asset.service.dart @@ -8,6 +8,7 @@ import 'package:immich_mobile/modules/backup/background_service/background.servi import 'package:immich_mobile/modules/backup/models/hive_backup_albums.model.dart'; import 'package:immich_mobile/modules/backup/services/backup.service.dart'; import 'package:immich_mobile/shared/models/asset.dart'; +import 'package:immich_mobile/shared/models/store.dart'; import 'package:immich_mobile/shared/providers/api.provider.dart'; import 'package:immich_mobile/shared/services/api.service.dart'; import 'package:immich_mobile/utils/openapi_extensions.dart'; @@ -37,7 +38,7 @@ class AssetService { final Pair, String?>? remote = await _apiService.assetApi.getAllAssetsWithETag(eTag: etag); if (remote == null) { - return const Pair(null, null); + return Pair(null, etag); } return Pair( remote.first.map(Asset.remote).toList(growable: false), @@ -45,7 +46,7 @@ class AssetService { ); } catch (e, stack) { log.severe('Error while getting remote assets', e, stack); - return const Pair(null, null); + return Pair(null, etag); } } @@ -62,7 +63,7 @@ class AssetService { } final box = await Hive.openBox(hiveBackupInfoBox); final HiveBackupAlbums? backupAlbumInfo = box.get(backupInfoKey); - final String userId = Hive.box(userInfoBox).get(userIdKey); + final String userId = Store.get(StoreKey.userRemoteId); if (backupAlbumInfo != null) { return (await _backupService .buildUploadCandidates(backupAlbumInfo.deepCopy())) @@ -105,12 +106,16 @@ class AssetService { } } - Future updateAsset(Asset asset, UpdateAssetDto updateAssetDto) async { - return await _apiService.assetApi.updateAsset(asset.id, updateAssetDto); + Future updateAsset( + Asset asset, + UpdateAssetDto updateAssetDto, + ) async { + final dto = + await _apiService.assetApi.updateAsset(asset.remoteId!, updateAssetDto); + return dto == null ? null : Asset.remote(dto); } - Future changeFavoriteStatus(Asset asset, bool isFavorite) { + Future changeFavoriteStatus(Asset asset, bool isFavorite) { return updateAsset(asset, UpdateAssetDto(isFavorite: isFavorite)); } - } diff --git a/mobile/lib/utils/migration.dart b/mobile/lib/utils/migration.dart new file mode 100644 index 0000000000..d74dff9027 --- /dev/null +++ b/mobile/lib/utils/migration.dart @@ -0,0 +1,24 @@ +import 'package:flutter/cupertino.dart'; +import 'package:hive/hive.dart'; +import 'package:immich_mobile/constants/hive_box.dart'; +import 'package:immich_mobile/shared/models/store.dart'; + +Future migrateHiveToStoreIfNecessary() async { + try { + if (await Hive.boxExists(userInfoBox)) { + final Box box = await Hive.openBox(userInfoBox); + await _migrateSingleKey(box, userIdKey, StoreKey.userRemoteId); + await _migrateSingleKey(box, assetEtagKey, StoreKey.assetETag); + } + } catch (e) { + debugPrint("Error while migrating userInfoBox $e"); + } +} + +_migrateSingleKey(Box box, String hiveKey, StoreKey key) async { + final String? value = box.get(hiveKey); + if (value != null) { + await Store.put(key, value); + await box.delete(hiveKey); + } +} diff --git a/mobile/pubspec.lock b/mobile/pubspec.lock index 54345703c2..24e49e8190 100644 --- a/mobile/pubspec.lock +++ b/mobile/pubspec.lock @@ -239,6 +239,13 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "2.2.3" + dartx: + dependency: transitive + description: + name: dartx + url: "https://pub.dartlang.org" + source: hosted + version: "1.1.0" easy_image_viewer: dependency: "direct main" description: @@ -547,6 +554,27 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "1.0.3" + isar: + dependency: "direct main" + description: + name: isar + url: "https://pub.dartlang.org" + source: hosted + version: "3.0.5" + isar_flutter_libs: + dependency: "direct main" + description: + name: isar_flutter_libs + url: "https://pub.dartlang.org" + source: hosted + version: "3.0.5" + isar_generator: + dependency: "direct dev" + description: + name: isar_generator + url: "https://pub.dartlang.org" + source: hosted + version: "3.0.5" js: dependency: transitive description: @@ -1063,6 +1091,13 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "0.4.12" + time: + dependency: transitive + description: + name: time + url: "https://pub.dartlang.org" + source: hosted + version: "2.1.3" timing: dependency: transitive description: @@ -1301,6 +1336,13 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "6.1.0" + xxh3: + dependency: transitive + description: + name: xxh3 + url: "https://pub.dartlang.org" + source: hosted + version: "1.0.1" yaml: dependency: transitive description: diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index 786d2342e1..1fd192fa2c 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -3,6 +3,7 @@ description: Immich - selfhosted backup media file on mobile phone publish_to: "none" version: 1.45.0+68 +isar_version: &isar_version 3.0.5 environment: sdk: ">=2.17.0 <3.0.0" @@ -41,6 +42,8 @@ dependencies: http_parser: ^4.0.1 flutter_web_auth: ^0.5.0 easy_image_viewer: ^1.2.0 + isar: *isar_version + isar_flutter_libs: *isar_version # contains Isar Core openapi: path: openapi @@ -58,6 +61,7 @@ dev_dependencies: auto_route_generator: ^5.0.2 flutter_launcher_icons: "^0.9.2" flutter_native_splash: ^2.2.16 + isar_generator: *isar_version integration_test: sdk: flutter