refactor(mobile): add Isar DB & Store class (#1574)

* refactor(mobile): add Isar DB & Store class

new Store: globally accessible key-value store like Hive (but based on Isar)

replace first few places of Hive usage with the new Store

* reduce max. DB size to prevent errors on older iOS devices

---------

Co-authored-by: Alex <alex.tran1502@gmail.com>

mobile/integration_test/test_utils/general_helper.dart

@@ -1,7 +1,9 @@
 import 'package:easy_localization/easy_localization.dart';
 import 'package:flutter_test/flutter_test.dart';
 import 'package:hive/hive.dart';
+import 'package:immich_mobile/shared/models/store.dart';
 import 'package:integration_test/integration_test.dart';
+import 'package:isar/isar.dart';
 // ignore: depend_on_referenced_packages
 import 'package:meta/meta.dart';
 import 'package:immich_mobile/main.dart' as app;
@@ -34,8 +36,12 @@ class ImmichTestHelper {
     // Clear all data from Hive
     await Hive.deleteFromDisk();
     await app.openBoxes();
+    // Clear all data from Isar (reuse existing instance if available)
+    final db = Isar.getInstance() ?? await app.loadDb();
+    await Store.clear();
+    await db.writeTxn(() => db.clear());
     // Load main Widget
-    await tester.pumpWidget(app.getMainWidget());
+    await tester.pumpWidget(app.getMainWidget(db));
     // Post run tasks
     await tester.pumpAndSettle();
     await EasyLocalization.ensureInitialized();

mobile/lib/main.dart

@@ -17,8 +17,10 @@ import 'package:immich_mobile/modules/login/providers/authentication.provider.da
 import 'package:immich_mobile/routing/router.dart';
 import 'package:immich_mobile/routing/tab_navigation_observer.dart';
 import 'package:immich_mobile/shared/models/immich_logger_message.model.dart';
+import 'package:immich_mobile/shared/models/store.dart';
 import 'package:immich_mobile/shared/providers/app_state.provider.dart';
 import 'package:immich_mobile/shared/providers/asset.provider.dart';
+import 'package:immich_mobile/shared/providers/db.provider.dart';
 import 'package:immich_mobile/shared/providers/release_info.provider.dart';
 import 'package:immich_mobile/shared/providers/server_info.provider.dart';
 import 'package:immich_mobile/shared/providers/websocket.provider.dart';
@@ -26,11 +28,16 @@ import 'package:immich_mobile/shared/services/immich_logger.service.dart';
 import 'package:immich_mobile/shared/views/immich_loading_overlay.dart';
 import 'package:immich_mobile/shared/views/version_announcement_overlay.dart';
 import 'package:immich_mobile/utils/immich_app_theme.dart';
+import 'package:immich_mobile/utils/migration.dart';
+import 'package:isar/isar.dart';
+import 'package:path_provider/path_provider.dart';
 import 'constants/hive_box.dart';
 
 void main() async {
   await initApp();
-  runApp(getMainWidget());
+  final db = await loadDb();
+  await migrateHiveToStoreIfNecessary();
+  runApp(getMainWidget(db));
 }
 
 Future<void> openBoxes() async {
@@ -70,13 +77,27 @@ Future<void> initApp() async {
   ImmichLogger().init();
 }
 
-Widget getMainWidget() {
+Future<Isar> loadDb() async {
+  final dir = await getApplicationDocumentsDirectory();
+  Isar db = await Isar.open(
+    [StoreValueSchema],
+    directory: dir.path,
+    maxSizeMiB: 256,
+  );
+  Store.init(db);
+  return db;
+}
+
+Widget getMainWidget(Isar db) {
   return EasyLocalization(
     supportedLocales: locales,
     path: translationsPath,
     useFallbackTranslations: true,
     fallbackLocale: locales.first,
-    child: const ProviderScope(child: ImmichApp()),
+    child: ProviderScope(
+      overrides: [dbProvider.overrideWithValue(db)],
+      child: const ImmichApp(),
+    ),
   );
 }
 

mobile/lib/modules/login/providers/authentication.provider.dart

@@ -4,6 +4,7 @@ import 'package:hive/hive.dart';
 import 'package:hooks_riverpod/hooks_riverpod.dart';
 import 'package:immich_mobile/constants/hive_box.dart';
 import 'package:immich_mobile/modules/album/services/album_cache.service.dart';
+import 'package:immich_mobile/shared/models/store.dart';
 import 'package:immich_mobile/shared/services/asset_cache.service.dart';
 import 'package:immich_mobile/modules/login/models/authentication_state.model.dart';
 import 'package:immich_mobile/modules/login/models/hive_saved_login_info.model.dart';
@@ -94,7 +95,8 @@ class AuthenticationNotifier extends StateNotifier<AuthenticationState> {
     await Future.wait([
       _apiService.authenticationApi.logout(),
       Hive.box(userInfoBox).delete(accessTokenKey),
-      Hive.box(userInfoBox).delete(assetEtagKey),
+      Store.delete(StoreKey.assetETag),
+      Store.delete(StoreKey.userRemoteId),
       _assetCacheService.invalidate(),
       _albumCacheService.invalidate(),
       _sharedAlbumCacheService.invalidate(),
@@ -153,7 +155,7 @@ class AuthenticationNotifier extends StateNotifier<AuthenticationState> {
       var deviceInfo = await _deviceInfoService.getDeviceInfo();
       userInfoHiveBox.put(deviceIdKey, deviceInfo["deviceId"]);
       userInfoHiveBox.put(accessTokenKey, accessToken);
-      userInfoHiveBox.put(userIdKey, userResponseDto.id);
+      Store.put(StoreKey.userRemoteId, userResponseDto.id);
 
       state = state.copyWith(
         isAuthenticated: true,

mobile/lib/shared/models/store.dart

@@ -0,0 +1,96 @@
+import 'package:isar/isar.dart';
+import 'dart:convert';
+
+part 'store.g.dart';
+
+/// Key-value store for individual items enumerated in StoreKey.
+/// Supports String, int and JSON-serializable Objects
+/// Can be used concurrently from multiple isolates
+class Store {
+  static late final Isar _db;
+  static final List<dynamic> _cache = List.filled(StoreKey.values.length, null);
+
+  /// Initializes the store (call exactly once per app start)
+  static void init(Isar db) {
+    _db = db;
+    _populateCache();
+    _db.storeValues.where().build().watch().listen(_onChangeListener);
+  }
+
+  /// clears all values from this store (cache and DB), only for testing!
+  static Future<void> clear() {
+    _cache.fillRange(0, _cache.length, null);
+    return _db.writeTxn(() => _db.storeValues.clear());
+  }
+
+  /// Returns the stored value for the given key, or the default value if null
+  static T? get<T>(StoreKey key, [T? defaultValue]) =>
+      _cache[key._id] ?? defaultValue;
+
+  /// Stores the value synchronously in the cache and asynchronously in the DB
+  static Future<void> put<T>(StoreKey key, T value) {
+    _cache[key._id] = value;
+    return _db.writeTxn(() => _db.storeValues.put(StoreValue._of(value, key)));
+  }
+
+  /// Removes the value synchronously from the cache and asynchronously from the DB
+  static Future<void> delete(StoreKey key) {
+    _cache[key._id] = null;
+    return _db.writeTxn(() => _db.storeValues.delete(key._id));
+  }
+
+  /// Fills the cache with the values from the DB
+  static _populateCache() {
+    for (StoreKey key in StoreKey.values) {
+      final StoreValue? value = _db.storeValues.getSync(key._id);
+      if (value != null) {
+        _cache[key._id] = value._extract(key);
+      }
+    }
+  }
+
+  /// updates the state if a value is updated in any isolate
+  static void _onChangeListener(List<StoreValue>? data) {
+    if (data != null) {
+      for (StoreValue value in data) {
+        _cache[value.id] = value._extract(StoreKey.values[value.id]);
+      }
+    }
+  }
+}
+
+/// Internal class for `Store`, do not use elsewhere.
+@Collection(inheritance: false)
+class StoreValue {
+  StoreValue(this.id, {this.intValue, this.strValue});
+  Id id;
+  int? intValue;
+  String? strValue;
+
+  T? _extract<T>(StoreKey key) => key._isInt
+      ? intValue
+      : (key._fromJson != null
+          ? key._fromJson!(json.decode(strValue!))
+          : strValue);
+  static StoreValue _of(dynamic value, StoreKey key) => StoreValue(
+        key._id,
+        intValue: key._isInt ? value : null,
+        strValue: key._isInt
+            ? null
+            : (key._fromJson == null ? value : json.encode(value.toJson())),
+      );
+}
+
+/// Key for each possible value in the `Store`.
+/// Defines the data type (int, String, JSON) for each value
+enum StoreKey {
+  userRemoteId(0),
+  assetETag(1),
+  ;
+
+  // ignore: unused_element
+  const StoreKey(this._id, [this._isInt = false, this._fromJson]);
+  final int _id;
+  final bool _isInt;
+  final Function(dynamic)? _fromJson;
+}

mobile/lib/shared/providers/asset.provider.dart

@@ -4,6 +4,7 @@ import 'package:flutter/foundation.dart';
 import 'package:hive/hive.dart';
 import 'package:hooks_riverpod/hooks_riverpod.dart';
 import 'package:immich_mobile/constants/hive_box.dart';
+import 'package:immich_mobile/shared/models/store.dart';
 import 'package:immich_mobile/shared/services/asset.service.dart';
 import 'package:immich_mobile/shared/services/asset_cache.service.dart';
 import 'package:immich_mobile/modules/home/ui/asset_grid/asset_grid_data_structure.dart';
@@ -106,7 +107,6 @@ class AssetNotifier extends StateNotifier<AssetsState> {
       _getAllAssetInProgress = true;
       bool isCacheValid = await _assetCacheService.isValid();
       stopwatch.start();
-      final Box box = Hive.box(userInfoBox);
       if (isCacheValid && state.allAssets.isEmpty) {
         final List<Asset>? cachedData = await _assetCacheService.get();
         if (cachedData == null) {
@@ -122,7 +122,7 @@ class AssetNotifier extends StateNotifier<AssetsState> {
       }
       final localTask = _assetService.getLocalAssets(urgent: !isCacheValid);
       final remoteTask = _assetService.getRemoteAssets(
-        etag: isCacheValid ? box.get(assetEtagKey) : null,
+        etag: isCacheValid ? Store.get(StoreKey.assetETag) : null,
       );
 
       int remoteBegin = state.allAssets.indexWhere((a) => a.isRemote);
@@ -151,7 +151,7 @@ class AssetNotifier extends StateNotifier<AssetsState> {
 
       log.info("Combining assets: ${stopwatch.elapsedMilliseconds}ms");
 
-      box.put(assetEtagKey, remoteResult.second);
+      Store.put(StoreKey.assetETag, remoteResult.second);
     } finally {
       _getAllAssetInProgress = false;
     }
@@ -279,8 +279,7 @@ class AssetNotifier extends StateNotifier<AssetsState> {
 
     final index = state.allAssets.indexWhere((a) => asset.id == a.id);
     if (index > 0) {
-      state.allAssets.removeAt(index);
+      state.allAssets[index] = newAsset;
-      state.allAssets.insert(index, Asset.remote(newAsset));
       _updateAssetsState(state.allAssets);
     }
 

mobile/lib/shared/providers/db.provider.dart

@@ -0,0 +1,5 @@
+import 'package:hooks_riverpod/hooks_riverpod.dart';
+import 'package:isar/isar.dart';
+
+// overwritten in main.dart due to async loading
+final dbProvider = Provider<Isar>((_) => throw UnimplementedError());

mobile/lib/shared/services/asset.service.dart

@@ -8,6 +8,7 @@ import 'package:immich_mobile/modules/backup/background_service/background.servi
 import 'package:immich_mobile/modules/backup/models/hive_backup_albums.model.dart';
 import 'package:immich_mobile/modules/backup/services/backup.service.dart';
 import 'package:immich_mobile/shared/models/asset.dart';
+import 'package:immich_mobile/shared/models/store.dart';
 import 'package:immich_mobile/shared/providers/api.provider.dart';
 import 'package:immich_mobile/shared/services/api.service.dart';
 import 'package:immich_mobile/utils/openapi_extensions.dart';
@@ -37,7 +38,7 @@ class AssetService {
       final Pair<List<AssetResponseDto>, String?>? remote =
           await _apiService.assetApi.getAllAssetsWithETag(eTag: etag);
       if (remote == null) {
-        return const Pair(null, null);
+        return Pair(null, etag);
       }
       return Pair(
         remote.first.map(Asset.remote).toList(growable: false),
@@ -45,7 +46,7 @@ class AssetService {
       );
     } catch (e, stack) {
       log.severe('Error while getting remote assets', e, stack);
-      return const Pair(null, null);
+      return Pair(null, etag);
     }
   }
 
@@ -62,7 +63,7 @@ class AssetService {
       }
       final box = await Hive.openBox<HiveBackupAlbums>(hiveBackupInfoBox);
       final HiveBackupAlbums? backupAlbumInfo = box.get(backupInfoKey);
-      final String userId = Hive.box(userInfoBox).get(userIdKey);
+      final String userId = Store.get(StoreKey.userRemoteId);
       if (backupAlbumInfo != null) {
         return (await _backupService
                 .buildUploadCandidates(backupAlbumInfo.deepCopy()))
@@ -105,12 +106,16 @@ class AssetService {
     }
   }
 
-  Future<AssetResponseDto?> updateAsset(Asset asset, UpdateAssetDto updateAssetDto) async {
+  Future<Asset?> updateAsset(
-    return await _apiService.assetApi.updateAsset(asset.id, updateAssetDto);
+    Asset asset,
+    UpdateAssetDto updateAssetDto,
+  ) async {
+    final dto =
+        await _apiService.assetApi.updateAsset(asset.remoteId!, updateAssetDto);
+    return dto == null ? null : Asset.remote(dto);
   }
 
-  Future<AssetResponseDto?> changeFavoriteStatus(Asset asset, bool isFavorite) {
+  Future<Asset?> changeFavoriteStatus(Asset asset, bool isFavorite) {
     return updateAsset(asset, UpdateAssetDto(isFavorite: isFavorite));
   }
-
 }

mobile/lib/utils/migration.dart

@@ -0,0 +1,24 @@
+import 'package:flutter/cupertino.dart';
+import 'package:hive/hive.dart';
+import 'package:immich_mobile/constants/hive_box.dart';
+import 'package:immich_mobile/shared/models/store.dart';
+
+Future<void> migrateHiveToStoreIfNecessary() async {
+  try {
+    if (await Hive.boxExists(userInfoBox)) {
+      final Box box = await Hive.openBox(userInfoBox);
+      await _migrateSingleKey(box, userIdKey, StoreKey.userRemoteId);
+      await _migrateSingleKey(box, assetEtagKey, StoreKey.assetETag);
+    }
+  } catch (e) {
+    debugPrint("Error while migrating userInfoBox $e");
+  }
+}
+
+_migrateSingleKey(Box box, String hiveKey, StoreKey key) async {
+  final String? value = box.get(hiveKey);
+  if (value != null) {
+    await Store.put(key, value);
+    await box.delete(hiveKey);
+  }
+}

mobile/pubspec.lock

@@ -239,6 +239,13 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "2.2.3"
+  dartx:
+    dependency: transitive
+    description:
+      name: dartx
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "1.1.0"
   easy_image_viewer:
     dependency: "direct main"
     description:
@@ -547,6 +554,27 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "1.0.3"
+  isar:
+    dependency: "direct main"
+    description:
+      name: isar
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "3.0.5"
+  isar_flutter_libs:
+    dependency: "direct main"
+    description:
+      name: isar_flutter_libs
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "3.0.5"
+  isar_generator:
+    dependency: "direct dev"
+    description:
+      name: isar_generator
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "3.0.5"
   js:
     dependency: transitive
     description:
@@ -1063,6 +1091,13 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "0.4.12"
+  time:
+    dependency: transitive
+    description:
+      name: time
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "2.1.3"
   timing:
     dependency: transitive
     description:
@@ -1301,6 +1336,13 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "6.1.0"
+  xxh3:
+    dependency: transitive
+    description:
+      name: xxh3
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "1.0.1"
   yaml:
     dependency: transitive
     description:

mobile/pubspec.yaml

@@ -3,6 +3,7 @@ description: Immich - selfhosted backup media file on mobile phone
 
 publish_to: "none"
 version: 1.45.0+68
+isar_version: &isar_version 3.0.5
 
 environment:
   sdk: ">=2.17.0 <3.0.0"
@@ -41,6 +42,8 @@ dependencies:
   http_parser: ^4.0.1
   flutter_web_auth: ^0.5.0
   easy_image_viewer: ^1.2.0
+  isar: *isar_version
+  isar_flutter_libs: *isar_version # contains Isar Core
 
   openapi:
     path: openapi
@@ -58,6 +61,7 @@ dev_dependencies:
   auto_route_generator: ^5.0.2
   flutter_launcher_icons: "^0.9.2"
   flutter_native_splash: ^2.2.16
+  isar_generator: *isar_version
   integration_test:
     sdk: flutter