chore(mobile): shared DataController for management of data sources (#31457)

* chore(mobile): static Store to unify and streamline reactivity

* Add tests of caching and new stores

* Don't instantiate PeopleDbRepository

* Slight DB rename and fix throwing store methods

* chore(mobile): shared DataController for management of data sources

* chore(mobile): shared DataController for management of data sources

* Fix lint with removed dependency

* Re-order Store bootstrap

* Remove extra import

mobile/integration_test/background_sync_teardown_test.dart

@@ -26,7 +26,8 @@ void main() {
 
   setUpAll(() async {
     await app.initApp();
-    (drift, _) = await Bootstrap.initDomain();
+    final (dataController, _) = await Bootstrap.initDomain();
+    drift = dataController.db;
   });
 
   setUp(() async {
@@ -119,7 +120,9 @@ void main() {
     final releaseTxn = Completer<void>();
     final txnHeld = Completer<void>();
     final txn = drift.transaction(() async {
-      await drift.into(drift.userEntity).insert(
+      await drift
+          .into(drift.userEntity)
+          .insert(
             UserEntityCompanion.insert(
               id: 'holder',
               name: 'holder',

mobile/integration_test/failed_sync_resume_e2e_test.dart

@@ -36,7 +36,8 @@ void main() {
 
   setUpAll(() async {
     await app.initApp();
-    (drift, _) = await Bootstrap.initDomain();
+    final (dataController, _) = await Bootstrap.initDomain();
+    drift = dataController.db;
     // A background-worker schedule persisted by real app use on this device can
     // launch a second engine mid-file (own isolate pool + full sync) and starve
     // these tests on a small device. Unregister it for the whole run.

mobile/integration_test/failed_sync_resume_test.dart

@@ -38,7 +38,8 @@ void main() {
 
   setUpAll(() async {
     await app.initApp();
-    (drift, _) = await Bootstrap.initDomain();
+    final (dataController, _) = await Bootstrap.initDomain();
+    drift = dataController.db;
     // A background-worker schedule persisted by real app use on this device can
     // launch a second engine mid-file (own isolate pool + full sync) and starve
     // these tests on a small device. Unregister it for the whole run.

mobile/integration_test/test_utils/general_helper.dart

@@ -3,9 +3,10 @@ import 'dart:async';
 import 'package:easy_localization/easy_localization.dart';
 import 'package:flutter_test/flutter_test.dart';
 import 'package:hooks_riverpod/hooks_riverpod.dart';
-import 'package:immich_mobile/entities/store.entity.dart';
+import 'package:immich_mobile/data/store.dart';
+// ignore: library_prefixes
+import 'package:immich_mobile/entities/store.entity.dart' as dbStore;
 import 'package:immich_mobile/main.dart' as app;
-import 'package:immich_mobile/providers/infrastructure/db.provider.dart';
 import 'package:immich_mobile/utils/bootstrap.dart';
 import 'package:integration_test/integration_test.dart';
 // ignore: depend_on_referenced_packages
@@ -38,11 +39,14 @@ class ImmichTestHelper {
   static Future<void> loadApp(WidgetTester tester) async {
     await EasyLocalization.ensureInitialized();
     // Clear all data from Isar (reuse existing instance if available)
-    final (drift, _) = await Bootstrap.initDomain();
+    final (dataController, apiService) = await Bootstrap.initDomain();
-    await Store.clear();
+    await dbStore.Store.clear();
     // Load main Widget
     await tester.pumpWidget(
-      ProviderScope(overrides: [driftProvider.overrideWith(driftOverride(drift))], child: const app.MainWidget()),
+      ProviderScope(
+        overrides: Store.overrideWith(dataController: dataController, apiService: apiService),
+        child: const app.MainWidget(),
+      ),
     );
     // Post run tasks
     await EasyLocalization.ensureInitialized();

mobile/lib/data/data_controller.dart

@@ -0,0 +1,83 @@
+import 'package:flutter/foundation.dart';
+import 'package:immich_mobile/data/db/logger/database.dart';
+import 'package:immich_mobile/data/db/main/dao/person.dart';
+import 'package:immich_mobile/data/db/main/database.dart';
+import 'package:immich_mobile/data/server/activity.dart';
+import 'package:immich_mobile/data/server/person.dart';
+import 'package:immich_mobile/domain/services/store.service.dart';
+import 'package:immich_mobile/infrastructure/repositories/store.repository.dart';
+import 'package:openapi/api.dart';
+import 'package:sqlite3/common.dart';
+
+/// Controls all data access. Serves request against the HTTP API and the Drift DB
+class DataController {
+  final Drift _db;
+  final DriftLogger _logDb;
+  final ApiClient _apiClient;
+
+  DataController._(this._db, this._logDb, this._apiClient);
+
+  /// Initalize the base data system. Sets up primary/logging DBs and the settings store
+  ///
+  /// `disableStoreWatching` prevents continually updating the setting store's cache on change
+  static Future<(DataController, bool)> init({required ApiClient apiClient, bool disableStoreWatching = false}) async {
+    await configureSqliteCache();
+
+    final (db, updatePool) = await openSqliteConnectionWithUpdatePool(name: 'immich');
+    final drift = Drift.sqlite(db, updatePool);
+
+    final (logDb, wasRecreated) = await _openLoggerDatabase();
+
+    await StoreService.init(storeRepository: StoreRepository(drift), listenUpdates: !disableStoreWatching);
+
+    return (DataController._(drift, logDb, apiClient), wasRecreated);
+  }
+
+  /// Open the logger database, recreating if corrupt. Returns the logger and whether it was recreated
+  static Future<(DriftLogger, bool)> _openLoggerDatabase() async {
+    Future<DriftLogger> open() async => DriftLogger.sqlite(await openSqliteConnection(name: 'immich_logs'));
+
+    final DriftLogger logDb = await open();
+
+    try {
+      await logDb.customSelect('SELECT COUNT(*) FROM logger_messages').get();
+    } on SqliteException catch (error) {
+      if (error.resultCode != SqlError.SQLITE_CORRUPT && error.resultCode != SqlError.SQLITE_NOTADB) {
+        await logDb.close();
+        rethrow;
+      }
+
+      if (kDebugMode) {
+        // ignore: banned-usage
+        debugPrint('Logs database is corrupt, recreating it');
+      }
+
+      await logDb.close();
+      await deleteSqliteDatabase(name: 'immich_logs');
+
+      return (await open(), true);
+    }
+
+    return (logDb, false);
+  }
+
+  late final PeopleDatabaseRepository peopleDb = PeopleDatabaseRepository(_db);
+  late final PersonApiRepository personApi = PersonApiRepository(PeopleApi(_apiClient));
+
+  late final ActivityApiRepository activityApi = ActivityApiRepository(ActivitiesApi(_apiClient));
+
+  /// Direct database access for the logic that has not yet been migrated
+  // TODO(rewrite): Remove once all repositories have been migrated
+  Drift get db => _db;
+
+  /// Direct logging access for the logic that has not yet been migrated
+  // TODO(rewrite): Remove once all repositories have been migrated
+  DriftLogger get logDb => _logDb;
+
+  Future<void> close() async {
+    await _db.close();
+
+    // Close after the primary DB to ensure all logs are captured
+    await _logDb.close();
+  }
+}

mobile/lib/data/store.dart

@@ -1,8 +1,38 @@
+import 'package:hooks_riverpod/hooks_riverpod.dart';
+import 'package:immich_mobile/data/data_controller.dart';
+import 'package:immich_mobile/data/db/main/database.dart';
 import 'package:immich_mobile/data/store/activity.dart';
 import 'package:immich_mobile/data/store/person.dart';
+import 'package:immich_mobile/services/api.service.dart';
+
+/// The [DataController] backing this container's store
+///
+/// Must be overridden with a constructed instance (`Store.overrideWithValue`)
+final _dataControllerProvider = Provider<DataController>(
+  (ref) => throw UnimplementedError("DataController instance must be set via Store.overrideWith"),
+);
+
+final _apiServiceProvider = Provider<ApiService>(
+  (ref) => throw UnimplementedError("ApiService instance must be set via Store.overrideWith"),
+);
 
 /// Global data layer, providing access to Drift and HTTP APIs, scoped by entity
 abstract final class Store {
+  static List<Override> overrideWith({required DataController dataController, required ApiService apiService}) => [
+    _dataControllerProvider.overrideWithValue(dataController),
+    _apiServiceProvider.overrideWithValue(apiService),
+  ];
+
+  /// Direct database access for the repositories that have not yet been migrated to `Store`
+  // TODO(rewrite): Remove this provider once all repositories have migrated to `Store`
+  static final db = Provider<Drift>((ref) => ref.watch(_dataControllerProvider).db);
+
+  /// Direct API access for the repositories that have not yet been migrated to `Store`
+  // TODO(rewrite): Remove this provider once all repositories have migrated to `Store`
+  static final api = Provider<ApiService>((ref) => ref.watch(_apiServiceProvider));
+
+  // ----
+
   static final people = PersonStore.instance;
 
   static final activity = ActivityStore.instance;

mobile/lib/domain/services/background_worker.service.dart

@@ -6,13 +6,14 @@ import 'package:background_downloader/background_downloader.dart';
 import 'package:flutter/material.dart';
 import 'package:hooks_riverpod/hooks_riverpod.dart';
 import 'package:immich_mobile/constants/constants.dart';
-import 'package:immich_mobile/data/db/logger/database.dart';
+import 'package:immich_mobile/data/data_controller.dart';
-import 'package:immich_mobile/data/db/main/database.dart';
+import 'package:immich_mobile/data/store.dart';
 import 'package:immich_mobile/domain/services/hash.service.dart';
 import 'package:immich_mobile/domain/services/local_sync.service.dart';
 import 'package:immich_mobile/domain/services/log.service.dart';
 import 'package:immich_mobile/domain/services/sync_stream.service.dart';
-import 'package:immich_mobile/entities/store.entity.dart';
+// ignore: library_prefixes
+import 'package:immich_mobile/entities/store.entity.dart' as dbStore;
 import 'package:immich_mobile/extensions/platform_extensions.dart';
 import 'package:immich_mobile/infrastructure/repositories/settings.repository.dart';
 import 'package:immich_mobile/platform/background_worker_api.g.dart';
@@ -25,6 +26,7 @@ import 'package:immich_mobile/providers/infrastructure/sync.provider.dart';
 import 'package:immich_mobile/providers/user.provider.dart';
 import 'package:immich_mobile/repositories/asset_media.repository.dart';
 import 'package:immich_mobile/repositories/permission.repository.dart';
+import 'package:immich_mobile/services/api.service.dart';
 import 'package:immich_mobile/services/auth.service.dart';
 import 'package:immich_mobile/services/foreground_upload.service.dart';
 import 'package:immich_mobile/services/localization.service.dart';
@@ -59,8 +61,7 @@ class BackgroundWorkerFgService {
 
 class BackgroundWorkerBgService extends BackgroundWorkerFlutterApi {
   ProviderContainer? _ref;
-  final Drift _drift;
+  final DataController _dataController;
-  final DriftLogger _driftLogger;
   final BackgroundWorkerBgHostApi _backgroundHostApi;
   final _cancellationToken = Completer<void>();
   final Logger _logger = Logger('BackgroundWorkerBgService');
@@ -70,9 +71,11 @@ class BackgroundWorkerBgService extends BackgroundWorkerFlutterApi {
 
   bool _isCleanedUp = false;
 
-  BackgroundWorkerBgService({required this._drift, required this._driftLogger})
+  BackgroundWorkerBgService({required this._dataController, required ApiService apiService})
     : _backgroundHostApi = BackgroundWorkerBgHostApi() {
-    final ref = ProviderContainer(overrides: [driftProvider.overrideWith(driftOverride(_drift))]);
+    final ref = ProviderContainer(
+      overrides: Store.overrideWith(dataController: _dataController, apiService: apiService),
+    );
     _ref = ref;
     final db = ref.read(driftProvider);
     _localSyncService = LocalSyncService(
@@ -248,7 +251,7 @@ class BackgroundWorkerBgService extends BackgroundWorkerFlutterApi {
 
   Future<void> _optimizeDB() async {
     try {
-      await (_drift.optimize(allTables: true), _driftLogger.optimize()).wait;
+      await (_dataController.db.optimize(allTables: true), _dataController.logDb.optimize()).wait;
     } catch (error, stack) {
       dPrint(() => "Error during background worker optimize: $error, $stack");
     }
@@ -278,9 +281,8 @@ class BackgroundWorkerBgService extends BackgroundWorkerFlutterApi {
       // Workers share one sqlite connection, so DB teardown must wait until every worker has stopped using it.
       await Future.wait([if (nativeSyncApi != null) nativeSyncApi.cancelHashing()]);
       await workerManagerPatch.dispose().catchError((_) async {});
-      await Future.wait([LogService.I.dispose(), Store.dispose()]);
+      await Future.wait([LogService.I.dispose(), dbStore.Store.dispose()]);
-      await _drift.close();
+      await _dataController.close();
-      await _driftLogger.close();
 
       _ref?.dispose();
       _ref = null;
@@ -371,6 +373,6 @@ Future<void> backgroundSyncNativeEntrypoint() async {
   WidgetsFlutterBinding.ensureInitialized();
   DartPluginRegistrant.ensureInitialized();
 
-  final (drift, logDB) = await Bootstrap.initDomain(shouldBufferLogs: false, listenStoreUpdates: false);
+  final (dataController, apiService) = await Bootstrap.initDomain(shouldBufferLogs: false, disableStoreWatching: true);
-  await BackgroundWorkerBgService(drift: drift, driftLogger: logDB).init();
+  await BackgroundWorkerBgService(dataController: dataController, apiService: apiService).init();
 }

mobile/lib/main.dart

@@ -14,6 +14,7 @@ import 'package:flutter_local_notifications/flutter_local_notifications.dart';
 import 'package:hooks_riverpod/hooks_riverpod.dart';
 import 'package:immich_mobile/constants/constants.dart';
 import 'package:immich_mobile/constants/locales.dart';
+import 'package:immich_mobile/data/store.dart';
 import 'package:immich_mobile/domain/services/background_worker.service.dart';
 import 'package:immich_mobile/extensions/build_context_extensions.dart';
 import 'package:immich_mobile/generated/codegen_loader.g.dart';
@@ -23,7 +24,6 @@ import 'package:immich_mobile/pages/common/splash_screen.page.dart';
 import 'package:immich_mobile/platform/background_worker_lock_api.g.dart';
 import 'package:immich_mobile/providers/app_life_cycle.provider.dart';
 import 'package:immich_mobile/providers/asset_viewer/share_intent_upload.provider.dart';
-import 'package:immich_mobile/providers/infrastructure/db.provider.dart';
 import 'package:immich_mobile/providers/infrastructure/platform.provider.dart';
 import 'package:immich_mobile/providers/infrastructure/settings.provider.dart';
 import 'package:immich_mobile/providers/locale_provider.dart';
@@ -51,13 +51,18 @@ void main() async {
     ImmichWidgetsBinding();
     unawaited(BackgroundWorkerLockService(BackgroundWorkerLockApi()).lock());
     await EasyLocalization.ensureInitialized();
-    final (drift, _) = await Bootstrap.initDomain();
+    final (dataController, apiService) = await Bootstrap.initDomain();
     await initApp();
     // Warm-up isolate pool for worker manager
     await workerManagerPatch.init(dynamicSpawning: true, isolatesCount: max(Platform.numberOfProcessors - 1, 5));
-    await migrateDatabaseIfNeeded(drift);
+    await migrateDatabaseIfNeeded(dataController.db);
 
-    runApp(ProviderScope(overrides: [driftProvider.overrideWith(driftOverride(drift))], child: const MainWidget()));
+    runApp(
+      ProviderScope(
+        overrides: Store.overrideWith(dataController: dataController, apiService: apiService),
+        child: const MainWidget(),
+      ),
+    );
   } catch (error, stack) {
     runApp(BootstrapErrorWidget(error: error.toString(), stack: stack.toString()));
   }

mobile/lib/providers/api.provider.dart

@@ -1,4 +1,5 @@
-import 'package:hooks_riverpod/hooks_riverpod.dart';
+import 'package:immich_mobile/data/store.dart';
-import 'package:immich_mobile/services/api.service.dart';
 
-final apiServiceProvider = Provider((_) => ApiService());
+/// Direct API access for the repositories that have not yet been migrated to `Store`
+// TODO(rewrite): Remove this provider once all repositories have migrated to `Store`
+final apiServiceProvider = Store.api;

mobile/lib/providers/infrastructure/db.provider.dart

@@ -2,6 +2,7 @@ import 'dart:async';
 
 import 'package:hooks_riverpod/hooks_riverpod.dart';
 import 'package:immich_mobile/data/db/main/database.dart';
+import 'package:immich_mobile/data/store.dart';
 
 Drift Function(Ref ref) driftOverride(Drift drift) => (ref) {
   ref.onDispose(() => unawaited(drift.close()));
@@ -9,6 +10,6 @@ Drift Function(Ref ref) driftOverride(Drift drift) => (ref) {
   return drift;
 };
 
-final driftProvider = Provider<Drift>(
+/// Direct database access for the repositories that have not yet been migrated to `Store`
-  (ref) => throw UnimplementedError("driftProvider must be overridden in the isolate's ProviderContainer before use"),
+// TODO(rewrite): Remove this provider once all repositories have migrated to `Store`
-);
+final driftProvider = Store.db;

mobile/lib/services/api.service.dart

@@ -41,10 +41,6 @@ class ApiService {
     // The below line ensures that the api clients are initialized when the service is instantiated
     // This is required to avoid late initialization errors when the clients are access before the endpoint is resolved
     setEndpoint('');
-    final endpoint = Store.tryGet(StoreKey.serverEndpoint);
-    if (endpoint != null && endpoint.isNotEmpty) {
-      setEndpoint(endpoint);
-    }
   }
   final _log = Logger("ApiService");
 

mobile/lib/utils/bootstrap.dart

@@ -1,18 +1,16 @@
 import 'package:background_downloader/background_downloader.dart';
 import 'package:immich_mobile/constants/constants.dart';
-import 'package:immich_mobile/data/db/logger/database.dart';
+import 'package:immich_mobile/data/data_controller.dart';
-import 'package:immich_mobile/data/db/main/database.dart';
+import 'package:immich_mobile/domain/models/store.model.dart';
 import 'package:immich_mobile/domain/services/log.service.dart';
-import 'package:immich_mobile/domain/services/store.service.dart';
+import 'package:immich_mobile/entities/store.entity.dart';
 import 'package:immich_mobile/generated/translations.g.dart';
 import 'package:immich_mobile/infrastructure/repositories/log.repository.dart';
 import 'package:immich_mobile/infrastructure/repositories/network.repository.dart';
 import 'package:immich_mobile/infrastructure/repositories/settings.repository.dart';
-import 'package:immich_mobile/infrastructure/repositories/store.repository.dart';
+import 'package:immich_mobile/services/api.service.dart';
-import 'package:immich_mobile/utils/debug_print.dart';
 import 'package:logging/logging.dart';
 import 'package:photo_manager/photo_manager.dart';
-import 'package:sqlite3/common.dart';
 
 void configureFileDownloaderNotifications() {
   final t = StaticTranslations.instance;
@@ -47,50 +45,42 @@ void configureFileDownloaderNotifications() {
 }
 
 abstract final class Bootstrap {
-  static Future<(Drift, DriftLogger)> initDomain({bool listenStoreUpdates = true, bool shouldBufferLogs = true}) async {
+  /// Initalize the base data system. Sets up primary/logging DBs, the [ApiService], and the settings store
-    await configureSqliteCache();
+  ///
-    final (db, updatePool) = await openSqliteConnectionWithUpdatePool(name: 'immich');
+  /// `disableStoreWatching` prevents continually updating the setting store's cache on change
-    final drift = Drift.sqlite(db, updatePool);
+  static Future<(DataController, ApiService)> initDomain({
-    final StoreRepository storeRepo = StoreRepository(drift);
+    bool shouldBufferLogs = true,
-
+    bool disableStoreWatching = false,
-    await StoreService.init(storeRepository: storeRepo, listenUpdates: listenStoreUpdates);
+  }) async {
-
-    final settingsRepo = await SettingsRepository.ensureInitialized(drift);
-    final logDb = await _initLogger(settingsRepository: settingsRepo, shouldBufferLogs: shouldBufferLogs);
-
     await NetworkRepository.init();
-    // Remove once all asset operations are migrated to Native APIs
-    await PhotoManager.setIgnorePermissionCheck(true);
-    return (drift, logDb);
-  }
-}
 
-Future<DriftLogger> _initLogger({required SettingsRepository settingsRepository, bool shouldBufferLogs = true}) async {
+    final apiService = ApiService();
-  Future<DriftLogger> open() async => DriftLogger.sqlite(await openSqliteConnection(name: 'immich_logs'));
+    final (dataController, loggerDatabaseWasRecreated) = await DataController.init(
+      apiClient: apiService.apiClient,
+      disableStoreWatching: disableStoreWatching,
+    );
 
-  DriftLogger logDb = await open();
+    final settingsRepo = await SettingsRepository.ensureInitialized(dataController.db);
-  bool wasCorrupt = false;
+
-  try {
+    // TODO(rewrite): This is bad DB coupling and should be removed
-    await logDb.customSelect('SELECT COUNT(*) FROM logger_messages').get();
+    final endpoint = Store.tryGet(StoreKey.serverEndpoint);
-  } on SqliteException catch (error) {
+    if (endpoint != null && endpoint.isNotEmpty) {
-    if (error.resultCode != SqlError.SQLITE_CORRUPT && error.resultCode != SqlError.SQLITE_NOTADB) {
+      apiService.setEndpoint(endpoint);
-      await logDb.close();
-      rethrow;
     }
-    dPrint(() => 'Logs database is corrupt, recreating it');
-    await logDb.close();
-    await deleteSqliteDatabase(name: 'immich_logs');
-    logDb = await open();
-    wasCorrupt = true;
-  }
 
-  await LogService.init(
+    // Take DataController's logging DB and register it with the logging service
-    logRepository: LogRepository(logDb),
+    await LogService.init(
-    settingsRepository: settingsRepository,
+      logRepository: LogRepository(dataController.logDb),
-    shouldBuffer: shouldBufferLogs,
+      settingsRepository: settingsRepo,
-  );
+      shouldBuffer: shouldBufferLogs,
-  if (wasCorrupt) {
+    );
-    Logger('bootstrap:initLogger').warning('Logs database was corrupt and has been recreated');
+
+    if (loggerDatabaseWasRecreated) {
+      Logger('bootstrap:initLogger').warning('Logs database was corrupt and has been recreated');
+    }
+
+    // TODO: Remove once all asset operations are migrated to Native APIs
+    await PhotoManager.setIgnorePermissionCheck(true);
+    return (dataController, apiService);
   }
-  return logDb;
 }

mobile/lib/utils/isolate.dart

@@ -3,10 +3,11 @@ import 'dart:ui';
 
 import 'package:flutter/services.dart';
 import 'package:hooks_riverpod/hooks_riverpod.dart';
+import 'package:immich_mobile/data/store.dart';
 import 'package:immich_mobile/domain/services/log.service.dart';
-import 'package:immich_mobile/entities/store.entity.dart';
+// ignore: library_prefixes
+import 'package:immich_mobile/entities/store.entity.dart' as dbStore;
 import 'package:immich_mobile/providers/infrastructure/cancel.provider.dart';
-import 'package:immich_mobile/providers/infrastructure/db.provider.dart';
 import 'package:immich_mobile/utils/bootstrap.dart';
 import 'package:immich_mobile/wm_executor.dart';
 import 'package:logging/logging.dart';
@@ -34,9 +35,15 @@ Cancelable<T?> runInIsolateGentle<T>({
     DartPluginRegistrant.ensureInitialized();
 
     final log = Logger("IsolateLogger");
-    final (drift, logDb) = await Bootstrap.initDomain(shouldBufferLogs: false, listenStoreUpdates: false);
+    final (dataController, apiService) = await Bootstrap.initDomain(
+      shouldBufferLogs: false,
+      disableStoreWatching: true,
+    );
     final ref = ProviderContainer(
-      overrides: [cancellationProvider.overrideWithValue(onCancel), driftProvider.overrideWith(driftOverride(drift))],
+      overrides: [
+        cancellationProvider.overrideWithValue(onCancel),
+        ...Store.overrideWith(dataController: dataController, apiService: apiService),
+      ],
     );
 
     try {
@@ -46,10 +53,9 @@ Cancelable<T?> runInIsolateGentle<T>({
       return null;
     } finally {
       ref.dispose();
-      await Store.dispose();
+      await dbStore.Store.dispose();
       await LogService.I.dispose();
-      await logDb.close();
+      await dataController.close();
-      await drift.close();
     }
   });
 }