From e8c621843138e52fa7c6ba272ae48b027387fb44 Mon Sep 17 00:00:00 2001 From: Santo Shakil Date: Thu, 17 Sep 2026 19:05:07 +0600 Subject: [PATCH] fix(mobile): show the real date for android local photos with no exif (#29193) * fix(mobile): show the real date for android local photos with no exif photos with no exif date showed their copy-to-phone date in the "on this device" albums instead of the real date. fall back to the earlier of date_modified/date_added (matches the server + ios), plus a migration to fix the rows already saved wrong. * persist migration progress when the date backfill fails * fix local date migration for trash and epoch rows * cover epoch dates in both local asset tables * simplify the created date pick and rework the migration test * heal no-exif dates from mediastore in the v27 migration * check gallery access from dart and skip dates that do not parse * let the date heal run once and never block the version * read the trashed assets through the same cast the sync uses --- .../alextran/immich/sync/MessagesImpl30.kt | 3 +- .../alextran/immich/sync/MessagesImplBase.kt | 18 ++- mobile/lib/main.dart | 4 +- mobile/lib/utils/migration.dart | 82 +++++++++- mobile/test/medium/repository_context.dart | 2 + mobile/test/modules/utils/migration_test.dart | 153 ++++++++++++++++++ 6 files changed, 251 insertions(+), 11 deletions(-) create mode 100644 mobile/test/modules/utils/migration_test.dart diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImpl30.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImpl30.kt index 4785b751c..4b09d97ea 100644 --- a/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImpl30.kt +++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImpl30.kt @@ -113,7 +113,8 @@ class NativeSyncApiImpl30(context: Context) : NativeSyncApiImplBase(context), Na putInt(MediaStore.QUERY_ARG_MATCH_TRASHED, MediaStore.MATCH_ONLY) } - getCursor(volume, queryArgs).use { cursor -> + val cursor = getCursor(volume, queryArgs) ?: error("MediaStore trash query failed") + cursor.use { getAssets(cursor).forEach { res -> if (res is AssetResult.ValidAsset) { result.getOrPut(res.albumId) { mutableListOf() }.add(res.asset) diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImplBase.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImplBase.kt index 25a849ad3..0710e114c 100644 --- a/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImplBase.kt +++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImplBase.kt @@ -175,11 +175,12 @@ open class NativeSyncApiImplBase(context: Context) : ImmichPlugin(), ActivityAwa MediaStore.Files.FileColumns.MEDIA_TYPE_VIDEO -> 2L else -> 0L } - // Date taken is milliseconds since epoch, Date added is seconds since epoch - val createdAt = (c.getLong(dateTakenColumn).takeIf { it > 0 }?.div(1000)) - ?: c.getLong(dateAddedColumn) - // Date modified is seconds since epoch + // Date taken is in ms; added/modified are in seconds, and modified can be 0 when unset. + // If EXIF date taken exists use it, else if modified is empty use added, else the earliest of the two. val modifiedAt = c.getLong(dateModifiedColumn) + val addedAt = c.getLong(dateAddedColumn) + val createdAt = (c.getLong(dateTakenColumn).takeIf { it > 0 }?.div(1000)) + ?: if (modifiedAt <= 0) addedAt else minOf(modifiedAt, addedAt) val width = c.getInt(widthColumn).toLong() val height = c.getInt(heightColumn).toLong() // Duration is milliseconds @@ -314,13 +315,14 @@ open class NativeSyncApiImplBase(context: Context) : ImmichPlugin(), ActivityAwa val selection = "(${MediaStore.Files.FileColumns.BUCKET_ID} IS NOT NULL) AND $MEDIA_SELECTION" - getCursor( + val cursor = getCursor( MediaStore.VOLUME_EXTERNAL, selection, MEDIA_SELECTION_ARGS, projection, "${MediaStore.Files.FileColumns.DATE_MODIFIED} DESC" - )?.use { cursor -> + ) ?: error("MediaStore album query failed") + cursor.use { val bucketIdColumn = cursor.getColumnIndexOrThrow(MediaStore.Files.FileColumns.BUCKET_ID) val bucketNameColumn = @@ -395,7 +397,9 @@ open class NativeSyncApiImplBase(context: Context) : ImmichPlugin(), ActivityAwa selectionArgs.addAll(listOf(updatedTimeCond.toString(), updatedTimeCond.toString())) } - return getAssets(getCursor(MediaStore.VOLUME_EXTERNAL, selection, selectionArgs.toTypedArray())) + val cursor = getCursor(MediaStore.VOLUME_EXTERNAL, selection, selectionArgs.toTypedArray()) + ?: error("MediaStore asset query failed") + return getAssets(cursor) .mapNotNull { result -> (result as? AssetResult.ValidAsset)?.asset } .toList() } diff --git a/mobile/lib/main.dart b/mobile/lib/main.dart index 5fc046d69..cc868de29 100644 --- a/mobile/lib/main.dart +++ b/mobile/lib/main.dart @@ -22,6 +22,8 @@ import 'package:immich_mobile/generated/translations.g.dart'; import 'package:immich_mobile/infrastructure/repositories/network.repository.dart'; import 'package:immich_mobile/pages/common/splash_screen.page.dart'; import 'package:immich_mobile/platform/background_worker_lock_api.g.dart'; +import 'package:immich_mobile/platform/native_sync_api.g.dart'; +import 'package:immich_mobile/platform/permission_api.g.dart'; import 'package:immich_mobile/providers/app_life_cycle.provider.dart'; import 'package:immich_mobile/providers/asset_viewer/share_intent_upload.provider.dart'; import 'package:immich_mobile/providers/infrastructure/platform.provider.dart'; @@ -55,7 +57,7 @@ void main() async { await initApp(); // Warm-up isolate pool for worker manager await workerManagerPatch.init(dynamicSpawning: true, isolatesCount: max(Platform.numberOfProcessors - 1, 5)); - await migrateDatabaseIfNeeded(dataController.db); + await migrateDatabaseIfNeeded(dataController.db, NativeSyncApi(), PermissionApi()); runApp( ProviderScope( diff --git a/mobile/lib/utils/migration.dart b/mobile/lib/utils/migration.dart index 4dfd4e1c2..d111a262f 100644 --- a/mobile/lib/utils/migration.dart +++ b/mobile/lib/utils/migration.dart @@ -5,9 +5,12 @@ import 'package:collection/collection.dart'; import 'package:drift/drift.dart'; import 'package:flutter/material.dart'; import 'package:immich_mobile/constants/colors.dart'; +import 'package:immich_mobile/constants/constants.dart'; import 'package:immich_mobile/constants/enums.dart'; import 'package:immich_mobile/data/db/main/database.dart'; import 'package:immich_mobile/data/db/main/table/app/settings.drift.dart'; +import 'package:immich_mobile/data/db/main/table/local/asset.drift.dart'; +import 'package:immich_mobile/data/db/main/table/local/trashed_asset.drift.dart'; import 'package:immich_mobile/domain/models/config/app_config.dart'; import 'package:immich_mobile/domain/models/log.model.dart'; import 'package:immich_mobile/domain/models/settings_key.dart'; @@ -15,14 +18,19 @@ import 'package:immich_mobile/domain/models/store.model.dart'; import 'package:immich_mobile/domain/models/timeline.model.dart'; import 'package:immich_mobile/domain/services/feature_message.service.dart'; import 'package:immich_mobile/entities/store.entity.dart'; +import 'package:immich_mobile/extensions/platform_extensions.dart'; import 'package:immich_mobile/infrastructure/repositories/network.repository.dart'; import 'package:immich_mobile/infrastructure/repositories/settings.repository.dart'; import 'package:immich_mobile/models/auth/auxilary_endpoint.model.dart'; +import 'package:immich_mobile/platform/native_sync_api.g.dart'; +import 'package:immich_mobile/platform/permission_api.g.dart'; import 'package:immich_mobile/providers/album/album_sort_by_options.provider.dart'; +import 'package:immich_mobile/utils/datetime_helpers.dart'; +import 'package:logging/logging.dart'; -const int targetVersion = 26; +const int targetVersion = 27; -Future migrateDatabaseIfNeeded(Drift drift) async { +Future migrateDatabaseIfNeeded(Drift drift, NativeSyncApi nativeSyncApi, PermissionApi permissionApi) async { final int? storedVersion = Store.tryGet(StoreKey.version); final version = storedVersion ?? targetVersion; @@ -34,6 +42,10 @@ Future migrateDatabaseIfNeeded(Drift drift) async { await _migrateTo26(drift); } + if (version < 27) { + await _migrateTo27(drift, nativeSyncApi, permissionApi); + } + if (storedVersion == null) { await FeatureMessageService(SettingsRepository.instance).markSeen(); } @@ -42,6 +54,72 @@ Future migrateDatabaseIfNeeded(Drift drift) async { return; } +// android photos without exif used to get the copy date as created_at, +// mediastore still has the real one so read it once more and fix only the rows that differ +Future _migrateTo27(Drift drift, NativeSyncApi nativeSyncApi, PermissionApi permissionApi) async { + if (!CurrentPlatform.isAndroid) { + return; + } + try { + final dates = {}; + void addDates(Iterable assets) { + for (final asset in assets) { + final date = tryFromSecondsSinceEpoch(asset.createdAt, isUtc: true); + if (date != null) { + dates[asset.id] = date; + } + } + } + + for (final album in await nativeSyncApi.getAlbums()) { + addDates(await nativeSyncApi.getAssetsForAlbum(album.id)); + } + if (await permissionApi.hasManageMediaPermission()) { + final trashed = await nativeSyncApi.getTrashedAssets(); + addDates(trashed.cast>().values.flattened.cast()); + } + + await drift.transaction(() async { + final localDates = { + for (final row in await (drift.selectOnly( + drift.localAssetEntity, + )..addColumns([drift.localAssetEntity.id, drift.localAssetEntity.createdAt])).get()) + row.read(drift.localAssetEntity.id)!: row.read(drift.localAssetEntity.createdAt)!, + }; + final trashedDates = { + for (final row in await (drift.selectOnly( + drift.trashedLocalAssetEntity, + )..addColumns([drift.trashedLocalAssetEntity.id, drift.trashedLocalAssetEntity.createdAt])).get()) + row.read(drift.trashedLocalAssetEntity.id)!: row.read(drift.trashedLocalAssetEntity.createdAt)!, + }; + for (final chunk in dates.entries.slices(kDriftMaxChunk)) { + await drift.batch((batch) { + for (final entry in chunk) { + final localDate = localDates[entry.key]; + if (localDate != null && !localDate.isAtSameMomentAs(entry.value)) { + batch.update( + drift.localAssetEntity, + LocalAssetEntityCompanion(createdAt: Value(entry.value)), + where: (row) => row.id.equals(entry.key), + ); + } + final trashedDate = trashedDates[entry.key]; + if (trashedDate != null && !trashedDate.isAtSameMomentAs(entry.value)) { + batch.update( + drift.trashedLocalAssetEntity, + TrashedLocalAssetEntityCompanion(createdAt: Value(entry.value)), + where: (row) => row.id.equals(entry.key), + ); + } + } + }); + } + }); + } catch (error, stackTrace) { + Logger('Migration').warning("Error migrating to version 27", error, stackTrace); + } +} + Future _migrateTo25() async { final accessToken = Store.tryGet(StoreKey.accessToken); if (accessToken == null || accessToken.isEmpty) { diff --git a/mobile/test/medium/repository_context.dart b/mobile/test/medium/repository_context.dart index c602c64ba..79e4426c6 100644 --- a/mobile/test/medium/repository_context.dart +++ b/mobile/test/medium/repository_context.dart @@ -317,6 +317,7 @@ class MediumRepositoryContext { TrashOrigin? source, AssetType? type, DateTime? createdAt, + DateTime? updatedAt, bool? isFavorite, }) async { id ??= TestUtils.uuid(); @@ -332,6 +333,7 @@ class MediumRepositoryContext { source: .new(source ?? TrashOrigin.remoteSync), isFavorite: .new(isFavorite ?? false), createdAt: .new(TestUtils.date(createdAt)), + updatedAt: .new(TestUtils.date(updatedAt)), ), ); } diff --git a/mobile/test/modules/utils/migration_test.dart b/mobile/test/modules/utils/migration_test.dart new file mode 100644 index 000000000..a1b5fdff5 --- /dev/null +++ b/mobile/test/modules/utils/migration_test.dart @@ -0,0 +1,153 @@ +import 'package:flutter/foundation.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:immich_mobile/data/db/main/table/local/trashed_asset.dart'; +import 'package:immich_mobile/domain/models/store.model.dart'; +import 'package:immich_mobile/domain/services/store.service.dart'; +import 'package:immich_mobile/entities/store.entity.dart'; +import 'package:immich_mobile/infrastructure/repositories/store.repository.dart'; +import 'package:immich_mobile/platform/native_sync_api.g.dart'; +import 'package:immich_mobile/platform/permission_api.g.dart'; +import 'package:immich_mobile/utils/migration.dart'; +import 'package:mocktail/mocktail.dart'; + +import '../../medium/repository_context.dart'; +import '../../service.mocks.dart'; + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + + late MediumRepositoryContext ctx; + late MockNativeSyncApi nativeSyncApi; + late MockPermissionApi permissionApi; + + setUpAll(() async { + debugDefaultTargetPlatformOverride = TargetPlatform.android; + ctx = MediumRepositoryContext(); + await StoreService.init(storeRepository: StoreRepository(ctx.db), listenUpdates: false); + }); + + setUp(() async { + nativeSyncApi = MockNativeSyncApi(); + permissionApi = MockPermissionApi(); + await Store.clear(); + await ctx.db.delete(ctx.db.localAssetEntity).go(); + await ctx.db.delete(ctx.db.trashedLocalAssetEntity).go(); + when(() => permissionApi.hasManageMediaPermission()).thenAnswer((_) async => true); + when( + () => nativeSyncApi.getAlbums(), + ).thenAnswer((_) async => [PlatformAlbum(id: 'album', name: 'album', isCloud: false, assetCount: 1)]); + when(() => nativeSyncApi.getAssetsForAlbum('album')).thenAnswer((_) async => []); + when(() => nativeSyncApi.getTrashedAssets()).thenAnswer((_) async => {}); + }); + + tearDownAll(() async { + debugDefaultTargetPlatformOverride = null; + await Store.clear(); + await ctx.dispose(); + }); + + test('heals dates from MediaStore', () async { + final wrongDate = DateTime.utc(2026); + final platformDate = DateTime.utc(2019, 3, 15, 10, 30); + await Store.put(StoreKey.version, 26); + await ctx.newLocalAsset(id: 'local', createdAt: wrongDate, updatedAt: platformDate); + await ctx.newTrashedLocalAsset( + id: 'trashed', + albumId: 'album', + createdAt: wrongDate, + updatedAt: platformDate, + source: TrashOrigin.localSync, + ); + when(() => nativeSyncApi.getAssetsForAlbum('album')).thenAnswer((_) async => [_asset('local', platformDate)]); + // the generated api hands back a cast view over plain lists + when(() => nativeSyncApi.getTrashedAssets()).thenAnswer( + (_) async => { + 'album': [_asset('trashed', platformDate)], + }.cast>(), + ); + + await migrateDatabaseIfNeeded(ctx.db, nativeSyncApi, permissionApi); + + final local = await (ctx.db.select(ctx.db.localAssetEntity)..where((row) => row.id.equals('local'))).getSingle(); + final trashed = await (ctx.db.select( + ctx.db.trashedLocalAssetEntity, + )..where((row) => row.id.equals('trashed'))).getSingle(); + expect(local.createdAt, platformDate); + expect(trashed.createdAt, platformDate); + expect(Store.tryGet(StoreKey.version), 27); + }); + + test('keeps EXIF date without trash access', () async { + final modifiedAt = DateTime.utc(2025); + final takenAt = DateTime.utc(2026); + await Store.put(StoreKey.version, 26); + await ctx.newLocalAsset(id: 'exif', createdAt: takenAt, updatedAt: modifiedAt); + when( + () => nativeSyncApi.getAssetsForAlbum('album'), + ).thenAnswer((_) async => [_asset('exif', takenAt, updatedAt: modifiedAt)]); + when(() => permissionApi.hasManageMediaPermission()).thenAnswer((_) async => false); + + await migrateDatabaseIfNeeded(ctx.db, nativeSyncApi, permissionApi); + + final asset = await (ctx.db.select(ctx.db.localAssetEntity)..where((row) => row.id.equals('exif'))).getSingle(); + expect(asset.createdAt, takenAt); + expect(Store.tryGet(StoreKey.version), 27); + verifyNever(() => nativeSyncApi.getTrashedAssets()); + }); + + test('completes when MediaStore is empty', () async { + final wrongDate = DateTime.utc(2026); + await Store.put(StoreKey.version, 26); + await ctx.newLocalAsset(id: 'local', createdAt: wrongDate); + when(() => nativeSyncApi.getAlbums()).thenAnswer((_) async => []); + + await migrateDatabaseIfNeeded(ctx.db, nativeSyncApi, permissionApi); + + final asset = await (ctx.db.select(ctx.db.localAssetEntity)..where((row) => row.id.equals('local'))).getSingle(); + expect(asset.createdAt, wrongDate); + expect(Store.tryGet(StoreKey.version), 27); + verify(() => nativeSyncApi.getTrashedAssets()).called(1); + }); + + test('completes when MediaStore read fails', () async { + final wrongDate = DateTime.utc(2026); + await Store.put(StoreKey.version, 26); + await ctx.newLocalAsset(id: 'local', createdAt: wrongDate); + when(() => nativeSyncApi.getAlbums()).thenThrow(StateError('query failed')); + + await migrateDatabaseIfNeeded(ctx.db, nativeSyncApi, permissionApi); + + final asset = await (ctx.db.select(ctx.db.localAssetEntity)..where((row) => row.id.equals('local'))).getSingle(); + expect(asset.createdAt, wrongDate); + expect(Store.tryGet(StoreKey.version), 27); + }); + + test('skips out-of-range date and completes migration', () async { + final wrongDate = DateTime.utc(2026); + await Store.put(StoreKey.version, 26); + await ctx.newLocalAsset(id: 'local', createdAt: wrongDate); + when( + () => nativeSyncApi.getAssetsForAlbum('album'), + ).thenAnswer((_) async => [_asset('local', wrongDate, createdAtSeconds: 8640000000001)]); + + await migrateDatabaseIfNeeded(ctx.db, nativeSyncApi, permissionApi); + + final asset = await (ctx.db.select(ctx.db.localAssetEntity)..where((row) => row.id.equals('local'))).getSingle(); + expect(asset.createdAt, wrongDate); + expect(Store.tryGet(StoreKey.version), 27); + }); +} + +class MockPermissionApi extends Mock implements PermissionApi {} + +PlatformAsset _asset(String id, DateTime createdAt, {DateTime? updatedAt, int? createdAtSeconds}) => PlatformAsset( + id: id, + name: '$id.jpg', + type: 1, + createdAt: createdAtSeconds ?? createdAt.millisecondsSinceEpoch ~/ 1000, + updatedAt: (updatedAt ?? createdAt).millisecondsSinceEpoch ~/ 1000, + durationMs: 0, + orientation: 0, + isFavorite: false, + playbackStyle: PlatformAssetPlaybackStyle.image, +);