sevenhill/Immich
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(mobile): stack edited photos over the original on upload (#31082)

Browse Source 
* feat(mobile): stack edited photos over the original on upload

* stack through the asset service and give the live photo still its task metadata

* add plain photo stacking tests and the 2x2 edit detection matrix
This commit is contained in:
Santo Shakil
2026-09-15 11:22:14 -07:00
committed by GitHub
parent ab5a9ae48c
commit 962ffaacc1
13 changed files with 3913 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mobile/drift_schemas/main/drift_schema_v32.json
Generated
+3636
View File
File diff suppressed because it is too large Load Diff
mobile/lib/data/db/main/database.dart
+4 -1
View File
@@ -159,7 +159,7 @@ class Drift extends $Drift {
}
@override
int get schemaVersion => 31;
int get schemaVersion => 32;
@override
MigrationStrategy get migration => MigrationStrategy(
@@ -357,6 +357,9 @@ class Drift extends $Drift {
from30To31: (m, v31) async {
await m.createIndex(v31.idxRemoteAssetUploaded);
},
from31To32: (m, v32) async {
await m.addColumn(v32.localAssetEntity, v32.localAssetEntity.previousChecksum);
},
),
),
);
mobile/lib/data/db/main/table/local/asset.dart
+4
View File
@@ -28,6 +28,10 @@ class LocalAssetEntity extends Table with DriftDefaultsMixin, AssetEntityMixin {
IntColumn get playbackStyle => intEnum<AssetPlaybackStyle>().withDefault(const Constant(0))();
/// Checksum this asset had before its bytes last changed, so the upload of the
/// edited asset can be stacked over the remote asset that still has the old bytes.
TextColumn get previousChecksum => text().nullable()();
@override
Set<Column> get primaryKey => {id};
}
mobile/lib/domain/services/asset.service.dart
+7
View File
@@ -197,4 +197,11 @@ class AssetService {
Future<LocalAsset?> getLocalAsset(String id) {
return _localRepository.get(id);
}
Future<void> stackEditedUpload(String localId, String remoteId) async {
final previousId = await _localRepository.getPreviousRemoteId(localId);
if (previousId != null) {
await _apiRepository.stack([remoteId, previousId]);
}
}
}
mobile/lib/infrastructure/repositories/local_album.repository.dart
+10 -4
View File
@@ -284,9 +284,10 @@ class LocalAlbumRepository extends DatabaseAccessor<Drift> with $LocalAlbumRepos
// Reset checksum if asset changed
await _db.batch((batch) async {
for (final asset in localAssets) {
final companion = LocalAssetEntityCompanion(
checksum: const Value(null),
adjustmentTime: Value(asset.adjustmentTime),
final companion = LocalAssetEntityCompanion.custom(
checksum: const Constant(null),
adjustmentTime: Variable(asset.adjustmentTime),
previousChecksum: _db.localAssetEntity.checksum,
);
batch.update(
_db.localAssetEntity,
@@ -347,7 +348,12 @@ class LocalAlbumRepository extends DatabaseAccessor<Drift> with $LocalAlbumRepos
batch.insert<$LocalAssetEntityTable, LocalAssetEntityData>(
_db.localAssetEntity,
companion,
onConflict: DoUpdate((_) => companion, where: (old) => old.updatedAt.isNotValue(asset.updatedAt)),
onConflict: DoUpdate(
// SET reads the existing row, so this keeps the checksum being cleared
(_) =>
RawValuesInsertable({...companion.toColumns(true), 'previous_checksum': _db.localAssetEntity.checksum}),
where: (old) => old.updatedAt.isNotValue(asset.updatedAt),
),
);
}
});
mobile/lib/infrastructure/repositories/local_asset.repository.dart
+16
View File
@@ -238,4 +238,20 @@ class LocalAssetRepository extends DatabaseAccessor<Drift> with $LocalAssetRepos
updateKind: UpdateKind.update,
);
}
Future<String?> getPreviousRemoteId(String id) {
final query = _db.selectOnly(_db.localAssetEntity)
..join([
innerJoin(
_db.remoteAssetEntity,
_db.remoteAssetEntity.checksum.equalsExp(_db.localAssetEntity.previousChecksum),
),
innerJoin(_db.authUserEntity, _db.authUserEntity.id.equalsExp(_db.remoteAssetEntity.ownerId)),
])
..addColumns([_db.remoteAssetEntity.id])
..where(_db.localAssetEntity.id.equals(id))
..limit(1);
return query.map((row) => row.read(_db.remoteAssetEntity.id)).getSingleOrNull();
}
}
mobile/lib/services/background_upload.service.dart
+29
View File
@@ -10,12 +10,14 @@ import 'package:immich_mobile/constants/constants.dart';
import 'package:immich_mobile/domain/models/asset/asset_metadata.model.dart';
import 'package:immich_mobile/domain/models/asset/base_asset.model.dart';
import 'package:immich_mobile/domain/models/store.model.dart';
import 'package:immich_mobile/domain/services/asset.service.dart';
import 'package:immich_mobile/entities/store.entity.dart';
import 'package:immich_mobile/extensions/platform_extensions.dart';
import 'package:immich_mobile/infrastructure/repositories/backup.repository.dart';
import 'package:immich_mobile/infrastructure/repositories/local_asset.repository.dart';
import 'package:immich_mobile/infrastructure/repositories/settings.repository.dart';
import 'package:immich_mobile/infrastructure/repositories/storage.repository.dart';
import 'package:immich_mobile/providers/infrastructure/asset.provider.dart';
import 'package:immich_mobile/providers/infrastructure/db.provider.dart';
import 'package:immich_mobile/providers/infrastructure/storage.provider.dart';
import 'package:immich_mobile/repositories/asset_media.repository.dart';
@@ -36,6 +38,7 @@ final backgroundUploadServiceProvider = Provider((ref) {
db.localAssetRepository,
db.backupRepository,
ref.watch(assetMediaRepositoryProvider),
ref.watch(assetServiceProvider),
);
ref.onDispose(service.dispose);
@@ -86,6 +89,7 @@ class BackgroundUploadService {
this._localAssetRepository,
this._backupRepository,
this._assetMediaRepository,
this._assetService,
) {
_uploadRepository.onUploadStatus = _onUploadCallback;
_uploadRepository.onTaskProgress = _onTaskProgressCallback;
@@ -96,6 +100,7 @@ class BackgroundUploadService {
final LocalAssetRepository _localAssetRepository;
final BackupRepository _backupRepository;
final AssetMediaRepository _assetMediaRepository;
final AssetService _assetService;
final Logger _logger = Logger('BackgroundUploadService');
final StreamController<TaskStatusUpdate> _taskStatusController = StreamController<TaskStatusUpdate>.broadcast();
@@ -190,6 +195,7 @@ class BackgroundUploadService {
switch (update.status) {
case TaskStatus.complete:
unawaited(_handleLivePhoto(update));
unawaited(_stackEditedAsset(update));
if (CurrentPlatform.isIOS) {
try {
@@ -238,6 +244,24 @@ class BackgroundUploadService {
}
}
Future<void> _stackEditedAsset(TaskStatusUpdate update) async {
try {
if (update.task.metaData.isEmpty || update.responseBody == null || update.responseBody!.isEmpty) {
return;
}
final metadata = UploadTaskMetadata.fromJson(update.task.metaData);
// The video half of a live photo carries isLivePhotos; the still that follows is the final asset
if (metadata.isLivePhotos) {
return;
}
await _assetService.stackEditedUpload(metadata.localAssetId, jsonDecode(update.responseBody!)['id'] as String);
} catch (error, stackTrace) {
dPrint(() => "Error stacking edited asset upload: $error $stackTrace");
}
}
@visibleForTesting
Future<UploadTask?> getUploadTask(LocalAsset asset, {String group = kBackupGroup, int? priority}) async {
final entity = await _storageRepository.getAssetEntityForAsset(asset);
@@ -325,6 +349,11 @@ class BackgroundUploadService {
modifiedAt: asset.updatedAt,
originalFileName: originalFileName,
deviceAssetId: asset.id,
metadata: UploadTaskMetadata(
localAssetId: asset.id,
isLivePhotos: false,
livePhotoVideoId: livePhotoVideoId,
).toJson(),
fields: fields,
group: kBackupLivePhotoGroup,
priority: 0, // Highest priority to get upload immediately
mobile/lib/services/foreground_upload.service.dart
+10
View File
@@ -7,6 +7,7 @@ import 'package:hooks_riverpod/hooks_riverpod.dart';
import 'package:immich_mobile/domain/models/asset/asset_metadata.model.dart';
import 'package:immich_mobile/domain/models/asset/base_asset.model.dart' hide AssetVisibility;
import 'package:immich_mobile/domain/models/store.model.dart';
import 'package:immich_mobile/domain/services/asset.service.dart';
import 'package:immich_mobile/entities/store.entity.dart';
import 'package:immich_mobile/extensions/network_capability_extensions.dart';
import 'package:immich_mobile/extensions/platform_extensions.dart';
@@ -15,6 +16,7 @@ import 'package:immich_mobile/infrastructure/repositories/backup.repository.dart
import 'package:immich_mobile/infrastructure/repositories/settings.repository.dart';
import 'package:immich_mobile/infrastructure/repositories/storage.repository.dart';
import 'package:immich_mobile/platform/connectivity_api.g.dart';
import 'package:immich_mobile/providers/infrastructure/asset.provider.dart';
import 'package:immich_mobile/providers/infrastructure/db.provider.dart';
import 'package:immich_mobile/providers/infrastructure/platform.provider.dart';
import 'package:immich_mobile/providers/infrastructure/storage.provider.dart';
@@ -43,6 +45,7 @@ final foregroundUploadServiceProvider = Provider((ref) {
ref.watch(driftProvider).backupRepository,
ref.watch(connectivityApiProvider),
ref.watch(assetMediaRepositoryProvider),
ref.watch(assetServiceProvider),
);
});
@@ -58,6 +61,7 @@ class ForegroundUploadService {
this._backupRepository,
this._connectivityApi,
this._assetMediaRepository,
this._assetService,
);
final UploadRepository _uploadRepository;
@@ -65,6 +69,7 @@ class ForegroundUploadService {
final BackupRepository _backupRepository;
final ConnectivityApi _connectivityApi;
final AssetMediaRepository _assetMediaRepository;
final AssetService _assetService;
final Logger _logger = Logger('ForegroundUploadService');
bool shouldAbortUpload = false;
@@ -379,6 +384,11 @@ class ForegroundUploadService {
if (result.isSuccess && result.remoteAssetId != null) {
callbacks.onSuccess?.call(asset.localId!, result.remoteAssetId!);
try {
await _assetService.stackEditedUpload(asset.localId!, result.remoteAssetId!);
} catch (error) {
_logger.warning("Failed to stack the upload of ${asset.localId}: $error");
}
} else if (result.isCancelled) {
shouldAbortUpload = true;
} else if (result.errorMessage != null) {
mobile/test/medium/repositories/local_album_repository_test.dart
+78
View File
@@ -0,0 +1,78 @@
import 'package:flutter/foundation.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:immich_mobile/domain/models/album/local_album.model.dart';
import 'package:immich_mobile/domain/models/asset/base_asset.model.dart';
import 'package:immich_mobile/infrastructure/repositories/local_album.repository.dart';
import '../repository_context.dart';
void main() {
late MediumRepositoryContext ctx;
late LocalAlbumRepository sut;
setUp(() {
ctx = MediumRepositoryContext();
sut = LocalAlbumRepository(ctx.db);
});
tearDown(() async {
debugDefaultTargetPlatformOverride = null;
await ctx.dispose();
});
group('upsert', () {
final album = LocalAlbum(id: 'album', name: 'Camera', updatedAt: DateTime(2024));
Future<({String? checksum, String? previousChecksum})> row(String id) async {
final data = await ctx.db.managers.localAssetEntity.filter((row) => row.id.equals(id)).getSingle();
return (checksum: data.checksum, previousChecksum: data.previousChecksum);
}
for (final (platform, scenario, edited, expected) in [
(
TargetPlatform.android,
'updatedAt changed clears the checksum and remembers it',
_localAsset('edited', updatedAt: DateTime(2025)),
(checksum: null, previousChecksum: 'a'),
),
(
TargetPlatform.android,
'adjustmentTime changed keeps the checksum',
_localAsset('edited', adjustmentTime: DateTime(2025)),
(checksum: 'a', previousChecksum: null),
),
(
TargetPlatform.iOS,
'adjustmentTime changed clears the checksum and remembers it',
_localAsset('edited', adjustmentTime: DateTime(2025)),
(checksum: null, previousChecksum: 'a'),
),
(
TargetPlatform.iOS,
'updatedAt changed keeps the checksum',
_localAsset('edited', updatedAt: DateTime(2025)),
(checksum: 'a', previousChecksum: null),
),
]) {
test('${platform.name}: $scenario', () async {
debugDefaultTargetPlatformOverride = platform;
await ctx.newLocalAsset(id: 'edited', checksum: 'a', updatedAt: DateTime(2024), adjustmentTime: DateTime(2024));
await sut.upsert(album, toUpsert: [edited]);
expect(await row('edited'), expected);
});
}
});
}
LocalAsset _localAsset(String id, {DateTime? updatedAt, DateTime? adjustmentTime}) => LocalAsset(
id: id,
name: '$id.jpg',
type: AssetType.image,
createdAt: DateTime(2024),
updatedAt: updatedAt ?? DateTime(2024),
adjustmentTime: adjustmentTime ?? DateTime(2024),
playbackStyle: AssetPlaybackStyle.image,
isEdited: false,
);
mobile/test/medium/repositories/local_asset_repository_test.dart
+17
View File
@@ -634,4 +634,21 @@ void main() {
expect(updated?.checksum, isNull);
});
});
group('getPreviousRemoteId', () {
test('finds the owned remote asset that has the previous checksum', () async {
final me = await ctx.newUser();
final partner = await ctx.newUser();
await ctx.newAuthUser(id: me.id);
final original = await ctx.newRemoteAsset(ownerId: me.id, checksum: 'a');
await ctx.newRemoteAsset(ownerId: partner.id, checksum: 'z');
final edited = await ctx.newLocalAsset(checksum: 'b', previousChecksum: 'a');
final partnerEdited = await ctx.newLocalAsset(checksum: 'c', previousChecksum: 'z');
final untouched = await ctx.newLocalAsset(checksum: 'd');
expect(await sut.getPreviousRemoteId(edited.id), original.id);
expect(await sut.getPreviousRemoteId(partnerEdited.id), isNull);
expect(await sut.getPreviousRemoteId(untouched.id), isNull);
});
});
}
mobile/test/medium/repository_context.dart
+2
View File
@@ -268,6 +268,7 @@ class MediumRepositoryContext {
String? name,
String? checksum,
Option<String>? checksumOption,
String? previousChecksum,
DateTime? createdAt,
AssetType? type,
bool? isFavorite,
@@ -296,6 +297,7 @@ class MediumRepositoryContext {
orientation: .new(orientation ?? 0),
updatedAt: .new(TestUtils.date(updatedAt)),
checksum: _resolveUndefined(checksum, checksumOption, const Uuid().v4()),
previousChecksum: .new(previousChecksum),
createdAt: .new(TestUtils.date(createdAt)),
type: .new(type ?? .image),
isFavorite: .new(isFavorite ?? false),
mobile/test/services/background_upload.service_test.dart
+57
View File
@@ -1,6 +1,7 @@
import 'dart:convert';
import 'dart:io';
import 'package:background_downloader/background_downloader.dart';
import 'package:drift/drift.dart' hide isNotNull, isNull;
import 'package:drift/native.dart';
import 'package:flutter/foundation.dart';
@@ -20,6 +21,7 @@ import '../fixtures/asset.stub.dart';
import '../infrastructure/repository.mock.dart';
import '../mocks/asset_entity.mock.dart';
import '../repository.mocks.dart';
import '../service.mocks.dart';
void main() {
late BackgroundUploadService sut;
@@ -28,6 +30,7 @@ void main() {
late MockLocalAssetRepository mockLocalAssetRepository;
late MockBackupRepository mockBackupRepository;
late MockAssetMediaRepository mockAssetMediaRepository;
late MockAssetService mockAssetService;
late Drift db;
setUpAll(() async {
@@ -50,6 +53,8 @@ void main() {
mockLocalAssetRepository = MockLocalAssetRepository();
mockBackupRepository = MockBackupRepository();
mockAssetMediaRepository = MockAssetMediaRepository();
mockAssetService = MockAssetService();
when(() => mockAssetService.stackEditedUpload(any(), any())).thenAnswer((_) async {});
sut = BackgroundUploadService(
mockUploadRepository,
@@ -57,6 +62,7 @@ void main() {
mockLocalAssetRepository,
mockBackupRepository,
mockAssetMediaRepository,
mockAssetService,
);
mockUploadRepository.onUploadStatus = (_) {};
@@ -234,6 +240,7 @@ void main() {
mockLocalAssetRepository,
mockBackupRepository,
mockAssetMediaRepository,
mockAssetService,
);
addTearDown(() => sutWithV24.dispose());
@@ -284,6 +291,7 @@ void main() {
mockLocalAssetRepository,
mockBackupRepository,
mockAssetMediaRepository,
mockAssetService,
);
addTearDown(() => sutAndroid.dispose());
@@ -324,6 +332,7 @@ void main() {
mockLocalAssetRepository,
mockBackupRepository,
mockAssetMediaRepository,
mockAssetService,
);
addTearDown(() => sutWithV24.dispose());
@@ -364,6 +373,7 @@ void main() {
mockLocalAssetRepository,
mockBackupRepository,
mockAssetMediaRepository,
mockAssetService,
);
addTearDown(() => sutWithV24.dispose());
@@ -403,4 +413,51 @@ void main() {
expect(metadata[0]['value']['iCloudId'], equals('cloud-id-livephoto'));
});
});
group('onUploadStatus', () {
test('stacks a plain photo after its upload', () async {
final asset = LocalAssetStub.image1;
final mockEntity = MockAssetEntity();
final mockFile = File('/path/to/photo.jpg');
final void Function(TaskStatusUpdate) onStatus = verify(
() => mockUploadRepository.onUploadStatus = captureAny(),
).captured.first;
when(() => mockEntity.isLivePhoto).thenReturn(false);
when(() => mockStorageRepository.getAssetEntityForAsset(asset)).thenAnswer((_) async => mockEntity);
when(() => mockStorageRepository.getFileForAsset(asset.id)).thenAnswer((_) async => mockFile);
when(() => mockAssetMediaRepository.getOriginalFilename(asset.id)).thenAnswer((_) async => 'photo.jpg');
final task = await sut.getUploadTask(asset);
onStatus(TaskStatusUpdate(task!, TaskStatus.complete, null, '{"id": "remote"}'));
verify(() => mockAssetService.stackEditedUpload(asset.id, 'remote')).called(1);
verifyNoMoreInteractions(mockAssetService);
});
test('stacks the still of a live photo, not its video', () async {
final asset = LocalAssetStub.image1;
final mockEntity = MockAssetEntity();
final stillFile = File('/path/to/still.heic');
final videoFile = File('/path/to/motion.mov');
final void Function(TaskStatusUpdate) onStatus = verify(
() => mockUploadRepository.onUploadStatus = captureAny(),
).captured.first;
when(() => mockEntity.isLivePhoto).thenReturn(true);
when(() => mockStorageRepository.getAssetEntityForAsset(asset)).thenAnswer((_) async => mockEntity);
when(() => mockStorageRepository.getMotionFileForAsset(asset)).thenAnswer((_) async => videoFile);
when(() => mockStorageRepository.getFileForAsset(asset.id)).thenAnswer((_) async => stillFile);
when(() => mockAssetMediaRepository.getOriginalFilename(asset.id)).thenAnswer((_) async => 'live.heic');
when(() => mockLocalAssetRepository.getById(asset.id)).thenAnswer((_) async => null);
final video = await sut.getUploadTask(asset);
final still = await sut.getLivePhotoUploadTask(asset, 'video');
onStatus(TaskStatusUpdate(video!, TaskStatus.complete, null, '{"id": "video"}'));
onStatus(TaskStatusUpdate(still!, TaskStatus.complete, null, '{"id": "still"}'));
verify(() => mockAssetService.stackEditedUpload(asset.id, 'still')).called(1);
verifyNoMoreInteractions(mockAssetService);
});
});
}
mobile/test/services/foreground_upload.service_test.dart
+43
View File
@@ -19,6 +19,7 @@ import '../fixtures/asset.stub.dart';
import '../infrastructure/repository.mock.dart';
import '../mocks/asset_entity.mock.dart';
import '../repository.mocks.dart';
import '../service.mocks.dart';
void main() {
late ForegroundUploadService sut;
@@ -27,6 +28,7 @@ void main() {
late MockBackupRepository mockBackupRepository;
late MockConnectivityApi mockConnectivityApi;
late MockAssetMediaRepository mockAssetMediaRepository;
late MockAssetService mockAssetService;
late Drift db;
setUpAll(() async {
@@ -52,6 +54,8 @@ void main() {
mockBackupRepository = MockBackupRepository();
mockConnectivityApi = MockConnectivityApi();
mockAssetMediaRepository = MockAssetMediaRepository();
mockAssetService = MockAssetService();
when(() => mockAssetService.stackEditedUpload(any(), any())).thenAnswer((_) async {});
sut = ForegroundUploadService(
mockUploadRepository,
@@ -59,6 +63,7 @@ void main() {
mockBackupRepository,
mockConnectivityApi,
mockAssetMediaRepository,
mockAssetService,
);
});
@@ -196,5 +201,43 @@ void main() {
expect(names, equals(['DJI_0001.jpg']));
});
test('stacks a plain photo after its upload', () async {
final asset = LocalAssetStub.image1;
final mockEntity = MockAssetEntity();
final stillFile = File('/path/to/photo.jpg');
when(() => mockEntity.isLivePhoto).thenReturn(false);
when(() => mockStorageRepository.getAssetEntityForAsset(asset)).thenAnswer((_) async => mockEntity);
when(() => mockStorageRepository.isAssetAvailableLocally(asset.id)).thenAnswer((_) async => true);
when(() => mockStorageRepository.getFileForAsset(asset.id)).thenAnswer((_) async => stillFile);
when(() => mockAssetMediaRepository.getOriginalFilename(asset.id)).thenAnswer((_) async => 'photo.jpg');
captureFields();
await sut.uploadSingleAsset(asset, null, callbacks: const UploadCallbacks());
verify(() => mockAssetService.stackEditedUpload(asset.localId!, 'remote-1')).called(1);
verifyNoMoreInteractions(mockAssetService);
});
test('stacks the still of a live photo, not its video', () async {
final asset = LocalAssetStub.image1;
final mockEntity = MockAssetEntity();
final stillFile = File('/path/to/still.heic');
final videoFile = File('/path/to/motion.mov');
when(() => mockEntity.isLivePhoto).thenReturn(true);
when(() => mockStorageRepository.getAssetEntityForAsset(asset)).thenAnswer((_) async => mockEntity);
when(() => mockStorageRepository.isAssetAvailableLocally(asset.id)).thenAnswer((_) async => true);
when(() => mockStorageRepository.getFileForAsset(asset.id)).thenAnswer((_) async => stillFile);
when(() => mockStorageRepository.getMotionFileForAsset(asset)).thenAnswer((_) async => videoFile);
when(() => mockAssetMediaRepository.getOriginalFilename(asset.id)).thenAnswer((_) async => 'live.heic');
captureFields();
await sut.uploadSingleAsset(asset, null, callbacks: const UploadCallbacks());
verify(() => mockAssetService.stackEditedUpload(asset.localId!, 'remote-2')).called(1);
verifyNoMoreInteractions(mockAssetService);
});
});
}