fix: face detection of edited assets (#31240)

server/src/queries/asset.job.repository.sql

@@ -465,7 +465,7 @@ select
   ) as "faces",
   (
     select
-      coalesce(json_agg(agg), '[]')
+      to_json(obj)
     from
       (
         select
@@ -477,14 +477,18 @@ select
           "asset_file"
         where
           "asset_file"."assetId" = "asset"."id"
-          and "asset_file"."type" = $1
-      ) as agg
-  ) as "files"
+          and "asset_file"."type" = 'preview'
+        order by
+          "asset_file"."isEdited" desc
+        limit
+          1
+      ) as obj
+  ) as "previewFile"
 from
   "asset"
   inner join "asset_exif" on "asset"."id" = "asset_exif"."assetId"
 where
-  "asset"."id" = $2
+  "asset"."id" = $1
 
 -- AssetJobRepository.getForOcr
 select

server/src/repositories/asset-job.repository.ts

@@ -1,6 +1,6 @@
 import { Injectable } from '@nestjs/common';
 import { Kysely, sql } from 'kysely';
-import { jsonArrayFrom } from 'kysely/helpers/postgres';
+import { jsonArrayFrom, jsonObjectFrom } from 'kysely/helpers/postgres';
 import { InjectKysely } from 'nestjs-kysely';
 import { columns } from 'src/database';
 import { DummyValue, GenerateSql } from 'src/decorators';
@@ -236,7 +236,17 @@ export class AssetJobRepository {
       .select(['asset.id', 'asset.visibility'])
       .$call(withExifInner)
       .select((eb) => withFaces(eb, true, true))
-      .select((eb) => withFiles(eb, AssetFileType.Preview))
+      .select((eb) =>
+        jsonObjectFrom(
+          eb
+            .selectFrom('asset_file')
+            .select(columns.assetFiles)
+            .whereRef('asset_file.assetId', '=', 'asset.id')
+            .where('asset_file.type', '=', sql.lit(AssetFileType.Preview))
+            .orderBy('asset_file.isEdited', 'desc')
+            .limit(sql.lit(1)),
+        ).as('previewFile'),
+      )
       .where('asset.id', '=', id)
       .executeTakeFirst();
   }

server/src/services/person.service.ts

@@ -320,8 +320,8 @@ export class PersonService extends BaseService {
     }
 
     const asset = await this.assetJobRepository.getForDetectFacesJob(id);
-    const previewFile = asset?.files[0];
-    if (!asset || asset.files.length !== 1 || !previewFile) {
+    const previewFile = asset?.previewFile;
+    if (!asset || !previewFile) {
       return JobStatus.Failed;
     }
 

server/test/mappers.ts

@@ -1,6 +1,7 @@
 import { Selectable, ShallowDehydrateObject } from 'kysely';
 import { MapAsset } from 'src/dtos/asset-response.dto';
 import { AssetEditActionItem } from 'src/dtos/editing.dto';
+import { AssetFileType } from 'src/enum';
 import { FaceSearchResult } from 'src/repositories/search.repository';
 import { ActivityTable } from 'src/schema/tables/activity.table';
 import { AssetTable } from 'src/schema/tables/asset.table';
@@ -188,7 +189,10 @@ export const getForDetectedFaces = (asset: ReturnType<AssetFactory['build']>) =>
   visibility: asset.visibility,
   exifInfo: getDehydrated(asset.exifInfo),
   faces: asset.faces.map((face) => getDehydrated(face)),
-  files: asset.files.map((file) => getDehydrated(file)),
+  previewFile: asset.files
+    .filter((file) => file.type === AssetFileType.Preview)
+    .toSorted((a) => (a.isEdited ? -1 : 1))
+    .map((file) => getDehydrated(file))[0],
 });
 
 export const getForSidecarWrite = (asset: ReturnType<AssetFactory['build']>) => ({

server/test/medium/specs/services/person.service.spec.ts

@@ -2,14 +2,16 @@ import { Kysely } from 'kysely';
 import { DateTime } from 'luxon';
 import { AssetEditAction, MirrorAxis } from 'src/dtos/editing.dto';
 import { AssetFaceCreateDto } from 'src/dtos/person.dto';
-import { JobName } from 'src/enum';
+import { AssetFileType, JobName } from 'src/enum';
 import { AccessRepository } from 'src/repositories/access.repository';
 import { AssetEditRepository } from 'src/repositories/asset-edit.repository';
+import { AssetJobRepository } from 'src/repositories/asset-job.repository';
 import { AssetRepository } from 'src/repositories/asset.repository';
 import { ConfigRepository } from 'src/repositories/config.repository';
 import { DatabaseRepository } from 'src/repositories/database.repository';
 import { JobRepository } from 'src/repositories/job.repository';
 import { LoggingRepository } from 'src/repositories/logging.repository';
+import { MachineLearningRepository } from 'src/repositories/machine-learning.repository';
 import { PersonRepository } from 'src/repositories/person.repository';
 import { StorageRepository } from 'src/repositories/storage.repository';
 import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository';
@@ -26,6 +28,7 @@ const setup = (db?: Kysely<DB>) => {
     database: db || defaultDatabase,
     real: [
       AccessRepository,
+      AssetJobRepository,
       ConfigRepository,
       DatabaseRepository,
       PersonRepository,
@@ -33,7 +36,7 @@ const setup = (db?: Kysely<DB>) => {
       AssetEditRepository,
       SystemMetadataRepository,
     ],
-    mock: [JobRepository, LoggingRepository, StorageRepository],
+    mock: [JobRepository, LoggingRepository, StorageRepository, MachineLearningRepository],
   });
 };
 
@@ -99,6 +102,38 @@ describe(PersonService.name, () => {
     });
   });
 
+  describe('handleDetectFaces', () => {
+    it('should prefer an edited preview file', async () => {
+      const { sut, ctx } = setup();
+      const config = await ctx.getConfig();
+      const { user } = await ctx.newUser();
+      const { asset } = await ctx.newAsset({ ownerId: user.id });
+      await ctx.newExif({ assetId: asset.id, description: '' });
+      await ctx.newAssetFile({
+        assetId: asset.id,
+        type: AssetFileType.Preview,
+        isEdited: true,
+        path: 'edited_file.jpg',
+      });
+      await ctx.newAssetFile({
+        assetId: asset.id,
+        type: AssetFileType.Preview,
+        isEdited: false,
+        path: 'unedited_file.jpg',
+      });
+      ctx
+        .getMock(MachineLearningRepository)
+        .detectFaces.mockResolvedValue({ imageHeight: 42, imageWidth: 69, faces: [] });
+
+      await sut.handleDetectFaces({ id: asset.id });
+
+      expect(ctx.getMock(MachineLearningRepository).detectFaces).toHaveBeenCalledWith(
+        'edited_file.jpg',
+        config.machineLearning.facialRecognition,
+      );
+    });
+  });
+
   describe('handleQueueRecognizeFaces', () => {
     it('should delete all people and queue faces for recognition', async () => {
       const { sut, ctx } = setup();