From 79b175465e7f23a695ea766a01c7fdc527984918 Mon Sep 17 00:00:00 2001 From: Baneina <47112988+pauverblom@users.noreply.github.com> Date: Thu, 29 Oct 2026 14:29:46 +0100 Subject: [PATCH] fix(server): normalize bounding boxes when matching detected faces to existing faces (#32486) (cherry picked from commit 5583cd7857a414f10039a9fb5bf03867bd4390c8) --- server/src/services/person.service.spec.ts | 60 ++++++++++++++++++++++ server/src/services/person.service.ts | 48 +++++++++++------ 2 files changed, 92 insertions(+), 16 deletions(-) diff --git a/server/src/services/person.service.spec.ts b/server/src/services/person.service.spec.ts index 29946004c..2f75230a9 100644 --- a/server/src/services/person.service.spec.ts +++ b/server/src/services/person.service.spec.ts @@ -1213,6 +1213,66 @@ describe(PersonService.name, () => { expect(mocks.person.reassignFaces).not.toHaveBeenCalled(); }); + it('should match metadata face stored at a different resolution', async () => { + const face = AssetFaceFactory.create({ + sourceType: SourceType.Exif, + imageWidth: 2736, + imageHeight: 3648, + boundingBoxX1: 1225, + boundingBoxY1: 1453, + boundingBoxX2: 1436, + boundingBoxY2: 1732, + }); + const asset = AssetFactory.from().face(face).file({ type: AssetFileType.Preview }).exif().build(); + mocks.machineLearning.detectFaces.mockResolvedValue({ + imageWidth: 1440, + imageHeight: 1920, + faces: [{ boundingBox: { x1: 645, y1: 765, x2: 756, y2: 912 }, embedding: '[1, 2, 3, 4]', score: 0.9 }], + }); + mocks.assetJob.getForDetectFacesJob.mockResolvedValue(getForDetectedFaces(asset)); + + await sut.handleDetectFaces({ id: asset.id }); + + expect(mocks.person.refreshFaces).toHaveBeenCalledWith([], [], [{ faceId: face.id, embedding: '[1, 2, 3, 4]' }]); + expect(mocks.job.queueAll).not.toHaveBeenCalled(); + }); + + it('should prefer metadata face over overlapping machine learning face', async () => { + const mlFace = AssetFaceFactory.create({ + imageWidth: 1440, + imageHeight: 1920, + boundingBoxX1: 645, + boundingBoxY1: 765, + boundingBoxX2: 756, + boundingBoxY2: 912, + }); + const exifFace = AssetFaceFactory.create({ + sourceType: SourceType.Exif, + imageWidth: 2736, + imageHeight: 3648, + boundingBoxX1: 1225, + boundingBoxY1: 1453, + boundingBoxX2: 1436, + boundingBoxY2: 1732, + }); + const asset = AssetFactory.from() + .face(mlFace) + .face(exifFace) + .file({ type: AssetFileType.Preview }) + .exif() + .build(); + mocks.machineLearning.detectFaces.mockResolvedValue(getAsDetectedFace(mlFace)); + mocks.assetJob.getForDetectFacesJob.mockResolvedValue(getForDetectedFaces(asset)); + + await sut.handleDetectFaces({ id: asset.id }); + + expect(mocks.person.refreshFaces).toHaveBeenCalledWith( + [], + [mlFace.id], + [{ faceId: exifFace.id, embedding: '[1, 2, 3, 4]' }], + ); + }); + it('should not add embedding to non-matching metadata face', async () => { const assetId = newUuid(); const face = AssetFaceFactory.create({ assetId, sourceType: SourceType.Exif }); diff --git a/server/src/services/person.service.ts b/server/src/services/person.service.ts index 3aaa34dea..1fcf6eb16 100644 --- a/server/src/services/person.service.ts +++ b/server/src/services/person.service.ts @@ -414,16 +414,10 @@ export class PersonService extends BaseService { } } - const heightScale = imageHeight / (asset.faces[0]?.imageHeight || 1); - const widthScale = imageWidth / (asset.faces[0]?.imageWidth || 1); for (const { boundingBox, embedding } of faces) { - const scaledBox = { - x1: boundingBox.x1 * widthScale, - y1: boundingBox.y1 * heightScale, - x2: boundingBox.x2 * widthScale, - y2: boundingBox.y2 * heightScale, - }; - const match = asset.faces.find((face) => this.iou(face, scaledBox) > 0.5); + const matches = asset.faces.filter((face) => this.iou(face, boundingBox, imageWidth, imageHeight) > 0.5); + // prefer metadata faces over machine learning duplicates + const match = matches.find((face) => face.sourceType !== SourceType.MachineLearning) ?? matches[0]; if (match && !mlFaceIds.delete(match.id)) { embeddings.push({ faceId: match.id, embedding }); @@ -466,17 +460,39 @@ export class PersonService extends BaseService { } private iou( - face: { boundingBoxX1: number; boundingBoxY1: number; boundingBoxX2: number; boundingBoxY2: number }, + face: { + boundingBoxX1: number; + boundingBoxY1: number; + boundingBoxX2: number; + boundingBoxY2: number; + imageWidth: number; + imageHeight: number; + }, newBox: BoundingBox, + imageWidth: number, + imageHeight: number, ): number { - const x1 = Math.max(face.boundingBoxX1, newBox.x1); - const y1 = Math.max(face.boundingBoxY1, newBox.y1); - const x2 = Math.min(face.boundingBoxX2, newBox.x2); - const y2 = Math.min(face.boundingBoxY2, newBox.y2); + const box1 = { + x1: face.boundingBoxX1 / face.imageWidth, + y1: face.boundingBoxY1 / face.imageHeight, + x2: face.boundingBoxX2 / face.imageWidth, + y2: face.boundingBoxY2 / face.imageHeight, + }; + const box2 = { + x1: newBox.x1 / imageWidth, + y1: newBox.y1 / imageHeight, + x2: newBox.x2 / imageWidth, + y2: newBox.y2 / imageHeight, + }; + + const x1 = Math.max(box1.x1, box2.x1); + const y1 = Math.max(box1.y1, box2.y1); + const x2 = Math.min(box1.x2, box2.x2); + const y2 = Math.min(box1.y2, box2.y2); const intersection = Math.max(0, x2 - x1) * Math.max(0, y2 - y1); - const area1 = (face.boundingBoxX2 - face.boundingBoxX1) * (face.boundingBoxY2 - face.boundingBoxY1); - const area2 = (newBox.x2 - newBox.x1) * (newBox.y2 - newBox.y1); + const area1 = (box1.x2 - box1.x1) * (box1.y2 - box1.y1); + const area2 = (box2.x2 - box2.x1) * (box2.y2 - box2.y1); const union = area1 + area2 - intersection; return intersection / union;