diff --git a/i18n/en.json b/i18n/en.json index 3f5927ee5..db95e6a02 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -2263,6 +2263,8 @@ "trigger": "Trigger", "trigger_asset_metadata_extraction": "Asset Metadata Extraction", "trigger_asset_metadata_extraction_description": "Triggered when the EXIF metadata of an asset is extracted", + "trigger_asset_ocr_extraction": "Asset OCR Extraction", + "trigger_asset_ocr_extraction_description": "Triggered when the OCR text of an asset is extracted", "trigger_asset_tagged": "Asset Tagged", "trigger_asset_tagged_description": "Triggered when a tag is added to an asset", "trigger_asset_uploaded": "Asset Upload", diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index ac9fe5811..eaeeefe46 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -34673,6 +34673,7 @@ "enum": [ "AssetCreate", "AssetMetadataExtraction", + "AssetOcrExtraction", "AssetTagged" ], "type": "string" diff --git a/packages/plugin-core/manifest.json b/packages/plugin-core/manifest.json index ac8ba474d..7866469e7 100644 --- a/packages/plugin-core/manifest.json +++ b/packages/plugin-core/manifest.json @@ -75,6 +75,32 @@ } ], "uiHints": ["SmartAlbum"] + }, + { + "name": "receipts", + "title": "Receipts", + "description": "Automatically add receipt assets to an album", + "trigger": "AssetOcrExtraction", + "steps": [ + { + "method": "immich-plugin-core#assetOcrFilter", + "config": { + "caseSensitive": false, + "pattern": "receipt.+total", + "matchType": "regex", + "matchScope": "full", + "minScore": 0 + } + }, + { + "method": "immich-plugin-core#assetAddToAlbums", + "config": { + "albumName": "Receipts", + "albumIds": [] + } + } + ], + "uiHints": ["SmartAlbum"] } ], "methods": [ @@ -428,6 +454,50 @@ }, "uiHints": ["Filter"] }, + { + "name": "assetOcrFilter", + "title": "Filter by OCR text", + "description": "Filter assets by OCR text pattern using text matching or regular expressions", + "types": ["AssetV1"], + "schema": { + "type": "object", + "properties": { + "pattern": { + "type": "string", + "title": "OCR text pattern", + "description": "Text or regex pattern to match against OCR text" + }, + "matchType": { + "type": "string", + "enum": ["contains", "startsWith", "exact", "regex"], + "default": "contains", + "title": "Match type", + "description": "Type of pattern matching to perform" + }, + "caseSensitive": { + "type": "boolean", + "default": false, + "title": "Case sensitive", + "description": "Whether matching should be case-sensitive" + }, + "minScore": { + "type": "number", + "default": 0, + "title": "Minimum score", + "description": "Minimum OCR confidence score to match" + }, + "matchScope": { + "type": "string", + "enum": ["full", "box"], + "default": "full", + "title": "Match scope", + "description": "Scope of the OCR text to match against" + } + }, + "required": ["pattern"] + }, + "uiHints": ["Filter"] + }, { "name": "assetArchive", "title": "Archive asset", diff --git a/packages/plugin-core/src/index.ts b/packages/plugin-core/src/index.ts index 915efbfb5..95239e29f 100644 --- a/packages/plugin-core/src/index.ts +++ b/packages/plugin-core/src/index.ts @@ -194,6 +194,13 @@ const methods = wrapper({ return { workflow: { continue: hasTimeZone === needsTimeZone } }; }, + assetOcrFilter: ({ config, data }) => { + const entries = (data.asset.ocr ?? []).filter((ocr) => ocr.textScore >= (config.minScore ?? 0)); + const texts = config.matchScope === 'full' ? [entries.map((ocr) => ocr.text).join(' ')] : entries.map((ocr) => ocr.text); + const matched = texts.some((t) => matchValueResult(t, config).workflow?.continue); + return { workflow: { continue: matched } }; + }, + assetTagFilter: ({ config, data }) => { const assetTags = data.asset.tags.map((tag) => tag.id); @@ -254,6 +261,7 @@ const { assetDateFilter, assetLock, assetMissingTimeZoneFilter, + assetOcrFilter, assetTagFilter, assetTypeFilter, assetVisibility, @@ -274,6 +282,7 @@ export { assetDateFilter, assetLock, assetMissingTimeZoneFilter, + assetOcrFilter, assetTagFilter, assetTypeFilter, assetVisibility, diff --git a/packages/plugin-sdk/src/types.ts b/packages/plugin-sdk/src/types.ts index 94c968978..23092de96 100644 --- a/packages/plugin-sdk/src/types.ts +++ b/packages/plugin-sdk/src/types.ts @@ -23,6 +23,7 @@ export type WorkflowEventData = WorkflowEventMap[T]; export enum WorkflowTrigger { AssetCreate = 'AssetCreate', AssetMetadataExtraction = 'AssetMetadataExtraction', + AssetOcrExtraction = 'AssetOcrExtraction', AssetTagged = 'AssetTagged', // PersonRecognized = 'PersonRecognized', } @@ -127,6 +128,11 @@ export type AssetV1 = { tags: string[] | null; updatedAt: string | null; } | null; + ocr?: { + text: string; + textScore: number; + boxScore: number; + }[] }; }; diff --git a/packages/sdk/src/fetch-client.ts b/packages/sdk/src/fetch-client.ts index d089a2033..bc68fcdc4 100644 --- a/packages/sdk/src/fetch-client.ts +++ b/packages/sdk/src/fetch-client.ts @@ -8603,6 +8603,7 @@ export enum WorkflowType { export enum WorkflowTrigger { AssetCreate = "AssetCreate", AssetMetadataExtraction = "AssetMetadataExtraction", + AssetOcrExtraction = "AssetOcrExtraction", AssetTagged = "AssetTagged" } export enum QueueJobStatus { diff --git a/server/src/queries/asset.job.repository.sql b/server/src/queries/asset.job.repository.sql index a383a725f..e068ac044 100644 --- a/server/src/queries/asset.job.repository.sql +++ b/server/src/queries/asset.job.repository.sql @@ -615,6 +615,7 @@ where -- AssetJobRepository.getForOcr select + "asset"."ownerId", "asset"."visibility", ( select diff --git a/server/src/repositories/asset-job.repository.ts b/server/src/repositories/asset-job.repository.ts index 2ce63e610..e440517b8 100644 --- a/server/src/repositories/asset-job.repository.ts +++ b/server/src/repositories/asset-job.repository.ts @@ -255,7 +255,7 @@ export class AssetJobRepository { getForOcr(id: string) { return this.db .selectFrom('asset') - .select((eb) => ['asset.visibility', withFilePath(eb, AssetFileType.Preview).as('previewFile')]) + .select((eb) => ['asset.ownerId', 'asset.visibility', withFilePath(eb, AssetFileType.Preview).as('previewFile')]) .where('asset.id', '=', id) .executeTakeFirst(); } diff --git a/server/src/repositories/event.repository.ts b/server/src/repositories/event.repository.ts index a1c6a4261..60a227a18 100644 --- a/server/src/repositories/event.repository.ts +++ b/server/src/repositories/event.repository.ts @@ -53,6 +53,7 @@ type EventMap = { AssetTrash: [{ assetId: string; userId: string }]; AssetDelete: [{ assetId: string; userId: string }]; AssetMetadataExtracted: [{ assetId: string; userId: string; source?: JobSource }]; + AssetOcrExtracted: [{ assetId: string; userId: string }]; // asset bulk events AssetTrashAll: [{ assetIds: string[]; userId: string }]; diff --git a/server/src/repositories/workflow.repository.ts b/server/src/repositories/workflow.repository.ts index b0c0c932c..7a7147d38 100644 --- a/server/src/repositories/workflow.repository.ts +++ b/server/src/repositories/workflow.repository.ts @@ -1,3 +1,4 @@ +import { WorkflowTrigger } from '@immich/plugin-sdk'; import { Injectable } from '@nestjs/common'; import { jsonArrayFrom, jsonObjectFrom } from 'kysely/helpers/postgres'; import { InjectKysely } from 'nestjs-kysely'; @@ -177,7 +178,7 @@ export class WorkflowRepository { await this.db.deleteFrom('workflow').where('id', '=', id).execute(); } - getForAssetV1(assetId: string) { + getForAssetV1(assetId: string, trigger: WorkflowTrigger) { return this.db .selectFrom('asset') .leftJoin('asset_exif', 'asset_exif.assetId', 'asset.id') @@ -221,6 +222,17 @@ export class WorkflowRepository { ]) .whereRef('asset_exif.assetId', '=', 'asset.id'), ).as('exifInfo'), + ...(trigger === WorkflowTrigger.AssetOcrExtraction + ? [ + jsonArrayFrom( + eb + .selectFrom('asset_ocr') + .select(['asset_ocr.text', 'asset_ocr.textScore', 'asset_ocr.boxScore']) + .whereRef('asset_ocr.assetId', '=', 'asset.id') + .where('isVisible', '=', true), + ).as('ocr'), + ] + : []), ]) .where('id', '=', assetId) .executeTakeFirstOrThrow(); diff --git a/server/src/services/ocr.service.spec.ts b/server/src/services/ocr.service.spec.ts index 9e6ff9a05..4ef2f61e1 100644 --- a/server/src/services/ocr.service.spec.ts +++ b/server/src/services/ocr.service.spec.ts @@ -13,6 +13,7 @@ describe(OcrService.name, () => { mocks.config.getWorker.mockReturnValue(ImmichWorker.Microservices); mocks.assetJob.getForOcr.mockResolvedValue({ + ownerId: 'user-id', visibility: AssetVisibility.Timeline, previewFile: '/uploads/user-id/thumbs/path.jpg', }); @@ -69,16 +70,22 @@ describe(OcrService.name, () => { expect(mocks.asset.getByIds).not.toHaveBeenCalled(); expect(mocks.machineLearning.encodeImage).not.toHaveBeenCalled(); + expect(mocks.event.emit).not.toHaveBeenCalled(); }); it('should skip assets without a resize path', async () => { const asset = AssetFactory.create(); - mocks.assetJob.getForOcr.mockResolvedValue({ visibility: AssetVisibility.Timeline, previewFile: null }); + mocks.assetJob.getForOcr.mockResolvedValue({ + ownerId: 'userId', + visibility: AssetVisibility.Timeline, + previewFile: null, + }); expect(await sut.handleOcr({ id: asset.id })).toEqual(JobStatus.Failed); expect(mocks.ocr.upsert).not.toHaveBeenCalled(); expect(mocks.machineLearning.ocr).not.toHaveBeenCalled(); + expect(mocks.event.emit).not.toHaveBeenCalled(); }); it('should save the returned objects', async () => { @@ -170,6 +177,7 @@ describe(OcrService.name, () => { it('should skip invisible assets', async () => { const asset = AssetFactory.from().file({ type: AssetFileType.Preview }).build(); mocks.assetJob.getForOcr.mockResolvedValue({ + ownerId: 'userId', visibility: AssetVisibility.Hidden, previewFile: asset.files[0].path, }); @@ -178,6 +186,7 @@ describe(OcrService.name, () => { expect(mocks.machineLearning.ocr).not.toHaveBeenCalled(); expect(mocks.ocr.upsert).not.toHaveBeenCalled(); + expect(mocks.event.emit).not.toHaveBeenCalled(); }); it('should fail if asset could not be found', async () => { @@ -187,6 +196,19 @@ describe(OcrService.name, () => { expect(mocks.machineLearning.ocr).not.toHaveBeenCalled(); expect(mocks.ocr.upsert).not.toHaveBeenCalled(); + expect(mocks.event.emit).not.toHaveBeenCalled(); + }); + + it('should emit an event', async () => { + const asset = AssetFactory.create(); + mockOcrResult('Hello World'); + + await sut.handleOcr({ id: asset.id }); + + expect(mocks.event.emit).toHaveBeenCalledWith('AssetOcrExtracted', { + userId: 'user-id', + assetId: asset.id, + }); }); describe('search tokenization', () => { diff --git a/server/src/services/ocr.service.ts b/server/src/services/ocr.service.ts index c91524e56..312076228 100644 --- a/server/src/services/ocr.service.ts +++ b/server/src/services/ocr.service.ts @@ -50,6 +50,11 @@ export class OcrService extends BaseService { await this.assetRepository.upsertJobStatus({ assetId: id, ocrAt: new Date() }); + await this.eventRepository.emit('AssetOcrExtracted', { + userId: asset.ownerId, + assetId: id, + }); + this.logger.debug(`Processed ${ocrResults.text.length} OCR result(s) for ${id}`); return JobStatus.Success; } diff --git a/server/src/services/workflow-execution.service.ts b/server/src/services/workflow-execution.service.ts index b8149aa9b..2185bfa8d 100644 --- a/server/src/services/workflow-execution.service.ts +++ b/server/src/services/workflow-execution.service.ts @@ -316,6 +316,11 @@ export class WorkflowExecutionService extends BaseService { return this.onAssetTrigger({ userId, assetId, trigger: WorkflowTrigger.AssetMetadataExtraction }); } + @OnEvent({ name: 'AssetOcrExtracted' }) + onAssetOcrExtracted({ userId, assetId }: ArgOf<'AssetOcrExtracted'>) { + return this.onAssetTrigger({ userId, assetId, trigger: WorkflowTrigger.AssetOcrExtraction }); + } + @OnEvent({ name: 'AssetTag' }) onAssetTagged({ assetId, userId }: ArgOf<'AssetTag'>) { return this.onAssetTrigger({ userId, assetId, trigger: WorkflowTrigger.AssetTagged }); @@ -332,7 +337,7 @@ export class WorkflowExecutionService extends BaseService { } @OnJob({ name: JobName.WorkflowAssetTrigger, queue: QueueName.Workflow }) - handleAssetTrigger({ workflowId, assetId }: JobOf) { + handleAssetTrigger({ workflowId, assetId, trigger }: JobOf) { return this.execute(workflowId, (type) => { const assetService = BaseService.create(AssetService, this); @@ -340,7 +345,7 @@ export class WorkflowExecutionService extends BaseService { case WorkflowType.AssetV1: { return { read: async () => { - const asset = await this.workflowRepository.getForAssetV1(assetId); + const asset = await this.workflowRepository.getForAssetV1(assetId, trigger); return { data: { asset } as any, authUserId: asset.ownerId, diff --git a/server/src/types.ts b/server/src/types.ts index 2dcf2ecc4..d6310dc40 100644 --- a/server/src/types.ts +++ b/server/src/types.ts @@ -1,3 +1,4 @@ +import { WorkflowTrigger } from '@immich/plugin-sdk'; import { ShallowDehydrateObject } from 'kysely'; import { Mocked } from 'vitest'; import { VECTOR_EXTENSIONS } from 'src/constants.js'; @@ -439,7 +440,7 @@ export type JobItem = | { name: JobName.Ocr; data: IEntityJob } // Workflow - | { name: JobName.WorkflowAssetTrigger; data: { workflowId: string; assetId: string } } + | { name: JobName.WorkflowAssetTrigger; data: { workflowId: string; assetId: string; trigger: WorkflowTrigger } } // Integrity | { name: JobName.IntegrityUntrackedFilesQueueAll; data?: IIntegrityJob } diff --git a/server/src/utils/workflow.ts b/server/src/utils/workflow.ts index c763eec2d..5de7100da 100644 --- a/server/src/utils/workflow.ts +++ b/server/src/utils/workflow.ts @@ -6,6 +6,7 @@ export const triggerMap: Record = { [WorkflowTrigger.AssetCreate]: [WorkflowType.AssetV1], // [WorkflowTrigger.PersonRecognized]: [WorkflowType.AssetPersonV1], [WorkflowTrigger.AssetMetadataExtraction]: [WorkflowType.AssetV1], + [WorkflowTrigger.AssetOcrExtraction]: [WorkflowType.AssetV1], [WorkflowTrigger.AssetTagged]: [WorkflowType.AssetV1], }; diff --git a/server/test/medium.factory.ts b/server/test/medium.factory.ts index 881a0f6fb..9dcde56e8 100644 --- a/server/test/medium.factory.ts +++ b/server/test/medium.factory.ts @@ -78,6 +78,7 @@ import { AssetExifTable } from 'src/schema/tables/asset-exif.table.js'; import { AssetFileTable } from 'src/schema/tables/asset-file.table.js'; import { AssetJobStatusTable } from 'src/schema/tables/asset-job-status.table.js'; import { AssetMetadataTable } from 'src/schema/tables/asset-metadata.table.js'; +import { AssetOcrTable } from 'src/schema/tables/asset-ocr.table.js'; import { AssetTable } from 'src/schema/tables/asset.table.js'; import { FaceSearchTable } from 'src/schema/tables/face-search.table.js'; import { MemoryTable } from 'src/schema/tables/memory.table.js'; @@ -230,6 +231,28 @@ export class MediumTestContext = return { assetFace, result }; } + async newAssetOcr( + dto: (Partial> & { assetId: string; text: string; textScore: number })[], + ) { + const result = await this.get(OcrRepository).upsert( + dto[0].assetId, + dto.map((item) => ({ + ...item, + x1: 0, + y1: 0, + x2: 1, + y2: 1, + x3: 2, + y3: 2, + x4: 3, + y4: 3, + boxScore: item.textScore, + })), + '', + ); + return { ocr: dto, result }; + } + async newMemory(dto: Partial> = {}) { const memory = mediumFactory.memoryInsert(dto); const result = await this.get(MemoryRepository).create(memory, new Set()); diff --git a/server/test/medium/specs/services/ocr.service.spec.ts b/server/test/medium/specs/services/ocr.service.spec.ts index 16ae3b9d5..837ae63fe 100644 --- a/server/test/medium/specs/services/ocr.service.spec.ts +++ b/server/test/medium/specs/services/ocr.service.spec.ts @@ -3,6 +3,7 @@ import { AssetFileType, JobStatus } from 'src/enum.js'; import { AssetJobRepository } from 'src/repositories/asset-job.repository.js'; import { AssetRepository } from 'src/repositories/asset.repository.js'; import { ConfigRepository } from 'src/repositories/config.repository.js'; +import { EventRepository } from 'src/repositories/event.repository.js'; import { JobRepository } from 'src/repositories/job.repository.js'; import { LoggingRepository } from 'src/repositories/logging.repository.js'; import { MachineLearningRepository } from 'src/repositories/machine-learning.repository.js'; @@ -20,7 +21,7 @@ const setup = (db?: Kysely) => { return newMediumService(OcrService, { database: db || defaultDatabase, real: [AssetRepository, AssetJobRepository, ConfigRepository, OcrRepository, SystemMetadataRepository], - mock: [JobRepository, LoggingRepository, MachineLearningRepository], + mock: [EventRepository, JobRepository, LoggingRepository, MachineLearningRepository], }); }; @@ -39,6 +40,7 @@ describe(OcrService.name, () => { const { user } = await ctx.newUser(); const { asset } = await ctx.newAsset({ ownerId: user.id }); await ctx.newAssetFile({ assetId: asset.id, type: AssetFileType.Preview, path: 'preview.jpg' }); + ctx.getMock(EventRepository).emit.mockResolvedValue(); const machineLearningMock = ctx.getMock(MachineLearningRepository); machineLearningMock.ocr.mockResolvedValue({ @@ -91,6 +93,7 @@ describe(OcrService.name, () => { const { user } = await ctx.newUser(); const { asset } = await ctx.newAsset({ ownerId: user.id }); await ctx.newAssetFile({ assetId: asset.id, type: AssetFileType.Preview, path: 'preview.jpg' }); + ctx.getMock(EventRepository).emit.mockResolvedValue(); const machineLearningMock = ctx.getMock(MachineLearningRepository); machineLearningMock.ocr.mockResolvedValue({ @@ -215,6 +218,7 @@ describe(OcrService.name, () => { const { user } = await ctx.newUser(); const { asset } = await ctx.newAsset({ ownerId: user.id }); await ctx.newAssetFile({ assetId: asset.id, type: AssetFileType.Preview, path: 'preview.jpg' }); + ctx.getMock(EventRepository).emit.mockResolvedValue(); const machineLearningMock = ctx.getMock(MachineLearningRepository); machineLearningMock.ocr.mockResolvedValue({ box: [], boxScore: [], text: [], textScore: [] }); @@ -240,6 +244,7 @@ describe(OcrService.name, () => { const { user } = await ctx.newUser(); const { asset } = await ctx.newAsset({ ownerId: user.id }); await ctx.newAssetFile({ assetId: asset.id, type: AssetFileType.Preview, path: 'preview.jpg' }); + ctx.getMock(EventRepository).emit.mockResolvedValue(); const machineLearningMock = ctx.getMock(MachineLearningRepository); machineLearningMock.ocr.mockResolvedValue({ diff --git a/server/test/medium/specs/workflow/workflow-core-plugin.spec.ts b/server/test/medium/specs/workflow/workflow-core-plugin.spec.ts index e4501f317..6e53c2513 100644 --- a/server/test/medium/specs/workflow/workflow-core-plugin.spec.ts +++ b/server/test/medium/specs/workflow/workflow-core-plugin.spec.ts @@ -146,7 +146,13 @@ describe('core plugin', () => { steps: [{ method: 'immich-plugin-core#assetArchive' }], }); - await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined(); + await expect( + ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset.id, + trigger: WorkflowTrigger.AssetCreate, + }), + ).resolves.toBeUndefined(); await expect(ctx.get(AssetRepository).getById(asset.id)).resolves.toMatchObject({ visibility: AssetVisibility.Archive, @@ -163,7 +169,13 @@ describe('core plugin', () => { steps: [{ method: 'immich-plugin-core#assetArchive', config: { inverse: true } }], }); - await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined(); + await expect( + ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset.id, + trigger: WorkflowTrigger.AssetCreate, + }), + ).resolves.toBeUndefined(); await expect(ctx.get(AssetRepository).getById(asset.id)).resolves.toMatchObject({ visibility: AssetVisibility.Timeline, @@ -182,7 +194,13 @@ describe('core plugin', () => { steps: [{ method: 'immich-plugin-core#assetLock' }], }); - await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined(); + await expect( + ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset.id, + trigger: WorkflowTrigger.AssetCreate, + }), + ).resolves.toBeUndefined(); await expect(ctx.get(AssetRepository).getById(asset.id)).resolves.toMatchObject({ visibility: AssetVisibility.Locked, @@ -199,7 +217,13 @@ describe('core plugin', () => { steps: [{ method: 'immich-plugin-core#assetLock', config: { inverse: true } }], }); - await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined(); + await expect( + ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset.id, + trigger: WorkflowTrigger.AssetCreate, + }), + ).resolves.toBeUndefined(); await expect(ctx.get(AssetRepository).getById(asset.id)).resolves.toMatchObject({ visibility: AssetVisibility.Timeline, @@ -218,7 +242,13 @@ describe('core plugin', () => { steps: [{ method: 'immich-plugin-core#assetFavorite' }], }); - await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined(); + await expect( + ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset.id, + trigger: WorkflowTrigger.AssetCreate, + }), + ).resolves.toBeUndefined(); await expect(ctx.get(AssetRepository).getById(asset.id)).resolves.toMatchObject({ isFavorite: true }); }); @@ -233,7 +263,13 @@ describe('core plugin', () => { steps: [{ method: 'immich-plugin-core#assetFavorite', config: { inverse: true } }], }); - await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined(); + await expect( + ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset.id, + trigger: WorkflowTrigger.AssetCreate, + }), + ).resolves.toBeUndefined(); await expect(ctx.get(AssetRepository).getById(asset.id)).resolves.toMatchObject({ isFavorite: false }); }); @@ -250,7 +286,13 @@ describe('core plugin', () => { steps: [{ method: 'immich-plugin-core#assetAddToAlbums', config: { albumIds: [], albumName: 'Screenshots' } }], }); - await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined(); + await expect( + ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset.id, + trigger: WorkflowTrigger.AssetCreate, + }), + ).resolves.toBeUndefined(); const albums = await ctx.get(AlbumRepository).getAll(user.id); expect(albums).toHaveLength(1); @@ -281,7 +323,13 @@ describe('core plugin', () => { expect(albums).toHaveLength(1); expect(albums[0].albumName).toEqual(album.albumName); - await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined(); + await expect( + ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset.id, + trigger: WorkflowTrigger.AssetCreate, + }), + ).resolves.toBeUndefined(); await expect(ctx.get(AlbumRepository).getAssetIds(album.id, [asset.id])).resolves.toContain(asset.id); }); @@ -297,7 +345,13 @@ describe('core plugin', () => { steps: [{ method: 'immich-plugin-core#assetAddToAlbums', config: { albumIds: [album.id] } }], }); - await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined(); + await expect( + ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset.id, + trigger: WorkflowTrigger.AssetCreate, + }), + ).resolves.toBeUndefined(); await expect(ctx.get(AlbumRepository).getAssetIds(album.id, [asset.id])).resolves.toContain(asset.id); }); @@ -316,7 +370,13 @@ describe('core plugin', () => { steps: [{ method: 'immich-plugin-core#assetAddToAlbums', config: { albumIds: [album1.id, album2.id] } }], }); - await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined(); + await expect( + ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset.id, + trigger: WorkflowTrigger.AssetCreate, + }), + ).resolves.toBeUndefined(); await expect(ctx.get(AlbumRepository).getAssetIds(album1.id, [asset.id])).resolves.toContain(asset.id); await expect(ctx.get(AlbumRepository).getAssetIds(album2.id, [asset.id])).resolves.toContain(asset.id); @@ -334,7 +394,13 @@ describe('core plugin', () => { steps: [{ method: 'immich-plugin-core#assetAddToAlbums', config: { albumIds: [album.id] } }], }); - await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeTruthy(); + await expect( + ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset.id, + trigger: WorkflowTrigger.AssetCreate, + }), + ).resolves.toBeTruthy(); await expect(ctx.get(AlbumRepository).getAssetIds(album.id, [asset.id])).resolves.not.toContain(asset.id); }); @@ -360,7 +426,13 @@ describe('core plugin', () => { ], }); - await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined(); + await expect( + ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset.id, + trigger: WorkflowTrigger.AssetCreate, + }), + ).resolves.toBeUndefined(); await expect(ctx.get(AssetRepository).getById(asset.id)).resolves.toMatchObject({ isFavorite: true }); }); @@ -383,7 +455,13 @@ describe('core plugin', () => { ], }); - await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined(); + await expect( + ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset.id, + trigger: WorkflowTrigger.AssetMetadataExtraction, + }), + ).resolves.toBeUndefined(); await expect(ctx.get(AssetRepository).getById(asset.id)).resolves.toMatchObject({ isFavorite: false }); }); @@ -406,7 +484,13 @@ describe('core plugin', () => { ], }); - await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined(); + await expect( + ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset.id, + trigger: WorkflowTrigger.AssetMetadataExtraction, + }), + ).resolves.toBeUndefined(); await expect(ctx.get(AssetRepository).getById(asset.id)).resolves.toMatchObject({ isFavorite: true }); }); }); @@ -434,10 +518,18 @@ describe('core plugin', () => { }); await expect( - ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset1.id }), + ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset1.id, + trigger: WorkflowTrigger.AssetCreate, + }), ).resolves.toBeUndefined(); await expect( - ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset2.id }), + ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset2.id, + trigger: WorkflowTrigger.AssetCreate, + }), ).resolves.toBeUndefined(); await expect(ctx.get(AssetRepository).getById(asset1.id)).resolves.toMatchObject({ isFavorite: true }); await expect(ctx.get(AssetRepository).getById(asset2.id)).resolves.toMatchObject({ isFavorite: true }); @@ -465,10 +557,18 @@ describe('core plugin', () => { }); await expect( - ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset1.id }), + ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset1.id, + trigger: WorkflowTrigger.AssetCreate, + }), ).resolves.toBeUndefined(); await expect( - ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset2.id }), + ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset2.id, + trigger: WorkflowTrigger.AssetCreate, + }), ).resolves.toBeUndefined(); await expect(ctx.get(AssetRepository).getById(asset1.id)).resolves.toMatchObject({ isFavorite: true }); await expect(ctx.get(AssetRepository).getById(asset2.id)).resolves.toMatchObject({ isFavorite: false }); @@ -496,10 +596,18 @@ describe('core plugin', () => { }); await expect( - ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset1.id }), + ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset1.id, + trigger: WorkflowTrigger.AssetCreate, + }), ).resolves.toBeUndefined(); await expect( - ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset2.id }), + ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset2.id, + trigger: WorkflowTrigger.AssetCreate, + }), ).resolves.toBeUndefined(); await expect(ctx.get(AssetRepository).getById(asset1.id)).resolves.toMatchObject({ isFavorite: true }); await expect(ctx.get(AssetRepository).getById(asset2.id)).resolves.toMatchObject({ isFavorite: false }); @@ -525,7 +633,13 @@ describe('core plugin', () => { ], }); - await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined(); + await expect( + ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset.id, + trigger: WorkflowTrigger.AssetCreate, + }), + ).resolves.toBeUndefined(); await expect(ctx.get(AssetRepository).getById(asset.id)).resolves.toMatchObject({ isFavorite: true }); }); }); @@ -663,13 +777,25 @@ describe('core plugin', () => { ], }); - await ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset1.id }); + await ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset1.id, + trigger: WorkflowTrigger.AssetCreate, + }); await expect(ctx.get(AssetRepository).getById(asset1.id)).resolves.toMatchObject({ isFavorite: true }); - await ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset2.id }); + await ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset2.id, + trigger: WorkflowTrigger.AssetCreate, + }); await expect(ctx.get(AssetRepository).getById(asset2.id)).resolves.toMatchObject({ isFavorite: true }); - await ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset3.id }); + await ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset3.id, + trigger: WorkflowTrigger.AssetCreate, + }); await expect(ctx.get(AssetRepository).getById(asset3.id)).resolves.toMatchObject({ isFavorite: false }); }); @@ -700,17 +826,157 @@ describe('core plugin', () => { ], }); - await ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset1.id }); + await ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset1.id, + trigger: WorkflowTrigger.AssetCreate, + }); await expect(ctx.get(AssetRepository).getById(asset1.id)).resolves.toMatchObject({ isFavorite: true }); - await ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset2.id }); + await ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset2.id, + trigger: WorkflowTrigger.AssetCreate, + }); await expect(ctx.get(AssetRepository).getById(asset2.id)).resolves.toMatchObject({ isFavorite: true }); - await ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset3.id }); + await ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset3.id, + trigger: WorkflowTrigger.AssetCreate, + }); await expect(ctx.get(AssetRepository).getById(asset3.id)).resolves.toMatchObject({ isFavorite: false }); }); }); + describe('assetOcrFilter', () => { + it('should match assets case-insensitively', async () => { + const { user } = await ctx.newUser(); + const [{ asset: asset1 }, { asset: asset2 }] = await Promise.all([ + ctx.newAsset({ ownerId: user.id }), + ctx.newAsset({ ownerId: user.id }), + ]); + await ctx.newAssetOcr([{ assetId: asset1.id, text: 'test', textScore: 0.9 }]); + await ctx.newAssetOcr([{ assetId: asset2.id, text: 'bogus', textScore: 0.9 }]); + + const workflow = await createWorkflow({ + ownerId: user.id, + trigger: WorkflowTrigger.AssetOcrExtraction, + steps: [ + { + method: 'immich-plugin-core#assetOcrFilter', + config: { matchType: 'contains', matchScope: 'box', pattern: 'Test', minScore: 0 }, + }, + { + method: 'immich-plugin-core#assetFavorite', + }, + ], + }); + + await expect( + ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset1.id, + trigger: WorkflowTrigger.AssetOcrExtraction, + }), + ).resolves.toBeUndefined(); + await expect( + ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset2.id, + trigger: WorkflowTrigger.AssetOcrExtraction, + }), + ).resolves.toBeUndefined(); + await expect(ctx.get(AssetRepository).getById(asset1.id)).resolves.toMatchObject({ isFavorite: true }); + await expect(ctx.get(AssetRepository).getById(asset2.id)).resolves.toMatchObject({ isFavorite: false }); + }); + + it('should match assets by regex in full text', async () => { + const { user } = await ctx.newUser(); + const [{ asset: asset1 }, { asset: asset2 }] = await Promise.all([ + ctx.newAsset({ ownerId: user.id }), + ctx.newAsset({ ownerId: user.id }), + ]); + await ctx.newAssetOcr([ + { assetId: asset1.id, text: 'Test', textScore: 0.9 }, + { assetId: asset1.id, text: 'Match', textScore: 0.5 }, + ]); + await ctx.newAssetOcr([{ assetId: asset2.id, text: 'Test', textScore: 0.9 }]); + + const workflow = await createWorkflow({ + ownerId: user.id, + trigger: WorkflowTrigger.AssetCreate, + steps: [ + { + method: 'immich-plugin-core#assetOcrFilter', + config: { matchType: 'regex', matchScope: 'full', pattern: 'Test.+Match', minScore: 0 }, + }, + { + method: 'immich-plugin-core#assetFavorite', + }, + ], + }); + + await expect( + ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset1.id, + trigger: WorkflowTrigger.AssetOcrExtraction, + }), + ).resolves.toBeUndefined(); + await expect( + ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset2.id, + trigger: WorkflowTrigger.AssetOcrExtraction, + }), + ).resolves.toBeUndefined(); + await expect(ctx.get(AssetRepository).getById(asset1.id)).resolves.toMatchObject({ isFavorite: true }); + await expect(ctx.get(AssetRepository).getById(asset2.id)).resolves.toMatchObject({ isFavorite: false }); + }); + + it('should filter assets with low score', async () => { + const { user } = await ctx.newUser(); + const [{ asset: asset1 }, { asset: asset2 }] = await Promise.all([ + ctx.newAsset({ ownerId: user.id }), + ctx.newAsset({ ownerId: user.id }), + ]); + await ctx.newAssetOcr([{ assetId: asset1.id, text: 'test', textScore: 0.9 }]); + await ctx.newAssetOcr([{ assetId: asset2.id, text: 'test', textScore: 0.5 }]); + + const workflow = await createWorkflow({ + ownerId: user.id, + trigger: WorkflowTrigger.AssetOcrExtraction, + steps: [ + { + method: 'immich-plugin-core#assetOcrFilter', + config: { matchType: 'contains', matchScope: 'box', pattern: 'test', minScore: 0.8 }, + }, + { + method: 'immich-plugin-core#assetFavorite', + }, + ], + }); + + await expect( + ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset1.id, + trigger: WorkflowTrigger.AssetOcrExtraction, + }), + ).resolves.toBeUndefined(); + await expect( + ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset2.id, + trigger: WorkflowTrigger.AssetOcrExtraction, + }), + ).resolves.toBeUndefined(); + await expect(ctx.get(AssetRepository).getById(asset1.id)).resolves.toMatchObject({ isFavorite: true }); + await expect(ctx.get(AssetRepository).getById(asset2.id)).resolves.toMatchObject({ isFavorite: false }); + }); + }); + describe('assetTagFilter', () => { it('should favorite asset if it matches any tag', async () => { const { user } = await ctx.newUser(); @@ -732,7 +998,13 @@ describe('core plugin', () => { ], }); - await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined(); + await expect( + ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset.id, + trigger: WorkflowTrigger.AssetCreate, + }), + ).resolves.toBeUndefined(); await expect(ctx.get(AssetRepository).getById(asset.id)).resolves.toMatchObject({ isFavorite: true }); }); @@ -755,7 +1027,13 @@ describe('core plugin', () => { ], }); - await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined(); + await expect( + ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset.id, + trigger: WorkflowTrigger.AssetCreate, + }), + ).resolves.toBeUndefined(); await expect(ctx.get(AssetRepository).getById(asset.id)).resolves.toMatchObject({ isFavorite: false }); }); @@ -779,7 +1057,13 @@ describe('core plugin', () => { ], }); - await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined(); + await expect( + ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset.id, + trigger: WorkflowTrigger.AssetCreate, + }), + ).resolves.toBeUndefined(); await expect(ctx.get(AssetRepository).getById(asset.id)).resolves.toMatchObject({ isFavorite: true }); }); @@ -802,7 +1086,13 @@ describe('core plugin', () => { ], }); - await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined(); + await expect( + ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset.id, + trigger: WorkflowTrigger.AssetCreate, + }), + ).resolves.toBeUndefined(); await expect(ctx.get(AssetRepository).getById(asset.id)).resolves.toMatchObject({ isFavorite: false }); }); @@ -826,7 +1116,13 @@ describe('core plugin', () => { ], }); - await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined(); + await expect( + ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset.id, + trigger: WorkflowTrigger.AssetCreate, + }), + ).resolves.toBeUndefined(); await expect(ctx.get(AssetRepository).getById(asset.id)).resolves.toMatchObject({ isFavorite: true }); }); }); @@ -850,7 +1146,13 @@ describe('core plugin', () => { ], }); - await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined(); + await expect( + ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset.id, + trigger: WorkflowTrigger.AssetCreate, + }), + ).resolves.toBeUndefined(); expect(fetchMock).toHaveBeenCalled(); }); @@ -887,7 +1189,13 @@ describe('core plugin', () => { .execute(); } - await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined(); + await expect( + ctx.sut.handleAssetTrigger({ + workflowId: workflow.id, + assetId: asset.id, + trigger: WorkflowTrigger.AssetCreate, + }), + ).resolves.toBeUndefined(); await expect(ctx.get(AssetRepository).getById(asset.id)).resolves.toMatchObject({ isFavorite: false }); }); }); diff --git a/web/src/lib/utils/workflow.ts b/web/src/lib/utils/workflow.ts index 9aff59153..66234d772 100644 --- a/web/src/lib/utils/workflow.ts +++ b/web/src/lib/utils/workflow.ts @@ -13,6 +13,9 @@ export const getTriggerName = ($t: MessageFormatter, type: WorkflowTrigger) => { case WorkflowTrigger.AssetMetadataExtraction: { return $t('trigger_asset_metadata_extraction'); } + case WorkflowTrigger.AssetOcrExtraction: { + return $t('trigger_asset_ocr_extraction'); + } case WorkflowTrigger.AssetTagged: { return $t('trigger_asset_tagged'); } @@ -33,6 +36,9 @@ export const getTriggerDescription = ($t: MessageFormatter, type: WorkflowTrigge case WorkflowTrigger.AssetMetadataExtraction: { return $t('trigger_asset_metadata_extraction_description'); } + case WorkflowTrigger.AssetOcrExtraction: { + return $t('trigger_asset_ocr_extraction_description'); + } case WorkflowTrigger.AssetTagged: { return $t('trigger_asset_tagged_description'); }