diff --git a/server/src/repositories/media.repository.spec.ts b/server/src/repositories/media.repository.spec.ts index 866b073e9..fad6201d1 100644 --- a/server/src/repositories/media.repository.spec.ts +++ b/server/src/repositories/media.repository.spec.ts @@ -68,9 +68,9 @@ describe(MediaRepository.name, () => { sut = new MediaRepository(automock(LoggingRepository, { args: [, { getEnv: () => ({}) }], strict: false })); }); - describe('applyEdits (single actions)', () => { + describe('edit (single actions)', () => { it('should apply crop edit correctly', async () => { - const result = sut['applyEdits']( + const result = sut['edit']( sharp({ create: { width: 1000, @@ -97,7 +97,7 @@ describe(MediaRepository.name, () => { expect(metadata.height).toBe(300); }); it('should apply rotate edit correctly', async () => { - const result = sut['applyEdits']( + const result = sut['edit']( sharp({ create: { width: 500, @@ -122,7 +122,7 @@ describe(MediaRepository.name, () => { }); it('should apply mirror edit correctly', async () => { - const resultHorizontal = sut['applyEdits'](sharp(await buildTestQuadImage()), [ + const resultHorizontal = sut['edit'](sharp(await buildTestQuadImage()), [ { action: AssetEditAction.Mirror, parameters: { @@ -141,7 +141,7 @@ describe(MediaRepository.name, () => { expect(await getPixelColor(bufferHorizontal, 10, 990)).toEqual({ r: 255, g: 255, b: 0 }); expect(await getPixelColor(bufferHorizontal, 990, 990)).toEqual({ r: 0, g: 0, b: 255 }); - const resultVertical = sut['applyEdits'](sharp(await buildTestQuadImage()), [ + const resultVertical = sut['edit'](sharp(await buildTestQuadImage()), [ { action: AssetEditAction.Mirror, parameters: { @@ -166,10 +166,10 @@ describe(MediaRepository.name, () => { }); }); - describe('applyEdits (multiple sequential edits)', () => { + describe('edit (multiple sequential edits)', () => { it('should apply horizontal mirror then vertical mirror (equivalent to 180° rotation)', async () => { const imageBuffer = await buildTestQuadImage(); - const result = sut['applyEdits'](sharp(imageBuffer), [ + const result = sut['edit'](sharp(imageBuffer), [ { action: AssetEditAction.Mirror, parameters: { axis: MirrorAxis.Horizontal } }, { action: AssetEditAction.Mirror, parameters: { axis: MirrorAxis.Vertical } }, ]); @@ -187,7 +187,7 @@ describe(MediaRepository.name, () => { it('should apply rotate 90° then horizontal mirror', async () => { const imageBuffer = await buildTestQuadImage(); - const result = sut['applyEdits'](sharp(imageBuffer), [ + const result = sut['edit'](sharp(imageBuffer), [ { action: AssetEditAction.Rotate, parameters: { angle: 90 } }, { action: AssetEditAction.Mirror, parameters: { axis: MirrorAxis.Horizontal } }, ]); @@ -205,9 +205,7 @@ describe(MediaRepository.name, () => { it('should apply 180° rotation', async () => { const imageBuffer = await buildTestQuadImage(); - const result = sut['applyEdits'](sharp(imageBuffer), [ - { action: AssetEditAction.Rotate, parameters: { angle: 180 } }, - ]); + const result = sut['edit'](sharp(imageBuffer), [{ action: AssetEditAction.Rotate, parameters: { angle: 180 } }]); const buffer = await result.png().toBuffer(); const metadata = await sharp(buffer).metadata(); @@ -222,9 +220,7 @@ describe(MediaRepository.name, () => { it('should apply 270° rotations', async () => { const imageBuffer = await buildTestQuadImage(); - const result = sut['applyEdits'](sharp(imageBuffer), [ - { action: AssetEditAction.Rotate, parameters: { angle: 270 } }, - ]); + const result = sut['edit'](sharp(imageBuffer), [{ action: AssetEditAction.Rotate, parameters: { angle: 270 } }]); const buffer = await result.png().toBuffer(); const metadata = await sharp(buffer).metadata(); @@ -239,7 +235,7 @@ describe(MediaRepository.name, () => { it('should apply crop then rotate 90°', async () => { const imageBuffer = await buildTestQuadImage(); - const result = sut['applyEdits'](sharp(imageBuffer), [ + const result = sut['edit'](sharp(imageBuffer), [ { action: AssetEditAction.Crop, parameters: { x: 0, y: 0, width: 1000, height: 500 } }, { action: AssetEditAction.Rotate, parameters: { angle: 90 } }, ]); @@ -255,7 +251,7 @@ describe(MediaRepository.name, () => { it('should apply rotate 90° then crop', async () => { const imageBuffer = await buildTestQuadImage(); - const result = sut['applyEdits'](sharp(imageBuffer), [ + const result = sut['edit'](sharp(imageBuffer), [ { action: AssetEditAction.Crop, parameters: { x: 0, y: 0, width: 500, height: 1000 } }, { action: AssetEditAction.Rotate, parameters: { angle: 90 } }, ]); @@ -271,7 +267,7 @@ describe(MediaRepository.name, () => { it('should apply vertical mirror then horizontal mirror then rotate 90°', async () => { const imageBuffer = await buildTestQuadImage(); - const result = sut['applyEdits'](sharp(imageBuffer), [ + const result = sut['edit'](sharp(imageBuffer), [ { action: AssetEditAction.Mirror, parameters: { axis: MirrorAxis.Vertical } }, { action: AssetEditAction.Mirror, parameters: { axis: MirrorAxis.Horizontal } }, { action: AssetEditAction.Rotate, parameters: { angle: 90 } }, @@ -290,7 +286,7 @@ describe(MediaRepository.name, () => { it('should apply crop to single quadrant then mirror', async () => { const imageBuffer = await buildTestQuadImage(); - const result = sut['applyEdits'](sharp(imageBuffer), [ + const result = sut['edit'](sharp(imageBuffer), [ { action: AssetEditAction.Crop, parameters: { x: 0, y: 0, width: 500, height: 500 } }, { action: AssetEditAction.Mirror, parameters: { axis: MirrorAxis.Horizontal } }, ]); @@ -308,7 +304,7 @@ describe(MediaRepository.name, () => { it('should apply all operations: crop, rotate, mirror', async () => { const imageBuffer = await buildTestQuadImage(); - const result = sut['applyEdits'](sharp(imageBuffer), [ + const result = sut['edit'](sharp(imageBuffer), [ { action: AssetEditAction.Crop, parameters: { x: 0, y: 0, width: 500, height: 1000 } }, { action: AssetEditAction.Rotate, parameters: { angle: 90 } }, { action: AssetEditAction.Mirror, parameters: { axis: MirrorAxis.Horizontal } }, @@ -332,8 +328,9 @@ describe(MediaRepository.name, () => { const buffer = Buffer.from(await response.arrayBuffer()); const dir = mkdtempDisposableSync(join(tmpdir(), 'media-repository-')); const file = join(dir.path, 'test.webp'); + const decoded = await sut.decodeImage(buffer, { colorspace: Colorspace.P3, processInvalidImages: false }); await sut.generateThumbnail( - buffer, + decoded, { colorspace: Colorspace.P3, quality: 80, format: ImageFormat.Webp, processInvalidImages: false }, file, ); diff --git a/server/src/repositories/media.repository.ts b/server/src/repositories/media.repository.ts index b0f6445e6..04ee0345f 100644 --- a/server/src/repositories/media.repository.ts +++ b/server/src/repositories/media.repository.ts @@ -8,12 +8,14 @@ import fs from 'node:fs/promises'; import { Writable } from 'node:stream'; import sharp, { Sharp } from 'sharp'; import type { + Bitmap, DecodeToBufferOptions, GenerateThumbhashOptions, GenerateThumbnailOptions, ImageDimensions, ProbeOptions, TranscodeCommand, + TransformOptions, VideoInfo, VideoPacketInfo, } from 'src/types.js'; @@ -148,11 +150,12 @@ export class MediaRepository { } } - decodeImage(input: string | Buffer, options: DecodeToBufferOptions) { - return this.getImageDecodingPipeline(input, options).raw().toBuffer({ resolveWithObject: true }); + async decodeImage(input: string | Buffer, options: DecodeToBufferOptions): Promise { + const decoded = await this.getImageDecodingPipeline(input, options).raw().toBuffer({ resolveWithObject: true }); + return await this.transform(decoded, options); } - private applyEdits(pipeline: Sharp, edits: AssetEditActionItem[]): Sharp { + private edit(pipeline: Sharp, edits: AssetEditActionItem[]): Sharp { const crop = edits.find((edit) => edit.action === 'crop'); if (crop) { pipeline = pipeline.extract({ @@ -175,8 +178,9 @@ export class MediaRepository { return pipeline; } - async generateThumbnail(input: string | Buffer, options: GenerateThumbnailOptions, output: string): Promise { - await this.getImageDecodingPipeline(input, options) + async generateThumbnail(image: Bitmap, options: GenerateThumbnailOptions, output: string): Promise { + const transformed = await this.transform(image, options); + await this.tag(transformed, options.colorspace) .toFormat(options.format, { quality: options.quality, // this is default in libvips (except the threshold is 90), but we need to set it manually in sharp @@ -187,51 +191,62 @@ export class MediaRepository { } private getImageDecodingPipeline(input: string | Buffer, options: DecodeToBufferOptions) { - let pipeline = sharp(input, { - // some invalid images can still be processed by sharp, but we want to fail on them by default to avoid crashes - failOn: options.processInvalidImages ? 'none' : 'error', - limitInputChannels: false, - limitInputPixels: false, - raw: options.raw, - unlimited: true, - }) - .pipelineColorspace(options.colorspace === Colorspace.Srgb ? 'srgb' : 'rgb16') + // some invalid images can still be processed by sharp, but we want to fail on them by default to avoid crashes + let pipeline = this.encoded(input, options.processInvalidImages ? 'none' : 'error') + .pipelineColorspace(this.getPipelineColorspace(options.colorspace)) .withIccProfile(options.colorspace); - if (!options.raw) { - const { angle, flip, flop } = options.orientation ? ORIENTATION_TO_SHARP_ROTATION[options.orientation] : {}; - pipeline = pipeline.rotate(angle); - if (flip) { - pipeline = pipeline.flip(); - } - - if (flop) { - pipeline = pipeline.flop(); - } + const { angle, flip, flop } = options.orientation ? ORIENTATION_TO_SHARP_ROTATION[options.orientation] : {}; + pipeline = pipeline.rotate(angle); + if (flip) { + pipeline = pipeline.flip(); } - if (options.edits && options.edits.length > 0) { - pipeline = this.applyEdits(pipeline, options.edits); + if (flop) { + pipeline = pipeline.flop(); } - if (options.size !== undefined) { - pipeline = pipeline.resize(options.size, options.size, { fit: 'outside', withoutEnlargement: true }); - } return pipeline; } - async generateThumbhash(input: string | Buffer, options: GenerateThumbhashOptions): Promise { + private getPipelineColorspace(colorspace: string) { + return colorspace === Colorspace.Srgb ? 'srgb' : 'rgb16'; + } + + /* Resamples in linear light; averaging gamma-encoded values darkens the result and loses detail. `colorspace` + * always has a sRGB transfer function and scRGB applies no primaries matrix, so linearising as sRGB is exact. */ + private transform(image: Bitmap, { size, fit = 'outside', edits = [] }: TransformOptions): Promise { + if (!size && edits.length === 0) { + return Promise.resolve(image); + } + + return this.edit(this.raw(image).pipelineColorspace('scrgb'), edits) + .resize(size, size, { fit, withoutEnlargement: true }) + .raw() + .toBuffer({ resolveWithObject: true }); + } + + private raw({ data, info: raw }: Bitmap) { + return sharp(data, { raw, limitInputChannels: false, limitInputPixels: false, unlimited: true }); + } + + private encoded(image: string | Buffer, failOn: 'none' | 'error') { + return sharp(image, { failOn, limitInputChannels: false, limitInputPixels: false, unlimited: true }); + } + + /** Re-attaches the profile, converting nothing: the pixels are already in that colourspace. */ + private tag(image: Bitmap, colorspace: string): Sharp { + return this.raw(image).pipelineColorspace(this.getPipelineColorspace(colorspace)).withIccProfile(colorspace); + } + + async generateThumbhash(image: Bitmap, options: GenerateThumbhashOptions): Promise { const { rgbaToThumbHash } = await import('thumbhash'); - const { data, info } = await this.getImageDecodingPipeline(input, { - colorspace: options.colorspace, - processInvalidImages: options.processInvalidImages, - raw: options.raw, - edits: options.edits, - }) - .resize(100, 100, { fit: 'inside', withoutEnlargement: true }) - .raw() + const transformed = await this.transform(image, { edits: options.edits, fit: 'inside', size: 100 }); + + const { data, info } = await sharp(transformed.data, { raw: transformed.info }) .ensureAlpha() + .raw() .toBuffer({ resolveWithObject: true }); return Buffer.from(rgbaToThumbHash(info.width, info.height, data)); diff --git a/server/src/services/auth.service.spec.ts b/server/src/services/auth.service.spec.ts index cc85a7902..0d3771b2e 100644 --- a/server/src/services/auth.service.spec.ts +++ b/server/src/services/auth.service.spec.ts @@ -1026,8 +1026,12 @@ describe(AuthService.name, () => { profileChangedAt: expect.any(Date), }); expect(mocks.oauth.getProfilePicture).toHaveBeenCalledWith(profile.picture); - expect(mocks.media.generateThumbnail).toHaveBeenCalledWith( + expect(mocks.media.decodeImage).toHaveBeenCalledWith( Buffer.from(pictureBytes.buffer, pictureBytes.byteOffset, pictureBytes.byteLength), + expect.objectContaining({ processInvalidImages: false }), + ); + expect(mocks.media.generateThumbnail).toHaveBeenCalledWith( + expect.anything(), expect.objectContaining({ format: 'webp', processInvalidImages: false }), expect.stringContaining(`/data/profile/${user.id}/${fileId}.webp`), ); diff --git a/server/src/services/media.service.spec.ts b/server/src/services/media.service.spec.ts index c54933130..faed3b5aa 100644 --- a/server/src/services/media.service.spec.ts +++ b/server/src/services/media.service.spec.ts @@ -445,7 +445,7 @@ describe(MediaService.name, () => { expect(mocks.media.generateThumbnail).toHaveBeenCalledTimes(2); expect(mocks.media.generateThumbnail).toHaveBeenCalledWith( - rawBuffer, + { data: rawBuffer, info: rawInfo }, { colorspace: Colorspace.P3, format: ImageFormat.Jpeg, @@ -453,13 +453,12 @@ describe(MediaService.name, () => { quality: 80, progressive: false, processInvalidImages: false, - raw: rawInfo, edits: [], }, expect.any(String), ); expect(mocks.media.generateThumbnail).toHaveBeenCalledWith( - rawBuffer, + { data: rawBuffer, info: rawInfo }, { colorspace: Colorspace.P3, format: ImageFormat.Webp, @@ -467,19 +466,20 @@ describe(MediaService.name, () => { quality: 80, progressive: false, processInvalidImages: false, - raw: rawInfo, edits: [], }, expect.any(String), ); expect(mocks.media.generateThumbhash).toHaveBeenCalledOnce(); - expect(mocks.media.generateThumbhash).toHaveBeenCalledWith(rawBuffer, { - colorspace: Colorspace.P3, - processInvalidImages: false, - raw: rawInfo, - edits: [], - }); + expect(mocks.media.generateThumbhash).toHaveBeenCalledWith( + { data: rawBuffer, info: rawInfo }, + { + colorspace: Colorspace.P3, + processInvalidImages: false, + edits: [], + }, + ); expect(mocks.asset.upsertFiles).toHaveBeenCalledWith([ { @@ -717,7 +717,7 @@ describe(MediaService.name, () => { expect(mocks.media.generateThumbnail).toHaveBeenCalledTimes(2); expect(mocks.media.generateThumbnail).toHaveBeenCalledWith( - rawBuffer, + { data: rawBuffer, info: rawInfo }, { colorspace: Colorspace.Srgb, format, @@ -725,13 +725,12 @@ describe(MediaService.name, () => { quality: 80, progressive: false, processInvalidImages: false, - raw: rawInfo, edits: [], }, previewPath, ); expect(mocks.media.generateThumbnail).toHaveBeenCalledWith( - rawBuffer, + { data: rawBuffer, info: rawInfo }, { colorspace: Colorspace.Srgb, format: ImageFormat.Webp, @@ -739,7 +738,6 @@ describe(MediaService.name, () => { quality: 80, progressive: false, processInvalidImages: false, - raw: rawInfo, edits: [], }, thumbnailPath, @@ -767,7 +765,7 @@ describe(MediaService.name, () => { expect(mocks.media.generateThumbnail).toHaveBeenCalledTimes(2); expect(mocks.media.generateThumbnail).toHaveBeenCalledWith( - rawBuffer, + { data: rawBuffer, info: rawInfo }, { colorspace: Colorspace.Srgb, format: ImageFormat.Jpeg, @@ -775,13 +773,12 @@ describe(MediaService.name, () => { quality: 80, progressive: false, processInvalidImages: false, - raw: rawInfo, edits: [], }, previewPath, ); expect(mocks.media.generateThumbnail).toHaveBeenCalledWith( - rawBuffer, + { data: rawBuffer, info: rawInfo }, { colorspace: Colorspace.Srgb, format, @@ -789,7 +786,6 @@ describe(MediaService.name, () => { quality: 80, progressive: false, processInvalidImages: false, - raw: rawInfo, edits: [], }, thumbnailPath, @@ -806,7 +802,7 @@ describe(MediaService.name, () => { await sut.handleGenerateThumbnails({ id: asset.id }); expect(mocks.media.generateThumbnail).toHaveBeenCalledWith( - rawBuffer, + { data: rawBuffer, info: rawInfo }, expect.objectContaining({ format: ImageFormat.Jpeg, progressive: true, @@ -814,7 +810,7 @@ describe(MediaService.name, () => { expect.stringContaining('preview.jpeg'), ); expect(mocks.media.generateThumbnail).toHaveBeenCalledWith( - rawBuffer, + { data: rawBuffer, info: rawInfo }, expect.objectContaining({ format: ImageFormat.Webp, progressive: false, @@ -845,7 +841,7 @@ describe(MediaService.name, () => { await sut.handleGenerateThumbnails({ id: asset.id }); expect(mocks.media.generateThumbnail).toHaveBeenCalledWith( - rawBuffer, + { data: rawBuffer, info: rawInfo }, expect.objectContaining({ format: ImageFormat.Jpeg, progressive: false, @@ -853,7 +849,7 @@ describe(MediaService.name, () => { expect.stringContaining('preview.jpeg'), ); expect(mocks.media.generateThumbnail).toHaveBeenCalledWith( - rawBuffer, + { data: rawBuffer, info: rawInfo }, expect.objectContaining({ format: ImageFormat.Jpeg, progressive: true, @@ -1032,19 +1028,19 @@ describe(MediaService.name, () => { expect(mocks.media.generateThumbnail).toHaveBeenCalledTimes(2); expect(mocks.media.generateThumbnail).toHaveBeenCalledWith( - rawBuffer, + { data: rawBuffer, info: rawInfo }, expect.objectContaining({ processInvalidImages: false }), expect.any(String), ); expect(mocks.media.generateThumbnail).toHaveBeenCalledWith( - rawBuffer, + { data: rawBuffer, info: rawInfo }, expect.objectContaining({ processInvalidImages: false }), expect.any(String), ); expect(mocks.media.generateThumbhash).toHaveBeenCalledOnce(); expect(mocks.media.generateThumbhash).toHaveBeenCalledWith( - rawBuffer, + { data: rawBuffer, info: rawInfo }, expect.objectContaining({ processInvalidImages: false }), ); @@ -1074,7 +1070,7 @@ describe(MediaService.name, () => { expect(mocks.media.generateThumbnail).toHaveBeenCalledTimes(2); expect(mocks.media.generateThumbnail).toHaveBeenCalledWith( - fullsizeBuffer, + { data: fullsizeBuffer, info: rawInfo }, { colorspace: Colorspace.P3, format: ImageFormat.Jpeg, @@ -1082,7 +1078,6 @@ describe(MediaService.name, () => { quality: 80, progressive: false, processInvalidImages: false, - raw: rawInfo, edits: [], }, expect.any(String), @@ -1111,20 +1106,19 @@ describe(MediaService.name, () => { expect(mocks.media.generateThumbnail).toHaveBeenCalledTimes(3); expect(mocks.media.generateThumbnail).toHaveBeenCalledWith( - fullsizeBuffer, + { data: fullsizeBuffer, info: rawInfo }, { colorspace: Colorspace.P3, format: ImageFormat.Webp, quality: 80, progressive: false, processInvalidImages: false, - raw: rawInfo, edits: [], }, expect.any(String), ); expect(mocks.media.generateThumbnail).toHaveBeenCalledWith( - fullsizeBuffer, + { data: fullsizeBuffer, info: rawInfo }, { colorspace: Colorspace.P3, format: ImageFormat.Jpeg, @@ -1132,7 +1126,6 @@ describe(MediaService.name, () => { quality: 80, progressive: false, processInvalidImages: false, - raw: rawInfo, edits: [], }, expect.any(String), @@ -1159,20 +1152,19 @@ describe(MediaService.name, () => { expect(mocks.media.generateThumbnail).toHaveBeenCalledTimes(3); expect(mocks.media.generateThumbnail).toHaveBeenCalledWith( - rawBuffer, + { data: rawBuffer, info: rawInfo }, { colorspace: Colorspace.P3, format: ImageFormat.Jpeg, quality: 80, progressive: false, processInvalidImages: false, - raw: rawInfo, edits: [], }, expect.any(String), ); expect(mocks.media.generateThumbnail).toHaveBeenCalledWith( - rawBuffer, + { data: rawBuffer, info: rawInfo }, { colorspace: Colorspace.P3, format: ImageFormat.Jpeg, @@ -1180,7 +1172,6 @@ describe(MediaService.name, () => { progressive: false, size: 1440, processInvalidImages: false, - raw: rawInfo, edits: [], }, expect.any(String), @@ -1211,14 +1202,13 @@ describe(MediaService.name, () => { expect(mocks.media.generateThumbnail).toHaveBeenCalledTimes(3); expect(mocks.media.generateThumbnail).toHaveBeenCalledWith( - rawBuffer, + { data: rawBuffer, info: rawInfo }, { colorspace: Colorspace.P3, format: ImageFormat.Jpeg, quality: 80, progressive: false, processInvalidImages: false, - raw: rawInfo, edits: [], }, expect.any(String), @@ -1276,14 +1266,13 @@ describe(MediaService.name, () => { expect(mocks.media.generateThumbnail).toHaveBeenCalledTimes(3); expect(mocks.media.generateThumbnail).toHaveBeenCalledWith( - rawBuffer, + { data: rawBuffer, info: rawInfo }, { colorspace: Colorspace.Srgb, format: ImageFormat.Jpeg, quality: 80, progressive: false, processInvalidImages: false, - raw: rawInfo, edits: [], }, expect.any(String), @@ -1319,14 +1308,13 @@ describe(MediaService.name, () => { expect(mocks.media.generateThumbnail).toHaveBeenCalledTimes(3); expect(mocks.media.generateThumbnail).toHaveBeenCalledWith( - rawBuffer, + { data: rawBuffer, info: rawInfo }, { colorspace: Colorspace.P3, format: ImageFormat.Webp, quality: 90, progressive: false, processInvalidImages: false, - raw: rawInfo, edits: [], }, expect.any(String), @@ -1352,7 +1340,7 @@ describe(MediaService.name, () => { expect(mocks.media.generateThumbnail).toHaveBeenCalledTimes(3); expect(mocks.media.generateThumbnail).toHaveBeenCalledWith( - rawBuffer, + { data: rawBuffer, info: rawInfo }, expect.objectContaining({ format: ImageFormat.Jpeg, progressive: true, @@ -1426,7 +1414,7 @@ describe(MediaService.name, () => { await sut.handleAssetEditThumbnailGeneration({ id: asset.id }); expect(mocks.media.generateThumbnail).toHaveBeenCalledWith( - rawBuffer, + { data: rawBuffer, info: rawInfo }, expect.objectContaining({ edits: [ expect.objectContaining({ @@ -1474,17 +1462,17 @@ describe(MediaService.name, () => { expect(mocks.media.generateThumbnail).toHaveBeenCalledTimes(3); expect(mocks.media.generateThumbnail).toHaveBeenCalledWith( - rawBuffer, + { data: rawBuffer, info: rawInfo }, expect.anything(), expect.stringContaining('preview_edited.jpeg'), ); expect(mocks.media.generateThumbnail).toHaveBeenCalledWith( - rawBuffer, + { data: rawBuffer, info: rawInfo }, expect.anything(), expect.stringContaining('thumbnail_edited.webp'), ); expect(mocks.media.generateThumbnail).toHaveBeenCalledWith( - rawBuffer, + { data: rawBuffer, info: rawInfo }, expect.anything(), expect.stringContaining('fullsize_edited.jpeg'), ); @@ -1570,7 +1558,7 @@ describe(MediaService.name, () => { processInvalidImages: false, }); expect(mocks.media.generateThumbnail).toHaveBeenCalledWith( - data, + { data, info }, { colorspace: Colorspace.P3, format: ImageFormat.Jpeg, @@ -1587,7 +1575,6 @@ describe(MediaService.name, () => { }, }, ], - raw: info, processInvalidImages: false, size: 250, }, @@ -1624,7 +1611,7 @@ describe(MediaService.name, () => { processInvalidImages: false, }); expect(mocks.media.generateThumbnail).toHaveBeenCalledWith( - data, + { data, info }, { colorspace: Colorspace.P3, format: ImageFormat.Jpeg, @@ -1641,7 +1628,6 @@ describe(MediaService.name, () => { }, }, ], - raw: info, processInvalidImages: false, size: 250, }, @@ -1673,7 +1659,7 @@ describe(MediaService.name, () => { processInvalidImages: false, }); expect(mocks.media.generateThumbnail).toHaveBeenCalledWith( - data, + { data, info }, { colorspace: Colorspace.P3, format: ImageFormat.Jpeg, @@ -1690,7 +1676,6 @@ describe(MediaService.name, () => { }, }, ], - raw: info, processInvalidImages: false, size: 250, }, @@ -1718,7 +1703,7 @@ describe(MediaService.name, () => { processInvalidImages: false, }); expect(mocks.media.generateThumbnail).toHaveBeenCalledWith( - data, + { data, info }, { colorspace: Colorspace.P3, format: ImageFormat.Jpeg, @@ -1735,7 +1720,6 @@ describe(MediaService.name, () => { }, }, ], - raw: info, processInvalidImages: false, size: 250, }, @@ -1763,7 +1747,7 @@ describe(MediaService.name, () => { processInvalidImages: false, }); expect(mocks.media.generateThumbnail).toHaveBeenCalledWith( - data, + { data, info }, { colorspace: Colorspace.P3, format: ImageFormat.Jpeg, @@ -1780,7 +1764,6 @@ describe(MediaService.name, () => { }, }, ], - raw: info, processInvalidImages: false, size: 250, }, @@ -1808,7 +1791,7 @@ describe(MediaService.name, () => { processInvalidImages: false, }); expect(mocks.media.generateThumbnail).toHaveBeenCalledWith( - data, + { data, info }, { colorspace: Colorspace.P3, format: ImageFormat.Jpeg, @@ -1825,7 +1808,6 @@ describe(MediaService.name, () => { }, }, ], - raw: info, processInvalidImages: false, size: 250, }, @@ -1858,7 +1840,7 @@ describe(MediaService.name, () => { processInvalidImages: false, }); expect(mocks.media.generateThumbnail).toHaveBeenCalledWith( - data, + { data, info }, { colorspace: Colorspace.P3, format: ImageFormat.Jpeg, @@ -1875,7 +1857,6 @@ describe(MediaService.name, () => { }, }, ], - raw: info, processInvalidImages: false, size: 250, }, diff --git a/server/src/services/media.service.ts b/server/src/services/media.service.ts index 9dc0bcfb8..129b99a10 100644 --- a/server/src/services/media.service.ts +++ b/server/src/services/media.service.ts @@ -172,12 +172,10 @@ export class MediaService extends BaseService { const extractedImage = await this.extractOriginalImage(asset, config.image); const { info, data, colorspace } = extractedImage; - thumbhash = await this.mediaRepository.generateThumbhash(data, { - colorspace, - processInvalidImages: false, - raw: info, - edits: [], - }); + thumbhash = await this.mediaRepository.generateThumbhash( + { data, info }, + { colorspace, processInvalidImages: false, edits: [] }, + ); } if (!asset.thumbhash || Buffer.compare(asset.thumbhash, thumbhash) !== 0) { @@ -318,13 +316,14 @@ export class MediaService extends BaseService { this.storageCore.ensureFolders(previewFile.path); // generate final images - const baseOptions = { colorspace, processInvalidImages: false, raw: info, edits: useEdits ? asset.edits : [] }; + const decoded = { data, info }; + const baseOptions = { colorspace, processInvalidImages: false, edits: useEdits ? asset.edits : [] }; const thumbnailOptions = { ...image.thumbnail, ...baseOptions, format: thumbnailFormat }; const previewOptions = { ...image.preview, ...baseOptions, format: previewFormat }; const promises = [ - this.mediaRepository.generateThumbhash(data, baseOptions), - this.mediaRepository.generateThumbnail(data, thumbnailOptions, thumbnailFile.path), - this.mediaRepository.generateThumbnail(data, previewOptions, previewFile.path), + this.mediaRepository.generateThumbhash(decoded, baseOptions), + this.mediaRepository.generateThumbnail(decoded, thumbnailOptions, thumbnailFile.path), + this.mediaRepository.generateThumbnail(decoded, previewOptions, previewFile.path), ]; let fullsizeFile: UpsertFileOptions | undefined; @@ -345,7 +344,7 @@ export class MediaService extends BaseService { quality: image.fullsize.quality, progressive: image.fullsize.progressive, }; - promises.push(this.mediaRepository.generateThumbnail(data, fullsizeOptions, fullsizeFile.path)); + promises.push(this.mediaRepository.generateThumbnail(decoded, fullsizeOptions, fullsizeFile.path)); } else if (generateFullsize && extracted && extracted.format === RawExtractedFormat.Jpeg) { fullsizeFile = this.getImageFile(asset, { fileType: AssetFileType.FullSize, @@ -416,7 +415,7 @@ export class MediaService extends BaseService { inputImage = originalPath; } - const { data: decodedImage, info } = await this.mediaRepository.decodeImage(inputImage, { + const decoded = await this.mediaRepository.decodeImage(inputImage, { colorspace: image.colorspace, processInvalidImages: process.env.IMMICH_PROCESS_INVALID_IMAGES === 'true', // if this is an extracted image, it may not have orientation metadata @@ -429,7 +428,6 @@ export class MediaService extends BaseService { const thumbnailOptions: GenerateThumbnailOptions = { colorspace: image.colorspace, format: ImageFormat.Jpeg, - raw: info, quality: image.thumbnail.quality, progressive: false, processInvalidImages: false, @@ -438,14 +436,17 @@ export class MediaService extends BaseService { { action: AssetEditAction.Crop, parameters: this.getCrop( - { old: { width: oldWidth, height: oldHeight }, new: { width: info.width, height: info.height } }, + { + old: { width: oldWidth, height: oldHeight }, + new: { width: decoded.info.width, height: decoded.info.height }, + }, { x1, y1, x2, y2 }, ), }, ], }; - await this.mediaRepository.generateThumbnail(decodedImage, thumbnailOptions, thumbnailPath); + await this.mediaRepository.generateThumbnail(decoded, thumbnailOptions, thumbnailPath); await this.personRepository.update({ ownerId, personGroupId, thumbnailPath }); return JobStatus.Success; @@ -519,9 +520,14 @@ export class MediaService extends BaseService { await this.mediaRepository.transcode(asset.originalPath, previewFile.path, previewOptions); await this.mediaRepository.transcode(asset.originalPath, thumbnailFile.path, thumbnailOptions); - const thumbhash = await this.mediaRepository.generateThumbhash(previewFile.path, { + const processInvalidImages = process.env.IMMICH_PROCESS_INVALID_IMAGES === 'true'; + const preview = await this.mediaRepository.decodeImage(previewFile.path, { colorspace: image.colorspace, - processInvalidImages: process.env.IMMICH_PROCESS_INVALID_IMAGES === 'true', + processInvalidImages, + }); + const thumbhash = await this.mediaRepository.generateThumbhash(preview, { + colorspace: image.colorspace, + processInvalidImages, }); return { diff --git a/server/src/types.ts b/server/src/types.ts index 197e97810..b8036a5de 100644 --- a/server/src/types.ts +++ b/server/src/types.ts @@ -62,20 +62,31 @@ export type RawImageInfo = { channels: 1 | 2 | 3 | 4; }; -type DecodeImageOptions = { - colorspace: string; - processInvalidImages: boolean; - raw?: RawImageInfo; - edits?: AssetEditActionItem[]; +export type Bitmap = { + data: Buffer; + info: RawImageInfo; }; -export interface DecodeToBufferOptions extends DecodeImageOptions { +type ImageColorOptions = { + colorspace: string; + processInvalidImages: boolean; +}; + +export interface DecodeToBufferOptions extends ImageColorOptions { size?: number; orientation?: ExifOrientation; } -export type GenerateThumbnailOptions = Pick & DecodeToBufferOptions; -export type GenerateThumbhashOptions = DecodeImageOptions; +export type TransformOptions = { + size?: number; + fit?: 'inside' | 'outside'; + edits?: AssetEditActionItem[]; +}; + +export type GenerateThumbnailOptions = Pick & + ImageColorOptions & + TransformOptions; +export type GenerateThumbhashOptions = ImageColorOptions & Pick; export interface VideoStreamInfo { index: number; diff --git a/server/src/utils/profile-image.ts b/server/src/utils/profile-image.ts index 60cc3fbe9..2fb3e9499 100644 --- a/server/src/utils/profile-image.ts +++ b/server/src/utils/profile-image.ts @@ -23,8 +23,9 @@ export const generateProfileImage = async ( ); storageCore.ensureFolders(outputPath); + const decoded = await media.decodeImage(input, { colorspace: image.colorspace, processInvalidImages: false }); await media.generateThumbnail( - input, + decoded, { colorspace: image.colorspace, format: image.thumbnail.format,