fix(server): convert to nv12 before hwupload for VAAPI with software decoding (#32474)

* fix(server): convert to nv12 before hwupload for VAAPI with software decoding

* test(server): cover nv12 conversion for VAAPI sw decode

* output nv12 directly

---------

Co-authored-by: mertalev <101130780+mertalev@users.noreply.github.com>
(cherry picked from commit 4bdc4480b2771a5a0dfed26e1149d680f6c204d9)

server/src/services/media.service.spec.ts

@@ -3099,7 +3099,7 @@ describe(MediaService.name, () => {
             '-v',
             'verbose',
             '-vf',
-            'hwupload=extra_hw_frames=64,scale_qsv=-1:720:mode=hq:format=nv12',
+            'format=nv12,hwupload=extra_hw_frames=64,scale_qsv=-1:720:mode=hq:format=nv12',
             '-preset',
             '7',
             '-global_quality:v',
@@ -3116,6 +3116,24 @@ describe(MediaService.name, () => {
       );
     });
 
+    it('should tone-map to nv12 before upload for sw decode qsv', async () => {
+      mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStreamHDR });
+      mocks.systemMetadata.get.mockResolvedValue({
+        ffmpeg: { accel: TranscodeHardwareAcceleration.Qsv, accelDecode: false },
+      });
+      await sut.handleVideoConversion({ id: 'video-id' });
+      expect(mocks.media.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        expect.any(String),
+        expect.objectContaining({
+          outputOptions: expect.arrayContaining([
+            '-vf',
+            'tonemapx=tonemap=hable:desat=0:p=bt709:t=bt709:m=bt709:r=pc:peak=100:format=nv12,hwupload=extra_hw_frames=64',
+          ]),
+        }),
+      );
+    });
+
     it('should set options for qsv with custom dri node with sw decode', async () => {
       mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
       mocks.systemMetadata.get.mockResolvedValue({
@@ -3347,7 +3365,7 @@ describe(MediaService.name, () => {
             '-v',
             'verbose',
             '-vf',
-            'hwupload=extra_hw_frames=64,scale_vaapi=-2:720:mode=hq:out_range=pc:format=nv12',
+            'format=nv12,hwupload=extra_hw_frames=64,scale_vaapi=-2:720:mode=hq:out_range=pc:format=nv12',
             '-compression_level',
             '7',
             '-rc_mode',
@@ -3358,6 +3376,39 @@ describe(MediaService.name, () => {
       );
     });
 
+    it('should convert to nv12 before upload for sw decode vaapi without scaling', async () => {
+      mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
+      mocks.systemMetadata.get.mockResolvedValue({
+        ffmpeg: { accel: TranscodeHardwareAcceleration.Vaapi, accelDecode: false, targetResolution: 'original' },
+      });
+      await sut.handleVideoConversion({ id: 'video-id' });
+      expect(mocks.media.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        expect.any(String),
+        expect.objectContaining({
+          outputOptions: expect.arrayContaining(['-vf', 'format=nv12,hwupload=extra_hw_frames=64']),
+        }),
+      );
+    });
+
+    it('should tone-map to nv12 before upload for sw decode vaapi', async () => {
+      mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.videoStreamHDR });
+      mocks.systemMetadata.get.mockResolvedValue({
+        ffmpeg: { accel: TranscodeHardwareAcceleration.Vaapi, accelDecode: false },
+      });
+      await sut.handleVideoConversion({ id: 'video-id' });
+      expect(mocks.media.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        expect.any(String),
+        expect.objectContaining({
+          outputOptions: expect.arrayContaining([
+            '-vf',
+            'tonemapx=tonemap=hable:desat=0:p=bt709:t=bt709:m=bt709:r=pc:peak=100:format=nv12,hwupload=extra_hw_frames=64',
+          ]),
+        }),
+      );
+    });
+
     it('should set vbr options for vaapi when max bitrate is enabled', async () => {
       mocks.assetJob.getForVideoConversion.mockResolvedValue({ ...asset, ...probeStub.matroskaContainer });
       mocks.systemMetadata.get.mockResolvedValue({

server/src/utils/media.ts

@@ -399,13 +399,13 @@ export class BaseConfig implements VideoCodecSWConfig {
     };
   }
 
-  getToneMapping(videoStream: VideoStreamInfo) {
+  getToneMapping(videoStream: VideoStreamInfo, format = 'yuv420p') {
     if (!this.shouldToneMap(videoStream)) {
       return [];
     }
 
     const { primaries, transfer, matrix } = this.getColors();
-    const options = `tonemapx=tonemap=${this.config.tonemap}:desat=0:p=${primaries}:t=${transfer}:m=${matrix}:r=pc:peak=100:format=yuv420p`;
+    const options = `tonemapx=tonemap=${this.config.tonemap}:desat=0:p=${primaries}:t=${transfer}:m=${matrix}:r=pc:peak=100:format=${format}`;
     return [options];
   }
 
@@ -818,6 +818,15 @@ export class QsvSwDecodeConfig extends BaseHWConfig {
     return options;
   }
 
+  getToneMapping(videoStream: VideoStreamInfo) {
+    // Output nv12 when tone-mapping instead of having ffmpeg convert from yuv420p
+    const options = super.getToneMapping(videoStream, 'nv12');
+    if (options.length === 0 && videoStream.pixelFormat !== 'nv12') {
+      options.push('format=nv12');
+    }
+    return options;
+  }
+
   getPresetOptions() {
     let presetIndex = this.getPresetIndex();
     if (presetIndex < 0) {
@@ -952,6 +961,15 @@ export class VaapiSwDecodeConfig extends BaseHWConfig {
     return options;
   }
 
+  getToneMapping(videoStream: VideoStreamInfo) {
+    // some drivers (e.g. Mesa radeonsi on AMD) reject yuv420p surfaces and FFmpeg doesn't auto-convert for VAAPI
+    const options = super.getToneMapping(videoStream, 'nv12');
+    if (options.length === 0 && videoStream.pixelFormat !== 'nv12') {
+      options.push('format=nv12');
+    }
+    return options;
+  }
+
   getPresetOptions() {
     let presetIndex = this.getPresetIndex();
     if (presetIndex < 0) {