From 62c128d1fb5e5534e7dedb8d8d54c27a921c2c1d Mon Sep 17 00:00:00 2001
From: Andre Sichelero <andre.gasolisichelero@rwdi.com>
Date: Mon, 5 May 2025 21:03:02 -0300
Subject: [PATCH] fix(transcode): conditionally fix VAAPI filter order for HEVC
 (tested on Haswel) l Places 'format=nv12' before 'hwupload' in the VAAPI
 video filter chain only when the input codec is HEVC and hardware decoding is
 disabled. This resolves filtergraph/driver issues observed on Intel Haswell
 GPUs when transcoding HEVC->H.264 VAAPI via software decode, ensuring the
 encoder receives the expected NV12 surface. Retains default filter logic for
 other codecs. Addresses issues discussed in GitHub issue #18103

---
 server/src/utils/media.ts | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/server/src/utils/media.ts b/server/src/utils/media.ts
index b00eb652ef..f251a5d8ed 100644
--- a/server/src/utils/media.ts
+++ b/server/src/utils/media.ts
@@ -781,7 +781,12 @@ export class VaapiSwDecodeConfig extends BaseHWConfig {
 
   getFilterOptions(videoStream: VideoStreamInfo) {
     const options = this.getToneMapping(videoStream);
-    options.push('hwupload=extra_hw_frames=64');
+    if (videoStream.codecName === VideoCodec.HEVC) {
+      options.push('format=nv12');
+      options.push('hwupload=extra_hw_frames=64');
+    } else {
+      options.push('hwupload=extra_hw_frames=64');
+    }
     if (this.shouldScale(videoStream)) {
       options.push(`scale_vaapi=${this.getScaling(videoStream)}:mode=hq:out_range=pc:format=nv12`);
     }