mirror of
https://github.com/jellyfin/jellyfin.git
synced 2024-06-29 02:13:36 +02:00
Merge pull request #11014 from gnattu/vf-videotoolbox
This commit is contained in:
commit
a92de9b2e3
|
@ -253,6 +253,14 @@ namespace MediaBrowser.Controller.MediaEncoding
|
||||||
&& _mediaEncoder.SupportsFilterWithOption(FilterOptionType.OverlayVulkanFrameSync);
|
&& _mediaEncoder.SupportsFilterWithOption(FilterOptionType.OverlayVulkanFrameSync);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private bool IsVideoToolboxFullSupported()
|
||||||
|
{
|
||||||
|
return _mediaEncoder.SupportsHwaccel("videotoolbox")
|
||||||
|
&& _mediaEncoder.SupportsFilter("yadif_videotoolbox")
|
||||||
|
&& _mediaEncoder.SupportsFilter("overlay_videotoolbox")
|
||||||
|
&& _mediaEncoder.SupportsFilter("scale_vt");
|
||||||
|
}
|
||||||
|
|
||||||
private bool IsHwTonemapAvailable(EncodingJobInfo state, EncodingOptions options)
|
private bool IsHwTonemapAvailable(EncodingJobInfo state, EncodingOptions options)
|
||||||
{
|
{
|
||||||
if (state.VideoStream is null
|
if (state.VideoStream is null
|
||||||
|
@ -272,7 +280,8 @@ namespace MediaBrowser.Controller.MediaEncoding
|
||||||
var isNvdecDecoder = vidDecoder.Contains("cuda", StringComparison.OrdinalIgnoreCase);
|
var isNvdecDecoder = vidDecoder.Contains("cuda", StringComparison.OrdinalIgnoreCase);
|
||||||
var isVaapiDecoder = vidDecoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase);
|
var isVaapiDecoder = vidDecoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase);
|
||||||
var isD3d11vaDecoder = vidDecoder.Contains("d3d11va", StringComparison.OrdinalIgnoreCase);
|
var isD3d11vaDecoder = vidDecoder.Contains("d3d11va", StringComparison.OrdinalIgnoreCase);
|
||||||
return isSwDecoder || isNvdecDecoder || isVaapiDecoder || isD3d11vaDecoder;
|
var isVideoToolBoxDecoder = vidDecoder.Contains("videotoolbox", StringComparison.OrdinalIgnoreCase);
|
||||||
|
return isSwDecoder || isNvdecDecoder || isVaapiDecoder || isD3d11vaDecoder || isVideoToolBoxDecoder;
|
||||||
}
|
}
|
||||||
|
|
||||||
return state.VideoStream.VideoRange == VideoRange.HDR
|
return state.VideoStream.VideoRange == VideoRange.HDR
|
||||||
|
@ -308,6 +317,21 @@ namespace MediaBrowser.Controller.MediaEncoding
|
||||||
&& state.VideoStream.VideoRangeType == VideoRangeType.HDR10;
|
&& state.VideoStream.VideoRangeType == VideoRangeType.HDR10;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private bool IsVideoToolboxTonemapAvailable(EncodingJobInfo state, EncodingOptions options)
|
||||||
|
{
|
||||||
|
if (state.VideoStream is null
|
||||||
|
|| !options.EnableVideoToolboxTonemapping
|
||||||
|
|| GetVideoColorBitDepth(state) != 10)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Certain DV profile 5 video works in Safari with direct playing, but the VideoToolBox does not produce correct mapping results with transcoding.
|
||||||
|
// All other HDR formats working.
|
||||||
|
return state.VideoStream.VideoRange == VideoRange.HDR
|
||||||
|
&& state.VideoStream.VideoRangeType is VideoRangeType.HDR10 or VideoRangeType.HLG or VideoRangeType.HDR10Plus;
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the name of the output video codec.
|
/// Gets the name of the output video codec.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
|
@ -4954,22 +4978,30 @@ namespace MediaBrowser.Controller.MediaEncoding
|
||||||
return (null, null, null);
|
return (null, null, null);
|
||||||
}
|
}
|
||||||
|
|
||||||
var swFilterChain = GetSwVidFilterChain(state, options, vidEncoder);
|
var isMacOS = OperatingSystem.IsMacOS();
|
||||||
|
var vidDecoder = GetHardwareVideoDecoder(state, options) ?? string.Empty;
|
||||||
|
var isVtEncoder = vidEncoder.Contains("videotoolbox", StringComparison.OrdinalIgnoreCase);
|
||||||
|
var isVtFullSupported = isMacOS && IsVideoToolboxFullSupported();
|
||||||
|
var isVtOclSupported = isVtFullSupported && IsOpenclFullSupported();
|
||||||
|
|
||||||
if (!options.EnableHardwareEncoding)
|
// legacy videotoolbox pipeline (disable hw filters)
|
||||||
|
if (!isVtEncoder
|
||||||
|
|| !isVtOclSupported
|
||||||
|
|| !_mediaEncoder.SupportsFilter("alphasrc"))
|
||||||
{
|
{
|
||||||
return swFilterChain;
|
return GetSwVidFilterChain(state, options, vidEncoder);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (_mediaEncoder.EncoderVersion.CompareTo(new Version("5.0.0")) < 0)
|
// preferred videotoolbox + vt/ocl filters pipeline
|
||||||
{
|
return GetAppleVidFiltersPreferred(state, options, vidDecoder, vidEncoder);
|
||||||
// All features used here requires ffmpeg 5.0 or later, fallback to software filters if using an old ffmpeg
|
}
|
||||||
return swFilterChain;
|
|
||||||
}
|
|
||||||
|
|
||||||
var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true);
|
public (List<string> MainFilters, List<string> SubFilters, List<string> OverlayFilters) GetAppleVidFiltersPreferred(
|
||||||
var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true);
|
EncodingJobInfo state,
|
||||||
var doDeintH2645 = doDeintH264 || doDeintHevc;
|
EncodingOptions options,
|
||||||
|
string vidDecoder,
|
||||||
|
string vidEncoder)
|
||||||
|
{
|
||||||
var inW = state.VideoStream?.Width;
|
var inW = state.VideoStream?.Width;
|
||||||
var inH = state.VideoStream?.Height;
|
var inH = state.VideoStream?.Height;
|
||||||
var reqW = state.BaseRequest.Width;
|
var reqW = state.BaseRequest.Width;
|
||||||
|
@ -4977,33 +5009,114 @@ namespace MediaBrowser.Controller.MediaEncoding
|
||||||
var reqMaxW = state.BaseRequest.MaxWidth;
|
var reqMaxW = state.BaseRequest.MaxWidth;
|
||||||
var reqMaxH = state.BaseRequest.MaxHeight;
|
var reqMaxH = state.BaseRequest.MaxHeight;
|
||||||
var threeDFormat = state.MediaSource.Video3DFormat;
|
var threeDFormat = state.MediaSource.Video3DFormat;
|
||||||
var newfilters = new List<string>();
|
|
||||||
var noOverlay = swFilterChain.OverlayFilters.Count == 0;
|
|
||||||
var supportsHwDeint = _mediaEncoder.SupportsFilter("yadif_videotoolbox");
|
|
||||||
// fallback to software filters if we are using filters not supported by hardware yet.
|
|
||||||
var useHardwareFilters = noOverlay && (!doDeintH2645 || supportsHwDeint);
|
|
||||||
|
|
||||||
if (!useHardwareFilters)
|
var isVtEncoder = vidEncoder.Contains("videotoolbox", StringComparison.OrdinalIgnoreCase);
|
||||||
|
|
||||||
|
var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true);
|
||||||
|
var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true);
|
||||||
|
var doDeintH2645 = doDeintH264 || doDeintHevc;
|
||||||
|
var doVtTonemap = IsVideoToolboxTonemapAvailable(state, options);
|
||||||
|
var doOclTonemap = !doVtTonemap && IsHwTonemapAvailable(state, options);
|
||||||
|
|
||||||
|
var scaleFormat = string.Empty;
|
||||||
|
if (!string.Equals(state.VideoStream.PixelFormat, "yuv420p", StringComparison.OrdinalIgnoreCase))
|
||||||
{
|
{
|
||||||
return swFilterChain;
|
// Use P010 for OpenCL tone mapping, otherwise force an 8bit output.
|
||||||
|
scaleFormat = doOclTonemap ? "p010le" : "nv12";
|
||||||
}
|
}
|
||||||
|
|
||||||
// ffmpeg cannot use videotoolbox to scale
|
var hwScaleFilter = GetHwScaleFilter("vt", scaleFormat, inW, inH, reqW, reqH, reqMaxW, reqMaxH);
|
||||||
var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH);
|
|
||||||
newfilters.Add(swScaleFilter);
|
|
||||||
|
|
||||||
// hwupload on videotoolbox encoders can automatically convert AVFrame into its CVPixelBuffer equivalent
|
var hasSubs = state.SubtitleStream is not null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode;
|
||||||
// videotoolbox will automatically convert the CVPixelBuffer to a pixel format the encoder supports, so we don't have to set a pixel format explicitly here
|
var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream;
|
||||||
// This will reduce CPU usage significantly on UHD videos with 10 bit colors because we bypassed the ffmpeg pixel format conversion
|
var hasGraphicalSubs = hasSubs && !state.SubtitleStream.IsTextSubtitleStream;
|
||||||
newfilters.Add("hwupload");
|
var hasAssSubs = hasSubs
|
||||||
|
&& (string.Equals(state.SubtitleStream.Codec, "ass", StringComparison.OrdinalIgnoreCase)
|
||||||
|
|| string.Equals(state.SubtitleStream.Codec, "ssa", StringComparison.OrdinalIgnoreCase));
|
||||||
|
|
||||||
|
if (!isVtEncoder)
|
||||||
|
{
|
||||||
|
// should not happen.
|
||||||
|
return (null, null, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Make main filters for video stream */
|
||||||
|
var mainFilters = new List<string>();
|
||||||
|
|
||||||
|
// Color override is only required for OpenCL where hardware surface is in use
|
||||||
|
if (doOclTonemap)
|
||||||
|
{
|
||||||
|
mainFilters.Add(GetOverwriteColorPropertiesParam(state, doOclTonemap));
|
||||||
|
}
|
||||||
|
|
||||||
|
// INPUT videotoolbox/memory surface(vram/uma)
|
||||||
|
// this will pass-through automatically if in/out format matches.
|
||||||
|
mainFilters.Add("format=nv12|p010le|videotoolbox_vld");
|
||||||
|
mainFilters.Add("hwupload=derive_device=videotoolbox");
|
||||||
|
|
||||||
|
// hw deint
|
||||||
if (doDeintH2645)
|
if (doDeintH2645)
|
||||||
{
|
{
|
||||||
var deintFilter = GetHwDeinterlaceFilter(state, options, "videotoolbox");
|
var deintFilter = GetHwDeinterlaceFilter(state, options, "videotoolbox");
|
||||||
newfilters.Add(deintFilter);
|
mainFilters.Add(deintFilter);
|
||||||
}
|
}
|
||||||
|
|
||||||
return (newfilters, swFilterChain.SubFilters, swFilterChain.OverlayFilters);
|
if (doVtTonemap)
|
||||||
|
{
|
||||||
|
const string VtTonemapArgs = "color_matrix=bt709:color_primaries=bt709:color_transfer=bt709";
|
||||||
|
|
||||||
|
// scale_vt can handle scaling & tonemapping in one shot, just like vpp_qsv.
|
||||||
|
hwScaleFilter = string.IsNullOrEmpty(hwScaleFilter)
|
||||||
|
? "scale_vt=" + VtTonemapArgs
|
||||||
|
: hwScaleFilter + ":" + VtTonemapArgs;
|
||||||
|
}
|
||||||
|
|
||||||
|
// hw scale & vt tonemap
|
||||||
|
mainFilters.Add(hwScaleFilter);
|
||||||
|
|
||||||
|
// ocl tonemap
|
||||||
|
if (doOclTonemap)
|
||||||
|
{
|
||||||
|
// map from videotoolbox to opencl via videotoolbox-opencl interop.
|
||||||
|
mainFilters.Add("hwmap=derive_device=opencl:mode=read");
|
||||||
|
|
||||||
|
var tonemapFilter = GetHwTonemapFilter(options, "opencl", "nv12");
|
||||||
|
mainFilters.Add(tonemapFilter);
|
||||||
|
|
||||||
|
// OUTPUT videotoolbox(nv12) surface(vram/uma)
|
||||||
|
// reverse-mapping via videotoolbox-opencl interop.
|
||||||
|
mainFilters.Add("hwmap=derive_device=videotoolbox:mode=write:reverse=1");
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Make sub and overlay filters for subtitle stream */
|
||||||
|
var subFilters = new List<string>();
|
||||||
|
var overlayFilters = new List<string>();
|
||||||
|
|
||||||
|
if (hasSubs)
|
||||||
|
{
|
||||||
|
if (hasGraphicalSubs)
|
||||||
|
{
|
||||||
|
var subPreProcFilters = GetGraphicalSubPreProcessFilters(inW, inH, reqW, reqH, reqMaxW, reqMaxH);
|
||||||
|
subFilters.Add(subPreProcFilters);
|
||||||
|
subFilters.Add("format=bgra");
|
||||||
|
}
|
||||||
|
else if (hasTextSubs)
|
||||||
|
{
|
||||||
|
var framerate = state.VideoStream?.RealFrameRate;
|
||||||
|
var subFramerate = hasAssSubs ? Math.Min(framerate ?? 25, 60) : 10;
|
||||||
|
|
||||||
|
var alphaSrcFilter = GetAlphaSrcFilter(state, inW, inH, reqW, reqH, reqMaxW, reqMaxH, subFramerate);
|
||||||
|
var subTextSubtitlesFilter = GetTextSubtitlesFilter(state, true, true);
|
||||||
|
subFilters.Add(alphaSrcFilter);
|
||||||
|
subFilters.Add("format=bgra");
|
||||||
|
subFilters.Add(subTextSubtitlesFilter);
|
||||||
|
}
|
||||||
|
|
||||||
|
subFilters.Add("hwupload=derive_device=videotoolbox");
|
||||||
|
overlayFilters.Add("overlay_videotoolbox=eof_action=pass:repeatlast=0");
|
||||||
|
}
|
||||||
|
|
||||||
|
return (mainFilters, subFilters, overlayFilters);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
@ -5995,22 +6108,37 @@ namespace MediaBrowser.Controller.MediaEncoding
|
||||||
|| string.Equals("yuvj420p", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase);
|
|| string.Equals("yuvj420p", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase);
|
||||||
var is8_10bitSwFormatsVt = is8bitSwFormatsVt || string.Equals("yuv420p10le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase);
|
var is8_10bitSwFormatsVt = is8bitSwFormatsVt || string.Equals("yuv420p10le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase);
|
||||||
|
|
||||||
|
// Hardware surface only make sense when interop with OpenCL
|
||||||
|
// VideoToolbox's Hardware surface in ffmpeg is not only slower than hwupload, but also breaks HDR in many cases.
|
||||||
|
// For example: https://trac.ffmpeg.org/ticket/10884
|
||||||
|
var useOclToneMapping = !IsVideoToolboxTonemapAvailable(state, options)
|
||||||
|
&& options.EnableTonemapping
|
||||||
|
&& state.VideoStream is not null
|
||||||
|
&& GetVideoColorBitDepth(state) == 10
|
||||||
|
&& state.VideoStream.VideoRange == VideoRange.HDR
|
||||||
|
&& (state.VideoStream.VideoRangeType == VideoRangeType.HDR10
|
||||||
|
|| state.VideoStream.VideoRangeType == VideoRangeType.HLG
|
||||||
|
|| (state.VideoStream.VideoRangeType == VideoRangeType.DOVI
|
||||||
|
&& string.Equals(state.VideoStream.Codec, "hevc", StringComparison.OrdinalIgnoreCase)));
|
||||||
|
|
||||||
|
var useHwSurface = useOclToneMapping && IsVideoToolboxFullSupported() && _mediaEncoder.SupportsFilter("alphasrc");
|
||||||
|
|
||||||
if (is8bitSwFormatsVt)
|
if (is8bitSwFormatsVt)
|
||||||
{
|
{
|
||||||
if (string.Equals("avc", videoStream.Codec, StringComparison.OrdinalIgnoreCase)
|
if (string.Equals("avc", videoStream.Codec, StringComparison.OrdinalIgnoreCase)
|
||||||
|| string.Equals("h264", videoStream.Codec, StringComparison.OrdinalIgnoreCase))
|
|| string.Equals("h264", videoStream.Codec, StringComparison.OrdinalIgnoreCase))
|
||||||
{
|
{
|
||||||
return GetHwaccelType(state, options, "h264", bitDepth, false);
|
return GetHwaccelType(state, options, "h264", bitDepth, useHwSurface);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (string.Equals("mpeg2video", videoStream.Codec, StringComparison.OrdinalIgnoreCase))
|
if (string.Equals("mpeg2video", videoStream.Codec, StringComparison.OrdinalIgnoreCase))
|
||||||
{
|
{
|
||||||
return GetHwaccelType(state, options, "mpeg2video", bitDepth, false);
|
return GetHwaccelType(state, options, "mpeg2video", bitDepth, useHwSurface);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (string.Equals("mpeg4", videoStream.Codec, StringComparison.OrdinalIgnoreCase))
|
if (string.Equals("mpeg4", videoStream.Codec, StringComparison.OrdinalIgnoreCase))
|
||||||
{
|
{
|
||||||
return GetHwaccelType(state, options, "mpeg4", bitDepth, false);
|
return GetHwaccelType(state, options, "mpeg4", bitDepth, useHwSurface);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -6019,12 +6147,12 @@ namespace MediaBrowser.Controller.MediaEncoding
|
||||||
if (string.Equals("hevc", videoStream.Codec, StringComparison.OrdinalIgnoreCase)
|
if (string.Equals("hevc", videoStream.Codec, StringComparison.OrdinalIgnoreCase)
|
||||||
|| string.Equals("h265", videoStream.Codec, StringComparison.OrdinalIgnoreCase))
|
|| string.Equals("h265", videoStream.Codec, StringComparison.OrdinalIgnoreCase))
|
||||||
{
|
{
|
||||||
return GetHwaccelType(state, options, "hevc", bitDepth, false);
|
return GetHwaccelType(state, options, "hevc", bitDepth, useHwSurface);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (string.Equals("vp9", videoStream.Codec, StringComparison.OrdinalIgnoreCase))
|
if (string.Equals("vp9", videoStream.Codec, StringComparison.OrdinalIgnoreCase))
|
||||||
{
|
{
|
||||||
return GetHwaccelType(state, options, "vp9", bitDepth, false);
|
return GetHwaccelType(state, options, "vp9", bitDepth, useHwSurface);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -128,6 +128,8 @@ namespace MediaBrowser.MediaEncoding.Encoder
|
||||||
"overlay_vulkan",
|
"overlay_vulkan",
|
||||||
// videotoolbox
|
// videotoolbox
|
||||||
"yadif_videotoolbox",
|
"yadif_videotoolbox",
|
||||||
|
"scale_vt",
|
||||||
|
"overlay_videotoolbox",
|
||||||
// rkrga
|
// rkrga
|
||||||
"scale_rkrga",
|
"scale_rkrga",
|
||||||
"vpp_rkrga",
|
"vpp_rkrga",
|
||||||
|
|
|
@ -28,6 +28,7 @@ public class EncodingOptions
|
||||||
VaapiDevice = "/dev/dri/renderD128";
|
VaapiDevice = "/dev/dri/renderD128";
|
||||||
EnableTonemapping = false;
|
EnableTonemapping = false;
|
||||||
EnableVppTonemapping = false;
|
EnableVppTonemapping = false;
|
||||||
|
EnableVideoToolboxTonemapping = false;
|
||||||
TonemappingAlgorithm = "bt2390";
|
TonemappingAlgorithm = "bt2390";
|
||||||
TonemappingMode = "auto";
|
TonemappingMode = "auto";
|
||||||
TonemappingRange = "auto";
|
TonemappingRange = "auto";
|
||||||
|
@ -146,6 +147,11 @@ public class EncodingOptions
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public bool EnableVppTonemapping { get; set; }
|
public bool EnableVppTonemapping { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets a value indicating whether videotoolbox tonemapping is enabled.
|
||||||
|
/// </summary>
|
||||||
|
public bool EnableVideoToolboxTonemapping { get; set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the tone-mapping algorithm.
|
/// Gets or sets the tone-mapping algorithm.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
|
|
Loading…
Reference in a new issue