diff options
Diffstat (limited to 'MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs')
| -rw-r--r-- | MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs | 2776 |
1 files changed, 1830 insertions, 946 deletions
diff --git a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs index 178a9999c8..9f7e35d1ea 100644 --- a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs +++ b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs @@ -1,19 +1,25 @@ #nullable disable #pragma warning disable CS1591 +// We need lowercase normalized string for ffmpeg +#pragma warning disable CA1308 using System; using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; +using System.Runtime.InteropServices; using System.Text; using System.Text.RegularExpressions; using System.Threading; +using Jellyfin.Data; using Jellyfin.Data.Enums; +using Jellyfin.Database.Implementations.Enums; using Jellyfin.Extensions; using MediaBrowser.Common.Configuration; using MediaBrowser.Controller.Extensions; +using MediaBrowser.Controller.IO; using MediaBrowser.Model.Configuration; using MediaBrowser.Model.Dlna; using MediaBrowser.Model.Dto; @@ -26,6 +32,22 @@ namespace MediaBrowser.Controller.MediaEncoding { public partial class EncodingHelper { + /// <summary> + /// The codec validation regex string. + /// This regular expression matches strings that consist of alphanumeric characters, hyphens, + /// periods, underscores, commas, and vertical bars, with a length between 0 and 40 characters. + /// This should matches all common valid codecs. + /// </summary> + public const string ContainerValidationRegexStr = @"^[a-zA-Z0-9\-\._,|]{0,40}$"; + + /// <summary> + /// The level validation regex string. + /// This regular expression matches strings representing a double. + /// </summary> + public const string LevelValidationRegexStr = @"-?[0-9]+(?:\.[0-9]+)?"; + + private const string _defaultMjpegEncoder = "mjpeg"; + private const string QsvAlias = "qs"; private const string VaapiAlias = "va"; private const string D3d11vaAlias = "dx11"; @@ -40,21 +62,33 @@ namespace MediaBrowser.Controller.MediaEncoding private readonly ISubtitleEncoder _subtitleEncoder; private readonly IConfiguration _config; private readonly IConfigurationManager _configurationManager; + private readonly IPathManager _pathManager; // i915 hang was fixed by linux 6.2 (3f882f2) private readonly Version _minKerneli915Hang = new Version(5, 18); private readonly Version _maxKerneli915Hang = new Version(6, 1, 3); private readonly Version _minFixedKernel60i915Hang = new Version(6, 0, 18); + private readonly Version _minKernelVersionAmdVkFmtModifier = new Version(5, 15); - private readonly Version _minFFmpegImplictHwaccel = new Version(6, 0); + private readonly Version _minFFmpegImplicitHwaccel = new Version(6, 0); private readonly Version _minFFmpegHwaUnsafeOutput = new Version(6, 0); private readonly Version _minFFmpegOclCuTonemapMode = new Version(5, 1, 3); private readonly Version _minFFmpegSvtAv1Params = new Version(5, 1); private readonly Version _minFFmpegVaapiH26xEncA53CcSei = new Version(6, 0); private readonly Version _minFFmpegReadrateOption = new Version(5, 0); - - private static readonly string[] _videoProfilesH264 = new[] - { + private readonly Version _minFFmpegWorkingVtHwSurface = new Version(7, 0, 1); + private readonly Version _minFFmpegDisplayRotationOption = new Version(6, 0); + private readonly Version _minFFmpegAdvancedTonemapMode = new Version(7, 0, 1); + private readonly Version _minFFmpegAlteredVaVkInterop = new Version(7, 0, 1); + private readonly Version _minFFmpegQsvVppTonemapOption = new Version(7, 0, 1); + private readonly Version _minFFmpegQsvVppOutRangeOption = new Version(7, 0, 1); + private readonly Version _minFFmpegVaapiDeviceVendorId = new Version(7, 0, 1); + private readonly Version _minFFmpegQsvVppScaleModeOption = new Version(6, 0); + private readonly Version _minFFmpegRkmppHevcDecDoviRpu = new Version(7, 1, 1); + private readonly Version _minFFmpegReadrateCatchupOption = new Version(8, 0); + + private static readonly string[] _videoProfilesH264 = + [ "ConstrainedBaseline", "Baseline", "Extended", @@ -63,20 +97,20 @@ namespace MediaBrowser.Controller.MediaEncoding "ProgressiveHigh", "ConstrainedHigh", "High10" - }; + ]; - private static readonly string[] _videoProfilesH265 = new[] - { + private static readonly string[] _videoProfilesH265 = + [ "Main", "Main10" - }; + ]; - private static readonly string[] _videoProfilesAv1 = new[] - { + private static readonly string[] _videoProfilesAv1 = + [ "Main", "High", "Professional", - }; + ]; private static readonly HashSet<string> _mp4ContainerNames = new(StringComparer.OrdinalIgnoreCase) { @@ -88,14 +122,15 @@ namespace MediaBrowser.Controller.MediaEncoding "m4v", }; + private static readonly TonemappingMode[] _legacyTonemapModes = [TonemappingMode.max, TonemappingMode.rgb]; + private static readonly TonemappingMode[] _advancedTonemapModes = [TonemappingMode.lum, TonemappingMode.itp]; + // Set max transcoding channels for encoders that can't handle more than a set amount of channels // AAC, FLAC, ALAC, libopus, libvorbis encoders all support at least 8 channels private static readonly Dictionary<string, int> _audioTranscodeChannelLookup = new(StringComparer.OrdinalIgnoreCase) { - { "wmav2", 2 }, { "libmp3lame", 2 }, { "libfdk_aac", 6 }, - { "aac_at", 6 }, { "ac3", 6 }, { "eac3", 6 }, { "dca", 6 }, @@ -103,37 +138,63 @@ namespace MediaBrowser.Controller.MediaEncoding { "truehd", 6 }, }; - private static readonly string _defaultMjpegEncoder = "mjpeg"; - private static readonly Dictionary<string, string> _mjpegCodecMap = new(StringComparer.OrdinalIgnoreCase) + private static readonly Dictionary<HardwareAccelerationType, string> _mjpegCodecMap = new() { - { "vaapi", _defaultMjpegEncoder + "_vaapi" }, - { "qsv", _defaultMjpegEncoder + "_qsv" } + { HardwareAccelerationType.vaapi, _defaultMjpegEncoder + "_vaapi" }, + { HardwareAccelerationType.qsv, _defaultMjpegEncoder + "_qsv" }, + { HardwareAccelerationType.videotoolbox, _defaultMjpegEncoder + "_videotoolbox" }, + { HardwareAccelerationType.rkmpp, _defaultMjpegEncoder + "_rkmpp" } }; - public static readonly string[] LosslessAudioCodecs = new string[] - { + public static readonly string[] LosslessAudioCodecs = + [ "alac", "ape", "flac", "mlp", "truehd", "wavpack" - }; + ]; public EncodingHelper( IApplicationPaths appPaths, IMediaEncoder mediaEncoder, ISubtitleEncoder subtitleEncoder, IConfiguration config, - IConfigurationManager configurationManager) + IConfigurationManager configurationManager, + IPathManager pathManager) { _appPaths = appPaths; _mediaEncoder = mediaEncoder; _subtitleEncoder = subtitleEncoder; _config = config; _configurationManager = configurationManager; + _pathManager = pathManager; + } + + private enum DynamicHdrMetadataRemovalPlan + { + None, + RemoveDovi, + RemoveHdr10Plus, } + /// <summary> + /// The codec validation regex. + /// This regular expression matches strings that consist of alphanumeric characters, hyphens, + /// periods, underscores, commas, and vertical bars, with a length between 0 and 40 characters. + /// This should matches all common valid codecs. + /// </summary> + [GeneratedRegex(ContainerValidationRegexStr)] + public static partial Regex ContainerValidationRegex(); + + /// <summary> + /// The level validation regex string. + /// This regular expression matches strings representing a double. + /// </summary> + [GeneratedRegex(LevelValidationRegexStr)] + public static partial Regex LevelValidationRegex(); + [GeneratedRegex(@"\s+")] private static partial Regex WhiteSpaceRegex(); @@ -155,18 +216,18 @@ namespace MediaBrowser.Controller.MediaEncoding { var hwType = encodingOptions.HardwareAccelerationType; - var codecMap = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase) + var codecMap = new Dictionary<HardwareAccelerationType, string>() { - { "amf", hwEncoder + "_amf" }, - { "nvenc", hwEncoder + "_nvenc" }, - { "qsv", hwEncoder + "_qsv" }, - { "vaapi", hwEncoder + "_vaapi" }, - { "videotoolbox", hwEncoder + "_videotoolbox" }, - { "v4l2m2m", hwEncoder + "_v4l2m2m" }, - { "rkmpp", hwEncoder + "_rkmpp" }, + { HardwareAccelerationType.amf, hwEncoder + "_amf" }, + { HardwareAccelerationType.nvenc, hwEncoder + "_nvenc" }, + { HardwareAccelerationType.qsv, hwEncoder + "_qsv" }, + { HardwareAccelerationType.vaapi, hwEncoder + "_vaapi" }, + { HardwareAccelerationType.videotoolbox, hwEncoder + "_videotoolbox" }, + { HardwareAccelerationType.v4l2m2m, hwEncoder + "_v4l2m2m" }, + { HardwareAccelerationType.rkmpp, hwEncoder + "_rkmpp" }, }; - if (!string.IsNullOrEmpty(hwType) + if (hwType != HardwareAccelerationType.none && encodingOptions.EnableHardwareEncoding && codecMap.TryGetValue(hwType, out var preferredEncoder) && _mediaEncoder.SupportsEncoder(preferredEncoder)) @@ -184,7 +245,15 @@ namespace MediaBrowser.Controller.MediaEncoding { var hwType = encodingOptions.HardwareAccelerationType; - if (!string.IsNullOrEmpty(hwType) + // Only enable VA-API MJPEG encoder on Intel iHD driver. + // Legacy platforms supported ONLY by i965 do not support MJPEG encoder. + if (hwType == HardwareAccelerationType.vaapi + && !_mediaEncoder.IsVaapiDeviceInteliHD) + { + return _defaultMjpegEncoder; + } + + if (hwType != HardwareAccelerationType.none && encodingOptions.EnableHardwareEncoding && _mjpegCodecMap.TryGetValue(hwType, out var preferredEncoder) && _mediaEncoder.SupportsEncoder(preferredEncoder)) @@ -217,6 +286,7 @@ namespace MediaBrowser.Controller.MediaEncoding && _mediaEncoder.SupportsFilter("tonemap_vaapi") && _mediaEncoder.SupportsFilter("procamp_vaapi") && _mediaEncoder.SupportsFilterWithOption(FilterOptionType.OverlayVaapiFrameSync) + && _mediaEncoder.SupportsFilter("transpose_vaapi") && _mediaEncoder.SupportsFilter("hwupload_vaapi"); } @@ -234,6 +304,8 @@ namespace MediaBrowser.Controller.MediaEncoding && _mediaEncoder.SupportsFilter("scale_opencl") && _mediaEncoder.SupportsFilterWithOption(FilterOptionType.TonemapOpenclBt2390) && _mediaEncoder.SupportsFilterWithOption(FilterOptionType.OverlayOpenclFrameSync); + + // Let transpose_opencl optional for the time being. } private bool IsCudaFullSupported() @@ -244,6 +316,8 @@ namespace MediaBrowser.Controller.MediaEncoding && _mediaEncoder.SupportsFilterWithOption(FilterOptionType.TonemapCudaName) && _mediaEncoder.SupportsFilter("overlay_cuda") && _mediaEncoder.SupportsFilter("hwupload_cuda"); + + // Let transpose_cuda optional for the time being. } private bool IsVulkanFullSupported() @@ -251,7 +325,9 @@ namespace MediaBrowser.Controller.MediaEncoding return _mediaEncoder.SupportsHwaccel("vulkan") && _mediaEncoder.SupportsFilter("libplacebo") && _mediaEncoder.SupportsFilter("scale_vulkan") - && _mediaEncoder.SupportsFilterWithOption(FilterOptionType.OverlayVulkanFrameSync); + && _mediaEncoder.SupportsFilterWithOption(FilterOptionType.OverlayVulkanFrameSync) + && _mediaEncoder.SupportsFilter("transpose_vulkan") + && _mediaEncoder.SupportsFilter("flip_vulkan"); } private bool IsVideoToolboxFullSupported() @@ -261,23 +337,45 @@ namespace MediaBrowser.Controller.MediaEncoding && _mediaEncoder.SupportsFilter("overlay_videotoolbox") && _mediaEncoder.SupportsFilter("tonemap_videotoolbox") && _mediaEncoder.SupportsFilter("scale_vt"); + + // Let transpose_vt optional for the time being. + } + + private bool IsSwTonemapAvailable(EncodingJobInfo state, EncodingOptions options) + { + if (state.VideoStream is null + || GetVideoColorBitDepth(state) < 10 + || !_mediaEncoder.SupportsFilter("tonemapx")) + { + return false; + } + + return state.VideoStream.VideoRange == VideoRange.HDR; } private bool IsHwTonemapAvailable(EncodingJobInfo state, EncodingOptions options) { if (state.VideoStream is null || !options.EnableTonemapping - || GetVideoColorBitDepth(state) != 10) + || GetVideoColorBitDepth(state) < 10) { return false; } - if (string.Equals(state.VideoStream.Codec, "hevc", StringComparison.OrdinalIgnoreCase) - && state.VideoStream.VideoRange == VideoRange.HDR + if (state.VideoStream.VideoRange == VideoRange.HDR && state.VideoStream.VideoRangeType == VideoRangeType.DOVI) { - // Only native SW decoder and HW accelerator can parse dovi rpu. + // Only native SW decoder, HW accelerator and hevc_rkmpp decoder can parse dovi rpu. var vidDecoder = GetHardwareVideoDecoder(state, options) ?? string.Empty; + + var isRkmppDecoder = vidDecoder.Contains("rkmpp", StringComparison.OrdinalIgnoreCase); + if (isRkmppDecoder + && _mediaEncoder.EncoderVersion >= _minFFmpegRkmppHevcDecDoviRpu + && string.Equals(state.VideoStream?.Codec, "hevc", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + var isSwDecoder = string.IsNullOrEmpty(vidDecoder); var isNvdecDecoder = vidDecoder.Contains("cuda", StringComparison.OrdinalIgnoreCase); var isVaapiDecoder = vidDecoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase); @@ -286,11 +384,8 @@ namespace MediaBrowser.Controller.MediaEncoding return isSwDecoder || isNvdecDecoder || isVaapiDecoder || isD3d11vaDecoder || isVideoToolBoxDecoder; } - return state.VideoStream.VideoRange == VideoRange.HDR - && (state.VideoStream.VideoRangeType == VideoRangeType.HDR10 - || state.VideoStream.VideoRangeType == VideoRangeType.HLG - || state.VideoStream.VideoRangeType == VideoRangeType.DOVIWithHDR10 - || state.VideoStream.VideoRangeType == VideoRangeType.DOVIWithHLG); + // GPU tonemapping supports all HDR RangeTypes + return state.VideoStream.VideoRange == VideoRange.HDR; } private bool IsVulkanHwTonemapAvailable(EncodingJobInfo state, EncodingOptions options) @@ -306,27 +401,33 @@ namespace MediaBrowser.Controller.MediaEncoding && GetVideoColorBitDepth(state) == 10; } - private bool IsVaapiVppTonemapAvailable(EncodingJobInfo state, EncodingOptions options) + private bool IsIntelVppTonemapAvailable(EncodingJobInfo state, EncodingOptions options) { if (state.VideoStream is null || !options.EnableVppTonemapping - || GetVideoColorBitDepth(state) != 10) + || GetVideoColorBitDepth(state) < 10) { return false; } - // Native VPP tonemapping may come to QSV in the future. + // prefer 'tonemap_vaapi' over 'vpp_qsv' on Linux for supporting Gen9/KBLx. + // 'vpp_qsv' requires VPL, which is only supported on Gen12/TGLx and newer. + if (OperatingSystem.IsWindows() + && options.HardwareAccelerationType == HardwareAccelerationType.qsv + && _mediaEncoder.EncoderVersion < _minFFmpegQsvVppTonemapOption) + { + return false; + } return state.VideoStream.VideoRange == VideoRange.HDR - && (state.VideoStream.VideoRangeType == VideoRangeType.HDR10 - || state.VideoStream.VideoRangeType == VideoRangeType.DOVIWithHDR10); + && IsDoviWithHdr10Bl(state.VideoStream); } private bool IsVideoToolboxTonemapAvailable(EncodingJobInfo state, EncodingOptions options) { if (state.VideoStream is null || !options.EnableVideoToolboxTonemapping - || GetVideoColorBitDepth(state) != 10) + || GetVideoColorBitDepth(state) < 10) { return false; } @@ -334,7 +435,27 @@ namespace MediaBrowser.Controller.MediaEncoding // Certain DV profile 5 video works in Safari with direct playing, but the VideoToolBox does not produce correct mapping results with transcoding. // All other HDR formats working. return state.VideoStream.VideoRange == VideoRange.HDR - && state.VideoStream.VideoRangeType is VideoRangeType.HDR10 or VideoRangeType.HLG or VideoRangeType.HDR10Plus or VideoRangeType.DOVIWithHDR10 or VideoRangeType.DOVIWithHLG; + && (IsDoviWithHdr10Bl(state.VideoStream) + || state.VideoStream.VideoRangeType is VideoRangeType.HLG); + } + + private bool IsVideoStreamHevcRext(EncodingJobInfo state) + { + var videoStream = state.VideoStream; + if (videoStream is null) + { + return false; + } + + return string.Equals(videoStream.Codec, "hevc", StringComparison.OrdinalIgnoreCase) + && (string.Equals(videoStream.Profile, "Rext", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoStream.PixelFormat, "yuv420p12le", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoStream.PixelFormat, "yuv422p", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoStream.PixelFormat, "yuv422p10le", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoStream.PixelFormat, "yuv422p12le", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoStream.PixelFormat, "yuv444p", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoStream.PixelFormat, "yuv444p10le", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoStream.PixelFormat, "yuv444p12le", StringComparison.OrdinalIgnoreCase)); } /// <summary> @@ -370,28 +491,10 @@ namespace MediaBrowser.Controller.MediaEncoding return GetMjpegEncoder(state, encodingOptions); } - if (string.Equals(codec, "vp8", StringComparison.OrdinalIgnoreCase) - || string.Equals(codec, "vpx", StringComparison.OrdinalIgnoreCase)) - { - return "libvpx"; - } - - if (string.Equals(codec, "vp9", StringComparison.OrdinalIgnoreCase)) - { - return "libvpx-vp9"; - } - - if (string.Equals(codec, "wmv", StringComparison.OrdinalIgnoreCase)) - { - return "wmv2"; - } - - if (string.Equals(codec, "theora", StringComparison.OrdinalIgnoreCase)) + if (ContainerValidationRegex().IsMatch(codec)) { - return "libtheora"; + return codec.ToLowerInvariant(); } - - return codec.ToLowerInvariant(); } return "copy"; @@ -429,7 +532,7 @@ namespace MediaBrowser.Controller.MediaEncoding public static string GetInputFormat(string container) { - if (string.IsNullOrEmpty(container)) + if (string.IsNullOrEmpty(container) || !ContainerValidationRegex().IsMatch(container)) { return null; } @@ -566,49 +669,21 @@ namespace MediaBrowser.Controller.MediaEncoding /// <returns>Codec string.</returns> public string InferAudioCodec(string container) { - var ext = "." + (container ?? string.Empty); - - if (string.Equals(ext, ".mp3", StringComparison.OrdinalIgnoreCase)) - { - return "mp3"; - } - - if (string.Equals(ext, ".aac", StringComparison.OrdinalIgnoreCase)) + if (string.IsNullOrWhiteSpace(container)) { + // this may not work, but if the client is that broken we cannot do anything better return "aac"; } - if (string.Equals(ext, ".wma", StringComparison.OrdinalIgnoreCase)) - { - return "wma"; - } - - if (string.Equals(ext, ".ogg", StringComparison.OrdinalIgnoreCase)) - { - return "vorbis"; - } - - if (string.Equals(ext, ".oga", StringComparison.OrdinalIgnoreCase)) - { - return "vorbis"; - } + var inferredCodec = container.ToLowerInvariant(); - if (string.Equals(ext, ".ogv", StringComparison.OrdinalIgnoreCase)) + return inferredCodec switch { - return "vorbis"; - } - - if (string.Equals(ext, ".webm", StringComparison.OrdinalIgnoreCase)) - { - return "vorbis"; - } - - if (string.Equals(ext, ".webma", StringComparison.OrdinalIgnoreCase)) - { - return "vorbis"; - } - - return "copy"; + "ogg" or "oga" or "ogv" or "webm" or "webma" => "opus", + "m4a" or "m4b" or "mp4" or "mov" or "mkv" or "mka" => "aac", + "ts" or "avi" or "flv" or "f4v" or "swf" => "mp3", + _ => inferredCodec + }; } /// <summary> @@ -666,16 +741,6 @@ namespace MediaBrowser.Controller.MediaEncoding return -1; } - public string GetInputPathArgument(EncodingJobInfo state) - { - return state.MediaSource.VideoType switch - { - VideoType.Dvd => _mediaEncoder.GetInputArgument(_mediaEncoder.GetPrimaryPlaylistVobFiles(state.MediaPath, null).ToList(), state.MediaSource), - VideoType.BluRay => _mediaEncoder.GetInputArgument(_mediaEncoder.GetPrimaryPlaylistM2tsFiles(state.MediaPath).ToList(), state.MediaSource), - _ => _mediaEncoder.GetInputArgument(state.MediaPath, state.MediaSource) - }; - } - /// <summary> /// Gets the audio encoder. /// </summary> @@ -685,6 +750,11 @@ namespace MediaBrowser.Controller.MediaEncoding { var codec = state.OutputAudioCodec; + if (!ContainerValidationRegex().IsMatch(codec)) + { + codec = "aac"; + } + if (string.Equals(codec, "aac", StringComparison.OrdinalIgnoreCase)) { // Use Apple's aac encoder if available as it provides best audio quality @@ -712,11 +782,6 @@ namespace MediaBrowser.Controller.MediaEncoding return "libvorbis"; } - if (string.Equals(codec, "wma", StringComparison.OrdinalIgnoreCase)) - { - return "wmav2"; - } - if (string.Equals(codec, "opus", StringComparison.OrdinalIgnoreCase)) { return "libopus"; @@ -732,6 +797,15 @@ namespace MediaBrowser.Controller.MediaEncoding return "dca"; } + if (string.Equals(codec, "alac", StringComparison.OrdinalIgnoreCase)) + { + // The ffmpeg upstream breaks the AudioToolbox ALAC encoder in version 6.1 but fixes it in version 7.0. + // Since ALAC is lossless in quality and the AudioToolbox encoder is not faster, + // its only benefit is a smaller file size. + // To prevent problems, use the ffmpeg native encoder instead. + return "alac"; + } + return codec.ToLowerInvariant(); } @@ -820,14 +894,16 @@ namespace MediaBrowser.Controller.MediaEncoding options); } - private string GetVaapiDeviceArgs(string renderNodePath, string driver, string kernelDriver, string srcDeviceAlias, string alias) + private string GetVaapiDeviceArgs(string renderNodePath, string driver, string kernelDriver, string vendorId, string srcDeviceAlias, string alias) { alias ??= VaapiAlias; + var haveVendorId = !string.IsNullOrEmpty(vendorId) + && _mediaEncoder.EncoderVersion >= _minFFmpegVaapiDeviceVendorId; - // 'renderNodePath' has higher priority than 'kernelDriver' - var driverOpts = string.IsNullOrEmpty(renderNodePath) - ? (string.IsNullOrEmpty(kernelDriver) ? string.Empty : ",kernel_driver=" + kernelDriver) - : renderNodePath; + // Priority: 'renderNodePath' > 'vendorId' > 'kernelDriver' + var driverOpts = File.Exists(renderNodePath) + ? renderNodePath + : (haveVendorId ? $",vendor_id={vendorId}" : (string.IsNullOrEmpty(kernelDriver) ? string.Empty : $",kernel_driver={kernelDriver}")); // 'driver' behaves similarly to env LIBVA_DRIVER_NAME driverOpts += string.IsNullOrEmpty(driver) ? string.Empty : ",driver=" + driver; @@ -855,17 +931,23 @@ namespace MediaBrowser.Controller.MediaEncoding renderNodePath); } - private string GetQsvDeviceArgs(string alias) + private string GetQsvDeviceArgs(string renderNodePath, string alias) { var arg = " -init_hw_device qsv=" + (alias ?? QsvAlias); if (OperatingSystem.IsLinux()) { // derive qsv from vaapi device - return GetVaapiDeviceArgs(null, "iHD", "i915", null, VaapiAlias) + arg + "@" + VaapiAlias; + return GetVaapiDeviceArgs(renderNodePath, "iHD", "i915", "0x8086", null, VaapiAlias) + arg + "@" + VaapiAlias; } if (OperatingSystem.IsWindows()) { + // on Windows, the deviceIndex is an int + if (int.TryParse(renderNodePath, NumberStyles.Integer, CultureInfo.InvariantCulture, out int deviceIndex)) + { + return GetD3d11vaDeviceArgs(deviceIndex, string.Empty, D3d11vaAlias) + arg + "@" + D3d11vaAlias; + } + // derive qsv from d3d11va device return GetD3d11vaDeviceArgs(0, "0x8086", D3d11vaAlias) + arg + "@" + D3d11vaAlias; } @@ -884,7 +966,7 @@ namespace MediaBrowser.Controller.MediaEncoding { // DVBSUB uses the fixed canvas size 720x576 if (state.SubtitleStream is not null - && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode + && ShouldEncodeSubtitle(state) && !state.SubtitleStream.IsTextSubtitleStream && !string.Equals(state.SubtitleStream.Codec, "DVBSUB", StringComparison.OrdinalIgnoreCase)) { @@ -934,7 +1016,7 @@ namespace MediaBrowser.Controller.MediaEncoding var vidDecoder = GetHardwareVideoDecoder(state, options) ?? string.Empty; var isHwTonemapAvailable = IsHwTonemapAvailable(state, options); - if (string.Equals(optHwaccelType, "vaapi", StringComparison.OrdinalIgnoreCase)) + if (optHwaccelType == HardwareAccelerationType.vaapi) { if (!isLinux || !_mediaEncoder.SupportsHwaccel("vaapi")) { @@ -950,14 +1032,14 @@ namespace MediaBrowser.Controller.MediaEncoding if (_mediaEncoder.IsVaapiDeviceInteliHD) { - args.Append(GetVaapiDeviceArgs(options.VaapiDevice, "iHD", null, null, VaapiAlias)); + args.Append(GetVaapiDeviceArgs(options.VaapiDevice, "iHD", null, null, null, VaapiAlias)); } else if (_mediaEncoder.IsVaapiDeviceInteli965) { // Only override i965 since it has lower priority than iHD in libva lookup. Environment.SetEnvironmentVariable("LIBVA_DRIVER_NAME", "i965"); Environment.SetEnvironmentVariable("LIBVA_DRIVER_NAME_JELLYFIN", "i965"); - args.Append(GetVaapiDeviceArgs(options.VaapiDevice, "i965", null, null, VaapiAlias)); + args.Append(GetVaapiDeviceArgs(options.VaapiDevice, "i965", null, null, null, VaapiAlias)); } var filterDevArgs = string.Empty; @@ -977,10 +1059,11 @@ namespace MediaBrowser.Controller.MediaEncoding Environment.SetEnvironmentVariable("AMD_DEBUG", "noefc"); if (IsVulkanFullSupported() - && _mediaEncoder.IsVaapiDeviceSupportVulkanDrmInterop) + && _mediaEncoder.IsVaapiDeviceSupportVulkanDrmInterop + && Environment.OSVersion.Version >= _minKernelVersionAmdVkFmtModifier) { args.Append(GetDrmDeviceArgs(options.VaapiDevice, DrmAlias)); - args.Append(GetVaapiDeviceArgs(null, null, null, DrmAlias, VaapiAlias)); + args.Append(GetVaapiDeviceArgs(null, null, null, null, DrmAlias, VaapiAlias)); args.Append(GetVulkanDeviceArgs(0, null, DrmAlias, VulkanAlias)); // libplacebo wants an explicitly set vulkan filter device. @@ -988,7 +1071,7 @@ namespace MediaBrowser.Controller.MediaEncoding } else { - args.Append(GetVaapiDeviceArgs(options.VaapiDevice, null, null, null, VaapiAlias)); + args.Append(GetVaapiDeviceArgs(options.VaapiDevice, null, null, null, null, VaapiAlias)); filterDevArgs = GetFilterHwDeviceArgs(VaapiAlias); if (doOclTonemap) @@ -1007,7 +1090,7 @@ namespace MediaBrowser.Controller.MediaEncoding args.Append(filterDevArgs); } - else if (string.Equals(optHwaccelType, "qsv", StringComparison.OrdinalIgnoreCase)) + else if (optHwaccelType == HardwareAccelerationType.qsv) { if ((!isLinux && !isWindows) || !_mediaEncoder.SupportsHwaccel("qsv")) { @@ -1024,7 +1107,7 @@ namespace MediaBrowser.Controller.MediaEncoding return string.Empty; } - args.Append(GetQsvDeviceArgs(QsvAlias)); + args.Append(GetQsvDeviceArgs(options.QsvDevice, QsvAlias)); var filterDevArgs = GetFilterHwDeviceArgs(QsvAlias); // child device used by qsv. if (_mediaEncoder.SupportsHwaccel("vaapi") || _mediaEncoder.SupportsHwaccel("d3d11va")) @@ -1042,7 +1125,7 @@ namespace MediaBrowser.Controller.MediaEncoding args.Append(filterDevArgs); } - else if (string.Equals(optHwaccelType, "nvenc", StringComparison.OrdinalIgnoreCase)) + else if (optHwaccelType == HardwareAccelerationType.nvenc) { if ((!isLinux && !isWindows) || !IsCudaFullSupported()) { @@ -1061,7 +1144,7 @@ namespace MediaBrowser.Controller.MediaEncoding args.Append(GetCudaDeviceArgs(0, CudaAlias)) .Append(GetFilterHwDeviceArgs(CudaAlias)); } - else if (string.Equals(optHwaccelType, "amf", StringComparison.OrdinalIgnoreCase)) + else if (optHwaccelType == HardwareAccelerationType.amf) { if (!isWindows || !_mediaEncoder.SupportsHwaccel("d3d11va")) { @@ -1086,7 +1169,7 @@ namespace MediaBrowser.Controller.MediaEncoding args.Append(filterDevArgs); } - else if (string.Equals(optHwaccelType, "videotoolbox", StringComparison.OrdinalIgnoreCase)) + else if (optHwaccelType == HardwareAccelerationType.videotoolbox) { if (!isMacOS || !_mediaEncoder.SupportsHwaccel("videotoolbox")) { @@ -1103,7 +1186,7 @@ namespace MediaBrowser.Controller.MediaEncoding // videotoolbox hw filter does not require device selection args.Append(GetVideoToolboxDeviceArgs(VideotoolboxAlias)); } - else if (string.Equals(optHwaccelType, "rkmpp", StringComparison.OrdinalIgnoreCase)) + else if (optHwaccelType == HardwareAccelerationType.rkmpp) { if (!isLinux || !_mediaEncoder.SupportsHwaccel("rkmpp")) { @@ -1136,9 +1219,6 @@ namespace MediaBrowser.Controller.MediaEncoding args.Append(vidDecoder); } - // hw transpose filters should be added manually. - args.Append(" -noautorotate"); - return args.ToString().Trim(); } @@ -1167,28 +1247,33 @@ namespace MediaBrowser.Controller.MediaEncoding if (state.MediaSource.VideoType == VideoType.Dvd || state.MediaSource.VideoType == VideoType.BluRay) { - var tmpConcatPath = Path.Join(_configurationManager.GetTranscodePath(), state.MediaSource.Id + ".concat"); - _mediaEncoder.GenerateConcatConfig(state.MediaSource, tmpConcatPath); - arg.Append(" -f concat -safe 0 -i ") - .Append(tmpConcatPath); + var concatFilePath = Path.Join(_configurationManager.CommonApplicationPaths.CachePath, "concat", state.MediaSource.Id + ".concat"); + if (!File.Exists(concatFilePath)) + { + _mediaEncoder.GenerateConcatConfig(state.MediaSource, concatFilePath); + } + + arg.Append(" -f concat -safe 0 -i \"") + .Append(concatFilePath) + .Append("\" "); } else { arg.Append(" -i ") - .Append(GetInputPathArgument(state)); + .Append(_mediaEncoder.GetInputPathArgument(state)); } // sub2video for external graphical subtitles if (state.SubtitleStream is not null - && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode + && ShouldEncodeSubtitle(state) && !state.SubtitleStream.IsTextSubtitleStream && state.SubtitleStream.IsExternal) { var subtitlePath = state.SubtitleStream.Path; var subtitleExtension = Path.GetExtension(subtitlePath.AsSpan()); - if (subtitleExtension.Equals(".sub", StringComparison.OrdinalIgnoreCase) - || subtitleExtension.Equals(".sup", StringComparison.OrdinalIgnoreCase)) + // dvdsub/vobsub graphical subtitles use .sub+.idx pairs + if (subtitleExtension.Equals(".sub", StringComparison.OrdinalIgnoreCase)) { var idxFile = Path.ChangeExtension(subtitlePath, ".idx"); if (File.Exists(idxFile)) @@ -1197,6 +1282,20 @@ namespace MediaBrowser.Controller.MediaEncoding } } + // Use analyzeduration also for subtitle streams to improve resolution detection with streams inside MKS files + var analyzeDurationArgument = GetFfmpegAnalyzeDurationArg(state); + if (!string.IsNullOrEmpty(analyzeDurationArgument)) + { + arg.Append(' ').Append(analyzeDurationArgument); + } + + // Apply probesize, too, if configured + var ffmpegProbeSizeArgument = GetFfmpegProbesizeArg(); + if (!string.IsNullOrEmpty(ffmpegProbeSizeArgument)) + { + arg.Append(' ').Append(ffmpegProbeSizeArgument); + } + // Also seek the external subtitles stream. var seekSubParam = GetFastSeekCommandLineParameter(state, options, segmentContainer); if (!string.IsNullOrEmpty(seekSubParam)) @@ -1243,27 +1342,151 @@ namespace MediaBrowser.Controller.MediaEncoding { var codec = stream.Codec ?? string.Empty; - return codec.IndexOf("264", StringComparison.OrdinalIgnoreCase) != -1 - || codec.IndexOf("avc", StringComparison.OrdinalIgnoreCase) != -1; + return codec.Contains("264", StringComparison.OrdinalIgnoreCase) + || codec.Contains("avc", StringComparison.OrdinalIgnoreCase); } public static bool IsH265(MediaStream stream) { var codec = stream.Codec ?? string.Empty; - return codec.IndexOf("265", StringComparison.OrdinalIgnoreCase) != -1 - || codec.IndexOf("hevc", StringComparison.OrdinalIgnoreCase) != -1; + return codec.Contains("265", StringComparison.OrdinalIgnoreCase) + || codec.Contains("hevc", StringComparison.OrdinalIgnoreCase); + } + + public static bool IsAv1(MediaStream stream) + { + var codec = stream.Codec ?? string.Empty; + + return codec.Contains("av1", StringComparison.OrdinalIgnoreCase); } public static bool IsAAC(MediaStream stream) { var codec = stream.Codec ?? string.Empty; - return codec.IndexOf("aac", StringComparison.OrdinalIgnoreCase) != -1; + return codec.Contains("aac", StringComparison.OrdinalIgnoreCase); + } + + public static bool IsDoviWithHdr10Bl(MediaStream stream) + { + var rangeType = stream?.VideoRangeType; + + return rangeType is VideoRangeType.DOVIWithHDR10 + or VideoRangeType.DOVIWithEL + or VideoRangeType.DOVIWithHDR10Plus + or VideoRangeType.DOVIWithELHDR10Plus + or VideoRangeType.DOVIInvalid; + } + + public static bool IsDovi(MediaStream stream) + { + var rangeType = stream?.VideoRangeType; + + return IsDoviWithHdr10Bl(stream) + || (rangeType is VideoRangeType.DOVI + or VideoRangeType.DOVIWithHLG + or VideoRangeType.DOVIWithSDR); + } + + public static bool IsHdr10Plus(MediaStream stream) + { + var rangeType = stream?.VideoRangeType; + + return rangeType is VideoRangeType.HDR10Plus + or VideoRangeType.DOVIWithHDR10Plus + or VideoRangeType.DOVIWithELHDR10Plus; } - public static string GetBitStreamArgs(MediaStream stream) + /// <summary> + /// Check if dynamic HDR metadata should be removed during stream copy. + /// Please note this check assumes the range check has already been done + /// and trivial fallbacks like HDR10+ to HDR10, DOVIWithHDR10 to HDR10 is already checked. + /// </summary> + private static DynamicHdrMetadataRemovalPlan ShouldRemoveDynamicHdrMetadata(EncodingJobInfo state) { + var videoStream = state.VideoStream; + if (videoStream.VideoRange is not VideoRange.HDR) + { + return DynamicHdrMetadataRemovalPlan.None; + } + + var requestedRangeTypes = state.GetRequestedRangeTypes(state.VideoStream.Codec); + if (requestedRangeTypes.Length == 0) + { + return DynamicHdrMetadataRemovalPlan.None; + } + + var requestHasHDR10 = requestedRangeTypes.Contains(VideoRangeType.HDR10.ToString(), StringComparison.OrdinalIgnoreCase); + var requestHasDOVI = requestedRangeTypes.Contains(VideoRangeType.DOVI.ToString(), StringComparison.OrdinalIgnoreCase); + var requestHasDOVIwithEL = requestedRangeTypes.Contains(VideoRangeType.DOVIWithEL.ToString(), StringComparison.OrdinalIgnoreCase); + var requestHasDOVIwithELHDR10plus = requestedRangeTypes.Contains(VideoRangeType.DOVIWithELHDR10Plus.ToString(), StringComparison.OrdinalIgnoreCase); + + var shouldRemoveHdr10Plus = false; + // Case 1: Client supports HDR10, does not support DOVI with EL but EL presets + var shouldRemoveDovi = (!requestHasDOVIwithEL && requestHasHDR10) && videoStream.VideoRangeType == VideoRangeType.DOVIWithEL; + + // Case 2: Client supports DOVI, does not support broken DOVI config + // Client does not report DOVI support should be allowed to copy bad data for remuxing as HDR10 players would not crash + shouldRemoveDovi = shouldRemoveDovi || (requestHasDOVI && videoStream.VideoRangeType == VideoRangeType.DOVIInvalid); + + // Special case: we have a video with both EL and HDR10+ + // If the client supports EL but not in the case of coexistence with HDR10+, remove HDR10+ for compatibility reasons. + // Otherwise, remove DOVI if the client is not a DOVI player + if (videoStream.VideoRangeType == VideoRangeType.DOVIWithELHDR10Plus) + { + shouldRemoveHdr10Plus = requestHasDOVIwithEL && !requestHasDOVIwithELHDR10plus; + shouldRemoveDovi = shouldRemoveDovi || !shouldRemoveHdr10Plus; + } + + if (shouldRemoveDovi) + { + return DynamicHdrMetadataRemovalPlan.RemoveDovi; + } + + // If the client is a Dolby Vision Player, remove the HDR10+ metadata to avoid playback issues + shouldRemoveHdr10Plus = shouldRemoveHdr10Plus || (requestHasDOVI && videoStream.VideoRangeType == VideoRangeType.DOVIWithHDR10Plus); + return shouldRemoveHdr10Plus ? DynamicHdrMetadataRemovalPlan.RemoveHdr10Plus : DynamicHdrMetadataRemovalPlan.None; + } + + private bool CanEncoderRemoveDynamicHdrMetadata(DynamicHdrMetadataRemovalPlan plan, MediaStream videoStream) + { + return plan switch + { + DynamicHdrMetadataRemovalPlan.RemoveDovi => _mediaEncoder.SupportsBitStreamFilterWithOption(BitStreamFilterOptionType.DoviRpuStrip) + || (IsH265(videoStream) && _mediaEncoder.SupportsBitStreamFilterWithOption(BitStreamFilterOptionType.HevcMetadataRemoveDovi)) + || (IsAv1(videoStream) && _mediaEncoder.SupportsBitStreamFilterWithOption(BitStreamFilterOptionType.Av1MetadataRemoveDovi)), + DynamicHdrMetadataRemovalPlan.RemoveHdr10Plus => (IsH265(videoStream) && _mediaEncoder.SupportsBitStreamFilterWithOption(BitStreamFilterOptionType.HevcMetadataRemoveHdr10Plus)) + || (IsAv1(videoStream) && _mediaEncoder.SupportsBitStreamFilterWithOption(BitStreamFilterOptionType.Av1MetadataRemoveHdr10Plus)), + _ => true, + }; + } + + public bool IsDoviRemoved(EncodingJobInfo state) + { + return state?.VideoStream is not null && ShouldRemoveDynamicHdrMetadata(state) == DynamicHdrMetadataRemovalPlan.RemoveDovi + && CanEncoderRemoveDynamicHdrMetadata(DynamicHdrMetadataRemovalPlan.RemoveDovi, state.VideoStream); + } + + public bool IsHdr10PlusRemoved(EncodingJobInfo state) + { + return state?.VideoStream is not null && ShouldRemoveDynamicHdrMetadata(state) == DynamicHdrMetadataRemovalPlan.RemoveHdr10Plus + && CanEncoderRemoveDynamicHdrMetadata(DynamicHdrMetadataRemovalPlan.RemoveHdr10Plus, state.VideoStream); + } + + public string GetBitStreamArgs(EncodingJobInfo state, MediaStreamType streamType) + { + if (state is null) + { + return null; + } + + var stream = streamType switch + { + MediaStreamType.Audio => state.AudioStream, + MediaStreamType.Video => state.VideoStream, + _ => state.VideoStream + }; // TODO This is auto inserted into the mpegts mux so it might not be needed. // https://www.ffmpeg.org/ffmpeg-bitstream-filters.html#h264_005fmp4toannexb if (IsH264(stream)) @@ -1271,32 +1494,68 @@ namespace MediaBrowser.Controller.MediaEncoding return "-bsf:v h264_mp4toannexb"; } + if (IsAAC(stream)) + { + // Convert adts header(mpegts) to asc header(mp4). + return "-bsf:a aac_adtstoasc"; + } + if (IsH265(stream)) { - return "-bsf:v hevc_mp4toannexb"; + var filter = "-bsf:v hevc_mp4toannexb"; + + // The following checks are not complete because the copy would be rejected + // if the encoder cannot remove required metadata. + // And if bsf is used, we must already be using copy codec. + switch (ShouldRemoveDynamicHdrMetadata(state)) + { + default: + case DynamicHdrMetadataRemovalPlan.None: + break; + case DynamicHdrMetadataRemovalPlan.RemoveDovi: + filter += _mediaEncoder.SupportsBitStreamFilterWithOption(BitStreamFilterOptionType.HevcMetadataRemoveDovi) + ? ",hevc_metadata=remove_dovi=1" + : ",dovi_rpu=strip=1"; + break; + case DynamicHdrMetadataRemovalPlan.RemoveHdr10Plus: + filter += ",hevc_metadata=remove_hdr10plus=1"; + break; + } + + return filter; } - if (IsAAC(stream)) + if (IsAv1(stream)) { - // Convert adts header(mpegts) to asc header(mp4). - return "-bsf:a aac_adtstoasc"; + switch (ShouldRemoveDynamicHdrMetadata(state)) + { + default: + case DynamicHdrMetadataRemovalPlan.None: + return null; + case DynamicHdrMetadataRemovalPlan.RemoveDovi: + return _mediaEncoder.SupportsBitStreamFilterWithOption(BitStreamFilterOptionType.Av1MetadataRemoveDovi) + ? "-bsf:v av1_metadata=remove_dovi=1" + : "-bsf:v dovi_rpu=strip=1"; + case DynamicHdrMetadataRemovalPlan.RemoveHdr10Plus: + return "-bsf:v av1_metadata=remove_hdr10plus=1"; + } } return null; } - public static string GetAudioBitStreamArguments(EncodingJobInfo state, string segmentContainer, string mediaSourceContainer) + public string GetAudioBitStreamArguments(EncodingJobInfo state, string segmentContainer, string mediaSourceContainer) { var bitStreamArgs = string.Empty; var segmentFormat = GetSegmentFileExtension(segmentContainer).TrimStart('.'); // Apply aac_adtstoasc bitstream filter when media source is in mpegts. if (string.Equals(segmentFormat, "mp4", StringComparison.OrdinalIgnoreCase) - && (string.Equals(mediaSourceContainer, "mpegts", StringComparison.OrdinalIgnoreCase) + && (string.Equals(mediaSourceContainer, "ts", StringComparison.OrdinalIgnoreCase) || string.Equals(mediaSourceContainer, "aac", StringComparison.OrdinalIgnoreCase) || string.Equals(mediaSourceContainer, "hls", StringComparison.OrdinalIgnoreCase))) { - bitStreamArgs = GetBitStreamArgs(state.AudioStream); + bitStreamArgs = GetBitStreamArgs(state, MediaStreamType.Audio); bitStreamArgs = string.IsNullOrEmpty(bitStreamArgs) ? string.Empty : " " + bitStreamArgs; } @@ -1313,7 +1572,7 @@ namespace MediaBrowser.Controller.MediaEncoding return ".ts"; } - public string GetVideoBitrateParam(EncodingJobInfo state, string videoCodec) + private string GetVideoBitrateParam(EncodingJobInfo state, string videoCodec) { if (state.OutputVideoBitrate is null) { @@ -1322,28 +1581,15 @@ namespace MediaBrowser.Controller.MediaEncoding int bitrate = state.OutputVideoBitrate.Value; - // Bit rate under 1000k is not allowed in h264_qsv + // Bit rate under 1000k is not allowed in h264_qsv. if (string.Equals(videoCodec, "h264_qsv", StringComparison.OrdinalIgnoreCase)) { bitrate = Math.Max(bitrate, 1000); } - // Currently use the same buffer size for all encoders - int bufsize = bitrate * 2; - - if (string.Equals(videoCodec, "libvpx", StringComparison.OrdinalIgnoreCase) - || string.Equals(videoCodec, "libvpx-vp9", StringComparison.OrdinalIgnoreCase)) - { - // When crf is used with vpx, b:v becomes a max rate - // https://trac.ffmpeg.org/wiki/Encode/VP8 - // https://trac.ffmpeg.org/wiki/Encode/VP9 - return FormattableString.Invariant($" -maxrate:v {bitrate} -bufsize:v {bufsize} -b:v {bitrate}"); - } - - if (string.Equals(videoCodec, "msmpeg4", StringComparison.OrdinalIgnoreCase)) - { - return FormattableString.Invariant($" -b:v {bitrate}"); - } + // Currently use the same buffer size for all non-QSV encoders. + // Use long arithmetic to prevent int32 overflow for very high bitrate values. + int bufsize = (int)Math.Min((long)bitrate * 2, int.MaxValue); if (string.Equals(videoCodec, "libsvtav1", StringComparison.OrdinalIgnoreCase)) { @@ -1356,6 +1602,32 @@ namespace MediaBrowser.Controller.MediaEncoding return FormattableString.Invariant($" -maxrate {bitrate} -bufsize {bufsize}"); } + if (string.Equals(videoCodec, "h264_qsv", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoCodec, "hevc_qsv", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoCodec, "av1_qsv", StringComparison.OrdinalIgnoreCase)) + { + // TODO: probe QSV encoders' capabilities and enable more tuning options + // See also https://github.com/intel/media-delivery/blob/master/doc/quality.rst + + // Enable MacroBlock level bitrate control for better subjective visual quality + var mbbrcOpt = string.Empty; + if (string.Equals(videoCodec, "h264_qsv", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoCodec, "hevc_qsv", StringComparison.OrdinalIgnoreCase)) + { + mbbrcOpt = " -mbbrc 1"; + } + + // Set (maxrate == bitrate + 1) to trigger VBR for better bitrate allocation + // Set (rc_init_occupancy == 2 * bitrate) and (bufsize == 4 * bitrate) to deal with drastic scene changes + // Use long arithmetic and clamp to int.MaxValue to prevent int32 overflow + // (e.g. bitrate * 4 wraps to a negative value for bitrates above ~537 million) + int qsvMaxrate = (int)Math.Min((long)bitrate + 1, int.MaxValue); + int qsvInitOcc = (int)Math.Min((long)bitrate * 2, int.MaxValue); + int qsvBufsize = (int)Math.Min((long)bitrate * 4, int.MaxValue); + + return FormattableString.Invariant($"{mbbrcOpt} -b:v {bitrate} -maxrate {qsvMaxrate} -rc_init_occupancy {qsvInitOcc} -bufsize {qsvBufsize}"); + } + if (string.Equals(videoCodec, "h264_amf", StringComparison.OrdinalIgnoreCase) || string.Equals(videoCodec, "hevc_amf", StringComparison.OrdinalIgnoreCase) || string.Equals(videoCodec, "av1_amf", StringComparison.OrdinalIgnoreCase)) @@ -1382,46 +1654,190 @@ namespace MediaBrowser.Controller.MediaEncoding { // The `maxrate` and `bufsize` options can potentially lead to performance regression // and even encoder hangs, especially when the value is very high. - return FormattableString.Invariant($" -b:v {bitrate}"); + return FormattableString.Invariant($" -b:v {bitrate} -qmin -1 -qmax -1"); } return FormattableString.Invariant($" -b:v {bitrate} -maxrate {bitrate} -bufsize {bufsize}"); } - public static string NormalizeTranscodingLevel(EncodingJobInfo state, string level) + private string GetEncoderParam(EncoderPreset? preset, EncoderPreset defaultPreset, EncodingOptions encodingOptions, string videoEncoder, bool isLibX265) { - if (double.TryParse(level, CultureInfo.InvariantCulture, out double requestLevel)) + var param = string.Empty; + var encoderPreset = preset ?? defaultPreset; + if (string.Equals(videoEncoder, "libx264", StringComparison.OrdinalIgnoreCase) || isLibX265) { - if (string.Equals(state.ActualOutputVideoCodec, "av1", StringComparison.OrdinalIgnoreCase)) + var presetString = encoderPreset switch { - // Transcode to level 5.3 (15) and lower for maximum compatibility. - // https://en.wikipedia.org/wiki/AV1#Levels - if (requestLevel < 0 || requestLevel >= 15) - { - return "15"; - } + EncoderPreset.auto => EncoderPreset.veryfast.ToString().ToLowerInvariant(), + _ => encoderPreset.ToString().ToLowerInvariant() + }; + + param += " -preset " + presetString; + + int encodeCrf = encodingOptions.H264Crf; + if (isLibX265) + { + encodeCrf = encodingOptions.H265Crf; } - else if (string.Equals(state.ActualOutputVideoCodec, "hevc", StringComparison.OrdinalIgnoreCase) - || string.Equals(state.ActualOutputVideoCodec, "h265", StringComparison.OrdinalIgnoreCase)) + + if (encodeCrf >= 0 && encodeCrf <= 51) { - // Transcode to level 5.0 and lower for maximum compatibility. - // Level 5.0 is suitable for up to 4k 30fps hevc encoding, otherwise let the encoder to handle it. - // https://en.wikipedia.org/wiki/High_Efficiency_Video_Coding_tiers_and_levels - // MaxLumaSampleRate = 3840*2160*30 = 248832000 < 267386880. - if (requestLevel < 0 || requestLevel >= 150) + param += " -crf " + encodeCrf.ToString(CultureInfo.InvariantCulture); + } + else + { + string defaultCrf = "23"; + if (isLibX265) { - return "150"; + defaultCrf = "28"; } + + param += " -crf " + defaultCrf; } - else if (string.Equals(state.ActualOutputVideoCodec, "h264", StringComparison.OrdinalIgnoreCase)) + } + else if (string.Equals(videoEncoder, "libsvtav1", StringComparison.OrdinalIgnoreCase)) + { + // Default to use the recommended preset 10. + // Omit presets < 5, which are too slow for on the fly encoding. + // https://gitlab.com/AOMediaCodec/SVT-AV1/-/blob/master/Docs/Ffmpeg.md + param += encoderPreset switch + { + EncoderPreset.veryslow => " -preset 5", + EncoderPreset.slower => " -preset 6", + EncoderPreset.slow => " -preset 7", + EncoderPreset.medium => " -preset 8", + EncoderPreset.fast => " -preset 9", + EncoderPreset.faster => " -preset 10", + EncoderPreset.veryfast => " -preset 11", + EncoderPreset.superfast => " -preset 12", + EncoderPreset.ultrafast => " -preset 13", + _ => " -preset 10" + }; + } + else if (string.Equals(videoEncoder, "h264_vaapi", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoEncoder, "hevc_vaapi", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoEncoder, "av1_vaapi", StringComparison.OrdinalIgnoreCase)) + { + // -compression_level is not reliable on AMD. + if (_mediaEncoder.IsVaapiDeviceInteliHD) { - // Transcode to level 5.1 and lower for maximum compatibility. - // h264 4k 30fps requires at least level 5.1 otherwise it will break on safari fmp4. - // https://en.wikipedia.org/wiki/Advanced_Video_Coding#Levels - if (requestLevel < 0 || requestLevel >= 51) + param += encoderPreset switch { - return "51"; - } + EncoderPreset.veryslow => " -compression_level 1", + EncoderPreset.slower => " -compression_level 2", + EncoderPreset.slow => " -compression_level 3", + EncoderPreset.medium => " -compression_level 4", + EncoderPreset.fast => " -compression_level 5", + EncoderPreset.faster => " -compression_level 6", + EncoderPreset.veryfast => " -compression_level 7", + EncoderPreset.superfast => " -compression_level 7", + EncoderPreset.ultrafast => " -compression_level 7", + _ => string.Empty + }; + } + } + else if (string.Equals(videoEncoder, "h264_qsv", StringComparison.OrdinalIgnoreCase) // h264 (h264_qsv) + || string.Equals(videoEncoder, "hevc_qsv", StringComparison.OrdinalIgnoreCase) // hevc (hevc_qsv) + || string.Equals(videoEncoder, "av1_qsv", StringComparison.OrdinalIgnoreCase)) // av1 (av1_qsv) + { + EncoderPreset[] valid_presets = [EncoderPreset.veryslow, EncoderPreset.slower, EncoderPreset.slow, EncoderPreset.medium, EncoderPreset.fast, EncoderPreset.faster, EncoderPreset.veryfast]; + + param += " -preset " + (valid_presets.Contains(encoderPreset) ? encoderPreset : EncoderPreset.veryfast).ToString().ToLowerInvariant(); + } + else if (string.Equals(videoEncoder, "h264_nvenc", StringComparison.OrdinalIgnoreCase) // h264 (h264_nvenc) + || string.Equals(videoEncoder, "hevc_nvenc", StringComparison.OrdinalIgnoreCase) // hevc (hevc_nvenc) + || string.Equals(videoEncoder, "av1_nvenc", StringComparison.OrdinalIgnoreCase) // av1 (av1_nvenc) + ) + { + param += encoderPreset switch + { + EncoderPreset.veryslow => " -preset p7", + EncoderPreset.slower => " -preset p6", + EncoderPreset.slow => " -preset p5", + EncoderPreset.medium => " -preset p4", + EncoderPreset.fast => " -preset p3", + EncoderPreset.faster => " -preset p2", + _ => " -preset p1" + }; + } + else if (string.Equals(videoEncoder, "h264_amf", StringComparison.OrdinalIgnoreCase) // h264 (h264_amf) + || string.Equals(videoEncoder, "hevc_amf", StringComparison.OrdinalIgnoreCase) // hevc (hevc_amf) + || string.Equals(videoEncoder, "av1_amf", StringComparison.OrdinalIgnoreCase) // av1 (av1_amf) + ) + { + param += encoderPreset switch + { + EncoderPreset.veryslow => " -quality quality", + EncoderPreset.slower => " -quality quality", + EncoderPreset.slow => " -quality quality", + EncoderPreset.medium => " -quality balanced", + _ => " -quality speed" + }; + + if (string.Equals(videoEncoder, "hevc_amf", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoEncoder, "av1_amf", StringComparison.OrdinalIgnoreCase)) + { + param += " -header_insertion_mode gop"; + } + + if (string.Equals(videoEncoder, "hevc_amf", StringComparison.OrdinalIgnoreCase)) + { + param += " -gops_per_idr 1"; + } + } + else if (string.Equals(videoEncoder, "h264_videotoolbox", StringComparison.OrdinalIgnoreCase) // h264 (h264_videotoolbox) + || string.Equals(videoEncoder, "hevc_videotoolbox", StringComparison.OrdinalIgnoreCase) // hevc (hevc_videotoolbox) + ) + { + param += encoderPreset switch + { + EncoderPreset.veryslow => " -prio_speed 0", + EncoderPreset.slower => " -prio_speed 0", + EncoderPreset.slow => " -prio_speed 0", + EncoderPreset.medium => " -prio_speed 0", + _ => " -prio_speed 1" + }; + } + + return param; + } + + public static string NormalizeTranscodingLevel(EncodingJobInfo state, string level) + { + if (!double.TryParse(level, CultureInfo.InvariantCulture, out double requestLevel)) + { + return null; + } + + if (string.Equals(state.ActualOutputVideoCodec, "av1", StringComparison.OrdinalIgnoreCase)) + { + // Transcode to level 5.3 (15) and lower for maximum compatibility. + // https://en.wikipedia.org/wiki/AV1#Levels + if (requestLevel < 0 || requestLevel >= 15) + { + return "15"; + } + } + else if (string.Equals(state.ActualOutputVideoCodec, "hevc", StringComparison.OrdinalIgnoreCase) + || string.Equals(state.ActualOutputVideoCodec, "h265", StringComparison.OrdinalIgnoreCase)) + { + // Transcode to level 5.0 and lower for maximum compatibility. + // Level 5.0 is suitable for up to 4k 30fps hevc encoding, otherwise let the encoder to handle it. + // https://en.wikipedia.org/wiki/High_Efficiency_Video_Coding_tiers_and_levels + // MaxLumaSampleRate = 3840*2160*30 = 248832000 < 267386880. + if (requestLevel < 0 || requestLevel >= 150) + { + return "150"; + } + } + else if (string.Equals(state.ActualOutputVideoCodec, "h264", StringComparison.OrdinalIgnoreCase)) + { + // Transcode to level 5.1 and lower for maximum compatibility. + // h264 4k 30fps requires at least level 5.1 otherwise it will break on safari fmp4. + // https://en.wikipedia.org/wiki/Advanced_Video_Coding#Levels + if (requestLevel < 0 || requestLevel >= 51) + { + return "51"; } } @@ -1447,7 +1863,7 @@ namespace MediaBrowser.Controller.MediaEncoding var alphaParam = enableAlpha ? ":alpha=1" : string.Empty; var sub2videoParam = enableSub2video ? ":sub2video=1" : string.Empty; - var fontPath = Path.Combine(_appPaths.CachePath, "attachments", state.MediaSource.Id); + var fontPath = _pathManager.GetAttachmentFolderPath(state.MediaSource.Id); var fontParam = string.Format( CultureInfo.InvariantCulture, ":fontsdir='{0}'", @@ -1471,7 +1887,6 @@ namespace MediaBrowser.Controller.MediaEncoding } } - // TODO: Perhaps also use original_size=1920x800 ?? return string.Format( CultureInfo.InvariantCulture, "subtitles=f='{0}'{1}{2}{3}{4}{5}", @@ -1483,17 +1898,18 @@ namespace MediaBrowser.Controller.MediaEncoding setPtsParam); } - var mediaPath = state.MediaPath ?? string.Empty; + var subtitlePath = _subtitleEncoder.GetSubtitleFilePath( + state.SubtitleStream, + state.MediaSource, + CancellationToken.None).GetAwaiter().GetResult(); return string.Format( CultureInfo.InvariantCulture, - "subtitles=f='{0}':si={1}{2}{3}{4}{5}", - _mediaEncoder.EscapeSubtitleFilterPath(mediaPath), - state.InternalSubtitleStreamOffset.ToString(CultureInfo.InvariantCulture), + "subtitles=f='{0}'{1}{2}{3}{4}", + _mediaEncoder.EscapeSubtitleFilterPath(subtitlePath), alphaParam, sub2videoParam, fontParam, - // fallbackFontParam, setPtsParam); } @@ -1510,7 +1926,7 @@ namespace MediaBrowser.Controller.MediaEncoding if (maxrate.HasValue && state.VideoStream is not null) { - var contentRate = state.VideoStream.AverageFrameRate ?? state.VideoStream.RealFrameRate; + var contentRate = state.VideoStream.ReferenceFrameRate; if (contentRate.HasValue && contentRate.Value > maxrate.Value) { @@ -1602,7 +2018,7 @@ namespace MediaBrowser.Controller.MediaEncoding /// <param name="encodingOptions">Encoding options.</param> /// <param name="defaultPreset">Default present to use for encoding.</param> /// <returns>Video bitrate.</returns> - public string GetVideoQualityParam(EncodingJobInfo state, string videoEncoder, EncodingOptions encodingOptions, string defaultPreset) + public string GetVideoQualityParam(EncodingJobInfo state, string videoEncoder, EncodingOptions encodingOptions, EncoderPreset defaultPreset) { var param = string.Empty; @@ -1617,7 +2033,9 @@ namespace MediaBrowser.Controller.MediaEncoding // https://github.com/intel/media-driver/issues/1456 var enableWaFori915Hang = false; - if (string.Equals(encodingOptions.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase)) + var hardwareAccelerationType = encodingOptions.HardwareAccelerationType; + + if (hardwareAccelerationType == HardwareAccelerationType.vaapi) { var isIntelVaapiDriver = _mediaEncoder.IsVaapiDeviceInteliHD || _mediaEncoder.IsVaapiDeviceInteli965; @@ -1630,7 +2048,7 @@ namespace MediaBrowser.Controller.MediaEncoding intelLowPowerHwEncoding = encodingOptions.EnableIntelLowPowerHevcHwEncoder && isIntelVaapiDriver; } } - else if (string.Equals(encodingOptions.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase)) + else if (hardwareAccelerationType == HardwareAccelerationType.qsv) { if (OperatingSystem.IsLinux()) { @@ -1646,7 +2064,7 @@ namespace MediaBrowser.Controller.MediaEncoding var doOclTonemap = _mediaEncoder.SupportsHwaccel("qsv") && IsVaapiSupported(state) && IsOpenclFullSupported() - && !IsVaapiVppTonemapAvailable(state, encodingOptions) + && !IsIntelVppTonemapAvailable(state, encodingOptions) && IsHwTonemapAvailable(state, encodingOptions); enableWaFori915Hang = isIntelDecoder && doOclTonemap; @@ -1677,315 +2095,47 @@ namespace MediaBrowser.Controller.MediaEncoding param += " -async_depth 1"; } - var isVc1 = string.Equals(state.VideoStream?.Codec, "vc1", StringComparison.OrdinalIgnoreCase); var isLibX265 = string.Equals(videoEncoder, "libx265", StringComparison.OrdinalIgnoreCase); + var encodingPreset = encodingOptions.EncoderPreset; - if (string.Equals(videoEncoder, "libx264", StringComparison.OrdinalIgnoreCase) || isLibX265) - { - if (!string.IsNullOrEmpty(encodingOptions.EncoderPreset)) - { - param += " -preset " + encodingOptions.EncoderPreset; - } - else - { - param += " -preset " + defaultPreset; - } - - int encodeCrf = encodingOptions.H264Crf; - if (isLibX265) - { - encodeCrf = encodingOptions.H265Crf; - } - - if (encodeCrf >= 0 && encodeCrf <= 51) - { - param += " -crf " + encodeCrf.ToString(CultureInfo.InvariantCulture); - } - else - { - string defaultCrf = "23"; - if (isLibX265) - { - defaultCrf = "28"; - } - - param += " -crf " + defaultCrf; - } - } - else if (string.Equals(videoEncoder, "libsvtav1", StringComparison.OrdinalIgnoreCase)) - { - // Default to use the recommended preset 10. - // Omit presets < 5, which are too slow for on the fly encoding. - // https://gitlab.com/AOMediaCodec/SVT-AV1/-/blob/master/Docs/Ffmpeg.md - param += encodingOptions.EncoderPreset switch - { - "veryslow" => " -preset 5", - "slower" => " -preset 6", - "slow" => " -preset 7", - "medium" => " -preset 8", - "fast" => " -preset 9", - "faster" => " -preset 10", - "veryfast" => " -preset 11", - "superfast" => " -preset 12", - "ultrafast" => " -preset 13", - _ => " -preset 10" - }; - } - else if (string.Equals(videoEncoder, "h264_vaapi", StringComparison.OrdinalIgnoreCase) - || string.Equals(videoEncoder, "hevc_vaapi", StringComparison.OrdinalIgnoreCase) - || string.Equals(videoEncoder, "av1_vaapi", StringComparison.OrdinalIgnoreCase)) - { - // -compression_level is not reliable on AMD. - if (_mediaEncoder.IsVaapiDeviceInteliHD) - { - param += encodingOptions.EncoderPreset switch - { - "veryslow" => " -compression_level 1", - "slower" => " -compression_level 2", - "slow" => " -compression_level 3", - "medium" => " -compression_level 4", - "fast" => " -compression_level 5", - "faster" => " -compression_level 6", - "veryfast" => " -compression_level 7", - "superfast" => " -compression_level 7", - "ultrafast" => " -compression_level 7", - _ => string.Empty - }; - } - } - else if (string.Equals(videoEncoder, "h264_qsv", StringComparison.OrdinalIgnoreCase) // h264 (h264_qsv) - || string.Equals(videoEncoder, "hevc_qsv", StringComparison.OrdinalIgnoreCase) // hevc (hevc_qsv) - || string.Equals(videoEncoder, "av1_qsv", StringComparison.OrdinalIgnoreCase)) // av1 (av1_qsv) - { - string[] valid_presets = { "veryslow", "slower", "slow", "medium", "fast", "faster", "veryfast" }; - - if (valid_presets.Contains(encodingOptions.EncoderPreset, StringComparison.OrdinalIgnoreCase)) - { - param += " -preset " + encodingOptions.EncoderPreset; - } - else - { - param += " -preset veryfast"; - } + param += GetEncoderParam(encodingPreset, defaultPreset, encodingOptions, videoEncoder, isLibX265); + param += GetVideoBitrateParam(state, videoEncoder); - // Only h264_qsv has look_ahead option - if (string.Equals(videoEncoder, "h264_qsv", StringComparison.OrdinalIgnoreCase)) - { - param += " -look_ahead 0"; - } - } - else if (string.Equals(videoEncoder, "h264_nvenc", StringComparison.OrdinalIgnoreCase) // h264 (h264_nvenc) - || string.Equals(videoEncoder, "hevc_nvenc", StringComparison.OrdinalIgnoreCase) // hevc (hevc_nvenc) - || string.Equals(videoEncoder, "av1_nvenc", StringComparison.OrdinalIgnoreCase)) // av1 (av1_nvenc) + var framerate = GetFramerateParam(state); + if (framerate.HasValue) { - switch (encodingOptions.EncoderPreset) - { - case "veryslow": - param += " -preset p7"; - break; - - case "slower": - param += " -preset p6"; - break; - - case "slow": - param += " -preset p5"; - break; - - case "medium": - param += " -preset p4"; - break; - - case "fast": - param += " -preset p3"; - break; - - case "faster": - param += " -preset p2"; - break; - - case "veryfast": - case "superfast": - case "ultrafast": - param += " -preset p1"; - break; - - default: - param += " -preset p1"; - break; - } + param += string.Format(CultureInfo.InvariantCulture, " -r {0}", framerate.Value.ToString(CultureInfo.InvariantCulture)); } - else if (string.Equals(videoEncoder, "h264_amf", StringComparison.OrdinalIgnoreCase) // h264 (h264_amf) - || string.Equals(videoEncoder, "hevc_amf", StringComparison.OrdinalIgnoreCase) // hevc (hevc_amf) - || string.Equals(videoEncoder, "av1_amf", StringComparison.OrdinalIgnoreCase)) // av1 (av1_amf) - { - switch (encodingOptions.EncoderPreset) - { - case "veryslow": - case "slower": - case "slow": - param += " -quality quality"; - break; - - case "medium": - param += " -quality balanced"; - break; - - case "fast": - case "faster": - case "veryfast": - case "superfast": - case "ultrafast": - param += " -quality speed"; - break; - - default: - param += " -quality speed"; - break; - } - if (string.Equals(videoEncoder, "hevc_amf", StringComparison.OrdinalIgnoreCase) - || string.Equals(videoEncoder, "av1_amf", StringComparison.OrdinalIgnoreCase)) - { - param += " -header_insertion_mode gop"; - } - - if (string.Equals(videoEncoder, "hevc_amf", StringComparison.OrdinalIgnoreCase)) - { - param += " -gops_per_idr 1"; - } - } - else if (string.Equals(videoEncoder, "h264_videotoolbox", StringComparison.OrdinalIgnoreCase) // h264 (h264_videotoolbox) - || string.Equals(videoEncoder, "hevc_videotoolbox", StringComparison.OrdinalIgnoreCase)) // hevc (hevc_videotoolbox) + var targetVideoCodec = state.ActualOutputVideoCodec; + if (string.Equals(targetVideoCodec, "h265", StringComparison.OrdinalIgnoreCase) + || string.Equals(targetVideoCodec, "hevc", StringComparison.OrdinalIgnoreCase)) { - switch (encodingOptions.EncoderPreset) - { - case "veryslow": - case "slower": - case "slow": - case "medium": - param += " -prio_speed 0"; - break; - - case "fast": - case "faster": - case "veryfast": - case "superfast": - case "ultrafast": - param += " -prio_speed 1"; - break; - - default: - param += " -prio_speed 1"; - break; - } + targetVideoCodec = "hevc"; } - else if (string.Equals(videoEncoder, "libvpx", StringComparison.OrdinalIgnoreCase)) // vp8 - { - // Values 0-3, 0 being highest quality but slower - var profileScore = 0; - - var qmin = "0"; - var qmax = "50"; - var crf = "10"; - - if (isVc1) - { - profileScore++; - } - // Max of 2 - profileScore = Math.Min(profileScore, 2); - - // http://www.webmproject.org/docs/encoder-parameters/ - param += string.Format( - CultureInfo.InvariantCulture, - " -speed 16 -quality good -profile:v {0} -slices 8 -crf {1} -qmin {2} -qmax {3}", - profileScore.ToString(CultureInfo.InvariantCulture), - crf, - qmin, - qmax); - } - else if (string.Equals(videoEncoder, "libvpx-vp9", StringComparison.OrdinalIgnoreCase)) // vp9 - { - // When `-deadline` is set to `good` or `best`, `-cpu-used` ranges from 0-5. - // When `-deadline` is set to `realtime`, `-cpu-used` ranges from 0-15. - // Resources: - // * https://trac.ffmpeg.org/wiki/Encode/VP9 - // * https://superuser.com/questions/1586934 - // * https://developers.google.com/media/vp9 - param += encodingOptions.EncoderPreset switch - { - "veryslow" => " -deadline best -cpu-used 0", - "slower" => " -deadline best -cpu-used 2", - "slow" => " -deadline best -cpu-used 3", - "medium" => " -deadline good -cpu-used 0", - "fast" => " -deadline good -cpu-used 1", - "faster" => " -deadline good -cpu-used 2", - "veryfast" => " -deadline good -cpu-used 3", - "superfast" => " -deadline good -cpu-used 4", - "ultrafast" => " -deadline good -cpu-used 5", - _ => " -deadline good -cpu-used 1" - }; - - // TODO: until VP9 gets its own CRF setting, base CRF on H.265. - int h265Crf = encodingOptions.H265Crf; - int defaultVp9Crf = 31; - if (h265Crf >= 0 && h265Crf <= 51) - { - // This conversion factor is chosen to match the default CRF for H.265 to the - // recommended 1080p CRF from Google. The factor also maps the logarithmic CRF - // scale of x265 [0, 51] to that of VP9 [0, 63] relatively well. - - // Resources: - // * https://developers.google.com/media/vp9/settings/vod - const float H265ToVp9CrfConversionFactor = 1.12F; - - var vp9Crf = Convert.ToInt32(h265Crf * H265ToVp9CrfConversionFactor); - - // Encoder allows for CRF values in the range [0, 63]. - vp9Crf = Math.Clamp(vp9Crf, 0, 63); - - param += FormattableString.Invariant($" -crf {vp9Crf}"); - } - else - { - param += FormattableString.Invariant($" -crf {defaultVp9Crf}"); - } + var profile = state.GetRequestedProfiles(targetVideoCodec).FirstOrDefault() ?? string.Empty; + profile = WhiteSpaceRegex().Replace(profile, string.Empty).ToLowerInvariant(); - param += " -row-mt 1 -profile 1"; - } - else if (string.Equals(videoEncoder, "mpeg4", StringComparison.OrdinalIgnoreCase)) + var videoProfiles = Array.Empty<string>(); + if (string.Equals("h264", targetVideoCodec, StringComparison.OrdinalIgnoreCase)) { - param += " -mbd rd -flags +mv4+aic -trellis 2 -cmp 2 -subcmp 2 -bf 2"; + videoProfiles = _videoProfilesH264; } - else if (string.Equals(videoEncoder, "wmv2", StringComparison.OrdinalIgnoreCase)) // asf/wmv + else if (string.Equals("hevc", targetVideoCodec, StringComparison.OrdinalIgnoreCase)) { - param += " -qmin 2"; + videoProfiles = _videoProfilesH265; } - else if (string.Equals(videoEncoder, "msmpeg4", StringComparison.OrdinalIgnoreCase)) + else if (string.Equals("av1", targetVideoCodec, StringComparison.OrdinalIgnoreCase)) { - param += " -mbd 2"; + videoProfiles = _videoProfilesAv1; } - param += GetVideoBitrateParam(state, videoEncoder); - - var framerate = GetFramerateParam(state); - if (framerate.HasValue) + if (!videoProfiles.Contains(profile, StringComparison.OrdinalIgnoreCase)) { - param += string.Format(CultureInfo.InvariantCulture, " -r {0}", framerate.Value.ToString(CultureInfo.InvariantCulture)); + profile = string.Empty; } - var targetVideoCodec = state.ActualOutputVideoCodec; - if (string.Equals(targetVideoCodec, "h265", StringComparison.OrdinalIgnoreCase) - || string.Equals(targetVideoCodec, "hevc", StringComparison.OrdinalIgnoreCase)) - { - targetVideoCodec = "hevc"; - } - - var profile = state.GetRequestedProfiles(targetVideoCodec).FirstOrDefault() ?? string.Empty; - profile = WhiteSpaceRegex().Replace(profile, string.Empty); - // We only transcode to HEVC 8-bit for now, force Main Profile. if (profile.Contains("main10", StringComparison.OrdinalIgnoreCase) || profile.Contains("mainstill", StringComparison.OrdinalIgnoreCase)) @@ -2055,6 +2205,18 @@ namespace MediaBrowser.Controller.MediaEncoding profile = "constrained_high"; } + if (string.Equals(videoEncoder, "h264_videotoolbox", StringComparison.OrdinalIgnoreCase) + && profile.Contains("constrainedbaseline", StringComparison.OrdinalIgnoreCase)) + { + profile = "constrained_baseline"; + } + + if (string.Equals(videoEncoder, "h264_videotoolbox", StringComparison.OrdinalIgnoreCase) + && profile.Contains("constrainedhigh", StringComparison.OrdinalIgnoreCase)) + { + profile = "constrained_high"; + } + if (!string.IsNullOrEmpty(profile)) { // Currently there's no profile option in av1_nvenc encoder @@ -2065,12 +2227,10 @@ namespace MediaBrowser.Controller.MediaEncoding } } - var level = state.GetRequestedLevel(targetVideoCodec); + var level = NormalizeTranscodingLevel(state, state.GetRequestedLevel(targetVideoCodec)); if (!string.IsNullOrEmpty(level)) { - level = NormalizeTranscodingLevel(state, level); - // libx264, QSV, AMF can adjust the given level to match the output. if (string.Equals(videoEncoder, "h264_qsv", StringComparison.OrdinalIgnoreCase) || string.Equals(videoEncoder, "libx264", StringComparison.OrdinalIgnoreCase)) @@ -2134,7 +2294,7 @@ namespace MediaBrowser.Controller.MediaEncoding if (string.Equals(videoEncoder, "libx264", StringComparison.OrdinalIgnoreCase)) { - param += " -x264opts:0 subme=0:me_range=4:rc_lookahead=10:me=dia:no_chroma_me:8x8dct=0:partitions=none"; + param += " -x264opts:0 subme=0:me_range=16:rc_lookahead=10:me=hex:open_gop=0"; } if (string.Equals(videoEncoder, "libx265", StringComparison.OrdinalIgnoreCase)) @@ -2142,8 +2302,13 @@ namespace MediaBrowser.Controller.MediaEncoding // libx265 only accept level option in -x265-params. // level option may cause libx265 to fail. // libx265 cannot adjust the given level, just throw an error. - // TODO: set fine tuned params. - param += " -x265-params:0 no-info=1"; + param += " -x265-params:0 no-scenecut=1:no-open-gop=1:no-info=1"; + + if (encodingOptions.EncoderPreset < EncoderPreset.ultrafast) + { + // The following params are slower than the ultrafast preset, don't use when ultrafast is selected. + param += ":subme=3:merange=25:rc-lookahead=10:me=star:ctu=32:max-tu-size=32:min-cu-size=16:rskip=2:rskip-edge-threshold=2:no-sao=1:no-strong-intra-smoothing=1"; + } } if (string.Equals(videoEncoder, "libsvtav1", StringComparison.OrdinalIgnoreCase) @@ -2244,17 +2409,44 @@ namespace MediaBrowser.Controller.MediaEncoding } // DOVIWithHDR10 should be compatible with HDR10 supporting players. Same goes with HLG and of course SDR. So allow copy of those formats - var requestHasHDR10 = requestedRangeTypes.Contains(VideoRangeType.HDR10.ToString(), StringComparison.OrdinalIgnoreCase); var requestHasHLG = requestedRangeTypes.Contains(VideoRangeType.HLG.ToString(), StringComparison.OrdinalIgnoreCase); var requestHasSDR = requestedRangeTypes.Contains(VideoRangeType.SDR.ToString(), StringComparison.OrdinalIgnoreCase); + var requestHasDOVI = requestedRangeTypes.Contains(VideoRangeType.DOVI.ToString(), StringComparison.OrdinalIgnoreCase); + + // If SDR is the only supported range, we should not copy any of the HDR streams. + // All the following copy check assumes at least one HDR format is supported. + if (requestedRangeTypes.Length == 1 && requestHasSDR && videoStream.VideoRangeType != VideoRangeType.SDR) + { + return false; + } + + // If the client does not support DOVI and the video stream is DOVI without fallback, we should not copy it. + if (!requestHasDOVI && videoStream.VideoRangeType == VideoRangeType.DOVI) + { + return false; + } if (!requestedRangeTypes.Contains(videoStream.VideoRangeType.ToString(), StringComparison.OrdinalIgnoreCase) && !((requestHasHDR10 && videoStream.VideoRangeType == VideoRangeType.DOVIWithHDR10) || (requestHasHLG && videoStream.VideoRangeType == VideoRangeType.DOVIWithHLG) - || (requestHasSDR && videoStream.VideoRangeType == VideoRangeType.DOVIWithSDR))) + || (requestHasSDR && videoStream.VideoRangeType == VideoRangeType.DOVIWithSDR) + || (requestHasHDR10 && videoStream.VideoRangeType == VideoRangeType.HDR10Plus))) { - return false; + // If the video stream is in HDR10+ or a static HDR format, don't allow copy if the client does not support HDR10 or HLG. + if (videoStream.VideoRangeType is VideoRangeType.HDR10Plus or VideoRangeType.HDR10 or VideoRangeType.HLG) + { + return false; + } + + // Check complicated cases where we need to remove dynamic metadata + // Conservatively refuse to copy if the encoder can't remove dynamic metadata, + // but a removal is required for compatability reasons. + var dynamicHdrMetadataRemovalPlan = ShouldRemoveDynamicHdrMetadata(state); + if (!CanEncoderRemoveDynamicHdrMetadata(dynamicHdrMetadataRemovalPlan, videoStream)) + { + return false; + } } } @@ -2276,9 +2468,12 @@ namespace MediaBrowser.Controller.MediaEncoding var requestedFramerate = request.MaxFramerate ?? request.Framerate; if (requestedFramerate.HasValue) { - var videoFrameRate = videoStream.AverageFrameRate ?? videoStream.RealFrameRate; + var videoFrameRate = videoStream.ReferenceFrameRate; - if (!videoFrameRate.HasValue || videoFrameRate.Value > requestedFramerate.Value) + // Add a little tolerance to the framerate check because some videos might record a framerate + // that is slightly greater than the intended framerate, but the device can still play it correctly. + // 0.05 fps tolerance should be safe enough. + if (!videoFrameRate.HasValue || videoFrameRate.Value > requestedFramerate.Value + 0.05f) { return false; } @@ -2288,7 +2483,11 @@ namespace MediaBrowser.Controller.MediaEncoding if (request.VideoBitRate.HasValue && (!videoStream.BitRate.HasValue || videoStream.BitRate.Value > request.VideoBitRate.Value)) { - return false; + // For LiveTV that has no bitrate, let's try copy if other conditions are met + if (string.IsNullOrWhiteSpace(request.LiveStreamId) || videoStream.BitRate.HasValue) + { + return false; + } } var maxBitDepth = state.GetRequestedVideoBitDepth(videoStream.Codec); @@ -2429,8 +2628,16 @@ namespace MediaBrowser.Controller.MediaEncoding } } - // Cap the max target bitrate to intMax/2 to satisfy the bufsize=bitrate*2. - return Math.Min(bitrate ?? 0, int.MaxValue / 2); + // Cap the max target bitrate to 400 Mbps. + // No consumer or professional hardware transcode target exceeds this value + // (Intel QSV tops out at ~300 Mbps for H.264; HEVC High Tier Level 5.x is ~240 Mbps). + // Without this cap, plugin-provided MPEG-TS streams with no usable bitrate metadata + // can produce unreasonably large -bufsize/-maxrate values for the encoder. + // Note: the existing FallbackMaxStreamingBitrate mechanism (default 30 Mbps) only + // applies when a LiveStreamId is set (M3U/HDHR sources). Plugin streams and other + // sources that bypass the LiveTV pipeline are not covered by it. + const int MaxSaneBitrate = 400_000_000; // 400 Mbps + return Math.Min(bitrate ?? 0, MaxSaneBitrate); } private int GetMinBitrate(int sourceBitrate, int requestedBitrate) @@ -2469,7 +2676,7 @@ namespace MediaBrowser.Controller.MediaEncoding return 1; } - private static int ScaleBitrate(int bitrate, string inputVideoCodec, string outputVideoCodec) + public static int ScaleBitrate(int bitrate, string inputVideoCodec, string outputVideoCodec) { var inputScaleFactor = GetVideoBitrateScaleFactor(inputVideoCodec); var outputScaleFactor = GetVideoBitrateScaleFactor(outputVideoCodec); @@ -2493,6 +2700,12 @@ namespace MediaBrowser.Controller.MediaEncoding { scaleFactor = Math.Max(scaleFactor, 2); } + else if (bitrate >= 30000000) + { + // Don't scale beyond 30Mbps, it is hardly visually noticeable for most codecs with our prefer speed encoding + // and will cause extremely high bitrate to be used for av1->h264 transcoding that will overload clients and encoders + scaleFactor = 1; + } return Convert.ToInt32(scaleFactor * bitrate); } @@ -2548,8 +2761,9 @@ namespace MediaBrowser.Controller.MediaEncoding return 128000 * (outputAudioChannels ?? audioStream.Channels ?? 2); } - public string GetAudioVbrModeParam(string encoder, int bitratePerChannel) + public string GetAudioVbrModeParam(string encoder, int bitrate, int channels) { + var bitratePerChannel = bitrate / Math.Max(channels, 1); if (string.Equals(encoder, "libfdk_aac", StringComparison.OrdinalIgnoreCase)) { return " -vbr:a " + bitratePerChannel switch @@ -2564,14 +2778,26 @@ namespace MediaBrowser.Controller.MediaEncoding if (string.Equals(encoder, "libmp3lame", StringComparison.OrdinalIgnoreCase)) { - return " -qscale:a " + bitratePerChannel switch + // lame's VBR is only good for a certain bitrate range + // For very low and very high bitrate, use abr mode + if (bitratePerChannel is < 122500 and > 48000) { - < 48000 => "8", - < 64000 => "6", - < 88000 => "4", - < 112000 => "2", - _ => "0" - }; + return " -qscale:a " + bitratePerChannel switch + { + < 64000 => "6", + < 88000 => "4", + < 112000 => "2", + _ => "0" + }; + } + + return " -abr:a 1" + " -b:a " + bitrate; + } + + if (string.Equals(encoder, "aac_at", StringComparison.OrdinalIgnoreCase)) + { + // aac_at's CVBR mode + return " -aac_at_mode:a 2" + " -b:a " + bitrate; } if (string.Equals(encoder, "libvorbis", StringComparison.OrdinalIgnoreCase)) @@ -2595,34 +2821,22 @@ namespace MediaBrowser.Controller.MediaEncoding var filters = new List<string>(); - if (channels.HasValue - && channels.Value == 2 - && state.AudioStream is not null - && state.AudioStream.Channels.HasValue - && state.AudioStream.Channels.Value > 5) + if (channels is 2 && state.AudioStream?.Channels is > 2) { - switch (encodingOptions.DownMixStereoAlgorithm) + var hasDownMixFilter = DownMixAlgorithmsHelper.AlgorithmFilterStrings.TryGetValue((encodingOptions.DownMixStereoAlgorithm, DownMixAlgorithmsHelper.InferChannelLayout(state.AudioStream)), out var downMixFilterString); + if (hasDownMixFilter) { - case DownMixStereoAlgorithms.Dave750: - filters.Add("volume=4.25"); - filters.Add("pan=stereo|c0=0.5*c2+0.707*c0+0.707*c4+0.5*c3|c1=0.5*c2+0.707*c1+0.707*c5+0.5*c3"); - break; - case DownMixStereoAlgorithms.NightmodeDialogue: - filters.Add("pan=stereo|c0=c2+0.30*c0+0.30*c4|c1=c2+0.30*c1+0.30*c5"); - break; - case DownMixStereoAlgorithms.None: - default: - if (!encodingOptions.DownMixAudioBoost.Equals(1)) - { - filters.Add("volume=" + encodingOptions.DownMixAudioBoost.ToString(CultureInfo.InvariantCulture)); - } + filters.Add(downMixFilterString); + } - break; + if (!encodingOptions.DownMixAudioBoost.Equals(1)) + { + filters.Add("volume=" + encodingOptions.DownMixAudioBoost.ToString(CultureInfo.InvariantCulture)); } } var isCopyingTimestamps = state.CopyTimestamps || state.TranscodingType != TranscodingJobType.Progressive; - if (state.SubtitleStream is not null && state.SubtitleStream.IsTextSubtitleStream && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode && !isCopyingTimestamps) + if (state.SubtitleStream is not null && state.SubtitleStream.IsTextSubtitleStream && ShouldEncodeSubtitle(state) && !isCopyingTimestamps) { var seconds = TimeSpan.FromTicks(state.StartTimeTicks ?? 0).TotalSeconds; @@ -2692,7 +2906,20 @@ namespace MediaBrowser.Controller.MediaEncoding if (state.TranscodingType != TranscodingJobType.Progressive && ((resultChannels > 2 && resultChannels < 6) || resultChannels == 7)) { - resultChannels = 2; + // We can let FFMpeg supply an extra LFE channel for 5ch and 7ch to make them 5.1 and 7.1 + if (resultChannels == 5) + { + resultChannels = 6; + } + else if (resultChannels == 7) + { + resultChannels = 8; + } + else + { + // For other weird layout, just downmix to stereo for compatibility + resultChannels = 2; + } } } @@ -2726,25 +2953,39 @@ namespace MediaBrowser.Controller.MediaEncoding public string GetFastSeekCommandLineParameter(EncodingJobInfo state, EncodingOptions options, string segmentContainer) { var time = state.BaseRequest.StartTimeTicks ?? 0; + var maxTime = state.RunTimeTicks ?? 0; var seekParam = string.Empty; if (time > 0) { - seekParam += string.Format(CultureInfo.InvariantCulture, "-ss {0}", _mediaEncoder.GetTimeParameter(time)); + // For direct streaming/remuxing, HLS segments start at keyframes. + // However, ffmpeg will seek to previous keyframe when the exact frame time is the input + // Workaround this by adding 0.5s offset to the seeking time to get the exact keyframe on most videos. + // This will help subtitle syncing. + var isHlsRemuxing = state.IsVideoRequest && state.TranscodingType is TranscodingJobType.Hls && IsCopyCodec(state.OutputVideoCodec); + var seekTick = isHlsRemuxing ? time + 5000000L : time; + + // Seeking beyond EOF makes no sense in transcoding. Clamp the seekTick value to + // [0, RuntimeTicks - 5.0s], so that the muxer gets packets and avoid error codes. + if (maxTime > 0) + { + seekTick = Math.Clamp(seekTick, 0, Math.Max(maxTime - 50000000L, 0)); + } + + seekParam += string.Format(CultureInfo.InvariantCulture, "-ss {0}", _mediaEncoder.GetTimeParameter(seekTick)); if (state.IsVideoRequest) { - var outputVideoCodec = GetVideoEncoder(state, options); - var segmentFormat = GetSegmentFileExtension(segmentContainer).TrimStart('.'); - - // Important: If this is ever re-enabled, make sure not to use it with wtv because it breaks seeking - // Disable -noaccurate_seek on mpegts container due to the timestamps issue on some clients, - // but it's still required for fMP4 container otherwise the audio can't be synced to the video. - if (!string.Equals(state.InputContainer, "wtv", StringComparison.OrdinalIgnoreCase) - && !string.Equals(segmentFormat, "ts", StringComparison.OrdinalIgnoreCase) - && state.TranscodingType != TranscodingJobType.Progressive - && !state.EnableBreakOnNonKeyFrames(outputVideoCodec) - && (state.BaseRequest.StartTimeTicks ?? 0) > 0) + // If we are remuxing, then the copied stream cannot be seeked accurately (it will seek to the nearest + // keyframe). If we are using fMP4, then force all other streams to use the same inaccurate seeking to + // avoid A/V sync issues which cause playback issues on some devices. + // When remuxing video, the segment start times correspond to key frames in the source stream, so this + // option shouldn't change the seeked point that much. + // Important: make sure not to use it with wtv because it breaks seeking + if (state.TranscodingType is TranscodingJobType.Hls + && string.Equals(segmentContainer, "mp4", StringComparison.OrdinalIgnoreCase) + && (IsCopyCodec(state.OutputVideoCodec) || IsCopyCodec(state.OutputAudioCodec)) + && !string.Equals(state.InputContainer, "wtv", StringComparison.OrdinalIgnoreCase)) { seekParam += " -noaccurate_seek"; } @@ -2804,7 +3045,7 @@ namespace MediaBrowser.Controller.MediaEncoding if (state.AudioStream.IsExternal) { bool hasExternalGraphicsSubs = state.SubtitleStream is not null - && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode + && ShouldEncodeSubtitle(state) && state.SubtitleStream.IsExternal && !state.SubtitleStream.IsTextSubtitleStream; int externalAudioMapIndex = hasExternalGraphicsSubs ? 2 : 1; @@ -2943,8 +3184,8 @@ namespace MediaBrowser.Controller.MediaEncoding var scaleW = (double)maximumWidth / outputWidth; var scaleH = (double)maximumHeight / outputHeight; var scale = Math.Min(scaleW, scaleH); - outputWidth = Math.Min(maximumWidth, (int)(outputWidth * scale)); - outputHeight = Math.Min(maximumHeight, (int)(outputHeight * scale)); + outputWidth = Math.Min(maximumWidth, Convert.ToInt32(outputWidth * scale)); + outputHeight = Math.Min(maximumHeight, Convert.ToInt32(outputHeight * scale)); } outputWidth = 2 * (outputWidth / 2); @@ -2996,8 +3237,10 @@ namespace MediaBrowser.Controller.MediaEncoding } public static string GetHwScaleFilter( + string hwScalePrefix, string hwScaleSuffix, string videoFormat, + bool swapOutputWandH, int? videoWidth, int? videoHeight, int? requestedWidth, @@ -3019,8 +3262,11 @@ namespace MediaBrowser.Controller.MediaEncoding || !videoHeight.HasValue || outHeight.Value != videoHeight.Value; - var arg1 = isSizeFixed ? ("=w=" + outWidth.Value + ":h=" + outHeight.Value) : string.Empty; - var arg2 = isFormatFixed ? ("format=" + videoFormat) : string.Empty; + var swpOutW = swapOutputWandH ? outHeight.Value : outWidth.Value; + var swpOutH = swapOutputWandH ? outWidth.Value : outHeight.Value; + + var arg1 = isSizeFixed ? $"=w={swpOutW}:h={swpOutH}" : string.Empty; + var arg2 = isFormatFixed ? $"format={videoFormat}" : string.Empty; if (isFormatFixed) { arg2 = (isSizeFixed ? ':' : '=') + arg2; @@ -3030,7 +3276,8 @@ namespace MediaBrowser.Controller.MediaEncoding { return string.Format( CultureInfo.InvariantCulture, - "scale_{0}{1}{2}", + "{0}_{1}{2}{3}", + hwScalePrefix ?? "scale", hwScaleSuffix, arg1, arg2); @@ -3042,6 +3289,8 @@ namespace MediaBrowser.Controller.MediaEncoding public static string GetGraphicalSubPreProcessFilters( int? videoWidth, int? videoHeight, + int? subtitleWidth, + int? subtitleHeight, int? requestedWidth, int? requestedHeight, int? requestedMaxWidth, @@ -3055,16 +3304,37 @@ namespace MediaBrowser.Controller.MediaEncoding requestedMaxWidth, requestedMaxHeight); - if (outWidth.HasValue && outHeight.HasValue) + if (!outWidth.HasValue + || !outHeight.HasValue + || outWidth.Value <= 0 + || outHeight.Value <= 0) { - return string.Format( - CultureInfo.InvariantCulture, - @"scale=-1:{1}:fast_bilinear,scale,crop,pad=max({0}\,iw):max({1}\,ih):(ow-iw)/2:(oh-ih)/2:black@0,crop={0}:{1}", - outWidth.Value, - outHeight.Value); + return string.Empty; } - return string.Empty; + // Automatically add padding based on subtitle input + var filters = @"scale,scale=-1:{1}:fast_bilinear,crop,pad=max({0}\,iw):max({1}\,ih):(ow-iw)/2:(oh-ih)/2:black@0,crop={0}:{1}"; + + if (subtitleWidth.HasValue + && subtitleHeight.HasValue + && subtitleWidth.Value > 0 + && subtitleHeight.Value > 0) + { + var videoDar = (double)outWidth.Value / outHeight.Value; + var subtitleDar = (double)subtitleWidth.Value / subtitleHeight.Value; + + // No need to add padding when DAR is the same -> 1080p PGSSUB on 2160p video + if (Math.Abs(videoDar - subtitleDar) < 0.01f) + { + filters = @"scale,scale={0}:{1}:fast_bilinear"; + } + } + + return string.Format( + CultureInfo.InvariantCulture, + filters, + outWidth.Value, + outHeight.Value); } public static string GetAlphaSrcFilter( @@ -3114,7 +3384,9 @@ namespace MediaBrowser.Controller.MediaEncoding int? requestedMaxHeight) { var isV4l2 = string.Equals(videoEncoder, "h264_v4l2m2m", StringComparison.OrdinalIgnoreCase); + var isMjpeg = videoEncoder is not null && videoEncoder.Contains("mjpeg", StringComparison.OrdinalIgnoreCase); var scaleVal = isV4l2 ? 64 : 2; + var targetAr = isMjpeg ? "(a*sar)" : "a"; // manually calculate AR when using mjpeg encoder // If fixed dimensions were supplied if (requestedWidth.HasValue && requestedHeight.HasValue) @@ -3143,10 +3415,11 @@ namespace MediaBrowser.Controller.MediaEncoding return string.Format( CultureInfo.InvariantCulture, - @"scale=trunc(min(max(iw\,ih*a)\,min({0}\,{1}*a))/{2})*{2}:trunc(min(max(iw/a\,ih)\,min({0}/a\,{1}))/2)*2", + @"scale=trunc(min(max(iw\,ih*{3})\,min({0}\,{1}*{3}))/{2})*{2}:trunc(min(max(iw/{3}\,ih)\,min({0}/{3}\,{1}))/2)*2", maxWidthParam, maxHeightParam, - scaleVal); + scaleVal, + targetAr); } // If a fixed width was requested @@ -3162,8 +3435,9 @@ namespace MediaBrowser.Controller.MediaEncoding return string.Format( CultureInfo.InvariantCulture, - "scale={0}:trunc(ow/a/2)*2", - widthParam); + "scale={0}:trunc(ow/{1}/2)*2", + widthParam, + targetAr); } // If a fixed height was requested @@ -3173,9 +3447,10 @@ namespace MediaBrowser.Controller.MediaEncoding return string.Format( CultureInfo.InvariantCulture, - "scale=trunc(oh*a/{1})*{1}:{0}", + "scale=trunc(oh*{2}/{1})*{1}:{0}", heightParam, - scaleVal); + scaleVal, + targetAr); } // If a max width was requested @@ -3185,9 +3460,10 @@ namespace MediaBrowser.Controller.MediaEncoding return string.Format( CultureInfo.InvariantCulture, - @"scale=trunc(min(max(iw\,ih*a)\,{0})/{1})*{1}:trunc(ow/a/2)*2", + @"scale=trunc(min(max(iw\,ih*{2})\,{0})/{1})*{1}:trunc(ow/{2}/2)*2", maxWidthParam, - scaleVal); + scaleVal, + targetAr); } // If a max height was requested @@ -3197,9 +3473,10 @@ namespace MediaBrowser.Controller.MediaEncoding return string.Format( CultureInfo.InvariantCulture, - @"scale=trunc(oh*a/{1})*{1}:min(max(iw/a\,ih)\,{0})", + @"scale=trunc(oh*{2}/{1})*{1}:min(max(iw/{2}\,ih)\,{0})", maxHeightParam, - scaleVal); + scaleVal, + targetAr); } return string.Empty; @@ -3255,25 +3532,43 @@ namespace MediaBrowser.Controller.MediaEncoding public static string GetSwDeinterlaceFilter(EncodingJobInfo state, EncodingOptions options) { - var doubleRateDeint = options.DeinterlaceDoubleRate && state.VideoStream?.AverageFrameRate <= 30; + var doubleRateDeint = options.DeinterlaceDoubleRate && state.VideoStream?.ReferenceFrameRate <= 30; return string.Format( CultureInfo.InvariantCulture, "{0}={1}:-1:0", - string.Equals(options.DeinterlaceMethod, "bwdif", StringComparison.OrdinalIgnoreCase) ? "bwdif" : "yadif", + options.DeinterlaceMethod.ToString().ToLowerInvariant(), doubleRateDeint ? "1" : "0"); } - public static string GetHwDeinterlaceFilter(EncodingJobInfo state, EncodingOptions options, string hwDeintSuffix) + public string GetHwDeinterlaceFilter(EncodingJobInfo state, EncodingOptions options, string hwDeintSuffix) { - var doubleRateDeint = options.DeinterlaceDoubleRate && (state.VideoStream?.AverageFrameRate ?? 60) <= 30; + var doubleRateDeint = options.DeinterlaceDoubleRate && (state.VideoStream?.ReferenceFrameRate ?? 60) <= 30; if (hwDeintSuffix.Contains("cuda", StringComparison.OrdinalIgnoreCase)) { + var useBwdif = options.DeinterlaceMethod == DeinterlaceMethod.bwdif && _mediaEncoder.SupportsFilter("bwdif_cuda"); + return string.Format( CultureInfo.InvariantCulture, - "yadif_cuda={0}:-1:0", + "{0}_cuda={1}:-1:0", + useBwdif ? "bwdif" : "yadif", doubleRateDeint ? "1" : "0"); } + if (hwDeintSuffix.Contains("opencl", StringComparison.OrdinalIgnoreCase)) + { + var useBwdif = options.DeinterlaceMethod == DeinterlaceMethod.bwdif; + + if (_mediaEncoder.SupportsFilter("yadif_opencl") + && _mediaEncoder.SupportsFilter("bwdif_opencl")) + { + return string.Format( + CultureInfo.InvariantCulture, + "{0}_opencl={1}:-1:0", + useBwdif ? "bwdif" : "yadif", + doubleRateDeint ? "1" : "0"); + } + } + if (hwDeintSuffix.Contains("vaapi", StringComparison.OrdinalIgnoreCase)) { return string.Format( @@ -3289,16 +3584,19 @@ namespace MediaBrowser.Controller.MediaEncoding if (hwDeintSuffix.Contains("videotoolbox", StringComparison.OrdinalIgnoreCase)) { + var useBwdif = options.DeinterlaceMethod == DeinterlaceMethod.bwdif && _mediaEncoder.SupportsFilter("bwdif_videotoolbox"); + return string.Format( CultureInfo.InvariantCulture, - "yadif_videotoolbox={0}:-1:0", + "{0}_videotoolbox={1}:-1:0", + useBwdif ? "bwdif" : "yadif", doubleRateDeint ? "1" : "0"); } return string.Empty; } - public string GetHwTonemapFilter(EncodingOptions options, string hwTonemapSuffix, string videoFormat) + private string GetHwTonemapFilter(EncodingOptions options, string hwTonemapSuffix, string videoFormat, bool forceFullRange) { if (string.IsNullOrEmpty(hwTonemapSuffix)) { @@ -3306,30 +3604,53 @@ namespace MediaBrowser.Controller.MediaEncoding } var args = string.Empty; - var algorithm = options.TonemappingAlgorithm; + var algorithm = options.TonemappingAlgorithm.ToString().ToLowerInvariant(); + var mode = options.TonemappingMode.ToString().ToLowerInvariant(); + var range = forceFullRange ? TonemappingRange.pc : options.TonemappingRange; + var rangeString = range.ToString().ToLowerInvariant(); if (string.Equals(hwTonemapSuffix, "vaapi", StringComparison.OrdinalIgnoreCase)) { - args = "procamp_vaapi=b={1}:c={2},tonemap_vaapi=format={0}:p=bt709:t=bt709:m=bt709:extra_hw_frames=32"; + var doVaVppProcamp = false; + var procampParams = string.Empty; + if (options.VppTonemappingBrightness != 0 + && options.VppTonemappingBrightness >= -100 + && options.VppTonemappingBrightness <= 100) + { + procampParams += "procamp_vaapi=b={0}"; + doVaVppProcamp = true; + } + + if (options.VppTonemappingContrast > 1 + && options.VppTonemappingContrast <= 10) + { + procampParams += doVaVppProcamp ? ":c={1}" : "procamp_vaapi=c={1}"; + doVaVppProcamp = true; + } + + args = procampParams + "{2}tonemap_vaapi=format={3}:p=bt709:t=bt709:m=bt709:extra_hw_frames=32"; return string.Format( CultureInfo.InvariantCulture, args, - videoFormat ?? "nv12", options.VppTonemappingBrightness, - options.VppTonemappingContrast); + options.VppTonemappingContrast, + doVaVppProcamp ? "," : string.Empty, + videoFormat ?? "nv12"); } else { args = "tonemap_{0}=format={1}:p=bt709:t=bt709:m=bt709:tonemap={2}:peak={3}:desat={4}"; - if (string.Equals(options.TonemappingMode, "max", StringComparison.OrdinalIgnoreCase) - || string.Equals(options.TonemappingMode, "rgb", StringComparison.OrdinalIgnoreCase)) + var useLegacyTonemapModes = _mediaEncoder.EncoderVersion >= _minFFmpegOclCuTonemapMode + && _legacyTonemapModes.Contains(options.TonemappingMode); + + var useAdvancedTonemapModes = _mediaEncoder.EncoderVersion >= _minFFmpegAdvancedTonemapMode + && _advancedTonemapModes.Contains(options.TonemappingMode); + + if (useLegacyTonemapModes || useAdvancedTonemapModes) { - if (_mediaEncoder.EncoderVersion >= _minFFmpegOclCuTonemapMode) - { - args += ":tonemap_mode={5}"; - } + args += ":tonemap_mode={5}"; } if (options.TonemappingParam != 0) @@ -3337,8 +3658,7 @@ namespace MediaBrowser.Controller.MediaEncoding args += ":param={6}"; } - if (string.Equals(options.TonemappingRange, "tv", StringComparison.OrdinalIgnoreCase) - || string.Equals(options.TonemappingRange, "pc", StringComparison.OrdinalIgnoreCase)) + if (range == TonemappingRange.tv || range == TonemappingRange.pc) { args += ":range={7}"; } @@ -3352,12 +3672,12 @@ namespace MediaBrowser.Controller.MediaEncoding algorithm, options.TonemappingPeak, options.TonemappingDesat, - options.TonemappingMode, + mode, options.TonemappingParam, - options.TonemappingRange); + rangeString); } - public string GetLibplaceboFilter( + private string GetLibplaceboFilter( EncodingOptions options, string videoFormat, bool doTonemap, @@ -3366,7 +3686,8 @@ namespace MediaBrowser.Controller.MediaEncoding int? requestedWidth, int? requestedHeight, int? requestedMaxWidth, - int? requestedMaxHeight) + int? requestedMaxHeight, + bool forceFullRange) { var (outWidth, outHeight) = GetFixedOutputSize( videoWidth, @@ -3389,32 +3710,24 @@ namespace MediaBrowser.Controller.MediaEncoding if (doTonemap) { var algorithm = options.TonemappingAlgorithm; + var algorithmString = "clip"; var mode = options.TonemappingMode; - var range = options.TonemappingRange; + var range = forceFullRange ? TonemappingRange.pc : options.TonemappingRange; - if (string.Equals(algorithm, "bt2390", StringComparison.OrdinalIgnoreCase)) - { - algorithm = "bt.2390"; - } - else if (string.Equals(algorithm, "none", StringComparison.OrdinalIgnoreCase)) + if (algorithm == TonemappingAlgorithm.bt2390) { - algorithm = "clip"; + algorithmString = "bt.2390"; } - - tonemapArg = ":tonemapping=" + algorithm; - - if (string.Equals(mode, "max", StringComparison.OrdinalIgnoreCase) - || string.Equals(mode, "rgb", StringComparison.OrdinalIgnoreCase)) + else if (algorithm != TonemappingAlgorithm.none) { - tonemapArg += ":tonemapping_mode=" + mode; + algorithmString = algorithm.ToString().ToLowerInvariant(); } - tonemapArg += ":peak_detect=0:color_primaries=bt709:color_trc=bt709:colorspace=bt709"; + tonemapArg = $":tonemapping={algorithmString}:peak_detect=0:color_primaries=bt709:color_trc=bt709:colorspace=bt709"; - if (string.Equals(range, "tv", StringComparison.OrdinalIgnoreCase) - || string.Equals(range, "pc", StringComparison.OrdinalIgnoreCase)) + if (range == TonemappingRange.tv || range == TonemappingRange.pc) { - tonemapArg += ":range=" + range; + tonemapArg += ":range=" + range.ToString().ToLowerInvariant(); } } @@ -3426,6 +3739,18 @@ namespace MediaBrowser.Controller.MediaEncoding tonemapArg); } + public string GetVideoTransposeDirection(EncodingJobInfo state) + { + return (state.VideoStream?.Rotation ?? 0) switch + { + 90 => "cclock", + 180 => "reversal", + -90 => "clock", + -180 => "reversal", + _ => string.Empty + }; + } + /// <summary> /// Gets the parameter of software filter chain. /// </summary> @@ -3454,15 +3779,22 @@ namespace MediaBrowser.Controller.MediaEncoding var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true); var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true); var doDeintH2645 = doDeintH264 || doDeintHevc; + var doToneMap = IsSwTonemapAvailable(state, options); + var requireDoviReshaping = doToneMap && state.VideoStream.VideoRangeType == VideoRangeType.DOVI; - var hasSubs = state.SubtitleStream is not null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; + var hasSubs = state.SubtitleStream is not null && ShouldEncodeSubtitle(state); var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream; var hasGraphicalSubs = hasSubs && !state.SubtitleStream.IsTextSubtitleStream; + var rotation = state.VideoStream?.Rotation ?? 0; + var swapWAndH = Math.Abs(rotation) == 90; + var swpInW = swapWAndH ? inH : inW; + var swpInH = swapWAndH ? inW : inH; + /* Make main filters for video stream */ var mainFilters = new List<string>(); - mainFilters.Add(GetOverwriteColorPropertiesParam(state, false)); + mainFilters.Add(GetOverwriteColorPropertiesParam(state, doToneMap)); // INPUT sw surface(memory/copy-back from vram) // sw deint @@ -3473,7 +3805,7 @@ namespace MediaBrowser.Controller.MediaEncoding } var outFormat = isSwDecoder ? "yuv420p" : "nv12"; - var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH); + var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, swpInW, swpInH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH); if (isVaapiEncoder) { outFormat = "nv12"; @@ -3485,11 +3817,42 @@ namespace MediaBrowser.Controller.MediaEncoding // sw scale mainFilters.Add(swScaleFilter); - mainFilters.Add("format=" + outFormat); - // sw tonemap <= TODO: finsh the fast tonemap filter + // sw tonemap + if (doToneMap) + { + // tonemapx requires yuv420p10 input for dovi reshaping, let ffmpeg convert the frame when necessary + var tonemapFormat = requireDoviReshaping ? "yuv420p" : outFormat; + var tonemapArgString = "tonemapx=tonemap={0}:desat={1}:peak={2}:t=bt709:m=bt709:p=bt709:format={3}"; - // OUTPUT yuv420p/nv12 surface(memory) + if (options.TonemappingParam != 0) + { + tonemapArgString += ":param={4}"; + } + + var range = options.TonemappingRange; + if (range == TonemappingRange.tv || range == TonemappingRange.pc) + { + tonemapArgString += ":range={5}"; + } + + var tonemapArgs = string.Format( + CultureInfo.InvariantCulture, + tonemapArgString, + options.TonemappingAlgorithm, + options.TonemappingDesat, + options.TonemappingPeak, + tonemapFormat, + options.TonemappingParam, + options.TonemappingRange); + + mainFilters.Add(tonemapArgs); + } + else + { + // OUTPUT yuv420p/nv12 surface(memory) + mainFilters.Add("format=" + outFormat); + } /* Make sub and overlay filters for subtitle stream */ var subFilters = new List<string>(); @@ -3502,7 +3865,9 @@ namespace MediaBrowser.Controller.MediaEncoding } else if (hasGraphicalSubs) { - var subPreProcFilters = GetGraphicalSubPreProcessFilters(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + var subW = state.SubtitleStream?.Width; + var subH = state.SubtitleStream?.Height; + var subPreProcFilters = GetGraphicalSubPreProcessFilters(swpInW, swpInH, subW, subH, reqW, reqH, reqMaxW, reqMaxH); subFilters.Add(subPreProcFilters); overlayFilters.Add("overlay=eof_action=pass:repeatlast=0"); } @@ -3522,7 +3887,7 @@ namespace MediaBrowser.Controller.MediaEncoding EncodingOptions options, string vidEncoder) { - if (!string.Equals(options.HardwareAccelerationType, "nvenc", StringComparison.OrdinalIgnoreCase)) + if (options.HardwareAccelerationType != HardwareAccelerationType.nvenc) { return (null, null, null); } @@ -3539,7 +3904,7 @@ namespace MediaBrowser.Controller.MediaEncoding return GetSwVidFilterChain(state, options, vidEncoder); } - // prefered nvdec/cuvid + cuda filters + nvenc pipeline + // preferred nvdec/cuvid + cuda filters + nvenc pipeline return GetNvidiaVidFiltersPrefered(state, options, vidDecoder, vidEncoder); } @@ -3561,20 +3926,30 @@ namespace MediaBrowser.Controller.MediaEncoding var isNvencEncoder = vidEncoder.Contains("nvenc", StringComparison.OrdinalIgnoreCase); var isSwDecoder = string.IsNullOrEmpty(vidDecoder); var isSwEncoder = !isNvencEncoder; + var isMjpegEncoder = vidEncoder.Contains("mjpeg", StringComparison.OrdinalIgnoreCase); var isCuInCuOut = isNvDecoder && isNvencEncoder; - var doubleRateDeint = options.DeinterlaceDoubleRate && (state.VideoStream?.AverageFrameRate ?? 60) <= 30; + var doubleRateDeint = options.DeinterlaceDoubleRate && (state.VideoStream?.ReferenceFrameRate ?? 60) <= 30; var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true); var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true); var doDeintH2645 = doDeintH264 || doDeintHevc; var doCuTonemap = IsHwTonemapAvailable(state, options); - var hasSubs = state.SubtitleStream is not null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; + var hasSubs = state.SubtitleStream is not null && ShouldEncodeSubtitle(state); var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream; var hasGraphicalSubs = hasSubs && !state.SubtitleStream.IsTextSubtitleStream; var hasAssSubs = hasSubs && (string.Equals(state.SubtitleStream.Codec, "ass", StringComparison.OrdinalIgnoreCase) || string.Equals(state.SubtitleStream.Codec, "ssa", StringComparison.OrdinalIgnoreCase)); + var subW = state.SubtitleStream?.Width; + var subH = state.SubtitleStream?.Height; + + var rotation = state.VideoStream?.Rotation ?? 0; + var transposeDir = rotation == 0 ? string.Empty : GetVideoTransposeDirection(state); + var doCuTranspose = !string.IsNullOrEmpty(transposeDir) && _mediaEncoder.SupportsFilter("transpose_cuda"); + var swapWAndH = Math.Abs(rotation) == 90 && (isSwDecoder || (isNvDecoder && doCuTranspose)); + var swpInW = swapWAndH ? inH : inW; + var swpInH = swapWAndH ? inW : inH; /* Make main filters for video stream */ var mainFilters = new List<string>(); @@ -3592,10 +3967,10 @@ namespace MediaBrowser.Controller.MediaEncoding } var outFormat = doCuTonemap ? "yuv420p10le" : "yuv420p"; - var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH); + var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, swpInW, swpInH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH); // sw scale mainFilters.Add(swScaleFilter); - mainFilters.Add("format=" + outFormat); + mainFilters.Add($"format={outFormat}"); // sw => hw if (doCuTonemap) @@ -3614,8 +3989,15 @@ namespace MediaBrowser.Controller.MediaEncoding mainFilters.Add(deintFilter); } - var outFormat = doCuTonemap ? string.Empty : "yuv420p"; - var hwScaleFilter = GetHwScaleFilter("cuda", outFormat, inW, inH, reqW, reqH, reqMaxW, reqMaxH); + // hw transpose + if (doCuTranspose) + { + mainFilters.Add($"transpose_cuda=dir={transposeDir}"); + } + + var isRext = IsVideoStreamHevcRext(state); + var outFormat = doCuTonemap ? (isRext ? "p010" : string.Empty) : "yuv420p"; + var hwScaleFilter = GetHwScaleFilter("scale", "cuda", outFormat, false, swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH); // hw scale mainFilters.Add(hwScaleFilter); } @@ -3623,7 +4005,7 @@ namespace MediaBrowser.Controller.MediaEncoding // hw tonemap if (doCuTonemap) { - var tonemapFilter = GetHwTonemapFilter(options, "cuda", "yuv420p"); + var tonemapFilter = GetHwTonemapFilter(options, "cuda", "yuv420p", isMjpegEncoder); mainFilters.Add(tonemapFilter); } @@ -3663,9 +4045,10 @@ namespace MediaBrowser.Controller.MediaEncoding { if (hasSubs) { + var alphaFormatOpt = string.Empty; if (hasGraphicalSubs) { - var subPreProcFilters = GetGraphicalSubPreProcessFilters(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + var subPreProcFilters = GetGraphicalSubPreProcessFilters(swpInW, swpInH, subW, subH, reqW, reqH, reqMaxW, reqMaxH); subFilters.Add(subPreProcFilters); subFilters.Add("format=yuva420p"); } @@ -3675,22 +4058,25 @@ namespace MediaBrowser.Controller.MediaEncoding var subFramerate = hasAssSubs ? Math.Min(framerate ?? 25, 60) : 10; // alphasrc=s=1280x720:r=10:start=0,format=yuva420p,subtitles,hwupload - var alphaSrcFilter = GetAlphaSrcFilter(state, inW, inH, reqW, reqH, reqMaxW, reqMaxH, subFramerate); + var alphaSrcFilter = GetAlphaSrcFilter(state, swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH, subFramerate); var subTextSubtitlesFilter = GetTextSubtitlesFilter(state, true, true); subFilters.Add(alphaSrcFilter); subFilters.Add("format=yuva420p"); subFilters.Add(subTextSubtitlesFilter); + + alphaFormatOpt = _mediaEncoder.SupportsFilterWithOption(FilterOptionType.OverlayCudaAlphaFormat) + ? ":alpha_format=premultiplied" : string.Empty; } subFilters.Add("hwupload=derive_device=cuda"); - overlayFilters.Add("overlay_cuda=eof_action=pass:repeatlast=0"); + overlayFilters.Add($"overlay_cuda=eof_action=pass:repeatlast=0{alphaFormatOpt}"); } } else { if (hasGraphicalSubs) { - var subPreProcFilters = GetGraphicalSubPreProcessFilters(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + var subPreProcFilters = GetGraphicalSubPreProcessFilters(swpInW, swpInH, subW, subH, reqW, reqH, reqMaxW, reqMaxH); subFilters.Add(subPreProcFilters); overlayFilters.Add("overlay=eof_action=pass:repeatlast=0"); } @@ -3711,7 +4097,7 @@ namespace MediaBrowser.Controller.MediaEncoding EncodingOptions options, string vidEncoder) { - if (!string.Equals(options.HardwareAccelerationType, "amf", StringComparison.OrdinalIgnoreCase)) + if (options.HardwareAccelerationType != HardwareAccelerationType.amf) { return (null, null, null); } @@ -3730,7 +4116,7 @@ namespace MediaBrowser.Controller.MediaEncoding return GetSwVidFilterChain(state, options, vidEncoder); } - // prefered d3d11va + opencl filters + amf pipeline + // preferred d3d11va + opencl filters + amf pipeline return GetAmdDx11VidFiltersPrefered(state, options, vidDecoder, vidEncoder); } @@ -3752,6 +4138,7 @@ namespace MediaBrowser.Controller.MediaEncoding var isAmfEncoder = vidEncoder.Contains("amf", StringComparison.OrdinalIgnoreCase); var isSwDecoder = string.IsNullOrEmpty(vidDecoder); var isSwEncoder = !isAmfEncoder; + var isMjpegEncoder = vidEncoder.Contains("mjpeg", StringComparison.OrdinalIgnoreCase); var isDxInDxOut = isD3d11vaDecoder && isAmfEncoder; var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true); @@ -3759,12 +4146,22 @@ namespace MediaBrowser.Controller.MediaEncoding var doDeintH2645 = doDeintH264 || doDeintHevc; var doOclTonemap = IsHwTonemapAvailable(state, options); - var hasSubs = state.SubtitleStream is not null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; + var hasSubs = state.SubtitleStream is not null && ShouldEncodeSubtitle(state); var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream; var hasGraphicalSubs = hasSubs && !state.SubtitleStream.IsTextSubtitleStream; var hasAssSubs = hasSubs && (string.Equals(state.SubtitleStream.Codec, "ass", StringComparison.OrdinalIgnoreCase) || string.Equals(state.SubtitleStream.Codec, "ssa", StringComparison.OrdinalIgnoreCase)); + var subW = state.SubtitleStream?.Width; + var subH = state.SubtitleStream?.Height; + + var rotation = state.VideoStream?.Rotation ?? 0; + var transposeDir = rotation == 0 ? string.Empty : GetVideoTransposeDirection(state); + var doOclTranspose = !string.IsNullOrEmpty(transposeDir) + && _mediaEncoder.SupportsFilterWithOption(FilterOptionType.TransposeOpenclReversal); + var swapWAndH = Math.Abs(rotation) == 90 && (isSwDecoder || (isD3d11vaDecoder && doOclTranspose)); + var swpInW = swapWAndH ? inH : inW; + var swpInH = swapWAndH ? inW : inH; /* Make main filters for video stream */ var mainFilters = new List<string>(); @@ -3782,19 +4179,19 @@ namespace MediaBrowser.Controller.MediaEncoding } var outFormat = doOclTonemap ? "yuv420p10le" : "yuv420p"; - var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH); + var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, swpInW, swpInH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH); // sw scale mainFilters.Add(swScaleFilter); - mainFilters.Add("format=" + outFormat); + mainFilters.Add($"format={outFormat}"); // keep video at memory except ocl tonemap, // since the overhead caused by hwupload >>> using sw filter. // sw => hw if (doOclTonemap) { - mainFilters.Add("hwupload=derive_device=d3d11va:extra_hw_frames=16"); + mainFilters.Add("hwupload=derive_device=d3d11va:extra_hw_frames=24"); mainFilters.Add("format=d3d11"); - mainFilters.Add("hwmap=derive_device=opencl"); + mainFilters.Add("hwmap=derive_device=opencl:mode=read"); } } @@ -3802,12 +4199,23 @@ namespace MediaBrowser.Controller.MediaEncoding { // INPUT d3d11 surface(vram) // map from d3d11va to opencl via d3d11-opencl interop. - mainFilters.Add("hwmap=derive_device=opencl"); + mainFilters.Add("hwmap=derive_device=opencl:mode=read"); + + // hw deint + if (doDeintH2645) + { + var deintFilter = GetHwDeinterlaceFilter(state, options, "opencl"); + mainFilters.Add(deintFilter); + } - // hw deint <= TODO: finsh the 'yadif_opencl' filter + // hw transpose + if (doOclTranspose) + { + mainFilters.Add($"transpose_opencl=dir={transposeDir}"); + } var outFormat = doOclTonemap ? string.Empty : "nv12"; - var hwScaleFilter = GetHwScaleFilter("opencl", outFormat, inW, inH, reqW, reqH, reqMaxW, reqMaxH); + var hwScaleFilter = GetHwScaleFilter("scale", "opencl", outFormat, false, swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH); // hw scale mainFilters.Add(hwScaleFilter); } @@ -3815,7 +4223,7 @@ namespace MediaBrowser.Controller.MediaEncoding // hw tonemap if (doOclTonemap) { - var tonemapFilter = GetHwTonemapFilter(options, "opencl", "nv12"); + var tonemapFilter = GetHwTonemapFilter(options, "opencl", "nv12", isMjpegEncoder); mainFilters.Add(tonemapFilter); } @@ -3852,7 +4260,7 @@ namespace MediaBrowser.Controller.MediaEncoding { // OUTPUT d3d11(nv12) surface(vram) // reverse-mapping via d3d11-opencl interop. - mainFilters.Add("hwmap=derive_device=d3d11va:reverse=1"); + mainFilters.Add("hwmap=derive_device=d3d11va:mode=write:reverse=1"); mainFilters.Add("format=d3d11"); } @@ -3863,9 +4271,10 @@ namespace MediaBrowser.Controller.MediaEncoding { if (hasSubs) { + var alphaFormatOpt = string.Empty; if (hasGraphicalSubs) { - var subPreProcFilters = GetGraphicalSubPreProcessFilters(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + var subPreProcFilters = GetGraphicalSubPreProcessFilters(swpInW, swpInH, subW, subH, reqW, reqH, reqMaxW, reqMaxH); subFilters.Add(subPreProcFilters); subFilters.Add("format=yuva420p"); } @@ -3875,16 +4284,19 @@ namespace MediaBrowser.Controller.MediaEncoding var subFramerate = hasAssSubs ? Math.Min(framerate ?? 25, 60) : 10; // alphasrc=s=1280x720:r=10:start=0,format=yuva420p,subtitles,hwupload - var alphaSrcFilter = GetAlphaSrcFilter(state, inW, inH, reqW, reqH, reqMaxW, reqMaxH, subFramerate); + var alphaSrcFilter = GetAlphaSrcFilter(state, swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH, subFramerate); var subTextSubtitlesFilter = GetTextSubtitlesFilter(state, true, true); subFilters.Add(alphaSrcFilter); subFilters.Add("format=yuva420p"); subFilters.Add(subTextSubtitlesFilter); + + alphaFormatOpt = _mediaEncoder.SupportsFilterWithOption(FilterOptionType.OverlayOpenclAlphaFormat) + ? ":alpha_format=premultiplied" : string.Empty; } subFilters.Add("hwupload=derive_device=opencl"); - overlayFilters.Add("overlay_opencl=eof_action=pass:repeatlast=0"); - overlayFilters.Add("hwmap=derive_device=d3d11va:reverse=1"); + overlayFilters.Add($"overlay_opencl=eof_action=pass:repeatlast=0{alphaFormatOpt}"); + overlayFilters.Add("hwmap=derive_device=d3d11va:mode=write:reverse=1"); overlayFilters.Add("format=d3d11"); } } @@ -3892,7 +4304,7 @@ namespace MediaBrowser.Controller.MediaEncoding { if (hasGraphicalSubs) { - var subPreProcFilters = GetGraphicalSubPreProcessFilters(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + var subPreProcFilters = GetGraphicalSubPreProcessFilters(swpInW, swpInH, subW, subH, reqW, reqH, reqMaxW, reqMaxH); subFilters.Add(subPreProcFilters); overlayFilters.Add("overlay=eof_action=pass:repeatlast=0"); } @@ -3913,7 +4325,7 @@ namespace MediaBrowser.Controller.MediaEncoding EncodingOptions options, string vidEncoder) { - if (!string.Equals(options.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase)) + if (options.HardwareAccelerationType != HardwareAccelerationType.qsv) { return (null, null, null); } @@ -3939,13 +4351,13 @@ namespace MediaBrowser.Controller.MediaEncoding return GetSwVidFilterChain(state, options, vidEncoder); } - // prefered qsv(vaapi) + opencl filters pipeline + // preferred qsv(vaapi) + opencl filters pipeline if (isIntelVaapiOclSupported) { return GetIntelQsvVaapiVidFiltersPrefered(state, options, vidDecoder, vidEncoder); } - // prefered qsv(d3d11) + opencl filters pipeline + // preferred qsv(d3d11) + opencl filters pipeline if (isIntelDx11OclSupported) { return GetIntelQsvDx11VidFiltersPrefered(state, options, vidDecoder, vidEncoder); @@ -3974,24 +4386,36 @@ namespace MediaBrowser.Controller.MediaEncoding var isHwDecoder = isD3d11vaDecoder || isQsvDecoder; var isSwDecoder = string.IsNullOrEmpty(vidDecoder); var isSwEncoder = !isQsvEncoder; + var isMjpegEncoder = vidEncoder.Contains("mjpeg", StringComparison.OrdinalIgnoreCase); var isQsvInQsvOut = isHwDecoder && isQsvEncoder; var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true); var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true); var doDeintH2645 = doDeintH264 || doDeintHevc; - var doOclTonemap = IsHwTonemapAvailable(state, options); + var doVppTonemap = IsIntelVppTonemapAvailable(state, options); + var doOclTonemap = !doVppTonemap && IsHwTonemapAvailable(state, options); + var doTonemap = doVppTonemap || doOclTonemap; - var hasSubs = state.SubtitleStream is not null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; + var hasSubs = state.SubtitleStream is not null && ShouldEncodeSubtitle(state); var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream; var hasGraphicalSubs = hasSubs && !state.SubtitleStream.IsTextSubtitleStream; var hasAssSubs = hasSubs && (string.Equals(state.SubtitleStream.Codec, "ass", StringComparison.OrdinalIgnoreCase) || string.Equals(state.SubtitleStream.Codec, "ssa", StringComparison.OrdinalIgnoreCase)); + var subW = state.SubtitleStream?.Width; + var subH = state.SubtitleStream?.Height; + + var rotation = state.VideoStream?.Rotation ?? 0; + var transposeDir = rotation == 0 ? string.Empty : GetVideoTransposeDirection(state); + var doVppTranspose = !string.IsNullOrEmpty(transposeDir); + var swapWAndH = Math.Abs(rotation) == 90 && (isSwDecoder || ((isD3d11vaDecoder || isQsvDecoder) && doVppTranspose)); + var swpInW = swapWAndH ? inH : inW; + var swpInH = swapWAndH ? inW : inH; /* Make main filters for video stream */ var mainFilters = new List<string>(); - mainFilters.Add(GetOverwriteColorPropertiesParam(state, doOclTonemap)); + mainFilters.Add(GetOverwriteColorPropertiesParam(state, doTonemap)); if (isSwDecoder) { @@ -4004,10 +4428,16 @@ namespace MediaBrowser.Controller.MediaEncoding } var outFormat = doOclTonemap ? "yuv420p10le" : (hasGraphicalSubs ? "yuv420p" : "nv12"); - var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH); + var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, swpInW, swpInH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH); + if (isMjpegEncoder && !doOclTonemap) + { + // sw decoder + hw mjpeg encoder + swScaleFilter = string.IsNullOrEmpty(swScaleFilter) ? "scale=out_range=pc" : $"{swScaleFilter}:out_range=pc"; + } + // sw scale mainFilters.Add(swScaleFilter); - mainFilters.Add("format=" + outFormat); + mainFilters.Add($"format={outFormat}"); // keep video at memory except ocl tonemap, // since the overhead caused by hwupload >>> using sw filter. @@ -4019,8 +4449,77 @@ namespace MediaBrowser.Controller.MediaEncoding } else if (isD3d11vaDecoder || isQsvDecoder) { - var outFormat = doOclTonemap ? string.Empty : "nv12"; - var hwScaleFilter = GetHwScaleFilter("qsv", outFormat, inW, inH, reqW, reqH, reqMaxW, reqMaxH); + var isRext = IsVideoStreamHevcRext(state); + var twoPassVppTonemap = false; + var doVppFullRangeOut = isMjpegEncoder + && _mediaEncoder.EncoderVersion >= _minFFmpegQsvVppOutRangeOption; + var doVppScaleModeHq = isMjpegEncoder + && _mediaEncoder.EncoderVersion >= _minFFmpegQsvVppScaleModeOption; + var doVppProcamp = false; + var procampParams = string.Empty; + var procampParamsString = string.Empty; + if (doVppTonemap) + { + if (isRext) + { + // VPP tonemap requires p010 input + twoPassVppTonemap = true; + } + + if (options.VppTonemappingBrightness != 0 + && options.VppTonemappingBrightness >= -100 + && options.VppTonemappingBrightness <= 100) + { + procampParamsString += ":brightness={0}"; + twoPassVppTonemap = doVppProcamp = true; + } + + if (options.VppTonemappingContrast > 1 + && options.VppTonemappingContrast <= 10) + { + procampParamsString += ":contrast={1}"; + twoPassVppTonemap = doVppProcamp = true; + } + + if (doVppProcamp) + { + procampParamsString += ":procamp=1:async_depth=2"; + procampParams = string.Format( + CultureInfo.InvariantCulture, + procampParamsString, + options.VppTonemappingBrightness, + options.VppTonemappingContrast); + } + } + + var outFormat = doOclTonemap ? ((doVppTranspose || isRext) ? "p010" : string.Empty) : "nv12"; + outFormat = twoPassVppTonemap ? "p010" : outFormat; + + var swapOutputWandH = doVppTranspose && swapWAndH; + var hwScaleFilter = GetHwScaleFilter("vpp", "qsv", outFormat, swapOutputWandH, swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH); + + // d3d11va doesn't support dynamic pool size, use vpp filter ctx to relay + // to prevent encoder async and bframes from exhausting the decoder pool. + if (!string.IsNullOrEmpty(hwScaleFilter) && isD3d11vaDecoder) + { + hwScaleFilter += ":passthrough=0"; + } + + if (!string.IsNullOrEmpty(hwScaleFilter) && doVppTranspose) + { + hwScaleFilter += $":transpose={transposeDir}"; + } + + if (!string.IsNullOrEmpty(hwScaleFilter) && isMjpegEncoder) + { + hwScaleFilter += (doVppFullRangeOut && !doOclTonemap) ? ":out_range=pc" : string.Empty; + hwScaleFilter += doVppScaleModeHq ? ":scale_mode=hq" : string.Empty; + } + + if (!string.IsNullOrEmpty(hwScaleFilter) && doVppTonemap) + { + hwScaleFilter += doVppProcamp ? procampParams : (twoPassVppTonemap ? string.Empty : ":tonemap=1"); + } if (isD3d11vaDecoder) { @@ -4039,20 +4538,32 @@ namespace MediaBrowser.Controller.MediaEncoding mainFilters.Add(deintFilter); } - // hw scale + // hw transpose & scale & tonemap(w/o procamp) mainFilters.Add(hwScaleFilter); + + // hw tonemap(w/ procamp) + if (doVppTonemap && twoPassVppTonemap) + { + mainFilters.Add("vpp_qsv=tonemap=1:format=nv12:async_depth=2"); + } + + // force bt709 just in case vpp tonemap is not triggered or using MSDK instead of VPL. + if (doVppTonemap) + { + mainFilters.Add(GetOverwriteColorPropertiesParam(state, false)); + } } if (doOclTonemap && isHwDecoder) { // map from qsv to opencl via qsv(d3d11)-opencl interop. - mainFilters.Add("hwmap=derive_device=opencl"); + mainFilters.Add("hwmap=derive_device=opencl:mode=read"); } // hw tonemap if (doOclTonemap) { - var tonemapFilter = GetHwTonemapFilter(options, "opencl", "nv12"); + var tonemapFilter = GetHwTonemapFilter(options, "opencl", "nv12", isMjpegEncoder); mainFilters.Add(tonemapFilter); } @@ -4090,7 +4601,7 @@ namespace MediaBrowser.Controller.MediaEncoding { // OUTPUT qsv(nv12) surface(vram) // reverse-mapping via qsv(d3d11)-opencl interop. - mainFilters.Add("hwmap=derive_device=qsv:reverse=1"); + mainFilters.Add("hwmap=derive_device=qsv:mode=write:reverse=1"); mainFilters.Add("format=qsv"); } @@ -4104,7 +4615,7 @@ namespace MediaBrowser.Controller.MediaEncoding if (hasGraphicalSubs) { // overlay_qsv can handle overlay scaling, setup a smaller height to reduce transfer overhead - var subPreProcFilters = GetGraphicalSubPreProcessFilters(inW, inH, reqW, reqH, reqMaxW, 1080); + var subPreProcFilters = GetGraphicalSubPreProcessFilters(swpInW, swpInH, subW, subH, reqW, reqH, reqMaxW, 1080); subFilters.Add(subPreProcFilters); subFilters.Add("format=bgra"); } @@ -4114,7 +4625,7 @@ namespace MediaBrowser.Controller.MediaEncoding var subFramerate = hasAssSubs ? Math.Min(framerate ?? 25, 60) : 10; // alphasrc=s=1280x720:r=10:start=0,format=bgra,subtitles,hwupload - var alphaSrcFilter = GetAlphaSrcFilter(state, inW, inH, reqW, reqH, reqMaxW, 1080, subFramerate); + var alphaSrcFilter = GetAlphaSrcFilter(state, swpInW, swpInH, reqW, reqH, reqMaxW, 1080, subFramerate); var subTextSubtitlesFilter = GetTextSubtitlesFilter(state, true, true); subFilters.Add(alphaSrcFilter); subFilters.Add("format=bgra"); @@ -4125,9 +4636,9 @@ namespace MediaBrowser.Controller.MediaEncoding // default to 64 otherwise it will fail on certain iGPU. subFilters.Add("hwupload=derive_device=qsv:extra_hw_frames=64"); - var (overlayW, overlayH) = GetFixedOutputSize(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + var (overlayW, overlayH) = GetFixedOutputSize(swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH); var overlaySize = (overlayW.HasValue && overlayH.HasValue) - ? (":w=" + overlayW.Value + ":h=" + overlayH.Value) + ? $":w={overlayW.Value}:h={overlayH.Value}" : string.Empty; var overlayQsvFilter = string.Format( CultureInfo.InvariantCulture, @@ -4140,7 +4651,7 @@ namespace MediaBrowser.Controller.MediaEncoding { if (hasGraphicalSubs) { - var subPreProcFilters = GetGraphicalSubPreProcessFilters(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + var subPreProcFilters = GetGraphicalSubPreProcessFilters(swpInW, swpInH, subW, subH, reqW, reqH, reqMaxW, reqMaxH); subFilters.Add(subPreProcFilters); overlayFilters.Add("overlay=eof_action=pass:repeatlast=0"); } @@ -4169,21 +4680,31 @@ namespace MediaBrowser.Controller.MediaEncoding var isHwDecoder = isVaapiDecoder || isQsvDecoder; var isSwDecoder = string.IsNullOrEmpty(vidDecoder); var isSwEncoder = !isQsvEncoder; + var isMjpegEncoder = vidEncoder.Contains("mjpeg", StringComparison.OrdinalIgnoreCase); var isQsvInQsvOut = isHwDecoder && isQsvEncoder; var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true); var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true); - var doVaVppTonemap = IsVaapiVppTonemapAvailable(state, options); + var doVaVppTonemap = IsIntelVppTonemapAvailable(state, options); var doOclTonemap = !doVaVppTonemap && IsHwTonemapAvailable(state, options); var doTonemap = doVaVppTonemap || doOclTonemap; var doDeintH2645 = doDeintH264 || doDeintHevc; - var hasSubs = state.SubtitleStream is not null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; + var hasSubs = state.SubtitleStream is not null && ShouldEncodeSubtitle(state); var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream; var hasGraphicalSubs = hasSubs && !state.SubtitleStream.IsTextSubtitleStream; var hasAssSubs = hasSubs && (string.Equals(state.SubtitleStream.Codec, "ass", StringComparison.OrdinalIgnoreCase) || string.Equals(state.SubtitleStream.Codec, "ssa", StringComparison.OrdinalIgnoreCase)); + var subW = state.SubtitleStream?.Width; + var subH = state.SubtitleStream?.Height; + + var rotation = state.VideoStream?.Rotation ?? 0; + var transposeDir = rotation == 0 ? string.Empty : GetVideoTransposeDirection(state); + var doVppTranspose = !string.IsNullOrEmpty(transposeDir); + var swapWAndH = Math.Abs(rotation) == 90 && (isSwDecoder || ((isVaapiDecoder || isQsvDecoder) && doVppTranspose)); + var swpInW = swapWAndH ? inH : inW; + var swpInH = swapWAndH ? inW : inH; /* Make main filters for video stream */ var mainFilters = new List<string>(); @@ -4201,10 +4722,16 @@ namespace MediaBrowser.Controller.MediaEncoding } var outFormat = doOclTonemap ? "yuv420p10le" : (hasGraphicalSubs ? "yuv420p" : "nv12"); - var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH); + var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, swpInW, swpInH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH); + if (isMjpegEncoder && !doOclTonemap) + { + // sw decoder + hw mjpeg encoder + swScaleFilter = string.IsNullOrEmpty(swScaleFilter) ? "scale=out_range=pc" : $"{swScaleFilter}:out_range=pc"; + } + // sw scale mainFilters.Add(swScaleFilter); - mainFilters.Add("format=" + outFormat); + mainFilters.Add($"format={outFormat}"); // keep video at memory except ocl tonemap, // since the overhead caused by hwupload >>> using sw filter. @@ -4216,24 +4743,50 @@ namespace MediaBrowser.Controller.MediaEncoding } else if (isVaapiDecoder || isQsvDecoder) { + var hwFilterSuffix = isVaapiDecoder ? "vaapi" : "qsv"; + var isRext = IsVideoStreamHevcRext(state); + var doVppFullRangeOut = isMjpegEncoder + && _mediaEncoder.EncoderVersion >= _minFFmpegQsvVppOutRangeOption; + var doVppScaleModeHq = isMjpegEncoder + && _mediaEncoder.EncoderVersion >= _minFFmpegQsvVppScaleModeOption; + // INPUT vaapi/qsv surface(vram) // hw deint if (doDeintH2645) { - var deintFilter = GetHwDeinterlaceFilter(state, options, isVaapiDecoder ? "vaapi" : "qsv"); + var deintFilter = GetHwDeinterlaceFilter(state, options, hwFilterSuffix); mainFilters.Add(deintFilter); } - var outFormat = doTonemap ? string.Empty : "nv12"; - var hwScaleFilter = GetHwScaleFilter(isVaapiDecoder ? "vaapi" : "qsv", outFormat, inW, inH, reqW, reqH, reqMaxW, reqMaxH); + // hw transpose(vaapi vpp) + if (isVaapiDecoder && doVppTranspose) + { + mainFilters.Add($"transpose_vaapi=dir={transposeDir}"); + } - // allocate extra pool sizes for vaapi vpp + var outFormat = doTonemap ? (((isQsvDecoder && doVppTranspose) || isRext) ? "p010" : string.Empty) : "nv12"; + var swapOutputWandH = isQsvDecoder && doVppTranspose && swapWAndH; + var hwScalePrefix = isQsvDecoder ? "vpp" : "scale"; + var hwScaleFilter = GetHwScaleFilter(hwScalePrefix, hwFilterSuffix, outFormat, swapOutputWandH, swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH); + + if (!string.IsNullOrEmpty(hwScaleFilter) && isQsvDecoder && doVppTranspose) + { + hwScaleFilter += $":transpose={transposeDir}"; + } + + if (!string.IsNullOrEmpty(hwScaleFilter) && isMjpegEncoder) + { + hwScaleFilter += ((isQsvDecoder && !doVppFullRangeOut) || doOclTonemap) ? string.Empty : ":out_range=pc"; + hwScaleFilter += isQsvDecoder ? (doVppScaleModeHq ? ":scale_mode=hq" : string.Empty) : ":mode=hq"; + } + + // allocate extra pool sizes for vaapi vpp scale if (!string.IsNullOrEmpty(hwScaleFilter) && isVaapiDecoder) { hwScaleFilter += ":extra_hw_frames=24"; } - // hw scale + // hw transpose(qsv vpp) & scale mainFilters.Add(hwScaleFilter); } @@ -4244,28 +4797,30 @@ namespace MediaBrowser.Controller.MediaEncoding { // map from qsv to vaapi. mainFilters.Add("hwmap=derive_device=vaapi"); + mainFilters.Add("format=vaapi"); } - var tonemapFilter = GetHwTonemapFilter(options, "vaapi", "nv12"); + var tonemapFilter = GetHwTonemapFilter(options, "vaapi", "nv12", isMjpegEncoder); mainFilters.Add(tonemapFilter); if (isQsvDecoder) { // map from vaapi to qsv. mainFilters.Add("hwmap=derive_device=qsv"); + mainFilters.Add("format=qsv"); } } if (doOclTonemap && isHwDecoder) { // map from qsv to opencl via qsv(vaapi)-opencl interop. - mainFilters.Add("hwmap=derive_device=opencl"); + mainFilters.Add("hwmap=derive_device=opencl:mode=read"); } // ocl tonemap if (doOclTonemap) { - var tonemapFilter = GetHwTonemapFilter(options, "opencl", "nv12"); + var tonemapFilter = GetHwTonemapFilter(options, "opencl", "nv12", isMjpegEncoder); mainFilters.Add(tonemapFilter); } @@ -4306,7 +4861,7 @@ namespace MediaBrowser.Controller.MediaEncoding // OUTPUT qsv(nv12) surface(vram) // reverse-mapping via qsv(vaapi)-opencl interop. // add extra pool size to avoid the 'cannot allocate memory' error on hevc_qsv. - mainFilters.Add("hwmap=derive_device=qsv:reverse=1:extra_hw_frames=16"); + mainFilters.Add("hwmap=derive_device=qsv:mode=write:reverse=1:extra_hw_frames=16"); mainFilters.Add("format=qsv"); } else if (isVaapiDecoder) @@ -4326,7 +4881,7 @@ namespace MediaBrowser.Controller.MediaEncoding if (hasGraphicalSubs) { // overlay_qsv can handle overlay scaling, setup a smaller height to reduce transfer overhead - var subPreProcFilters = GetGraphicalSubPreProcessFilters(inW, inH, reqW, reqH, reqMaxW, 1080); + var subPreProcFilters = GetGraphicalSubPreProcessFilters(swpInW, swpInH, subW, subH, reqW, reqH, reqMaxW, 1080); subFilters.Add(subPreProcFilters); subFilters.Add("format=bgra"); } @@ -4335,7 +4890,7 @@ namespace MediaBrowser.Controller.MediaEncoding var framerate = state.VideoStream?.RealFrameRate; var subFramerate = hasAssSubs ? Math.Min(framerate ?? 25, 60) : 10; - var alphaSrcFilter = GetAlphaSrcFilter(state, inW, inH, reqW, reqH, reqMaxW, 1080, subFramerate); + var alphaSrcFilter = GetAlphaSrcFilter(state, swpInW, swpInH, reqW, reqH, reqMaxW, 1080, subFramerate); var subTextSubtitlesFilter = GetTextSubtitlesFilter(state, true, true); subFilters.Add(alphaSrcFilter); subFilters.Add("format=bgra"); @@ -4346,9 +4901,9 @@ namespace MediaBrowser.Controller.MediaEncoding // default to 64 otherwise it will fail on certain iGPU. subFilters.Add("hwupload=derive_device=qsv:extra_hw_frames=64"); - var (overlayW, overlayH) = GetFixedOutputSize(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + var (overlayW, overlayH) = GetFixedOutputSize(swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH); var overlaySize = (overlayW.HasValue && overlayH.HasValue) - ? (":w=" + overlayW.Value + ":h=" + overlayH.Value) + ? $":w={overlayW.Value}:h={overlayH.Value}" : string.Empty; var overlayQsvFilter = string.Format( CultureInfo.InvariantCulture, @@ -4361,7 +4916,7 @@ namespace MediaBrowser.Controller.MediaEncoding { if (hasGraphicalSubs) { - var subPreProcFilters = GetGraphicalSubPreProcessFilters(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + var subPreProcFilters = GetGraphicalSubPreProcessFilters(swpInW, swpInH, subW, subH, reqW, reqH, reqMaxW, reqMaxH); subFilters.Add(subPreProcFilters); overlayFilters.Add("overlay=eof_action=pass:repeatlast=0"); } @@ -4382,7 +4937,7 @@ namespace MediaBrowser.Controller.MediaEncoding EncodingOptions options, string vidEncoder) { - if (!string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase)) + if (options.HardwareAccelerationType != HardwareAccelerationType.vaapi) { return (null, null, null); } @@ -4417,17 +4972,18 @@ namespace MediaBrowser.Controller.MediaEncoding return swFilterChain; } - // prefered vaapi + opencl filters pipeline + // preferred vaapi + opencl filters pipeline if (_mediaEncoder.IsVaapiDeviceInteliHD) { // Intel iHD path, with extra vpp tonemap and overlay support. return GetIntelVaapiFullVidFiltersPrefered(state, options, vidDecoder, vidEncoder); } - // prefered vaapi + vulkan filters pipeline + // preferred vaapi + vulkan filters pipeline if (_mediaEncoder.IsVaapiDeviceAmd && isVaapiVkSupported - && _mediaEncoder.IsVaapiDeviceSupportVulkanDrmInterop) + && _mediaEncoder.IsVaapiDeviceSupportVulkanDrmInterop + && Environment.OSVersion.Version >= _minKernelVersionAmdVkFmtModifier) { // AMD radeonsi path(targeting Polaris/gfx8+), with extra vulkan tonemap and overlay support. return GetAmdVaapiFullVidFiltersPrefered(state, options, vidDecoder, vidEncoder); @@ -4455,21 +5011,31 @@ namespace MediaBrowser.Controller.MediaEncoding var isVaapiEncoder = vidEncoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase); var isSwDecoder = string.IsNullOrEmpty(vidDecoder); var isSwEncoder = !isVaapiEncoder; + var isMjpegEncoder = vidEncoder.Contains("mjpeg", StringComparison.OrdinalIgnoreCase); var isVaInVaOut = isVaapiDecoder && isVaapiEncoder; var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true); var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true); - var doVaVppTonemap = isVaapiDecoder && IsVaapiVppTonemapAvailable(state, options); + var doVaVppTonemap = isVaapiDecoder && IsIntelVppTonemapAvailable(state, options); var doOclTonemap = !doVaVppTonemap && IsHwTonemapAvailable(state, options); var doTonemap = doVaVppTonemap || doOclTonemap; var doDeintH2645 = doDeintH264 || doDeintHevc; - var hasSubs = state.SubtitleStream is not null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; + var hasSubs = state.SubtitleStream is not null && ShouldEncodeSubtitle(state); var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream; var hasGraphicalSubs = hasSubs && !state.SubtitleStream.IsTextSubtitleStream; var hasAssSubs = hasSubs && (string.Equals(state.SubtitleStream.Codec, "ass", StringComparison.OrdinalIgnoreCase) || string.Equals(state.SubtitleStream.Codec, "ssa", StringComparison.OrdinalIgnoreCase)); + var subW = state.SubtitleStream?.Width; + var subH = state.SubtitleStream?.Height; + + var rotation = state.VideoStream?.Rotation ?? 0; + var transposeDir = rotation == 0 ? string.Empty : GetVideoTransposeDirection(state); + var doVaVppTranspose = !string.IsNullOrEmpty(transposeDir); + var swapWAndH = Math.Abs(rotation) == 90 && (isSwDecoder || (isVaapiDecoder && doVaVppTranspose)); + var swpInW = swapWAndH ? inH : inW; + var swpInH = swapWAndH ? inW : inH; /* Make main filters for video stream */ var mainFilters = new List<string>(); @@ -4487,10 +5053,16 @@ namespace MediaBrowser.Controller.MediaEncoding } var outFormat = doOclTonemap ? "yuv420p10le" : "nv12"; - var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH); + var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, swpInW, swpInH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH); + if (isMjpegEncoder && !doOclTonemap) + { + // sw decoder + hw mjpeg encoder + swScaleFilter = string.IsNullOrEmpty(swScaleFilter) ? "scale=out_range=pc" : $"{swScaleFilter}:out_range=pc"; + } + // sw scale mainFilters.Add(swScaleFilter); - mainFilters.Add("format=" + outFormat); + mainFilters.Add($"format={outFormat}"); // keep video at memory except ocl tonemap, // since the overhead caused by hwupload >>> using sw filter. @@ -4502,6 +5074,8 @@ namespace MediaBrowser.Controller.MediaEncoding } else if (isVaapiDecoder) { + var isRext = IsVideoStreamHevcRext(state); + // INPUT vaapi surface(vram) // hw deint if (doDeintH2645) @@ -4510,8 +5084,20 @@ namespace MediaBrowser.Controller.MediaEncoding mainFilters.Add(deintFilter); } - var outFormat = doTonemap ? string.Empty : "nv12"; - var hwScaleFilter = GetHwScaleFilter("vaapi", outFormat, inW, inH, reqW, reqH, reqMaxW, reqMaxH); + // hw transpose + if (doVaVppTranspose) + { + mainFilters.Add($"transpose_vaapi=dir={transposeDir}"); + } + + var outFormat = doTonemap ? (isRext ? "p010" : string.Empty) : "nv12"; + var hwScaleFilter = GetHwScaleFilter("scale", "vaapi", outFormat, false, swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH); + + if (!string.IsNullOrEmpty(hwScaleFilter) && isMjpegEncoder) + { + hwScaleFilter += doOclTonemap ? string.Empty : ":out_range=pc"; + hwScaleFilter += ":mode=hq"; + } // allocate extra pool sizes for vaapi vpp if (!string.IsNullOrEmpty(hwScaleFilter)) @@ -4526,20 +5112,20 @@ namespace MediaBrowser.Controller.MediaEncoding // vaapi vpp tonemap if (doVaVppTonemap && isVaapiDecoder) { - var tonemapFilter = GetHwTonemapFilter(options, "vaapi", "nv12"); + var tonemapFilter = GetHwTonemapFilter(options, "vaapi", "nv12", isMjpegEncoder); mainFilters.Add(tonemapFilter); } if (doOclTonemap && isVaapiDecoder) { // map from vaapi to opencl via vaapi-opencl interop(Intel only). - mainFilters.Add("hwmap=derive_device=opencl"); + mainFilters.Add("hwmap=derive_device=opencl:mode=read"); } // ocl tonemap if (doOclTonemap) { - var tonemapFilter = GetHwTonemapFilter(options, "opencl", "nv12"); + var tonemapFilter = GetHwTonemapFilter(options, "opencl", "nv12", isMjpegEncoder); mainFilters.Add(tonemapFilter); } @@ -4547,7 +5133,7 @@ namespace MediaBrowser.Controller.MediaEncoding { // OUTPUT vaapi(nv12) surface(vram) // reverse-mapping via vaapi-opencl interop. - mainFilters.Add("hwmap=derive_device=vaapi:reverse=1"); + mainFilters.Add("hwmap=derive_device=vaapi:mode=write:reverse=1"); mainFilters.Add("format=vaapi"); } @@ -4598,7 +5184,7 @@ namespace MediaBrowser.Controller.MediaEncoding if (hasGraphicalSubs) { // overlay_vaapi can handle overlay scaling, setup a smaller height to reduce transfer overhead - var subPreProcFilters = GetGraphicalSubPreProcessFilters(inW, inH, reqW, reqH, reqMaxW, 1080); + var subPreProcFilters = GetGraphicalSubPreProcessFilters(swpInW, swpInH, subW, subH, reqW, reqH, reqMaxW, 1080); subFilters.Add(subPreProcFilters); subFilters.Add("format=bgra"); } @@ -4607,7 +5193,7 @@ namespace MediaBrowser.Controller.MediaEncoding var framerate = state.VideoStream?.RealFrameRate; var subFramerate = hasAssSubs ? Math.Min(framerate ?? 25, 60) : 10; - var alphaSrcFilter = GetAlphaSrcFilter(state, inW, inH, reqW, reqH, reqMaxW, 1080, subFramerate); + var alphaSrcFilter = GetAlphaSrcFilter(state, swpInW, swpInH, reqW, reqH, reqMaxW, 1080, subFramerate); var subTextSubtitlesFilter = GetTextSubtitlesFilter(state, true, true); subFilters.Add(alphaSrcFilter); subFilters.Add("format=bgra"); @@ -4616,9 +5202,9 @@ namespace MediaBrowser.Controller.MediaEncoding subFilters.Add("hwupload=derive_device=vaapi"); - var (overlayW, overlayH) = GetFixedOutputSize(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + var (overlayW, overlayH) = GetFixedOutputSize(swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH); var overlaySize = (overlayW.HasValue && overlayH.HasValue) - ? (":w=" + overlayW.Value + ":h=" + overlayH.Value) + ? $":w={overlayW.Value}:h={overlayH.Value}" : string.Empty; var overlayVaapiFilter = string.Format( CultureInfo.InvariantCulture, @@ -4631,7 +5217,7 @@ namespace MediaBrowser.Controller.MediaEncoding { if (hasGraphicalSubs) { - var subPreProcFilters = GetGraphicalSubPreProcessFilters(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + var subPreProcFilters = GetGraphicalSubPreProcessFilters(swpInW, swpInH, subW, subH, reqW, reqH, reqMaxW, reqMaxH); subFilters.Add(subPreProcFilters); overlayFilters.Add("overlay=eof_action=pass:repeatlast=0"); @@ -4663,19 +5249,27 @@ namespace MediaBrowser.Controller.MediaEncoding var isVaapiEncoder = vidEncoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase); var isSwDecoder = string.IsNullOrEmpty(vidDecoder); var isSwEncoder = !isVaapiEncoder; + var isMjpegEncoder = vidEncoder.Contains("mjpeg", StringComparison.OrdinalIgnoreCase); var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true); var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true); var doVkTonemap = IsVulkanHwTonemapAvailable(state, options); var doDeintH2645 = doDeintH264 || doDeintHevc; - var hasSubs = state.SubtitleStream is not null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; + var hasSubs = state.SubtitleStream is not null && ShouldEncodeSubtitle(state); var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream; var hasGraphicalSubs = hasSubs && !state.SubtitleStream.IsTextSubtitleStream; var hasAssSubs = hasSubs && (string.Equals(state.SubtitleStream.Codec, "ass", StringComparison.OrdinalIgnoreCase) || string.Equals(state.SubtitleStream.Codec, "ssa", StringComparison.OrdinalIgnoreCase)); + var rotation = state.VideoStream?.Rotation ?? 0; + var transposeDir = rotation == 0 ? string.Empty : GetVideoTransposeDirection(state); + var doVkTranspose = isVaapiDecoder && !string.IsNullOrEmpty(transposeDir); + var swapWAndH = Math.Abs(rotation) == 90 && (isSwDecoder || (isVaapiDecoder && doVkTranspose)); + var swpInW = swapWAndH ? inH : inW; + var swpInH = swapWAndH ? inW : inH; + /* Make main filters for video stream */ var mainFilters = new List<string>(); @@ -4700,7 +5294,7 @@ namespace MediaBrowser.Controller.MediaEncoding else { // sw scale - var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH); + var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, swpInW, swpInH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH); mainFilters.Add(swScaleFilter); mainFilters.Add("format=nv12"); } @@ -4708,11 +5302,37 @@ namespace MediaBrowser.Controller.MediaEncoding else if (isVaapiDecoder) { // INPUT vaapi surface(vram) - if (doVkTonemap || hasSubs) + if (doVkTranspose || doVkTonemap || hasSubs) { // map from vaapi to vulkan/drm via interop (Polaris/gfx8+). - mainFilters.Add("hwmap=derive_device=vulkan"); - mainFilters.Add("format=vulkan"); + if (_mediaEncoder.EncoderVersion >= _minFFmpegAlteredVaVkInterop) + { + if (doVkTranspose || !_mediaEncoder.IsVaapiDeviceSupportVulkanDrmModifier) + { + // disable the indirect va-drm-vk mapping since it's no longer reliable. + mainFilters.Add("hwmap=derive_device=drm"); + mainFilters.Add("format=drm_prime"); + mainFilters.Add("hwmap=derive_device=vulkan"); + mainFilters.Add("format=vulkan"); + + // workaround for libplacebo using the imported vulkan frame on gfx8. + if (!_mediaEncoder.IsVaapiDeviceSupportVulkanDrmModifier) + { + mainFilters.Add("scale_vulkan"); + } + } + else if (doVkTonemap || hasSubs) + { + // non ad-hoc libplacebo also accepts drm_prime direct input. + mainFilters.Add("hwmap=derive_device=drm"); + mainFilters.Add("format=drm_prime"); + } + } + else // legacy va-vk mapping that works only in jellyfin-ffmpeg6 + { + mainFilters.Add("hwmap=derive_device=vulkan"); + mainFilters.Add("format=vulkan"); + } } else { @@ -4724,16 +5344,36 @@ namespace MediaBrowser.Controller.MediaEncoding } // hw scale - var hwScaleFilter = GetHwScaleFilter("vaapi", "nv12", inW, inH, reqW, reqH, reqMaxW, reqMaxH); + var hwScaleFilter = GetHwScaleFilter("scale", "vaapi", "nv12", false, inW, inH, reqW, reqH, reqMaxW, reqMaxH); + + if (!string.IsNullOrEmpty(hwScaleFilter) && isMjpegEncoder && !doVkTonemap) + { + hwScaleFilter += ":out_range=pc:mode=hq"; + } + mainFilters.Add(hwScaleFilter); } } + // vk transpose + if (doVkTranspose) + { + if (string.Equals(transposeDir, "reversal", StringComparison.OrdinalIgnoreCase)) + { + mainFilters.Add("flip_vulkan"); + } + else + { + mainFilters.Add($"transpose_vulkan=dir={transposeDir}"); + } + } + // vk libplacebo if (doVkTonemap || hasSubs) { - var libplaceboFilter = GetLibplaceboFilter(options, "bgra", doVkTonemap, inW, inH, reqW, reqH, reqMaxW, reqMaxH); + var libplaceboFilter = GetLibplaceboFilter(options, "bgra", doVkTonemap, swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH, isMjpegEncoder); mainFilters.Add(libplaceboFilter); + mainFilters.Add("format=vulkan"); } if (doVkTonemap && !hasSubs) @@ -4776,7 +5416,9 @@ namespace MediaBrowser.Controller.MediaEncoding { if (hasGraphicalSubs) { - var subPreProcFilters = GetGraphicalSubPreProcessFilters(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + var subW = state.SubtitleStream?.Width; + var subH = state.SubtitleStream?.Height; + var subPreProcFilters = GetGraphicalSubPreProcessFilters(swpInW, swpInH, subW, subH, reqW, reqH, reqMaxW, reqMaxH); subFilters.Add(subPreProcFilters); subFilters.Add("format=bgra"); } @@ -4785,7 +5427,7 @@ namespace MediaBrowser.Controller.MediaEncoding var framerate = state.VideoStream?.RealFrameRate; var subFramerate = hasAssSubs ? Math.Min(framerate ?? 25, 60) : 10; - var alphaSrcFilter = GetAlphaSrcFilter(state, inW, inH, reqW, reqH, reqMaxW, reqMaxH, subFramerate); + var alphaSrcFilter = GetAlphaSrcFilter(state, swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH, subFramerate); var subTextSubtitlesFilter = GetTextSubtitlesFilter(state, true, true); subFilters.Add(alphaSrcFilter); subFilters.Add("format=bgra"); @@ -4844,6 +5486,7 @@ namespace MediaBrowser.Controller.MediaEncoding var isVaapiEncoder = vidEncoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase); var isSwDecoder = string.IsNullOrEmpty(vidDecoder); var isSwEncoder = !isVaapiEncoder; + var isMjpegEncoder = vidEncoder.Contains("mjpeg", StringComparison.OrdinalIgnoreCase); var isVaInVaOut = isVaapiDecoder && isVaapiEncoder; var isi965Driver = _mediaEncoder.IsVaapiDeviceInteli965; var isAmdDriver = _mediaEncoder.IsVaapiDeviceAmd; @@ -4853,10 +5496,15 @@ namespace MediaBrowser.Controller.MediaEncoding var doDeintH2645 = doDeintH264 || doDeintHevc; var doOclTonemap = IsHwTonemapAvailable(state, options); - var hasSubs = state.SubtitleStream is not null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; + var hasSubs = state.SubtitleStream is not null && ShouldEncodeSubtitle(state); var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream; var hasGraphicalSubs = hasSubs && !state.SubtitleStream.IsTextSubtitleStream; + var rotation = state.VideoStream?.Rotation ?? 0; + var swapWAndH = Math.Abs(rotation) == 90 && isSwDecoder; + var swpInW = swapWAndH ? inH : inW; + var swpInH = swapWAndH ? inW : inH; + /* Make main filters for video stream */ var mainFilters = new List<string>(); @@ -4874,7 +5522,13 @@ namespace MediaBrowser.Controller.MediaEncoding } outFormat = doOclTonemap ? "yuv420p10le" : "nv12"; - var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH); + var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, swpInW, swpInH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH); + if (isMjpegEncoder && !doOclTonemap) + { + // sw decoder + hw mjpeg encoder + swScaleFilter = string.IsNullOrEmpty(swScaleFilter) ? "scale=out_range=pc" : $"{swScaleFilter}:out_range=pc"; + } + // sw scale mainFilters.Add(swScaleFilter); mainFilters.Add("format=" + outFormat); @@ -4898,7 +5552,13 @@ namespace MediaBrowser.Controller.MediaEncoding } outFormat = doOclTonemap ? string.Empty : "nv12"; - var hwScaleFilter = GetHwScaleFilter("vaapi", outFormat, inW, inH, reqW, reqH, reqMaxW, reqMaxH); + var hwScaleFilter = GetHwScaleFilter("scale", "vaapi", outFormat, false, inW, inH, reqW, reqH, reqMaxW, reqMaxH); + + if (!string.IsNullOrEmpty(hwScaleFilter) && isMjpegEncoder) + { + hwScaleFilter += doOclTonemap ? string.Empty : ":out_range=pc"; + hwScaleFilter += ":mode=hq"; + } // allocate extra pool sizes for vaapi vpp if (!string.IsNullOrEmpty(hwScaleFilter)) @@ -4928,7 +5588,7 @@ namespace MediaBrowser.Controller.MediaEncoding // ocl tonemap if (doOclTonemap) { - var tonemapFilter = GetHwTonemapFilter(options, "opencl", "nv12"); + var tonemapFilter = GetHwTonemapFilter(options, "opencl", "nv12", isMjpegEncoder); mainFilters.Add(tonemapFilter); } @@ -4994,7 +5654,9 @@ namespace MediaBrowser.Controller.MediaEncoding { if (hasGraphicalSubs) { - var subPreProcFilters = GetGraphicalSubPreProcessFilters(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + var subW = state.SubtitleStream?.Width; + var subH = state.SubtitleStream?.Height; + var subPreProcFilters = GetGraphicalSubPreProcessFilters(swpInW, swpInH, subW, subH, reqW, reqH, reqMaxW, reqMaxH); subFilters.Add(subPreProcFilters); overlayFilters.Add("overlay=eof_action=pass:repeatlast=0"); @@ -5020,18 +5682,20 @@ namespace MediaBrowser.Controller.MediaEncoding EncodingOptions options, string vidEncoder) { - if (!string.Equals(options.HardwareAccelerationType, "videotoolbox", StringComparison.OrdinalIgnoreCase)) + if (options.HardwareAccelerationType != HardwareAccelerationType.videotoolbox) { return (null, null, null); } + // ReSharper disable once InconsistentNaming var isMacOS = OperatingSystem.IsMacOS(); var vidDecoder = GetHardwareVideoDecoder(state, options) ?? string.Empty; + var isVtDecoder = vidDecoder.Contains("videotoolbox", StringComparison.OrdinalIgnoreCase); var isVtEncoder = vidEncoder.Contains("videotoolbox", StringComparison.OrdinalIgnoreCase); var isVtFullSupported = isMacOS && IsVideoToolboxFullSupported(); // legacy videotoolbox pipeline (disable hw filters) - if (!isVtEncoder + if (!(isVtEncoder || isVtDecoder) || !isVtFullSupported || !_mediaEncoder.SupportsFilter("alphasrc")) { @@ -5048,6 +5712,10 @@ namespace MediaBrowser.Controller.MediaEncoding string vidDecoder, string vidEncoder) { + var isVtEncoder = vidEncoder.Contains("videotoolbox", StringComparison.OrdinalIgnoreCase); + var isVtDecoder = vidDecoder.Contains("videotoolbox", StringComparison.OrdinalIgnoreCase); + var isMjpegEncoder = vidEncoder.Contains("mjpeg", StringComparison.OrdinalIgnoreCase); + var inW = state.VideoStream?.Width; var inH = state.VideoStream?.Height; var reqW = state.BaseRequest.Width; @@ -5056,13 +5724,19 @@ namespace MediaBrowser.Controller.MediaEncoding var reqMaxH = state.BaseRequest.MaxHeight; var threeDFormat = state.MediaSource.Video3DFormat; - var isVtEncoder = vidEncoder.Contains("videotoolbox", StringComparison.OrdinalIgnoreCase); - var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true); var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true); var doDeintH2645 = doDeintH264 || doDeintHevc; var doVtTonemap = IsVideoToolboxTonemapAvailable(state, options); var doMetalTonemap = !doVtTonemap && IsHwTonemapAvailable(state, options); + var usingHwSurface = isVtDecoder && (_mediaEncoder.EncoderVersion >= _minFFmpegWorkingVtHwSurface); + + var rotation = state.VideoStream?.Rotation ?? 0; + var transposeDir = rotation == 0 ? string.Empty : GetVideoTransposeDirection(state); + var doVtTranspose = !string.IsNullOrEmpty(transposeDir) && _mediaEncoder.SupportsFilter("transpose_vt"); + var swapWAndH = Math.Abs(rotation) == 90 && doVtTranspose; + var swpInW = swapWAndH ? inH : inW; + var swpInH = swapWAndH ? inW : inH; var scaleFormat = string.Empty; // Use P010 for Metal tone mapping, otherwise force an 8bit output. @@ -5081,29 +5755,18 @@ namespace MediaBrowser.Controller.MediaEncoding } } - var hwScaleFilter = GetHwScaleFilter("vt", scaleFormat, inW, inH, reqW, reqH, reqMaxW, reqMaxH); + var hwScaleFilter = GetHwScaleFilter("scale", "vt", scaleFormat, false, swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH); - var hasSubs = state.SubtitleStream is not null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; + var hasSubs = state.SubtitleStream is not null && ShouldEncodeSubtitle(state); var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream; var hasGraphicalSubs = hasSubs && !state.SubtitleStream.IsTextSubtitleStream; var hasAssSubs = hasSubs && (string.Equals(state.SubtitleStream.Codec, "ass", StringComparison.OrdinalIgnoreCase) || string.Equals(state.SubtitleStream.Codec, "ssa", StringComparison.OrdinalIgnoreCase)); - if (!isVtEncoder) - { - // should not happen. - return (null, null, null); - } - /* Make main filters for video stream */ var mainFilters = new List<string>(); - // INPUT videotoolbox/memory surface(vram/uma) - // this will pass-through automatically if in/out format matches. - mainFilters.Add("format=nv12|p010le|videotoolbox_vld"); - mainFilters.Add("hwupload=derive_device=videotoolbox"); - // hw deint if (doDeintH2645) { @@ -5111,6 +5774,12 @@ namespace MediaBrowser.Controller.MediaEncoding mainFilters.Add(deintFilter); } + // hw transpose + if (doVtTranspose) + { + mainFilters.Add($"transpose_vt=dir={transposeDir}"); + } + if (doVtTonemap) { const string VtTonemapArgs = "color_matrix=bt709:color_primaries=bt709:color_transfer=bt709"; @@ -5127,7 +5796,7 @@ namespace MediaBrowser.Controller.MediaEncoding // Metal tonemap if (doMetalTonemap) { - var tonemapFilter = GetHwTonemapFilter(options, "videotoolbox", "nv12"); + var tonemapFilter = GetHwTonemapFilter(options, "videotoolbox", "nv12", isMjpegEncoder); mainFilters.Add(tonemapFilter); } @@ -5139,7 +5808,9 @@ namespace MediaBrowser.Controller.MediaEncoding { if (hasGraphicalSubs) { - var subPreProcFilters = GetGraphicalSubPreProcessFilters(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + var subW = state.SubtitleStream?.Width; + var subH = state.SubtitleStream?.Height; + var subPreProcFilters = GetGraphicalSubPreProcessFilters(swpInW, swpInH, subW, subH, reqW, reqH, reqMaxW, reqMaxH); subFilters.Add(subPreProcFilters); subFilters.Add("format=bgra"); } @@ -5148,17 +5819,46 @@ namespace MediaBrowser.Controller.MediaEncoding var framerate = state.VideoStream?.RealFrameRate; var subFramerate = hasAssSubs ? Math.Min(framerate ?? 25, 60) : 10; - var alphaSrcFilter = GetAlphaSrcFilter(state, inW, inH, reqW, reqH, reqMaxW, reqMaxH, subFramerate); + var alphaSrcFilter = GetAlphaSrcFilter(state, swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH, subFramerate); var subTextSubtitlesFilter = GetTextSubtitlesFilter(state, true, true); subFilters.Add(alphaSrcFilter); subFilters.Add("format=bgra"); subFilters.Add(subTextSubtitlesFilter); } - subFilters.Add("hwupload=derive_device=videotoolbox"); + subFilters.Add("hwupload"); overlayFilters.Add("overlay_videotoolbox=eof_action=pass:repeatlast=0"); } + if (usingHwSurface) + { + if (!isVtEncoder) + { + mainFilters.Add("hwdownload"); + mainFilters.Add("format=nv12"); + } + + return (mainFilters, subFilters, overlayFilters); + } + + // For old jellyfin-ffmpeg that has broken hwsurface, add a hwupload + var needFiltering = mainFilters.Any(f => !string.IsNullOrEmpty(f)) || + subFilters.Any(f => !string.IsNullOrEmpty(f)) || + overlayFilters.Any(f => !string.IsNullOrEmpty(f)); + if (needFiltering) + { + // INPUT videotoolbox/memory surface(vram/uma) + // this will pass-through automatically if in/out format matches. + mainFilters.Insert(0, "hwupload"); + mainFilters.Insert(0, "format=nv12|p010le|videotoolbox_vld"); + + if (!isVtEncoder) + { + mainFilters.Add("hwdownload"); + mainFilters.Add("format=nv12"); + } + } + return (mainFilters, subFilters, overlayFilters); } @@ -5174,7 +5874,7 @@ namespace MediaBrowser.Controller.MediaEncoding EncodingOptions options, string vidEncoder) { - if (!string.Equals(options.HardwareAccelerationType, "rkmpp", StringComparison.OrdinalIgnoreCase)) + if (options.HardwareAccelerationType != HardwareAccelerationType.rkmpp) { return (null, null, null); } @@ -5192,7 +5892,7 @@ namespace MediaBrowser.Controller.MediaEncoding return GetSwVidFilterChain(state, options, vidEncoder); } - // prefered rkmpp + rkrga + opencl filters pipeline + // preferred rkmpp + rkrga + opencl filters pipeline if (isRkmppOclSupported) { return GetRkmppVidFiltersPrefered(state, options, vidDecoder, vidEncoder); @@ -5219,19 +5919,32 @@ namespace MediaBrowser.Controller.MediaEncoding var isRkmppEncoder = vidEncoder.Contains("rkmpp", StringComparison.OrdinalIgnoreCase); var isSwDecoder = !isRkmppDecoder; var isSwEncoder = !isRkmppEncoder; + var isMjpegEncoder = vidEncoder.Contains("mjpeg", StringComparison.OrdinalIgnoreCase); var isDrmInDrmOut = isRkmppDecoder && isRkmppEncoder; + var isEncoderSupportAfbc = isRkmppEncoder + && (vidEncoder.Contains("h264", StringComparison.OrdinalIgnoreCase) + || vidEncoder.Contains("hevc", StringComparison.OrdinalIgnoreCase)); var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true); var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true); var doDeintH2645 = doDeintH264 || doDeintHevc; var doOclTonemap = IsHwTonemapAvailable(state, options); - var hasSubs = state.SubtitleStream != null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; + var hasSubs = state.SubtitleStream is not null && ShouldEncodeSubtitle(state); var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream; var hasGraphicalSubs = hasSubs && !state.SubtitleStream.IsTextSubtitleStream; var hasAssSubs = hasSubs && (string.Equals(state.SubtitleStream.Codec, "ass", StringComparison.OrdinalIgnoreCase) || string.Equals(state.SubtitleStream.Codec, "ssa", StringComparison.OrdinalIgnoreCase)); + var subW = state.SubtitleStream?.Width; + var subH = state.SubtitleStream?.Height; + + var rotation = state.VideoStream?.Rotation ?? 0; + var transposeDir = rotation == 0 ? string.Empty : GetVideoTransposeDirection(state); + var doRkVppTranspose = !string.IsNullOrEmpty(transposeDir); + var swapWAndH = Math.Abs(rotation) == 90 && (isSwDecoder || (isRkmppDecoder && doRkVppTranspose)); + var swpInW = swapWAndH ? inH : inW; + var swpInH = swapWAndH ? inW : inH; /* Make main filters for video stream */ var mainFilters = new List<string>(); @@ -5249,7 +5962,13 @@ namespace MediaBrowser.Controller.MediaEncoding } var outFormat = doOclTonemap ? "yuv420p10le" : (hasGraphicalSubs ? "yuv420p" : "nv12"); - var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH); + var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, swpInW, swpInH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH); + if (isMjpegEncoder && !doOclTonemap) + { + // sw decoder + hw mjpeg encoder + swScaleFilter = string.IsNullOrEmpty(swScaleFilter) ? "scale=out_range=pc" : $"{swScaleFilter}:out_range=pc"; + } + if (!string.IsNullOrEmpty(swScaleFilter)) { swScaleFilter += ":flags=fast_bilinear"; @@ -5257,7 +5976,7 @@ namespace MediaBrowser.Controller.MediaEncoding // sw scale mainFilters.Add(swScaleFilter); - mainFilters.Add("format=" + outFormat); + mainFilters.Add($"format={outFormat}"); // keep video at memory except ocl tonemap, // since the overhead caused by hwupload >>> using sw filter. @@ -5271,22 +5990,51 @@ namespace MediaBrowser.Controller.MediaEncoding { // INPUT rkmpp/drm surface(gem/dma-heap) - var isFullAfbcPipeline = isDrmInDrmOut && !doOclTonemap; + var isFullAfbcPipeline = isEncoderSupportAfbc && isDrmInDrmOut && !doOclTonemap; + var swapOutputWandH = doRkVppTranspose && swapWAndH; var outFormat = doOclTonemap ? "p010" : "nv12"; - var hwScaleFilter = GetHwScaleFilter("rkrga", outFormat, inW, inH, reqW, reqH, reqMaxW, reqMaxH); - var hwScaleFilter2 = GetHwScaleFilter("rkrga", string.Empty, inW, inH, reqW, reqH, reqMaxW, reqMaxH); + var hwScaleFilter = GetHwScaleFilter("vpp", "rkrga", outFormat, swapOutputWandH, swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH); + var doScaling = !string.IsNullOrEmpty(GetHwScaleFilter("vpp", "rkrga", string.Empty, swapOutputWandH, swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH)); if (!hasSubs + || doRkVppTranspose || !isFullAfbcPipeline - || !string.IsNullOrEmpty(hwScaleFilter2)) + || doScaling) { + var isScaleRatioSupported = IsScaleRatioSupported(inW, inH, reqW, reqH, reqMaxW, reqMaxH, 8.0f); + + // RGA3 hardware only support (1/8 ~ 8) scaling in each blit operation, + // but in Trickplay there's a case: (3840/320 == 12), enable 2pass for it + if (doScaling && !isScaleRatioSupported) + { + // Vendor provided BSP kernel has an RGA driver bug that causes the output to be corrupted for P010 format. + // Use NV15 instead of P010 to avoid the issue. + // SDR inputs are using BGRA formats already which is not affected. + var intermediateFormat = doOclTonemap ? "nv15" : (isMjpegEncoder ? "bgra" : outFormat); + var hwScaleFilterFirstPass = $"scale_rkrga=w=iw/7.9:h=ih/7.9:format={intermediateFormat}:force_original_aspect_ratio=increase:force_divisible_by=4:afbc=1"; + mainFilters.Add(hwScaleFilterFirstPass); + } + + // The RKMPP MJPEG encoder on some newer chip models no longer supports RGB input. + // Use 2pass here to enable RGA output of full-range YUV in the 2nd pass. + if (isMjpegEncoder && !doOclTonemap && ((doScaling && isScaleRatioSupported) || !doScaling)) + { + var hwScaleFilterFirstPass = "vpp_rkrga=format=bgra:afbc=1"; + mainFilters.Add(hwScaleFilterFirstPass); + } + + if (!string.IsNullOrEmpty(hwScaleFilter) && doRkVppTranspose) + { + hwScaleFilter += $":transpose={transposeDir}"; + } + // try enabling AFBC to save DDR bandwidth if (!string.IsNullOrEmpty(hwScaleFilter) && isFullAfbcPipeline) { hwScaleFilter += ":afbc=1"; } - // hw scale + // hw transpose & scale mainFilters.Add(hwScaleFilter); } } @@ -5294,19 +6042,13 @@ namespace MediaBrowser.Controller.MediaEncoding if (doOclTonemap && isRkmppDecoder) { // map from rkmpp/drm to opencl via drm-opencl interop. - mainFilters.Add("hwmap=derive_device=opencl:mode=read"); + mainFilters.Add("hwmap=derive_device=opencl"); } // ocl tonemap if (doOclTonemap) { - var tonemapFilter = GetHwTonemapFilter(options, "opencl", "nv12"); - // enable tradeoffs for performance - if (!string.IsNullOrEmpty(tonemapFilter)) - { - tonemapFilter += ":tradeoff=1"; - } - + var tonemapFilter = GetHwTonemapFilter(options, "opencl", "nv12", isMjpegEncoder); mainFilters.Add(tonemapFilter); } @@ -5343,7 +6085,7 @@ namespace MediaBrowser.Controller.MediaEncoding { // OUTPUT drm(nv12) surface(gem/dma-heap) // reverse-mapping via drm-opencl interop. - mainFilters.Add("hwmap=derive_device=rkmpp:mode=write:reverse=1"); + mainFilters.Add("hwmap=derive_device=rkmpp:reverse=1"); mainFilters.Add("format=drm_prime"); } } @@ -5355,9 +6097,10 @@ namespace MediaBrowser.Controller.MediaEncoding { if (hasSubs) { + var subMaxH = 1080; if (hasGraphicalSubs) { - var subPreProcFilters = GetGraphicalSubPreProcessFilters(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + var subPreProcFilters = GetGraphicalSubPreProcessFilters(swpInW, swpInH, subW, subH, reqW, reqH, reqMaxW, subMaxH); subFilters.Add(subPreProcFilters); subFilters.Add("format=bgra"); } @@ -5367,7 +6110,7 @@ namespace MediaBrowser.Controller.MediaEncoding var subFramerate = hasAssSubs ? Math.Min(framerate ?? 25, 60) : 10; // alphasrc=s=1280x720:r=10:start=0,format=bgra,subtitles,hwupload - var alphaSrcFilter = GetAlphaSrcFilter(state, inW, inH, reqW, reqH, reqMaxW, reqMaxH, subFramerate); + var alphaSrcFilter = GetAlphaSrcFilter(state, swpInW, swpInH, reqW, reqH, reqMaxW, subMaxH, subFramerate); var subTextSubtitlesFilter = GetTextSubtitlesFilter(state, true, true); subFilters.Add(alphaSrcFilter); subFilters.Add("format=bgra"); @@ -5376,15 +6119,28 @@ namespace MediaBrowser.Controller.MediaEncoding subFilters.Add("hwupload=derive_device=rkmpp"); + // offload 1080p+ subtitles swscale upscaling from CPU to RGA + var (overlayW, overlayH) = GetFixedOutputSize(swpInW, swpInH, reqW, reqH, reqMaxW, reqMaxH); + if (overlayW.HasValue && overlayH.HasValue && overlayH.Value > subMaxH) + { + subFilters.Add($"vpp_rkrga=w={overlayW.Value}:h={overlayH.Value}:format=bgra:afbc=1"); + } + // try enabling AFBC to save DDR bandwidth - overlayFilters.Add("overlay_rkrga=eof_action=pass:repeatlast=0:format=nv12:afbc=1"); + var hwOverlayFilter = "overlay_rkrga=eof_action=pass:repeatlast=0:format=nv12"; + if (isEncoderSupportAfbc) + { + hwOverlayFilter += ":afbc=1"; + } + + overlayFilters.Add(hwOverlayFilter); } } else if (memoryOutput) { if (hasGraphicalSubs) { - var subPreProcFilters = GetGraphicalSubPreProcessFilters(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + var subPreProcFilters = GetGraphicalSubPreProcessFilters(swpInW, swpInH, subW, subH, reqW, reqH, reqMaxW, reqMaxH); subFilters.Add(subPreProcFilters); overlayFilters.Add("overlay=eof_action=pass:repeatlast=0"); } @@ -5411,7 +6167,7 @@ namespace MediaBrowser.Controller.MediaEncoding return string.Empty; } - var hasSubs = state.SubtitleStream is not null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; + var hasSubs = state.SubtitleStream is not null && ShouldEncodeSubtitle(state); var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream; var hasGraphicalSubs = hasSubs && !state.SubtitleStream.IsTextSubtitleStream; @@ -5419,38 +6175,20 @@ namespace MediaBrowser.Controller.MediaEncoding List<string> subFilters; List<string> overlayFilters; - if (string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase)) - { - (mainFilters, subFilters, overlayFilters) = GetVaapiVidFilterChain(state, options, outputVideoCodec); - } - else if (string.Equals(options.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase)) - { - (mainFilters, subFilters, overlayFilters) = GetIntelVidFilterChain(state, options, outputVideoCodec); - } - else if (string.Equals(options.HardwareAccelerationType, "nvenc", StringComparison.OrdinalIgnoreCase)) - { - (mainFilters, subFilters, overlayFilters) = GetNvidiaVidFilterChain(state, options, outputVideoCodec); - } - else if (string.Equals(options.HardwareAccelerationType, "amf", StringComparison.OrdinalIgnoreCase)) - { - (mainFilters, subFilters, overlayFilters) = GetAmdVidFilterChain(state, options, outputVideoCodec); - } - else if (string.Equals(options.HardwareAccelerationType, "videotoolbox", StringComparison.OrdinalIgnoreCase)) - { - (mainFilters, subFilters, overlayFilters) = GetAppleVidFilterChain(state, options, outputVideoCodec); - } - else if (string.Equals(options.HardwareAccelerationType, "rkmpp", StringComparison.OrdinalIgnoreCase)) + (mainFilters, subFilters, overlayFilters) = options.HardwareAccelerationType switch { - (mainFilters, subFilters, overlayFilters) = GetRkmppVidFilterChain(state, options, outputVideoCodec); - } - else - { - (mainFilters, subFilters, overlayFilters) = GetSwVidFilterChain(state, options, outputVideoCodec); - } + HardwareAccelerationType.vaapi => GetVaapiVidFilterChain(state, options, outputVideoCodec), + HardwareAccelerationType.amf => GetAmdVidFilterChain(state, options, outputVideoCodec), + HardwareAccelerationType.qsv => GetIntelVidFilterChain(state, options, outputVideoCodec), + HardwareAccelerationType.nvenc => GetNvidiaVidFilterChain(state, options, outputVideoCodec), + HardwareAccelerationType.videotoolbox => GetAppleVidFilterChain(state, options, outputVideoCodec), + HardwareAccelerationType.rkmpp => GetRkmppVidFilterChain(state, options, outputVideoCodec), + _ => GetSwVidFilterChain(state, options, outputVideoCodec), + }; - mainFilters?.RemoveAll(filter => string.IsNullOrEmpty(filter)); - subFilters?.RemoveAll(filter => string.IsNullOrEmpty(filter)); - overlayFilters?.RemoveAll(filter => string.IsNullOrEmpty(filter)); + mainFilters?.RemoveAll(string.IsNullOrEmpty); + subFilters?.RemoveAll(string.IsNullOrEmpty); + overlayFilters?.RemoveAll(string.IsNullOrEmpty); var framerate = GetFramerateParam(state); if (framerate.HasValue) @@ -5630,69 +6368,70 @@ namespace MediaBrowser.Controller.MediaEncoding return null; } - if (!string.IsNullOrEmpty(videoStream.Codec) && !string.IsNullOrEmpty(options.HardwareAccelerationType)) + var hardwareAccelerationType = options.HardwareAccelerationType; + + if (!string.IsNullOrEmpty(videoStream.Codec) && hardwareAccelerationType != HardwareAccelerationType.none) { var bitDepth = GetVideoColorBitDepth(state); - // Only HEVC, VP9 and AV1 formats have 10-bit hardware decoder support now. + // Only HEVC, VP9 and AV1 formats have 10-bit hardware decoder support for most platforms if (bitDepth == 10 && !(string.Equals(videoStream.Codec, "hevc", StringComparison.OrdinalIgnoreCase) || string.Equals(videoStream.Codec, "h265", StringComparison.OrdinalIgnoreCase) || string.Equals(videoStream.Codec, "vp9", StringComparison.OrdinalIgnoreCase) || string.Equals(videoStream.Codec, "av1", StringComparison.OrdinalIgnoreCase))) { - // One exception is that RKMPP decoder can handle H.264 High 10. - if (!(string.Equals(options.HardwareAccelerationType, "rkmpp", StringComparison.OrdinalIgnoreCase) - && string.Equals(videoStream.Codec, "h264", StringComparison.OrdinalIgnoreCase))) + // RKMPP has H.264 Hi10P decoder + bool hasHardwareHi10P = hardwareAccelerationType == HardwareAccelerationType.rkmpp; + + // VideoToolbox on Apple Silicon has H.264 Hi10P mode enabled after macOS 14.6 + if (hardwareAccelerationType == HardwareAccelerationType.videotoolbox) { - return null; + var ver = Environment.OSVersion.Version; + var arch = RuntimeInformation.OSArchitecture; + if (arch.Equals(Architecture.Arm64) && ver >= new Version(14, 6)) + { + hasHardwareHi10P = true; + } } - } - - if (string.Equals(options.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase)) - { - return GetQsvHwVidDecoder(state, options, videoStream, bitDepth); - } - if (string.Equals(options.HardwareAccelerationType, "nvenc", StringComparison.OrdinalIgnoreCase)) - { - return GetNvdecVidDecoder(state, options, videoStream, bitDepth); - } - - if (string.Equals(options.HardwareAccelerationType, "amf", StringComparison.OrdinalIgnoreCase)) - { - return GetAmfVidDecoder(state, options, videoStream, bitDepth); + if (!hasHardwareHi10P + && string.Equals(videoStream.Codec, "h264", StringComparison.OrdinalIgnoreCase)) + { + return null; + } } - if (string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase)) + // Block unsupported H.264 Hi422P and Hi444PP profiles, which can be encoded with 4:2:0 pixel format + if (string.Equals(videoStream.Codec, "h264", StringComparison.OrdinalIgnoreCase) + && ((videoStream.Profile?.Contains("4:2:2", StringComparison.OrdinalIgnoreCase) ?? false) + || (videoStream.Profile?.Contains("4:4:4", StringComparison.OrdinalIgnoreCase) ?? false))) { - return GetVaapiVidDecoder(state, options, videoStream, bitDepth); + // VideoToolbox on Apple Silicon has H.264 Hi444PP and theoretically also has Hi422P + if (!(hardwareAccelerationType == HardwareAccelerationType.videotoolbox + && RuntimeInformation.OSArchitecture.Equals(Architecture.Arm64))) + { + return null; + } } - if (string.Equals(options.HardwareAccelerationType, "videotoolbox", StringComparison.OrdinalIgnoreCase)) + var decoder = hardwareAccelerationType switch { - return GetVideotoolboxVidDecoder(state, options, videoStream, bitDepth); - } + HardwareAccelerationType.vaapi => GetVaapiVidDecoder(state, options, videoStream, bitDepth), + HardwareAccelerationType.amf => GetAmfVidDecoder(state, options, videoStream, bitDepth), + HardwareAccelerationType.qsv => GetQsvHwVidDecoder(state, options, videoStream, bitDepth), + HardwareAccelerationType.nvenc => GetNvdecVidDecoder(state, options, videoStream, bitDepth), + HardwareAccelerationType.videotoolbox => GetVideotoolboxVidDecoder(state, options, videoStream, bitDepth), + HardwareAccelerationType.rkmpp => GetRkmppVidDecoder(state, options, videoStream, bitDepth), + _ => string.Empty + }; - if (string.Equals(options.HardwareAccelerationType, "rkmpp", StringComparison.OrdinalIgnoreCase)) + if (!string.IsNullOrEmpty(decoder)) { - return GetRkmppVidDecoder(state, options, videoStream, bitDepth); + return decoder; } } - var whichCodec = videoStream.Codec; - if (string.Equals(whichCodec, "avc", StringComparison.OrdinalIgnoreCase)) - { - whichCodec = "h264"; - } - else if (string.Equals(whichCodec, "h265", StringComparison.OrdinalIgnoreCase)) - { - whichCodec = "hevc"; - } - - // Avoid a second attempt if no hardware acceleration is being used - options.HardwareDecodingCodecs = Array.FindAll(options.HardwareDecodingCodecs, val => !string.Equals(val, whichCodec, StringComparison.OrdinalIgnoreCase)); - // leave blank so ffmpeg will decide return null; } @@ -5716,7 +6455,11 @@ namespace MediaBrowser.Controller.MediaEncoding var decoderName = decoderPrefix + '_' + decoderSuffix; var isCodecAvailable = _mediaEncoder.SupportsDecoder(decoderName) && options.HardwareDecodingCodecs.Contains(videoCodec, StringComparison.OrdinalIgnoreCase); - if (bitDepth == 10 && isCodecAvailable) + + // VideoToolbox decoders have built-in SW fallback + if (bitDepth == 10 + && isCodecAvailable + && (options.HardwareAccelerationType != HardwareAccelerationType.videotoolbox)) { if (string.Equals(videoCodec, "hevc", StringComparison.OrdinalIgnoreCase) && options.HardwareDecodingCodecs.Contains("hevc", StringComparison.OrdinalIgnoreCase) @@ -5772,11 +6515,12 @@ namespace MediaBrowser.Controller.MediaEncoding var isVideotoolboxSupported = isMacOS && _mediaEncoder.SupportsHwaccel("videotoolbox"); var isRkmppSupported = isLinux && IsRkmppFullSupported(); var isCodecAvailable = options.HardwareDecodingCodecs.Contains(videoCodec, StringComparison.OrdinalIgnoreCase); + var hardwareAccelerationType = options.HardwareAccelerationType; var ffmpegVersion = _mediaEncoder.EncoderVersion; // Set the av1 codec explicitly to trigger hw accelerator, otherwise libdav1d will be used. - var isAv1 = ffmpegVersion < _minFFmpegImplictHwaccel + var isAv1 = ffmpegVersion < _minFFmpegImplicitHwaccel && string.Equals(videoCodec, "av1", StringComparison.OrdinalIgnoreCase); // Allow profile mismatch if decoding H.264 baseline with d3d11va and vaapi hwaccels. @@ -5786,17 +6530,45 @@ namespace MediaBrowser.Controller.MediaEncoding // Disable the extra internal copy in nvdec. We already handle it in filter chain. var nvdecNoInternalCopy = ffmpegVersion >= _minFFmpegHwaUnsafeOutput; - if (bitDepth == 10 && isCodecAvailable) + // Strip the display rotation side data from the transposed fmp4 output stream. + var stripRotationData = (state.VideoStream?.Rotation ?? 0) != 0 + && ffmpegVersion >= _minFFmpegDisplayRotationOption; + var stripRotationDataArgs = stripRotationData ? " -display_rotation 0" : string.Empty; + + // VideoToolbox decoders have built-in SW fallback + if (isCodecAvailable + && (options.HardwareAccelerationType != HardwareAccelerationType.videotoolbox)) { if (string.Equals(videoCodec, "hevc", StringComparison.OrdinalIgnoreCase) - && options.HardwareDecodingCodecs.Contains("hevc", StringComparison.OrdinalIgnoreCase) - && !options.EnableDecodingColorDepth10Hevc) + && options.HardwareDecodingCodecs.Contains("hevc", StringComparison.OrdinalIgnoreCase)) { - return null; + if (IsVideoStreamHevcRext(state)) + { + if (bitDepth <= 10 && !options.EnableDecodingColorDepth10HevcRext) + { + return null; + } + + if (bitDepth == 12 && !options.EnableDecodingColorDepth12HevcRext) + { + return null; + } + + if (hardwareAccelerationType == HardwareAccelerationType.vaapi + && !_mediaEncoder.IsVaapiDeviceInteliHD) + { + return null; + } + } + else if (bitDepth == 10 && !options.EnableDecodingColorDepth10Hevc) + { + return null; + } } if (string.Equals(videoCodec, "vp9", StringComparison.OrdinalIgnoreCase) && options.HardwareDecodingCodecs.Contains("vp9", StringComparison.OrdinalIgnoreCase) + && bitDepth == 10 && !options.EnableDecodingColorDepth10Vp9) { return null; @@ -5804,19 +6576,19 @@ namespace MediaBrowser.Controller.MediaEncoding } // Intel qsv/d3d11va/vaapi - if (string.Equals(options.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase)) + if (hardwareAccelerationType == HardwareAccelerationType.qsv) { if (options.PreferSystemNativeHwDecoder) { if (isVaapiSupported && isCodecAvailable) { - return " -hwaccel vaapi" + (outputHwSurface ? " -hwaccel_output_format vaapi" : string.Empty) + return " -hwaccel vaapi" + (outputHwSurface ? " -hwaccel_output_format vaapi -noautorotate" + stripRotationDataArgs : string.Empty) + (profileMismatch ? " -hwaccel_flags +allow_profile_mismatch" : string.Empty) + (isAv1 ? " -c:v av1" : string.Empty); } if (isD3d11Supported && isCodecAvailable) { - return " -hwaccel d3d11va" + (outputHwSurface ? " -hwaccel_output_format d3d11" : string.Empty) + return " -hwaccel d3d11va" + (outputHwSurface ? " -hwaccel_output_format d3d11 -noautorotate" + stripRotationDataArgs : string.Empty) + (profileMismatch ? " -hwaccel_flags +allow_profile_mismatch" : string.Empty) + " -threads 2" + (isAv1 ? " -c:v av1" : string.Empty); } } @@ -5824,61 +6596,61 @@ namespace MediaBrowser.Controller.MediaEncoding { if (isQsvSupported && isCodecAvailable) { - return " -hwaccel qsv" + (outputHwSurface ? " -hwaccel_output_format qsv" : string.Empty); + return " -hwaccel qsv" + (outputHwSurface ? " -hwaccel_output_format qsv -noautorotate" + stripRotationDataArgs : string.Empty); } } } // Nvidia cuda - if (string.Equals(options.HardwareAccelerationType, "nvenc", StringComparison.OrdinalIgnoreCase)) + if (hardwareAccelerationType == HardwareAccelerationType.nvenc) { if (isCudaSupported && isCodecAvailable) { if (options.EnableEnhancedNvdecDecoder) { // set -threads 1 to nvdec decoder explicitly since it doesn't implement threading support. - return " -hwaccel cuda" + (outputHwSurface ? " -hwaccel_output_format cuda" : string.Empty) + return " -hwaccel cuda" + (outputHwSurface ? " -hwaccel_output_format cuda -noautorotate" + stripRotationDataArgs : string.Empty) + (nvdecNoInternalCopy ? " -hwaccel_flags +unsafe_output" : string.Empty) + " -threads 1" + (isAv1 ? " -c:v av1" : string.Empty); } // cuvid decoder doesn't have threading issue. - return " -hwaccel cuda" + (outputHwSurface ? " -hwaccel_output_format cuda" : string.Empty); + return " -hwaccel cuda" + (outputHwSurface ? " -hwaccel_output_format cuda -noautorotate" + stripRotationDataArgs : string.Empty); } } // Amd d3d11va - if (string.Equals(options.HardwareAccelerationType, "amf", StringComparison.OrdinalIgnoreCase)) + if (hardwareAccelerationType == HardwareAccelerationType.amf) { if (isD3d11Supported && isCodecAvailable) { - return " -hwaccel d3d11va" + (outputHwSurface ? " -hwaccel_output_format d3d11" : string.Empty) - + (profileMismatch ? " -hwaccel_flags +allow_profile_mismatch" : string.Empty) + (isAv1 ? " -c:v av1" : string.Empty); + return " -hwaccel d3d11va" + (outputHwSurface ? " -hwaccel_output_format d3d11 -noautorotate" + stripRotationDataArgs : string.Empty) + + (profileMismatch ? " -hwaccel_flags +allow_profile_mismatch" : string.Empty) + " -threads 2" + (isAv1 ? " -c:v av1" : string.Empty); } } // Vaapi - if (string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase) + if (hardwareAccelerationType == HardwareAccelerationType.vaapi && isVaapiSupported && isCodecAvailable) { - return " -hwaccel vaapi" + (outputHwSurface ? " -hwaccel_output_format vaapi" : string.Empty) + return " -hwaccel vaapi" + (outputHwSurface ? " -hwaccel_output_format vaapi -noautorotate" + stripRotationDataArgs : string.Empty) + (profileMismatch ? " -hwaccel_flags +allow_profile_mismatch" : string.Empty) + (isAv1 ? " -c:v av1" : string.Empty); } // Apple videotoolbox - if (string.Equals(options.HardwareAccelerationType, "videotoolbox", StringComparison.OrdinalIgnoreCase) + if (hardwareAccelerationType == HardwareAccelerationType.videotoolbox && isVideotoolboxSupported && isCodecAvailable) { - return " -hwaccel videotoolbox" + (outputHwSurface ? " -hwaccel_output_format videotoolbox_vld" : string.Empty); + return " -hwaccel videotoolbox" + (outputHwSurface ? " -hwaccel_output_format videotoolbox_vld" : string.Empty) + " -noautorotate" + stripRotationDataArgs; } // Rockchip rkmpp - if (string.Equals(options.HardwareAccelerationType, "rkmpp", StringComparison.OrdinalIgnoreCase) + if (hardwareAccelerationType == HardwareAccelerationType.rkmpp && isRkmppSupported && isCodecAvailable) { - return " -hwaccel rkmpp" + (outputHwSurface ? " -hwaccel_output_format drm_prime" : string.Empty); + return " -hwaccel rkmpp" + (outputHwSurface ? " -hwaccel_output_format drm_prime -noautorotate" + stripRotationDataArgs : string.Empty); } return null; @@ -5890,7 +6662,7 @@ namespace MediaBrowser.Controller.MediaEncoding var isLinux = OperatingSystem.IsLinux(); if ((!isWindows && !isLinux) - || !string.Equals(options.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase)) + || options.HardwareAccelerationType != HardwareAccelerationType.qsv) { return null; } @@ -5908,6 +6680,14 @@ namespace MediaBrowser.Controller.MediaEncoding var is8bitSwFormatsQsv = string.Equals("yuv420p", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase) || string.Equals("yuvj420p", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); var is8_10bitSwFormatsQsv = is8bitSwFormatsQsv || string.Equals("yuv420p10le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); + var is8_10_12bitSwFormatsQsv = is8_10bitSwFormatsQsv + || string.Equals("yuv422p", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase) + || string.Equals("yuv444p", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase) + || string.Equals("yuv422p10le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase) + || string.Equals("yuv444p10le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase) + || string.Equals("yuv420p12le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase) + || string.Equals("yuv422p12le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase) + || string.Equals("yuv444p12le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); // TODO: add more 8/10bit and 4:4:4 formats for Qsv after finishing the ffcheck tool if (is8bitSwFormatsQsv) @@ -5936,12 +6716,6 @@ namespace MediaBrowser.Controller.MediaEncoding if (is8_10bitSwFormatsQsv) { - if (string.Equals(videoStream.Codec, "hevc", StringComparison.OrdinalIgnoreCase) - || string.Equals(videoStream.Codec, "h265", StringComparison.OrdinalIgnoreCase)) - { - return GetHwaccelType(state, options, "hevc", bitDepth, hwSurface) + GetHwDecoderName(options, "hevc", "qsv", "hevc", bitDepth); - } - if (string.Equals(videoStream.Codec, "vp9", StringComparison.OrdinalIgnoreCase)) { return GetHwaccelType(state, options, "vp9", bitDepth, hwSurface) + GetHwDecoderName(options, "vp9", "qsv", "vp9", bitDepth); @@ -5953,13 +6727,22 @@ namespace MediaBrowser.Controller.MediaEncoding } } + if (is8_10_12bitSwFormatsQsv) + { + if (string.Equals(videoStream.Codec, "hevc", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoStream.Codec, "h265", StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "hevc", bitDepth, hwSurface) + GetHwDecoderName(options, "hevc", "qsv", "hevc", bitDepth); + } + } + return null; } public string GetNvdecVidDecoder(EncodingJobInfo state, EncodingOptions options, MediaStream videoStream, int bitDepth) { if ((!OperatingSystem.IsWindows() && !OperatingSystem.IsLinux()) - || !string.Equals(options.HardwareAccelerationType, "nvenc", StringComparison.OrdinalIgnoreCase)) + || options.HardwareAccelerationType != HardwareAccelerationType.nvenc) { return null; } @@ -5968,6 +6751,11 @@ namespace MediaBrowser.Controller.MediaEncoding var is8bitSwFormatsNvdec = string.Equals("yuv420p", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase) || string.Equals("yuvj420p", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); var is8_10bitSwFormatsNvdec = is8bitSwFormatsNvdec || string.Equals("yuv420p10le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); + var is8_10_12bitSwFormatsNvdec = is8_10bitSwFormatsNvdec + || string.Equals("yuv444p", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase) + || string.Equals("yuv444p10le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase) + || string.Equals("yuv420p12le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase) + || string.Equals("yuv444p12le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); // TODO: add more 8/10/12bit and 4:4:4 formats for Nvdec after finishing the ffcheck tool if (is8bitSwFormatsNvdec) @@ -6001,12 +6789,6 @@ namespace MediaBrowser.Controller.MediaEncoding if (is8_10bitSwFormatsNvdec) { - if (string.Equals("hevc", videoStream.Codec, StringComparison.OrdinalIgnoreCase) - || string.Equals("h265", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) - { - return GetHwaccelType(state, options, "hevc", bitDepth, hwSurface) + GetHwDecoderName(options, "hevc", "cuvid", "hevc", bitDepth); - } - if (string.Equals("vp9", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) { return GetHwaccelType(state, options, "vp9", bitDepth, hwSurface) + GetHwDecoderName(options, "vp9", "cuvid", "vp9", bitDepth); @@ -6018,13 +6800,22 @@ namespace MediaBrowser.Controller.MediaEncoding } } + if (is8_10_12bitSwFormatsNvdec) + { + if (string.Equals("hevc", videoStream.Codec, StringComparison.OrdinalIgnoreCase) + || string.Equals("h265", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "hevc", bitDepth, hwSurface) + GetHwDecoderName(options, "hevc", "cuvid", "hevc", bitDepth); + } + } + return null; } public string GetAmfVidDecoder(EncodingJobInfo state, EncodingOptions options, MediaStream videoStream, int bitDepth) { if (!OperatingSystem.IsWindows() - || !string.Equals(options.HardwareAccelerationType, "amf", StringComparison.OrdinalIgnoreCase)) + || options.HardwareAccelerationType != HardwareAccelerationType.amf) { return null; } @@ -6080,7 +6871,7 @@ namespace MediaBrowser.Controller.MediaEncoding public string GetVaapiVidDecoder(EncodingJobInfo state, EncodingOptions options, MediaStream videoStream, int bitDepth) { if (!OperatingSystem.IsLinux() - || !string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase)) + || options.HardwareAccelerationType != HardwareAccelerationType.vaapi) { return null; } @@ -6092,6 +6883,14 @@ namespace MediaBrowser.Controller.MediaEncoding var is8bitSwFormatsVaapi = string.Equals("yuv420p", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase) || string.Equals("yuvj420p", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); var is8_10bitSwFormatsVaapi = is8bitSwFormatsVaapi || string.Equals("yuv420p10le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); + var is8_10_12bitSwFormatsVaapi = is8_10bitSwFormatsVaapi + || string.Equals("yuv422p", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase) + || string.Equals("yuv444p", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase) + || string.Equals("yuv422p10le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase) + || string.Equals("yuv444p10le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase) + || string.Equals("yuv420p12le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase) + || string.Equals("yuv422p12le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase) + || string.Equals("yuv444p12le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); if (is8bitSwFormatsVaapi) { @@ -6119,12 +6918,6 @@ namespace MediaBrowser.Controller.MediaEncoding if (is8_10bitSwFormatsVaapi) { - if (string.Equals("hevc", videoStream.Codec, StringComparison.OrdinalIgnoreCase) - || string.Equals("h265", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) - { - return GetHwaccelType(state, options, "hevc", bitDepth, hwSurface); - } - if (string.Equals("vp9", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) { return GetHwaccelType(state, options, "vp9", bitDepth, hwSurface); @@ -6136,13 +6929,22 @@ namespace MediaBrowser.Controller.MediaEncoding } } + if (is8_10_12bitSwFormatsVaapi) + { + if (string.Equals("hevc", videoStream.Codec, StringComparison.OrdinalIgnoreCase) + || string.Equals("h265", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "hevc", bitDepth, hwSurface); + } + } + return null; } public string GetVideotoolboxVidDecoder(EncodingJobInfo state, EncodingOptions options, MediaStream videoStream, int bitDepth) { if (!OperatingSystem.IsMacOS() - || !string.Equals(options.HardwareAccelerationType, "videotoolbox", StringComparison.OrdinalIgnoreCase)) + || options.HardwareAccelerationType != HardwareAccelerationType.videotoolbox) { return null; } @@ -6150,37 +6952,54 @@ namespace MediaBrowser.Controller.MediaEncoding var is8bitSwFormatsVt = string.Equals("yuv420p", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase) || string.Equals("yuvj420p", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); var is8_10bitSwFormatsVt = is8bitSwFormatsVt || string.Equals("yuv420p10le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); - - // VideoToolbox's Hardware surface in ffmpeg is not only slower than hwupload, but also breaks HDR in many cases. - // For example: https://trac.ffmpeg.org/ticket/10884 - // Disable it for now. - const bool UseHwSurface = false; + var is8_10_12bitSwFormatsVt = is8_10bitSwFormatsVt + || string.Equals("yuv422p", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase) + || string.Equals("yuv444p", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase) + || string.Equals("yuv422p10le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase) + || string.Equals("yuv444p10le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase) + || string.Equals("yuv420p12le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase) + || string.Equals("yuv422p12le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase) + || string.Equals("yuv444p12le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); + var isAv1SupportedSwFormatsVt = is8_10bitSwFormatsVt || string.Equals("yuv420p12le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); + + // The related patches make videotoolbox hardware surface working is only available in jellyfin-ffmpeg 7.0.1 at the moment. + bool useHwSurface = (_mediaEncoder.EncoderVersion >= _minFFmpegWorkingVtHwSurface) && IsVideoToolboxFullSupported(); if (is8bitSwFormatsVt) { + if (string.Equals("vp8", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "vp8", bitDepth, useHwSurface); + } + } + + if (is8_10bitSwFormatsVt) + { if (string.Equals("avc", videoStream.Codec, StringComparison.OrdinalIgnoreCase) || string.Equals("h264", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) { - return GetHwaccelType(state, options, "h264", bitDepth, UseHwSurface); + return GetHwaccelType(state, options, "h264", bitDepth, useHwSurface); } - if (string.Equals("vp8", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + if (string.Equals("vp9", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) { - return GetHwaccelType(state, options, "vp8", bitDepth, UseHwSurface); + return GetHwaccelType(state, options, "vp9", bitDepth, useHwSurface); } } - if (is8_10bitSwFormatsVt) + if (is8_10_12bitSwFormatsVt) { if (string.Equals("hevc", videoStream.Codec, StringComparison.OrdinalIgnoreCase) || string.Equals("h265", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) { - return GetHwaccelType(state, options, "hevc", bitDepth, UseHwSurface); + return GetHwaccelType(state, options, "hevc", bitDepth, useHwSurface); } - if (string.Equals("vp9", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + if (string.Equals("av1", videoStream.Codec, StringComparison.OrdinalIgnoreCase) + && isAv1SupportedSwFormatsVt + && _mediaEncoder.IsVideoToolboxAv1DecodeAvailable) { - return GetHwaccelType(state, options, "vp9", bitDepth, UseHwSurface); + return GetHwaccelType(state, options, "av1", bitDepth, useHwSurface); } } @@ -6192,7 +7011,7 @@ namespace MediaBrowser.Controller.MediaEncoding var isLinux = OperatingSystem.IsLinux(); if (!isLinux - || !string.Equals(options.HardwareAccelerationType, "rkmpp", StringComparison.OrdinalIgnoreCase)) + || options.HardwareAccelerationType != HardwareAccelerationType.rkmpp) { return null; } @@ -6276,6 +7095,7 @@ namespace MediaBrowser.Controller.MediaEncoding if (string.Equals(videoStream.Codec, "av1", StringComparison.OrdinalIgnoreCase)) { + // there's an issue about AV1 AFBC on RK3588, disable it for now until it's fixed upstream return GetHwaccelType(state, options, "av1", bitDepth, hwSurface); } } @@ -6293,28 +7113,19 @@ namespace MediaBrowser.Controller.MediaEncoding #nullable enable public static int GetNumberOfThreads(EncodingJobInfo? state, EncodingOptions encodingOptions, string? outputVideoCodec) { - // VP8 and VP9 encoders must have their thread counts set. - bool mustSetThreadCount = string.Equals(outputVideoCodec, "libvpx", StringComparison.OrdinalIgnoreCase) - || string.Equals(outputVideoCodec, "libvpx-vp9", StringComparison.OrdinalIgnoreCase); - var threads = state?.BaseRequest.CpuCoreLimit ?? encodingOptions.EncodingThreadCount; if (threads <= 0) { // Automatically set thread count - return mustSetThreadCount ? Math.Max(Environment.ProcessorCount - 1, 1) : 0; - } - - if (threads >= Environment.ProcessorCount) - { - return Environment.ProcessorCount; + return 0; } - return threads; + return Math.Min(threads, Environment.ProcessorCount); } #nullable disable - public void TryStreamCopy(EncodingJobInfo state) + public void TryStreamCopy(EncodingJobInfo state, EncodingOptions options) { if (state.VideoStream is not null && CanStreamCopyVideo(state, state.VideoStream)) { @@ -6331,8 +7142,14 @@ namespace MediaBrowser.Controller.MediaEncoding } } + var preventHlsAudioCopy = state.TranscodingType is TranscodingJobType.Hls + && state.VideoStream is not null + && !IsCopyCodec(state.OutputVideoCodec) + && options.HlsAudioSeekStrategy is HlsAudioSeekStrategy.TranscodeAudio; + if (state.AudioStream is not null - && CanStreamCopyAudio(state, state.AudioStream, state.SupportedAudioCodecs)) + && CanStreamCopyAudio(state, state.AudioStream, state.SupportedAudioCodecs) + && !preventHlsAudioCopy) { state.OutputAudioCodec = "copy"; } @@ -6348,9 +7165,8 @@ namespace MediaBrowser.Controller.MediaEncoding } } - public string GetInputModifier(EncodingJobInfo state, EncodingOptions encodingOptions, string segmentContainer) + private string GetFfmpegAnalyzeDurationArg(EncodingJobInfo state) { - var inputModifier = string.Empty; var analyzeDurationArgument = string.Empty; // Apply -analyzeduration as per the environment variable, @@ -6366,6 +7182,26 @@ namespace MediaBrowser.Controller.MediaEncoding analyzeDurationArgument = "-analyzeduration " + ffmpegAnalyzeDuration; } + return analyzeDurationArgument; + } + + private string GetFfmpegProbesizeArg() + { + var ffmpegProbeSize = _config.GetFFmpegProbeSize(); + + if (!string.IsNullOrEmpty(ffmpegProbeSize)) + { + return $"-probesize {ffmpegProbeSize}"; + } + + return string.Empty; + } + + public string GetInputModifier(EncodingJobInfo state, EncodingOptions encodingOptions, string segmentContainer) + { + var inputModifier = string.Empty; + var analyzeDurationArgument = GetFfmpegAnalyzeDurationArg(state); + if (!string.IsNullOrEmpty(analyzeDurationArgument)) { inputModifier += " " + analyzeDurationArgument; @@ -6374,11 +7210,11 @@ namespace MediaBrowser.Controller.MediaEncoding inputModifier = inputModifier.Trim(); // Apply -probesize if configured - var ffmpegProbeSize = _config.GetFFmpegProbeSize(); + var ffmpegProbeSizeArgument = GetFfmpegProbesizeArg(); - if (!string.IsNullOrEmpty(ffmpegProbeSize)) + if (!string.IsNullOrEmpty(ffmpegProbeSizeArgument)) { - inputModifier += $" -probesize {ffmpegProbeSize}"; + inputModifier += " " + ffmpegProbeSizeArgument; } var userAgentParam = GetUserAgentParam(state); @@ -6412,13 +7248,16 @@ namespace MediaBrowser.Controller.MediaEncoding inputModifier += " -async " + state.InputAudioSync; } - if (!string.IsNullOrEmpty(state.InputVideoSync)) + // The -fps_mode option cannot be applied to input + if (!string.IsNullOrEmpty(state.InputVideoSync) && _mediaEncoder.EncoderVersion < new Version(5, 1)) { - inputModifier += " -vsync " + state.InputVideoSync; + inputModifier += GetVideoSyncOption(state.InputVideoSync, _mediaEncoder.EncoderVersion); } + int readrate = 0; if (state.ReadInputAtNativeFramerate && state.InputProtocol != MediaProtocol.Rtsp) { + readrate = 1; inputModifier += " -re"; } else if (encodingOptions.EnableSegmentDeletion @@ -6429,7 +7268,15 @@ namespace MediaBrowser.Controller.MediaEncoding { // Set an input read rate limit 10x for using SegmentDeletion with stream-copy // to prevent ffmpeg from exiting prematurely (due to fast drive) - inputModifier += " -readrate 10"; + readrate = 10; + inputModifier += $" -readrate {readrate}"; + } + + // Set a larger catchup value to revert to the old behavior, + // otherwise, remuxing might stall due to this new option + if (readrate > 0 && _mediaEncoder.EncoderVersion >= _minFFmpegReadrateCatchupOption) + { + inputModifier += $" -readrate_catchup {readrate * 100}"; } var flags = new List<string>(); @@ -6465,7 +7312,7 @@ namespace MediaBrowser.Controller.MediaEncoding if (state.IsVideoRequest) { - if (!string.IsNullOrEmpty(state.InputContainer) && state.VideoType == VideoType.VideoFile && string.IsNullOrEmpty(encodingOptions.HardwareAccelerationType)) + if (!string.IsNullOrEmpty(state.InputContainer) && state.VideoType == VideoType.VideoFile && encodingOptions.HardwareAccelerationType != HardwareAccelerationType.none) { var inputFormat = GetInputFormat(state.InputContainer); if (!string.IsNullOrEmpty(inputFormat)) @@ -6519,7 +7366,7 @@ namespace MediaBrowser.Controller.MediaEncoding state.RemoteHttpHeaders = mediaSource.RequiredHttpHeaders; state.ReadInputAtNativeFramerate = mediaSource.ReadAtNativeFramerate; - if (state.ReadInputAtNativeFramerate + if ((state.ReadInputAtNativeFramerate && !state.IsSegmentedLiveStream) || (mediaSource.Protocol == MediaProtocol.File && string.Equals(mediaSource.Container, "wtv", StringComparison.OrdinalIgnoreCase))) { @@ -6581,7 +7428,7 @@ namespace MediaBrowser.Controller.MediaEncoding state.SupportedAudioCodecs = supportedAudioCodecsList.ToArray(); - request.AudioCodec = state.SupportedAudioCodecs.FirstOrDefault(i => _mediaEncoder.CanEncodeToAudioCodec(i)) + request.AudioCodec = state.SupportedAudioCodecs.FirstOrDefault(_mediaEncoder.CanEncodeToAudioCodec) ?? state.SupportedAudioCodecs.FirstOrDefault(); } @@ -6612,7 +7459,7 @@ namespace MediaBrowser.Controller.MediaEncoding { // DTS and TrueHD are not supported by HLS // Keep them in the supported codecs list, but shift them to the end of the list so that if transcoding happens, another codec is used - shiftAudioCodecs.Add("dca"); + shiftAudioCodecs.Add("dts"); shiftAudioCodecs.Add("truehd"); } else @@ -6630,7 +7477,7 @@ namespace MediaBrowser.Controller.MediaEncoding while (shiftAudioCodecs.Contains(audioCodecs[0], StringComparison.OrdinalIgnoreCase)) { - var removed = shiftAudioCodecs[0]; + var removed = audioCodecs[0]; audioCodecs.RemoveAt(0); audioCodecs.Add(removed); } @@ -6664,7 +7511,7 @@ namespace MediaBrowser.Controller.MediaEncoding while (shiftVideoCodecs.Contains(videoCodecs[0], StringComparison.OrdinalIgnoreCase)) { - var removed = shiftVideoCodecs[0]; + var removed = videoCodecs[0]; videoCodecs.RemoveAt(0); videoCodecs.Add(removed); } @@ -6707,7 +7554,7 @@ namespace MediaBrowser.Controller.MediaEncoding return " -codec:s:0 " + codec + " -disposition:s:0 default"; } - public string GetProgressiveVideoFullCommandLine(EncodingJobInfo state, EncodingOptions encodingOptions, string defaultPreset) + public string GetProgressiveVideoFullCommandLine(EncodingJobInfo state, EncodingOptions encodingOptions, EncoderPreset defaultPreset) { // Get the output codec name var videoCodec = GetVideoEncoder(state, encodingOptions); @@ -6758,7 +7605,7 @@ namespace MediaBrowser.Controller.MediaEncoding return string.Empty; } - public string GetProgressiveVideoArguments(EncodingJobInfo state, EncodingOptions encodingOptions, string videoCodec, string defaultPreset) + public string GetProgressiveVideoArguments(EncodingJobInfo state, EncodingOptions encodingOptions, string videoCodec, EncoderPreset defaultPreset) { var args = "-codec:v:0 " + videoCodec; @@ -6773,7 +7620,7 @@ namespace MediaBrowser.Controller.MediaEncoding && string.Equals(state.OutputContainer, "ts", StringComparison.OrdinalIgnoreCase) && !string.Equals(state.VideoStream.NalLengthSize, "0", StringComparison.OrdinalIgnoreCase)) { - string bitStreamArgs = GetBitStreamArgs(state.VideoStream); + string bitStreamArgs = GetBitStreamArgs(state, MediaStreamType.Video); if (!string.IsNullOrEmpty(bitStreamArgs)) { args += " " + bitStreamArgs; @@ -6799,7 +7646,7 @@ namespace MediaBrowser.Controller.MediaEncoding args += keyFrameArg; - var hasGraphicalSubs = state.SubtitleStream is not null && !state.SubtitleStream.IsTextSubtitleStream && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; + var hasGraphicalSubs = state.SubtitleStream is not null && !state.SubtitleStream.IsTextSubtitleStream && ShouldEncodeSubtitle(state); var hasCopyTs = false; @@ -6837,7 +7684,7 @@ namespace MediaBrowser.Controller.MediaEncoding if (!string.IsNullOrEmpty(state.OutputVideoSync)) { - args += " -vsync " + state.OutputVideoSync; + args += GetVideoSyncOption(state.OutputVideoSync, _mediaEncoder.EncoderVersion); } args += GetOutputFFlags(state); @@ -6865,7 +7712,10 @@ namespace MediaBrowser.Controller.MediaEncoding var channels = state.OutputAudioChannels; - if (channels.HasValue && ((channels.Value != 2 && state.AudioStream.Channels <= 5) || encodingOptions.DownMixStereoAlgorithm == DownMixStereoAlgorithms.None)) + var useDownMixAlgorithm = state.AudioStream is not null + && DownMixAlgorithmsHelper.AlgorithmFilterStrings.ContainsKey((encodingOptions.DownMixStereoAlgorithm, DownMixAlgorithmsHelper.InferChannelLayout(state.AudioStream))); + + if (channels.HasValue && !useDownMixAlgorithm) { args += " -ac " + channels.Value; } @@ -6873,8 +7723,8 @@ namespace MediaBrowser.Controller.MediaEncoding var bitrate = state.OutputAudioBitrate; if (bitrate.HasValue && !LosslessAudioCodecs.Contains(codec, StringComparison.OrdinalIgnoreCase)) { - var vbrParam = GetAudioVbrModeParam(codec, bitrate.Value / (channels ?? 2)); - if (encodingOptions.EnableAudioVbr && vbrParam is not null) + var vbrParam = GetAudioVbrModeParam(codec, bitrate.Value, channels ?? 2); + if (encodingOptions.EnableAudioVbr && state.EnableAudioVbrEncoding && vbrParam is not null) { args += vbrParam; } @@ -6904,8 +7754,8 @@ namespace MediaBrowser.Controller.MediaEncoding if (bitrate.HasValue && !LosslessAudioCodecs.Contains(outputCodec, StringComparison.OrdinalIgnoreCase)) { - var vbrParam = GetAudioVbrModeParam(GetAudioEncoder(state), bitrate.Value / (channels ?? 2)); - if (encodingOptions.EnableAudioVbr && vbrParam is not null) + var vbrParam = GetAudioVbrModeParam(GetAudioEncoder(state), bitrate.Value, channels ?? 2); + if (encodingOptions.EnableAudioVbr && state.EnableAudioVbrEncoding && vbrParam is not null) { audioTranscodeParams.Add(vbrParam); } @@ -7001,5 +7851,39 @@ namespace MediaBrowser.Controller.MediaEncoding { return string.Equals(codec, "copy", StringComparison.OrdinalIgnoreCase); } + + private static bool ShouldEncodeSubtitle(EncodingJobInfo state) + { + return state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode + || (state.BaseRequest.AlwaysBurnInSubtitleWhenTranscoding && !IsCopyCodec(state.OutputVideoCodec)); + } + + public static string GetVideoSyncOption(string videoSync, Version encoderVersion) + { + if (string.IsNullOrEmpty(videoSync)) + { + return string.Empty; + } + + if (encoderVersion >= new Version(5, 1)) + { + if (int.TryParse(videoSync, CultureInfo.InvariantCulture, out var vsync)) + { + return vsync switch + { + -1 => " -fps_mode auto", + 0 => " -fps_mode passthrough", + 1 => " -fps_mode cfr", + 2 => " -fps_mode vfr", + _ => string.Empty + }; + } + + return string.Empty; + } + + // -vsync is deprecated in FFmpeg 5.1 and will be removed in the future. + return $" -vsync {videoSync}"; + } } } |