Merge branch 'master' into fix-hwa-video-rotation

2 files changed, 101 insertions, 32 deletions

diff --git a/MediaBrowser.Controller/MediaEncoding/DownMixAlgorithmsHelper.cs b/MediaBrowser.Controller/MediaEncoding/DownMixAlgorithmsHelper.cs
new file mode 100644
index 000000000..749f87271
--- /dev/null
+++ b/MediaBrowser.Controller/MediaEncoding/DownMixAlgorithmsHelper.cs
@@ -0,0 +1,74 @@
+using System.Collections.Generic;
+using MediaBrowser.Model.Entities;
+
+namespace MediaBrowser.Controller.MediaEncoding;
+
+/// <summary>
+/// Describes the downmix algorithms capabilities.
+/// </summary>
+public static class DownMixAlgorithmsHelper
+{
+    /// <summary>
+    /// The filter string of the DownMixStereoAlgorithms.
+    /// The index is the tuple of (algorithm, layout).
+    /// </summary>
+    public static readonly Dictionary<(DownMixStereoAlgorithms, string), string> AlgorithmFilterStrings = new()
+    {
+        { (DownMixStereoAlgorithms.Dave750, "5.1"), "pan=stereo|c0=0.5*c2+0.707*c0+0.707*c4+0.5*c3|c1=0.5*c2+0.707*c1+0.707*c5+0.5*c3" },
+        // Use AC-4 algorithm to downmix 7.1 inputs to 5.1 first
+        { (DownMixStereoAlgorithms.Dave750, "7.1"), "pan=5.1(side)|c0=c0|c1=c1|c2=c2|c3=c3|c4=0.707*c4+0.707*c6|c5=0.707*c5+0.707*c7,pan=stereo|c0=0.5*c2+0.707*c0+0.707*c4+0.5*c3|c1=0.5*c2+0.707*c1+0.707*c5+0.5*c3" },
+        { (DownMixStereoAlgorithms.NightmodeDialogue, "5.1"), "pan=stereo|c0=c2+0.30*c0+0.30*c4|c1=c2+0.30*c1+0.30*c5" },
+        // Use AC-4 algorithm to downmix 7.1 inputs to 5.1 first
+        { (DownMixStereoAlgorithms.NightmodeDialogue, "7.1"), "pan=5.1(side)|c0=c0|c1=c1|c2=c2|c3=c3|c4=0.707*c4+0.707*c6|c5=0.707*c5+0.707*c7,pan=stereo|c0=c2+0.30*c0+0.30*c4|c1=c2+0.30*c1+0.30*c5" },
+        { (DownMixStereoAlgorithms.Rfc7845, "3.0"), "pan=stereo|c0=0.414214*c2+0.585786*c0|c1=0.414214*c2+0.585786*c1" },
+        { (DownMixStereoAlgorithms.Rfc7845, "quad"), "pan=stereo|c0=0.422650*c0+0.366025*c2+0.211325*c3|c1=0.422650*c1+0.366025*c3+0.211325*c2" },
+        { (DownMixStereoAlgorithms.Rfc7845, "5.0"), "pan=stereo|c0=0.460186*c2+0.650802*c0+0.563611*c3+0.325401*c4|c1=0.460186*c2+0.650802*c1+0.563611*c4+0.325401*c3" },
+        { (DownMixStereoAlgorithms.Rfc7845, "5.1"), "pan=stereo|c0=0.374107*c2+0.529067*c0+0.458186*c4+0.264534*c5+0.374107*c3|c1=0.374107*c2+0.529067*c1+0.458186*c5+0.264534*c4+0.374107*c3" },
+        { (DownMixStereoAlgorithms.Rfc7845, "6.1"), "pan=stereo|c0=0.321953*c2+0.455310*c0+0.394310*c5+0.227655*c6+0.278819*c4+0.321953*c3|c1=0.321953*c2+0.455310*c1+0.394310*c6+0.227655*c5+0.278819*c4+0.321953*c3" },
+        { (DownMixStereoAlgorithms.Rfc7845, "7.1"), "pan=stereo|c0=0.274804*c2+0.388631*c0+0.336565*c6+0.194316*c7+0.336565*c4+0.194316*c5+0.274804*c3|c1=0.274804*c2+0.388631*c1+0.336565*c7+0.194316*c6+0.336565*c5+0.194316*c4+0.274804*c3" },
+        { (DownMixStereoAlgorithms.Ac4, "3.0"), "pan=stereo|c0=c0+0.707*c2|c1=c1+0.707*c2" },
+        { (DownMixStereoAlgorithms.Ac4, "5.0"), "pan=stereo|c0=c0+0.707*c2+0.707*c3|c1=c1+0.707*c2+0.707*c4" },
+        { (DownMixStereoAlgorithms.Ac4, "5.1"), "pan=stereo|c0=c0+0.707*c2+0.707*c4|c1=c1+0.707*c2+0.707*c5" },
+        { (DownMixStereoAlgorithms.Ac4, "7.0"), "pan=5.0(side)|c0=c0|c1=c1|c2=c2|c3=0.707*c3+0.707*c5|c4=0.707*c4+0.707*c6,pan=stereo|c0=c0+0.707*c2+0.707*c3|c1=c1+0.707*c2+0.707*c4" },
+        { (DownMixStereoAlgorithms.Ac4, "7.1"), "pan=5.1(side)|c0=c0|c1=c1|c2=c2|c3=c3|c4=0.707*c4+0.707*c6|c5=0.707*c5+0.707*c7,pan=stereo|c0=c0+0.707*c2+0.707*c4|c1=c1+0.707*c2+0.707*c5" },
+    };
+
+    /// <summary>
+    /// Get the audio channel layout string from the audio stream
+    /// If the input audio string does not have a valid layout string, guess from channel count.
+    /// </summary>
+    /// <param name="audioStream">The audio stream to get layout.</param>
+    /// <returns>Channel Layout string.</returns>
+    public static string InferChannelLayout(MediaStream audioStream)
+    {
+        if (!string.IsNullOrWhiteSpace(audioStream.ChannelLayout))
+        {
+            // Note: BDMVs do not derive this string from ffmpeg, which would cause ambiguity with 4-channel audio
+            // "quad" => 2 front and 2 rear, "4.0" => 3 front and 1 rear
+            // BDMV will always use "4.0" in this case
+            // Because the quad layout is super rare in BDs, we will use "4.0" as is here
+            return audioStream.ChannelLayout;
+        }
+
+        if (audioStream.Channels is null)
+        {
+            return string.Empty;
+        }
+
+        // When we don't have definitive channel layout, we have to guess from the channel count
+        // Guessing is not always correct, but for most videos we don't have to guess like this as the definitive layout is recorded during scan
+        var inferredLayout = audioStream.Channels.Value switch
+        {
+            1 => "mono",
+            2 => "stereo",
+            3 => "2.1", // Could also be 3.0, prefer 2.1
+            4 => "4.0", // Could also be quad (with rear left and rear right) and 3.1 with LFE. prefer 4.0 with front center and back center
+            5 => "5.0",
+            6 => "5.1", // Could also be 6.0 or hexagonal, prefer 5.1
+            7 => "6.1", // Could also be 7.0, prefer 6.1
+            8 => "7.1", // Could also be 8.0, prefer 7.1
+            _ => string.Empty // Return empty string for not supported layout
+        };
+        return inferredLayout;
+    }
+}
diff --git a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs
index eb80bab2d..c068cf055 100644
--- a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs
+++ b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs
@@ -64,6 +64,7 @@ namespace MediaBrowser.Controller.MediaEncoding
         private readonly Version _minFFmpegSvtAv1Params = new Version(5, 1);
         private readonly Version _minFFmpegVaapiH26xEncA53CcSei = new Version(6, 0);
         private readonly Version _minFFmpegReadrateOption = new Version(5, 0);
+        private readonly Version _minFFmpegWorkingVtHwSurface = new Version(7, 0, 1);
         private readonly Version _minFFmpegDisplayRotationOption = new Version(6, 0);
 
         private static readonly Regex _validationRegex = new(ValidationRegex, RegexOptions.Compiled);
@@ -2673,28 +2674,17 @@ namespace MediaBrowser.Controller.MediaEncoding
 
             var filters = new List<string>();
 
-            if (channels.HasValue
-                && channels.Value == 2
-                && state.AudioStream is not null
-                && state.AudioStream.Channels.HasValue
-                && state.AudioStream.Channels.Value == 6)
+            if (channels is 2 && state.AudioStream?.Channels is > 2)
             {
-                if (!encodingOptions.DownMixAudioBoost.Equals(1))
+                var hasDownMixFilter = DownMixAlgorithmsHelper.AlgorithmFilterStrings.TryGetValue((encodingOptions.DownMixStereoAlgorithm, DownMixAlgorithmsHelper.InferChannelLayout(state.AudioStream)), out var downMixFilterString);
+                if (hasDownMixFilter)
                 {
-                    filters.Add("volume=" + encodingOptions.DownMixAudioBoost.ToString(CultureInfo.InvariantCulture));
+                    filters.Add(downMixFilterString);
                 }
 
-                switch (encodingOptions.DownMixStereoAlgorithm)
+                if (!encodingOptions.DownMixAudioBoost.Equals(1))
                 {
-                    case DownMixStereoAlgorithms.Dave750:
-                        filters.Add("pan=stereo|c0=0.5*c2+0.707*c0+0.707*c4+0.5*c3|c1=0.5*c2+0.707*c1+0.707*c5+0.5*c3");
-                        break;
-                    case DownMixStereoAlgorithms.NightmodeDialogue:
-                        filters.Add("pan=stereo|c0=c2+0.30*c0+0.30*c4|c1=c2+0.30*c1+0.30*c5");
-                        break;
-                    case DownMixStereoAlgorithms.None:
-                    default:
-                        break;
+                    filters.Add("volume=" + encodingOptions.DownMixAudioBoost.ToString(CultureInfo.InvariantCulture));
                 }
             }
 
@@ -5300,6 +5290,7 @@ namespace MediaBrowser.Controller.MediaEncoding
             string vidEncoder)
         {
             var isVtEncoder = vidEncoder.Contains("videotoolbox", StringComparison.OrdinalIgnoreCase);
+            var isVtDecoder = vidDecoder.Contains("videotoolbox", StringComparison.OrdinalIgnoreCase);
 
             if (!isVtEncoder)
             {
@@ -5320,6 +5311,7 @@ namespace MediaBrowser.Controller.MediaEncoding
             var doDeintH2645 = doDeintH264 || doDeintHevc;
             var doVtTonemap = IsVideoToolboxTonemapAvailable(state, options);
             var doMetalTonemap = !doVtTonemap && IsHwTonemapAvailable(state, options);
+            var usingHwSurface = isVtDecoder && (_mediaEncoder.EncoderVersion >= _minFFmpegWorkingVtHwSurface);
 
             var rotation = state.VideoStream?.Rotation ?? 0;
             var tranposeDir = rotation == 0 ? string.Empty : GetVideoTransposeDirection(state);
@@ -5414,23 +5406,25 @@ namespace MediaBrowser.Controller.MediaEncoding
                     subFilters.Add(subTextSubtitlesFilter);
                 }
 
-                subFilters.Add("hwupload=derive_device=videotoolbox");
+                subFilters.Add("hwupload");
                 overlayFilters.Add("overlay_videotoolbox=eof_action=pass:repeatlast=0");
             }
 
+            if (usingHwSurface)
+            {
+                return (mainFilters, subFilters, overlayFilters);
+            }
+
+            // For old jellyfin-ffmpeg that has broken hwsurface, add a hwupload
             var needFiltering = mainFilters.Any(f => !string.IsNullOrEmpty(f)) ||
                                 subFilters.Any(f => !string.IsNullOrEmpty(f)) ||
                                 overlayFilters.Any(f => !string.IsNullOrEmpty(f));
-
-            // This is a workaround for ffmpeg's hwupload implementation
-            // For VideoToolbox encoders, a hwupload without a valid filter actually consuming its frame
-            // will cause the encoder to produce incorrect frames.
             if (needFiltering)
             {
                 // INPUT videotoolbox/memory surface(vram/uma)
                 // this will pass-through automatically if in/out format matches.
                 mainFilters.Insert(0, "format=nv12|p010le|videotoolbox_vld");
-                mainFilters.Insert(0, "hwupload=derive_device=videotoolbox");
+                mainFilters.Insert(0, "hwupload");
             }
 
             return (mainFilters, subFilters, overlayFilters);
@@ -6458,22 +6452,20 @@ namespace MediaBrowser.Controller.MediaEncoding
                                     || string.Equals("yuvj420p", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase);
             var is8_10bitSwFormatsVt = is8bitSwFormatsVt || string.Equals("yuv420p10le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase);
 
-            // VideoToolbox's Hardware surface in ffmpeg is not only slower than hwupload, but also breaks HDR in many cases.
-            // For example: https://trac.ffmpeg.org/ticket/10884
-            // Disable it for now.
-            const bool UseHwSurface = false;
+            // The related patches make videotoolbox hardware surface working is only available in jellyfin-ffmpeg 7.0.1 at the moment.
+            bool useHwSurface = (_mediaEncoder.EncoderVersion >= _minFFmpegWorkingVtHwSurface) && IsVideoToolboxFullSupported();
 
             if (is8bitSwFormatsVt)
             {
                 if (string.Equals("avc", videoStream.Codec, StringComparison.OrdinalIgnoreCase)
                     || string.Equals("h264", videoStream.Codec, StringComparison.OrdinalIgnoreCase))
                 {
-                    return GetHwaccelType(state, options, "h264", bitDepth, UseHwSurface);
+                    return GetHwaccelType(state, options, "h264", bitDepth, useHwSurface);
                 }
 
                 if (string.Equals("vp8", videoStream.Codec, StringComparison.OrdinalIgnoreCase))
                 {
-                    return GetHwaccelType(state, options, "vp8", bitDepth, UseHwSurface);
+                    return GetHwaccelType(state, options, "vp8", bitDepth, useHwSurface);
                 }
             }
 
@@ -6482,12 +6474,12 @@ namespace MediaBrowser.Controller.MediaEncoding
                 if (string.Equals("hevc", videoStream.Codec, StringComparison.OrdinalIgnoreCase)
                     || string.Equals("h265", videoStream.Codec, StringComparison.OrdinalIgnoreCase))
                 {
-                    return GetHwaccelType(state, options, "hevc", bitDepth, UseHwSurface);
+                    return GetHwaccelType(state, options, "hevc", bitDepth, useHwSurface);
                 }
 
                 if (string.Equals("vp9", videoStream.Codec, StringComparison.OrdinalIgnoreCase))
                 {
-                    return GetHwaccelType(state, options, "vp9", bitDepth, UseHwSurface);
+                    return GetHwaccelType(state, options, "vp9", bitDepth, useHwSurface);
                 }
             }
 
@@ -7172,7 +7164,10 @@ namespace MediaBrowser.Controller.MediaEncoding
 
             var channels = state.OutputAudioChannels;
 
-            if (channels.HasValue && ((channels.Value != 2 && state.AudioStream?.Channels != 6) || encodingOptions.DownMixStereoAlgorithm == DownMixStereoAlgorithms.None))
+            var useDownMixAlgorithm = state.AudioStream is not null
+                                      && DownMixAlgorithmsHelper.AlgorithmFilterStrings.ContainsKey((encodingOptions.DownMixStereoAlgorithm, DownMixAlgorithmsHelper.InferChannelLayout(state.AudioStream)));
+
+            if (channels.HasValue && !useDownMixAlgorithm)
             {
                 args += " -ac " + channels.Value;
             }