Merge pull request #17806 from jellyfin/fix/dovi-color-validation

Enforce dolby vision transfer check

6 files changed, 327 insertions, 12 deletions

diff --git a/Jellyfin.Api/Helpers/DynamicHlsHelper.cs b/Jellyfin.Api/Helpers/DynamicHlsHelper.cs
index b09b279699..4995d37f4b 100644
--- a/Jellyfin.Api/Helpers/DynamicHlsHelper.cs
+++ b/Jellyfin.Api/Helpers/DynamicHlsHelper.cs
@@ -458,6 +458,7 @@ public class DynamicHlsHelper
                     {
                         case VideoRangeType.HLG:
                         case VideoRangeType.DOVIWithHLG:
+                        case VideoRangeType.DOVIInvalid when string.Equals(state.VideoStream.ColorTransfer, "arib-std-b67", StringComparison.OrdinalIgnoreCase):
                             builder.Append(",VIDEO-RANGE=HLG");
                             break;
                         default:
diff --git a/Jellyfin.Data/Enums/VideoRangeType.cs b/Jellyfin.Data/Enums/VideoRangeType.cs
index ce232d73c3..e7cc340d21 100644
--- a/Jellyfin.Data/Enums/VideoRangeType.cs
+++ b/Jellyfin.Data/Enums/VideoRangeType.cs
@@ -61,8 +61,9 @@ public enum VideoRangeType
     DOVIWithELHDR10Plus,
 
     /// <summary>
-    /// Dolby Vision with invalid configuration. e.g. Profile 8 compat id 6.
-    /// When using this range, the server would assume the video is still HDR10 after removing the Dolby Vision metadata.
+    /// Dolby Vision with invalid configuration, e.g. Profile 8 compat id 6 or inconsistent base-layer color metadata.
+    /// The base layer is classified as HDR only when its transfer characteristics signal PQ or HLG.
+    /// Otherwise, it is classified as SDR.
     /// </summary>
     DOVIInvalid,
 
diff --git a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs
index 9952c10c3a..a0f4087395 100644
--- a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs
+++ b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs
@@ -442,7 +442,8 @@ namespace MediaBrowser.Controller.MediaEncoding
                    && (state.VideoStream.VideoRangeType == VideoRangeType.HDR10
                        || IsHdr10Plus(state.VideoStream)
                        || IsDoviWithHdr10Bl(state.VideoStream)
-                       || state.VideoStream.VideoRangeType == VideoRangeType.HLG);
+                       || state.VideoStream.VideoRangeType == VideoRangeType.HLG
+                       || state.VideoStream.VideoRangeType == VideoRangeType.DOVIInvalid);
         }
 
         private static bool IsDeinterlaceAvailable(EncodingJobInfo state)
@@ -1390,7 +1391,8 @@ namespace MediaBrowser.Controller.MediaEncoding
                 or VideoRangeType.DOVIWithEL
                 or VideoRangeType.DOVIWithHDR10Plus
                 or VideoRangeType.DOVIWithELHDR10Plus
-                or VideoRangeType.DOVIInvalid;
+                || (rangeType == VideoRangeType.DOVIInvalid
+                    && string.Equals(stream.ColorTransfer, "smpte2084", StringComparison.OrdinalIgnoreCase)); // invalid may be hlg now
         }
 
         public static bool IsDovi(MediaStream stream)
@@ -1400,7 +1402,8 @@ namespace MediaBrowser.Controller.MediaEncoding
             return IsDoviWithHdr10Bl(stream)
                    || (rangeType is VideoRangeType.DOVI
                        or VideoRangeType.DOVIWithHLG
-                       or VideoRangeType.DOVIWithSDR);
+                       or VideoRangeType.DOVIWithSDR
+                       or VideoRangeType.DOVIInvalid);
         }
 
         public static bool IsHdr10Plus(MediaStream stream)
@@ -1420,7 +1423,8 @@ namespace MediaBrowser.Controller.MediaEncoding
         private static DynamicHdrMetadataRemovalPlan ShouldRemoveDynamicHdrMetadata(EncodingJobInfo state)
         {
             var videoStream = state.VideoStream;
-            if (videoStream.VideoRange is not VideoRange.HDR)
+            if (videoStream.VideoRange is not VideoRange.HDR
+                && videoStream.VideoRangeType != VideoRangeType.DOVIInvalid)
             {
                 return DynamicHdrMetadataRemovalPlan.None;
             }
diff --git a/MediaBrowser.Model/Entities/MediaStream.cs b/MediaBrowser.Model/Entities/MediaStream.cs
index f057714bea..67af843626 100644
--- a/MediaBrowser.Model/Entities/MediaStream.cs
+++ b/MediaBrowser.Model/Entities/MediaStream.cs
@@ -810,6 +810,11 @@ namespace MediaBrowser.Model.Entities
                 return (VideoRange.Unknown, VideoRangeType.Unknown);
             }
 
+            var isPq = string.Equals(ColorTransfer, "smpte2084", StringComparison.OrdinalIgnoreCase);
+            var isHlg = string.Equals(ColorTransfer, "arib-std-b67", StringComparison.OrdinalIgnoreCase);
+            // Invalid DV only retains HDR when the base layer explicitly signals PQ or HLG.
+            var baseVideoRange = isPq || isHlg ? VideoRange.HDR : VideoRange.SDR;
+
             var codecTag = CodecTag;
             var dvProfile = DvProfile;
             var rpuPresentFlag = RpuPresentFlag == 1;
@@ -834,7 +839,7 @@ namespace MediaBrowser.Model.Entities
                         4 => (VideoRange.HDR, VideoRangeType.DOVIWithHLG),
                         2 => (VideoRange.SDR, VideoRangeType.DOVIWithSDR),
                         // Out of Dolby Spec files should be marked as invalid
-                        _ => (VideoRange.HDR, VideoRangeType.DOVIInvalid)
+                        _ => (baseVideoRange, VideoRangeType.DOVIInvalid)
                     },
                     7 => (VideoRange.HDR, VideoRangeType.DOVIWithEL),
                     10 => dvBlCompatId switch
@@ -844,11 +849,26 @@ namespace MediaBrowser.Model.Entities
                         2 => (VideoRange.SDR, VideoRangeType.DOVIWithSDR),
                         4 => (VideoRange.HDR, VideoRangeType.DOVIWithHLG),
                         // Out of Dolby Spec files should be marked as invalid
-                        _ => (VideoRange.HDR, VideoRangeType.DOVIInvalid)
+                        _ => (baseVideoRange, VideoRangeType.DOVIInvalid)
                     },
                     _ => (VideoRange.SDR, VideoRangeType.SDR)
                 };
 
+                var expectedTransfer = dvRangeSet.Item2 switch
+                {
+                    VideoRangeType.DOVIWithHDR10 or VideoRangeType.DOVIWithEL => "smpte2084",
+                    VideoRangeType.DOVIWithHLG => "arib-std-b67",
+                    _ => null
+                };
+
+                if (expectedTransfer is not null
+                    && (!string.Equals(ColorSpace, "bt2020nc", StringComparison.OrdinalIgnoreCase)
+                        || !string.Equals(ColorTransfer, expectedTransfer, StringComparison.OrdinalIgnoreCase)
+                        || !string.Equals(ColorPrimaries, "bt2020", StringComparison.OrdinalIgnoreCase)))
+                {
+                    return (baseVideoRange, VideoRangeType.DOVIInvalid);
+                }
+
                 if (Hdr10PlusPresentFlag == true)
                 {
                     return dvRangeSet.Item2 switch
@@ -862,13 +882,11 @@ namespace MediaBrowser.Model.Entities
                 return dvRangeSet;
             }
 
-            var colorTransfer = ColorTransfer;
-
-            if (string.Equals(colorTransfer, "smpte2084", StringComparison.OrdinalIgnoreCase))
+            if (isPq)
             {
                 return Hdr10PlusPresentFlag == true ? (VideoRange.HDR, VideoRangeType.HDR10Plus) : (VideoRange.HDR, VideoRangeType.HDR10);
             }
-            else if (string.Equals(colorTransfer, "arib-std-b67", StringComparison.OrdinalIgnoreCase))
+            else if (isHlg)
             {
                 return (VideoRange.HDR, VideoRangeType.HLG);
             }
diff --git a/tests/Jellyfin.Controller.Tests/MediaEncoding/EncodingHelperDoviTests.cs b/tests/Jellyfin.Controller.Tests/MediaEncoding/EncodingHelperDoviTests.cs
new file mode 100644
index 0000000000..557035e2d1
--- /dev/null
+++ b/tests/Jellyfin.Controller.Tests/MediaEncoding/EncodingHelperDoviTests.cs
@@ -0,0 +1,162 @@
+using System;
+using Jellyfin.Data.Enums;
+using MediaBrowser.Common.Configuration;
+using MediaBrowser.Controller.IO;
+using MediaBrowser.Controller.MediaEncoding;
+using MediaBrowser.Controller.Streaming;
+using MediaBrowser.Model.Configuration;
+using MediaBrowser.Model.Dto;
+using MediaBrowser.Model.Entities;
+using Moq;
+using Xunit;
+
+using IConfiguration = Microsoft.Extensions.Configuration.IConfiguration;
+
+namespace Jellyfin.Controller.Tests.MediaEncoding;
+
+public class EncodingHelperDoviTests
+{
+    [Theory]
+    [InlineData(null, false)]
+    [InlineData("bt709", false)]
+    [InlineData("unknown", false)]
+    [InlineData("bt2020-10", false)]
+    [InlineData("smpte2084", true)]
+    [InlineData("arib-std-b67", true)]
+    public void GetSwVidFilterChain_InvalidDovi_OnlyTonemapsHdrBaseLayer(string? transfer, bool tonemap)
+    {
+        var state = CreateState("hevc", transfer);
+        var helper = CreateHelper(true);
+
+        var (filters, _, _) = helper.GetSwVidFilterChain(state, new EncodingOptions(), "libx264");
+        var args = string.Join(',', filters);
+
+        Assert.Equal(VideoRangeType.DOVIInvalid, state.VideoStream.VideoRangeType);
+        Assert.Equal(tonemap, args.Contains("tonemapx=", StringComparison.Ordinal));
+        Assert.Contains(tonemap ? "color_trc=" + transfer : "color_trc=bt709", args, StringComparison.Ordinal);
+    }
+
+    [Theory]
+    [InlineData(null, false)]
+    [InlineData("bt709", false)]
+    [InlineData("arib-std-b67", false)]
+    [InlineData("smpte2084", true)]
+    [InlineData("SMPTE2084", true)]
+    public void IsDoviWithHdr10Bl_InvalidDovi_RequiresPq(string? transfer, bool expected)
+    {
+        var stream = CreateState("hevc", transfer).VideoStream;
+
+        Assert.True(EncodingHelper.IsDovi(stream));
+        Assert.Equal(expected, EncodingHelper.IsDoviWithHdr10Bl(stream));
+    }
+
+    [Theory]
+    [InlineData("hevc", null, "hevc_metadata=remove_dovi=1")]
+    [InlineData("hevc", "bt709", "hevc_metadata=remove_dovi=1")]
+    [InlineData("hevc", "smpte2084", "hevc_metadata=remove_dovi=1")]
+    [InlineData("hevc", "arib-std-b67", "hevc_metadata=remove_dovi=1")]
+    [InlineData("av1", null, "av1_metadata=remove_dovi=1")]
+    [InlineData("av1", "bt709", "av1_metadata=remove_dovi=1")]
+    [InlineData("av1", "smpte2084", "av1_metadata=remove_dovi=1")]
+    [InlineData("av1", "arib-std-b67", "av1_metadata=remove_dovi=1")]
+    public void GetBitStreamArgs_InvalidDovi_PreservesClientDependentRemoval(string codec, string? transfer, string expected)
+    {
+        var state = CreateState(codec, transfer);
+        var helper = CreateHelper(true);
+
+        foreach (var (requestedRanges, removeDovi) in new[] { (null, false), ("SDR", false), ("HDR10", false), ("DOVIWithEL", false), ("DOVI", true), ("SDR,DOVI", true) })
+        {
+            state.BaseRequest.VideoRangeType = requestedRanges;
+
+            Assert.Equal(removeDovi, helper.IsDoviRemoved(state));
+            if (removeDovi)
+            {
+                Assert.Contains(expected, helper.GetBitStreamArgs(state, MediaStreamType.Video), StringComparison.Ordinal);
+            }
+            else
+            {
+                Assert.Equal(codec == "hevc" ? "-bsf:v hevc_mp4toannexb" : null, helper.GetBitStreamArgs(state, MediaStreamType.Video));
+            }
+
+            Assert.False(CreateHelper(false).IsDoviRemoved(state));
+        }
+    }
+
+    [Theory]
+    [InlineData(null, true)]
+    [InlineData("HDR10", true)]
+    [InlineData("DOVI", false)]
+    [InlineData("SDR,DOVI", false)]
+    public void CanStreamCopyVideo_InvalidDovi_RequiresRemovalSupportOnlyForDoviClients(string? requestedRanges, bool copyWithoutRemovalSupport)
+    {
+        foreach (var codec in new[] { "hevc", "av1" })
+        {
+            foreach (var transfer in new[] { "bt709", "smpte2084" })
+            {
+                var state = CreateState(codec, transfer);
+                state.BaseRequest.VideoRangeType = requestedRanges;
+
+                Assert.True(CreateHelper(true).CanStreamCopyVideo(state, state.VideoStream));
+                Assert.Equal(copyWithoutRemovalSupport, CreateHelper(false).CanStreamCopyVideo(state, state.VideoStream));
+            }
+        }
+    }
+
+    [Fact]
+    public void GetBitStreamArgs_ValidDovi_PreservesMetadata()
+    {
+        var state = CreateState("hevc", "smpte2084");
+        state.VideoStream.ColorSpace = "bt2020nc";
+        state.VideoStream.ColorPrimaries = "bt2020";
+        state.BaseRequest.VideoRangeType = "DOVIWithEL";
+        var helper = CreateHelper(true);
+
+        Assert.False(helper.IsDoviRemoved(state));
+        Assert.Equal("-bsf:v hevc_mp4toannexb", helper.GetBitStreamArgs(state, MediaStreamType.Video));
+    }
+
+    private static EncodingJobInfo CreateState(string codec, string? transfer)
+    {
+        var stream = new MediaStream
+        {
+            Type = MediaStreamType.Video,
+            Codec = codec,
+            Width = 1920,
+            Height = 1080,
+            BitDepth = 10,
+            DvProfile = codec == "hevc" ? 7 : 10,
+            DvBlSignalCompatibilityId = codec == "hevc" ? 6 : 1,
+            RpuPresentFlag = 1,
+            BlPresentFlag = 1,
+            ColorSpace = "bt709",
+            ColorPrimaries = "bt709",
+            ColorTransfer = transfer
+        };
+
+        return new EncodingJobInfo(TranscodingJobType.Hls)
+        {
+            VideoStream = stream,
+            MediaSource = new MediaSourceInfo { Container = "mkv", MediaStreams = [stream] },
+            BaseRequest = new VideoRequestDto(),
+            OutputVideoCodec = "copy",
+            IsVideoRequest = true,
+            IsInputVideo = true
+        };
+    }
+
+    private static EncodingHelper CreateHelper(bool supportsRemoval)
+    {
+        var encoder = new Mock<IMediaEncoder>();
+        encoder.Setup(x => x.SupportsBitStreamFilterWithOption(It.IsAny<BitStreamFilterOptionType>())).Returns(supportsRemoval);
+        encoder.Setup(x => x.SupportsFilter("tonemapx")).Returns(true);
+        encoder.SetupGet(x => x.EncoderVersion).Returns(new Version(8, 1));
+
+        return new EncodingHelper(
+            Mock.Of<IApplicationPaths>(),
+            encoder.Object,
+            Mock.Of<ISubtitleEncoder>(),
+            Mock.Of<IConfiguration>(),
+            Mock.Of<IConfigurationManager>(),
+            Mock.Of<IPathManager>());
+    }
+}
diff --git a/tests/Jellyfin.Model.Tests/Entities/MediaStreamVideoRangeTests.cs b/tests/Jellyfin.Model.Tests/Entities/MediaStreamVideoRangeTests.cs
new file mode 100644
index 0000000000..f264e5a019
--- /dev/null
+++ b/tests/Jellyfin.Model.Tests/Entities/MediaStreamVideoRangeTests.cs
@@ -0,0 +1,129 @@
+using Jellyfin.Data.Enums;
+using MediaBrowser.Model.Entities;
+using Xunit;
+
+namespace Jellyfin.Model.Tests.Entities;
+
+public class MediaStreamVideoRangeTests
+{
+    [Theory]
+    [InlineData(7, 6, "smpte2084", false, VideoRangeType.DOVIWithEL)]
+    [InlineData(7, 6, "smpte2084", true, VideoRangeType.DOVIWithELHDR10Plus)]
+    [InlineData(8, 1, "smpte2084", false, VideoRangeType.DOVIWithHDR10)]
+    [InlineData(8, 1, "smpte2084", true, VideoRangeType.DOVIWithHDR10Plus)]
+    [InlineData(8, 4, "arib-std-b67", false, VideoRangeType.DOVIWithHLG)]
+    [InlineData(10, 1, "smpte2084", false, VideoRangeType.DOVIWithHDR10)]
+    [InlineData(10, 1, "smpte2084", true, VideoRangeType.DOVIWithHDR10Plus)]
+    [InlineData(10, 4, "arib-std-b67", false, VideoRangeType.DOVIWithHLG)]
+    [InlineData(8, 1, "SMPTE2084", false, VideoRangeType.DOVIWithHDR10)]
+    [InlineData(8, 4, "ARIB-STD-B67", false, VideoRangeType.DOVIWithHLG)]
+    public void GetVideoColorRange_ValidDovi_PreservesRangeType(
+        int profile, int compatibilityId, string transfer, bool hdr10Plus, VideoRangeType expected)
+    {
+        var stream = CreateDovi(profile, compatibilityId, "BT2020NC", transfer, "BT2020", hdr10Plus);
+
+        Assert.Equal((VideoRange.HDR, expected), stream.GetVideoColorRange());
+    }
+
+    [Theory]
+    [InlineData("bt709", "bt709", "bt709", VideoRange.SDR)]
+    [InlineData("bt2020nc", "bt709", "bt2020", VideoRange.SDR)]
+    [InlineData("bt2020nc", null, "bt2020", VideoRange.SDR)]
+    [InlineData("bt2020nc", "", "bt2020", VideoRange.SDR)]
+    [InlineData("bt2020nc", "unknown", "bt2020", VideoRange.SDR)]
+    [InlineData("bt2020nc", "bt2020-10", "bt2020", VideoRange.SDR)]
+    [InlineData(null, null, null, VideoRange.SDR)]
+    [InlineData("bt709", "smpte2084", "bt2020", VideoRange.HDR)]
+    [InlineData("bt2020nc", "smpte2084", "bt709", VideoRange.HDR)]
+    [InlineData(null, "smpte2084", "bt2020", VideoRange.HDR)]
+    [InlineData("bt2020nc", "smpte2084", null, VideoRange.HDR)]
+    [InlineData("bt709", "arib-std-b67", "bt2020", VideoRange.HDR)]
+    [InlineData("bt2020nc", "arib-std-b67", "bt709", VideoRange.HDR)]
+    [InlineData(null, "arib-std-b67", "bt2020", VideoRange.HDR)]
+    [InlineData("bt2020nc", "arib-std-b67", null, VideoRange.HDR)]
+    public void GetVideoColorRange_InvalidDoviColors_UsesBaseLayerRange(
+        string? space, string? transfer, string? primaries, VideoRange expected)
+    {
+        // Cover every HDR-compatible DV profile, including the HDR10+ variants.
+        foreach (var (profile, compatibilityId) in new[] { (7, 6), (8, 1), (8, 4), (10, 1), (10, 4) })
+        {
+            foreach (var hdr10Plus in new[] { false, true })
+            {
+                var stream = CreateDovi(profile, compatibilityId, space, transfer, primaries, hdr10Plus);
+
+                Assert.Equal(expected, stream.VideoRange);
+                Assert.Equal(VideoRangeType.DOVIInvalid, stream.VideoRangeType);
+            }
+        }
+    }
+
+    [Theory]
+    [InlineData(7, 6, "arib-std-b67")]
+    [InlineData(8, 1, "arib-std-b67")]
+    [InlineData(8, 4, "smpte2084")]
+    [InlineData(10, 1, "arib-std-b67")]
+    [InlineData(10, 4, "smpte2084")]
+    public void GetVideoColorRange_WrongHdrTransfer_InvalidButStillHdr(int profile, int compatibilityId, string transfer)
+    {
+        var stream = CreateDovi(profile, compatibilityId, "bt2020nc", transfer, "bt2020", true);
+
+        Assert.Equal((VideoRange.HDR, VideoRangeType.DOVIInvalid), stream.GetVideoColorRange());
+    }
+
+    [Theory]
+    [InlineData(5, 0, null, VideoRange.HDR, VideoRangeType.DOVI)]
+    [InlineData(10, 0, null, VideoRange.HDR, VideoRangeType.DOVI)]
+    [InlineData(8, 2, "bt709", VideoRange.SDR, VideoRangeType.DOVIWithSDR)]
+    [InlineData(10, 2, "bt709", VideoRange.SDR, VideoRangeType.DOVIWithSDR)]
+    public void GetVideoColorRange_OtherDoviProfiles_PreservesClassification(
+        int profile, int compatibilityId, string? transfer, VideoRange range, VideoRangeType rangeType)
+    {
+        var stream = CreateDovi(profile, compatibilityId, "bt709", transfer, "bt709", false);
+
+        Assert.Equal((range, rangeType), stream.GetVideoColorRange());
+    }
+
+    [Theory]
+    [InlineData(8, null, VideoRange.SDR)]
+    [InlineData(8, "bt709", VideoRange.SDR)]
+    [InlineData(8, "smpte2084", VideoRange.HDR)]
+    [InlineData(10, null, VideoRange.SDR)]
+    [InlineData(10, "arib-std-b67", VideoRange.HDR)]
+    public void GetVideoColorRange_InvalidCompatibilityId_UsesBaseLayerRange(int profile, string? transfer, VideoRange expected)
+    {
+        var stream = CreateDovi(profile, 6, "bt2020nc", transfer, "bt2020", false);
+
+        Assert.Equal((expected, VideoRangeType.DOVIInvalid), stream.GetVideoColorRange());
+    }
+
+    [Theory]
+    [InlineData("bt709", false, VideoRange.SDR, VideoRangeType.SDR)]
+    [InlineData(null, false, VideoRange.SDR, VideoRangeType.SDR)]
+    [InlineData("smpte2084", false, VideoRange.HDR, VideoRangeType.HDR10)]
+    [InlineData("smpte2084", true, VideoRange.HDR, VideoRangeType.HDR10Plus)]
+    [InlineData("arib-std-b67", false, VideoRange.HDR, VideoRangeType.HLG)]
+    public void GetVideoColorRange_WithoutDovi_PreservesClassification(
+        string? transfer, bool hdr10Plus, VideoRange range, VideoRangeType rangeType)
+    {
+        var stream = new MediaStream { Type = MediaStreamType.Video, ColorTransfer = transfer, Hdr10PlusPresentFlag = hdr10Plus };
+
+        Assert.Equal((range, rangeType), stream.GetVideoColorRange());
+        stream.Type = MediaStreamType.Audio;
+        Assert.Equal((VideoRange.Unknown, VideoRangeType.Unknown), stream.GetVideoColorRange());
+    }
+
+    private static MediaStream CreateDovi(int profile, int compatibilityId, string? space, string? transfer, string? primaries, bool hdr10Plus)
+        => new()
+        {
+            Type = MediaStreamType.Video,
+            DvProfile = profile,
+            DvBlSignalCompatibilityId = compatibilityId,
+            RpuPresentFlag = 1,
+            BlPresentFlag = 1,
+            ElPresentFlag = profile == 7 ? 1 : 0,
+            ColorSpace = space,
+            ColorTransfer = transfer,
+            ColorPrimaries = primaries,
+            Hdr10PlusPresentFlag = hdr10Plus
+        };
+}