From 49ac705867234c48e79ceb1cd84bc4394c65313d Mon Sep 17 00:00:00 2001 From: gnattu Date: Thu, 3 Apr 2025 08:06:02 +0800 Subject: Improve dynamic HDR metadata handling (#13277) * Add support for bitstream filter to remove dynamic hdr metadata * Add support for ffprobe's only_first_vframe for HDR10+ detection * Add BitStreamFilterOptionType for metadata removal check * Map HDR10+ metadata to VideoRangeType.cs Current implementation uses a hack that abuses the EL flag to avoid database schema changes. Should add proper field once EFCore migration is merged. * Add more Dolby Vision Range types Out of spec ones are problematic and should be marked as a dedicated invalid type and handled by the server to not crash the player. Profile 7 videos should not be treated as normal HDR10 videos at all and should remove the metadata before serving. * Remove dynamic hdr metadata when necessary * Allow direct playback of HDR10+ videos on HDR10 clients * Only use dovi codec tag when dovi metadata is not removed * Handle DV Profile 7 Videos better * Fix HDR10+ with new bitmask * Indicate the presence of HDR10+ in HLS SUPPLEMENTAL-CODECS * Fix Dovi 8.4 not labeled as HLG in HLS * Fallback to dovi_rpu bsf for av1 when possible * Fix dovi_rpu cli for av1 * Use correct EFCore db column for HDR10+ * Undo outdated migration * Add proper hdr10+ migration * Remove outdated migration * Rebase to new db code * Add migrations for Hdr10PlusPresentFlag * Directly use bsf enum * Add xmldocs for SupportsBitStreamFilterWithOption * Make `VideoRangeType.Unknown` explicitly default on api models. * Unset default for non-api model class * Use tuples for bsf dictionary for now --- .../Encoder/EncoderValidator.cs | 46 ++++++ MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs | 20 +++ .../Probing/InternalMediaInfoResult.cs | 7 + .../Probing/MediaFrameInfo.cs | 184 +++++++++++++++++++++ .../Probing/MediaFrameSideDataInfo.cs | 16 ++ .../Probing/ProbeResultNormalizer.cs | 15 +- 6 files changed, 286 insertions(+), 2 deletions(-) create mode 100644 MediaBrowser.MediaEncoding/Probing/MediaFrameInfo.cs create mode 100644 MediaBrowser.MediaEncoding/Probing/MediaFrameSideDataInfo.cs (limited to 'MediaBrowser.MediaEncoding') diff --git a/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs b/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs index 54d0eb4b5..d28cd70ef 100644 --- a/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs +++ b/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs @@ -7,6 +7,7 @@ using System.Globalization; using System.Linq; using System.Runtime.Versioning; using System.Text.RegularExpressions; +using MediaBrowser.Controller.MediaEncoding; using Microsoft.Extensions.Logging; namespace MediaBrowser.MediaEncoding.Encoder @@ -160,6 +161,15 @@ namespace MediaBrowser.MediaEncoding.Encoder { 6, new string[] { "transpose_opencl", "rotate by half-turn" } } }; + private static readonly Dictionary _bsfOptionsDict = new Dictionary + { + { BitStreamFilterOptionType.HevcMetadataRemoveDovi, ("hevc_metadata", "remove_dovi") }, + { BitStreamFilterOptionType.HevcMetadataRemoveHdr10Plus, ("hevc_metadata", "remove_hdr10plus") }, + { BitStreamFilterOptionType.Av1MetadataRemoveDovi, ("av1_metadata", "remove_dovi") }, + { BitStreamFilterOptionType.Av1MetadataRemoveHdr10Plus, ("av1_metadata", "remove_hdr10plus") }, + { BitStreamFilterOptionType.DoviRpuStrip, ("dovi_rpu", "strip") } + }; + // These are the library versions that corresponds to our minimum ffmpeg version 4.4 according to the version table below // Refers to the versions in https://ffmpeg.org/download.html private static readonly Dictionary _ffmpegMinimumLibraryVersions = new Dictionary @@ -286,6 +296,9 @@ namespace MediaBrowser.MediaEncoding.Encoder public IDictionary GetFiltersWithOption() => GetFFmpegFiltersWithOption(); + public IDictionary GetBitStreamFiltersWithOption() => _bsfOptionsDict + .ToDictionary(item => item.Key, item => CheckBitStreamFilterWithOption(item.Value.Item1, item.Value.Item2)); + public Version? GetFFmpegVersion() { string output; @@ -495,6 +508,34 @@ namespace MediaBrowser.MediaEncoding.Encoder return false; } + public bool CheckBitStreamFilterWithOption(string filter, string option) + { + if (string.IsNullOrEmpty(filter) || string.IsNullOrEmpty(option)) + { + return false; + } + + string output; + try + { + output = GetProcessOutput(_encoderPath, "-h bsf=" + filter, false, null); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error detecting the given bit stream filter"); + return false; + } + + if (output.Contains("Bit stream filter " + filter, StringComparison.Ordinal)) + { + return output.Contains(option, StringComparison.Ordinal); + } + + _logger.LogWarning("Bit stream filter: {Name} with option {Option} is not available", filter, option); + + return false; + } + public bool CheckSupportedRuntimeKey(string keyDesc, Version? ffmpegVersion) { if (string.IsNullOrEmpty(keyDesc)) @@ -523,6 +564,11 @@ namespace MediaBrowser.MediaEncoding.Encoder return !string.IsNullOrEmpty(flag) && GetProcessExitCode(_encoderPath, $"-loglevel quiet -hwaccel_flags +{flag} -hide_banner -f lavfi -i nullsrc=s=1x1:d=100 -f null -"); } + public bool CheckSupportedProberOption(string option, string proberPath) + { + return !string.IsNullOrEmpty(option) && GetProcessExitCode(proberPath, $"-loglevel quiet -f lavfi -i nullsrc=s=1x1:d=1 -{option}"); + } + private IEnumerable GetCodecs(Codec codec) { string codecstr = codec == Codec.Encoder ? "encoders" : "decoders"; diff --git a/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs b/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs index e96040506..9a759ba41 100644 --- a/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs +++ b/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs @@ -73,9 +73,11 @@ namespace MediaBrowser.MediaEncoding.Encoder private List _hwaccels = new List(); private List _filters = new List(); private IDictionary _filtersWithOption = new Dictionary(); + private IDictionary _bitStreamFiltersWithOption = new Dictionary(); private bool _isPkeyPauseSupported = false; private bool _isLowPriorityHwDecodeSupported = false; + private bool _proberSupportsFirstVideoFrame = false; private bool _isVaapiDeviceAmd = false; private bool _isVaapiDeviceInteliHD = false; @@ -222,6 +224,7 @@ namespace MediaBrowser.MediaEncoding.Encoder SetAvailableEncoders(validator.GetEncoders()); SetAvailableFilters(validator.GetFilters()); SetAvailableFiltersWithOption(validator.GetFiltersWithOption()); + SetAvailableBitStreamFiltersWithOption(validator.GetBitStreamFiltersWithOption()); SetAvailableHwaccels(validator.GetHwaccels()); SetMediaEncoderVersion(validator); @@ -229,6 +232,7 @@ namespace MediaBrowser.MediaEncoding.Encoder _isPkeyPauseSupported = validator.CheckSupportedRuntimeKey("p pause transcoding", _ffmpegVersion); _isLowPriorityHwDecodeSupported = validator.CheckSupportedHwaccelFlag("low_priority"); + _proberSupportsFirstVideoFrame = validator.CheckSupportedProberOption("only_first_vframe", _ffprobePath); // Check the Vaapi device vendor if (OperatingSystem.IsLinux() @@ -342,6 +346,11 @@ namespace MediaBrowser.MediaEncoding.Encoder _filtersWithOption = dict; } + public void SetAvailableBitStreamFiltersWithOption(IDictionary dict) + { + _bitStreamFiltersWithOption = dict; + } + public void SetMediaEncoderVersion(EncoderValidator validator) { _ffmpegVersion = validator.GetFFmpegVersion(); @@ -382,6 +391,11 @@ namespace MediaBrowser.MediaEncoding.Encoder return false; } + public bool SupportsBitStreamFilterWithOption(BitStreamFilterOptionType option) + { + return _bitStreamFiltersWithOption.TryGetValue(option, out var val) && val; + } + public bool CanEncodeToAudioCodec(string codec) { if (string.Equals(codec, "opus", StringComparison.OrdinalIgnoreCase)) @@ -501,6 +515,12 @@ namespace MediaBrowser.MediaEncoding.Encoder var args = extractChapters ? "{0} -i {1} -threads {2} -v warning -print_format json -show_streams -show_chapters -show_format" : "{0} -i {1} -threads {2} -v warning -print_format json -show_streams -show_format"; + + if (_proberSupportsFirstVideoFrame) + { + args += " -show_frames -only_first_vframe"; + } + args = string.Format(CultureInfo.InvariantCulture, args, probeSizeArgument, inputPath, _threads).Trim(); var process = new Process diff --git a/MediaBrowser.MediaEncoding/Probing/InternalMediaInfoResult.cs b/MediaBrowser.MediaEncoding/Probing/InternalMediaInfoResult.cs index d4d153b08..53eea64db 100644 --- a/MediaBrowser.MediaEncoding/Probing/InternalMediaInfoResult.cs +++ b/MediaBrowser.MediaEncoding/Probing/InternalMediaInfoResult.cs @@ -30,5 +30,12 @@ namespace MediaBrowser.MediaEncoding.Probing /// The chapters. [JsonPropertyName("chapters")] public IReadOnlyList Chapters { get; set; } + + /// + /// Gets or sets the frames. + /// + /// The streams. + [JsonPropertyName("frames")] + public IReadOnlyList Frames { get; set; } } } diff --git a/MediaBrowser.MediaEncoding/Probing/MediaFrameInfo.cs b/MediaBrowser.MediaEncoding/Probing/MediaFrameInfo.cs new file mode 100644 index 000000000..bed4368ed --- /dev/null +++ b/MediaBrowser.MediaEncoding/Probing/MediaFrameInfo.cs @@ -0,0 +1,184 @@ +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace MediaBrowser.MediaEncoding.Probing; + +/// +/// Class MediaFrameInfo. +/// +public class MediaFrameInfo +{ + /// + /// Gets or sets the media type. + /// + [JsonPropertyName("media_type")] + public string? MediaType { get; set; } + + /// + /// Gets or sets the StreamIndex. + /// + [JsonPropertyName("stream_index")] + public int? StreamIndex { get; set; } + + /// + /// Gets or sets the KeyFrame. + /// + [JsonPropertyName("key_frame")] + public int? KeyFrame { get; set; } + + /// + /// Gets or sets the Pts. + /// + [JsonPropertyName("pts")] + public long? Pts { get; set; } + + /// + /// Gets or sets the PtsTime. + /// + [JsonPropertyName("pts_time")] + public string? PtsTime { get; set; } + + /// + /// Gets or sets the BestEffortTimestamp. + /// + [JsonPropertyName("best_effort_timestamp")] + public long BestEffortTimestamp { get; set; } + + /// + /// Gets or sets the BestEffortTimestampTime. + /// + [JsonPropertyName("best_effort_timestamp_time")] + public string? BestEffortTimestampTime { get; set; } + + /// + /// Gets or sets the Duration. + /// + [JsonPropertyName("duration")] + public int Duration { get; set; } + + /// + /// Gets or sets the DurationTime. + /// + [JsonPropertyName("duration_time")] + public string? DurationTime { get; set; } + + /// + /// Gets or sets the PktPos. + /// + [JsonPropertyName("pkt_pos")] + public string? PktPos { get; set; } + + /// + /// Gets or sets the PktSize. + /// + [JsonPropertyName("pkt_size")] + public string? PktSize { get; set; } + + /// + /// Gets or sets the Width. + /// + [JsonPropertyName("width")] + public int? Width { get; set; } + + /// + /// Gets or sets the Height. + /// + [JsonPropertyName("height")] + public int? Height { get; set; } + + /// + /// Gets or sets the CropTop. + /// + [JsonPropertyName("crop_top")] + public int? CropTop { get; set; } + + /// + /// Gets or sets the CropBottom. + /// + [JsonPropertyName("crop_bottom")] + public int? CropBottom { get; set; } + + /// + /// Gets or sets the CropLeft. + /// + [JsonPropertyName("crop_left")] + public int? CropLeft { get; set; } + + /// + /// Gets or sets the CropRight. + /// + [JsonPropertyName("crop_right")] + public int? CropRight { get; set; } + + /// + /// Gets or sets the PixFmt. + /// + [JsonPropertyName("pix_fmt")] + public string? PixFmt { get; set; } + + /// + /// Gets or sets the SampleAspectRatio. + /// + [JsonPropertyName("sample_aspect_ratio")] + public string? SampleAspectRatio { get; set; } + + /// + /// Gets or sets the PictType. + /// + [JsonPropertyName("pict_type")] + public string? PictType { get; set; } + + /// + /// Gets or sets the InterlacedFrame. + /// + [JsonPropertyName("interlaced_frame")] + public int? InterlacedFrame { get; set; } + + /// + /// Gets or sets the TopFieldFirst. + /// + [JsonPropertyName("top_field_first")] + public int? TopFieldFirst { get; set; } + + /// + /// Gets or sets the RepeatPict. + /// + [JsonPropertyName("repeat_pict")] + public int? RepeatPict { get; set; } + + /// + /// Gets or sets the ColorRange. + /// + [JsonPropertyName("color_range")] + public string? ColorRange { get; set; } + + /// + /// Gets or sets the ColorSpace. + /// + [JsonPropertyName("color_space")] + public string? ColorSpace { get; set; } + + /// + /// Gets or sets the ColorPrimaries. + /// + [JsonPropertyName("color_primaries")] + public string? ColorPrimaries { get; set; } + + /// + /// Gets or sets the ColorTransfer. + /// + [JsonPropertyName("color_transfer")] + public string? ColorTransfer { get; set; } + + /// + /// Gets or sets the ChromaLocation. + /// + [JsonPropertyName("chroma_location")] + public string? ChromaLocation { get; set; } + + /// + /// Gets or sets the SideDataList. + /// + [JsonPropertyName("side_data_list")] + public IReadOnlyList? SideDataList { get; set; } +} diff --git a/MediaBrowser.MediaEncoding/Probing/MediaFrameSideDataInfo.cs b/MediaBrowser.MediaEncoding/Probing/MediaFrameSideDataInfo.cs new file mode 100644 index 000000000..3f7dd9a69 --- /dev/null +++ b/MediaBrowser.MediaEncoding/Probing/MediaFrameSideDataInfo.cs @@ -0,0 +1,16 @@ +using System.Text.Json.Serialization; + +namespace MediaBrowser.MediaEncoding.Probing; + +/// +/// Class MediaFrameSideDataInfo. +/// Currently only records the SideDataType for HDR10+ detection. +/// +public class MediaFrameSideDataInfo +{ + /// + /// Gets or sets the SideDataType. + /// + [JsonPropertyName("side_data_type")] + public string? SideDataType { get; set; } +} diff --git a/MediaBrowser.MediaEncoding/Probing/ProbeResultNormalizer.cs b/MediaBrowser.MediaEncoding/Probing/ProbeResultNormalizer.cs index 6b0fd9a14..a98dbe597 100644 --- a/MediaBrowser.MediaEncoding/Probing/ProbeResultNormalizer.cs +++ b/MediaBrowser.MediaEncoding/Probing/ProbeResultNormalizer.cs @@ -105,8 +105,9 @@ namespace MediaBrowser.MediaEncoding.Probing SetSize(data, info); var internalStreams = data.Streams ?? Array.Empty(); + var internalFrames = data.Frames ?? Array.Empty(); - info.MediaStreams = internalStreams.Select(s => GetMediaStream(isAudio, s, data.Format)) + info.MediaStreams = internalStreams.Select(s => GetMediaStream(isAudio, s, data.Format, internalFrames)) .Where(i => i is not null) // Drop subtitle streams if we don't know the codec because it will just cause failures if we don't know how to handle them .Where(i => i.Type != MediaStreamType.Subtitle || !string.IsNullOrWhiteSpace(i.Codec)) @@ -685,8 +686,9 @@ namespace MediaBrowser.MediaEncoding.Probing /// if set to true [is info]. /// The stream info. /// The format info. + /// The frame info. /// MediaStream. - private MediaStream GetMediaStream(bool isAudio, MediaStreamInfo streamInfo, MediaFormatInfo formatInfo) + private MediaStream GetMediaStream(bool isAudio, MediaStreamInfo streamInfo, MediaFormatInfo formatInfo, IReadOnlyList frameInfoList) { // These are mp4 chapters if (string.Equals(streamInfo.CodecName, "mov_text", StringComparison.OrdinalIgnoreCase)) @@ -904,6 +906,15 @@ namespace MediaBrowser.MediaEncoding.Probing } } } + + var frameInfo = frameInfoList?.FirstOrDefault(i => i.StreamIndex == stream.Index); + if (frameInfo?.SideDataList != null) + { + if (frameInfo.SideDataList.Any(data => string.Equals(data.SideDataType, "HDR Dynamic Metadata SMPTE2094-40 (HDR10+)", StringComparison.OrdinalIgnoreCase))) + { + stream.Hdr10PlusPresentFlag = true; + } + } } else if (streamInfo.CodecType == CodecType.Data) { -- cgit v1.2.3