From 9c15f96e12a0d48a70cbca8380bf78a4f2512b03 Mon Sep 17 00:00:00 2001 From: cvium Date: Thu, 23 Sep 2021 15:29:12 +0200 Subject: Add first draft of keyframe extraction for Matroska --- .../FfProbe/FfProbeKeyframeExtractor.cs | 10 ++ .../FfTool/FfToolKeyframeExtractor.cs | 10 ++ .../Jellyfin.MediaEncoding.Keyframes.csproj | 24 +++ .../KeyframeData.cs | 28 ++++ .../KeyframeExtractor.cs | 56 +++++++ .../Matroska/Extensions/EbmlReaderExtensions.cs | 181 +++++++++++++++++++++ .../Matroska/MatroskaConstants.cs | 31 ++++ .../Matroska/MatroskaKeyframeExtractor.cs | 76 +++++++++ .../Matroska/Models/Info.cs | 29 ++++ .../Matroska/Models/SeekHead.cs | 36 ++++ 10 files changed, 481 insertions(+) create mode 100644 src/Jellyfin.MediaEncoding.Keyframes/FfProbe/FfProbeKeyframeExtractor.cs create mode 100644 src/Jellyfin.MediaEncoding.Keyframes/FfTool/FfToolKeyframeExtractor.cs create mode 100644 src/Jellyfin.MediaEncoding.Keyframes/Jellyfin.MediaEncoding.Keyframes.csproj create mode 100644 src/Jellyfin.MediaEncoding.Keyframes/KeyframeData.cs create mode 100644 src/Jellyfin.MediaEncoding.Keyframes/KeyframeExtractor.cs create mode 100644 src/Jellyfin.MediaEncoding.Keyframes/Matroska/Extensions/EbmlReaderExtensions.cs create mode 100644 src/Jellyfin.MediaEncoding.Keyframes/Matroska/MatroskaConstants.cs create mode 100644 src/Jellyfin.MediaEncoding.Keyframes/Matroska/MatroskaKeyframeExtractor.cs create mode 100644 src/Jellyfin.MediaEncoding.Keyframes/Matroska/Models/Info.cs create mode 100644 src/Jellyfin.MediaEncoding.Keyframes/Matroska/Models/SeekHead.cs (limited to 'src/Jellyfin.MediaEncoding.Keyframes') diff --git a/src/Jellyfin.MediaEncoding.Keyframes/FfProbe/FfProbeKeyframeExtractor.cs b/src/Jellyfin.MediaEncoding.Keyframes/FfProbe/FfProbeKeyframeExtractor.cs new file mode 100644 index 000000000..249608ef9 --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Keyframes/FfProbe/FfProbeKeyframeExtractor.cs @@ -0,0 +1,10 @@ +using System; + +namespace Jellyfin.MediaEncoding.Keyframes.FfProbe +{ + public static class FfProbeKeyframeExtractor + { + // TODO + public static KeyframeData GetKeyframeData(string ffProbePath, string filePath) => throw new NotImplementedException(); + } +} diff --git a/src/Jellyfin.MediaEncoding.Keyframes/FfTool/FfToolKeyframeExtractor.cs b/src/Jellyfin.MediaEncoding.Keyframes/FfTool/FfToolKeyframeExtractor.cs new file mode 100644 index 000000000..89c149ff4 --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Keyframes/FfTool/FfToolKeyframeExtractor.cs @@ -0,0 +1,10 @@ +using System; + +namespace Jellyfin.MediaEncoding.Keyframes.FfTool +{ + public static class FfToolKeyframeExtractor + { + // TODO + public static KeyframeData GetKeyframeData(string ffProbePath, string filePath) => throw new NotImplementedException(); + } +} diff --git a/src/Jellyfin.MediaEncoding.Keyframes/Jellyfin.MediaEncoding.Keyframes.csproj b/src/Jellyfin.MediaEncoding.Keyframes/Jellyfin.MediaEncoding.Keyframes.csproj new file mode 100644 index 000000000..7a984658b --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Keyframes/Jellyfin.MediaEncoding.Keyframes.csproj @@ -0,0 +1,24 @@ + + + + net5.0 + Jellyfin.MediaEncoding.Keyframes + + + + + + + + + + + + + + + ..\..\..\..\..\..\Program Files\dotnet\packs\Microsoft.AspNetCore.App.Ref\5.0.0\ref\net5.0\Microsoft.Extensions.Logging.Abstractions.dll + + + + diff --git a/src/Jellyfin.MediaEncoding.Keyframes/KeyframeData.cs b/src/Jellyfin.MediaEncoding.Keyframes/KeyframeData.cs new file mode 100644 index 000000000..3122f827c --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Keyframes/KeyframeData.cs @@ -0,0 +1,28 @@ +using System.Collections.Generic; + +namespace Jellyfin.MediaEncoding.Keyframes +{ + public class KeyframeData + { + /// + /// Initializes a new instance of the class. + /// + /// The total duration of the video stream in ticks. + /// The video keyframes in ticks. + public KeyframeData(long totalDuration, IReadOnlyList keyframeTicks) + { + TotalDuration = totalDuration; + KeyframeTicks = keyframeTicks; + } + + /// + /// Gets the total duration of the stream in ticks. + /// + public long TotalDuration { get; } + + /// + /// Gets the keyframes in ticks. + /// + public IReadOnlyList KeyframeTicks { get; } + } +} diff --git a/src/Jellyfin.MediaEncoding.Keyframes/KeyframeExtractor.cs b/src/Jellyfin.MediaEncoding.Keyframes/KeyframeExtractor.cs new file mode 100644 index 000000000..2ee6b43e6 --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Keyframes/KeyframeExtractor.cs @@ -0,0 +1,56 @@ +using System; +using System.IO; +using Jellyfin.MediaEncoding.Keyframes.FfProbe; +using Jellyfin.MediaEncoding.Keyframes.FfTool; +using Jellyfin.MediaEncoding.Keyframes.Matroska; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.MediaEncoding.Keyframes +{ + /// + /// Manager class for the set of keyframe extractors. + /// + public class KeyframeExtractor + { + private readonly ILogger _logger; + + /// + /// Initializes a new instance of the class. + /// + /// An instance of the interface. + public KeyframeExtractor(ILogger logger) + { + _logger = logger; + } + + /// + /// Extracts the keyframe positions from a video file. + /// + /// Absolute file path to the media file. + /// Absolute file path to the ffprobe executable. + /// Absolute file path to the fftool executable. + /// + public KeyframeData GetKeyframeData(string filePath, string ffProbePath, string ffToolPath) + { + var extension = Path.GetExtension(filePath); + if (string.Equals(extension, ".mkv", StringComparison.OrdinalIgnoreCase)) + { + try + { + return MatroskaKeyframeExtractor.GetKeyframeData(filePath); + } + catch (InvalidOperationException ex) + { + _logger.LogError(ex, "{MatroskaKeyframeExtractor} failed to extract keyframes", nameof(MatroskaKeyframeExtractor)); + } + } + + if (!string.IsNullOrEmpty(ffToolPath)) + { + return FfToolKeyframeExtractor.GetKeyframeData(ffToolPath, filePath); + } + + return FfProbeKeyframeExtractor.GetKeyframeData(ffProbePath, filePath); + } + } +} diff --git a/src/Jellyfin.MediaEncoding.Keyframes/Matroska/Extensions/EbmlReaderExtensions.cs b/src/Jellyfin.MediaEncoding.Keyframes/Matroska/Extensions/EbmlReaderExtensions.cs new file mode 100644 index 000000000..0de0f996c --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Keyframes/Matroska/Extensions/EbmlReaderExtensions.cs @@ -0,0 +1,181 @@ +using System; +using Jellyfin.MediaEncoding.Keyframes.Matroska.Models; +using NEbml.Core; + +namespace Jellyfin.MediaEncoding.Keyframes.Matroska.Extensions +{ + /// + /// Extension methods for the class. + /// + internal static class EbmlReaderExtensions + { + /// + /// Traverses the current container to find the element with identifier. + /// + /// An instance of . + /// The element identifier. + /// A value indicating whether the element was found. + internal static bool FindElement(this EbmlReader reader, ulong identifier) + { + while (reader.ReadNext()) + { + if (reader.ElementId.EncodedValue == identifier) + { + return true; + } + } + + return false; + } + + /// + /// Reads the current position in the file as an unsigned integer converted from binary. + /// + /// An instance of . + /// The unsigned integer. + internal static uint ReadUIntFromBinary(this EbmlReader reader) + { + var buffer = new byte[4]; + reader.ReadBinary(buffer, 0, 4); + if (BitConverter.IsLittleEndian) + { + Array.Reverse(buffer); + } + + return BitConverter.ToUInt32(buffer); + } + + /// + /// Reads from the start of the file to retrieve the SeekHead segment. + /// + /// An instance of . + /// Instance of + internal static SeekHead ReadSeekHead(this EbmlReader reader) + { + reader = reader ?? throw new ArgumentNullException(nameof(reader)); + + if (reader.ElementPosition != 0) + { + throw new InvalidOperationException("File position must be at 0"); + } + + // Skip the header + if (!reader.FindElement(MatroskaConstants.SegmentContainer)) + { + throw new InvalidOperationException("Expected a segment container"); + } + + reader.EnterContainer(); + + long? tracksPosition = null; + long? cuesPosition = null; + long? infoPosition = null; + // The first element should be a SeekHead otherwise we'll have to search manually + if (!reader.FindElement(MatroskaConstants.SeekHead)) + { + throw new InvalidOperationException("Expected a SeekHead"); + } + + reader.EnterContainer(); + while (reader.FindElement(MatroskaConstants.Seek)) + { + reader.EnterContainer(); + reader.ReadNext(); + var type = (ulong)reader.ReadUIntFromBinary(); + switch (type) + { + case MatroskaConstants.Tracks: + reader.ReadNext(); + tracksPosition = (long)reader.ReadUInt(); + break; + case MatroskaConstants.Cues: + reader.ReadNext(); + cuesPosition = (long)reader.ReadUInt(); + break; + case MatroskaConstants.Info: + reader.ReadNext(); + infoPosition = (long)reader.ReadUInt(); + break; + } + + reader.LeaveContainer(); + + if (tracksPosition.HasValue && cuesPosition.HasValue && infoPosition.HasValue) + { + break; + } + } + + reader.LeaveContainer(); + + if (!tracksPosition.HasValue || !cuesPosition.HasValue || !infoPosition.HasValue) + { + throw new InvalidOperationException("SeekHead is missing or does not contain Info, Tracks and Cues positions"); + } + + return new SeekHead(infoPosition.Value, tracksPosition.Value, cuesPosition.Value); + } + + /// + /// Reads from SegmentContainer to retrieve the Info segment. + /// + /// An instance of . + /// Instance of + internal static Info ReadInfo(this EbmlReader reader, long position) + { + reader.ReadAt(position); + + double? duration = null; + reader.EnterContainer(); + // Mandatory element + reader.FindElement(MatroskaConstants.TimestampScale); + var timestampScale = reader.ReadUInt(); + + if (reader.FindElement(MatroskaConstants.Duration)) + { + duration = reader.ReadFloat(); + } + + reader.LeaveContainer(); + + return new Info((long)timestampScale, duration); + } + + /// + /// Enters the Tracks segment and reads all tracks to find the specified type. + /// + /// Instance of . + /// The relative position of the tracks segment. + /// The track type identifier. + /// The first track number with the specified type. + /// Stream type is not found. + internal static ulong FindFirstTrackNumberByType(this EbmlReader reader, long tracksPosition, ulong type) + { + reader.ReadAt(tracksPosition); + + reader.EnterContainer(); + while (reader.FindElement(MatroskaConstants.TrackEntry)) + { + reader.EnterContainer(); + // Mandatory element + reader.FindElement(MatroskaConstants.TrackNumber); + var trackNumber = reader.ReadUInt(); + + // Mandatory element + reader.FindElement(MatroskaConstants.TrackType); + var trackType = reader.ReadUInt(); + + reader.LeaveContainer(); + if (trackType == MatroskaConstants.TrackTypeVideo) + { + reader.LeaveContainer(); + return trackNumber; + } + } + + reader.LeaveContainer(); + + throw new InvalidOperationException($"No stream with type {type} found"); + } + } +} diff --git a/src/Jellyfin.MediaEncoding.Keyframes/Matroska/MatroskaConstants.cs b/src/Jellyfin.MediaEncoding.Keyframes/Matroska/MatroskaConstants.cs new file mode 100644 index 000000000..d18418d45 --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Keyframes/Matroska/MatroskaConstants.cs @@ -0,0 +1,31 @@ +namespace Jellyfin.MediaEncoding.Keyframes.Matroska +{ + /// + /// Constants for the Matroska identifiers. + /// + public static class MatroskaConstants + { + internal const ulong SegmentContainer = 0x18538067; + + internal const ulong SeekHead = 0x114D9B74; + internal const ulong Seek = 0x4DBB; + + internal const ulong Info = 0x1549A966; + internal const ulong TimestampScale = 0x2AD7B1; + internal const ulong Duration = 0x4489; + + internal const ulong Tracks = 0x1654AE6B; + internal const ulong TrackEntry = 0xAE; + internal const ulong TrackNumber = 0xD7; + internal const ulong TrackType = 0x83; + + internal const ulong TrackTypeVideo = 0x1; + internal const ulong TrackTypeSubtitle = 0x11; + + internal const ulong Cues = 0x1C53BB6B; + internal const ulong CueTime = 0xB3; + internal const ulong CuePoint = 0xBB; + internal const ulong CueTrackPositions = 0xB7; + internal const ulong CuePointTrackNumber = 0xF7; + } +} diff --git a/src/Jellyfin.MediaEncoding.Keyframes/Matroska/MatroskaKeyframeExtractor.cs b/src/Jellyfin.MediaEncoding.Keyframes/Matroska/MatroskaKeyframeExtractor.cs new file mode 100644 index 000000000..10d017d2a --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Keyframes/Matroska/MatroskaKeyframeExtractor.cs @@ -0,0 +1,76 @@ +using System; +using System.Collections.Generic; +using System.IO; +using Jellyfin.MediaEncoding.Keyframes.Matroska.Extensions; +using NEbml.Core; + +namespace Jellyfin.MediaEncoding.Keyframes.Matroska +{ + /// + /// The keyframe extractor for the matroska container. + /// + public static class MatroskaKeyframeExtractor + { + /// + /// Extracts the keyframes in ticks (scaled using the container timestamp scale) from the matroska container. + /// + /// The file path. + /// An instance of . + public static KeyframeData GetKeyframeData(string filePath) + { + using var stream = File.OpenRead(filePath); + using var reader = new EbmlReader(stream); + + var seekHead = reader.ReadSeekHead(); + var info = reader.ReadInfo(seekHead.InfoPosition); + var videoTrackNumber = reader.FindFirstTrackNumberByType(seekHead.TracksPosition, MatroskaConstants.TrackTypeVideo); + + var keyframes = new List(); + reader.ReadAt(seekHead.CuesPosition); + reader.EnterContainer(); + + while (reader.FindElement(MatroskaConstants.CuePoint)) + { + reader.EnterContainer(); + ulong? trackNumber = null; + // Mandatory element + reader.FindElement(MatroskaConstants.CueTime); + var cueTime = reader.ReadUInt(); + + // Mandatory element + reader.FindElement(MatroskaConstants.CueTrackPositions); + reader.EnterContainer(); + if (reader.FindElement(MatroskaConstants.CuePointTrackNumber)) + { + trackNumber = reader.ReadUInt(); + } + + reader.LeaveContainer(); + + if (trackNumber == videoTrackNumber) + { + keyframes.Add(ScaleToNanoseconds(cueTime, info.TimestampScale)); + } + + reader.LeaveContainer(); + } + + reader.LeaveContainer(); + + var result = new KeyframeData(ScaleToNanoseconds(info.Duration ?? 0, info.TimestampScale), keyframes); + return result; + } + + private static long ScaleToNanoseconds(ulong unscaledValue, long timestampScale) + { + // TimestampScale is in nanoseconds, scale it to get the value in ticks, 1 tick == 100 ns + return (long)unscaledValue * timestampScale / 100; + } + + private static long ScaleToNanoseconds(double unscaledValue, long timestampScale) + { + // TimestampScale is in nanoseconds, scale it to get the value in ticks, 1 tick == 100 ns + return Convert.ToInt64(unscaledValue * timestampScale / 100); + } + } +} diff --git a/src/Jellyfin.MediaEncoding.Keyframes/Matroska/Models/Info.cs b/src/Jellyfin.MediaEncoding.Keyframes/Matroska/Models/Info.cs new file mode 100644 index 000000000..02c6741ec --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Keyframes/Matroska/Models/Info.cs @@ -0,0 +1,29 @@ +namespace Jellyfin.MediaEncoding.Keyframes.Matroska.Models +{ + /// + /// The matroska Info segment. + /// + internal class Info + { + /// + /// Initializes a new instance of the class. + /// + /// The timestamp scale in nanoseconds. + /// The duration of the entire file. + public Info(long timestampScale, double? duration) + { + TimestampScale = timestampScale; + Duration = duration; + } + + /// + /// Gets the timestamp scale in nanoseconds. + /// + public long TimestampScale { get; } + + /// + /// Gets the total duration of the file. + /// + public double? Duration { get; } + } +} diff --git a/src/Jellyfin.MediaEncoding.Keyframes/Matroska/Models/SeekHead.cs b/src/Jellyfin.MediaEncoding.Keyframes/Matroska/Models/SeekHead.cs new file mode 100644 index 000000000..d9e346c03 --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Keyframes/Matroska/Models/SeekHead.cs @@ -0,0 +1,36 @@ +namespace Jellyfin.MediaEncoding.Keyframes.Matroska.Models +{ + /// + /// The matroska SeekHead segment. All positions are relative to the Segment container. + /// + internal class SeekHead + { + /// + /// Initializes a new instance of the class. + /// + /// The relative file position of the info segment. + /// The relative file position of the tracks segment. + /// The relative file position of the cues segment. + public SeekHead(long infoPosition, long tracksPosition, long cuesPosition) + { + InfoPosition = infoPosition; + TracksPosition = tracksPosition; + CuesPosition = cuesPosition; + } + + /// + /// Gets relative file position of the info segment. + /// + public long InfoPosition { get; } + + /// + /// Gets the relative file position of the tracks segment. + /// + public long TracksPosition { get; } + + /// + /// Gets the relative file position of the cues segment. + /// + public long CuesPosition { get; } + } +} -- cgit v1.2.3