Migrate to trickplay table to EF. Rename vars/methods/members to have consistent use of tile and thumbnail

1 files changed, 468 insertions, 0 deletions

diff --git a/Jellyfin.Server.Implementations/Trickplay/TrickplayManager.cs b/Jellyfin.Server.Implementations/Trickplay/TrickplayManager.cs
new file mode 100644
index 000000000..34b27e21a
--- /dev/null
+++ b/Jellyfin.Server.Implementations/Trickplay/TrickplayManager.cs
@@ -0,0 +1,468 @@
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using Jellyfin.Data.Entities;
+using MediaBrowser.Controller.Configuration;
+using MediaBrowser.Controller.Drawing;
+using MediaBrowser.Controller.Entities;
+using MediaBrowser.Controller.Library;
+using MediaBrowser.Controller.MediaEncoding;
+using MediaBrowser.Controller.Trickplay;
+using MediaBrowser.Model.Configuration;
+using MediaBrowser.Model.Entities;
+using MediaBrowser.Model.IO;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
+
+namespace Jellyfin.Server.Implementations.Trickplay;
+
+/// <summary>
+/// ITrickplayManager implementation.
+/// </summary>
+public class TrickplayManager : ITrickplayManager
+{
+    private readonly ILogger<TrickplayManager> _logger;
+    private readonly IMediaEncoder _mediaEncoder;
+    private readonly IFileSystem _fileSystem;
+    private readonly EncodingHelper _encodingHelper;
+    private readonly ILibraryManager _libraryManager;
+    private readonly IServerConfigurationManager _config;
+    private readonly IImageEncoder _imageEncoder;
+    private readonly IDbContextFactory<JellyfinDbContext> _dbProvider;
+
+    private static readonly SemaphoreSlim _resourcePool = new(1, 1);
+    private static readonly string[] _trickplayImgExtensions = { ".jpg" };
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="TrickplayManager"/> class.
+    /// </summary>
+    /// <param name="logger">The logger.</param>
+    /// <param name="mediaEncoder">The media encoder.</param>
+    /// <param name="fileSystem">The file systen.</param>
+    /// <param name="encodingHelper">The encoding helper.</param>
+    /// <param name="libraryManager">The library manager.</param>
+    /// <param name="config">The server configuration manager.</param>
+    /// <param name="imageEncoder">The image encoder.</param>
+    /// <param name="dbProvider">The database provider.</param>
+    public TrickplayManager(
+        ILogger<TrickplayManager> logger,
+        IMediaEncoder mediaEncoder,
+        IFileSystem fileSystem,
+        EncodingHelper encodingHelper,
+        ILibraryManager libraryManager,
+        IServerConfigurationManager config,
+        IImageEncoder imageEncoder,
+        IDbContextFactory<JellyfinDbContext> dbProvider)
+    {
+        _logger = logger;
+        _mediaEncoder = mediaEncoder;
+        _fileSystem = fileSystem;
+        _encodingHelper = encodingHelper;
+        _libraryManager = libraryManager;
+        _config = config;
+        _imageEncoder = imageEncoder;
+        _dbProvider = dbProvider;
+    }
+
+    /// <inheritdoc />
+    public async Task RefreshTrickplayDataAsync(Video video, bool replace, CancellationToken cancellationToken)
+    {
+        _logger.LogDebug("Trickplay refresh for {ItemId} (replace existing: {Replace})", video.Id, replace);
+
+        var options = _config.Configuration.TrickplayOptions;
+        foreach (var width in options.WidthResolutions)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+            await RefreshTrickplayDataInternal(
+                video,
+                replace,
+                width,
+                options,
+                cancellationToken).ConfigureAwait(false);
+        }
+    }
+
+    private async Task RefreshTrickplayDataInternal(
+        Video video,
+        bool replace,
+        int width,
+        TrickplayOptions options,
+        CancellationToken cancellationToken)
+    {
+        if (!CanGenerateTrickplay(video, options.Interval))
+        {
+            return;
+        }
+
+        var imgTempDir = string.Empty;
+        var outputDir = GetTrickplayDirectory(video, width);
+
+        await _resourcePool.WaitAsync(cancellationToken).ConfigureAwait(false);
+
+        try
+        {
+            if (!replace && Directory.Exists(outputDir) && (await GetTrickplayResolutions(video.Id).ConfigureAwait(false)).ContainsKey(width))
+            {
+                _logger.LogDebug("Found existing trickplay files for {ItemId}. Exiting.", video.Id);
+                return;
+            }
+
+            // Extract images
+            // Note: Media sources under parent items exist as their own video/item as well. Only use this video stream for trickplay.
+            var mediaSource = video.GetMediaSources(false).Find(source => Guid.Parse(source.Id).Equals(video.Id));
+
+            if (mediaSource is null)
+            {
+                _logger.LogDebug("Found no matching media source for item {ItemId}", video.Id);
+                return;
+            }
+
+            var mediaPath = mediaSource.Path;
+            var mediaStream = mediaSource.VideoStream;
+            var container = mediaSource.Container;
+
+            _logger.LogInformation("Creating trickplay files at {Width} width, for {Path} [ID: {ItemId}]", width, mediaPath, video.Id);
+            imgTempDir = await _mediaEncoder.ExtractVideoImagesOnIntervalAccelerated(
+                mediaPath,
+                container,
+                mediaSource,
+                mediaStream,
+                width,
+                TimeSpan.FromMilliseconds(options.Interval),
+                options.EnableHwAcceleration,
+                options.ProcessThreads,
+                options.Qscale,
+                options.ProcessPriority,
+                _encodingHelper,
+                cancellationToken).ConfigureAwait(false);
+
+            if (string.IsNullOrEmpty(imgTempDir) || !Directory.Exists(imgTempDir))
+            {
+                throw new InvalidOperationException("Null or invalid directory from media encoder.");
+            }
+
+            var images = _fileSystem.GetFiles(imgTempDir, _trickplayImgExtensions, false, false)
+                .Select(i => i.FullName)
+                .OrderBy(i => i)
+                .ToList();
+
+            // Create tiles
+            var tilesTempDir = Path.Combine(imgTempDir, Guid.NewGuid().ToString("N"));
+            var trickplayInfo = CreateTiles(images, width, options, tilesTempDir, outputDir);
+
+            // Save tiles info
+            try
+            {
+                if (trickplayInfo is not null)
+                {
+                    trickplayInfo.ItemId = video.Id;
+                    await SaveTrickplayInfo(trickplayInfo).ConfigureAwait(false);
+
+                    _logger.LogInformation("Finished creation of trickplay files for {0}", mediaPath);
+                }
+                else
+                {
+                    throw new InvalidOperationException("Null trickplay tiles info from CreateTiles.");
+                }
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "Error while saving trickplay tiles info.");
+
+                // Make sure no files stay in metadata folders on failure
+                // if tiles info wasn't saved.
+                Directory.Delete(outputDir, true);
+            }
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Error creating trickplay images.");
+        }
+        finally
+        {
+            _resourcePool.Release();
+
+            if (!string.IsNullOrEmpty(imgTempDir))
+            {
+                Directory.Delete(imgTempDir, true);
+            }
+        }
+    }
+
+    private TrickplayInfo CreateTiles(List<string> images, int width, TrickplayOptions options, string workDir, string outputDir)
+    {
+        if (images.Count == 0)
+        {
+            throw new ArgumentException("Can't create trickplay from 0 images.");
+        }
+
+        Directory.CreateDirectory(workDir);
+
+        var trickplayInfo = new TrickplayInfo
+        {
+            Width = width,
+            Interval = options.Interval,
+            TileWidth = options.TileWidth,
+            TileHeight = options.TileHeight,
+            ThumbnailCount = images.Count,
+            // Set during image generation
+            Height = 0,
+            Bandwidth = 0
+        };
+
+        /*
+         * Generate trickplay tiles from sets of thumbnails
+         */
+        var imageOptions = new ImageCollageOptions
+        {
+            Width = trickplayInfo.TileWidth,
+            Height = trickplayInfo.TileHeight
+        };
+
+        var thumbnailsPerTile = trickplayInfo.TileWidth * trickplayInfo.TileHeight;
+        var requiredTiles = (int)Math.Ceiling((double)images.Count / thumbnailsPerTile);
+
+        for (int i = 0; i < requiredTiles; i++)
+        {
+            // Set output/input paths
+            var tilePath = Path.Combine(workDir, $"{i}.jpg");
+
+            imageOptions.OutputPath = tilePath;
+            imageOptions.InputPaths = images.Skip(i * thumbnailsPerTile).Take(thumbnailsPerTile).ToList();
+
+            // Generate image and use returned height for tiles info
+            var height = _imageEncoder.CreateTrickplayTile(imageOptions, options.JpegQuality, trickplayInfo.Width, trickplayInfo.Height != 0 ? trickplayInfo.Height : null);
+            if (trickplayInfo.Height == 0)
+            {
+                trickplayInfo.Height = height;
+            }
+
+            // Update bitrate
+            var bitrate = (int)Math.Ceiling((decimal)new FileInfo(tilePath).Length * 8 / trickplayInfo.TileWidth / trickplayInfo.TileHeight / (trickplayInfo.Interval / 1000));
+            trickplayInfo.Bandwidth = Math.Max(trickplayInfo.Bandwidth, bitrate);
+        }
+
+        /*
+         * Move trickplay tiles to output directory
+         */
+        Directory.CreateDirectory(Directory.GetParent(outputDir)!.FullName);
+
+        // Replace existing tiles if they already exist
+        if (Directory.Exists(outputDir))
+        {
+            Directory.Delete(outputDir, true);
+        }
+
+        MoveDirectory(workDir, outputDir);
+
+        return trickplayInfo;
+    }
+
+    private bool CanGenerateTrickplay(Video video, int interval)
+    {
+        var videoType = video.VideoType;
+        if (videoType == VideoType.Iso || videoType == VideoType.Dvd || videoType == VideoType.BluRay)
+        {
+            return false;
+        }
+
+        if (video.IsPlaceHolder)
+        {
+            return false;
+        }
+
+        if (video.IsShortcut)
+        {
+            return false;
+        }
+
+        if (!video.IsCompleteMedia)
+        {
+            return false;
+        }
+
+        if (!video.RunTimeTicks.HasValue || video.RunTimeTicks.Value < TimeSpan.FromMilliseconds(interval).Ticks)
+        {
+            return false;
+        }
+
+        var libraryOptions = _libraryManager.GetLibraryOptions(video);
+        if (libraryOptions is null || !libraryOptions.EnableTrickplayImageExtraction)
+        {
+            return false;
+        }
+
+        // Can't extract images if there are no video streams
+        return video.GetMediaStreams().Count > 0;
+    }
+
+    /// <inheritdoc />
+    public async Task<Dictionary<int, TrickplayInfo>> GetTrickplayResolutions(Guid itemId)
+    {
+        var trickplayResolutions = new Dictionary<int, TrickplayInfo>();
+
+        var dbContext = await _dbProvider.CreateDbContextAsync().ConfigureAwait(false);
+        await using (dbContext.ConfigureAwait(false))
+        {
+            var trickplayInfos = await dbContext.TrickplayInfos
+                .AsNoTracking()
+                .Where(i => i.ItemId.Equals(itemId))
+                .ToListAsync()
+                .ConfigureAwait(false);
+
+            foreach (var info in trickplayInfos)
+            {
+                trickplayResolutions[info.Width] = info;
+            }
+        }
+
+        return trickplayResolutions;
+    }
+
+    /// <inheritdoc />
+    public async Task SaveTrickplayInfo(TrickplayInfo info)
+    {
+        var dbContext = await _dbProvider.CreateDbContextAsync().ConfigureAwait(false);
+        await using (dbContext.ConfigureAwait(false))
+        {
+            var oldInfo = await dbContext.TrickplayInfos.FindAsync(info.ItemId, info.Width).ConfigureAwait(false);
+            if (oldInfo is not null)
+            {
+                dbContext.TrickplayInfos.Remove(oldInfo);
+            }
+
+            dbContext.Add(info);
+
+            await dbContext.SaveChangesAsync().ConfigureAwait(false);
+        }
+    }
+
+    /// <inheritdoc />
+    public async Task<Dictionary<Guid, Dictionary<int, TrickplayInfo>>> GetTrickplayManifest(BaseItem item)
+    {
+        var trickplayManifest = new Dictionary<Guid, Dictionary<int, TrickplayInfo>>();
+        foreach (var mediaSource in item.GetMediaSources(false))
+        {
+            var mediaSourceId = Guid.Parse(mediaSource.Id);
+            var trickplayResolutions = await GetTrickplayResolutions(mediaSourceId).ConfigureAwait(false);
+
+            if (trickplayResolutions.Count > 0)
+            {
+                trickplayManifest[mediaSourceId] = trickplayResolutions;
+            }
+        }
+
+        return trickplayManifest;
+    }
+
+    /// <inheritdoc />
+    public string GetTrickplayTilePath(BaseItem item, int width, int index)
+    {
+        return Path.Combine(GetTrickplayDirectory(item, width), index + ".jpg");
+    }
+
+    /// <inheritdoc />
+    public async Task<string?> GetHlsPlaylist(Guid itemId, int width, string? apiKey)
+    {
+        var trickplayResolutions = await GetTrickplayResolutions(itemId).ConfigureAwait(false);
+        if (trickplayResolutions is not null && trickplayResolutions.TryGetValue(width, out var trickplayInfo))
+        {
+            var builder = new StringBuilder(128);
+
+            if (trickplayInfo.ThumbnailCount > 0)
+            {
+                const string urlFormat = "Trickplay/{0}/{1}.jpg?MediaSourceId={2}&api_key={3}";
+                const string decimalFormat = "{0:0.###}";
+
+                var resolution = $"{trickplayInfo.Width}x{trickplayInfo.Height}";
+                var layout = $"{trickplayInfo.TileWidth}x{trickplayInfo.TileHeight}";
+                var thumbnailsPerTile = trickplayInfo.TileWidth * trickplayInfo.TileHeight;
+                var thumbnailDuration = trickplayInfo.Interval / 1000d;
+                var infDuration = thumbnailDuration * thumbnailsPerTile;
+                var tileCount = (int)Math.Ceiling((decimal)trickplayInfo.ThumbnailCount / thumbnailsPerTile);
+
+                builder
+                    .AppendLine("#EXTM3U")
+                    .Append("#EXT-X-TARGETDURATION:")
+                    .AppendLine(tileCount.ToString(CultureInfo.InvariantCulture))
+                    .AppendLine("#EXT-X-VERSION:7")
+                    .AppendLine("#EXT-X-MEDIA-SEQUENCE:1")
+                    .AppendLine("#EXT-X-PLAYLIST-TYPE:VOD")
+                    .AppendLine("#EXT-X-IMAGES-ONLY");
+
+                for (int i = 0; i < tileCount; i++)
+                {
+                    // All tiles prior to the last must contain full amount of thumbnails (no black).
+                    if (i == tileCount - 1)
+                    {
+                        thumbnailsPerTile = trickplayInfo.ThumbnailCount - (i * thumbnailsPerTile);
+                        infDuration = thumbnailDuration * thumbnailsPerTile;
+                    }
+
+                    // EXTINF
+                    builder
+                        .Append("#EXTINF:")
+                        .AppendFormat(CultureInfo.InvariantCulture, decimalFormat, infDuration)
+                        .AppendLine(",");
+
+                    // EXT-X-TILES
+                    builder
+                        .Append("#EXT-X-TILES:RESOLUTION=")
+                        .Append(resolution)
+                        .Append(",LAYOUT=")
+                        .Append(layout)
+                        .Append(",DURATION=")
+                        .AppendFormat(CultureInfo.InvariantCulture, decimalFormat, thumbnailDuration)
+                        .AppendLine();
+
+                    // URL
+                    builder
+                        .AppendFormat(
+                            CultureInfo.InvariantCulture,
+                            urlFormat,
+                            width.ToString(CultureInfo.InvariantCulture),
+                            i.ToString(CultureInfo.InvariantCulture),
+                            itemId.ToString("N"),
+                            apiKey)
+                        .AppendLine();
+                }
+
+                builder.AppendLine("#EXT-X-ENDLIST");
+                return builder.ToString();
+            }
+        }
+
+        return null;
+    }
+
+    private string GetTrickplayDirectory(BaseItem item, int? width = null)
+    {
+        var path = Path.Combine(item.GetInternalMetadataPath(), "trickplay");
+
+        return width.HasValue ? Path.Combine(path, width.Value.ToString(CultureInfo.InvariantCulture)) : path;
+    }
+
+    private void MoveDirectory(string source, string destination)
+    {
+        try
+        {
+            Directory.Move(source, destination);
+        }
+        catch (IOException)
+        {
+            // Cross device move requires a copy
+            Directory.CreateDirectory(destination);
+            foreach (string file in Directory.GetFiles(source))
+            {
+                File.Copy(file, Path.Join(destination, Path.GetFileName(file)), true);
+            }
+
+            Directory.Delete(source, true);
+        }
+    }
+}