diff options
Diffstat (limited to 'src')
15 files changed, 486 insertions, 109 deletions
diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/DescendantQueryHelper.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/DescendantQueryHelper.cs index b821476390..2ba3faf5f7 100644 --- a/src/Jellyfin.Database/Jellyfin.Database.Implementations/DescendantQueryHelper.cs +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/DescendantQueryHelper.cs @@ -22,6 +22,65 @@ public static class DescendantQueryHelper b => !b.IsFolder && !b.IsVirtualItem; /// <summary> + /// Gets the predicate identifying the items that stand on their own in a library. An alternate + /// version is a second file for the item that links it rather than an item beside it, and an owned + /// item belongs to its owner unless it is an extra (a trailer and the like, which carries both an + /// owner and an extra type). Nothing here turns on who is asking, so a count that applies it + /// answers the same with a user and without one. + /// </summary> + public static Expression<Func<BaseItemEntity, bool>> IsDistinctLibraryItem { get; } = + b => !b.PrimaryVersionId.HasValue && (!b.OwnerId.HasValue || b.ExtraType != null); + + /// <summary> + /// Builds the predicate identifying the items a user has played, counting a multi-version item as + /// played when any of its alternate versions is. Mirrors the aggregation + /// <c>VersionResumeData.ApplyTo</c> performs on the played flag a single item reports, so that a + /// folder's unplayed count cannot disagree with the watched state its members render with. + /// </summary> + /// <param name="userId">The id of the user whose played state to test.</param> + /// <returns>The predicate matching the items that user has played.</returns> + public static Expression<Func<BaseItemEntity, bool>> IsPlayedBy(Guid userId) => + b => b.UserData!.Any(u => u.UserId.Equals(userId) && u.Played) + || b.LinkedChildEntities!.Any(lc => + (lc.ChildType == LinkedChildType.LocalAlternateVersion || lc.ChildType == LinkedChildType.LinkedAlternateVersion) + && lc.Child!.UserData!.Any(u => u.UserId.Equals(userId) && u.Played)); + + /// <summary> + /// Builds the projection pairing an item's id with <see cref="IsPlayedBy"/> evaluated on that same + /// row. A caller that needs the flag alongside the id composes it rather than testing membership of + /// the played set: as a sub-select the set is unbounded by whatever the caller joins it to, so the + /// database builds it from the whole table once per place it appears. + /// </summary> + /// <param name="userId">The id of the user whose played state to test.</param> + /// <returns>The projection of each item onto its id and that user's played state.</returns> + public static Expression<Func<BaseItemEntity, LeafPlayedState>> PlayedStateBy(Guid userId) + { + var played = IsPlayedBy(userId); + var item = played.Parameters[0]; + + // Named members, as the compiler emits for an anonymous type: without them the query provider + // cannot read a later `x.Id` back to the column it was built from and gives up translating. + return Expression.Lambda<Func<BaseItemEntity, LeafPlayedState>>( + Expression.New( + typeof(LeafPlayedState).GetConstructor([typeof(Guid), typeof(bool)])!, + [Expression.Property(item, nameof(BaseItemEntity.Id)), played.Body], + [typeof(LeafPlayedState).GetProperty(nameof(LeafPlayedState.Id))!, typeof(LeafPlayedState).GetProperty(nameof(LeafPlayedState.Played))!]), + item); + } + + /// <summary> + /// Builds the negation of <see cref="IsPlayedBy"/>, so a caller filtering for unplayed items reads + /// the same definition of played as one filtering for played items. + /// </summary> + /// <param name="userId">The id of the user whose played state to test.</param> + /// <returns>The predicate matching the items that user has not played.</returns> + public static Expression<Func<BaseItemEntity, bool>> IsUnplayedBy(Guid userId) + { + var played = IsPlayedBy(userId); + return Expression.Lambda<Func<BaseItemEntity, bool>>(Expression.Not(played.Body), played.Parameters); + } + + /// <summary> /// Gets a queryable of all descendant IDs for a parent item. /// Traverses AncestorIds and LinkedChildren to find all descendants. /// </summary> diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/IJellyfinDatabaseProvider.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/IJellyfinDatabaseProvider.cs index 77abb45f2a..0a72287ba9 100644 --- a/src/Jellyfin.Database/Jellyfin.Database.Implementations/IJellyfinDatabaseProvider.cs +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/IJellyfinDatabaseProvider.cs @@ -45,8 +45,10 @@ public interface IJellyfinDatabaseProvider Task RunScheduledOptimisation(CancellationToken cancellationToken); /// <summary> - /// If supported this should perform any actions that are required on stopping the jellyfin server, including the - /// same maintenance as <see cref="RunScheduledOptimisation(CancellationToken)"/>. + /// If supported this should perform any actions that are required on stopping the jellyfin server. This runs + /// against a deadline imposed by the service manager, so unlike + /// <see cref="RunScheduledOptimisation(CancellationToken)"/> it should only do work whose cost does not grow with + /// the size of the database. /// </summary> /// <param name="cancellationToken">The token that will be used to abort the operation.</param> /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns> diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/LeafPlayedState.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/LeafPlayedState.cs new file mode 100644 index 0000000000..0846013c45 --- /dev/null +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/LeafPlayedState.cs @@ -0,0 +1,10 @@ +using System; + +namespace Jellyfin.Database.Implementations; + +/// <summary> +/// An item's id paired with whether a given user has played it. +/// </summary> +/// <param name="Id">The id of the item.</param> +/// <param name="Played">Whether the user has played the item.</param> +public readonly record struct LeafPlayedState(Guid Id, bool Played); diff --git a/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/Migrations/20250913211637_AddProperParentChildRelationBaseItemWithCascade.cs b/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/Migrations/20250913211637_AddProperParentChildRelationBaseItemWithCascade.cs index a7f5e369ab..156f553fb3 100644 --- a/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/Migrations/20250913211637_AddProperParentChildRelationBaseItemWithCascade.cs +++ b/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/Migrations/20250913211637_AddProperParentChildRelationBaseItemWithCascade.cs @@ -11,27 +11,19 @@ namespace Jellyfin.Server.Implementations.Migrations protected override void Up(MigrationBuilder migrationBuilder) { migrationBuilder.Sql(""" -DELETE FROM BaseItems - WHERE - ParentId IS NOT NULL - AND - NOT EXISTS(SELECT 1 FROM BaseItems parent WHERE parent.Id = BaseItems.ParentId); -DELETE FROM BaseItems - WHERE - ParentId IS NOT NULL - AND - NOT EXISTS(SELECT 1 FROM BaseItems parent WHERE parent.Id = BaseItems.ParentId); -DELETE FROM BaseItems - WHERE - ParentId IS NOT NULL - AND - NOT EXISTS(SELECT 1 FROM BaseItems parent WHERE parent.Id = BaseItems.ParentId); -DELETE FROM BaseItems - WHERE - ParentId IS NOT NULL - AND - NOT EXISTS(SELECT 1 FROM BaseItems parent WHERE parent.Id = BaseItems.ParentId); -"""); + WITH RECURSIVE Orphan ("Id") AS ( + SELECT Child."Id" + FROM "BaseItems" AS Child + WHERE Child."ParentId" IS NOT NULL + AND NOT EXISTS (SELECT 1 FROM "BaseItems" AS Parent WHERE Parent."Id" = Child."ParentId") + UNION + SELECT Descendant."Id" + FROM "BaseItems" AS Descendant + INNER JOIN Orphan ON Descendant."ParentId" = Orphan."Id" + ) + DELETE FROM "BaseItems" WHERE "Id" IN (SELECT "Id" FROM Orphan); + """); + migrationBuilder.AddForeignKey( name: "FK_BaseItems_BaseItems_ParentId", table: "BaseItems", diff --git a/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/Migrations/20260113203012_ChangeOwnerIdToGuid.cs b/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/Migrations/20260113203012_ChangeOwnerIdToGuid.cs index 4927b0e78d..379da0e9be 100644 --- a/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/Migrations/20260113203012_ChangeOwnerIdToGuid.cs +++ b/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/Migrations/20260113203012_ChangeOwnerIdToGuid.cs @@ -11,6 +11,61 @@ namespace Jellyfin.Database.Providers.Sqlite.Migrations /// <inheritdoc /> protected override void Up(MigrationBuilder migrationBuilder) { + migrationBuilder.Sql( + """ + DROP TABLE IF EXISTS "OrphanedBaseItemIds"; + CREATE TEMPORARY TABLE "OrphanedBaseItemIds" ("Id" TEXT NOT NULL PRIMARY KEY); + + INSERT INTO "OrphanedBaseItemIds" ("Id") + WITH RECURSIVE Orphan ("Id") AS ( + SELECT Child."Id" + FROM "BaseItems" AS Child + WHERE Child."ParentId" IS NOT NULL + AND NOT EXISTS (SELECT 1 FROM "BaseItems" AS Parent WHERE Parent."Id" = Child."ParentId") + UNION + SELECT Descendant."Id" + FROM "BaseItems" AS Descendant + INNER JOIN Orphan ON Descendant."ParentId" = Orphan."Id" + ) + SELECT "Id" FROM Orphan; + + -- Keep the play state of the doomed items the way ItemPersistenceService does when it + -- deletes an item: reattach it to the placeholder item instead of letting the + -- FK_UserData_BaseItems_ItemId cascade wipe it. The placeholder can only hold one row + -- per (UserId, CustomDataKey), so resolve collisions before repointing anything. + DELETE FROM "UserData" + WHERE "ItemId" = '00000000-0000-0000-0000-000000000001' + AND EXISTS ( + SELECT 1 + FROM "UserData" AS Doomed + INNER JOIN "OrphanedBaseItemIds" AS Orphan ON Orphan."Id" = Doomed."ItemId" + WHERE Doomed."UserId" = "UserData"."UserId" + AND Doomed."CustomDataKey" = "UserData"."CustomDataKey"); + + DELETE FROM "UserData" + WHERE "ItemId" IN (SELECT "Id" FROM "OrphanedBaseItemIds") + AND "rowid" NOT IN ( + SELECT MIN("rowid") + FROM "UserData" + WHERE "ItemId" IN (SELECT "Id" FROM "OrphanedBaseItemIds") + GROUP BY "UserId", "CustomDataKey"); + + UPDATE "UserData" + SET "ItemId" = '00000000-0000-0000-0000-000000000001', + "RetentionDate" = datetime('now') + WHERE "ItemId" IN (SELECT "Id" FROM "OrphanedBaseItemIds"); + + -- FK_LinkedChildren_BaseItems_{ParentId,ChildId} are NO ACTION, so these rows have to + -- go by hand or the delete below fails on them. + DELETE FROM "LinkedChildren" + WHERE "ParentId" IN (SELECT "Id" FROM "OrphanedBaseItemIds") + OR "ChildId" IN (SELECT "Id" FROM "OrphanedBaseItemIds"); + + DELETE FROM "BaseItems" WHERE "Id" IN (SELECT "Id" FROM "OrphanedBaseItemIds"); + + DROP TABLE "OrphanedBaseItemIds"; + """); + // Normalize OwnerId to uppercase GUID format migrationBuilder.Sql( @"UPDATE BaseItems diff --git a/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/Migrations/20260815063607_RemoveOrphanedUserPermissionsAndPreferences.cs b/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/Migrations/20260815063607_RemoveOrphanedUserPermissionsAndPreferences.cs index 3d4cf90441..2530e84af6 100644 --- a/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/Migrations/20260815063607_RemoveOrphanedUserPermissionsAndPreferences.cs +++ b/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/Migrations/20260815063607_RemoveOrphanedUserPermissionsAndPreferences.cs @@ -11,8 +11,8 @@ namespace Jellyfin.Database.Providers.Sqlite.Migrations /// <inheritdoc /> protected override void Up(MigrationBuilder migrationBuilder) { - migrationBuilder.Sql("DELETE FROM Permissions WHERE UserId IS NULL;"); - migrationBuilder.Sql("DELETE FROM Preferences WHERE UserId IS NULL;"); + migrationBuilder.Sql("DELETE FROM Permissions WHERE UserId IS NULL OR UserId NOT IN (SELECT Id FROM Users);"); + migrationBuilder.Sql("DELETE FROM Preferences WHERE UserId IS NULL OR UserId NOT IN (SELECT Id FROM Users);"); migrationBuilder.DropIndex( name: "IX_Preferences_UserId_Kind", diff --git a/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/SqliteDatabaseProvider.cs b/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/SqliteDatabaseProvider.cs index f11cde7e48..3330b64b69 100644 --- a/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/SqliteDatabaseProvider.cs +++ b/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/SqliteDatabaseProvider.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Data.Common; using System.Globalization; using System.IO; using System.Linq; @@ -117,17 +118,32 @@ public sealed class SqliteDatabaseProvider : IJellyfinDatabaseProvider /// <inheritdoc/> public async Task RunShutdownTask(CancellationToken cancellationToken) { - // Run before disposing the application + // Run before disposing the application. Only a checkpoint: stopping is on a deadline. + + // Empty the pool first. Anything still parked in it can start reading again between here and the + // checkpoint, and a reader that holds the write-ahead log open is exactly what makes the truncation + // fail. Connections handed out already cannot be taken away, but they get disposed on return. + SqliteConnection.ClearAllPools(); + try { - await OptimizeAsync(cancellationToken).ConfigureAwait(false); + if (DbContextFactory is not null) + { + var context = await DbContextFactory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (context.ConfigureAwait(false)) + { + await context.Database.ExecuteSqlRawAsync("PRAGMA wal_checkpoint(TRUNCATE)", cancellationToken).ConfigureAwait(false); + } + } } catch (Exception ex) { - // A missed optimization only costs performance, so never fail the shutdown over this. - _logger.LogError(ex, "Error while optimizing jellyfin.db"); + // A missed checkpoint only leaves a write-ahead log for the next start to replay, so never fail the + // shutdown over this. + _logger.LogError(ex, "Error while checkpointing jellyfin.db"); } + // The checkpointing connection went back into the pool, so retire that one as well. SqliteConnection.ClearAllPools(); } @@ -141,15 +157,72 @@ public sealed class SqliteDatabaseProvider : IJellyfinDatabaseProvider var context = await DbContextFactory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); await using (context.ConfigureAwait(false)) { - await context.Database.ExecuteSqlRawAsync("PRAGMA wal_checkpoint(TRUNCATE)", cancellationToken).ConfigureAwait(false); - await context.Database.ExecuteSqlRawAsync("VACUUM", cancellationToken).ConfigureAwait(false); - await context.Database.ExecuteSqlRawAsync("PRAGMA analysis_limit=0", cancellationToken).ConfigureAwait(false); - await context.Database.ExecuteSqlRawAsync("ANALYZE", cancellationToken).ConfigureAwait(false); - await context.Database.ExecuteSqlRawAsync("PRAGMA wal_checkpoint(TRUNCATE)", cancellationToken).ConfigureAwait(false); - _logger.LogInformation("jellyfin.db optimized successfully!"); + await context.Database.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + try + { + long? tempStore; + long? analysisLimit; + var pragmaCommand = context.Database.GetDbConnection().CreateCommand(); + await using (pragmaCommand.ConfigureAwait(false)) + { + pragmaCommand.CommandText = "PRAGMA temp_store"; + tempStore = await ReadPragmaValueAsync(pragmaCommand, cancellationToken).ConfigureAwait(false); + pragmaCommand.CommandText = "PRAGMA analysis_limit"; + analysisLimit = await ReadPragmaValueAsync(pragmaCommand, cancellationToken).ConfigureAwait(false); + } + + await context.Database.ExecuteSqlRawAsync("PRAGMA wal_checkpoint(TRUNCATE)", cancellationToken).ConfigureAwait(false); + + _logger.LogDebug( + "Rebuilding jellyfin.db on disk, scratch space goes to {TempDirectory}", + Environment.GetEnvironmentVariable("SQLITE_TMPDIR") ?? "SQLite's default temporary directory"); + await context.Database.ExecuteSqlRawAsync("PRAGMA temp_store=1", cancellationToken).ConfigureAwait(false); + try + { + await context.Database.ExecuteSqlRawAsync("VACUUM", cancellationToken).ConfigureAwait(false); + } + finally + { + // The connection goes back to the pool, so hand it over the way it was handed to us. + if (tempStore is not null) + { + await context.Database.ExecuteSqlRawAsync( + FormattableString.Invariant($"PRAGMA temp_store={tempStore.Value}"), + CancellationToken.None).ConfigureAwait(false); + } + } + + await context.Database.ExecuteSqlRawAsync("PRAGMA analysis_limit=0", cancellationToken).ConfigureAwait(false); + try + { + await context.Database.ExecuteSqlRawAsync("ANALYZE", cancellationToken).ConfigureAwait(false); + } + finally + { + if (analysisLimit is not null) + { + await context.Database.ExecuteSqlRawAsync( + FormattableString.Invariant($"PRAGMA analysis_limit={analysisLimit.Value}"), + CancellationToken.None).ConfigureAwait(false); + } + } + + await context.Database.ExecuteSqlRawAsync("PRAGMA wal_checkpoint(TRUNCATE)", cancellationToken).ConfigureAwait(false); + _logger.LogInformation("jellyfin.db optimized successfully!"); + } + finally + { + await context.Database.CloseConnectionAsync().ConfigureAwait(false); + } } } + private static async Task<long?> ReadPragmaValueAsync(DbCommand command, CancellationToken cancellationToken) + { + var value = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false); + return value is null or DBNull ? null : Convert.ToInt64(value, CultureInfo.InvariantCulture); + } + /// <inheritdoc/> public void ConfigureConventions(ModelConfigurationBuilder configurationBuilder) { @@ -157,16 +230,31 @@ public sealed class SqliteDatabaseProvider : IJellyfinDatabaseProvider } /// <inheritdoc /> - public Task<string> MigrationBackupFast(CancellationToken cancellationToken) + public async Task<string> MigrationBackupFast(CancellationToken cancellationToken) { - var key = DateTime.UtcNow.ToString("yyyyMMddhhmmss", CultureInfo.InvariantCulture); var path = Path.Combine(_applicationPaths.DataPath, "jellyfin.db"); - var backupFile = Path.Combine(_applicationPaths.DataPath, BackupFolderName); - Directory.CreateDirectory(backupFile); + var backupFolder = Path.Combine(_applicationPaths.DataPath, BackupFolderName); + Directory.CreateDirectory(backupFolder); + + if (DbContextFactory is not null && File.Exists(path)) + { + var context = await DbContextFactory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (context.ConfigureAwait(false)) + { + await context.Database.ExecuteSqlRawAsync("PRAGMA wal_checkpoint(TRUNCATE)", cancellationToken).ConfigureAwait(false); + } + } + + var key = DateTime.UtcNow.ToString("yyyyMMddHHmmss", CultureInfo.InvariantCulture); + var backupFile = Path.Combine(backupFolder, $"{key}_jellyfin.db"); + for (var attempt = 1; File.Exists(backupFile); attempt++) + { + key = string.Create(CultureInfo.InvariantCulture, $"{DateTime.UtcNow:yyyyMMddHHmmss}_{attempt}"); + backupFile = Path.Combine(backupFolder, $"{key}_jellyfin.db"); + } - backupFile = Path.Combine(backupFile, $"{key}_jellyfin.db"); File.Copy(path, backupFile); - return Task.FromResult(key); + return key; } /// <inheritdoc /> @@ -183,10 +271,55 @@ public sealed class SqliteDatabaseProvider : IJellyfinDatabaseProvider return Task.CompletedTask; } + if (!TryRetireWriteAheadLog(path)) + { + _logger.LogCritical( + "Refusing to restore jellyfin.db: the write-ahead log at {WriteAheadLog} could not be retired, which " + + "means the database is still open and replacing it now would silently bring back the data this " + + "rollback is undoing. Stop the server and copy {Backup} over {Path} by hand.", + path + "-wal", + backupFile, + path); + return Task.CompletedTask; + } + File.Copy(backupFile, path, true); + return Task.CompletedTask; } + private bool TryRetireWriteAheadLog(string path) + { + var writeAheadLogPath = path + "-wal"; + if (!File.Exists(path) || !File.Exists(writeAheadLogPath)) + { + return true; + } + + try + { + var connectionString = new SqliteConnectionStringBuilder + { + DataSource = path, + Mode = SqliteOpenMode.ReadWrite, + Pooling = false + }.ToString(); + + using var connection = new SqliteConnection(connectionString); + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = "PRAGMA wal_checkpoint(TRUNCATE)"; + command.ExecuteNonQuery(); + } + catch (SqliteException ex) + { + // Either something else holds the database or it is too damaged to open. The check below covers both. + _logger.LogError(ex, "Could not open jellyfin.db to retire its write-ahead log"); + } + + return !File.Exists(writeAheadLogPath); + } + /// <inheritdoc /> public Task DeleteBackup(string key) { diff --git a/src/Jellyfin.Drawing.Skia/SkiaEncoder.cs b/src/Jellyfin.Drawing.Skia/SkiaEncoder.cs index 4cdff055f4..b8d40614d2 100644 --- a/src/Jellyfin.Drawing.Skia/SkiaEncoder.cs +++ b/src/Jellyfin.Drawing.Skia/SkiaEncoder.cs @@ -556,6 +556,13 @@ public class SkiaEncoder : IImageEncoder /// <returns>The resized image.</returns> internal static SKImage ResizeImage(SKBitmap source, SKImageInfo targetInfo, bool isAntialias = false, bool isDither = false) { + if (source.Width == targetInfo.Width && source.Height == targetInfo.Height) + { + return SKImage.FromBitmap(source); + } + + var isDownscale = source.Width > targetInfo.Width || source.Height > targetInfo.Height; + using var target = new SKBitmap(targetInfo); using var canvas = new SKCanvas(target); using var paint = new SKPaint(); @@ -565,7 +572,7 @@ public class SkiaEncoder : IImageEncoder // Historically, kHigh implied cubic filtering, but only when upsampling. // If specified kHigh, and were down-sampling, Skia used to switch back to kMedium (bilinear filtering plus mipmaps). // With current skia API, passing Mitchell cubic when down-sampling will cause serious quality degradation. - var samplingOptions = source.Width > targetInfo.Width || source.Height > targetInfo.Height + var samplingOptions = isDownscale ? DefaultSamplingOptions : UpscaleSamplingOptions; @@ -576,7 +583,10 @@ public class SkiaEncoder : IImageEncoder samplingOptions, paint); - SharpenInPlace(target); + if (isDownscale) + { + SharpenInPlace(target); + } return SKImage.FromBitmap(target); } diff --git a/src/Jellyfin.LiveTv/Channels/ChannelManager.cs b/src/Jellyfin.LiveTv/Channels/ChannelManager.cs index ed02fe6a1d..256a652fd2 100644 --- a/src/Jellyfin.LiveTv/Channels/ChannelManager.cs +++ b/src/Jellyfin.LiveTv/Channels/ChannelManager.cs @@ -372,9 +372,7 @@ namespace Jellyfin.LiveTv.Channels { IEnumerable<MediaSourceInfo> results = GetSavedMediaSources(item); - return results - .Select(i => NormalizeMediaSource(item, i)) - .ToList(); + return NormalizeMediaSources(item, results); } /// <summary> @@ -400,9 +398,7 @@ namespace Jellyfin.LiveTv.Channels results = Enumerable.Empty<MediaSourceInfo>(); } - return results - .Select(i => NormalizeMediaSource(item, i)) - .ToList(); + return NormalizeMediaSources(item, results); } private async Task<IEnumerable<MediaSourceInfo>> GetChannelItemMediaSourcesInternal(IRequiresMediaInfoCallback channel, string id, CancellationToken cancellationToken) @@ -420,11 +416,36 @@ namespace Jellyfin.LiveTv.Channels return list; } - private static MediaSourceInfo NormalizeMediaSource(BaseItem item, MediaSourceInfo info) + private static IReadOnlyList<MediaSourceInfo> NormalizeMediaSources(BaseItem item, IEnumerable<MediaSourceInfo> infos) { - info.RunTimeTicks ??= item.RunTimeTicks; + var list = infos.ToList(); + var itemId = item.Id.ToString("N", CultureInfo.InvariantCulture); + var hasDefaultSource = list.Any(i => string.Equals(i.Id, itemId, StringComparison.OrdinalIgnoreCase)); - return info; + for (var index = 0; index < list.Count; index++) + { + var info = list[index]; + info.RunTimeTicks ??= item.RunTimeTicks; + + if (!string.IsNullOrEmpty(info.Id)) + { + continue; + } + + if (!hasDefaultSource) + { + // The source carrying the item id sorts first and becomes the client's default. + info.Id = itemId; + hasDefaultSource = true; + continue; + } + + // Remaining sources need ids that are distinct but stable, as clients send them back to request playback. + var key = string.IsNullOrEmpty(info.Path) ? index.ToString(CultureInfo.InvariantCulture) : info.Path; + info.Id = (itemId + key).GetMD5().ToString("N", CultureInfo.InvariantCulture); + } + + return list; } private async Task<Channel> GetChannel(IChannel channelInfo, CancellationToken cancellationToken) diff --git a/src/Jellyfin.LiveTv/Guide/GuideManager.cs b/src/Jellyfin.LiveTv/Guide/GuideManager.cs index 41520f8789..a11f83f2f8 100644 --- a/src/Jellyfin.LiveTv/Guide/GuideManager.cs +++ b/src/Jellyfin.LiveTv/Guide/GuideManager.cs @@ -125,12 +125,16 @@ public class GuideManager : IGuideManager { var innerProgress = new Progress<double>(p => progress.Report(p * progressPerService)); - var idList = await RefreshChannelsInternal(service, innerProgress, cancellationToken).ConfigureAwait(false); + var (channelIds, programIds, hasErrors) = await RefreshChannelsInternal(service, innerProgress, cancellationToken).ConfigureAwait(false); - newChannelIdList.AddRange(idList.Item1); - newProgramIdList.AddRange(idList.Item2); + newChannelIdList.AddRange(channelIds); + newProgramIdList.AddRange(programIds); + + // The channels that failed did not report any programs, so cleaning the database + // would delete every program they provide instead of keeping the previous ones. + cleanDatabase &= !hasErrors; } - catch (OperationCanceledException) + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; } @@ -172,10 +176,12 @@ public class GuideManager : IGuideManager : 7; } - private async Task<Tuple<List<Guid>, List<Guid>>> RefreshChannelsInternal(ILiveTvService service, IProgress<double> progress, CancellationToken cancellationToken) + private async Task<(List<Guid> ChannelIds, List<Guid> ProgramIds, bool HasErrors)> RefreshChannelsInternal(ILiveTvService service, IProgress<double> progress, CancellationToken cancellationToken) { progress.Report(10); + var hasErrors = false; + var allChannelsList = (await service.GetChannelsAsync(cancellationToken).ConfigureAwait(false)) .Select(i => new Tuple<string, ChannelInfo>(service.Name, i)) .ToList(); @@ -195,12 +201,13 @@ public class GuideManager : IGuideManager list.Add(item); } - catch (OperationCanceledException) + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; } catch (Exception ex) { + hasErrors = true; _logger.LogError(ex, "Error getting channel information for {Name}", channelInfo.Item2.Name); } @@ -314,12 +321,13 @@ public class GuideManager : IGuideManager }, cancellationToken).ConfigureAwait(false); } - catch (OperationCanceledException) + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; } catch (Exception ex) { + hasErrors = true; _logger.LogError(ex, "Error getting programs for channel {Name}", currentChannel.Name); } @@ -330,7 +338,7 @@ public class GuideManager : IGuideManager } progress.Report(100); - return new Tuple<List<Guid>, List<Guid>>(channels, programIds); + return (channels, programIds, hasErrors); } private void CleanDatabase(Guid[] currentIdList, BaseItemKind[] validTypes, IProgress<double> progress, CancellationToken cancellationToken) diff --git a/src/Jellyfin.LiveTv/Listings/ListingsManager.cs b/src/Jellyfin.LiveTv/Listings/ListingsManager.cs index 15e20d6f64..7274a3030c 100644 --- a/src/Jellyfin.LiveTv/Listings/ListingsManager.cs +++ b/src/Jellyfin.LiveTv/Listings/ListingsManager.cs @@ -352,9 +352,12 @@ public class ListingsManager : IListingsManager var xmltvCacheFile = Path.Combine(cachePath, "xmltv", safeId + ".xml"); try { - File.Delete(xmltvCacheFile); + if (File.Exists(xmltvCacheFile)) + { + File.Delete(xmltvCacheFile); + } } - catch (IOException ex) + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { _logger.LogWarning(ex, "Error deleting XMLTV cache file for provider {ProviderId}", safeId); } diff --git a/src/Jellyfin.LiveTv/Listings/SchedulesDirect.cs b/src/Jellyfin.LiveTv/Listings/SchedulesDirect.cs index c93d1f039c..40ec99e9a5 100644 --- a/src/Jellyfin.LiveTv/Listings/SchedulesDirect.cs +++ b/src/Jellyfin.LiveTv/Listings/SchedulesDirect.cs @@ -549,46 +549,52 @@ namespace Jellyfin.LiveTv.Listings { var token = await GetToken(info, cancellationToken).ConfigureAwait(false); - var lineups = new List<NameIdPair>(); - if (string.IsNullOrWhiteSpace(token)) { - return lineups; + throw new AuthenticationException("Could not authenticate with Schedules Direct"); } + var lineups = new List<NameIdPair>(); + using var options = new HttpRequestMessage(HttpMethod.Get, ApiUrl + "/headends?country=" + country + "&postalcode=" + location); options.Headers.TryAddWithoutValidation("token", token); - try + var root = await Request<IReadOnlyList<HeadendsDto>>(options, false, info, cancellationToken).ConfigureAwait(false); + foreach (HeadendsDto headend in root ?? []) { - var root = await Request<IReadOnlyList<HeadendsDto>>(options, false, info, cancellationToken).ConfigureAwait(false); - if (root is not null) + foreach (LineupDto lineup in headend.Lineups ?? []) { - foreach (HeadendsDto headend in root) + lineups.Add(new NameIdPair { - foreach (LineupDto lineup in headend.Lineups) - { - lineups.Add(new NameIdPair - { - Name = string.IsNullOrWhiteSpace(lineup.Name) ? lineup.Lineup : lineup.Name, - Id = lineup.Uri?[18..] - }); - } - } - } - else - { - _logger.LogInformation("No lineups available"); + Name = string.IsNullOrWhiteSpace(lineup.Name) ? lineup.Lineup : lineup.Name, + Id = string.IsNullOrWhiteSpace(lineup.Lineup) ? lineup.Uri?.Split('/')[^1] : lineup.Lineup + }); } } - catch (Exception ex) + + if (lineups.Count == 0) { - _logger.LogError(ex, "Error getting headends"); + _logger.LogWarning( + "Schedules Direct has no lineups for country {Country} and postal code {PostalCode}", + country, + location); } return lineups; } + private void ResetErrorState(ListingsProviderInfo info) + { + _accountError = false; + Interlocked.Exchange(ref _lastErrorResponseTicks, 0); + + // Only the account being saved is retried, the tokens of the other accounts stay valid. + if (!string.IsNullOrWhiteSpace(info.Username)) + { + _tokens.TryRemove(info.Username, out _); + } + } + private async Task<string> GetToken(ListingsProviderInfo info, CancellationToken cancellationToken) { var username = info.Username; @@ -605,15 +611,19 @@ namespace Jellyfin.LiveTv.Listings return null; } - // Permanent account error — SD is disabled for this server lifetime. + // Account error — SD stays disabled until the provider is saved again or the server restarts. if (_accountError) { + _logger.LogWarning("Skipping Schedules Direct request because of an earlier account error. Save the listings provider again to retry."); + return null; } // Avoid hammering SD after transient login failures (e.g. max attempts / temporary lockout) if ((DateTime.UtcNow - new DateTime(Interlocked.Read(ref _lastErrorResponseTicks), DateTimeKind.Utc)).TotalMinutes < 30) { + _logger.LogWarning("Skipping Schedules Direct request because of a recent login failure. Retrying no earlier than 30 minutes after it."); + return null; } @@ -776,7 +786,7 @@ namespace Jellyfin.LiveTv.Listings return root.Token; } - throw new AuthenticationException("Could not authenticate with Schedules Direct Error: " + root.Message); + throw new AuthenticationException("Could not authenticate with Schedules Direct Error: " + (root?.Message ?? "empty response")); } private async Task AddLineupToAccount(ListingsProviderInfo info, CancellationToken cancellationToken) @@ -992,10 +1002,18 @@ namespace Jellyfin.LiveTv.Listings public async Task Validate(ListingsProviderInfo info, bool validateLogin, bool validateListings) { + ResetErrorState(info); + if (validateLogin) { ArgumentException.ThrowIfNullOrEmpty(info.Username); ArgumentException.ThrowIfNullOrEmpty(info.Password); + + var token = await GetToken(info, CancellationToken.None).ConfigureAwait(false); + if (string.IsNullOrWhiteSpace(token)) + { + throw new AuthenticationException("Could not authenticate with Schedules Direct"); + } } if (validateListings) diff --git a/src/Jellyfin.LiveTv/Listings/XmlTvListingsProvider.cs b/src/Jellyfin.LiveTv/Listings/XmlTvListingsProvider.cs index 0aeb7ad05d..f78d464659 100644 --- a/src/Jellyfin.LiveTv/Listings/XmlTvListingsProvider.cs +++ b/src/Jellyfin.LiveTv/Listings/XmlTvListingsProvider.cs @@ -1,6 +1,7 @@ #pragma warning disable CS1591 using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Globalization; using System.IO; @@ -27,11 +28,14 @@ namespace Jellyfin.LiveTv.Listings public class XmlTvListingsProvider : IListingsProvider { private static readonly TimeSpan _maxCacheAge = TimeSpan.FromHours(1); + private static readonly TimeSpan _downloadTimeout = TimeSpan.FromMinutes(15); private readonly IServerConfigurationManager _config; private readonly IHttpClientFactory _httpClientFactory; private readonly ILogger<XmlTvListingsProvider> _logger; + private readonly ConcurrentDictionary<string, DateTime> _lastDownloadFailures = new(StringComparer.Ordinal); + public XmlTvListingsProvider( IServerConfigurationManager config, IHttpClientFactory httpClientFactory, @@ -64,32 +68,51 @@ namespace Jellyfin.LiveTv.Listings string cacheDir = Path.Join(_config.ApplicationPaths.CachePath, "xmltv"); string cacheFile = Path.Join(cacheDir, cacheFilename); - if (File.Exists(cacheFile)) + if (File.Exists(cacheFile) && File.GetLastWriteTimeUtc(cacheFile) >= DateTime.UtcNow.Subtract(_maxCacheAge)) + { + return cacheFile; + } + + var isRemote = info.Path.StartsWith("http", StringComparison.OrdinalIgnoreCase); + + if (isRemote + && _lastDownloadFailures.TryGetValue(info.Path, out var lastFailure) + && DateTime.UtcNow - lastFailure < _maxCacheAge) { - if (File.GetLastWriteTimeUtc(cacheFile) >= DateTime.UtcNow.Subtract(_maxCacheAge)) + if (File.Exists(cacheFile)) { return cacheFile; } - File.Delete(cacheFile); - } - else - { - Directory.CreateDirectory(cacheDir); + throw new InvalidOperationException("Skipping the XMLTV download after a recent failure: " + info.Path); } + Directory.CreateDirectory(cacheDir); + + var tempFile = cacheFile + ".tmp"; + try { - if (info.Path.StartsWith("http", StringComparison.OrdinalIgnoreCase)) + using var timeout = new CancellationTokenSource(_downloadTimeout); + using var linkedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeout.Token); + var downloadCancellationToken = linkedTokenSource.Token; + + if (isRemote) { _logger.LogInformation("Downloading xmltv listings from {Path}", info.Path); - using var response = await _httpClientFactory.CreateClient(NamedClient.Default).GetAsync(info.Path, cancellationToken).ConfigureAwait(false); + var httpClient = _httpClientFactory.CreateClient(NamedClient.Default); + httpClient.Timeout = _downloadTimeout; + + using var response = await httpClient + .GetAsync(info.Path, HttpCompletionOption.ResponseHeadersRead, downloadCancellationToken) + .ConfigureAwait(false); + response.EnsureSuccessStatusCode(); var redirectedUrl = response.RequestMessage?.RequestUri?.ToString() ?? info.Path; - var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + var stream = await response.Content.ReadAsStreamAsync(downloadCancellationToken).ConfigureAwait(false); await using (stream.ConfigureAwait(false)) { - return await UnzipIfNeededAndCopy(redirectedUrl, stream, cacheFile, cancellationToken).ConfigureAwait(false); + await UnzipIfNeededAndCopy(redirectedUrl, stream, tempFile, downloadCancellationToken).ConfigureAwait(false); } } else @@ -97,28 +120,63 @@ namespace Jellyfin.LiveTv.Listings var stream = AsyncFile.OpenRead(info.Path); await using (stream.ConfigureAwait(false)) { - return await UnzipIfNeededAndCopy(info.Path, stream, cacheFile, cancellationToken).ConfigureAwait(false); + await UnzipIfNeededAndCopy(info.Path, stream, tempFile, downloadCancellationToken).ConfigureAwait(false); } } + + File.Move(tempFile, cacheFile, true); + _lastDownloadFailures.TryRemove(info.Path, out _); + + return cacheFile; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + TryDeleteTempFile(tempFile); + + throw; } catch (Exception ex) { + TryDeleteTempFile(tempFile); + _lastDownloadFailures[info.Path] = DateTime.UtcNow; + _logger.LogError(ex, "Error downloading or processing XMLTV file from {Path}", info.Path); if (File.Exists(cacheFile)) { - File.Delete(cacheFile); + _logger.LogWarning("Falling back to the previously downloaded XMLTV file for {Path}", info.Path); + + return cacheFile; + } + + if (ex is OperationCanceledException) + { + throw new TimeoutException( + string.Format(CultureInfo.InvariantCulture, "Timed out downloading the XMLTV file from {0}", info.Path), + ex); } throw; } } - private async Task<string> UnzipIfNeededAndCopy(string originalUrl, Stream stream, string file, CancellationToken cancellationToken) + private void TryDeleteTempFile(string tempFile) + { + try + { + File.Delete(tempFile); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + _logger.LogWarning(ex, "Error deleting temporary XMLTV file {File}", tempFile); + } + } + + private async Task UnzipIfNeededAndCopy(string originalUrl, Stream stream, string file, CancellationToken cancellationToken) { var fileStream = new FileStream( file, - FileMode.CreateNew, + FileMode.Create, FileAccess.Write, FileShare.None, IODefaults.FileStreamBufferSize, @@ -148,15 +206,8 @@ namespace Jellyfin.LiveTv.Listings var fileInfo = new FileInfo(file); if (!fileInfo.Exists || fileInfo.Length == 0) { - if (fileInfo.Exists) - { - File.Delete(file); - } - throw new InvalidOperationException("Downloaded XMLTV file is empty: " + originalUrl); } - - return file; } public async Task<IEnumerable<ProgramInfo>> GetProgramsAsync(ListingsProviderInfo info, string channelId, DateTime startDateUtc, DateTime endDateUtc, CancellationToken cancellationToken) @@ -281,6 +332,13 @@ namespace Jellyfin.LiveTv.Listings public Task Validate(ListingsProviderInfo info, bool validateLogin, bool validateListings) { + // Saving the provider is an explicit retry, so the download backoff has to be dropped + // together with the cached file the listings manager deletes. + if (!string.IsNullOrEmpty(info.Path)) + { + _lastDownloadFailures.TryRemove(info.Path, out _); + } + // Assume all urls are valid. check files for existence if (!info.Path.StartsWith("http", StringComparison.OrdinalIgnoreCase) && !File.Exists(info.Path)) { diff --git a/src/Jellyfin.LiveTv/LiveTvManager.cs b/src/Jellyfin.LiveTv/LiveTvManager.cs index 2edf7681db..93f9083178 100644 --- a/src/Jellyfin.LiveTv/LiveTvManager.cs +++ b/src/Jellyfin.LiveTv/LiveTvManager.cs @@ -1229,6 +1229,14 @@ namespace Jellyfin.LiveTv .Where(IsLiveTvEnabled); } + /// <inheritdoc /> + public bool IsEnabledForUser(User user) + { + ArgumentNullException.ThrowIfNull(user); + + return IsLiveTvEnabled(user); + } + /// <summary> /// Resets the tuner. /// </summary> diff --git a/src/Jellyfin.LiveTv/Recordings/RecordingsManager.cs b/src/Jellyfin.LiveTv/Recordings/RecordingsManager.cs index 62a06370da..8a5aeccba5 100644 --- a/src/Jellyfin.LiveTv/Recordings/RecordingsManager.cs +++ b/src/Jellyfin.LiveTv/Recordings/RecordingsManager.cs @@ -286,7 +286,7 @@ public sealed class RecordingsManager : IRecordingsManager, IDisposable if (requiresRefresh) { - await _libraryManager.ValidateMediaLibrary(new Progress<double>(), CancellationToken.None).ConfigureAwait(false); + _libraryManager.QueueLibraryScan(); } } |