From 0a2cf69a5500ef3feb2964011e27c5c5bb599de7 Mon Sep 17 00:00:00 2001 From: evan314159 <110177090+evan314159@users.noreply.github.com> Date: Mon, 8 Sep 2025 04:40:27 +0800 Subject: Additional debug logging for SQLite connections (#14753) --- .../SqliteDatabaseProvider.cs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'src/Jellyfin.Database') diff --git a/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/SqliteDatabaseProvider.cs b/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/SqliteDatabaseProvider.cs index e52ab69d71..ccf84e6012 100644 --- a/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/SqliteDatabaseProvider.cs +++ b/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/SqliteDatabaseProvider.cs @@ -47,13 +47,25 @@ public sealed class SqliteDatabaseProvider : IJellyfinDatabaseProvider sqliteConnectionBuilder.Cache = Enum.Parse(databaseConfiguration.CustomProviderOptions?.Options.FirstOrDefault(e => e.Key.Equals("cache", StringComparison.OrdinalIgnoreCase))?.Value ?? nameof(SqliteCacheMode.Default)); sqliteConnectionBuilder.Pooling = (databaseConfiguration.CustomProviderOptions?.Options.FirstOrDefault(e => e.Key.Equals("pooling", StringComparison.OrdinalIgnoreCase))?.Value ?? bool.FalseString).Equals(bool.TrueString, StringComparison.OrdinalIgnoreCase); + var connectionString = sqliteConnectionBuilder.ToString(); + + // Log SQLite connection parameters + _logger.LogInformation("SQLite connection string: {ConnectionString}", connectionString); + options .UseSqlite( - sqliteConnectionBuilder.ToString(), + connectionString, sqLiteOptions => sqLiteOptions.MigrationsAssembly(GetType().Assembly)) // TODO: Remove when https://github.com/dotnet/efcore/pull/35873 is merged & released .ConfigureWarnings(warnings => warnings.Ignore(RelationalEventId.NonTransactionalMigrationOperationWarning)); + + var enableSensitiveDataLoggingOption = databaseConfiguration.CustomProviderOptions?.Options.FirstOrDefault(e => e.Key.Equals("EnableSensitiveDataLogging", StringComparison.OrdinalIgnoreCase))?.Value; + if (!string.IsNullOrEmpty(enableSensitiveDataLoggingOption) && bool.TryParse(enableSensitiveDataLoggingOption, out bool enableSensitiveDataLogging) && enableSensitiveDataLogging) + { + options.EnableSensitiveDataLogging(enableSensitiveDataLogging); + _logger.LogInformation("EnableSensitiveDataLogging is enabled on SQLite connection"); + } } /// -- cgit v1.2.3 From a99e67544a82da388ce041f836355f86dbc03609 Mon Sep 17 00:00:00 2001 From: JPVenson Date: Fri, 12 Sep 2025 21:57:33 +0200 Subject: Reenable pooling (#14778) --- .../ScheduledTasks/Tasks/OptimizeDatabaseTask.cs | 2 +- .../SqliteDatabaseProvider.cs | 17 ++++++----------- 2 files changed, 7 insertions(+), 12 deletions(-) (limited to 'src/Jellyfin.Database') diff --git a/Emby.Server.Implementations/ScheduledTasks/Tasks/OptimizeDatabaseTask.cs b/Emby.Server.Implementations/ScheduledTasks/Tasks/OptimizeDatabaseTask.cs index bf8ffaf479..92d7a3907a 100644 --- a/Emby.Server.Implementations/ScheduledTasks/Tasks/OptimizeDatabaseTask.cs +++ b/Emby.Server.Implementations/ScheduledTasks/Tasks/OptimizeDatabaseTask.cs @@ -61,7 +61,7 @@ public class OptimizeDatabaseTask : IScheduledTask, IConfigurableScheduledTask yield return new TaskTriggerInfo { Type = TaskTriggerInfoType.IntervalTrigger, - IntervalTicks = TimeSpan.FromHours(24).Ticks + IntervalTicks = TimeSpan.FromHours(6).Ticks }; } diff --git a/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/SqliteDatabaseProvider.cs b/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/SqliteDatabaseProvider.cs index ccf84e6012..d51e8fd645 100644 --- a/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/SqliteDatabaseProvider.cs +++ b/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/SqliteDatabaseProvider.cs @@ -45,7 +45,7 @@ public sealed class SqliteDatabaseProvider : IJellyfinDatabaseProvider var sqliteConnectionBuilder = new SqliteConnectionStringBuilder(); sqliteConnectionBuilder.DataSource = Path.Combine(_applicationPaths.DataPath, "jellyfin.db"); sqliteConnectionBuilder.Cache = Enum.Parse(databaseConfiguration.CustomProviderOptions?.Options.FirstOrDefault(e => e.Key.Equals("cache", StringComparison.OrdinalIgnoreCase))?.Value ?? nameof(SqliteCacheMode.Default)); - sqliteConnectionBuilder.Pooling = (databaseConfiguration.CustomProviderOptions?.Options.FirstOrDefault(e => e.Key.Equals("pooling", StringComparison.OrdinalIgnoreCase))?.Value ?? bool.FalseString).Equals(bool.TrueString, StringComparison.OrdinalIgnoreCase); + sqliteConnectionBuilder.Pooling = (databaseConfiguration.CustomProviderOptions?.Options.FirstOrDefault(e => e.Key.Equals("pooling", StringComparison.OrdinalIgnoreCase))?.Value ?? bool.TrueString).Equals(bool.TrueString, StringComparison.OrdinalIgnoreCase); var connectionString = sqliteConnectionBuilder.ToString(); @@ -74,16 +74,11 @@ public sealed class SqliteDatabaseProvider : IJellyfinDatabaseProvider var context = await DbContextFactory!.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); await using (context.ConfigureAwait(false)) { - if (context.Database.IsSqlite()) - { - await context.Database.ExecuteSqlRawAsync("PRAGMA optimize", cancellationToken).ConfigureAwait(false); - await context.Database.ExecuteSqlRawAsync("VACUUM", cancellationToken).ConfigureAwait(false); - _logger.LogInformation("jellyfin.db optimized successfully!"); - } - else - { - _logger.LogInformation("This database doesn't support optimization"); - } + await context.Database.ExecuteSqlRawAsync("PRAGMA wal_checkpoint(TRUNCATE)", cancellationToken).ConfigureAwait(false); + await context.Database.ExecuteSqlRawAsync("PRAGMA optimize", cancellationToken).ConfigureAwait(false); + await context.Database.ExecuteSqlRawAsync("VACUUM", cancellationToken).ConfigureAwait(false); + await context.Database.ExecuteSqlRawAsync("PRAGMA wal_checkpoint(TRUNCATE)", cancellationToken).ConfigureAwait(false); + _logger.LogInformation("jellyfin.db optimized successfully!"); } } -- cgit v1.2.3 From f0e60a7ff332601b4cc955969c09bdf6f1dd0118 Mon Sep 17 00:00:00 2001 From: JPVenson Date: Fri, 12 Sep 2025 21:57:40 +0200 Subject: Improve optimistic locking behavior (#14779) --- .../Locking/OptimisticLockBehavior.cs | 24 ++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) (limited to 'src/Jellyfin.Database') diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/Locking/OptimisticLockBehavior.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/Locking/OptimisticLockBehavior.cs index 9395b2e2dd..b90a2e056f 100644 --- a/src/Jellyfin.Database/Jellyfin.Database.Implementations/Locking/OptimisticLockBehavior.cs +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/Locking/OptimisticLockBehavior.cs @@ -1,6 +1,7 @@ using System; using System.Data.Common; using System.Linq; +using System.Security.Cryptography; using System.Threading; using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; @@ -28,15 +29,34 @@ public class OptimisticLockBehavior : IEntityFrameworkCoreLockingBehavior TimeSpan[] sleepDurations = [ TimeSpan.FromMilliseconds(50), TimeSpan.FromMilliseconds(50), + TimeSpan.FromMilliseconds(50), + TimeSpan.FromMilliseconds(50), + TimeSpan.FromMilliseconds(250), TimeSpan.FromMilliseconds(250), + TimeSpan.FromMilliseconds(250), + TimeSpan.FromMilliseconds(150), + TimeSpan.FromMilliseconds(150), TimeSpan.FromMilliseconds(150), TimeSpan.FromMilliseconds(500), + TimeSpan.FromMilliseconds(150), TimeSpan.FromMilliseconds(500), + TimeSpan.FromMilliseconds(150), TimeSpan.FromSeconds(3) ]; + + Func backoffProvider = (index, context) => + { + var backoff = sleepDurations[index]; + return backoff + TimeSpan.FromMilliseconds(RandomNumberGenerator.GetInt32(0, (int)(backoff.TotalMilliseconds * .5))); + }; + _logger = logger; - _writePolicy = Policy.HandleInner(e => e.Message.Contains("database is locked", StringComparison.InvariantCultureIgnoreCase)).WaitAndRetry(sleepDurations, RetryHandle); - _writeAsyncPolicy = Policy.HandleInner(e => e.Message.Contains("database is locked", StringComparison.InvariantCultureIgnoreCase)).WaitAndRetryAsync(sleepDurations, RetryHandle); + _writePolicy = Policy + .HandleInner(e => e.Message.Contains("database is locked", StringComparison.InvariantCultureIgnoreCase)) + .WaitAndRetry(sleepDurations.Length, backoffProvider, RetryHandle); + _writeAsyncPolicy = Policy + .HandleInner(e => e.Message.Contains("database is locked", StringComparison.InvariantCultureIgnoreCase)) + .WaitAndRetryAsync(sleepDurations.Length, backoffProvider, RetryHandle); void RetryHandle(Exception exception, TimeSpan timespan, int retryNo, Context context) { -- cgit v1.2.3 From 24410d8a2e05e7abd552fae2732c337afe0d63bd Mon Sep 17 00:00:00 2001 From: JPVenson Date: Sun, 14 Sep 2025 19:24:35 +0200 Subject: Reenable common PRAGMA setters (#14791) --- .../PragmaConnectionInterceptor.cs | 108 +++++++++++++++++++++ .../SqliteDatabaseProvider.cs | 38 ++++++-- 2 files changed, 140 insertions(+), 6 deletions(-) create mode 100644 src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/PragmaConnectionInterceptor.cs (limited to 'src/Jellyfin.Database') diff --git a/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/PragmaConnectionInterceptor.cs b/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/PragmaConnectionInterceptor.cs new file mode 100644 index 0000000000..47e44d97b9 --- /dev/null +++ b/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/PragmaConnectionInterceptor.cs @@ -0,0 +1,108 @@ +using System.Collections.Generic; +using System.Data.Common; +using System.Globalization; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore.Diagnostics; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Database.Providers.Sqlite; + +/// +/// Injects a series of PRAGMA on each connection starts. +/// +public class PragmaConnectionInterceptor : DbConnectionInterceptor +{ + private readonly ILogger _logger; + private readonly int? _cacheSize; + private readonly string _lockingMode; + private readonly int? _journalSizeLimit; + private readonly int _tempStoreMode; + private readonly int _syncMode; + private readonly IDictionary _customPragma; + + /// + /// Initializes a new instance of the class. + /// + /// The logger. + /// Cache size. + /// Locking mode. + /// Journal Size. + /// The https://sqlite.org/pragma.html#pragma_temp_store pragma. + /// The https://sqlite.org/pragma.html#pragma_synchronous pragma. + /// A list of custom provided Pragma in the list of CustomOptions starting with "#PRAGMA:". + public PragmaConnectionInterceptor(ILogger logger, int? cacheSize, string lockingMode, int? journalSizeLimit, int tempStoreMode, int syncMode, IDictionary customPragma) + { + _logger = logger; + _cacheSize = cacheSize; + _lockingMode = lockingMode; + _journalSizeLimit = journalSizeLimit; + _tempStoreMode = tempStoreMode; + _syncMode = syncMode; + _customPragma = customPragma; + + InitialCommand = BuildCommandText(); + _logger.LogInformation("SQLITE connection pragma command set to: \r\n {PragmaCommand}", InitialCommand); + } + + private string? InitialCommand { get; set; } + + /// + public override void ConnectionOpened(DbConnection connection, ConnectionEndEventData eventData) + { + base.ConnectionOpened(connection, eventData); + + using (var command = connection.CreateCommand()) + { +#pragma warning disable CA2100 // Review SQL queries for security vulnerabilities + command.CommandText = InitialCommand; +#pragma warning restore CA2100 // Review SQL queries for security vulnerabilities + command.ExecuteNonQuery(); + } + } + + /// + public override async Task ConnectionOpenedAsync(DbConnection connection, ConnectionEndEventData eventData, CancellationToken cancellationToken = default) + { + await base.ConnectionOpenedAsync(connection, eventData, cancellationToken).ConfigureAwait(false); + + var command = connection.CreateCommand(); + await using (command.ConfigureAwait(false)) + { +#pragma warning disable CA2100 // Review SQL queries for security vulnerabilities + command.CommandText = InitialCommand; +#pragma warning restore CA2100 // Review SQL queries for security vulnerabilities + await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + } + + private string BuildCommandText() + { + var sb = new StringBuilder(); + if (_cacheSize.HasValue) + { + sb.AppendLine(CultureInfo.InvariantCulture, $"PRAGMA cache_size={_cacheSize.Value};"); + } + + if (!string.IsNullOrWhiteSpace(_lockingMode)) + { + sb.AppendLine(CultureInfo.InvariantCulture, $"PRAGMA locking_mode={_lockingMode};"); + } + + if (_journalSizeLimit.HasValue) + { + sb.AppendLine(CultureInfo.InvariantCulture, $"PRAGMA journal_size_limit={_journalSizeLimit};"); + } + + sb.AppendLine(CultureInfo.InvariantCulture, $"PRAGMA synchronous={_syncMode};"); + sb.AppendLine(CultureInfo.InvariantCulture, $"PRAGMA temp_store={_tempStoreMode};"); + + foreach (var item in _customPragma) + { + sb.AppendLine(CultureInfo.InvariantCulture, $"PRAGMA {item.Key}={item.Value};"); + } + + return sb.ToString(); + } +} diff --git a/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/SqliteDatabaseProvider.cs b/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/SqliteDatabaseProvider.cs index d51e8fd645..2b000b257b 100644 --- a/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/SqliteDatabaseProvider.cs +++ b/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/SqliteDatabaseProvider.cs @@ -42,10 +42,28 @@ public sealed class SqliteDatabaseProvider : IJellyfinDatabaseProvider /// public void Initialise(DbContextOptionsBuilder options, DatabaseConfigurationOptions databaseConfiguration) { + static T? GetOption(ICollection? options, string key, Func converter, Func? defaultValue = null) + { + if (options is null) + { + return defaultValue is not null ? defaultValue() : default; + } + + var value = options.FirstOrDefault(e => e.Key.Equals(key, StringComparison.OrdinalIgnoreCase)); + if (value is null) + { + return defaultValue is not null ? defaultValue() : default; + } + + return converter(value.Value); + } + + var customOptions = databaseConfiguration.CustomProviderOptions?.Options; + var sqliteConnectionBuilder = new SqliteConnectionStringBuilder(); sqliteConnectionBuilder.DataSource = Path.Combine(_applicationPaths.DataPath, "jellyfin.db"); - sqliteConnectionBuilder.Cache = Enum.Parse(databaseConfiguration.CustomProviderOptions?.Options.FirstOrDefault(e => e.Key.Equals("cache", StringComparison.OrdinalIgnoreCase))?.Value ?? nameof(SqliteCacheMode.Default)); - sqliteConnectionBuilder.Pooling = (databaseConfiguration.CustomProviderOptions?.Options.FirstOrDefault(e => e.Key.Equals("pooling", StringComparison.OrdinalIgnoreCase))?.Value ?? bool.TrueString).Equals(bool.TrueString, StringComparison.OrdinalIgnoreCase); + sqliteConnectionBuilder.Cache = GetOption(customOptions, "cache", Enum.Parse, () => SqliteCacheMode.Default); + sqliteConnectionBuilder.Pooling = GetOption(customOptions, "pooling", e => e.Equals(bool.TrueString, StringComparison.OrdinalIgnoreCase), () => true); var connectionString = sqliteConnectionBuilder.ToString(); @@ -58,10 +76,18 @@ public sealed class SqliteDatabaseProvider : IJellyfinDatabaseProvider sqLiteOptions => sqLiteOptions.MigrationsAssembly(GetType().Assembly)) // TODO: Remove when https://github.com/dotnet/efcore/pull/35873 is merged & released .ConfigureWarnings(warnings => - warnings.Ignore(RelationalEventId.NonTransactionalMigrationOperationWarning)); - - var enableSensitiveDataLoggingOption = databaseConfiguration.CustomProviderOptions?.Options.FirstOrDefault(e => e.Key.Equals("EnableSensitiveDataLogging", StringComparison.OrdinalIgnoreCase))?.Value; - if (!string.IsNullOrEmpty(enableSensitiveDataLoggingOption) && bool.TryParse(enableSensitiveDataLoggingOption, out bool enableSensitiveDataLogging) && enableSensitiveDataLogging) + warnings.Ignore(RelationalEventId.NonTransactionalMigrationOperationWarning)) + .AddInterceptors(new PragmaConnectionInterceptor( + _logger, + GetOption(customOptions, "cacheSize", e => int.Parse(e, CultureInfo.InvariantCulture)), + GetOption(customOptions, "lockingmode", e => e, () => "NORMAL")!, + GetOption(customOptions, "journalsizelimit", int.Parse, () => 134_217_728), + GetOption(customOptions, "tempstoremode", int.Parse, () => 2), + GetOption(customOptions, "syncmode", int.Parse, () => 1), + customOptions?.Where(e => e.Key.StartsWith("#PRAGMA:", StringComparison.OrdinalIgnoreCase)).ToDictionary(e => e.Key["#PRAGMA:".Length..], e => e.Value) ?? [])); + + var enableSensitiveDataLogging = GetOption(customOptions, "EnableSensitiveDataLogging", e => e.Equals(bool.TrueString, StringComparison.OrdinalIgnoreCase), () => false); + if (enableSensitiveDataLogging) { options.EnableSensitiveDataLogging(enableSensitiveDataLogging); _logger.LogInformation("EnableSensitiveDataLogging is enabled on SQLite connection"); -- cgit v1.2.3 From a0b3e2b071509f440db10768f6f8984c7ea382d6 Mon Sep 17 00:00:00 2001 From: JPVenson Date: Tue, 16 Sep 2025 21:08:04 +0200 Subject: Optimize internal querying of UserData, other fixes (#14795) --- .../Library/LibraryManager.cs | 10 +- .../Library/MusicManager.cs | 13 +- .../Library/UserDataManager.cs | 8 +- Jellyfin.Api/Controllers/YearsController.cs | 5 +- .../Item/BaseItemRepository.cs | 104 +- .../Item/PeopleRepository.cs | 6 +- MediaBrowser.Controller/Entities/BaseItem.cs | 7 + MediaBrowser.Controller/Entities/Folder.cs | 122 +- MediaBrowser.Controller/Entities/Movies/BoxSet.cs | 4 +- MediaBrowser.Controller/Entities/TV/Season.cs | 2 +- MediaBrowser.Controller/Entities/TV/Series.cs | 1 + MediaBrowser.Controller/Entities/UserView.cs | 10 +- .../Persistence/IItemRepository.cs | 10 + MediaBrowser.Controller/Playlists/Playlist.cs | 6 +- .../Entities/BaseItemEntity.cs | 4 + .../ModelConfiguration/BaseItemConfiguration.cs | 1 + ...entChildRelationBaseItemWithCascade.Designer.cs | 1721 ++++++++++++++++++++ ...ProperParentChildRelationBaseItemWithCascade.cs | 30 + .../Migrations/JellyfinDbModelSnapshot.cs | 14 +- .../Controllers/LibraryStructureControllerTests.cs | 2 + 20 files changed, 1988 insertions(+), 92 deletions(-) create mode 100644 src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/Migrations/20250913211637_AddProperParentChildRelationBaseItemWithCascade.Designer.cs create mode 100644 src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/Migrations/20250913211637_AddProperParentChildRelationBaseItemWithCascade.cs (limited to 'src/Jellyfin.Database') diff --git a/Emby.Server.Implementations/Library/LibraryManager.cs b/Emby.Server.Implementations/Library/LibraryManager.cs index 58a971f62a..0074df80a0 100644 --- a/Emby.Server.Implementations/Library/LibraryManager.cs +++ b/Emby.Server.Implementations/Library/LibraryManager.cs @@ -1090,6 +1090,7 @@ namespace Emby.Server.Implementations.Library public async Task ValidateTopLibraryFolders(CancellationToken cancellationToken, bool removeRoot = false) { + RootFolder.Children = null; await RootFolder.RefreshMetadata(cancellationToken).ConfigureAwait(false); // Start by just validating the children of the root, but go no further @@ -1100,9 +1101,12 @@ namespace Emby.Server.Implementations.Library allowRemoveRoot: removeRoot, cancellationToken: cancellationToken).ConfigureAwait(false); - await GetUserRootFolder().RefreshMetadata(cancellationToken).ConfigureAwait(false); + var rootFolder = GetUserRootFolder(); + rootFolder.Children = null; - await GetUserRootFolder().ValidateChildren( + await rootFolder.RefreshMetadata(cancellationToken).ConfigureAwait(false); + + await rootFolder.ValidateChildren( new Progress(), new MetadataRefreshOptions(new DirectoryService(_fileSystem)), recursive: false, @@ -1110,7 +1114,7 @@ namespace Emby.Server.Implementations.Library cancellationToken: cancellationToken).ConfigureAwait(false); // Quickly scan CollectionFolders for changes - foreach (var child in GetUserRootFolder().Children.OfType()) + foreach (var child in rootFolder.Children!.OfType()) { // If the user has somehow deleted the collection directory, remove the metadata from the database. if (child is CollectionFolder collectionFolder && !Directory.Exists(collectionFolder.Path)) diff --git a/Emby.Server.Implementations/Library/MusicManager.cs b/Emby.Server.Implementations/Library/MusicManager.cs index 28cf695007..e0c8ae371b 100644 --- a/Emby.Server.Implementations/Library/MusicManager.cs +++ b/Emby.Server.Implementations/Library/MusicManager.cs @@ -45,11 +45,14 @@ namespace Emby.Server.Implementations.Library public IReadOnlyList GetInstantMixFromFolder(Folder item, User? user, DtoOptions dtoOptions) { var genres = item - .GetRecursiveChildren(user, new InternalItemsQuery(user) - { - IncludeItemTypes = [BaseItemKind.Audio], - DtoOptions = dtoOptions - }) + .GetRecursiveChildren( + user, + new InternalItemsQuery(user) + { + IncludeItemTypes = [BaseItemKind.Audio], + DtoOptions = dtoOptions + }, + out _) .Cast