diff options
| author | Cody Robibero <cody@robibe.ro> | 2026-07-26 16:18:49 -0400 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2026-07-26 16:18:49 -0400 |
| commit | 1e4d126cb9f1590cfdc0e731647b93836d1a8867 (patch) | |
| tree | baf2e7ba19fc2026508d7e67567cc4f2f66c9336 /Jellyfin.Server.Implementations | |
| parent | b04614e18d2aeed8d3d3de920c381c6bd5b97bdb (diff) | |
| parent | f3a1d56c563c71573ed2e7681b347533897ee70c (diff) | |
Merge pull request #17422 from Shadowghost/performance
Reduce correlated subqueries to improve query performance
Diffstat (limited to 'Jellyfin.Server.Implementations')
6 files changed, 208 insertions, 108 deletions
diff --git a/Jellyfin.Server.Implementations/Item/BaseItemRepository.ByName.cs b/Jellyfin.Server.Implementations/Item/BaseItemRepository.ByName.cs index 6fe8563bf5..8e917f6951 100644 --- a/Jellyfin.Server.Implementations/Item/BaseItemRepository.ByName.cs +++ b/Jellyfin.Server.Implementations/Item/BaseItemRepository.ByName.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.Linq; using Jellyfin.Data.Enums; +using Jellyfin.Database.Implementations; using Jellyfin.Database.Implementations.Entities; using MediaBrowser.Controller.Entities; using MediaBrowser.Model.Dto; @@ -109,7 +110,6 @@ public sealed partial class BaseItemRepository IsNews = filter.IsNews, IsSeries = filter.IsSeries }) - .Where(e => e.MediaStreams != null) .SelectMany(e => e.MediaStreams!) .Where(e => e.StreamType == (MediaStreamTypeEntity)mediaStreamType) .Select(s => string.IsNullOrEmpty(s.Language) ? "und" : s.Language) // und = undetermined @@ -168,21 +168,16 @@ public sealed partial class BaseItemRepository IsSeries = filter.IsSeries }); - // Keep this as an IQueryable sub-select. Materializing to a list would inline one - // bound parameter per CleanValue and hit SQLite's variable cap on libraries with - // high-cardinality value types (e.g. tens of thousands of artists). - var matchingCleanValues = context.ItemValuesMap - .Where(ivm => itemValueTypes.Contains(ivm.ItemValue.Type)) - .Join( - innerQueryFilter, - ivm => ivm.ItemId, - g => g.Id, - (ivm, g) => ivm.ItemValue.CleanValue) - .Distinct(); - var innerQuery = PrepareItemQuery(context, filter) .Where(e => e.Type == returnType) - .Where(e => matchingCleanValues.Contains(e.CleanName!)); + .Where(e => context.ItemValuesMap + .Where(ivm => itemValueTypes.Contains(ivm.ItemValue.Type) && ivm.ItemValue.CleanValue == e.CleanName) + .Join( + innerQueryFilter, + ivm => ivm.ItemId, + g => g.Id, + (ivm, g) => ivm.ItemId) + .Any()); var outerQueryFilter = new InternalItemsQuery(filter.User) { @@ -205,32 +200,42 @@ public sealed partial class BaseItemRepository ExcludeItemIds = filter.ExcludeItemIds }; - // Collapse rows that share a PresentationUniqueKey (e.g. alternate versions) by picking - // the lowest Id per group. For MusicArtist, prefer the entity from a library the user - // can actually access,since the same artist can have a folder in multiple libraries. - // Keep as an IQueryable sub-select so paging is applied AFTER - // ApplyOrder runs the caller's actual sort. + // Collapse rows that share a PresentationUniqueKey (e.g. alternate versions) into one + // representative id per group, then materialize the representative ids once. var masterQuery = TranslateQuery(innerQuery, context, outerQueryFilter); var isMusicArtist = returnType == _itemTypeLookup.BaseItemKindNames[BaseItemKind.MusicArtist]; - var representativeIds = isMusicArtist - ? masterQuery + List<Guid> representativeIds; + if (isMusicArtist) + { + // For MusicArtist, prefer the entity from a library the user can actually access. + // Materialize to prevent correlated per-group first-row queries which hurt performance. + var topParentIds = filter.TopParentIds; + representativeIds = masterQuery + .Select(e => new { e.Id, e.PresentationUniqueKey, e.TopParentId }) + .AsEnumerable() .GroupBy(e => e.PresentationUniqueKey) .Select(g => g - .OrderBy(e => filter.TopParentIds.Contains(e.TopParentId ?? Guid.Empty) ? 0 : 1) + .OrderBy(e => topParentIds.Contains(e.TopParentId ?? Guid.Empty) ? 0 : 1) .ThenBy(e => e.Id) .First().Id) - : masterQuery + .ToList(); + } + else + { + representativeIds = masterQuery .GroupBy(e => e.PresentationUniqueKey) - .Select(g => g.Min(e => e.Id)); + .Select(g => g.Min(e => e.Id)) + .ToList(); + } var result = new QueryResult<(BaseItemDto, ItemCounts?)>(); if (filter.EnableTotalRecordCount) { - result.TotalRecordCount = representativeIds.Count(); + result.TotalRecordCount = representativeIds.Count; } var query = ApplyNavigations( - context.BaseItems.AsNoTracking().AsSingleQuery().Where(e => representativeIds.Contains(e.Id)), + context.BaseItems.AsNoTracking().AsSingleQuery().WhereOneOrMany(representativeIds, e => e.Id), filter); query = ApplyOrder(query, filter, context); @@ -311,8 +316,8 @@ public sealed partial class BaseItemRepository var itemIds = itemCountQuery.Select(e => e.Id); // Rewrite query to avoid SelectMany on navigation properties (which requires SQL APPLY, not supported on SQLite) - // Instead, start from ItemValueMaps and join with BaseItems - return context.ItemValuesMap + // Instead, start from ItemValueMaps and join with BaseItems. + var rawCounts = context.ItemValuesMap .Where(ivm => itemValueTypes.Contains(ivm.ItemValue.Type)) .Where(ivm => itemIds.Contains(ivm.ItemId)) .Join( @@ -322,18 +327,47 @@ public sealed partial class BaseItemRepository (ivm, e) => new { CleanName = ivm.ItemValue.CleanValue, e.Type }) .GroupBy(x => new { x.CleanName, x.Type }) .Select(g => new { g.Key.CleanName, g.Key.Type, Count = g.Count() }) - .GroupBy(x => x.CleanName) - .ToDictionary( - g => g.Key, - g => new ItemCounts + .AsEnumerable(); + + var countsByCleanName = new Dictionary<string, ItemCounts>(); + foreach (var group in rawCounts.GroupBy(x => x.CleanName)) + { + var counts = new ItemCounts(); + foreach (var row in group) + { + if (row.Type == seriesTypeName) + { + counts.SeriesCount += row.Count; + } + else if (row.Type == episodeTypeName) + { + counts.EpisodeCount += row.Count; + } + else if (row.Type == movieTypeName) + { + counts.MovieCount += row.Count; + } + else if (row.Type == musicAlbumTypeName) + { + counts.AlbumCount += row.Count; + } + else if (row.Type == musicArtistTypeName) { - SeriesCount = g.Where(x => x.Type == seriesTypeName).Sum(x => x.Count), - EpisodeCount = g.Where(x => x.Type == episodeTypeName).Sum(x => x.Count), - MovieCount = g.Where(x => x.Type == movieTypeName).Sum(x => x.Count), - AlbumCount = g.Where(x => x.Type == musicAlbumTypeName).Sum(x => x.Count), - ArtistCount = g.Where(x => x.Type == musicArtistTypeName).Sum(x => x.Count), - SongCount = g.Where(x => x.Type == audioTypeName).Sum(x => x.Count), - TrailerCount = g.Where(x => x.Type == trailerTypeName).Sum(x => x.Count), - }); + counts.ArtistCount += row.Count; + } + else if (row.Type == audioTypeName) + { + counts.SongCount += row.Count; + } + else if (row.Type == trailerTypeName) + { + counts.TrailerCount += row.Count; + } + } + + countsByCleanName[group.Key] = counts; + } + + return countsByCleanName; } } diff --git a/Jellyfin.Server.Implementations/Item/BaseItemRepository.Querying.cs b/Jellyfin.Server.Implementations/Item/BaseItemRepository.Querying.cs index dc16c3b1b3..1ff8d8f863 100644 --- a/Jellyfin.Server.Implementations/Item/BaseItemRepository.Querying.cs +++ b/Jellyfin.Server.Implementations/Item/BaseItemRepository.Querying.cs @@ -126,38 +126,54 @@ public sealed partial class BaseItemRepository if (collectionType is CollectionType.movies) { - // Group by PresentationUniqueKey, pick the newest item per group. - var topGroupItems = baseQuery + // Pick, per PresentationUniqueKey, the newest item; return the newest `limit` of those. + // Build up until limit by streaming through results and deduplicating on the fly. + var orderedIds = baseQuery .Where(e => e.PresentationUniqueKey != null) - .GroupBy(e => e.PresentationUniqueKey) - .Select(g => new - { - MaxDate = g.Max(e => e.DateCreated), - FirstId = g.OrderByDescending(e => e.DateCreated).ThenByDescending(e => e.Id).Select(e => e.Id).First() - }) - .OrderByDescending(g => g.MaxDate); + .OrderByDescending(e => e.DateCreated) + .ThenByDescending(e => e.Id) + .Select(e => new { e.Id, e.PresentationUniqueKey }); - var firstIdsQuery = filter.Limit.HasValue - ? topGroupItems.Take(filter.Limit.Value).Select(g => g.FirstId) - : topGroupItems.Select(g => g.FirstId); + // DistinctBy and Take are lazy, so enumeration stops as soon as limit distinct keys are read. + var firstIds = orderedIds + .AsEnumerable() + .DistinctBy(row => row.PresentationUniqueKey) + .Select(row => row.Id) + .Take(limit ?? int.MaxValue) + .ToList(); - return LoadLatestByIds(context, firstIdsQuery, filter); + return LoadLatestByIds(context, firstIds, filter); } - // Albums whose Id is the parent of any track matching the user's filter. - var albumIdsWithMatchingTrack = context.AncestorIds - .Join(baseQuery, ai => ai.ItemId, t => t.Id, (ai, _) => ai.ParentItemId); - var musicAlbumTypeName = _itemTypeLookup.BaseItemKindNames[BaseItemKind.MusicAlbum]!; - var topAlbumsQuery = context.BaseItems.AsNoTracking() - .Where(album => album.Type == musicAlbumTypeName) - .Where(album => albumIdsWithMatchingTrack.Contains(album.Id)) + IQueryable<BaseItemEntity> topAlbumsQuery; + + // When the query is scoped to whole libraries, read the newest albums directly by their own TopParentId. + if (filter.TopParentIds.Length > 0) + { + topAlbumsQuery = context.BaseItems.AsNoTracking() + .Where(album => album.Type == musicAlbumTypeName + && !album.IsVirtualItem + && album.TopParentId.HasValue) + .WhereOneOrMany(filter.TopParentIds, album => album.TopParentId!.Value); + } + else + { + // Fallback (e.g. AncestorIds-scoped callers): albums that are the parent of a matching track. + var albumIdsWithMatchingTrack = context.AncestorIds + .Join(baseQuery, ai => ai.ItemId, t => t.Id, (ai, _) => ai.ParentItemId); + topAlbumsQuery = context.BaseItems.AsNoTracking() + .Where(album => album.Type == musicAlbumTypeName) + .Where(album => albumIdsWithMatchingTrack.Contains(album.Id)); + } + + var orderedAlbums = topAlbumsQuery .OrderByDescending(album => album.DateCreated) .ThenByDescending(album => album.Id); - var albumIdsQuery = filter.Limit.HasValue - ? topAlbumsQuery.Take(filter.Limit.Value).Select(a => a.Id) - : topAlbumsQuery.Select(a => a.Id); + var albumIdsQuery = limit.HasValue + ? orderedAlbums.Take(limit.Value).Select(a => a.Id) + : orderedAlbums.Select(a => a.Id); return LoadLatestByIds(context, albumIdsQuery, filter); } @@ -181,6 +197,29 @@ public sealed partial class BaseItemRepository .ToArray()!; } + private IReadOnlyList<BaseItemDto> LoadLatestByIds( + JellyfinDbContext context, + List<Guid> ids, + InternalItemsQuery filter) + { + if (ids.Count == 0) + { + return []; + } + + var itemsQuery = ApplyNavigations( + context.BaseItems.AsNoTracking().WhereOneOrMany(ids, e => e.Id), + filter); + + return itemsQuery + .OrderByDescending(e => e.DateCreated) + .ThenByDescending(e => e.Id) + .AsEnumerable() + .Select(w => DeserializeBaseItem(w, filter.SkipDeserialization)) + .Where(dto => dto != null) + .ToArray()!; + } + /// <summary> /// Gets the latest TV show items with smart Season/Series container selection. /// </summary> diff --git a/Jellyfin.Server.Implementations/Item/BaseItemRepository.TranslateQuery.cs b/Jellyfin.Server.Implementations/Item/BaseItemRepository.TranslateQuery.cs index f19df6259e..47f8a40b9c 100644 --- a/Jellyfin.Server.Implementations/Item/BaseItemRepository.TranslateQuery.cs +++ b/Jellyfin.Server.Implementations/Item/BaseItemRepository.TranslateQuery.cs @@ -434,20 +434,34 @@ public sealed partial class BaseItemRepository if (filter.IsLiked.HasValue) { - var isLiked = filter.IsLiked.Value; - baseQuery = baseQuery.Where(e => e.UserData!.Any(ud => ud.UserId == filter.User!.Id && ud.Rating >= UserItemData.MinLikeValue) == isLiked); - } + var likedIds = context.UserData + .Where(ud => ud.UserId == filter.User!.Id && ud.Rating >= UserItemData.MinLikeValue) + .Select(ud => ud.ItemId); - if (filter.IsFavoriteOrLiked.HasValue) - { - var isFavoriteOrLiked = filter.IsFavoriteOrLiked.Value; - baseQuery = baseQuery.Where(e => e.UserData!.Any(ud => ud.UserId == filter.User!.Id && ud.IsFavorite) == isFavoriteOrLiked); + baseQuery = filter.IsLiked.Value + ? baseQuery.Where(e => likedIds.Contains(e.Id)) + : baseQuery.Where(e => !likedIds.Contains(e.Id)); } - if (filter.IsFavorite.HasValue) + if (filter.IsFavoriteOrLiked.HasValue || filter.IsFavorite.HasValue) { - var isFavorite = filter.IsFavorite.Value; - baseQuery = baseQuery.Where(e => e.UserData!.Any(ud => ud.UserId == filter.User!.Id && ud.IsFavorite) == isFavorite); + var favoriteIds = context.UserData + .Where(ud => ud.UserId == filter.User!.Id && ud.IsFavorite) + .Select(ud => ud.ItemId); + + if (filter.IsFavoriteOrLiked.HasValue) + { + baseQuery = filter.IsFavoriteOrLiked.Value + ? baseQuery.Where(e => favoriteIds.Contains(e.Id)) + : baseQuery.Where(e => !favoriteIds.Contains(e.Id)); + } + + if (filter.IsFavorite.HasValue) + { + baseQuery = filter.IsFavorite.Value + ? baseQuery.Where(e => favoriteIds.Contains(e.Id)) + : baseQuery.Where(e => !favoriteIds.Contains(e.Id)); + } } if (filter.IsPlayed.HasValue) @@ -560,16 +574,19 @@ public sealed partial class BaseItemRepository // Only in-progress siblings can eliminate a candidate: a version without progress has a NULL max LastPlayedDate, // which is never greater and never ties. Restricting the sibling scan to the in-progress set keeps this bounded by // the user's Continue Watching count instead of forcing a full BaseItems scan (COALESCE keys are non-indexable) per row. - baseQuery = baseQuery.Where(e => e.Type == seriesTypeName || !context.BaseItems - .Where(s => s.Id != e.Id - && inProgressIds.Contains(s.Id) - && (s.PrimaryVersionId ?? s.Id) == (e.PrimaryVersionId ?? e.Id)) - .Any(s => - inProgress.Where(su => su.ItemId == s.Id).Max(su => su.LastPlayedDate) - > inProgress.Where(eu => eu.ItemId == e.Id).Max(eu => eu.LastPlayedDate) - || (inProgress.Where(su => su.ItemId == s.Id).Max(su => su.LastPlayedDate) - == inProgress.Where(eu => eu.ItemId == e.Id).Max(eu => eu.LastPlayedDate) - && s.Id.CompareTo(e.Id) < 0))); + // Items in no version group at all have no sibling that could eliminate them, so short-circuit the scan for those. + baseQuery = baseQuery.Where(e => e.Type == seriesTypeName + || (e.PrimaryVersionId == null && !context.BaseItems.Any(a => a.PrimaryVersionId == e.Id)) + || !context.BaseItems + .Where(s => s.Id != e.Id + && inProgressIds.Contains(s.Id) + && (s.PrimaryVersionId ?? s.Id) == (e.PrimaryVersionId ?? e.Id)) + .Any(s => + inProgress.Where(su => su.ItemId == s.Id).Max(su => su.LastPlayedDate) + > inProgress.Where(eu => eu.ItemId == e.Id).Max(eu => eu.LastPlayedDate) + || (inProgress.Where(su => su.ItemId == s.Id).Max(su => su.LastPlayedDate) + == inProgress.Where(eu => eu.ItemId == e.Id).Max(eu => eu.LastPlayedDate) + && s.Id.CompareTo(e.Id) < 0))); } else { diff --git a/Jellyfin.Server.Implementations/Item/NextUpService.cs b/Jellyfin.Server.Implementations/Item/NextUpService.cs index 725b4cfaac..f478daef23 100644 --- a/Jellyfin.Server.Implementations/Item/NextUpService.cs +++ b/Jellyfin.Server.Implementations/Item/NextUpService.cs @@ -98,7 +98,7 @@ public class NextUpService : INextUpService .Where(e => e.UserData!.Any(ud => ud.UserId == userId && ud.Played)); lastWatchedBase = _queryHelpers.ApplyAccessFiltering(context, lastWatchedBase, filter); - // Use lightweight projection + client-side grouping to avoid correlated scalar subquery + // Use lightweight projection + client-side dedup to avoid the correlated scalar subquery // per group that EF generates for GroupBy+OrderByDescending+FirstOrDefault. var allPlayedLite = lastWatchedBase .Select(e => new @@ -110,15 +110,11 @@ public class NextUpService : INextUpService }) .ToList(); - var lastWatchedInfo = new Dictionary<string, Guid>(); - foreach (var group in allPlayedLite.GroupBy(e => e.SeriesPresentationUniqueKey)) - { - var lastWatched = group - .OrderByDescending(e => e.ParentIndexNumber) - .ThenByDescending(e => e.IndexNumber) - .First(); - lastWatchedInfo[group.Key!] = lastWatched.Id; - } + var lastWatchedInfo = allPlayedLite + .OrderByDescending(e => e.ParentIndexNumber) + .ThenByDescending(e => e.IndexNumber) + .DistinctBy(e => e.SeriesPresentationUniqueKey) + .ToDictionary(e => e.SeriesPresentationUniqueKey!, e => e.Id); Dictionary<string, Guid> lastWatchedByDateInfo = new(); if (includeWatchedForRewatching) @@ -144,11 +140,10 @@ public class NextUpService : INextUpService (e, ud) => new { EpisodeId = e.Id, e.SeriesPresentationUniqueKey, ud.LastPlayedDate }) .ToList(); - foreach (var group in playedWithDates.GroupBy(x => x.SeriesPresentationUniqueKey)) - { - var mostRecent = group.OrderByDescending(x => x.LastPlayedDate).First(); - lastWatchedByDateInfo[group.Key!] = mostRecent.EpisodeId; - } + lastWatchedByDateInfo = playedWithDates + .OrderByDescending(x => x.LastPlayedDate) + .DistinctBy(x => x.SeriesPresentationUniqueKey) + .ToDictionary(x => x.SeriesPresentationUniqueKey!, x => x.EpisodeId); } var allLastWatchedIds = lastWatchedInfo.Values diff --git a/Jellyfin.Server.Implementations/Item/OrderMapper.cs b/Jellyfin.Server.Implementations/Item/OrderMapper.cs index aac85d0131..25ad81ec6c 100644 --- a/Jellyfin.Server.Implementations/Item/OrderMapper.cs +++ b/Jellyfin.Server.Implementations/Item/OrderMapper.cs @@ -29,19 +29,30 @@ public static class OrderMapper /// <returns>Func to be executed later for sorting query.</returns> public static Expression<Func<BaseItemEntity, object?>> MapOrderByField(ItemSortBy sortBy, InternalItemsQuery query, JellyfinDbContext jellyfinDbContext) { + if (sortBy == ItemSortBy.DatePlayed) + { + // An item's played date is the newest of its own progress and that of its alternate versions, + // which track progress under their own ids. Matching both in one predicate ORs them together, + // which no index can serve: the user's whole UserData table gets scanned per sorted row. + // Two indexed lookups combined by MAX cost a seek each instead. + var userData = query.User is null + ? jellyfinDbContext.UserData + : jellyfinDbContext.UserData.Where(w => w.UserId == query.User.Id); + + return e => userData + .Where(w => w.ItemId == e.Id) + .Select(w => w.LastPlayedDate) + .Concat(userData + .Where(w => w.Item!.PrimaryVersionId == e.Id) + .Select(w => w.LastPlayedDate)) + .Max(); + } + return (sortBy, query.User) switch { (ItemSortBy.AirTime, _) => e => e.SortName, (ItemSortBy.Runtime, _) => e => e.RunTimeTicks, (ItemSortBy.Random, _) => e => EF.Functions.Random(), - (ItemSortBy.DatePlayed, not null) => e => - jellyfinDbContext.UserData - .Where(w => w.UserId == query.User.Id && (w.ItemId == e.Id || w.Item!.PrimaryVersionId == e.Id)) - .Max(f => f.LastPlayedDate), - (ItemSortBy.DatePlayed, null) => e => - jellyfinDbContext.UserData - .Where(w => w.ItemId == e.Id || w.Item!.PrimaryVersionId == e.Id) - .Max(f => f.LastPlayedDate), (ItemSortBy.PlayCount, _) => e => e.UserData!.Where(f => f.UserId.Equals(query.User!.Id)).OrderBy(f => f.CustomDataKey).FirstOrDefault()!.PlayCount, (ItemSortBy.IsFavoriteOrLiked, _) => e => e.UserData!.Where(f => f.UserId.Equals(query.User!.Id)).OrderBy(f => f.CustomDataKey).Select(f => (bool?)f.IsFavorite).FirstOrDefault() ?? false, (ItemSortBy.IsFolder, _) => e => e.IsFolder, diff --git a/Jellyfin.Server.Implementations/Item/PeopleRepository.cs b/Jellyfin.Server.Implementations/Item/PeopleRepository.cs index eb87b525fe..9611c5c13a 100644 --- a/Jellyfin.Server.Implementations/Item/PeopleRepository.cs +++ b/Jellyfin.Server.Implementations/Item/PeopleRepository.cs @@ -79,7 +79,11 @@ public class PeopleRepository(IDbContextFactory<JellyfinDbContext> dbProvider, I public IReadOnlyList<string> GetPeopleNames(InternalPeopleQuery filter) { using var context = _dbProvider.CreateDbContext(); - var dbQuery = TranslateQuery(context.Peoples.AsNoTracking(), context, filter).Select(e => e.Name).Distinct(); + + IQueryable<string> dbQuery = TranslateQuery(context.Peoples.AsNoTracking(), context, filter) + .Select(e => e.Name) + .Distinct() + .OrderBy(e => e); if (filter.StartIndex.HasValue && filter.StartIndex > 0) { @@ -88,7 +92,7 @@ public class PeopleRepository(IDbContextFactory<JellyfinDbContext> dbProvider, I if (filter.Limit > 0) { - dbQuery = dbQuery.OrderBy(e => e).Take(filter.Limit); + dbQuery = dbQuery.Take(filter.Limit); } return dbQuery.ToArray(); |
