diff options
Diffstat (limited to 'Emby.Server.Implementations/Library')
18 files changed, 1069 insertions, 252 deletions
diff --git a/Emby.Server.Implementations/Library/LibraryManager.cs b/Emby.Server.Implementations/Library/LibraryManager.cs index 3691f4e19d..16616dd328 100644 --- a/Emby.Server.Implementations/Library/LibraryManager.cs +++ b/Emby.Server.Implementations/Library/LibraryManager.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Globalization; using System.IO; using System.Linq; @@ -15,7 +16,6 @@ using Emby.Naming.Common; using Emby.Naming.TV; using Emby.Naming.Video; using Emby.Server.Implementations.Library.Resolvers; -using Emby.Server.Implementations.Library.Validators; using Emby.Server.Implementations.Playlists; using Emby.Server.Implementations.ScheduledTasks.Tasks; using Emby.Server.Implementations.Sorting; @@ -35,7 +35,6 @@ using MediaBrowser.Controller.Entities.Movies; using MediaBrowser.Controller.IO; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.LiveTv; -using MediaBrowser.Controller.MediaEncoding; using MediaBrowser.Controller.Persistence; using MediaBrowser.Controller.Playlists; using MediaBrowser.Controller.Providers; @@ -45,6 +44,7 @@ using MediaBrowser.Model.Configuration; using MediaBrowser.Model.Drawing; using MediaBrowser.Model.Dto; using MediaBrowser.Model.Entities; +using MediaBrowser.Model.Globalization; using MediaBrowser.Model.IO; using MediaBrowser.Model.Library; using MediaBrowser.Model.Querying; @@ -74,7 +74,6 @@ namespace Emby.Server.Implementations.Library private readonly Lazy<IProviderManager> _providerManagerFactory; private readonly Lazy<IUserViewManager> _userViewManagerFactory; private readonly IServerApplicationHost _appHost; - private readonly IMediaEncoder _mediaEncoder; private readonly IFileSystem _fileSystem; private readonly IItemRepository _itemRepository; private readonly IItemPersistenceService _persistenceService; @@ -86,6 +85,8 @@ namespace Emby.Server.Implementations.Library private readonly IPeopleRepository _peopleRepository; private readonly ExtraResolver _extraResolver; private readonly IPathManager _pathManager; + private readonly ILocalizationManager _localization; + private readonly IDirectoryService _directoryService; private readonly FastConcurrentLru<Guid, BaseItem> _cache; private readonly DotIgnoreIgnoreRule _dotIgnoreIgnoreRule; private readonly IMediaStreamRepository _mediaStreamRepository; @@ -120,7 +121,6 @@ namespace Emby.Server.Implementations.Library /// <param name="fileSystem">The file system.</param> /// <param name="providerManagerFactory">The provider manager.</param> /// <param name="userViewManagerFactory">The user view manager.</param> - /// <param name="mediaEncoder">The media encoder.</param> /// <param name="itemRepository">The item repository.</param> /// <param name="persistenceService">The item persistence service.</param> /// <param name="nextUpService">The next up service.</param> @@ -132,6 +132,7 @@ namespace Emby.Server.Implementations.Library /// <param name="peopleRepository">The people repository.</param> /// <param name="pathManager">The path manager.</param> /// <param name="dotIgnoreIgnoreRule">The .ignore rule handler.</param> + /// <param name="localization">The localization manager.</param> /// <param name="mediaStreamRepository">The media stream repository.</param> /// <param name="externalDataManagerFactory">The external data manager (lazy, to break the DI cycle through ChapterManager).</param> public LibraryManager( @@ -145,7 +146,6 @@ namespace Emby.Server.Implementations.Library IFileSystem fileSystem, Lazy<IProviderManager> providerManagerFactory, Lazy<IUserViewManager> userViewManagerFactory, - IMediaEncoder mediaEncoder, IItemRepository itemRepository, IItemPersistenceService persistenceService, INextUpService nextUpService, @@ -157,6 +157,7 @@ namespace Emby.Server.Implementations.Library IPeopleRepository peopleRepository, IPathManager pathManager, DotIgnoreIgnoreRule dotIgnoreIgnoreRule, + ILocalizationManager localization, IMediaStreamRepository mediaStreamRepository, Lazy<IExternalDataManager> externalDataManagerFactory) { @@ -170,7 +171,6 @@ namespace Emby.Server.Implementations.Library _fileSystem = fileSystem; _providerManagerFactory = providerManagerFactory; _userViewManagerFactory = userViewManagerFactory; - _mediaEncoder = mediaEncoder; _itemRepository = itemRepository; _persistenceService = persistenceService; _nextUpService = nextUpService; @@ -184,6 +184,8 @@ namespace Emby.Server.Implementations.Library _peopleRepository = peopleRepository; _pathManager = pathManager; _dotIgnoreIgnoreRule = dotIgnoreIgnoreRule; + _localization = localization; + _directoryService = directoryService; _extraResolver = new ExtraResolver(loggerFactory.CreateLogger<ExtraResolver>(), namingOptions, directoryService); _configurationManager.ConfigurationUpdated += ConfigurationUpdated; @@ -316,7 +318,7 @@ namespace Emby.Server.Implementations.Library if (wizardChanged) { - _taskManager.CancelIfRunningAndQueue<RefreshMediaLibraryTask>(); + QueueLibraryScan(); } } @@ -407,6 +409,13 @@ namespace Emby.Server.Implementations.Library } _persistenceService.DeleteItem([.. pathMaps.Select(f => f.Item.Id)]); + + // Evict the deleted items from the cache and announce each removal. + foreach (var (item, _, _) in pathMaps) + { + _cache.TryRemove(item.Id, out _); + ReportItemRemoved(item, item.GetOwner() ?? item.GetParent()); + } } public void DeleteItem(BaseItem item, DeleteOptions options, BaseItem parent, bool notifyParentItem) @@ -606,6 +615,12 @@ namespace Emby.Server.Implementations.Library folder.UserData = null; } + // Announce the descendants before the item itself. + foreach (var child in children) + { + ReportItemRemoved(child, item); + } + ReportItemRemoved(item, parent); } @@ -863,7 +878,18 @@ namespace Emby.Server.Implementations.Library wrongTypeItem.GetType().Name, expectedVideoType.Name, path); - DeleteItem(wrongTypeItem, new DeleteOptions { DeleteFileLocation = false }); + + // A full DeleteItem would save the primary version, which resolves its + // alternates again and re-enters here before this row is gone. + DeleteItemsUnsafeFast([wrongTypeItem]); + + // The fast path skips the parent bookkeeping, and the stale item is listed + // under its ParentId, so that folder's cached listing has to be dropped. + if (wrongTypeItem.GetParent() is Folder staleParent) + { + staleParent.Children = null; + staleParent.UserData = null; + } } } @@ -1192,6 +1218,12 @@ namespace Emby.Server.Implementations.Library } /// <inheritdoc /> + public Guid GetPersonId(string name) + { + return GetItemByNameId<Person>(Person.GetPath(name)); + } + + /// <inheritdoc /> public Person? GetPerson(string name) { var path = Person.GetPath(name); @@ -1204,6 +1236,33 @@ namespace Emby.Server.Implementations.Library return null; } + /// <inheritdoc /> + public Person GetOrCreatePerson(string name) + { + var existing = GetPerson(name); + if (existing is not null) + { + return existing; + } + + var path = Person.GetPath(name); + var info = Directory.CreateDirectory(path); + var item = new Person + { + Name = name, + Id = GetItemByNameId<Person>(path), + DateCreated = info.CreationTimeUtc, + DateModified = info.LastWriteTimeUtc, + Path = path + }; + + item.PresentationUniqueKey = item.CreatePresentationUniqueKey(); + + CreateItem(item, null); + + return item; + } + /// <summary> /// Gets the studio. /// </summary> @@ -1295,7 +1354,6 @@ namespace Emby.Server.Implementations.Library { IncludeItemTypes = [BaseItemKind.MusicArtist], Name = name, - UseRawName = true, DtoOptions = options }).Cast<MusicArtist>() .OrderBy(i => i.IsAccessedByName ? 1 : 0) @@ -1336,15 +1394,6 @@ namespace Emby.Server.Implementations.Library return GetNewItemIdInternal(path, typeof(T), forceCaseInsensitiveId); } - /// <inheritdoc /> - public Task ValidatePeopleAsync(IProgress<double> progress, CancellationToken cancellationToken) - { - // Ensure the location is available. - Directory.CreateDirectory(_configurationManager.ApplicationPaths.PeoplePath); - - return new PeopleValidator(this, _logger, _fileSystem).ValidatePeople(cancellationToken, progress); - } - /// <summary> /// Reloads the root media folder. /// </summary> @@ -1471,6 +1520,10 @@ namespace Emby.Server.Implementations.Library var numComplete = 0; var numTasks = tasks.Count; + _logger.LogInformation("Running {TaskCount} post-scan task(s)", numTasks); + + var phaseStart = Stopwatch.GetTimestamp(); + foreach (var task in tasks) { // Prevent access to modified closure @@ -1488,20 +1541,45 @@ namespace Emby.Server.Implementations.Library progress.Report(innerPercent); }); - _logger.LogDebug("Running post-scan task {0}", task.GetType().Name); + var taskName = task.GetType().Name; + var taskStart = Stopwatch.GetTimestamp(); + + _logger.LogInformation( + "Running post-scan task {TaskNumber}/{TaskCount}: {TaskName}", + currentNumComplete + 1, + numTasks, + taskName); try { await task.Run(innerProgress, cancellationToken).ConfigureAwait(false); + + var elapsed = Stopwatch.GetElapsedTime(taskStart); + _logger.LogInformation( + "Post-scan task {TaskName} completed after {Minutes} minute(s) and {Seconds} seconds", + taskName, + Math.Truncate(elapsed.TotalMinutes), + elapsed.Seconds); } catch (OperationCanceledException) { - _logger.LogInformation("Post-scan task cancelled: {0}", task.GetType().Name); + var elapsed = Stopwatch.GetElapsedTime(taskStart); + _logger.LogInformation( + "Post-scan task {TaskName} cancelled after {Minutes} minute(s) and {Seconds} seconds", + taskName, + Math.Truncate(elapsed.TotalMinutes), + elapsed.Seconds); throw; } catch (Exception ex) { - _logger.LogError(ex, "Error running post-scan task"); + var elapsed = Stopwatch.GetElapsedTime(taskStart); + _logger.LogError( + ex, + "Post-scan task {TaskName} failed after {Minutes} minute(s) and {Seconds} seconds", + taskName, + Math.Truncate(elapsed.TotalMinutes), + elapsed.Seconds); } numComplete++; @@ -1510,6 +1588,12 @@ namespace Emby.Server.Implementations.Library progress.Report(percent * 100); } + var phaseElapsed = Stopwatch.GetElapsedTime(phaseStart); + _logger.LogInformation( + "All post-scan tasks completed after {Minutes} minute(s) and {Seconds} seconds", + Math.Truncate(phaseElapsed.TotalMinutes), + phaseElapsed.Seconds); + _persistenceService.UpdateInheritedValues(); progress.Report(100); @@ -1727,9 +1811,21 @@ namespace Emby.Server.Implementations.Library return _countService.GetItemCountsForNameItem(kind, id, relatedItemKinds, query); } - public Dictionary<Guid, int> GetChildCountBatch(IReadOnlyList<Guid> parentIds, Guid? userId) + /// <inheritdoc/> + public Dictionary<Guid, ItemCounts> GetItemCountsForNameItems(BaseItemKind kind, IReadOnlyList<Guid> ids, BaseItemKind[] relatedItemKinds, User? user) + { + var query = new InternalItemsQuery(user); + if (user is not null) + { + AddUserToQuery(query, user); + } + + return _countService.GetItemCountsForNameItems(kind, ids, relatedItemKinds, query); + } + + public Dictionary<Guid, int> GetChildCountBatch(IReadOnlyList<Guid> parentIds, User? user) { - return _countService.GetChildCountBatch(parentIds, userId); + return _countService.GetChildCountBatch(parentIds, user); } /// <inheritdoc/> @@ -1896,14 +1992,14 @@ namespace Emby.Server.Implementations.Library } // Optimize by querying against top level views - query.TopParentIds = parents.SelectMany(i => GetTopParentIdsForQuery(i, query.User)).ToArray(); - query.AncestorIds = []; - - // Prevent searching in all libraries due to empty filter - if (query.TopParentIds.Length == 0) + var topParentIds = parents.SelectMany(i => GetTopParentIdsForQuery(i, query.User)).ToArray(); + if (topParentIds.Length == 0) { - query.TopParentIds = [Guid.NewGuid()]; + return; } + + query.TopParentIds = topParentIds; + query.AncestorIds = []; } public QueryResult<(BaseItem Item, ItemCounts ItemCounts)> GetAlbumArtists(InternalItemsQuery query) @@ -1949,12 +2045,15 @@ namespace Emby.Server.Implementations.Library if (parents.All(i => i is ICollectionFolder || i is UserView)) { // Optimize by querying against top level views - query.TopParentIds = parents.SelectMany(i => GetTopParentIdsForQuery(i, query.User)).ToArray(); + var topParentIds = parents.SelectMany(i => GetTopParentIdsForQuery(i, query.User)).ToArray(); - // Prevent searching in all libraries due to empty filter - if (query.TopParentIds.Length == 0) + if (topParentIds.Length > 0) { - query.TopParentIds = [Guid.NewGuid()]; + query.TopParentIds = topParentIds; + } + else + { + SetAncestorIds(query, parents); } } else if (parents.Count == 1 && parents.First() is Folder folder @@ -1963,34 +2062,31 @@ namespace Emby.Server.Implementations.Library { // Playlists and BoxSets store their contents in LinkedChildren and never // populate AncestorIds for those items, so a recursive AncestorIds query - // would return zero rows. Resolve to the linked child IDs up front and - // route through the existing indexed ItemIds filter. - query.ItemIds = folder.LinkedChildren - .Where(lc => lc.ItemId.HasValue && !lc.ItemId.Value.IsEmpty()) - .Select(lc => lc.ItemId!.Value) - .ToArray(); - - // Empty linked-children should still return empty rather than scanning everything. - if (query.ItemIds.Length == 0) - { - query.ItemIds = [Guid.NewGuid()]; - } + // would return zero rows. Filter by the descendant set instead, which follows + // the links and keeps descending, so a linked folder contributes what is below + // it as well - the episodes of a Series added to a collection, for example. + query.DescendantOfId = folder.Id; } else { - // We need to be able to query from any arbitrary ancestor up the tree - query.AncestorIds = parents.SelectMany(i => i.GetIdsForAncestorQuery()).ToArray(); - - // Prevent searching in all libraries due to empty filter - if (query.AncestorIds.Length == 0) - { - query.AncestorIds = [Guid.NewGuid()]; - } + SetAncestorIds(query, parents); } query.Parent = null; } + private static void SetAncestorIds(InternalItemsQuery query, IReadOnlyCollection<BaseItem> parents) + { + // We need to be able to query from any arbitrary ancestor up the tree + query.AncestorIds = parents.SelectMany(i => i.GetIdsForAncestorQuery()).ToArray(); + + // Prevent searching in all libraries due to empty filter + if (query.AncestorIds.Length == 0) + { + query.AncestorIds = [Guid.NewGuid()]; + } + } + private void AddUserToQuery(InternalItemsQuery query, User user, bool allowExternalContent = true) { if (query.User is null) @@ -2230,6 +2326,12 @@ namespace Emby.Server.Implementations.Library } /// <inheritdoc /> + public IReadOnlySet<Guid> GetItemIdsWithAlternateVersions(IReadOnlyList<Guid> itemIds) + { + return _linkedChildrenService.GetItemIdsWithAlternateVersions(itemIds); + } + + /// <inheritdoc /> public void UpsertLinkedChild(Guid parentId, Guid childId, MediaBrowser.Controller.Entities.LinkedChildType childType) { _linkedChildrenService.UpsertLinkedChild(parentId, childId, childType); @@ -2315,9 +2417,13 @@ namespace Emby.Server.Implementations.Library { var comparer = Comparers.FirstOrDefault(c => name == c.Type); - // If it requires a user, create a new one, and assign the user if (comparer is IUserBaseItemComparer) { + if (user is null) + { + throw new ArgumentException($"Sort key '{name}' requires a user, but none was provided."); + } + var userComparer = (IUserBaseItemComparer)Activator.CreateInstance(comparer.GetType())!; // only null for Nullable<T> instances userComparer.User = user; @@ -2346,6 +2452,9 @@ namespace Emby.Server.Implementations.Library var parentCollectionType = parent is not null ? GetTopFolderContentType(parent) : null; foreach (var item in items) { + // Parental controls filter on the stored score, so it has to match the ratings being saved. + item.OnMetadataChanged(); + if (item is Video video && video.LocalAlternateVersions.Length > 0) { var videoType = video.GetType(); @@ -2367,6 +2476,7 @@ namespace Emby.Server.Implementations.Library { altVideo.OwnerId = video.Id; altVideo.SetPrimaryVersionId(video.Id); + altVideo.IsInMixedFolder = video.IsInMixedFolder; // ResolveAlternateVersion only sees the alternate's primary file. // If the alternate is itself a stack (e.g. 1080p part1 + part2), // detect its parts from sibling files so its AdditionalParts persist. @@ -2490,9 +2600,15 @@ namespace Emby.Server.Implementations.Library } } - if (!File.Exists(image.Path)) + if (string.IsNullOrEmpty(image.Path) || !File.Exists(image.Path)) { - _logger.LogWarning("Image not found at {ImagePath}", image.Path); + _logger.LogWarning( + "{ImageType} image for {ItemName} ({ItemId}) not found at \"{ImagePath}\", source was {SourcePath}", + img.Type, + item.Name, + item.Id, + image.Path, + img.Path); continue; } @@ -2545,6 +2661,9 @@ namespace Emby.Server.Implementations.Library { foreach (var item in items) { + // Parental controls filter on the stored score, so it has to match the ratings being saved. + item.OnMetadataChanged(); + item.DateLastSaved = DateTime.UtcNow; await RunMetadataSavers(item, updateReason).ConfigureAwait(false); @@ -2552,6 +2671,8 @@ namespace Emby.Server.Implementations.Library item.DateLastSaved = DateTime.UtcNow; } + ForgetDroppedLocalAlternateVersions(items); + // Resolve and add any local alternate version items that don't exist yet // This ensures they exist in the database when LinkedChildren are processed var allItems = new List<BaseItem>(items); @@ -2580,6 +2701,7 @@ namespace Emby.Server.Implementations.Library { altVideo.OwnerId = video.Id; altVideo.SetPrimaryVersionId(video.Id); + altVideo.IsInMixedFolder = video.IsInMixedFolder; // ResolveAlternateVersion only sees the alternate's primary file. // If the alternate is itself a stack (e.g. 1080p part1 + part2), // detect its parts from sibling files so its AdditionalParts persist. @@ -2640,6 +2762,30 @@ namespace Emby.Server.Implementations.Library public Task UpdateItemAsync(BaseItem item, BaseItem parent, ItemUpdateType updateReason, CancellationToken cancellationToken) => UpdateItemsAsync([item], parent, updateReason, cancellationToken); + /// <summary> + /// Forgets the cached local alternate versions of the supplied items that they no longer list. + /// </summary> + /// <param name="items">The items about to be saved.</param> + private void ForgetDroppedLocalAlternateVersions(IReadOnlyList<BaseItem> items) + { + foreach (var video in items.OfType<Video>()) + { + var videoType = video.GetType(); + var keptIds = video.LocalAlternateVersions + .Where(path => !string.IsNullOrEmpty(path)) + .Select(path => GetNewItemId(path, videoType)) + .ToHashSet(); + + foreach (var versionId in GetLocalAlternateVersionIds(video)) + { + if (!keptIds.Contains(versionId)) + { + _cache.TryRemove(versionId, out _); + } + } + } + } + /// <inheritdoc /> public async Task ReattachUserDataAsync(BaseItem item, CancellationToken cancellationToken) { @@ -2863,7 +3009,8 @@ namespace Emby.Server.Implementations.Library "views", _fileSystem.GetValidFilename(viewType.ToString())); - var id = GetNewItemId(path + "_namedview_" + name, typeof(UserView)); + // The display name is localized, so it must not take part in the id. + var id = GetNewItemId(path + "_namedview_" + viewType.ToString(), typeof(UserView)); var item = GetItemById(id) as UserView; @@ -2887,6 +3034,13 @@ namespace Emby.Server.Implementations.Library refresh = true; } + else if (!string.Equals(item.Name, name, StringComparison.Ordinal)) + { + item.Name = name; + item.ForcedSortName = sortName; + + refresh = true; + } if (refresh) { @@ -2904,12 +3058,7 @@ namespace Emby.Server.Implementations.Library CollectionType? viewType, string sortName) { - var parentIdString = parentId.IsEmpty() - ? null - : parentId.ToString("N", CultureInfo.InvariantCulture); - var idValues = "38_namedview_" + name + user.Id.ToString("N", CultureInfo.InvariantCulture) + (parentIdString ?? string.Empty) + (viewType?.ToString() ?? string.Empty); - - var id = GetNewItemId(idValues, typeof(UserView)); + var id = GetNamedViewId(user, parentId, viewType); var path = Path.Combine(_configurationManager.ApplicationPaths.InternalMetadataPath, "views", id.ToString("N", CultureInfo.InvariantCulture)); @@ -2937,6 +3086,11 @@ namespace Emby.Server.Implementations.Library isNew = true; } + else if (!string.Equals(item.Name, name, StringComparison.Ordinal)) + { + item.Name = name; + item.UpdateToRepositoryAsync(ItemUpdateType.MetadataEdit, CancellationToken.None).GetAwaiter().GetResult(); + } var lastRefreshedUtc = item.DateLastRefreshed; var refresh = isNew || DateTime.UtcNow - lastRefreshedUtc >= _viewRefreshInterval; @@ -2962,6 +3116,18 @@ namespace Emby.Server.Implementations.Library return item; } + public Guid GetNamedViewId(User user, Guid parentId, CollectionType? viewType) + { + var parentIdString = parentId.IsEmpty() + ? null + : parentId.ToString("N", CultureInfo.InvariantCulture); + + // The name is either localized (grouped views) or the library folder's own name. + var idValues = "38_namedview_" + user.Id.ToString("N", CultureInfo.InvariantCulture) + (parentIdString ?? string.Empty) + (viewType?.ToString() ?? string.Empty); + + return GetNewItemId(idValues, typeof(UserView)); + } + public UserView GetShadowView( BaseItem parent, CollectionType? viewType, @@ -2971,10 +3137,7 @@ namespace Emby.Server.Implementations.Library var name = parent.Name; var parentId = parent.Id; - - var idValues = "38_namedview_" + name + parentId + (viewType?.ToString() ?? string.Empty); - - var id = GetNewItemId(idValues, typeof(UserView)); + var id = GetShadowViewId(name, parentId, viewType); var path = parent.Path; @@ -3026,6 +3189,13 @@ namespace Emby.Server.Implementations.Library return item; } + public Guid GetShadowViewId(string name, Guid parentId, CollectionType? viewType) + { + var idValues = "38_namedview_" + name + parentId + (viewType?.ToString() ?? string.Empty); + + return GetNewItemId(idValues, typeof(UserView)); + } + public UserView GetNamedView( string name, Guid parentId, @@ -3038,7 +3208,7 @@ namespace Emby.Server.Implementations.Library var parentIdString = parentId.IsEmpty() ? null : parentId.ToString("N", CultureInfo.InvariantCulture); - var idValues = "37_namedview_" + name + (parentIdString ?? string.Empty) + (viewType?.ToString() ?? string.Empty); + var idValues = "37_namedview_" + (parentIdString ?? string.Empty) + (viewType?.ToString() ?? string.Empty); if (!string.IsNullOrEmpty(uniqueId)) { idValues += uniqueId; @@ -3072,9 +3242,10 @@ namespace Emby.Server.Implementations.Library isNew = true; } - if (viewType != item.ViewType) + if (viewType != item.ViewType || !string.Equals(item.Name, name, StringComparison.Ordinal)) { item.ViewType = viewType; + item.Name = name; item.UpdateToRepositoryAsync(ItemUpdateType.MetadataEdit, CancellationToken.None).GetAwaiter().GetResult(); } @@ -3195,11 +3366,11 @@ namespace Emby.Server.Implementations.Library } } - if (!episode.ProductionYear.HasValue) + if (episode.ProductionYear is null) { episode.ProductionYear = episodeInfo.Year; - if (episode.ProductionYear.HasValue) + if (episode.ProductionYear is not null) { changed = true; } @@ -3276,9 +3447,11 @@ namespace Emby.Server.Implementations.Library var ownerVideoInfo = VideoResolver.Resolve(owner.Path, isFolder, _namingOptions, libraryRoot: owner.ContainingFolderPath); if (ownerVideoInfo is null) { - yield break; + return []; } + var candidates = new List<ExtraCandidate>(); + var count = filtered.Count; for (var i = 0; i < count; i++) { @@ -3292,35 +3465,51 @@ namespace Emby.Server.Implementations.Library foreach (var file in filesInSubFolderList) { - if (!_extraResolver.TryGetExtraTypeForOwner(file.FullName, ownerVideoInfo, out var extraType)) + if (!_extraResolver.TryGetExtraTypeForOwner(file.FullName, ownerVideoInfo, out var extraType, out var extraRule)) { continue; } - var extra = GetExtra(file, extraType.Value, subFolderIsMixedFolder); - if (extra is not null) - { - yield return extra; - } + AddCandidate(file, extraType.Value, extraRule, subFolderIsMixedFolder); } } - else if (!current.IsDirectory && _extraResolver.TryGetExtraTypeForOwner(current.FullName, ownerVideoInfo, out var extraType)) + else if (!current.IsDirectory && _extraResolver.TryGetExtraTypeForOwner(current.FullName, ownerVideoInfo, out var extraType, out var extraRule)) { - var extra = GetExtra(current, extraType.Value, false); - if (extra is not null) - { - yield return extra; - } + AddCandidate(current, extraType.Value, extraRule, false); } } - BaseItem? GetExtra(FileSystemMetadata file, ExtraType extraType, bool isInMixedFolder) + var extras = new List<BaseItem>(); + var typeCounters = new Dictionary<ExtraType, int>(); + var generatedNames = new Dictionary<ExtraType, HashSet<string>>(); + + // Order by path so that the numbering handed out below does not depend on the + // order the file system happened to list the folder in + foreach (var candidate in candidates.OrderBy(c => c.Extra.Path, StringComparer.Ordinal)) + { + var extra = PrepareExtra(candidate); + if (extra is not null) + { + extras.Add(extra); + } + } + + return extras; + + void AddCandidate(FileSystemMetadata file, ExtraType extraType, ExtraRule extraRule, bool isInMixedFolder) { var extra = ResolvePath(_fileSystem.GetFileInfo(file.FullName), directoryService, _extraResolver.GetResolversForExtraType(extraType)); - if (extra is not Video && extra is not Audio) + if (extra is Video or Audio) { - return null; + candidates.Add(new ExtraCandidate(extra, extraType, extraRule, isInMixedFolder)); } + } + + BaseItem? PrepareExtra(ExtraCandidate candidate) + { + var resolved = candidate.Extra; + var extra = resolved; + var name = GetExtraName(candidate, ownerVideoInfo, typeCounters); // Try to retrieve it from the db. If we don't find it, use the resolved version var itemById = GetItemById(extra.Id); @@ -3329,10 +3518,20 @@ namespace Emby.Server.Implementations.Library extra = itemById; } + // The name derived from the file is only a default. A name that came from anywhere else, + // such as a local metadata file, is the user's and has to survive the scan, so only a + // name this method handed out itself is renewed; RefreshExtras persists the change. + if (!string.IsNullOrEmpty(name) + && extra.LockedFields?.Contains(MetadataField.Name) != true + && (itemById is null || IsGeneratedExtraName(extra.Name, candidate))) + { + extra.Name = name; + } + // Only update extra type if it is more specific then the currently known extra type - if (extra.ExtraType is null or ExtraType.Unknown || extraType != ExtraType.Unknown) + if (extra.ExtraType is null or ExtraType.Unknown || candidate.ExtraType != ExtraType.Unknown) { - extra.ExtraType = extraType; + extra.ExtraType = candidate.ExtraType; } // Only return items that are actual extras (have ExtraType set) @@ -3340,14 +3539,101 @@ namespace Emby.Server.Implementations.Library // so that RefreshExtras can detect when they need updating and set ForceSave. if (extra.ExtraType is not null) { - extra.IsInMixedFolder = isInMixedFolder; + extra.IsInMixedFolder = candidate.IsInMixedFolder; return extra; } return null; } + + bool IsGeneratedExtraName(string currentName, ExtraCandidate candidate) + { + // The file name is what an extra was called before it was given a name of its type + if (string.Equals(currentName, candidate.Extra.Name, StringComparison.Ordinal)) + { + return true; + } + + if (!generatedNames.TryGetValue(candidate.ExtraType, out var names)) + { + // Any of the numbers of this type may have been handed out, as the order the extras + // of a type are numbered in shifts as files appear beside them or are taken away + names = new HashSet<string>(StringComparer.Ordinal); + var count = candidates.Count(c => c.ExtraType == candidate.ExtraType); + for (var seen = 0; seen < count; seen++) + { + names.Add(GetNumberedExtraName(candidate.ExtraType, seen)); + } + + generatedNames[candidate.ExtraType] = names; + } + + return names.Contains(currentName); + } } + /// <summary> + /// Gets the name to give an extra. + /// </summary> + /// <param name="candidate">The resolved extra.</param> + /// <param name="ownerVideoInfo">The naming info of the owner.</param> + /// <param name="typeCounters">Number of extras named after their type so far, per type.</param> + /// <returns>The name.</returns> + private string GetExtraName(ExtraCandidate candidate, VideoFileInfo ownerVideoInfo, Dictionary<ExtraType, int> typeCounters) + { + var isNamedAfterOwner = candidate.ExtraRule.RuleType switch + { + ExtraRuleType.Filename => true, + ExtraRuleType.Suffix => string.Equals(candidate.Extra.Name, ownerVideoInfo.Name, StringComparison.OrdinalIgnoreCase), + _ => false + }; + + if (!isNamedAfterOwner) + { + return candidate.Extra.Name; + } + + typeCounters.TryGetValue(candidate.ExtraType, out var seen); + typeCounters[candidate.ExtraType] = seen + 1; + + return GetNumberedExtraName(candidate.ExtraType, seen); + } + + /// <summary> + /// Gets the name given to the n-th extra of a type that is named after its type. + /// </summary> + /// <param name="extraType">The extra type.</param> + /// <param name="seen">Number of extras of the type named before this one.</param> + /// <returns>The name.</returns> + private string GetNumberedExtraName(ExtraType extraType, int seen) + { + var typeName = _localization.GetServerLocalizedString(GetExtraTypeNameKey(extraType)); + + return seen == 0 + ? typeName + : string.Format( + CultureInfo.InvariantCulture, + _localization.GetServerLocalizedString("NameExtraNumbered"), + typeName, + seen + 1); + } + + private static string GetExtraTypeNameKey(ExtraType extraType) => extraType switch + { + ExtraType.Clip => "NameExtraClip", + ExtraType.Trailer => "NameExtraTrailer", + ExtraType.BehindTheScenes => "NameExtraBehindTheScenes", + ExtraType.DeletedScene => "NameExtraDeletedScene", + ExtraType.Interview => "NameExtraInterview", + ExtraType.Scene => "NameExtraScene", + ExtraType.Sample => "NameExtraSample", + ExtraType.ThemeSong => "NameExtraThemeSong", + ExtraType.ThemeVideo => "NameExtraThemeVideo", + ExtraType.Featurette => "NameExtraFeaturette", + ExtraType.Short => "NameExtraShort", + _ => "NameExtraUnknown" + }; + public string GetPathAfterNetworkSubstitution(string path, BaseItem? ownerItem) { foreach (var map in _configurationManager.Configuration.PathSubstitutions) @@ -3386,6 +3672,11 @@ namespace Emby.Server.Implementations.Library public QueryResult<BaseItem> GetPeopleItems(InternalPeopleQuery query) { + ArgumentNullException.ThrowIfNull(query); + + // This hands back by-name items, so the people without one are not ours to report. + query.MustHaveItem = true; + var queryResult = _peopleRepository.GetPeople(query); var baseItems = queryResult.Items.Select(i => { @@ -3419,11 +3710,23 @@ namespace Emby.Server.Implementations.Library } /// <inheritdoc/> + public int DeleteOrphanedCredits() + { + return _peopleRepository.DeleteOrphanedCredits(); + } + + /// <inheritdoc/> public IReadOnlyDictionary<Guid, IReadOnlyList<string>> GetPeopleNamesByItems(IReadOnlyList<Guid> itemIds, IReadOnlyList<string> personTypes) { return _peopleRepository.GetPeopleNamesByItems(itemIds, personTypes); } + /// <inheritdoc/> + public IReadOnlyDictionary<Guid, IReadOnlyList<PersonInfo>> GetPeopleByItems(IReadOnlyList<Guid> itemIds) + { + return _peopleRepository.GetPeopleByItems(itemIds); + } + public void UpdatePeople(BaseItem item, List<PersonInfo> people) { UpdatePeopleAsync(item, people, CancellationToken.None).GetAwaiter().GetResult(); @@ -3457,7 +3760,20 @@ namespace Emby.Server.Implementations.Library await item.UpdateToRepositoryAsync(ItemUpdateType.ImageUpdate, CancellationToken.None).ConfigureAwait(false); - return item.GetImageInfo(image.Type, imageIndex); + var localImage = item.GetImageInfo(image.Type, imageIndex); + if (localImage is null) + { + throw new InvalidOperationException(string.Format( + CultureInfo.InvariantCulture, + "Downloaded {0} image {1} from {2} is not attached to {3} ({4})", + image.Type, + imageIndex, + url, + item.Name, + item.Id)); + } + + return localImage; } catch (HttpRequestException ex) { @@ -3479,7 +3795,13 @@ namespace Emby.Server.Implementations.Library await item.UpdateToRepositoryAsync(ItemUpdateType.ImageUpdate, CancellationToken.None).ConfigureAwait(false); } - throw new InvalidOperationException("Unable to convert any images to local"); + throw new InvalidOperationException(string.Format( + CultureInfo.InvariantCulture, + "Unable to convert any {0} image url in \"{1}\" to a local file for {2} ({3})", + image.Type, + image.Path, + item.Name, + item.Id)); } public async Task AddVirtualFolder(string name, CollectionTypeOptions? collectionType, LibraryOptions options, bool refreshLibrary) @@ -3535,6 +3857,10 @@ namespace Emby.Server.Implementations.Library AddMediaPathInternal(name, path, false); } } + + // The libraries root was listed before this folder existed, so drop that listing: + // anything still reading it resolves the library set without the new folder. + _directoryService.Invalidate(virtualFolderPath); } finally { @@ -3542,7 +3868,7 @@ namespace Emby.Server.Implementations.Library if (refreshLibrary) { - StartScanInBackground(); + _ = StartScanInBackground(); } else { @@ -3561,27 +3887,14 @@ namespace Emby.Server.Implementations.Library var itemUpdateType = ItemUpdateType.MetadataDownload; var saveEntity = false; - var createEntity = false; var personEntity = GetPerson(person.Name); if (personEntity is null) { try { - var path = Person.GetPath(person.Name); - var info = Directory.CreateDirectory(path); - personEntity = new Person() - { - Name = person.Name, - Id = GetItemByNameId<Person>(path), - DateCreated = info.CreationTimeUtc, - DateModified = info.LastWriteTimeUtc, - Path = path - }; - - personEntity.PresentationUniqueKey = personEntity.CreatePresentationUniqueKey(); + personEntity = GetOrCreatePerson(person.Name); saveEntity = true; - createEntity = true; } catch (Exception ex) { @@ -3615,11 +3928,6 @@ namespace Emby.Server.Implementations.Library if (saveEntity) { - if (createEntity) - { - CreateItems([personEntity], null, CancellationToken.None); - } - await RunMetadataSavers(personEntity, itemUpdateType).ConfigureAwait(false); personEntity.DateLastSaved = DateTime.UtcNow; @@ -3628,13 +3936,16 @@ namespace Emby.Server.Implementations.Library } } - private void StartScanInBackground() + internal Task StartScanInBackground() { - Task.Run(() => + // An active scan already handles library structure changes, so this request can be dropped. + if (IsScanRunning) { - // No need to start if scanning the library because it will handle it - ValidateMediaLibrary(new Progress<double>(), CancellationToken.None); - }); + return Task.CompletedTask; + } + + // Queue instead of restarting so a scan that starts after the check is allowed to finish. + return Task.Run(QueueLibraryScan); } public void AddMediaPath(string virtualFolderName, MediaPathInfo mediaPath) @@ -3659,7 +3970,9 @@ namespace Emby.Server.Implementations.Library } var rootFolderPath = _configurationManager.ApplicationPaths.DefaultUserViewsPath; - var virtualFolderPath = Path.Combine(rootFolderPath, virtualFolderName); + var virtualFolderPath = FileSystemHelper.GetChildPath(rootFolderPath, virtualFolderName) + ?? throw new FileNotFoundException( + string.Format(CultureInfo.InvariantCulture, "The media collection {0} does not exist", virtualFolderName)); CreateShortcut(virtualFolderPath, pathInfo); @@ -3680,7 +3993,9 @@ namespace Emby.Server.Implementations.Library ArgumentNullException.ThrowIfNull(mediaPath); var rootFolderPath = _configurationManager.ApplicationPaths.DefaultUserViewsPath; - var virtualFolderPath = Path.Combine(rootFolderPath, virtualFolderName); + var virtualFolderPath = FileSystemHelper.GetChildPath(rootFolderPath, virtualFolderName) + ?? throw new FileNotFoundException( + string.Format(CultureInfo.InvariantCulture, "The media collection {0} does not exist", virtualFolderName)); var libraryOptions = CollectionFolder.GetLibraryOptions(virtualFolderPath); @@ -3719,9 +4034,9 @@ namespace Emby.Server.Implementations.Library var rootFolderPath = _configurationManager.ApplicationPaths.DefaultUserViewsPath; - var path = Path.Combine(rootFolderPath, name); + var path = FileSystemHelper.GetChildPath(rootFolderPath, name); - if (!Directory.Exists(path)) + if (path is null || !Directory.Exists(path)) { throw new FileNotFoundException("The media folder does not exist"); } @@ -3731,6 +4046,7 @@ namespace Emby.Server.Implementations.Library try { Directory.Delete(path, true); + _directoryService.Invalidate(path); } finally { @@ -3740,7 +4056,7 @@ namespace Emby.Server.Implementations.Library { await ValidateTopLibraryFolders(CancellationToken.None, true).ConfigureAwait(false); - StartScanInBackground(); + _ = StartScanInBackground(); } else { @@ -3785,9 +4101,9 @@ namespace Emby.Server.Implementations.Library ArgumentException.ThrowIfNullOrEmpty(mediaPath); var rootFolderPath = _configurationManager.ApplicationPaths.DefaultUserViewsPath; - var virtualFolderPath = Path.Combine(rootFolderPath, virtualFolderName); + var virtualFolderPath = FileSystemHelper.GetChildPath(rootFolderPath, virtualFolderName); - if (!Directory.Exists(virtualFolderPath)) + if (virtualFolderPath is null || !Directory.Exists(virtualFolderPath)) { throw new FileNotFoundException( string.Format(CultureInfo.InvariantCulture, "The media collection {0} does not exist", virtualFolderName)); @@ -3800,6 +4116,7 @@ namespace Emby.Server.Implementations.Library if (!string.IsNullOrEmpty(shortcut)) { _fileSystem.DeleteFile(shortcut); + _directoryService.Invalidate(shortcut); } var libraryOptions = CollectionFolder.GetLibraryOptions(virtualFolderPath); @@ -3843,6 +4160,7 @@ namespace Emby.Server.Implementations.Library } _fileSystem.CreateShortcut(lnk, _appHost.ReverseVirtualPath(path)); + _directoryService.Invalidate(lnk); RemoveContentTypeOverrides(path); } @@ -3882,9 +4200,35 @@ namespace Emby.Server.Implementations.Library } /// <inheritdoc /> + public IReadOnlyList<string> GetTagNames(InternalItemsQuery query) + { + if (query.User is not null) + { + AddUserToQuery(query, query.User); + } + + SetTopParentOrAncestorIds(query); + return _itemRepository.GetTagNames(query); + } + + /// <inheritdoc /> public IReadOnlyList<string> GetMediaStreamLanguages(MediaStreamType mediaStreamType) { return _mediaStreamRepository.GetMediaStreamLanguages(mediaStreamType); } + + /// <inheritdoc /> + public IReadOnlyList<string> GetMediaStreamLanguages(MediaStreamType mediaStreamType, InternalItemsQuery query) + { + if (query.User is not null) + { + AddUserToQuery(query, query.User); + } + + SetTopParentOrAncestorIds(query); + return _itemRepository.GetMediaStreamLanguages(query, mediaStreamType); + } + + private sealed record ExtraCandidate(BaseItem Extra, ExtraType ExtraType, ExtraRule ExtraRule, bool IsInMixedFolder); } } diff --git a/Emby.Server.Implementations/Library/MediaSourceManager.cs b/Emby.Server.Implementations/Library/MediaSourceManager.cs index 97e00177b6..e9bba05839 100644 --- a/Emby.Server.Implementations/Library/MediaSourceManager.cs +++ b/Emby.Server.Implementations/Library/MediaSourceManager.cs @@ -384,7 +384,13 @@ namespace Emby.Server.Implementations.Library { ArgumentNullException.ThrowIfNull(item); - var hasMediaSources = (IHasMediaSources)item; + // Clients can ask for the sources of an item that has none (a container queued by mistake). + if (item is not IHasMediaSources hasMediaSources) + { + throw new ArgumentException( + string.Format(CultureInfo.InvariantCulture, "{0} {1} has no media sources and cannot be played.", item.GetType().Name, item.Id), + nameof(item)); + } var sources = hasMediaSources.GetMediaSources(enablePathSubstitution); @@ -494,7 +500,12 @@ namespace Emby.Server.Implementations.Library { var index = userData.SubtitleStreamIndex.Value; // Make sure the saved index is still valid - if (index == -1 || source.MediaStreams.Any(i => i.Type == MediaStreamType.Subtitle && i.Index == index)) + var savedStream = source.MediaStreams.FirstOrDefault(i => i.Type == MediaStreamType.Subtitle && i.Index == index); + // "Only forced" rules out full tracks entirely, so a remembered one must not resurrect them. + // The client reports whatever is playing, so an index remembered under another mode sticks forever otherwise. + if (index == -1 + || (savedStream is not null + && (user.SubtitleMode != SubtitlePlaybackMode.OnlyForced || savedStream.IsForced))) { source.DefaultSubtitleStreamIndex = index; return; diff --git a/Emby.Server.Implementations/Library/PathManager.cs b/Emby.Server.Implementations/Library/PathManager.cs index fad948ad97..2a50fcc7fe 100644 --- a/Emby.Server.Implementations/Library/PathManager.cs +++ b/Emby.Server.Implementations/Library/PathManager.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using System.Globalization; using System.IO; +using Jellyfin.Extensions; using MediaBrowser.Common.Configuration; using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Entities; @@ -43,7 +44,19 @@ public class PathManager : IPathManager public string? GetAttachmentPath(string mediaSourceId, string fileName) { var folder = GetAttachmentFolderPath(mediaSourceId); - return folder is null ? null : Path.Combine(folder, fileName); + if (folder is null) + { + return null; + } + + var safeName = PathHelper.GetSafeLeafFileName(fileName); + if (safeName is null) + { + _logger.LogWarning("Rejecting attachment filename '{FileName}' for MediaSource {MediaSourceId}: not a valid leaf name.", fileName, mediaSourceId); + return null; + } + + return Path.Combine(folder, safeName); } /// <inheritdoc /> diff --git a/Emby.Server.Implementations/Library/Resolvers/ExtraResolver.cs b/Emby.Server.Implementations/Library/Resolvers/ExtraResolver.cs index b9f9f29723..a0f75e4ddb 100644 --- a/Emby.Server.Implementations/Library/Resolvers/ExtraResolver.cs +++ b/Emby.Server.Implementations/Library/Resolvers/ExtraResolver.cs @@ -32,8 +32,8 @@ namespace Emby.Server.Implementations.Library.Resolvers : base(logger, namingOptions, directoryService) { _namingOptions = namingOptions; - _trailerResolvers = new IItemResolver[] { new GenericVideoResolver<Trailer>(logger, namingOptions, directoryService) }; - _videoResolvers = new IItemResolver[] { this }; + _trailerResolvers = [new GenericVideoResolver<Trailer>(logger, namingOptions, directoryService, parseName: true)]; + _videoResolvers = [this]; } protected override Video Resolve(ItemResolveArgs args) @@ -54,12 +54,13 @@ namespace Emby.Server.Implementations.Library.Resolvers _ => _videoResolvers }; - public bool TryGetExtraTypeForOwner(string path, VideoFileInfo ownerVideoFileInfo, [NotNullWhen(true)] out ExtraType? extraType, string? libraryRoot = "") + public bool TryGetExtraTypeForOwner(string path, VideoFileInfo ownerVideoFileInfo, [NotNullWhen(true)] out ExtraType? extraType, [NotNullWhen(true)] out ExtraRule? extraRule, string? libraryRoot = "") { var extraResult = GetExtraInfo(path, _namingOptions, libraryRoot); - if (extraResult.ExtraType is null) + if (extraResult.ExtraType is null || extraResult.Rule is null) { extraType = null; + extraRule = null; return false; } @@ -88,6 +89,7 @@ namespace Emby.Server.Implementations.Library.Resolvers } extraType = extraResult.ExtraType; + extraRule = extraResult.Rule; return isValid; } diff --git a/Emby.Server.Implementations/Library/Resolvers/GenericVideoResolver.cs b/Emby.Server.Implementations/Library/Resolvers/GenericVideoResolver.cs index ba320266a4..b3bdea704a 100644 --- a/Emby.Server.Implementations/Library/Resolvers/GenericVideoResolver.cs +++ b/Emby.Server.Implementations/Library/Resolvers/GenericVideoResolver.cs @@ -2,6 +2,7 @@ using Emby.Naming.Common; using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Library; using MediaBrowser.Controller.Providers; using Microsoft.Extensions.Logging; @@ -14,15 +15,25 @@ namespace Emby.Server.Implementations.Library.Resolvers public class GenericVideoResolver<T> : BaseVideoResolver<T> where T : Video, new() { + private readonly bool _parseName; + /// <summary> /// Initializes a new instance of the <see cref="GenericVideoResolver{T}"/> class. /// </summary> /// <param name="logger">The logger.</param> /// <param name="namingOptions">The naming options.</param> /// <param name="directoryService">The directory service.</param> - public GenericVideoResolver(ILogger logger, NamingOptions namingOptions, IDirectoryService directoryService) + /// <param name="parseName">Whether to parse the file name for metadata such as the year.</param> + public GenericVideoResolver(ILogger logger, NamingOptions namingOptions, IDirectoryService directoryService, bool parseName = false) : base(logger, namingOptions, directoryService) { + _parseName = parseName; + } + + /// <inheritdoc /> + protected override T Resolve(ItemResolveArgs args) + { + return ResolveVideo<T>(args, _parseName); } } } diff --git a/Emby.Server.Implementations/Library/Resolvers/PlaylistResolver.cs b/Emby.Server.Implementations/Library/Resolvers/PlaylistResolver.cs index 74c1f69616..d6513fc79c 100644 --- a/Emby.Server.Implementations/Library/Resolvers/PlaylistResolver.cs +++ b/Emby.Server.Implementations/Library/Resolvers/PlaylistResolver.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; +using Emby.Server.Implementations.Playlists; using Jellyfin.Data.Enums; using Jellyfin.Extensions; using MediaBrowser.Controller.Library; @@ -46,6 +47,19 @@ namespace Emby.Server.Implementations.Library.Resolvers }; } + // Anything directly inside the internal playlists folder is a playlist, even when its + // playlist.xml is missing: failing to resolve here makes the library scan treat the + // playlist as deleted from disk and remove it, taking its items with it. + if (args.Parent is PlaylistsFolder) + { + return new Playlist + { + Path = args.Path, + Name = filename, + OpenAccess = true + }; + } + // It's a directory-based playlist if the directory contains a playlist file IEnumerable<string> filePaths; try diff --git a/Emby.Server.Implementations/Library/Resolvers/TV/SeasonResolver.cs b/Emby.Server.Implementations/Library/Resolvers/TV/SeasonResolver.cs index 6e9a38fd34..a8bd832cc8 100644 --- a/Emby.Server.Implementations/Library/Resolvers/TV/SeasonResolver.cs +++ b/Emby.Server.Implementations/Library/Resolvers/TV/SeasonResolver.cs @@ -99,7 +99,7 @@ namespace Emby.Server.Implementations.Library.Resolvers.TV args.LibraryOptions.SeasonZeroDisplayName : string.Format( CultureInfo.InvariantCulture, - _localization.GetLocalizedString("NameSeasonNumber"), + _localization.GetServerLocalizedString("NameSeasonNumber"), seasonNumber, args.LibraryOptions.PreferredMetadataLanguage); } @@ -129,6 +129,17 @@ namespace Emby.Server.Implementations.Library.Resolvers.TV var tmdbId = justName.GetAttributeValue("tmdbid"); item.TrySetProviderId(MetadataProvider.Tmdb, tmdbId); + + // Anime databases model a single cour as its own entry, so a multi-season + // series maps to one of these ids per season rather than one per series. + var anidbId = justName.GetAttributeValue("anidbid"); + item.TrySetProviderId("AniDB", anidbId); + + var aniListId = justName.GetAttributeValue("anilistid"); + item.TrySetProviderId("AniList", aniListId); + + var aniSearchId = justName.GetAttributeValue("anisearchid"); + item.TrySetProviderId("AniSearch", aniSearchId); } } } diff --git a/Emby.Server.Implementations/Library/Search/SearchManager.cs b/Emby.Server.Implementations/Library/Search/SearchManager.cs index a5be3f07bd..a8ee416b31 100644 --- a/Emby.Server.Implementations/Library/Search/SearchManager.cs +++ b/Emby.Server.Implementations/Library/Search/SearchManager.cs @@ -92,49 +92,45 @@ public class SearchManager : ISearchManager await Task.WhenAll(externalTask, internalTask).ConfigureAwait(false); var externalResults = await externalTask.ConfigureAwait(false); - var fromExternal = externalResults.Count > 0; - IReadOnlyList<SearchResult> results; - if (fromExternal) - { - results = externalResults; - } - else - { - results = await internalTask.ConfigureAwait(false); - if (_internalProviders.Length > 0) - { - _logger.LogDebug("No results from external providers, using internal provider results"); - } - } // Internal providers apply user-access filtering inline in their queries. External // providers don't know about user permissions, so they may return IDs from hidden - // libraries or items the user is otherwise blocked from. Run the post-filter only - // when results came from externals to close that gap. The Items controller's second - // roundtrip via folder.GetItems applies most of these again, but it does not restrict - // by TopParentIds when ItemIds is set. - if (fromExternal && results.Count > 0 && query.UserId.HasValue && !query.UserId.Value.IsEmpty()) + // libraries or items the user is otherwise blocked from. Filter them here to close + // that gap. The Items controller's second roundtrip via folder.GetItems applies most + // of these again, but it does not restrict by TopParentIds when ItemIds is set. + if (externalResults.Count > 0 && query.UserId.HasValue && !query.UserId.Value.IsEmpty()) { var user = _userManager.GetUserById(query.UserId.Value); if (user is not null) { - results = await FilterByUserAccessAsync(results, user, cancellationToken).ConfigureAwait(false); + externalResults = await FilterByUserAccessAsync(externalResults, user, query, cancellationToken).ConfigureAwait(false); } } - return results; + if (externalResults.Count > 0) + { + return externalResults; + } + + if (_internalProviders.Length > 0) + { + _logger.LogDebug("No results from external providers, using internal provider results"); + } + + return await internalTask.ConfigureAwait(false); } private async Task<IReadOnlyList<SearchResult>> FilterByUserAccessAsync( IReadOnlyList<SearchResult> candidates, User user, + SearchProviderQuery query, CancellationToken cancellationToken) { - // SetUser populates parental rating + blocked/allowed tags. ConfigureUserAccess populates - // TopParentIds for the user's accessible libraries — we call it before assigning ItemIds - // because LibraryManager.AddUserToQuery skips TopParentIds when ItemIds is non-empty. - var accessFilter = new InternalItemsQuery(user); - _libraryManager.ConfigureUserAccess(accessFilter, user); + // SetUser populates parental rating + blocked/allowed tags, Build populates TopParentIds + // for the user's accessible libraries. The candidate ids are applied to the query below + // rather than to the filter because LibraryManager.AddUserToQuery skips TopParentIds when + // ItemIds is non-empty. + var accessFilter = SearchQueryAccessFilter.Build(user, query, _libraryManager); Guid[] candidateIds = [.. candidates.Select(c => c.ItemId)]; @@ -147,17 +143,24 @@ public class SearchManager : ISearchManager baseQuery = _queryHelpers.ApplyAccessFiltering(dbContext, baseQuery, accessFilter); - var allowedCount = await baseQuery.CountAsync(cancellationToken).ConfigureAwait(false); - if (allowedCount == candidates.Count) + var allowed = await baseQuery + .Select(e => new { e.Id, e.PrimaryVersionId }) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + var allowedIds = allowed.Select(e => e.Id).ToHashSet(); + + // A provider can return both an alternate version and the primary it belongs to, and the + // two are one item to the user. + allowedIds.ExceptWith(allowed + .Where(e => e.PrimaryVersionId.HasValue && allowedIds.Contains(e.PrimaryVersionId.Value)) + .Select(e => e.Id)); + + if (allowedIds.Count == candidates.Count) { return candidates; } - var allowedIds = await baseQuery - .Select(e => e.Id) - .ToHashSetAsync(cancellationToken) - .ConfigureAwait(false); - var filtered = candidates.Where(c => allowedIds.Contains(c.ItemId)).ToList(); if (filtered.Count < candidates.Count) { diff --git a/Emby.Server.Implementations/Library/Search/SearchQueryAccessFilter.cs b/Emby.Server.Implementations/Library/Search/SearchQueryAccessFilter.cs new file mode 100644 index 0000000000..6e3f01de13 --- /dev/null +++ b/Emby.Server.Implementations/Library/Search/SearchQueryAccessFilter.cs @@ -0,0 +1,38 @@ +using Jellyfin.Database.Implementations.Entities; +using Jellyfin.Extensions; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Library; + +namespace Emby.Server.Implementations.Library.Search; + +/// <summary> +/// Builds the access filter that decides which items a search may return for a user. +/// </summary> +internal static class SearchQueryAccessFilter +{ + /// <summary> + /// Builds an access filter carrying the search's library access and type filters. + /// </summary> + /// <param name="user">The user the search runs for.</param> + /// <param name="query">The search query.</param> + /// <param name="libraryManager">The library manager.</param> + /// <returns>The access filter.</returns> + public static InternalItemsQuery Build(User user, SearchProviderQuery query, ILibraryManager libraryManager) + { + // The type filters have to travel with the access filter: a by-name item belongs to no + // library, so it carries no TopParentId to match, and the library filter only knows to + // exempt it when the query says those types are wanted. A search scoped to a parent gets + // no exemption because a by-name item has no parent to descend from either. + var accessFilter = new InternalItemsQuery(user) + { + IncludeItemTypes = query.IncludeItemTypes, + ExcludeItemTypes = query.ExcludeItemTypes, + IncludeItemsByName = !query.ParentId.HasValue || query.ParentId.Value.IsEmpty() + }; + + // ConfigureUserAccess populates TopParentIds for the libraries the user may open. + libraryManager.ConfigureUserAccess(accessFilter, user); + + return accessFilter; + } +} diff --git a/Emby.Server.Implementations/Library/Search/SqlSearchProvider.cs b/Emby.Server.Implementations/Library/Search/SqlSearchProvider.cs index bc766f1c8c..a5ea02df51 100644 --- a/Emby.Server.Implementations/Library/Search/SqlSearchProvider.cs +++ b/Emby.Server.Implementations/Library/Search/SqlSearchProvider.cs @@ -10,6 +10,7 @@ using Jellyfin.Data.Enums; using Jellyfin.Database.Implementations; using Jellyfin.Database.Implementations.Entities; using Jellyfin.Extensions; +using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.Persistence; @@ -36,6 +37,7 @@ public class SqlSearchProvider : IInternalSearchProvider private readonly ILibraryManager _libraryManager; private readonly IUserManager _userManager; private readonly IItemQueryHelpers _queryHelpers; + private readonly IServerConfigurationManager _configurationManager; /// <summary> /// Initializes a new instance of the <see cref="SqlSearchProvider"/> class. @@ -45,18 +47,21 @@ public class SqlSearchProvider : IInternalSearchProvider /// <param name="libraryManager">The library manager.</param> /// <param name="userManager">The user manager.</param> /// <param name="queryHelpers">The shared item query helpers.</param> + /// <param name="configurationManager">The configuration manager.</param> public SqlSearchProvider( IDbContextFactory<JellyfinDbContext> dbProvider, IItemTypeLookup itemTypeLookup, ILibraryManager libraryManager, IUserManager userManager, - IItemQueryHelpers queryHelpers) + IItemQueryHelpers queryHelpers, + IServerConfigurationManager configurationManager) { _dbProvider = dbProvider; _itemTypeLookup = itemTypeLookup; _libraryManager = libraryManager; _userManager = userManager; _queryHelpers = queryHelpers; + _configurationManager = configurationManager; } /// <inheritdoc/> @@ -98,6 +103,12 @@ public class SqlSearchProvider : IInternalSearchProvider // so match it via a case-insensitive LIKE rather than a per-row case conversion // that may not translate to SQL on every provider. var likeOriginal = $"%{rawSearchTerm}%"; + + // Great benefit for users with metadata in non-latin script + // since SortName is among other things stored transliterated. + var sortNameShape = BaseItem.GetSortName(rawSearchTerm, true, _configurationManager.Configuration); + var likeSortName = string.IsNullOrWhiteSpace(sortNameShape) ? null : $"%{sortNameShape}%"; + var limit = query.Limit ?? DefaultSearchLimit; var dbContext = await _dbProvider.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); @@ -107,14 +118,27 @@ public class SqlSearchProvider : IInternalSearchProvider var dbQuery = dbContext.BaseItems .AsNoTracking() .Where(e => e.Id != _placeholderId) - .Where(e => !e.IsVirtualItem) - .Where(e => e.CleanName!.Contains(cleanSearchTerm) - || (e.OriginalTitle != null && EF.Functions.Like(e.OriginalTitle, likeOriginal))); + .Where(e => !e.IsVirtualItem); + + if (likeSortName is null) + { + dbQuery = dbQuery + .Where(e => e.CleanName!.Contains(cleanSearchTerm) + || (e.OriginalTitle != null && EF.Functions.Like(e.OriginalTitle, likeOriginal))); + } + else + { + dbQuery = dbQuery + .Where(e => e.CleanName!.Contains(cleanSearchTerm) + || (e.OriginalTitle != null && EF.Functions.Like(e.OriginalTitle, likeOriginal)) + || (e.SortName != null && EF.Functions.Like(e.SortName, likeSortName))); + } dbQuery = ApplyTypeFilter(dbQuery, query.IncludeItemTypes, query.ExcludeItemTypes); dbQuery = ApplyMediaTypeFilter(dbQuery, query.MediaTypes); dbQuery = ApplyParentFilter(dbQuery, query.ParentId); - dbQuery = ApplyUserAccessFilter(dbContext, dbQuery, query.UserId); + dbQuery = ApplyUserAccessFilter(dbContext, dbQuery, query); + dbQuery = ExcludeVersionsOfMatchedPrimaries(dbQuery); // Compute the score in SQL: the ternary translates to a CASE WHEN. CleanName is // the pre-normalized (lowercase, diacritic-stripped) form, so we score against it @@ -193,11 +217,18 @@ public class SqlSearchProvider : IInternalSearchProvider return query.Where(e => e.ParentId == pid || e.Parents!.Any(p => p.ParentItemId == pid)); } + private static IQueryable<BaseItemEntity> ExcludeVersionsOfMatchedPrimaries(IQueryable<BaseItemEntity> query) + { + var matched = query; + return query.Where(e => e.PrimaryVersionId == null || !matched.Any(p => p.Id == e.PrimaryVersionId)); + } + private IQueryable<BaseItemEntity> ApplyUserAccessFilter( JellyfinDbContext dbContext, IQueryable<BaseItemEntity> query, - Guid? userId) + SearchProviderQuery searchQuery) { + var userId = searchQuery.UserId; if (!userId.HasValue || userId.Value.IsEmpty()) { return query; @@ -209,8 +240,7 @@ public class SqlSearchProvider : IInternalSearchProvider return query; } - var accessFilter = new InternalItemsQuery(user); - _libraryManager.ConfigureUserAccess(accessFilter, user); + var accessFilter = SearchQueryAccessFilter.Build(user, searchQuery, _libraryManager); return _queryHelpers.ApplyAccessFiltering(dbContext, query, accessFilter); } diff --git a/Emby.Server.Implementations/Library/SimilarItems/MovieSimilarItemsProvider.cs b/Emby.Server.Implementations/Library/SimilarItems/MovieSimilarItemsProvider.cs index 57d1f7c770..cc8f0fd24e 100644 --- a/Emby.Server.Implementations/Library/SimilarItems/MovieSimilarItemsProvider.cs +++ b/Emby.Server.Implementations/Library/SimilarItems/MovieSimilarItemsProvider.cs @@ -1,3 +1,5 @@ +#pragma warning disable RS0030 // Do not use banned APIs: Guid == is required inside EF expression trees. + using System; using System.Collections.Generic; using System.Linq; @@ -172,7 +174,7 @@ public sealed class MovieSimilarItemsProvider : ILocalSimilarItemsProvider<Movie var allCandidateIdsList = allCandidateIds.ToList(); var accessibleItems = await baseQuery .WhereOneOrMany(allCandidateIdsList, e => e.Id) - .Select(e => new { e.Id, e.PresentationUniqueKey }) + .Select(e => new { e.Id, e.PresentationUniqueKey, e.PrimaryVersionId }) .ToListAsync(cancellationToken).ConfigureAwait(false); // Phase 3: Pick top IDs per source, dedup by PresentationUniqueKey @@ -189,6 +191,9 @@ public sealed class MovieSimilarItemsProvider : ILocalSimilarItemsProvider<Movie var orderedIds = accessibleItems .Where(x => scores.ContainsKey(x.Id)) .OrderByDescending(x => scores.GetValueOrDefault(x.Id)) + // Two versions of one movie score the same, so name the primary as the + // representative of the group rather than whichever came back first. + .ThenBy(x => x.PrimaryVersionId.HasValue) .DistinctBy(x => x.PresentationUniqueKey) .Take(limit) .Select(x => x.Id) @@ -245,6 +250,11 @@ public sealed class MovieSimilarItemsProvider : ILocalSimilarItemsProvider<Movie result[id] = []; } + var hiddenVersionIds = context.BaseItems.AsNoTracking() + .Where(e => e.PrimaryVersionId != null + && context.BaseItems.Any(p => p.Id == e.PrimaryVersionId && p.TopParentId == e.TopParentId)) + .Select(e => e.Id); + foreach (var (valueType, weight) in _itemValueDimensions) { var sourceRows = await context.ItemValuesMap.AsNoTracking() @@ -260,7 +270,7 @@ public sealed class MovieSimilarItemsProvider : ILocalSimilarItemsProvider<Movie } var candidateRows = await context.ItemValuesMap.AsNoTracking() - .Where(m => m.ItemValue.Type == valueType && allKeys.Contains(m.ItemValue.CleanValue)) + .Where(m => !hiddenVersionIds.Contains(m.ItemId) && m.ItemValue.Type == valueType && allKeys.Contains(m.ItemValue.CleanValue)) .Select(m => new { m.ItemId, Key = m.ItemValue.CleanValue }) .ToListAsync(cancellationToken).ConfigureAwait(false); @@ -276,6 +286,7 @@ public sealed class MovieSimilarItemsProvider : ILocalSimilarItemsProvider<Movie if (personSourceRows.Count > 0) { var personCandidateRows = await context.PeopleBaseItemMap.AsNoTracking() + .Where(m => !hiddenVersionIds.Contains(m.ItemId)) .Where(m => context.PeopleBaseItemMap .Where(s => sourceIds.Contains(s.ItemId) && _scoredPersonTypes.Contains(s.People.PersonType)) .Select(s => s.PeopleId) diff --git a/Emby.Server.Implementations/Library/SimilarItems/SimilarItemsAccessFilter.cs b/Emby.Server.Implementations/Library/SimilarItems/SimilarItemsAccessFilter.cs new file mode 100644 index 0000000000..75aea0eab6 --- /dev/null +++ b/Emby.Server.Implementations/Library/SimilarItems/SimilarItemsAccessFilter.cs @@ -0,0 +1,42 @@ +using Jellyfin.Data.Enums; +using Jellyfin.Database.Implementations.Entities; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Library; + +namespace Emby.Server.Implementations.Library.SimilarItems; + +/// <summary> +/// Builds the access filter that decides which items a similar-items lookup may return for a user. +/// </summary> +internal static class SimilarItemsAccessFilter +{ + private static readonly BaseItemKind[] _itemByNameKinds = + [ + BaseItemKind.Person, + BaseItemKind.Genre, + BaseItemKind.MusicGenre, + BaseItemKind.MusicArtist, + BaseItemKind.Studio + ]; + + /// <summary> + /// Builds an access filter carrying the user's library access and parental restrictions. + /// </summary> + /// <param name="user">The user the lookup runs for.</param> + /// <param name="libraryManager">The library manager.</param> + /// <returns>The access filter.</returns> + public static InternalItemsQuery Build(User user, ILibraryManager libraryManager) + { + // IncludeItemTypes is read only for the by-name exemption here; the caller applies this + // filter through ApplyAccessFiltering, which does not translate it into a type restriction. + var accessFilter = new InternalItemsQuery(user) + { + IncludeItemTypes = _itemByNameKinds + }; + + // ConfigureUserAccess populates TopParentIds for the libraries the user may open. + libraryManager.ConfigureUserAccess(accessFilter, user); + + return accessFilter; + } +} diff --git a/Emby.Server.Implementations/Library/SimilarItems/SimilarItemsManager.cs b/Emby.Server.Implementations/Library/SimilarItems/SimilarItemsManager.cs index d923cff07e..a18a17b593 100644 --- a/Emby.Server.Implementations/Library/SimilarItems/SimilarItemsManager.cs +++ b/Emby.Server.Implementations/Library/SimilarItems/SimilarItemsManager.cs @@ -7,8 +7,10 @@ using System.Text.Json; using System.Threading; using System.Threading.Tasks; using Jellyfin.Data.Enums; +using Jellyfin.Database.Implementations; using Jellyfin.Database.Implementations.Entities; using Jellyfin.Database.Implementations.Enums; +using Jellyfin.Extensions; using Jellyfin.Extensions.Json; using MediaBrowser.Common.Extensions; using MediaBrowser.Controller; @@ -16,11 +18,13 @@ using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Dto; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; +using MediaBrowser.Controller.Persistence; using MediaBrowser.Model.Configuration; using MediaBrowser.Model.Dto; using MediaBrowser.Model.Entities; using MediaBrowser.Model.IO; using MediaBrowser.Model.Querying; +using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; namespace Emby.Server.Implementations.Library.SimilarItems; @@ -35,6 +39,8 @@ public class SimilarItemsManager : ISimilarItemsManager private readonly ILibraryManager _libraryManager; private readonly IFileSystem _fileSystem; private readonly IServerConfigurationManager _serverConfigurationManager; + private readonly IDbContextFactory<JellyfinDbContext> _dbProvider; + private readonly IItemQueryHelpers _queryHelpers; private ISimilarItemsProvider[] _similarItemsProviders = []; /// <summary> @@ -45,18 +51,24 @@ public class SimilarItemsManager : ISimilarItemsManager /// <param name="libraryManager">The library manager.</param> /// <param name="fileSystem">The file system.</param> /// <param name="serverConfigurationManager">The server configuration manager.</param> + /// <param name="dbProvider">The database context factory.</param> + /// <param name="queryHelpers">The shared item query helpers.</param> public SimilarItemsManager( ILogger<SimilarItemsManager> logger, IServerApplicationPaths appPaths, ILibraryManager libraryManager, IFileSystem fileSystem, - IServerConfigurationManager serverConfigurationManager) + IServerConfigurationManager serverConfigurationManager, + IDbContextFactory<JellyfinDbContext> dbProvider, + IItemQueryHelpers queryHelpers) { _logger = logger; _appPaths = appPaths; _libraryManager = libraryManager; _fileSystem = fileSystem; _serverConfigurationManager = serverConfigurationManager; + _dbProvider = dbProvider; + _queryHelpers = queryHelpers; } /// <inheritdoc/> @@ -183,6 +195,7 @@ public class SimilarItemsManager : ISimilarItemsManager // Collect references in batches and resolve against local library. // Stop fetching once we have enough resolved local items. const int BatchSize = 20; + const int MaxRemoteReferenceFetchLimit = 500; var remaining = requestedLimit - allResults.Count; var collectedReferences = new List<SimilarItemReference>(); var pendingBatch = new List<SimilarItemReference>(); @@ -199,7 +212,7 @@ public class SimilarItemsManager : ISimilarItemsManager remaining -= resolvedItems.Count; pendingBatch.Clear(); - if (remaining <= 0) + if (remaining <= 0 || collectedReferences.Count >= MaxRemoteReferenceFetchLimit) { break; } @@ -229,11 +242,64 @@ public class SimilarItemsManager : ISimilarItemsManager } } - return allResults + var ordered = allResults .OrderByDescending(x => x.Score) .Select(x => x.Item) .Take(requestedLimit) .ToList(); + + return await FilterByLibraryAccessAsync(ordered, user, cancellationToken).ConfigureAwait(false); + } + + private async Task<IReadOnlyList<BaseItem>> FilterByLibraryAccessAsync( + IReadOnlyList<BaseItem> candidates, + User? user, + CancellationToken cancellationToken) + { + if (candidates.Count == 0 || user is null) + { + return candidates; + } + + var accessFilter = SimilarItemsAccessFilter.Build(user, _libraryManager); + + // No accessible libraries means nothing to compare against, and an empty TopParentIds set + // would disable the filter rather than reject everything. + if (accessFilter.TopParentIds.Length == 0) + { + return candidates; + } + + Guid[] candidateIds = [.. candidates.Select(c => c.Id)]; + + var dbContext = await _dbProvider.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (dbContext.ConfigureAwait(false)) + { + var baseQuery = dbContext.BaseItems + .AsNoTracking() + .WhereOneOrMany(candidateIds, e => e.Id); + + baseQuery = _queryHelpers.ApplyAccessFiltering(dbContext, baseQuery, accessFilter); + + var allowedCount = await baseQuery.CountAsync(cancellationToken).ConfigureAwait(false); + if (allowedCount == candidates.Count) + { + return candidates; + } + + var allowedIds = await baseQuery + .Select(e => e.Id) + .ToHashSetAsync(cancellationToken) + .ConfigureAwait(false); + + var filtered = candidates.Where(c => allowedIds.Contains(c.Id)).ToList(); + _logger.LogDebug( + "Dropped {Dropped} of {Total} similar-item candidates due to user access filtering", + candidates.Count - filtered.Count, + candidates.Count); + + return filtered; + } } /// <inheritdoc/> @@ -375,19 +441,39 @@ public class SimilarItemsManager : ISimilarItemsManager var batchResults = await batchProvider.GetBatchSimilarItemsAsync(baselineItems, query, cancellationToken).ConfigureAwait(false); + // Filter once across every category rather than per baseline, so a batch provider costs one + // access query no matter how many categories it produced. + var allItems = batchResults.Values.SelectMany(items => items).DistinctBy(item => item.Id).ToList(); + var allowed = await FilterByLibraryAccessAsync(allItems, query.User, cancellationToken).ConfigureAwait(false); + + HashSet<Guid>? allowedIds = allowed.Count == allItems.Count + ? null + : [.. allowed.Select(item => item.Id)]; + var recommendations = new List<SimilarItemsRecommendation>(baselineItems.Count); foreach (var baseline in baselineItems) { - if (batchResults.TryGetValue(baseline.Id, out var similar) && similar.Count > 0) + if (!batchResults.TryGetValue(baseline.Id, out var similar) || similar.Count == 0) + { + continue; + } + + if (allowedIds is not null) { - recommendations.Add(new SimilarItemsRecommendation + similar = similar.Where(item => allowedIds.Contains(item.Id)).ToList(); + if (similar.Count == 0) { - BaselineItemName = baseline.Name, - CategoryId = baseline.Id, - RecommendationType = recommendationType, - Items = similar - }); + continue; + } } + + recommendations.Add(new SimilarItemsRecommendation + { + BaselineItemName = baseline.Name, + CategoryId = baseline.Id, + RecommendationType = recommendationType, + Items = similar + }); } return recommendations; @@ -565,7 +651,13 @@ public class SimilarItemsManager : ISimilarItemsManager try { - var stream = File.OpenRead(cachePath); + var stream = new FileStream( + cachePath, + FileMode.Open, + FileAccess.Read, + FileShare.ReadWrite | FileShare.Delete, + IODefaults.FileStreamBufferSize, + FileOptions.Asynchronous | FileOptions.SequentialScan); await using (stream.ConfigureAwait(false)) { var cache = await JsonSerializer.DeserializeAsync<SimilarItemsCache>(stream, JsonDefaults.Options, cancellationToken).ConfigureAwait(false); @@ -589,6 +681,7 @@ public class SimilarItemsManager : ISimilarItemsManager private async Task SaveSimilarItemsCacheAsync(string cachePath, List<SimilarItemReference> references, TimeSpan cacheDuration, CancellationToken cancellationToken) { + string? tempPath = null; try { var directory = Path.GetDirectoryName(cachePath); @@ -603,16 +696,34 @@ public class SimilarItemsManager : ISimilarItemsManager ExpiresAt = DateTime.UtcNow.Add(cacheDuration) }; - var stream = File.Create(cachePath); + tempPath = cachePath + "." + Guid.NewGuid().ToString("N", CultureInfo.InvariantCulture) + ".tmp"; + var stream = File.Create(tempPath); await using (stream.ConfigureAwait(false)) { await JsonSerializer.SerializeAsync(stream, cache, JsonDefaults.Options, cancellationToken).ConfigureAwait(false); } + + File.Move(tempPath, cachePath, true); + tempPath = null; } catch (IOException ex) { _logger.LogWarning(ex, "Failed to save similar items cache to {CachePath}", cachePath); } + finally + { + if (tempPath is not null) + { + try + { + File.Delete(tempPath); + } + catch (IOException ex) + { + _logger.LogDebug(ex, "Failed to delete temporary similar items cache file {TempPath}", tempPath); + } + } + } } private sealed class SimilarItemsCache diff --git a/Emby.Server.Implementations/Library/UserDataManager.cs b/Emby.Server.Implementations/Library/UserDataManager.cs index f5c41e5670..1e7068b24b 100644 --- a/Emby.Server.Implementations/Library/UserDataManager.cs +++ b/Emby.Server.Implementations/Library/UserDataManager.cs @@ -75,6 +75,22 @@ namespace Emby.Server.Implementations.Library } dbContext.SaveChanges(); + + // Rows under keys the item no longer reports are kept, not deleted: the key set is briefly + // incomplete mid-refresh, and those rows are what a deleted and re-added item reattaches by. + dbContext.UserData + .Where(e => e.ItemId == item.Id && e.UserId == user.Id && !keys.Contains(e.CustomDataKey)) + .ExecuteUpdate(e => e + .SetProperty(f => f.AudioStreamIndex, userData.AudioStreamIndex) + .SetProperty(f => f.IsFavorite, userData.IsFavorite) + .SetProperty(f => f.LastPlayedDate, userData.LastPlayedDate) + .SetProperty(f => f.Likes, userData.Likes) + .SetProperty(f => f.PlaybackPositionTicks, userData.PlaybackPositionTicks) + .SetProperty(f => f.PlayCount, userData.PlayCount) + .SetProperty(f => f.Played, userData.Played) + .SetProperty(f => f.Rating, userData.Rating) + .SetProperty(f => f.SubtitleStreamIndex, userData.SubtitleStreamIndex)); + transaction.Commit(); var userId = user.InternalId; @@ -352,6 +368,7 @@ namespace Emby.Server.Implementations.Library /// <inheritdoc /> public UserItemData? GetUserData(User user, BaseItem item) { + ArgumentNullException.ThrowIfNull(user); var row = ResolveUserDataRow(item, item.UserData?.Where(e => e.UserId.Equals(user.Id))); return row is not null ? Map(row) : new UserItemData() { diff --git a/Emby.Server.Implementations/Library/UserViewManager.cs b/Emby.Server.Implementations/Library/UserViewManager.cs index 9512b0ffd7..6d52ce7440 100644 --- a/Emby.Server.Implementations/Library/UserViewManager.cs +++ b/Emby.Server.Implementations/Library/UserViewManager.cs @@ -62,12 +62,11 @@ namespace Emby.Server.Implementations.Library // Playlist and BoxSet libraries require special handling because the folder only references linked items if (folderViewType == CollectionType.playlists || folderViewType == CollectionType.boxsets) { - var items = folder.GetItemList(new InternalItemsQuery(user) - { - ParentId = folder.ParentId - }); + var itemKind = folderViewType == CollectionType.playlists + ? BaseItemKind.Playlist + : BaseItemKind.BoxSet; - if (!items.Any(item => item.IsVisible(user))) + if (!HasVisibleItem(itemKind, folders, user)) { continue; } @@ -112,7 +111,7 @@ namespace Emby.Server.Implementations.Library if (_config.Configuration.EnableFolderView) { - var name = _localizationManager.GetLocalizedString("Folders"); + var name = _localizationManager.GetServerLocalizedString("Folders"); list.Add(_libraryManager.GetNamedView(name, CollectionType.folders, string.Empty)); } @@ -127,7 +126,7 @@ namespace Emby.Server.Implementations.Library list.AddRange(channels); - if (_liveTvManager.GetEnabledUsers().Select(i => i.Id).Contains(user.Id)) + if (_liveTvManager.IsEnabledForUser(user)) { list.Add(_liveTvManager.GetInternalLiveTvFolder(CancellationToken.None)); } @@ -159,6 +158,32 @@ namespace Emby.Server.Implementations.Library .ToArray(); } + private bool HasVisibleItem(BaseItemKind itemKind, IReadOnlyList<Folder> folders, User user) + { + var topParentIds = folders.SelectMany(GetTopParentIds).ToArray(); + if (topParentIds.Length == 0) + { + return false; + } + + var items = _libraryManager.GetItemList(new InternalItemsQuery(user) + { + IncludeItemTypes = [itemKind], + TopParentIds = topParentIds, + GroupByPresentationUniqueKey = false, + DtoOptions = DtoOptions.StoredColumnsOnly + }); + + return items.Any(item => item.IsVisible(user)); + } + + private static IEnumerable<Guid> GetTopParentIds(Folder folder) + { + return folder is CollectionFolder collectionFolder && collectionFolder.PhysicalFolderIds.Length > 0 + ? collectionFolder.PhysicalFolderIds + : [folder.Id]; + } + public UserView GetUserSubViewWithName(string name, Guid parentId, CollectionType? type, string sortName) { var uniqueId = parentId + "subview" + type; @@ -168,7 +193,7 @@ namespace Emby.Server.Implementations.Library public UserView GetUserSubView(Guid parentId, CollectionType? type, string localizationKey, string sortName) { - var name = _localizationManager.GetLocalizedString(localizationKey); + var name = _localizationManager.GetServerLocalizedString(localizationKey); return GetUserSubViewWithName(name, parentId, type, sortName); } @@ -191,7 +216,7 @@ namespace Emby.Server.Implementations.Library return GetUserView((Folder)parents[0], viewType, string.Empty); } - var name = _localizationManager.GetLocalizedString(localizationKey); + var name = _localizationManager.GetServerLocalizedString(localizationKey); return _libraryManager.GetNamedView(user, name, viewType, sortName); } @@ -396,6 +421,12 @@ namespace Emby.Server.Implementations.Library query.Limit = limit; return _libraryManager.GetLatestItemList(query, parents, CollectionType.movies); } + + if (collectionType is null) + { + query.Limit = limit; + return _libraryManager.GetLatestItemList(query, parents, CollectionType.unknown); + } } return _libraryManager.GetItemList(query, parents); diff --git a/Emby.Server.Implementations/Library/Validators/ArtistsValidator.cs b/Emby.Server.Implementations/Library/Validators/ArtistsValidator.cs index fa7112eb90..690466be70 100644 --- a/Emby.Server.Implementations/Library/Validators/ArtistsValidator.cs +++ b/Emby.Server.Implementations/Library/Validators/ArtistsValidator.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Globalization; using System.Linq; using System.Threading; @@ -61,6 +62,9 @@ public class ArtistsValidator var count = names.Count; var refreshed = 0; + var liveIds = new HashSet<Guid>(); + var unresolved = 0; + foreach (var name in names) { try @@ -73,13 +77,20 @@ public class ArtistsValidator // Fall back to GetArtist if not found (creates new item if needed) item ??= _libraryManager.GetArtist(name); - var isNew = !existingArtistIds.Contains(item.Id); - var neverRefreshed = item.DateLastRefreshed == default; - if (isNew || neverRefreshed) + // A name with no item is nothing to refresh, and nothing to keep alive either. + if (item is not null) { - await item.RefreshMetadata(cancellationToken).ConfigureAwait(false); - refreshed++; + liveIds.Add(item.Id); + + var isNew = !existingArtistIds.Contains(item.Id); + var neverRefreshed = item.DateLastRefreshed == default; + + if (isNew || neverRefreshed) + { + await item.RefreshMetadata(cancellationToken).ConfigureAwait(false); + refreshed++; + } } } catch (OperationCanceledException) @@ -88,6 +99,7 @@ public class ArtistsValidator } catch (Exception ex) { + unresolved++; _logger.LogError(ex, "Error refreshing {ArtistName}", name); } @@ -101,13 +113,26 @@ public class ArtistsValidator _logger.LogInformation("Refreshed metadata for {RefreshedCount} new artists out of {TotalCount} total", refreshed, count); + // Every name that threw is a name whose artist is missing from the live set, and deleting against + // a live set with holes in it deletes artists the library still refers to. Leave the sweep to a + // run that got a clean read of them. + if (unresolved > 0) + { + _logger.LogWarning( + "Not removing dead artists: {Count} of {TotalCount} names could not be resolved this run", + unresolved, + count); + + progress.Report(100); + return; + } + var deadEntities = _libraryManager.GetItemList(new InternalItemsQuery { IncludeItemTypes = [BaseItemKind.MusicArtist], - IsDeadArtist = true, IsLocked = false - }).Cast<MusicArtist>() - .Where(item => item.IsAccessedByName) + }).OfType<MusicArtist>() + .Where(item => item.IsAccessedByName && !liveIds.Contains(item.Id)) .ToList(); foreach (var item in deadEntities) diff --git a/Emby.Server.Implementations/Library/Validators/CollectionPosterVerifyPostScanTask.cs b/Emby.Server.Implementations/Library/Validators/CollectionPosterVerifyPostScanTask.cs new file mode 100644 index 0000000000..2cfa446862 --- /dev/null +++ b/Emby.Server.Implementations/Library/Validators/CollectionPosterVerifyPostScanTask.cs @@ -0,0 +1,64 @@ +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Library; +using MediaBrowser.Model.Entities; +using Microsoft.Extensions.Logging; + +namespace Emby.Server.Implementations.Library.Validators; + +/// <summary> +/// Ensures top-level library folders have a primary poster after scans. +/// Poster extraction is attempted before library scanning. When a library is +/// empty at that point, no poster can be extracted. This post-scan task reruns +/// metadata extraction for top-level folders that are still missing images. +/// </summary> +public class CollectionPosterVerifyPostScanTask : ILibraryPostScanTask +{ + private readonly ILibraryManager _libraryManager; + private readonly ILogger<CollectionPosterVerifyPostScanTask> _logger; + + /// <summary> + /// Initializes a new instance of the <see cref="CollectionPosterVerifyPostScanTask" /> class. + /// </summary> + /// <param name="libraryManager">The library manager.</param> + /// <param name="logger">The logger.</param> + public CollectionPosterVerifyPostScanTask( + ILibraryManager libraryManager, + ILogger<CollectionPosterVerifyPostScanTask> logger) + { + _libraryManager = libraryManager; + _logger = logger; + } + + /// <summary> + /// Runs the specified progress. + /// </summary> + /// <param name="progress">The progress.</param> + /// <param name="cancellationToken">The cancellation token.</param> + /// <returns>Task.</returns> + public async Task Run(IProgress<double> progress, CancellationToken cancellationToken) + { + var libraries = _libraryManager.GetUserRootFolder().Children.OfType<CollectionFolder>().ToList(); + var totalLibraries = libraries.Count; + var processedLibraries = 0; + + foreach (var library in libraries) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (!library.HasImage(ImageType.Primary)) + { + _logger.LogDebug("Library {LibraryName} is missing a primary image. Refreshing metadata.", library.Name); + await library.RefreshMetadata(cancellationToken).ConfigureAwait(false); + } + + processedLibraries++; + progress.Report((double)processedLibraries / totalLibraries * 100); + } + + progress.Report(100); + } +} diff --git a/Emby.Server.Implementations/Library/Validators/PeopleValidator.cs b/Emby.Server.Implementations/Library/Validators/PeopleValidator.cs index dacef102dd..7d53f40ce7 100644 --- a/Emby.Server.Implementations/Library/Validators/PeopleValidator.cs +++ b/Emby.Server.Implementations/Library/Validators/PeopleValidator.cs @@ -1,12 +1,12 @@ using System; +using System.Collections.Generic; +using System.Globalization; using System.Linq; using System.Threading; using System.Threading.Tasks; using Jellyfin.Data.Enums; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; -using MediaBrowser.Controller.Providers; -using MediaBrowser.Model.IO; using Microsoft.Extensions.Logging; namespace Emby.Server.Implementations.Library.Validators; @@ -17,104 +17,143 @@ namespace Emby.Server.Implementations.Library.Validators; public class PeopleValidator { /// <summary> - /// The _library manager. + /// The library manager. /// </summary> private readonly ILibraryManager _libraryManager; /// <summary> - /// The _logger. + /// The logger. /// </summary> - private readonly ILogger _logger; - - private readonly IFileSystem _fileSystem; + private readonly ILogger<PeopleValidator> _logger; /// <summary> /// Initializes a new instance of the <see cref="PeopleValidator" /> class. /// </summary> /// <param name="libraryManager">The library manager.</param> /// <param name="logger">The logger.</param> - /// <param name="fileSystem">The file system.</param> - public PeopleValidator(ILibraryManager libraryManager, ILogger logger, IFileSystem fileSystem) + public PeopleValidator(ILibraryManager libraryManager, ILogger<PeopleValidator> logger) { _libraryManager = libraryManager; _logger = logger; - _fileSystem = fileSystem; } /// <summary> /// Validates the people. /// </summary> - /// <param name="cancellationToken">The cancellation token.</param> /// <param name="progress">The progress.</param> + /// <param name="cancellationToken">The cancellation token.</param> /// <returns>Task.</returns> - public async Task ValidatePeople(CancellationToken cancellationToken, IProgress<double> progress) + public async Task Run(IProgress<double> progress, CancellationToken cancellationToken) { - var people = _libraryManager.GetPeopleNames(new InternalPeopleQuery()); - - var numComplete = 0; + // Before the refresh below walks them: a credit no item maps to any more stands for nothing, + // and while it is there the person it names cannot reach the dead-person sweep either. + var numOrphaned = _libraryManager.DeleteOrphanedCredits(); + if (numOrphaned > 0) + { + _logger.LogInformation("Deleted {Amount} credits no item maps to", numOrphaned); + } - var numPeople = people.Count; + var names = _libraryManager.GetPeopleNames(new InternalPeopleQuery()); + var existingPersonIds = _libraryManager.GetItemIds(new InternalItemsQuery + { + IncludeItemTypes = [BaseItemKind.Person] + }).ToHashSet(); - IProgress<double> subProgress = new Progress<double>((val) => progress.Report(val / 2)); + var (newNames, deadIds) = PartitionCreditsByPersonId(names, _libraryManager.GetPersonId, existingPersonIds); - _logger.LogDebug("Will refresh {Amount} people", numPeople); + var numComplete = 0; + var count = names.Count; + var refreshed = 0; - foreach (var person in people) + foreach (var name in names) { cancellationToken.ThrowIfCancellationRequested(); try { - var item = _libraryManager.GetPerson(person); - if (item is null) - { - _logger.LogWarning("Failed to get person: {Name}", person); - continue; - } + var item = _libraryManager.GetOrCreatePerson(name); + var isNew = !existingPersonIds.Contains(item.Id); + var neverRefreshed = item.DateLastRefreshed == default; - var options = new MetadataRefreshOptions(new DirectoryService(_fileSystem)) + if (isNew || neverRefreshed) { - ImageRefreshMode = MetadataRefreshMode.ValidationOnly, - MetadataRefreshMode = MetadataRefreshMode.ValidationOnly - }; - - await item.RefreshMetadata(options, cancellationToken).ConfigureAwait(false); + await item.RefreshMetadata(cancellationToken).ConfigureAwait(false); + refreshed++; + } } catch (OperationCanceledException) { + // Don't clutter the log throw; } catch (Exception ex) { - _logger.LogError(ex, "Error validating IBN entry {Person}", person); + _logger.LogError(ex, "Error refreshing {PersonName}", name); } - // Update progress numComplete++; double percent = numComplete; - percent /= numPeople; + percent /= count; + percent *= 100; - subProgress.Report(100 * percent); + progress.Report(percent); } - var deadEntities = _libraryManager.GetItemList(new InternalItemsQuery - { - IncludeItemTypes = [BaseItemKind.Person], - IsDeadPerson = true, - IsLocked = false - }); + _logger.LogInformation( + "Refreshed metadata for {RefreshedCount} people out of {TotalCount} total, {NewCount} of which had no item yet", + refreshed, + count, + newNames.Count); - subProgress = new Progress<double>((val) => progress.Report((val / 2) + 50)); + // A person somebody locked is theirs, not ours, however little the library still credits them. + var deadEntities = deadIds + .Select(_libraryManager.GetItemById) + .OfType<Person>() + .Where(item => !item.IsLocked) + .ToList(); - var i = 0; - foreach (var item in deadEntities.Chunk(500)) + foreach (var item in deadEntities) { - _libraryManager.DeleteItemsUnsafeFast(item, true); - subProgress.Report(100f / deadEntities.Count * (i++ * 100)); + _logger.LogInformation("Deleting dead {ItemType} {ItemId} {ItemName}", item.GetType().Name, item.Id.ToString("N", CultureInfo.InvariantCulture), item.Name); } + _libraryManager.DeleteItemsUnsafeFast(deadEntities, deleteSourceFiles: true); + progress.Report(100); + } + + /// <summary> + /// Splits the person items into the ones a credit still calls for and the ones nothing does. + /// </summary> + /// <param name="creditNames">Every name credited on an item, from the people table.</param> + /// <param name="getPersonId">Maps a credit name to the id its person item has.</param> + /// <param name="existingPersonIds">The ids of the person items that exist.</param> + /// <returns>The credits needing an item, and the ids of the items nothing credits.</returns> + internal static (List<string> NewNames, List<Guid> DeadIds) PartitionCreditsByPersonId( + IReadOnlyList<string> creditNames, + Func<string, Guid> getPersonId, + IReadOnlySet<Guid> existingPersonIds) + { + ArgumentNullException.ThrowIfNull(creditNames); + ArgumentNullException.ThrowIfNull(getPersonId); + ArgumentNullException.ThrowIfNull(existingPersonIds); + + var newNames = new List<string>(); + var liveIds = new HashSet<Guid>(); + + foreach (var name in creditNames) + { + var personId = getPersonId(name); + + // Distinct credit names can normalize onto one id; only the first of them needs an item. + if (liveIds.Add(personId) && !existingPersonIds.Contains(personId)) + { + newNames.Add(name); + } + } + + var deadIds = existingPersonIds.Where(id => !liveIds.Contains(id)).ToList(); - _logger.LogInformation("People validation complete"); + return (newNames, deadIds); } } |