diff options
Diffstat (limited to 'Emby.Server.Implementations/Library')
12 files changed, 630 insertions, 175 deletions
diff --git a/Emby.Server.Implementations/Library/LibraryManager.cs b/Emby.Server.Implementations/Library/LibraryManager.cs index dd8c883684..16616dd328 100644 --- a/Emby.Server.Implementations/Library/LibraryManager.cs +++ b/Emby.Server.Implementations/Library/LibraryManager.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Globalization; using System.IO; using System.Linq; @@ -15,7 +16,6 @@ using Emby.Naming.Common; using Emby.Naming.TV; using Emby.Naming.Video; using Emby.Server.Implementations.Library.Resolvers; -using Emby.Server.Implementations.Library.Validators; using Emby.Server.Implementations.Playlists; using Emby.Server.Implementations.ScheduledTasks.Tasks; using Emby.Server.Implementations.Sorting; @@ -35,7 +35,6 @@ using MediaBrowser.Controller.Entities.Movies; using MediaBrowser.Controller.IO; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.LiveTv; -using MediaBrowser.Controller.MediaEncoding; using MediaBrowser.Controller.Persistence; using MediaBrowser.Controller.Playlists; using MediaBrowser.Controller.Providers; @@ -75,7 +74,6 @@ namespace Emby.Server.Implementations.Library private readonly Lazy<IProviderManager> _providerManagerFactory; private readonly Lazy<IUserViewManager> _userViewManagerFactory; private readonly IServerApplicationHost _appHost; - private readonly IMediaEncoder _mediaEncoder; private readonly IFileSystem _fileSystem; private readonly IItemRepository _itemRepository; private readonly IItemPersistenceService _persistenceService; @@ -88,6 +86,7 @@ namespace Emby.Server.Implementations.Library private readonly ExtraResolver _extraResolver; private readonly IPathManager _pathManager; private readonly ILocalizationManager _localization; + private readonly IDirectoryService _directoryService; private readonly FastConcurrentLru<Guid, BaseItem> _cache; private readonly DotIgnoreIgnoreRule _dotIgnoreIgnoreRule; private readonly IMediaStreamRepository _mediaStreamRepository; @@ -122,7 +121,6 @@ namespace Emby.Server.Implementations.Library /// <param name="fileSystem">The file system.</param> /// <param name="providerManagerFactory">The provider manager.</param> /// <param name="userViewManagerFactory">The user view manager.</param> - /// <param name="mediaEncoder">The media encoder.</param> /// <param name="itemRepository">The item repository.</param> /// <param name="persistenceService">The item persistence service.</param> /// <param name="nextUpService">The next up service.</param> @@ -148,7 +146,6 @@ namespace Emby.Server.Implementations.Library IFileSystem fileSystem, Lazy<IProviderManager> providerManagerFactory, Lazy<IUserViewManager> userViewManagerFactory, - IMediaEncoder mediaEncoder, IItemRepository itemRepository, IItemPersistenceService persistenceService, INextUpService nextUpService, @@ -174,7 +171,6 @@ namespace Emby.Server.Implementations.Library _fileSystem = fileSystem; _providerManagerFactory = providerManagerFactory; _userViewManagerFactory = userViewManagerFactory; - _mediaEncoder = mediaEncoder; _itemRepository = itemRepository; _persistenceService = persistenceService; _nextUpService = nextUpService; @@ -189,6 +185,7 @@ namespace Emby.Server.Implementations.Library _pathManager = pathManager; _dotIgnoreIgnoreRule = dotIgnoreIgnoreRule; _localization = localization; + _directoryService = directoryService; _extraResolver = new ExtraResolver(loggerFactory.CreateLogger<ExtraResolver>(), namingOptions, directoryService); _configurationManager.ConfigurationUpdated += ConfigurationUpdated; @@ -321,7 +318,7 @@ namespace Emby.Server.Implementations.Library if (wizardChanged) { - _taskManager.CancelIfRunningAndQueue<RefreshMediaLibraryTask>(); + QueueLibraryScan(); } } @@ -881,7 +878,18 @@ namespace Emby.Server.Implementations.Library wrongTypeItem.GetType().Name, expectedVideoType.Name, path); - DeleteItem(wrongTypeItem, new DeleteOptions { DeleteFileLocation = false }); + + // A full DeleteItem would save the primary version, which resolves its + // alternates again and re-enters here before this row is gone. + DeleteItemsUnsafeFast([wrongTypeItem]); + + // The fast path skips the parent bookkeeping, and the stale item is listed + // under its ParentId, so that folder's cached listing has to be dropped. + if (wrongTypeItem.GetParent() is Folder staleParent) + { + staleParent.Children = null; + staleParent.UserData = null; + } } } @@ -1210,6 +1218,12 @@ namespace Emby.Server.Implementations.Library } /// <inheritdoc /> + public Guid GetPersonId(string name) + { + return GetItemByNameId<Person>(Person.GetPath(name)); + } + + /// <inheritdoc /> public Person? GetPerson(string name) { var path = Person.GetPath(name); @@ -1222,6 +1236,33 @@ namespace Emby.Server.Implementations.Library return null; } + /// <inheritdoc /> + public Person GetOrCreatePerson(string name) + { + var existing = GetPerson(name); + if (existing is not null) + { + return existing; + } + + var path = Person.GetPath(name); + var info = Directory.CreateDirectory(path); + var item = new Person + { + Name = name, + Id = GetItemByNameId<Person>(path), + DateCreated = info.CreationTimeUtc, + DateModified = info.LastWriteTimeUtc, + Path = path + }; + + item.PresentationUniqueKey = item.CreatePresentationUniqueKey(); + + CreateItem(item, null); + + return item; + } + /// <summary> /// Gets the studio. /// </summary> @@ -1313,7 +1354,6 @@ namespace Emby.Server.Implementations.Library { IncludeItemTypes = [BaseItemKind.MusicArtist], Name = name, - UseRawName = true, DtoOptions = options }).Cast<MusicArtist>() .OrderBy(i => i.IsAccessedByName ? 1 : 0) @@ -1354,15 +1394,6 @@ namespace Emby.Server.Implementations.Library return GetNewItemIdInternal(path, typeof(T), forceCaseInsensitiveId); } - /// <inheritdoc /> - public Task ValidatePeopleAsync(IProgress<double> progress, CancellationToken cancellationToken) - { - // Ensure the location is available. - Directory.CreateDirectory(_configurationManager.ApplicationPaths.PeoplePath); - - return new PeopleValidator(this, _logger, _fileSystem).ValidatePeople(cancellationToken, progress); - } - /// <summary> /// Reloads the root media folder. /// </summary> @@ -1489,6 +1520,10 @@ namespace Emby.Server.Implementations.Library var numComplete = 0; var numTasks = tasks.Count; + _logger.LogInformation("Running {TaskCount} post-scan task(s)", numTasks); + + var phaseStart = Stopwatch.GetTimestamp(); + foreach (var task in tasks) { // Prevent access to modified closure @@ -1506,20 +1541,45 @@ namespace Emby.Server.Implementations.Library progress.Report(innerPercent); }); - _logger.LogDebug("Running post-scan task {0}", task.GetType().Name); + var taskName = task.GetType().Name; + var taskStart = Stopwatch.GetTimestamp(); + + _logger.LogInformation( + "Running post-scan task {TaskNumber}/{TaskCount}: {TaskName}", + currentNumComplete + 1, + numTasks, + taskName); try { await task.Run(innerProgress, cancellationToken).ConfigureAwait(false); + + var elapsed = Stopwatch.GetElapsedTime(taskStart); + _logger.LogInformation( + "Post-scan task {TaskName} completed after {Minutes} minute(s) and {Seconds} seconds", + taskName, + Math.Truncate(elapsed.TotalMinutes), + elapsed.Seconds); } catch (OperationCanceledException) { - _logger.LogInformation("Post-scan task cancelled: {0}", task.GetType().Name); + var elapsed = Stopwatch.GetElapsedTime(taskStart); + _logger.LogInformation( + "Post-scan task {TaskName} cancelled after {Minutes} minute(s) and {Seconds} seconds", + taskName, + Math.Truncate(elapsed.TotalMinutes), + elapsed.Seconds); throw; } catch (Exception ex) { - _logger.LogError(ex, "Error running post-scan task"); + var elapsed = Stopwatch.GetElapsedTime(taskStart); + _logger.LogError( + ex, + "Post-scan task {TaskName} failed after {Minutes} minute(s) and {Seconds} seconds", + taskName, + Math.Truncate(elapsed.TotalMinutes), + elapsed.Seconds); } numComplete++; @@ -1528,6 +1588,12 @@ namespace Emby.Server.Implementations.Library progress.Report(percent * 100); } + var phaseElapsed = Stopwatch.GetElapsedTime(phaseStart); + _logger.LogInformation( + "All post-scan tasks completed after {Minutes} minute(s) and {Seconds} seconds", + Math.Truncate(phaseElapsed.TotalMinutes), + phaseElapsed.Seconds); + _persistenceService.UpdateInheritedValues(); progress.Report(100); @@ -1745,9 +1811,21 @@ namespace Emby.Server.Implementations.Library return _countService.GetItemCountsForNameItem(kind, id, relatedItemKinds, query); } - public Dictionary<Guid, int> GetChildCountBatch(IReadOnlyList<Guid> parentIds, Guid? userId) + /// <inheritdoc/> + public Dictionary<Guid, ItemCounts> GetItemCountsForNameItems(BaseItemKind kind, IReadOnlyList<Guid> ids, BaseItemKind[] relatedItemKinds, User? user) { - return _countService.GetChildCountBatch(parentIds, userId); + var query = new InternalItemsQuery(user); + if (user is not null) + { + AddUserToQuery(query, user); + } + + return _countService.GetItemCountsForNameItems(kind, ids, relatedItemKinds, query); + } + + public Dictionary<Guid, int> GetChildCountBatch(IReadOnlyList<Guid> parentIds, User? user) + { + return _countService.GetChildCountBatch(parentIds, user); } /// <inheritdoc/> @@ -1984,18 +2062,10 @@ namespace Emby.Server.Implementations.Library { // Playlists and BoxSets store their contents in LinkedChildren and never // populate AncestorIds for those items, so a recursive AncestorIds query - // would return zero rows. Resolve to the linked child IDs up front and - // route through the existing indexed ItemIds filter. - query.ItemIds = folder.LinkedChildren - .Where(lc => lc.ItemId.HasValue && !lc.ItemId.Value.IsEmpty()) - .Select(lc => lc.ItemId!.Value) - .ToArray(); - - // Empty linked-children should still return empty rather than scanning everything. - if (query.ItemIds.Length == 0) - { - query.ItemIds = [Guid.NewGuid()]; - } + // would return zero rows. Filter by the descendant set instead, which follows + // the links and keeps descending, so a linked folder contributes what is below + // it as well - the episodes of a Series added to a collection, for example. + query.DescendantOfId = folder.Id; } else { @@ -2382,6 +2452,9 @@ namespace Emby.Server.Implementations.Library var parentCollectionType = parent is not null ? GetTopFolderContentType(parent) : null; foreach (var item in items) { + // Parental controls filter on the stored score, so it has to match the ratings being saved. + item.OnMetadataChanged(); + if (item is Video video && video.LocalAlternateVersions.Length > 0) { var videoType = video.GetType(); @@ -2588,6 +2661,9 @@ namespace Emby.Server.Implementations.Library { foreach (var item in items) { + // Parental controls filter on the stored score, so it has to match the ratings being saved. + item.OnMetadataChanged(); + item.DateLastSaved = DateTime.UtcNow; await RunMetadataSavers(item, updateReason).ConfigureAwait(false); @@ -2982,14 +3058,7 @@ namespace Emby.Server.Implementations.Library CollectionType? viewType, string sortName) { - var parentIdString = parentId.IsEmpty() - ? null - : parentId.ToString("N", CultureInfo.InvariantCulture); - - // The name is either localized (grouped views) or the library folder's own name. - var idValues = "38_namedview_" + user.Id.ToString("N", CultureInfo.InvariantCulture) + (parentIdString ?? string.Empty) + (viewType?.ToString() ?? string.Empty); - - var id = GetNewItemId(idValues, typeof(UserView)); + var id = GetNamedViewId(user, parentId, viewType); var path = Path.Combine(_configurationManager.ApplicationPaths.InternalMetadataPath, "views", id.ToString("N", CultureInfo.InvariantCulture)); @@ -3047,6 +3116,18 @@ namespace Emby.Server.Implementations.Library return item; } + public Guid GetNamedViewId(User user, Guid parentId, CollectionType? viewType) + { + var parentIdString = parentId.IsEmpty() + ? null + : parentId.ToString("N", CultureInfo.InvariantCulture); + + // The name is either localized (grouped views) or the library folder's own name. + var idValues = "38_namedview_" + user.Id.ToString("N", CultureInfo.InvariantCulture) + (parentIdString ?? string.Empty) + (viewType?.ToString() ?? string.Empty); + + return GetNewItemId(idValues, typeof(UserView)); + } + public UserView GetShadowView( BaseItem parent, CollectionType? viewType, @@ -3056,10 +3137,7 @@ namespace Emby.Server.Implementations.Library var name = parent.Name; var parentId = parent.Id; - - var idValues = "38_namedview_" + name + parentId + (viewType?.ToString() ?? string.Empty); - - var id = GetNewItemId(idValues, typeof(UserView)); + var id = GetShadowViewId(name, parentId, viewType); var path = parent.Path; @@ -3111,6 +3189,13 @@ namespace Emby.Server.Implementations.Library return item; } + public Guid GetShadowViewId(string name, Guid parentId, CollectionType? viewType) + { + var idValues = "38_namedview_" + name + parentId + (viewType?.ToString() ?? string.Empty); + + return GetNewItemId(idValues, typeof(UserView)); + } + public UserView GetNamedView( string name, Guid parentId, @@ -3396,6 +3481,7 @@ namespace Emby.Server.Implementations.Library var extras = new List<BaseItem>(); var typeCounters = new Dictionary<ExtraType, int>(); + var generatedNames = new Dictionary<ExtraType, HashSet<string>>(); // Order by path so that the numbering handed out below does not depend on the // order the file system happened to list the folder in @@ -3432,10 +3518,12 @@ namespace Emby.Server.Implementations.Library extra = itemById; } - // An extra is named after its file, so the file is the source of truth. Items created - // by older versions, or renamed by a metadata provider, are corrected here; - // RefreshExtras persists the change. - if (!string.IsNullOrEmpty(name) && extra.LockedFields?.Contains(MetadataField.Name) != true) + // The name derived from the file is only a default. A name that came from anywhere else, + // such as a local metadata file, is the user's and has to survive the scan, so only a + // name this method handed out itself is renewed; RefreshExtras persists the change. + if (!string.IsNullOrEmpty(name) + && extra.LockedFields?.Contains(MetadataField.Name) != true + && (itemById is null || IsGeneratedExtraName(extra.Name, candidate))) { extra.Name = name; } @@ -3457,6 +3545,31 @@ namespace Emby.Server.Implementations.Library return null; } + + bool IsGeneratedExtraName(string currentName, ExtraCandidate candidate) + { + // The file name is what an extra was called before it was given a name of its type + if (string.Equals(currentName, candidate.Extra.Name, StringComparison.Ordinal)) + { + return true; + } + + if (!generatedNames.TryGetValue(candidate.ExtraType, out var names)) + { + // Any of the numbers of this type may have been handed out, as the order the extras + // of a type are numbered in shifts as files appear beside them or are taken away + names = new HashSet<string>(StringComparer.Ordinal); + var count = candidates.Count(c => c.ExtraType == candidate.ExtraType); + for (var seen = 0; seen < count; seen++) + { + names.Add(GetNumberedExtraName(candidate.ExtraType, seen)); + } + + generatedNames[candidate.ExtraType] = names; + } + + return names.Contains(currentName); + } } /// <summary> @@ -3483,7 +3596,18 @@ namespace Emby.Server.Implementations.Library typeCounters.TryGetValue(candidate.ExtraType, out var seen); typeCounters[candidate.ExtraType] = seen + 1; - var typeName = _localization.GetServerLocalizedString(GetExtraTypeNameKey(candidate.ExtraType)); + return GetNumberedExtraName(candidate.ExtraType, seen); + } + + /// <summary> + /// Gets the name given to the n-th extra of a type that is named after its type. + /// </summary> + /// <param name="extraType">The extra type.</param> + /// <param name="seen">Number of extras of the type named before this one.</param> + /// <returns>The name.</returns> + private string GetNumberedExtraName(ExtraType extraType, int seen) + { + var typeName = _localization.GetServerLocalizedString(GetExtraTypeNameKey(extraType)); return seen == 0 ? typeName @@ -3548,6 +3672,11 @@ namespace Emby.Server.Implementations.Library public QueryResult<BaseItem> GetPeopleItems(InternalPeopleQuery query) { + ArgumentNullException.ThrowIfNull(query); + + // This hands back by-name items, so the people without one are not ours to report. + query.MustHaveItem = true; + var queryResult = _peopleRepository.GetPeople(query); var baseItems = queryResult.Items.Select(i => { @@ -3728,6 +3857,10 @@ namespace Emby.Server.Implementations.Library AddMediaPathInternal(name, path, false); } } + + // The libraries root was listed before this folder existed, so drop that listing: + // anything still reading it resolves the library set without the new folder. + _directoryService.Invalidate(virtualFolderPath); } finally { @@ -3735,7 +3868,7 @@ namespace Emby.Server.Implementations.Library if (refreshLibrary) { - StartScanInBackground(); + _ = StartScanInBackground(); } else { @@ -3754,27 +3887,14 @@ namespace Emby.Server.Implementations.Library var itemUpdateType = ItemUpdateType.MetadataDownload; var saveEntity = false; - var createEntity = false; var personEntity = GetPerson(person.Name); if (personEntity is null) { try { - var path = Person.GetPath(person.Name); - var info = Directory.CreateDirectory(path); - personEntity = new Person() - { - Name = person.Name, - Id = GetItemByNameId<Person>(path), - DateCreated = info.CreationTimeUtc, - DateModified = info.LastWriteTimeUtc, - Path = path - }; - - personEntity.PresentationUniqueKey = personEntity.CreatePresentationUniqueKey(); + personEntity = GetOrCreatePerson(person.Name); saveEntity = true; - createEntity = true; } catch (Exception ex) { @@ -3808,11 +3928,6 @@ namespace Emby.Server.Implementations.Library if (saveEntity) { - if (createEntity) - { - CreateItems([personEntity], null, CancellationToken.None); - } - await RunMetadataSavers(personEntity, itemUpdateType).ConfigureAwait(false); personEntity.DateLastSaved = DateTime.UtcNow; @@ -3821,13 +3936,16 @@ namespace Emby.Server.Implementations.Library } } - private void StartScanInBackground() + internal Task StartScanInBackground() { - Task.Run(() => + // An active scan already handles library structure changes, so this request can be dropped. + if (IsScanRunning) { - // No need to start if scanning the library because it will handle it - ValidateMediaLibrary(new Progress<double>(), CancellationToken.None); - }); + return Task.CompletedTask; + } + + // Queue instead of restarting so a scan that starts after the check is allowed to finish. + return Task.Run(QueueLibraryScan); } public void AddMediaPath(string virtualFolderName, MediaPathInfo mediaPath) @@ -3852,7 +3970,9 @@ namespace Emby.Server.Implementations.Library } var rootFolderPath = _configurationManager.ApplicationPaths.DefaultUserViewsPath; - var virtualFolderPath = Path.Combine(rootFolderPath, virtualFolderName); + var virtualFolderPath = FileSystemHelper.GetChildPath(rootFolderPath, virtualFolderName) + ?? throw new FileNotFoundException( + string.Format(CultureInfo.InvariantCulture, "The media collection {0} does not exist", virtualFolderName)); CreateShortcut(virtualFolderPath, pathInfo); @@ -3873,7 +3993,9 @@ namespace Emby.Server.Implementations.Library ArgumentNullException.ThrowIfNull(mediaPath); var rootFolderPath = _configurationManager.ApplicationPaths.DefaultUserViewsPath; - var virtualFolderPath = Path.Combine(rootFolderPath, virtualFolderName); + var virtualFolderPath = FileSystemHelper.GetChildPath(rootFolderPath, virtualFolderName) + ?? throw new FileNotFoundException( + string.Format(CultureInfo.InvariantCulture, "The media collection {0} does not exist", virtualFolderName)); var libraryOptions = CollectionFolder.GetLibraryOptions(virtualFolderPath); @@ -3912,9 +4034,9 @@ namespace Emby.Server.Implementations.Library var rootFolderPath = _configurationManager.ApplicationPaths.DefaultUserViewsPath; - var path = Path.Combine(rootFolderPath, name); + var path = FileSystemHelper.GetChildPath(rootFolderPath, name); - if (!Directory.Exists(path)) + if (path is null || !Directory.Exists(path)) { throw new FileNotFoundException("The media folder does not exist"); } @@ -3924,6 +4046,7 @@ namespace Emby.Server.Implementations.Library try { Directory.Delete(path, true); + _directoryService.Invalidate(path); } finally { @@ -3933,7 +4056,7 @@ namespace Emby.Server.Implementations.Library { await ValidateTopLibraryFolders(CancellationToken.None, true).ConfigureAwait(false); - StartScanInBackground(); + _ = StartScanInBackground(); } else { @@ -3978,9 +4101,9 @@ namespace Emby.Server.Implementations.Library ArgumentException.ThrowIfNullOrEmpty(mediaPath); var rootFolderPath = _configurationManager.ApplicationPaths.DefaultUserViewsPath; - var virtualFolderPath = Path.Combine(rootFolderPath, virtualFolderName); + var virtualFolderPath = FileSystemHelper.GetChildPath(rootFolderPath, virtualFolderName); - if (!Directory.Exists(virtualFolderPath)) + if (virtualFolderPath is null || !Directory.Exists(virtualFolderPath)) { throw new FileNotFoundException( string.Format(CultureInfo.InvariantCulture, "The media collection {0} does not exist", virtualFolderName)); @@ -3993,6 +4116,7 @@ namespace Emby.Server.Implementations.Library if (!string.IsNullOrEmpty(shortcut)) { _fileSystem.DeleteFile(shortcut); + _directoryService.Invalidate(shortcut); } var libraryOptions = CollectionFolder.GetLibraryOptions(virtualFolderPath); @@ -4036,6 +4160,7 @@ namespace Emby.Server.Implementations.Library } _fileSystem.CreateShortcut(lnk, _appHost.ReverseVirtualPath(path)); + _directoryService.Invalidate(lnk); RemoveContentTypeOverrides(path); } @@ -4075,6 +4200,18 @@ namespace Emby.Server.Implementations.Library } /// <inheritdoc /> + public IReadOnlyList<string> GetTagNames(InternalItemsQuery query) + { + if (query.User is not null) + { + AddUserToQuery(query, query.User); + } + + SetTopParentOrAncestorIds(query); + return _itemRepository.GetTagNames(query); + } + + /// <inheritdoc /> public IReadOnlyList<string> GetMediaStreamLanguages(MediaStreamType mediaStreamType) { return _mediaStreamRepository.GetMediaStreamLanguages(mediaStreamType); diff --git a/Emby.Server.Implementations/Library/MediaSourceManager.cs b/Emby.Server.Implementations/Library/MediaSourceManager.cs index 97e00177b6..e9bba05839 100644 --- a/Emby.Server.Implementations/Library/MediaSourceManager.cs +++ b/Emby.Server.Implementations/Library/MediaSourceManager.cs @@ -384,7 +384,13 @@ namespace Emby.Server.Implementations.Library { ArgumentNullException.ThrowIfNull(item); - var hasMediaSources = (IHasMediaSources)item; + // Clients can ask for the sources of an item that has none (a container queued by mistake). + if (item is not IHasMediaSources hasMediaSources) + { + throw new ArgumentException( + string.Format(CultureInfo.InvariantCulture, "{0} {1} has no media sources and cannot be played.", item.GetType().Name, item.Id), + nameof(item)); + } var sources = hasMediaSources.GetMediaSources(enablePathSubstitution); @@ -494,7 +500,12 @@ namespace Emby.Server.Implementations.Library { var index = userData.SubtitleStreamIndex.Value; // Make sure the saved index is still valid - if (index == -1 || source.MediaStreams.Any(i => i.Type == MediaStreamType.Subtitle && i.Index == index)) + var savedStream = source.MediaStreams.FirstOrDefault(i => i.Type == MediaStreamType.Subtitle && i.Index == index); + // "Only forced" rules out full tracks entirely, so a remembered one must not resurrect them. + // The client reports whatever is playing, so an index remembered under another mode sticks forever otherwise. + if (index == -1 + || (savedStream is not null + && (user.SubtitleMode != SubtitlePlaybackMode.OnlyForced || savedStream.IsForced))) { source.DefaultSubtitleStreamIndex = index; return; diff --git a/Emby.Server.Implementations/Library/Resolvers/TV/SeasonResolver.cs b/Emby.Server.Implementations/Library/Resolvers/TV/SeasonResolver.cs index 6624d0125f..a8bd832cc8 100644 --- a/Emby.Server.Implementations/Library/Resolvers/TV/SeasonResolver.cs +++ b/Emby.Server.Implementations/Library/Resolvers/TV/SeasonResolver.cs @@ -129,6 +129,17 @@ namespace Emby.Server.Implementations.Library.Resolvers.TV var tmdbId = justName.GetAttributeValue("tmdbid"); item.TrySetProviderId(MetadataProvider.Tmdb, tmdbId); + + // Anime databases model a single cour as its own entry, so a multi-season + // series maps to one of these ids per season rather than one per series. + var anidbId = justName.GetAttributeValue("anidbid"); + item.TrySetProviderId("AniDB", anidbId); + + var aniListId = justName.GetAttributeValue("anilistid"); + item.TrySetProviderId("AniList", aniListId); + + var aniSearchId = justName.GetAttributeValue("anisearchid"); + item.TrySetProviderId("AniSearch", aniSearchId); } } } diff --git a/Emby.Server.Implementations/Library/Search/SearchManager.cs b/Emby.Server.Implementations/Library/Search/SearchManager.cs index 0e180753a6..a8ee416b31 100644 --- a/Emby.Server.Implementations/Library/Search/SearchManager.cs +++ b/Emby.Server.Implementations/Library/Search/SearchManager.cs @@ -112,13 +112,12 @@ public class SearchManager : ISearchManager return externalResults; } - var internalResults = await internalTask.ConfigureAwait(false); if (_internalProviders.Length > 0) { _logger.LogDebug("No results from external providers, using internal provider results"); } - return internalResults; + return await internalTask.ConfigureAwait(false); } private async Task<IReadOnlyList<SearchResult>> FilterByUserAccessAsync( @@ -144,17 +143,24 @@ public class SearchManager : ISearchManager baseQuery = _queryHelpers.ApplyAccessFiltering(dbContext, baseQuery, accessFilter); - var allowedCount = await baseQuery.CountAsync(cancellationToken).ConfigureAwait(false); - if (allowedCount == candidates.Count) + var allowed = await baseQuery + .Select(e => new { e.Id, e.PrimaryVersionId }) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + var allowedIds = allowed.Select(e => e.Id).ToHashSet(); + + // A provider can return both an alternate version and the primary it belongs to, and the + // two are one item to the user. + allowedIds.ExceptWith(allowed + .Where(e => e.PrimaryVersionId.HasValue && allowedIds.Contains(e.PrimaryVersionId.Value)) + .Select(e => e.Id)); + + if (allowedIds.Count == candidates.Count) { return candidates; } - var allowedIds = await baseQuery - .Select(e => e.Id) - .ToHashSetAsync(cancellationToken) - .ConfigureAwait(false); - var filtered = candidates.Where(c => allowedIds.Contains(c.ItemId)).ToList(); if (filtered.Count < candidates.Count) { diff --git a/Emby.Server.Implementations/Library/Search/SqlSearchProvider.cs b/Emby.Server.Implementations/Library/Search/SqlSearchProvider.cs index c4d3b249d5..a5ea02df51 100644 --- a/Emby.Server.Implementations/Library/Search/SqlSearchProvider.cs +++ b/Emby.Server.Implementations/Library/Search/SqlSearchProvider.cs @@ -10,6 +10,7 @@ using Jellyfin.Data.Enums; using Jellyfin.Database.Implementations; using Jellyfin.Database.Implementations.Entities; using Jellyfin.Extensions; +using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.Persistence; @@ -36,6 +37,7 @@ public class SqlSearchProvider : IInternalSearchProvider private readonly ILibraryManager _libraryManager; private readonly IUserManager _userManager; private readonly IItemQueryHelpers _queryHelpers; + private readonly IServerConfigurationManager _configurationManager; /// <summary> /// Initializes a new instance of the <see cref="SqlSearchProvider"/> class. @@ -45,18 +47,21 @@ public class SqlSearchProvider : IInternalSearchProvider /// <param name="libraryManager">The library manager.</param> /// <param name="userManager">The user manager.</param> /// <param name="queryHelpers">The shared item query helpers.</param> + /// <param name="configurationManager">The configuration manager.</param> public SqlSearchProvider( IDbContextFactory<JellyfinDbContext> dbProvider, IItemTypeLookup itemTypeLookup, ILibraryManager libraryManager, IUserManager userManager, - IItemQueryHelpers queryHelpers) + IItemQueryHelpers queryHelpers, + IServerConfigurationManager configurationManager) { _dbProvider = dbProvider; _itemTypeLookup = itemTypeLookup; _libraryManager = libraryManager; _userManager = userManager; _queryHelpers = queryHelpers; + _configurationManager = configurationManager; } /// <inheritdoc/> @@ -98,6 +103,12 @@ public class SqlSearchProvider : IInternalSearchProvider // so match it via a case-insensitive LIKE rather than a per-row case conversion // that may not translate to SQL on every provider. var likeOriginal = $"%{rawSearchTerm}%"; + + // Great benefit for users with metadata in non-latin script + // since SortName is among other things stored transliterated. + var sortNameShape = BaseItem.GetSortName(rawSearchTerm, true, _configurationManager.Configuration); + var likeSortName = string.IsNullOrWhiteSpace(sortNameShape) ? null : $"%{sortNameShape}%"; + var limit = query.Limit ?? DefaultSearchLimit; var dbContext = await _dbProvider.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); @@ -107,14 +118,27 @@ public class SqlSearchProvider : IInternalSearchProvider var dbQuery = dbContext.BaseItems .AsNoTracking() .Where(e => e.Id != _placeholderId) - .Where(e => !e.IsVirtualItem) - .Where(e => e.CleanName!.Contains(cleanSearchTerm) - || (e.OriginalTitle != null && EF.Functions.Like(e.OriginalTitle, likeOriginal))); + .Where(e => !e.IsVirtualItem); + + if (likeSortName is null) + { + dbQuery = dbQuery + .Where(e => e.CleanName!.Contains(cleanSearchTerm) + || (e.OriginalTitle != null && EF.Functions.Like(e.OriginalTitle, likeOriginal))); + } + else + { + dbQuery = dbQuery + .Where(e => e.CleanName!.Contains(cleanSearchTerm) + || (e.OriginalTitle != null && EF.Functions.Like(e.OriginalTitle, likeOriginal)) + || (e.SortName != null && EF.Functions.Like(e.SortName, likeSortName))); + } dbQuery = ApplyTypeFilter(dbQuery, query.IncludeItemTypes, query.ExcludeItemTypes); dbQuery = ApplyMediaTypeFilter(dbQuery, query.MediaTypes); dbQuery = ApplyParentFilter(dbQuery, query.ParentId); dbQuery = ApplyUserAccessFilter(dbContext, dbQuery, query); + dbQuery = ExcludeVersionsOfMatchedPrimaries(dbQuery); // Compute the score in SQL: the ternary translates to a CASE WHEN. CleanName is // the pre-normalized (lowercase, diacritic-stripped) form, so we score against it @@ -193,6 +217,12 @@ public class SqlSearchProvider : IInternalSearchProvider return query.Where(e => e.ParentId == pid || e.Parents!.Any(p => p.ParentItemId == pid)); } + private static IQueryable<BaseItemEntity> ExcludeVersionsOfMatchedPrimaries(IQueryable<BaseItemEntity> query) + { + var matched = query; + return query.Where(e => e.PrimaryVersionId == null || !matched.Any(p => p.Id == e.PrimaryVersionId)); + } + private IQueryable<BaseItemEntity> ApplyUserAccessFilter( JellyfinDbContext dbContext, IQueryable<BaseItemEntity> query, diff --git a/Emby.Server.Implementations/Library/SimilarItems/MovieSimilarItemsProvider.cs b/Emby.Server.Implementations/Library/SimilarItems/MovieSimilarItemsProvider.cs index 57d1f7c770..cc8f0fd24e 100644 --- a/Emby.Server.Implementations/Library/SimilarItems/MovieSimilarItemsProvider.cs +++ b/Emby.Server.Implementations/Library/SimilarItems/MovieSimilarItemsProvider.cs @@ -1,3 +1,5 @@ +#pragma warning disable RS0030 // Do not use banned APIs: Guid == is required inside EF expression trees. + using System; using System.Collections.Generic; using System.Linq; @@ -172,7 +174,7 @@ public sealed class MovieSimilarItemsProvider : ILocalSimilarItemsProvider<Movie var allCandidateIdsList = allCandidateIds.ToList(); var accessibleItems = await baseQuery .WhereOneOrMany(allCandidateIdsList, e => e.Id) - .Select(e => new { e.Id, e.PresentationUniqueKey }) + .Select(e => new { e.Id, e.PresentationUniqueKey, e.PrimaryVersionId }) .ToListAsync(cancellationToken).ConfigureAwait(false); // Phase 3: Pick top IDs per source, dedup by PresentationUniqueKey @@ -189,6 +191,9 @@ public sealed class MovieSimilarItemsProvider : ILocalSimilarItemsProvider<Movie var orderedIds = accessibleItems .Where(x => scores.ContainsKey(x.Id)) .OrderByDescending(x => scores.GetValueOrDefault(x.Id)) + // Two versions of one movie score the same, so name the primary as the + // representative of the group rather than whichever came back first. + .ThenBy(x => x.PrimaryVersionId.HasValue) .DistinctBy(x => x.PresentationUniqueKey) .Take(limit) .Select(x => x.Id) @@ -245,6 +250,11 @@ public sealed class MovieSimilarItemsProvider : ILocalSimilarItemsProvider<Movie result[id] = []; } + var hiddenVersionIds = context.BaseItems.AsNoTracking() + .Where(e => e.PrimaryVersionId != null + && context.BaseItems.Any(p => p.Id == e.PrimaryVersionId && p.TopParentId == e.TopParentId)) + .Select(e => e.Id); + foreach (var (valueType, weight) in _itemValueDimensions) { var sourceRows = await context.ItemValuesMap.AsNoTracking() @@ -260,7 +270,7 @@ public sealed class MovieSimilarItemsProvider : ILocalSimilarItemsProvider<Movie } var candidateRows = await context.ItemValuesMap.AsNoTracking() - .Where(m => m.ItemValue.Type == valueType && allKeys.Contains(m.ItemValue.CleanValue)) + .Where(m => !hiddenVersionIds.Contains(m.ItemId) && m.ItemValue.Type == valueType && allKeys.Contains(m.ItemValue.CleanValue)) .Select(m => new { m.ItemId, Key = m.ItemValue.CleanValue }) .ToListAsync(cancellationToken).ConfigureAwait(false); @@ -276,6 +286,7 @@ public sealed class MovieSimilarItemsProvider : ILocalSimilarItemsProvider<Movie if (personSourceRows.Count > 0) { var personCandidateRows = await context.PeopleBaseItemMap.AsNoTracking() + .Where(m => !hiddenVersionIds.Contains(m.ItemId)) .Where(m => context.PeopleBaseItemMap .Where(s => sourceIds.Contains(s.ItemId) && _scoredPersonTypes.Contains(s.People.PersonType)) .Select(s => s.PeopleId) diff --git a/Emby.Server.Implementations/Library/SimilarItems/SimilarItemsAccessFilter.cs b/Emby.Server.Implementations/Library/SimilarItems/SimilarItemsAccessFilter.cs new file mode 100644 index 0000000000..75aea0eab6 --- /dev/null +++ b/Emby.Server.Implementations/Library/SimilarItems/SimilarItemsAccessFilter.cs @@ -0,0 +1,42 @@ +using Jellyfin.Data.Enums; +using Jellyfin.Database.Implementations.Entities; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Library; + +namespace Emby.Server.Implementations.Library.SimilarItems; + +/// <summary> +/// Builds the access filter that decides which items a similar-items lookup may return for a user. +/// </summary> +internal static class SimilarItemsAccessFilter +{ + private static readonly BaseItemKind[] _itemByNameKinds = + [ + BaseItemKind.Person, + BaseItemKind.Genre, + BaseItemKind.MusicGenre, + BaseItemKind.MusicArtist, + BaseItemKind.Studio + ]; + + /// <summary> + /// Builds an access filter carrying the user's library access and parental restrictions. + /// </summary> + /// <param name="user">The user the lookup runs for.</param> + /// <param name="libraryManager">The library manager.</param> + /// <returns>The access filter.</returns> + public static InternalItemsQuery Build(User user, ILibraryManager libraryManager) + { + // IncludeItemTypes is read only for the by-name exemption here; the caller applies this + // filter through ApplyAccessFiltering, which does not translate it into a type restriction. + var accessFilter = new InternalItemsQuery(user) + { + IncludeItemTypes = _itemByNameKinds + }; + + // ConfigureUserAccess populates TopParentIds for the libraries the user may open. + libraryManager.ConfigureUserAccess(accessFilter, user); + + return accessFilter; + } +} diff --git a/Emby.Server.Implementations/Library/SimilarItems/SimilarItemsManager.cs b/Emby.Server.Implementations/Library/SimilarItems/SimilarItemsManager.cs index 4e482c174a..a18a17b593 100644 --- a/Emby.Server.Implementations/Library/SimilarItems/SimilarItemsManager.cs +++ b/Emby.Server.Implementations/Library/SimilarItems/SimilarItemsManager.cs @@ -7,8 +7,10 @@ using System.Text.Json; using System.Threading; using System.Threading.Tasks; using Jellyfin.Data.Enums; +using Jellyfin.Database.Implementations; using Jellyfin.Database.Implementations.Entities; using Jellyfin.Database.Implementations.Enums; +using Jellyfin.Extensions; using Jellyfin.Extensions.Json; using MediaBrowser.Common.Extensions; using MediaBrowser.Controller; @@ -16,11 +18,13 @@ using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Dto; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; +using MediaBrowser.Controller.Persistence; using MediaBrowser.Model.Configuration; using MediaBrowser.Model.Dto; using MediaBrowser.Model.Entities; using MediaBrowser.Model.IO; using MediaBrowser.Model.Querying; +using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; namespace Emby.Server.Implementations.Library.SimilarItems; @@ -35,6 +39,8 @@ public class SimilarItemsManager : ISimilarItemsManager private readonly ILibraryManager _libraryManager; private readonly IFileSystem _fileSystem; private readonly IServerConfigurationManager _serverConfigurationManager; + private readonly IDbContextFactory<JellyfinDbContext> _dbProvider; + private readonly IItemQueryHelpers _queryHelpers; private ISimilarItemsProvider[] _similarItemsProviders = []; /// <summary> @@ -45,18 +51,24 @@ public class SimilarItemsManager : ISimilarItemsManager /// <param name="libraryManager">The library manager.</param> /// <param name="fileSystem">The file system.</param> /// <param name="serverConfigurationManager">The server configuration manager.</param> + /// <param name="dbProvider">The database context factory.</param> + /// <param name="queryHelpers">The shared item query helpers.</param> public SimilarItemsManager( ILogger<SimilarItemsManager> logger, IServerApplicationPaths appPaths, ILibraryManager libraryManager, IFileSystem fileSystem, - IServerConfigurationManager serverConfigurationManager) + IServerConfigurationManager serverConfigurationManager, + IDbContextFactory<JellyfinDbContext> dbProvider, + IItemQueryHelpers queryHelpers) { _logger = logger; _appPaths = appPaths; _libraryManager = libraryManager; _fileSystem = fileSystem; _serverConfigurationManager = serverConfigurationManager; + _dbProvider = dbProvider; + _queryHelpers = queryHelpers; } /// <inheritdoc/> @@ -230,11 +242,64 @@ public class SimilarItemsManager : ISimilarItemsManager } } - return allResults + var ordered = allResults .OrderByDescending(x => x.Score) .Select(x => x.Item) .Take(requestedLimit) .ToList(); + + return await FilterByLibraryAccessAsync(ordered, user, cancellationToken).ConfigureAwait(false); + } + + private async Task<IReadOnlyList<BaseItem>> FilterByLibraryAccessAsync( + IReadOnlyList<BaseItem> candidates, + User? user, + CancellationToken cancellationToken) + { + if (candidates.Count == 0 || user is null) + { + return candidates; + } + + var accessFilter = SimilarItemsAccessFilter.Build(user, _libraryManager); + + // No accessible libraries means nothing to compare against, and an empty TopParentIds set + // would disable the filter rather than reject everything. + if (accessFilter.TopParentIds.Length == 0) + { + return candidates; + } + + Guid[] candidateIds = [.. candidates.Select(c => c.Id)]; + + var dbContext = await _dbProvider.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (dbContext.ConfigureAwait(false)) + { + var baseQuery = dbContext.BaseItems + .AsNoTracking() + .WhereOneOrMany(candidateIds, e => e.Id); + + baseQuery = _queryHelpers.ApplyAccessFiltering(dbContext, baseQuery, accessFilter); + + var allowedCount = await baseQuery.CountAsync(cancellationToken).ConfigureAwait(false); + if (allowedCount == candidates.Count) + { + return candidates; + } + + var allowedIds = await baseQuery + .Select(e => e.Id) + .ToHashSetAsync(cancellationToken) + .ConfigureAwait(false); + + var filtered = candidates.Where(c => allowedIds.Contains(c.Id)).ToList(); + _logger.LogDebug( + "Dropped {Dropped} of {Total} similar-item candidates due to user access filtering", + candidates.Count - filtered.Count, + candidates.Count); + + return filtered; + } } /// <inheritdoc/> @@ -376,19 +441,39 @@ public class SimilarItemsManager : ISimilarItemsManager var batchResults = await batchProvider.GetBatchSimilarItemsAsync(baselineItems, query, cancellationToken).ConfigureAwait(false); + // Filter once across every category rather than per baseline, so a batch provider costs one + // access query no matter how many categories it produced. + var allItems = batchResults.Values.SelectMany(items => items).DistinctBy(item => item.Id).ToList(); + var allowed = await FilterByLibraryAccessAsync(allItems, query.User, cancellationToken).ConfigureAwait(false); + + HashSet<Guid>? allowedIds = allowed.Count == allItems.Count + ? null + : [.. allowed.Select(item => item.Id)]; + var recommendations = new List<SimilarItemsRecommendation>(baselineItems.Count); foreach (var baseline in baselineItems) { - if (batchResults.TryGetValue(baseline.Id, out var similar) && similar.Count > 0) + if (!batchResults.TryGetValue(baseline.Id, out var similar) || similar.Count == 0) + { + continue; + } + + if (allowedIds is not null) { - recommendations.Add(new SimilarItemsRecommendation + similar = similar.Where(item => allowedIds.Contains(item.Id)).ToList(); + if (similar.Count == 0) { - BaselineItemName = baseline.Name, - CategoryId = baseline.Id, - RecommendationType = recommendationType, - Items = similar - }); + continue; + } } + + recommendations.Add(new SimilarItemsRecommendation + { + BaselineItemName = baseline.Name, + CategoryId = baseline.Id, + RecommendationType = recommendationType, + Items = similar + }); } return recommendations; @@ -566,7 +651,13 @@ public class SimilarItemsManager : ISimilarItemsManager try { - var stream = File.OpenRead(cachePath); + var stream = new FileStream( + cachePath, + FileMode.Open, + FileAccess.Read, + FileShare.ReadWrite | FileShare.Delete, + IODefaults.FileStreamBufferSize, + FileOptions.Asynchronous | FileOptions.SequentialScan); await using (stream.ConfigureAwait(false)) { var cache = await JsonSerializer.DeserializeAsync<SimilarItemsCache>(stream, JsonDefaults.Options, cancellationToken).ConfigureAwait(false); @@ -590,6 +681,7 @@ public class SimilarItemsManager : ISimilarItemsManager private async Task SaveSimilarItemsCacheAsync(string cachePath, List<SimilarItemReference> references, TimeSpan cacheDuration, CancellationToken cancellationToken) { + string? tempPath = null; try { var directory = Path.GetDirectoryName(cachePath); @@ -604,16 +696,34 @@ public class SimilarItemsManager : ISimilarItemsManager ExpiresAt = DateTime.UtcNow.Add(cacheDuration) }; - var stream = File.Create(cachePath); + tempPath = cachePath + "." + Guid.NewGuid().ToString("N", CultureInfo.InvariantCulture) + ".tmp"; + var stream = File.Create(tempPath); await using (stream.ConfigureAwait(false)) { await JsonSerializer.SerializeAsync(stream, cache, JsonDefaults.Options, cancellationToken).ConfigureAwait(false); } + + File.Move(tempPath, cachePath, true); + tempPath = null; } catch (IOException ex) { _logger.LogWarning(ex, "Failed to save similar items cache to {CachePath}", cachePath); } + finally + { + if (tempPath is not null) + { + try + { + File.Delete(tempPath); + } + catch (IOException ex) + { + _logger.LogDebug(ex, "Failed to delete temporary similar items cache file {TempPath}", tempPath); + } + } + } } private sealed class SimilarItemsCache diff --git a/Emby.Server.Implementations/Library/UserDataManager.cs b/Emby.Server.Implementations/Library/UserDataManager.cs index 0680046c11..1e7068b24b 100644 --- a/Emby.Server.Implementations/Library/UserDataManager.cs +++ b/Emby.Server.Implementations/Library/UserDataManager.cs @@ -75,6 +75,22 @@ namespace Emby.Server.Implementations.Library } dbContext.SaveChanges(); + + // Rows under keys the item no longer reports are kept, not deleted: the key set is briefly + // incomplete mid-refresh, and those rows are what a deleted and re-added item reattaches by. + dbContext.UserData + .Where(e => e.ItemId == item.Id && e.UserId == user.Id && !keys.Contains(e.CustomDataKey)) + .ExecuteUpdate(e => e + .SetProperty(f => f.AudioStreamIndex, userData.AudioStreamIndex) + .SetProperty(f => f.IsFavorite, userData.IsFavorite) + .SetProperty(f => f.LastPlayedDate, userData.LastPlayedDate) + .SetProperty(f => f.Likes, userData.Likes) + .SetProperty(f => f.PlaybackPositionTicks, userData.PlaybackPositionTicks) + .SetProperty(f => f.PlayCount, userData.PlayCount) + .SetProperty(f => f.Played, userData.Played) + .SetProperty(f => f.Rating, userData.Rating) + .SetProperty(f => f.SubtitleStreamIndex, userData.SubtitleStreamIndex)); + transaction.Commit(); var userId = user.InternalId; diff --git a/Emby.Server.Implementations/Library/UserViewManager.cs b/Emby.Server.Implementations/Library/UserViewManager.cs index 47b3891901..6d52ce7440 100644 --- a/Emby.Server.Implementations/Library/UserViewManager.cs +++ b/Emby.Server.Implementations/Library/UserViewManager.cs @@ -62,12 +62,11 @@ namespace Emby.Server.Implementations.Library // Playlist and BoxSet libraries require special handling because the folder only references linked items if (folderViewType == CollectionType.playlists || folderViewType == CollectionType.boxsets) { - var items = folder.GetItemList(new InternalItemsQuery(user) - { - ParentId = folder.ParentId - }); + var itemKind = folderViewType == CollectionType.playlists + ? BaseItemKind.Playlist + : BaseItemKind.BoxSet; - if (!items.Any(item => item.IsVisible(user))) + if (!HasVisibleItem(itemKind, folders, user)) { continue; } @@ -127,7 +126,7 @@ namespace Emby.Server.Implementations.Library list.AddRange(channels); - if (_liveTvManager.GetEnabledUsers().Select(i => i.Id).Contains(user.Id)) + if (_liveTvManager.IsEnabledForUser(user)) { list.Add(_liveTvManager.GetInternalLiveTvFolder(CancellationToken.None)); } @@ -159,6 +158,32 @@ namespace Emby.Server.Implementations.Library .ToArray(); } + private bool HasVisibleItem(BaseItemKind itemKind, IReadOnlyList<Folder> folders, User user) + { + var topParentIds = folders.SelectMany(GetTopParentIds).ToArray(); + if (topParentIds.Length == 0) + { + return false; + } + + var items = _libraryManager.GetItemList(new InternalItemsQuery(user) + { + IncludeItemTypes = [itemKind], + TopParentIds = topParentIds, + GroupByPresentationUniqueKey = false, + DtoOptions = DtoOptions.StoredColumnsOnly + }); + + return items.Any(item => item.IsVisible(user)); + } + + private static IEnumerable<Guid> GetTopParentIds(Folder folder) + { + return folder is CollectionFolder collectionFolder && collectionFolder.PhysicalFolderIds.Length > 0 + ? collectionFolder.PhysicalFolderIds + : [folder.Id]; + } + public UserView GetUserSubViewWithName(string name, Guid parentId, CollectionType? type, string sortName) { var uniqueId = parentId + "subview" + type; diff --git a/Emby.Server.Implementations/Library/Validators/ArtistsValidator.cs b/Emby.Server.Implementations/Library/Validators/ArtistsValidator.cs index fa7112eb90..690466be70 100644 --- a/Emby.Server.Implementations/Library/Validators/ArtistsValidator.cs +++ b/Emby.Server.Implementations/Library/Validators/ArtistsValidator.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Globalization; using System.Linq; using System.Threading; @@ -61,6 +62,9 @@ public class ArtistsValidator var count = names.Count; var refreshed = 0; + var liveIds = new HashSet<Guid>(); + var unresolved = 0; + foreach (var name in names) { try @@ -73,13 +77,20 @@ public class ArtistsValidator // Fall back to GetArtist if not found (creates new item if needed) item ??= _libraryManager.GetArtist(name); - var isNew = !existingArtistIds.Contains(item.Id); - var neverRefreshed = item.DateLastRefreshed == default; - if (isNew || neverRefreshed) + // A name with no item is nothing to refresh, and nothing to keep alive either. + if (item is not null) { - await item.RefreshMetadata(cancellationToken).ConfigureAwait(false); - refreshed++; + liveIds.Add(item.Id); + + var isNew = !existingArtistIds.Contains(item.Id); + var neverRefreshed = item.DateLastRefreshed == default; + + if (isNew || neverRefreshed) + { + await item.RefreshMetadata(cancellationToken).ConfigureAwait(false); + refreshed++; + } } } catch (OperationCanceledException) @@ -88,6 +99,7 @@ public class ArtistsValidator } catch (Exception ex) { + unresolved++; _logger.LogError(ex, "Error refreshing {ArtistName}", name); } @@ -101,13 +113,26 @@ public class ArtistsValidator _logger.LogInformation("Refreshed metadata for {RefreshedCount} new artists out of {TotalCount} total", refreshed, count); + // Every name that threw is a name whose artist is missing from the live set, and deleting against + // a live set with holes in it deletes artists the library still refers to. Leave the sweep to a + // run that got a clean read of them. + if (unresolved > 0) + { + _logger.LogWarning( + "Not removing dead artists: {Count} of {TotalCount} names could not be resolved this run", + unresolved, + count); + + progress.Report(100); + return; + } + var deadEntities = _libraryManager.GetItemList(new InternalItemsQuery { IncludeItemTypes = [BaseItemKind.MusicArtist], - IsDeadArtist = true, IsLocked = false - }).Cast<MusicArtist>() - .Where(item => item.IsAccessedByName) + }).OfType<MusicArtist>() + .Where(item => item.IsAccessedByName && !liveIds.Contains(item.Id)) .ToList(); foreach (var item in deadEntities) diff --git a/Emby.Server.Implementations/Library/Validators/PeopleValidator.cs b/Emby.Server.Implementations/Library/Validators/PeopleValidator.cs index 078a0b921d..7d53f40ce7 100644 --- a/Emby.Server.Implementations/Library/Validators/PeopleValidator.cs +++ b/Emby.Server.Implementations/Library/Validators/PeopleValidator.cs @@ -1,12 +1,12 @@ using System; +using System.Collections.Generic; +using System.Globalization; using System.Linq; using System.Threading; using System.Threading.Tasks; using Jellyfin.Data.Enums; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; -using MediaBrowser.Controller.Providers; -using MediaBrowser.Model.IO; using Microsoft.Extensions.Logging; namespace Emby.Server.Implementations.Library.Validators; @@ -17,112 +17,143 @@ namespace Emby.Server.Implementations.Library.Validators; public class PeopleValidator { /// <summary> - /// The _library manager. + /// The library manager. /// </summary> private readonly ILibraryManager _libraryManager; /// <summary> - /// The _logger. + /// The logger. /// </summary> - private readonly ILogger _logger; - - private readonly IFileSystem _fileSystem; + private readonly ILogger<PeopleValidator> _logger; /// <summary> /// Initializes a new instance of the <see cref="PeopleValidator" /> class. /// </summary> /// <param name="libraryManager">The library manager.</param> /// <param name="logger">The logger.</param> - /// <param name="fileSystem">The file system.</param> - public PeopleValidator(ILibraryManager libraryManager, ILogger logger, IFileSystem fileSystem) + public PeopleValidator(ILibraryManager libraryManager, ILogger<PeopleValidator> logger) { _libraryManager = libraryManager; _logger = logger; - _fileSystem = fileSystem; } /// <summary> /// Validates the people. /// </summary> - /// <param name="cancellationToken">The cancellation token.</param> /// <param name="progress">The progress.</param> + /// <param name="cancellationToken">The cancellation token.</param> /// <returns>Task.</returns> - public async Task ValidatePeople(CancellationToken cancellationToken, IProgress<double> progress) + public async Task Run(IProgress<double> progress, CancellationToken cancellationToken) { // Before the refresh below walks them: a credit no item maps to any more stands for nothing, // and while it is there the person it names cannot reach the dead-person sweep either. var numOrphaned = _libraryManager.DeleteOrphanedCredits(); if (numOrphaned > 0) { - _logger.LogDebug("Deleted {Amount} credits no item maps to", numOrphaned); + _logger.LogInformation("Deleted {Amount} credits no item maps to", numOrphaned); } - var people = _libraryManager.GetPeopleNames(new InternalPeopleQuery()); - - var numComplete = 0; - - var numPeople = people.Count; + var names = _libraryManager.GetPeopleNames(new InternalPeopleQuery()); + var existingPersonIds = _libraryManager.GetItemIds(new InternalItemsQuery + { + IncludeItemTypes = [BaseItemKind.Person] + }).ToHashSet(); - IProgress<double> subProgress = new Progress<double>((val) => progress.Report(val / 2)); + var (newNames, deadIds) = PartitionCreditsByPersonId(names, _libraryManager.GetPersonId, existingPersonIds); - _logger.LogDebug("Will refresh {Amount} people", numPeople); + var numComplete = 0; + var count = names.Count; + var refreshed = 0; - foreach (var person in people) + foreach (var name in names) { cancellationToken.ThrowIfCancellationRequested(); try { - var item = _libraryManager.GetPerson(person); - if (item is null) - { - _logger.LogWarning("Failed to get person: {Name}", person); - continue; - } + var item = _libraryManager.GetOrCreatePerson(name); + var isNew = !existingPersonIds.Contains(item.Id); + var neverRefreshed = item.DateLastRefreshed == default; - var options = new MetadataRefreshOptions(new DirectoryService(_fileSystem)) + if (isNew || neverRefreshed) { - ImageRefreshMode = MetadataRefreshMode.ValidationOnly, - MetadataRefreshMode = MetadataRefreshMode.ValidationOnly - }; - - await item.RefreshMetadata(options, cancellationToken).ConfigureAwait(false); + await item.RefreshMetadata(cancellationToken).ConfigureAwait(false); + refreshed++; + } } catch (OperationCanceledException) { + // Don't clutter the log throw; } catch (Exception ex) { - _logger.LogError(ex, "Error validating IBN entry {Person}", person); + _logger.LogError(ex, "Error refreshing {PersonName}", name); } - // Update progress numComplete++; double percent = numComplete; - percent /= numPeople; + percent /= count; + percent *= 100; - subProgress.Report(100 * percent); + progress.Report(percent); } - var deadEntities = _libraryManager.GetItemList(new InternalItemsQuery - { - IncludeItemTypes = [BaseItemKind.Person], - IsDeadPerson = true, - IsLocked = false - }); + _logger.LogInformation( + "Refreshed metadata for {RefreshedCount} people out of {TotalCount} total, {NewCount} of which had no item yet", + refreshed, + count, + newNames.Count); - subProgress = new Progress<double>((val) => progress.Report((val / 2) + 50)); + // A person somebody locked is theirs, not ours, however little the library still credits them. + var deadEntities = deadIds + .Select(_libraryManager.GetItemById) + .OfType<Person>() + .Where(item => !item.IsLocked) + .ToList(); - var i = 0; - foreach (var item in deadEntities.Chunk(500)) + foreach (var item in deadEntities) { - _libraryManager.DeleteItemsUnsafeFast(item, true); - subProgress.Report(100f / deadEntities.Count * (i++ * 100)); + _logger.LogInformation("Deleting dead {ItemType} {ItemId} {ItemName}", item.GetType().Name, item.Id.ToString("N", CultureInfo.InvariantCulture), item.Name); } + _libraryManager.DeleteItemsUnsafeFast(deadEntities, deleteSourceFiles: true); + progress.Report(100); + } + + /// <summary> + /// Splits the person items into the ones a credit still calls for and the ones nothing does. + /// </summary> + /// <param name="creditNames">Every name credited on an item, from the people table.</param> + /// <param name="getPersonId">Maps a credit name to the id its person item has.</param> + /// <param name="existingPersonIds">The ids of the person items that exist.</param> + /// <returns>The credits needing an item, and the ids of the items nothing credits.</returns> + internal static (List<string> NewNames, List<Guid> DeadIds) PartitionCreditsByPersonId( + IReadOnlyList<string> creditNames, + Func<string, Guid> getPersonId, + IReadOnlySet<Guid> existingPersonIds) + { + ArgumentNullException.ThrowIfNull(creditNames); + ArgumentNullException.ThrowIfNull(getPersonId); + ArgumentNullException.ThrowIfNull(existingPersonIds); + + var newNames = new List<string>(); + var liveIds = new HashSet<Guid>(); + + foreach (var name in creditNames) + { + var personId = getPersonId(name); + + // Distinct credit names can normalize onto one id; only the first of them needs an item. + if (liveIds.Add(personId) && !existingPersonIds.Contains(personId)) + { + newNames.Add(name); + } + } + + var deadIds = existingPersonIds.Where(id => !liveIds.Contains(id)).ToList(); - _logger.LogInformation("People validation complete, deleted {Orphaned} orphaned credits", numOrphaned); + return (newNames, deadIds); } } |