diff options
8 files changed, 213 insertions, 40 deletions
diff --git a/Emby.Server.Implementations/Dto/DtoService.cs b/Emby.Server.Implementations/Dto/DtoService.cs index 6fa057702c..2462a754ae 100644 --- a/Emby.Server.Implementations/Dto/DtoService.cs +++ b/Emby.Server.Implementations/Dto/DtoService.cs @@ -611,7 +611,11 @@ namespace Emby.Server.Implementations.Dto // For these types we can try to optimize and assume these values will be equal if (item is MusicAlbum || item is Season || item is Playlist) { - dto.ChildCount = dto.RecursiveItemCount; + if (dto.RecursiveItemCount > 0) + { + dto.ChildCount = dto.RecursiveItemCount; + } + var folderChildCount = folder.LinkedChildren.Length; // The default is an empty array, so we can't reliably use the count when it's empty if (folderChildCount > 0) diff --git a/Emby.Server.Implementations/Localization/Core/be.json b/Emby.Server.Implementations/Localization/Core/be.json index 5d0ef65842..49ebc45f06 100644 --- a/Emby.Server.Implementations/Localization/Core/be.json +++ b/Emby.Server.Implementations/Localization/Core/be.json @@ -106,5 +106,11 @@ "TaskExtractMediaSegments": "Сканіраванне медыя-сегмента", "TaskMoveTrickplayImages": "Перанесці месцазнаходжанне выявы Trickplay", "CleanupUserDataTask": "Задача па ачыстцы даных карыстальніка", - "CleanupUserDataTaskDescription": "Ачышчае ўсе даныя карыстальніка (стан прагляду, абранае і г.д.) для медыяфайлаў, што адсутнічаюць больш за 90 дзён." + "CleanupUserDataTaskDescription": "Ачышчае ўсе даныя карыстальніка (стан прагляду, абранае і г.д.) для медыяфайлаў, што адсутнічаюць больш за 90 дзён.", + "LyricDownloadFailureFromForItem": "Не ўдалося загрузіць тэкст песні з {0} для {1}", + "NameExtraDeletedScene": "Выдаленая сцэна", + "NameExtraInterview": "Інтэрв'ю", + "NameExtraNumbered": "{0} {1}", + "NameExtraScene": "Сцэна", + "NameExtraTrailer": "Трэйлер" } diff --git a/Emby.Server.Implementations/Localization/Core/en-GB.json b/Emby.Server.Implementations/Localization/Core/en-GB.json index 1f69fc1f55..a053fc2da9 100644 --- a/Emby.Server.Implementations/Localization/Core/en-GB.json +++ b/Emby.Server.Implementations/Localization/Core/en-GB.json @@ -120,5 +120,6 @@ "NameExtraThemeSong": "Theme Song", "NameExtraThemeVideo": "Theme Video", "NameExtraTrailer": "Trailer", - "NameExtraUnknown": "Extra" + "NameExtraUnknown": "Extra", + "NameExtraNumbered": "{0} {1}" } diff --git a/Emby.Server.Implementations/ScheduledTasks/Tasks/PeopleValidationTask.cs b/Emby.Server.Implementations/ScheduledTasks/Tasks/PeopleValidationTask.cs index bd73f63aa7..afb27ddf9e 100644 --- a/Emby.Server.Implementations/ScheduledTasks/Tasks/PeopleValidationTask.cs +++ b/Emby.Server.Implementations/ScheduledTasks/Tasks/PeopleValidationTask.cs @@ -177,56 +177,51 @@ public class PeopleValidationTask : IScheduledTask, IConfigurableScheduledTask var thirtyDaysAgo = DateTime.UtcNow.AddDays(-30); var personTypeName = _itemTypeLookup.BaseItemKindNames[BaseItemKind.Person]; + List<Guid> peopleIds; + var context = await _dbContextFactory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); await using (context.ConfigureAwait(false)) { - const int PartitionSize = 100; - - var numPeople = await context.BaseItems + // Read the candidates in one go rather than paging them. A refresh stamps the person and takes + // it out of this set, so a growing offset over a shrinking set walks past people it never visits. + peopleIds = await context.BaseItems .AsNoTracking() .Where(b => b.Type == personTypeName) .Where(b => b.DateLastRefreshed == null || b.DateLastRefreshed < thirtyDaysAgo) .Where(b => !b.Images!.Any(i => i.ImageType == ImageInfoImageType.Primary) || string.IsNullOrEmpty(b.Overview)) - .CountAsync(cancellationToken) + .OrderBy(b => b.Id) + .Select(b => b.Id) + .ToListAsync(cancellationToken) .ConfigureAwait(false); + } - _logger.LogDebug("Found {Count} people needing image/overview refresh", numPeople); + _logger.LogDebug("Found {Count} people needing image/overview refresh", peopleIds.Count); - if (numPeople == 0) - { - progress.Report(100); - return; - } + if (peopleIds.Count == 0) + { + progress.Report(100); + return; + } - var numComplete = 0; - var numRefreshed = 0; + var numComplete = 0; + var numRefreshed = 0; - await foreach (var entry in context.BaseItems - .AsNoTracking() - .Where(b => b.Type == personTypeName) - .Where(b => b.DateLastRefreshed == null || b.DateLastRefreshed < thirtyDaysAgo) - .Where(b => - !b.Images!.Any(i => i.ImageType == ImageInfoImageType.Primary) || - string.IsNullOrEmpty(b.Overview)) - .OrderBy(b => b.Id) - .WithPartitionProgress(partition => _logger.LogDebug("Processing people partition {Partition}", partition)) - .PartitionEagerAsync(PartitionSize, cancellationToken) - .WithCancellation(cancellationToken) - .ConfigureAwait(false)) - { - if (await RefreshPersonAsync(entry.Id, cancellationToken).ConfigureAwait(false)) - { - numRefreshed++; - } + foreach (var personId in peopleIds) + { + cancellationToken.ThrowIfCancellationRequested(); - numComplete++; - progress.Report(100.0 * numComplete / numPeople); + if (await RefreshPersonAsync(personId, cancellationToken).ConfigureAwait(false)) + { + numRefreshed++; } - _logger.LogInformation("Refreshed metadata for {Count} people missing images or overview", numRefreshed); + numComplete++; + progress.Report(100.0 * numComplete / peopleIds.Count); } + + _logger.LogInformation("Refreshed metadata for {Count} people missing images or overview", numRefreshed); } private async Task<bool> RefreshPersonAsync(Guid personId, CancellationToken cancellationToken) diff --git a/MediaBrowser.Providers/Manager/MetadataService.cs b/MediaBrowser.Providers/Manager/MetadataService.cs index 26dc8f9930..fe5285bf65 100644 --- a/MediaBrowser.Providers/Manager/MetadataService.cs +++ b/MediaBrowser.Providers/Manager/MetadataService.cs @@ -212,22 +212,30 @@ namespace MediaBrowser.Providers.Manager var attemptedFetch = refreshOptions.MetadataRefreshMode > MetadataRefreshMode.ValidationOnly || refreshOptions.ImageRefreshMode > MetadataRefreshMode.ValidationOnly; + var refreshStampNeedsSaving = false; + if (hasRefreshedMetadata && hasRefreshedImages && attemptedFetch) { item.DateLastRefreshed = DateTime.UtcNow; updateType |= item.OnMetadataChanged(); + + // A full refresh queries every provider whether or not anything looks stale. When they all + // come back empty the stamp is the only thing that changed, and without it nothing records + // that the lookup happened, so the next pass repeats the same fruitless queries forever. + refreshStampNeedsSaving = refreshOptions.MetadataRefreshMode == MetadataRefreshMode.FullRefresh + || refreshOptions.ImageRefreshMode == MetadataRefreshMode.FullRefresh; } - updateType = await SaveInternal(item, refreshOptions, updateType, isFirstRefresh, requiresRefresh, metadataResult, cancellationToken).ConfigureAwait(false); + updateType = await SaveInternal(item, refreshOptions, updateType, isFirstRefresh, requiresRefresh, refreshStampNeedsSaving, metadataResult, cancellationToken).ConfigureAwait(false); await AfterMetadataRefresh(itemOfType, refreshOptions, cancellationToken).ConfigureAwait(false); return updateType; - async Task<ItemUpdateType> SaveInternal(BaseItem item, MetadataRefreshOptions refreshOptions, ItemUpdateType updateType, bool isFirstRefresh, bool requiresRefresh, MetadataResult<TItemType> metadataResult, CancellationToken cancellationToken) + async Task<ItemUpdateType> SaveInternal(BaseItem item, MetadataRefreshOptions refreshOptions, ItemUpdateType updateType, bool isFirstRefresh, bool requiresRefresh, bool refreshStampNeedsSaving, MetadataResult<TItemType> metadataResult, CancellationToken cancellationToken) { // Save if changes were made, or it's never been saved before - if (refreshOptions.ForceSave || updateType > ItemUpdateType.None || isFirstRefresh || refreshOptions.ReplaceAllMetadata || requiresRefresh) + if (refreshOptions.ForceSave || updateType > ItemUpdateType.None || isFirstRefresh || refreshOptions.ReplaceAllMetadata || requiresRefresh || refreshStampNeedsSaving) { if (item.IsFileProtocol) { diff --git a/MediaBrowser.Providers/TV/SeriesMetadataService.cs b/MediaBrowser.Providers/TV/SeriesMetadataService.cs index b350f482c3..803fab538f 100644 --- a/MediaBrowser.Providers/TV/SeriesMetadataService.cs +++ b/MediaBrowser.Providers/TV/SeriesMetadataService.cs @@ -364,7 +364,7 @@ public class SeriesMetadataService : MetadataService<Series, SeriesInfo> foreach (var episode in episodes) { var season = seasons.FirstOrDefault(i => i.IndexNumber == episode.ParentIndexNumber); - if (season is null || episode.SeasonId.Equals(season.Id)) + if (season is null || (episode.SeasonId.Equals(season.Id) && episode.ParentId.Equals(season.Id))) { continue; } @@ -372,6 +372,11 @@ public class SeriesMetadataService : MetadataService<Series, SeriesInfo> // Assign the correct season id and name to episode. episode.SeasonId = season.Id; episode.SeasonName = season.Name; + + // We need to set ParentId here for episodes in virtual seasons (e.g., flat structures), otherwise it retains the + // ParentId from the series. + episode.SetParent(season); + await episode.UpdateToRepositoryAsync(ItemUpdateType.MetadataImport, cancellationToken).ConfigureAwait(false); } } diff --git a/tests/Jellyfin.Providers.Tests/Manager/MetadataServiceRefreshTests.cs b/tests/Jellyfin.Providers.Tests/Manager/MetadataServiceRefreshTests.cs index 1d2fb2e760..3b4d6fc9bb 100644 --- a/tests/Jellyfin.Providers.Tests/Manager/MetadataServiceRefreshTests.cs +++ b/tests/Jellyfin.Providers.Tests/Manager/MetadataServiceRefreshTests.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.Net.Http; using System.Threading; using System.Threading.Tasks; @@ -11,6 +12,7 @@ using MediaBrowser.Controller.IO; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.Persistence; using MediaBrowser.Controller.Providers; +using MediaBrowser.Model.Configuration; using MediaBrowser.Model.Entities; using MediaBrowser.Model.IO; using MediaBrowser.Providers.Manager; @@ -228,6 +230,88 @@ namespace Jellyfin.Providers.Tests.Manager Assert.Equal("nm0000123", mergedPerson.GetProviderId(MetadataProvider.Imdb)); } + [Theory] + [InlineData(MetadataRefreshMode.FullRefresh, true)] + [InlineData(MetadataRefreshMode.Default, false)] + public async Task RefreshMetadata_ProvidersFoundNothing_PersistsRefreshDateOnFullRefresh(MetadataRefreshMode mode, bool expectSaved) + { + var item = new TestItem + { + Id = Guid.NewGuid(), + Name = "Test Item", + PreferredMetadataLanguage = "en", + PreferredMetadataCountryCode = "US", + DateLastRefreshed = DateTime.UtcNow.AddDays(-60), + DateLastSaved = DateTime.UtcNow.AddDays(-60) + }; + item.PresentationUniqueKey = item.CreatePresentationUniqueKey(); + + var stampBefore = item.DateLastRefreshed; + + var provider = new Mock<IRemoteMetadataProvider<TestItem, ItemLookupInfo>>(MockBehavior.Loose); + provider.Setup(p => p.Name).Returns("Provider"); + provider.Setup(p => p.GetMetadata(It.IsAny<ItemLookupInfo>(), It.IsAny<CancellationToken>())) + .ReturnsAsync(new MetadataResult<TestItem> { HasMetadata = false }); + + var libraryManager = new Mock<ILibraryManager>(MockBehavior.Loose); + libraryManager.Setup(l => l.GetLibraryOptions(It.IsAny<BaseItem>())).Returns(new LibraryOptions()); + + var providerManager = new Mock<IProviderManager>(MockBehavior.Loose); + providerManager.Setup(p => p.GetImageProviders(It.IsAny<BaseItem>(), It.IsAny<ImageRefreshOptions>())) + .Returns(Array.Empty<IImageProvider>()); + providerManager.Setup(p => p.GetMetadataProviders<TestItem>(It.IsAny<BaseItem>(), It.IsAny<LibraryOptions>())) + .Returns(new[] { (IMetadataProvider<TestItem>)provider.Object }); + providerManager.Setup(p => p.GetMetadataSavers(It.IsAny<BaseItem>(), It.IsAny<LibraryOptions>())) + .Returns(Array.Empty<IMetadataSaver>()); + + var itemRepository = new Mock<IItemRepository>(MockBehavior.Loose); + itemRepository.Setup(r => r.ItemExistsAsync(It.IsAny<Guid>())).ReturnsAsync(true); + + var service = new TestItemMetadataService(libraryManager.Object, providerManager.Object, itemRepository.Object); + + await service.RefreshMetadata( + item, + new MetadataRefreshOptions(Mock.Of<IDirectoryService>()) + { + MetadataRefreshMode = mode, + ImageRefreshMode = mode + }, + CancellationToken.None).ConfigureAwait(true); + + // Nothing was found, so on a full refresh the advanced stamp is the only reason to write the row. + Assert.Equal(expectSaved, item.Saved); + + if (expectSaved) + { + Assert.True(item.DateLastRefreshed > stampBefore); + } + } + + /// <summary> + /// Stands in for a real item so the refresh stays off the shared BaseItem statics, which other + /// test classes in this assembly overwrite while xUnit runs them in parallel. + /// </summary> + internal sealed class TestItem : BaseItem + { + public bool Saved { get; private set; } + + public override bool RequiresRefresh() => false; + + public override bool IsSaveLocalMetadataEnabled() => false; + + public override string CreatePresentationUniqueKey() => Id.ToString("N", CultureInfo.InvariantCulture); + + public override ItemUpdateType OnMetadataChanged() => ItemUpdateType.None; + + public override bool BeforeMetadataRefresh(bool replaceAllMetadata) => false; + + public override Task UpdateToRepositoryAsync(ItemUpdateType updateReason, CancellationToken cancellationToken) + { + Saved = true; + return Task.CompletedTask; + } + } + private sealed class TestMetadataService : MetadataService<Movie, MovieInfo> { public TestMetadataService() @@ -249,5 +333,20 @@ namespace Jellyfin.Providers.Tests.Manager ICollection<IMetadataProvider> providers) => RefreshWithProviders(metadata, id, options, providers, ImageProvider, false, CancellationToken.None); } + + private sealed class TestItemMetadataService : MetadataService<TestItem, ItemLookupInfo> + { + public TestItemMetadataService(ILibraryManager libraryManager, IProviderManager providerManager, IItemRepository itemRepository) + : base( + Mock.Of<IServerConfigurationManager>(), + NullLogger<MetadataService<TestItem, ItemLookupInfo>>.Instance, + providerManager, + Mock.Of<IFileSystem>(), + libraryManager, + Mock.Of<IExternalDataManager>(), + itemRepository) + { + } + } } } diff --git a/tests/Jellyfin.Server.Implementations.Tests/Dto/DtoServiceTests.cs b/tests/Jellyfin.Server.Implementations.Tests/Dto/DtoServiceTests.cs index 9c247d54b9..bdac59c013 100644 --- a/tests/Jellyfin.Server.Implementations.Tests/Dto/DtoServiceTests.cs +++ b/tests/Jellyfin.Server.Implementations.Tests/Dto/DtoServiceTests.cs @@ -1,5 +1,7 @@ using System; +using System.Collections.Generic; using Emby.Server.Implementations.Dto; +using Jellyfin.Database.Implementations.Entities; using MediaBrowser.Common; using MediaBrowser.Controller.Chapters; using MediaBrowser.Controller.Drawing; @@ -21,11 +23,13 @@ namespace Jellyfin.Server.Implementations.Tests.Dto; public class DtoServiceTests { private readonly Mock<ILibraryManager> _libraryManagerMock; + private readonly Mock<IUserDataManager> _userDataManagerMock; private readonly DtoService _dtoService; public DtoServiceTests() { _libraryManagerMock = new Mock<ILibraryManager>(); + _userDataManagerMock = new Mock<IUserDataManager>(); var imageProcessor = new Mock<IImageProcessor>(); // Deterministic tag derived from the image so each item gets a distinct, assertable tag. @@ -42,7 +46,7 @@ public class DtoServiceTests _dtoService = new DtoService( NullLogger<DtoService>.Instance, _libraryManagerMock.Object, - new Mock<IUserDataManager>().Object, + _userDataManagerMock.Object, imageProcessor.Object, new Mock<IProviderManager>().Object, new Mock<IRecordingsManager>().Object, @@ -105,6 +109,57 @@ public class DtoServiceTests Assert.Null(dto.ParentPrimaryImageItemId); } + [Fact] + public void GetBaseItemDtos_SeasonWithNoRealEpisodes_ReportsVirtualEpisodesAsChildCount() + { + // No episode has aired yet, so RecursiveItemCount is 0. ChildCount must still report the + // virtual episodes clients get back for the season. This deliberately does not track + // Season.IsVirtualItem: that flag is recomputed only on a full refresh, so a season can + // carry it while already holding real episodes. + var (season, user) = BuildSeason(playedCount: 0, totalCount: 0, childCount: 10); + var options = new DtoOptions(false) { EnableImages = false, Fields = [ItemFields.ChildCount, ItemFields.RecursiveItemCount] }; + + var dto = _dtoService.GetBaseItemDtos([season], options, user, skipVisibilityCheck: true)[0]; + + Assert.Equal(0, dto.RecursiveItemCount); + Assert.Equal(10, dto.ChildCount); + } + + [Fact] + public void GetBaseItemDtos_SeasonWithRealEpisodes_KeepsRecursiveItemCountAsChildCount() + { + var (season, user) = BuildSeason(playedCount: 2, totalCount: 9, childCount: 11); + var options = new DtoOptions(false) { EnableImages = false, Fields = [ItemFields.ChildCount, ItemFields.RecursiveItemCount] }; + + var dto = _dtoService.GetBaseItemDtos([season], options, user, skipVisibilityCheck: true)[0]; + + Assert.Equal(9, dto.RecursiveItemCount); + // The shortcut still wins over the batched child count, which also counts virtual episodes. + Assert.Equal(9, dto.ChildCount); + } + + private (Season Season, User User) BuildSeason(int playedCount, int totalCount, int childCount) + { + var user = new User("user", "auth-provider", "reset-provider"); + var season = new Season { Id = Guid.NewGuid(), Name = "Season 2", SeriesId = Guid.NewGuid() }; + + _userDataManagerMock + .Setup(x => x.GetUserDataBatch(It.IsAny<IReadOnlyList<BaseItem>>(), user)) + .Returns(new Dictionary<Guid, UserItemData> { [season.Id] = new UserItemData { Key = "key" } }); + _userDataManagerMock + .Setup(x => x.GetResumeUserDataBatch(It.IsAny<IReadOnlyList<BaseItem>>(), user)) + .Returns(new Dictionary<Guid, VersionResumeData>()); + + _libraryManagerMock + .Setup(x => x.GetPlayedAndTotalCountBatch(It.IsAny<IReadOnlyList<Guid>>(), user)) + .Returns(new Dictionary<Guid, (int Played, int Total)> { [season.Id] = (playedCount, totalCount) }); + _libraryManagerMock + .Setup(x => x.GetChildCountBatch(It.IsAny<IReadOnlyList<Guid>>(), It.IsAny<Guid?>())) + .Returns(new Dictionary<Guid, int> { [season.Id] = childCount }); + + return (season, user); + } + private (Episode Episode, Season Season, Series Series) BuildEpisode(bool seasonHasPoster, bool seriesHasPoster = true) { // Non-local (http) paths keep aspect-ratio resolution off the image processor and on the |