Merge pull request #17819 from Shadowghost/fix-tmdb-search

Fix TMDb search result ranking

4 files changed, 333 insertions, 9 deletions

diff --git a/MediaBrowser.Providers/Plugins/Tmdb/Movies/TmdbMovieProvider.cs b/MediaBrowser.Providers/Plugins/Tmdb/Movies/TmdbMovieProvider.cs
index ef952082da..9501da7ec6 100644
--- a/MediaBrowser.Providers/Plugins/Tmdb/Movies/TmdbMovieProvider.cs
+++ b/MediaBrowser.Providers/Plugins/Tmdb/Movies/TmdbMovieProvider.cs
@@ -163,12 +163,15 @@ namespace MediaBrowser.Providers.Plugins.Tmdb.Movies
                 // Caller provides the filename with extension stripped and NOT the parsed filename
                 var parsedName = _libraryManager.ParseName(info.Name);
                 var cleanedName = TmdbUtils.CleanName(parsedName.Name);
+                var searchYear = info.Year ?? parsedName.Year ?? 0;
 
-                var searchResults = await _tmdbClientManager.SearchMovieAsync(cleanedName, info.Year ?? parsedName.Year ?? 0, info.MetadataLanguage, info.MetadataCountryCode, cancellationToken).ConfigureAwait(false);
+                var searchResults = await _tmdbClientManager.SearchMovieAsync(cleanedName, searchYear, info.MetadataLanguage, info.MetadataCountryCode, cancellationToken).ConfigureAwait(false);
 
-                if (searchResults?.Count > 0)
+                var match = TmdbUtils.FindBestMatch(searchResults, parsedName.Name, searchYear);
+
+                if (match is not null)
                 {
-                    tmdbId = searchResults[0].Id;
+                    tmdbId = match.Id;
                 }
             }
 
diff --git a/MediaBrowser.Providers/Plugins/Tmdb/TV/TmdbSeriesProvider.cs b/MediaBrowser.Providers/Plugins/Tmdb/TV/TmdbSeriesProvider.cs
index 6163e20194..b6bb7aa013 100755
--- a/MediaBrowser.Providers/Plugins/Tmdb/TV/TmdbSeriesProvider.cs
+++ b/MediaBrowser.Providers/Plugins/Tmdb/TV/TmdbSeriesProvider.cs
@@ -202,11 +202,14 @@ namespace MediaBrowser.Providers.Plugins.Tmdb.TV
                 // Caller provides the filename with extension stripped and NOT the parsed filename
                 var parsedName = _libraryManager.ParseName(info.Name);
                 var cleanedName = TmdbUtils.CleanName(parsedName.Name);
-                var searchResults = await _tmdbClientManager.SearchSeriesAsync(cleanedName, info.MetadataLanguage, info.MetadataCountryCode, info.Year ?? parsedName.Year ?? 0, cancellationToken).ConfigureAwait(false);
+                var searchYear = info.Year ?? parsedName.Year ?? 0;
+                var searchResults = await _tmdbClientManager.SearchSeriesAsync(cleanedName, info.MetadataLanguage, info.MetadataCountryCode, searchYear, cancellationToken).ConfigureAwait(false);
 
-                if (searchResults?.Count > 0)
+                var match = TmdbUtils.FindBestMatch(searchResults, parsedName.Name, searchYear);
+
+                if (match is not null)
                 {
-                    tmdbId = searchResults[0].Id.ToString(CultureInfo.InvariantCulture);
+                    tmdbId = match.Id.ToString(CultureInfo.InvariantCulture);
                 }
             }
 
diff --git a/MediaBrowser.Providers/Plugins/Tmdb/TmdbUtils.cs b/MediaBrowser.Providers/Plugins/Tmdb/TmdbUtils.cs
index 44a2f7291e..a002140d1e 100644
--- a/MediaBrowser.Providers/Plugins/Tmdb/TmdbUtils.cs
+++ b/MediaBrowser.Providers/Plugins/Tmdb/TmdbUtils.cs
@@ -8,6 +8,7 @@ using System.Text.RegularExpressions;
 using Jellyfin.Data.Enums;
 using MediaBrowser.Model.Entities;
 using TMDbLib.Objects.General;
+using TMDbLib.Objects.Search;
 using TMDbLib.Objects.TvShows;
 using PersonInfo = MediaBrowser.Controller.Entities.PersonInfo;
 
@@ -33,6 +34,11 @@ namespace MediaBrowser.Providers.Plugins.Tmdb
         /// </summary>
         public const string ApiKey = "4219e299c89411838049ab0dab19ebd5";
 
+        private const int TitleExactScore = 8;
+        private const int TitlePrefixScore = 4;
+        private const int YearExactScore = 2;
+        private const int YearAdjacentScore = 1;
+
         /// <summary>
         /// The crew types to keep.
         /// </summary>
@@ -63,8 +69,20 @@ namespace MediaBrowser.Providers.Plugins.Tmdb
             "novel"
         }.ToFrozenSet(StringComparer.OrdinalIgnoreCase);
 
-        [GeneratedRegex(@"[\W_-[·]]+")]
-        private static partial Regex NonWordRegex();
+        /// <summary>
+        /// Everything that is not a letter, a number or a combining mark separates two search terms. The
+        /// interpunct is kept because TMDb uses it inside titles such as "WALL·E", where it matches better
+        /// than a space does.
+        /// </summary>
+        [GeneratedRegex(@"[^\p{L}\p{N}\p{M}·]+")]
+        private static partial Regex NonSearchTermRegex();
+
+        /// <summary>
+        /// As <see cref="NonSearchTermRegex"/>, but the interpunct is a separator too, so a "WALL-E" folder
+        /// and the "WALL·E" title TMDb returns compare equal.
+        /// </summary>
+        [GeneratedRegex(@"[^\p{L}\p{N}\p{M}]+")]
+        private static partial Regex NonComparableRegex();
 
         /// <summary>
         /// Gets the TMDb id of an item, if it has one TMDb can be queried with.
@@ -101,7 +119,140 @@ namespace MediaBrowser.Providers.Plugins.Tmdb
         public static string CleanName(string name)
         {
             // TMDb expects a space separated list of words make sure that is the case
-            return NonWordRegex().Replace(name, " ");
+            return NonSearchTermRegex().Replace(name, " ").Trim();
+        }
+
+        /// <summary>
+        /// Reduces a title to the form used to compare a local name against a TMDb search result.
+        /// </summary>
+        /// <param name="title">The title to normalize.</param>
+        /// <returns>The normalized title, or an empty string if there was nothing to normalize.</returns>
+        public static string NormalizeTitle(string? title)
+        {
+            return string.IsNullOrEmpty(title)
+                ? string.Empty
+                : NonComparableRegex().Replace(title, " ").Trim().ToLowerInvariant();
+        }
+
+        /// <summary>
+        /// Picks the movie search result that best matches the name and year an item was looked up by.
+        /// </summary>
+        /// <param name="results">The search results, in the order TMDb returned them.</param>
+        /// <param name="name">The parsed name of the local item.</param>
+        /// <param name="year">The year of the local item, or 0 if it is unknown.</param>
+        /// <returns>The best match, or <c>null</c> if there were no results.</returns>
+        public static SearchMovie? FindBestMatch(IReadOnlyList<SearchMovie>? results, string? name, int year)
+        {
+            return FindBestMatch(
+                results,
+                name,
+                year,
+                static movie => movie.Title,
+                static movie => movie.OriginalTitle,
+                static movie => movie.ReleaseDate);
+        }
+
+        /// <summary>
+        /// Picks the series search result that best matches the name and year an item was looked up by.
+        /// </summary>
+        /// <param name="results">The search results, in the order TMDb returned them.</param>
+        /// <param name="name">The parsed name of the local item.</param>
+        /// <param name="year">The year of the local item, or 0 if it is unknown.</param>
+        /// <returns>The best match, or <c>null</c> if there were no results.</returns>
+        public static SearchTv? FindBestMatch(IReadOnlyList<SearchTv>? results, string? name, int year)
+        {
+            return FindBestMatch(
+                results,
+                name,
+                year,
+                static series => series.Name,
+                static series => series.OriginalName,
+                static series => series.FirstAirDate);
+        }
+
+        /// <summary>
+        /// Picks the search result that best matches the name and year an item was looked up by.
+        /// </summary>
+        /// <remarks>
+        /// TMDb's year parameter only nudges relevance, it does not filter, so the first hit is regularly a
+        /// different film or show that happens to share the title - searching for "Mulan" with year 2020
+        /// returns the 1998 film first. A title that matches outranks one that does not, and the year only
+        /// separates candidates that are otherwise equally good. When nothing matches at all TMDb's own
+        /// ordering is kept, so a name that needs fuzzy matching, such as "A Christmas No. 1" for
+        /// "A Christmas Number One", still resolves.
+        /// </remarks>
+        private static T? FindBestMatch<T>(
+            IReadOnlyList<T>? results,
+            string? name,
+            int year,
+            Func<T, string?> titleSelector,
+            Func<T, string?> originalTitleSelector,
+            Func<T, DateTime?> releaseDateSelector)
+            where T : class
+        {
+            if (results is null || results.Count == 0)
+            {
+                return null;
+            }
+
+            var normalizedName = NormalizeTitle(name);
+            if (normalizedName.Length == 0)
+            {
+                return results[0];
+            }
+
+            var best = results[0];
+            var bestScore = 0;
+
+            foreach (var result in results)
+            {
+                var score = Math.Max(
+                        ScoreTitle(normalizedName, titleSelector(result)),
+                        ScoreTitle(normalizedName, originalTitleSelector(result)))
+                    + ScoreYear(year, releaseDateSelector(result)?.Year);
+
+                // Strictly greater, so ties keep the earlier, more relevant result.
+                if (score > bestScore)
+                {
+                    bestScore = score;
+                    best = result;
+                }
+            }
+
+            return best;
+        }
+
+        private static int ScoreTitle(string normalizedName, string? title)
+        {
+            var normalizedTitle = NormalizeTitle(title);
+
+            if (string.Equals(normalizedName, normalizedTitle, StringComparison.Ordinal))
+            {
+                return TitleExactScore;
+            }
+
+            // Whole words only, otherwise "Wall" half matches "Wall Street".
+            return normalizedTitle.Length > normalizedName.Length
+                && normalizedTitle[normalizedName.Length] == ' '
+                && normalizedTitle.StartsWith(normalizedName, StringComparison.Ordinal)
+                    ? TitlePrefixScore
+                    : 0;
+        }
+
+        private static int ScoreYear(int year, int? resultYear)
+        {
+            if (year <= 0 || resultYear is not int candidateYear)
+            {
+                return 0;
+            }
+
+            return Math.Abs(candidateYear - year) switch
+            {
+                0 => YearExactScore,
+                // Regional release dates routinely straddle a new year.
+                1 => YearAdjacentScore,
+                _ => 0
+            };
         }
 
         /// <summary>
diff --git a/tests/Jellyfin.Providers.Tests/Tmdb/TmdbUtilsTests.cs b/tests/Jellyfin.Providers.Tests/Tmdb/TmdbUtilsTests.cs
index 4c4dd5e92f..8926a7b13c 100644
--- a/tests/Jellyfin.Providers.Tests/Tmdb/TmdbUtilsTests.cs
+++ b/tests/Jellyfin.Providers.Tests/Tmdb/TmdbUtilsTests.cs
@@ -1,6 +1,9 @@
+using System;
+using System.Collections.Generic;
 using MediaBrowser.Controller.Entities.Movies;
 using MediaBrowser.Model.Entities;
 using MediaBrowser.Providers.Plugins.Tmdb;
+using TMDbLib.Objects.Search;
 using Xunit;
 
 namespace Jellyfin.Providers.Tests.Tmdb
@@ -71,5 +74,169 @@ namespace Jellyfin.Providers.Tests.Tmdb
             Assert.False(new Movie().TryGetTmdbId(out var tmdbId));
             Assert.Equal(0, tmdbId);
         }
+
+        [Theory]
+        [InlineData("The Amityville Horror", "The Amityville Horror")]
+        [InlineData("WALL-E", "WALL E")]
+        // The interpunct is kept, it matches the TMDb title better than a space does.
+        [InlineData("WALL·E", "WALL·E")]
+        [InlineData("50-50", "50 50")]
+        [InlineData("A Christmas No. 1", "A Christmas No 1")]
+        // Vulgar fractions are numbers, dropping them turned "8½" into a search for "8".
+        [InlineData("8½", "8½")]
+        [InlineData("9½ Weeks", "9½ Weeks")]
+        [InlineData("  Léon: The Professional  ", "Léon The Professional")]
+        public static void CleanName_Valid_Success(string name, string expected)
+        {
+            Assert.Equal(expected, TmdbUtils.CleanName(name));
+        }
+
+        [Theory]
+        [InlineData("WALL-E", "wall e")]
+        [InlineData("WALL·E", "wall e")]
+        [InlineData("WALL E", "wall e")]
+        [InlineData("8½", "8½")]
+        [InlineData("Ocean's Eleven", "ocean s eleven")]
+        [InlineData(null, "")]
+        [InlineData("   ", "")]
+        public static void NormalizeTitle_Valid_Success(string? title, string expected)
+        {
+            Assert.Equal(expected, TmdbUtils.NormalizeTitle(title));
+        }
+
+        [Theory]
+        [MemberData(nameof(FindBestMatch_Movies_TestData))]
+        public static void FindBestMatch_Movies_PicksExpected(string description, string name, int year, IReadOnlyList<SearchMovie> results, int expectedId)
+        {
+            var match = TmdbUtils.FindBestMatch(results, name, year);
+
+            Assert.NotNull(match);
+            Assert.True(expectedId == match.Id, $"{description}: expected {expectedId} but matched {match.Id}");
+        }
+
+        [Fact]
+        public static void FindBestMatch_Series_PicksMatchingFirstAirYear()
+        {
+            IReadOnlyList<SearchTv> results =
+            [
+                Series(10042, "Doc", "Doc", 2001),
+                Series(101048, "Doc", "Doc", 2020),
+                Series(255055, "Doc", "Doc", 2025),
+                Series(2430, "Doc Martin", "Doc Martin", 2004)
+            ];
+
+            var match = TmdbUtils.FindBestMatch(results, "Doc", 2025);
+
+            Assert.NotNull(match);
+            Assert.Equal(255055, match.Id);
+        }
+
+        [Fact]
+        public static void FindBestMatch_NoResults_ReturnsNull()
+        {
+            Assert.Null(TmdbUtils.FindBestMatch(Array.Empty<SearchMovie>(), "Mulan", 2020));
+            Assert.Null(TmdbUtils.FindBestMatch(Array.Empty<SearchTv>(), "Doc", 2025));
+            Assert.Null(TmdbUtils.FindBestMatch((IReadOnlyList<SearchMovie>?)null, "Mulan", 2020));
+            Assert.Null(TmdbUtils.FindBestMatch((IReadOnlyList<SearchTv>?)null, "Doc", 2025));
+        }
+
+        public static TheoryData<string, string, int, IReadOnlyList<SearchMovie>, int> FindBestMatch_Movies_TestData()
+            => new()
+            {
+                // TMDb's year parameter does not filter, so the remake and the original both come back and
+                // the wrong one is first. Results are in the order the live API returned them.
+                {
+                    "Mulan (2020)", "Mulan", 2020,
+                    [Movie(10674, "Mulan", "Mulan", 1998), Movie(337401, "Mulan", "Mulan", 2020), Movie(752662, "Hua Mulan", "花木兰", 2020)],
+                    337401
+                },
+                {
+                    "Mulan (1998)", "Mulan", 1998,
+                    [Movie(10674, "Mulan", "Mulan", 1998), Movie(337401, "Mulan", "Mulan", 2020), Movie(752662, "Hua Mulan", "花木兰", 2020)],
+                    10674
+                },
+                {
+                    "Aladdin (2019)", "Aladdin", 2019,
+                    [Movie(812, "Aladdin", "Aladdin", 1992), Movie(420817, "Aladdin", "Aladdin", 2019), Movie(602411, "Adventures of Aladdin", "Adventures of Aladdin", 2019)],
+                    420817
+                },
+                {
+                    "The Lion King (2019)", "The Lion King", 2019,
+                    [Movie(8587, "The Lion King", "The Lion King", 1994), Movie(420818, "The Lion King", "The Lion King", 2019)],
+                    420818
+                },
+                {
+                    "The Amityville Horror (1979)", "The Amityville Horror", 1979,
+                    [Movie(10065, "The Amityville Horror", "The Amityville Horror", 2005), Movie(11449, "The Amityville Horror", "The Amityville Horror", 1979)],
+                    11449
+                },
+                // A featurette outranks the film it belongs to. The interpunct must not stop "WALL-E" from
+                // matching "WALL·E", or the prefix match on the featurette wins.
+                {
+                    "WALL-E (2008)", "WALL-E", 2008,
+                    [Movie(877268, "WALL·E's Treasures & Trinkets", "WALL·E's Treasures & Trinkets", 2008), Movie(10681, "WALL·E", "WALL·E", 2008), Movie(10673, "Wall Street", "Wall Street", 1987)],
+                    10681
+                },
+                // The name only survives as "8" if the fraction is stripped, and then every 1963 result ties.
+                {
+                    "8½ (1963)", "8½", 1963,
+                    [Movie(422801, "Interpol Code 8", "国際秘密警察　指令第８号", 1963), Movie(520251, "Um 8 Uhr kommt Sadowski", "Um 8 Uhr kommt Sadowski", 1963), Movie(422, "8½", "8½", 1963)],
+                    422
+                },
+                // Matched on the original title, the localized one is unrecognizable.
+                {
+                    "Ściany mają uszy (1966)", "Ściany mają uszy", 1966,
+                    [Movie(1, "Something Else", "Something Else", 1966), Movie(2, "Walls Have Ears", "Ściany mają uszy", 1966)],
+                    2
+                },
+                // Regional release dates straddle the new year, so a year that is off by one still matches.
+                {
+                    "Off by one year", "Some Movie", 2011,
+                    [Movie(1, "Some Movie", "Some Movie", 2015), Movie(2, "Some Movie", "Some Movie", 2010)],
+                    2
+                },
+                // Nothing matches the name, so TMDb's own ordering is kept.
+                {
+                    "A Christmas No. 1 (2021)", "A Christmas No. 1", 2021,
+                    [Movie(878111, "A Christmas Number One", "A Christmas Number One", 2021), Movie(2, "Ten Hours for Christmas", "10 Horas para o Natal", 2021)],
+                    878111
+                },
+                // A title that matches always beats one that only shares the year.
+                {
+                    "Title outranks year", "Some Movie", 2020,
+                    [Movie(1, "A Different Movie", "A Different Movie", 2020), Movie(2, "Some Movie", "Some Movie", 1994)],
+                    2
+                },
+                // Without a year the title alone decides, and equally good titles keep TMDb's order.
+                {
+                    "No year known", "Mulan", 0,
+                    [Movie(10674, "Mulan", "Mulan", 1998), Movie(337401, "Mulan", "Mulan", 2020)],
+                    10674
+                },
+                // An unparsable name must not throw or reorder anything.
+                {
+                    "Empty name", "  ", 2020,
+                    [Movie(1, "Some Movie", "Some Movie", 1994), Movie(2, "Some Movie", "Some Movie", 2020)],
+                    1
+                }
+            };
+
+        private static SearchMovie Movie(int id, string title, string originalTitle, int year)
+            => new()
+            {
+                Id = id,
+                Title = title,
+                OriginalTitle = originalTitle,
+                ReleaseDate = new DateTime(year, 6, 1, 0, 0, 0, DateTimeKind.Utc)
+            };
+
+        private static SearchTv Series(int id, string name, string originalName, int year)
+            => new()
+            {
+                Id = id,
+                Name = name,
+                OriginalName = originalName,
+                FirstAirDate = new DateTime(year, 6, 1, 0, 0, 0, DateTimeKind.Utc)
+            };
     }
 }