diff options
| author | Shadowghost <Shadowghost@users.noreply.github.com> | 2026-11-20 07:23:42 -0500 |
|---|---|---|
| committer | Cody Robibero <cody@robibe.ro> | 2026-11-20 07:23:42 -0500 |
| commit | 511f1ef882d594df64810d2c36b59ad381fd6e17 (patch) | |
| tree | 2a4dfabd381bbd23a1e1d65f9f9317c7deb659fb /src | |
| parent | 00d8050bf661f6f9050a8de6e55585a8fe6d8beb (diff) | |
Backport pull request #18048 from jellyfin/release-12.z
Follow the Schedules Direct API contract when refreshing the guide
Original-merge: 85fa90cb758dabfb434fbd00c1128091007cf6a4
Merged-by: Bond-009 <bond.009@outlook.com>
Backported-by: Cody Robibero <cody@robibe.ro>
Diffstat (limited to 'src')
15 files changed, 1820 insertions, 191 deletions
diff --git a/src/Jellyfin.LiveTv/Guide/GuideManager.cs b/src/Jellyfin.LiveTv/Guide/GuideManager.cs index a11f83f2f8..15486081b0 100644 --- a/src/Jellyfin.LiveTv/Guide/GuideManager.cs +++ b/src/Jellyfin.LiveTv/Guide/GuideManager.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Net.Http; using System.Threading; using System.Threading.Tasks; using Jellyfin.Data.Enums; @@ -8,6 +9,7 @@ using Jellyfin.Extensions; using Jellyfin.LiveTv.Configuration; using Jellyfin.LiveTv.Listings; using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.Net; using MediaBrowser.Controller.Dto; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; @@ -747,26 +749,17 @@ public class GuideManager : IGuideManager return false; } - private async Task PreCacheImages(IReadOnlyList<BaseItem> programs, DateTime maxCacheDate) + internal async Task PreCacheImages(IReadOnlyList<BaseItem> programs, DateTime maxCacheDate) { - var sdLimitActive = _schedulesDirectService.IsImageDailyLimitActive(); - await Parallel.ForEachAsync( programs .Where(p => p.EndDate.HasValue && p.EndDate.Value < maxCacheDate) - .Where(p => !sdLimitActive || !p.ImageInfos.All( - img => img.IsLocalFile || img.Path.Contains("schedulesdirect", StringComparison.OrdinalIgnoreCase))) + .Where(p => p.ImageInfos.Any(img => !img.IsLocalFile)) .DistinctBy(p => p.Id), _cacheParallelOptions, async (program, cancellationToken) => { - // Re-check: limit may have been set by a parallel task since the LINQ filter ran. - if (_schedulesDirectService.IsImageDailyLimitActive() - && program.ImageInfos.All( - img => img.IsLocalFile || img.Path.Contains("schedulesdirect", StringComparison.OrdinalIgnoreCase))) - { - return; - } + List<ItemImageInfo>? deadImages = null; for (var i = 0; i < program.ImageInfos.Length; i++) { @@ -781,10 +774,12 @@ public class GuideManager : IGuideManager continue; } - // Skip SD downloads once the daily limit has been hit. - if (imageInfo.Path.Contains("schedulesdirect", StringComparison.OrdinalIgnoreCase) - && _schedulesDirectService.IsImageDailyLimitActive()) + // Re-checked per image: the limit may have been hit by a parallel task. A blocked + // url left on the program would still be fetched when a client asks for it. + if (!_schedulesDirectService.CanDownloadImage(imageInfo.Path)) { + deadImages ??= []; + deadImages.Add(imageInfo); continue; } @@ -797,12 +792,35 @@ public class GuideManager : IGuideManager imageIndex: 0, removeOnFailure: false) .ConfigureAwait(false); + _schedulesDirectService.ReportImageDownloadSuccess(imageInfo.Path); } catch (Exception ex) { + // A rejected download never surfaces as an error on its own, so feed the + // response back to the provider to stop us hammering an exhausted account. + var httpException = (ex as HttpRequestException) ?? ex.InnerException as HttpRequestException; + var action = _schedulesDirectService.ReportImageDownloadFailure( + imageInfo.Path, + httpException?.StatusCode, + (httpException as HttpResponseBodyException)?.ResponseBody); + + if (action == ImageDownloadFailureAction.RemoveImage) + { + deadImages ??= []; + deadImages.Add(imageInfo); + } + _logger.LogWarning(ex, "Unable to pre-cache {Url}", imageInfo.Path); } } + + if (deadImages is not null) + { + // Requesting a url Schedules Direct has already rejected is what gets an + // account blocked, so it must not survive to the next refresh. + program.RemoveImages(deadImages); + await program.UpdateToRepositoryAsync(ItemUpdateType.ImageUpdate, cancellationToken).ConfigureAwait(false); + } }).ConfigureAwait(false); } } diff --git a/src/Jellyfin.LiveTv/Listings/ListingsManager.cs b/src/Jellyfin.LiveTv/Listings/ListingsManager.cs index 7274a3030c..2f4afc2b27 100644 --- a/src/Jellyfin.LiveTv/Listings/ListingsManager.cs +++ b/src/Jellyfin.LiveTv/Listings/ListingsManager.cs @@ -124,6 +124,8 @@ public class ListingsManager : IListingsManager { ArgumentNullException.ThrowIfNull(channel); + Exception? failure = null; + foreach (var (provider, providerInfo) in GetListingProviders()) { if (!IsListingProviderEnabledForTuner(providerInfo, channel.TunerHostId)) @@ -144,19 +146,42 @@ public class ListingsManager : IListingsManager provider.Name, providerInfo.ListingsId ?? string.Empty); - var epgChannels = await GetEpgChannels(provider, providerInfo, true, cancellationToken).ConfigureAwait(false); + List<ProgramInfo> programs; + try + { + // Listing the provider's channels can fail as well, and that must not stop the + // fallback to the next provider either. + var epgChannels = await GetEpgChannels(provider, providerInfo, true, cancellationToken).ConfigureAwait(false); - var epgChannel = GetEpgChannelFromTunerChannel(providerInfo.ChannelMappings, channel, epgChannels); - if (epgChannel is null) + var epgChannel = GetEpgChannelFromTunerChannel(providerInfo.ChannelMappings, channel, epgChannels); + if (epgChannel is null) + { + _logger.LogDebug("EPG channel not found for tuner channel {0}-{1} from {2}-{3}", channel.Number, channel.Name, provider.Name, providerInfo.ListingsId ?? string.Empty); + continue; + } + + programs = (await provider + .GetProgramsAsync(providerInfo, epgChannel.Id, startDateUtc, endDateUtc, cancellationToken).ConfigureAwait(false)) + .ToList(); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) { - _logger.LogDebug("EPG channel not found for tuner channel {0}-{1} from {2}-{3}", channel.Number, channel.Name, provider.Name, providerInfo.ListingsId ?? string.Empty); + // Another provider may still be able to serve this channel, so the failure is + // held back rather than thrown, and only surfaces if nothing else answers. + _logger.LogError( + ex, + "Error getting programs for channel {Number}-{Name} from {Provider}", + channel.Number, + channel.Name, + provider.Name); + failure ??= ex; continue; } - var programs = (await provider - .GetProgramsAsync(providerInfo, epgChannel.Id, startDateUtc, endDateUtc, cancellationToken).ConfigureAwait(false)) - .ToList(); - // Replace the value that came from the provider with a normalized value foreach (var program in programs) { @@ -170,7 +195,16 @@ public class ListingsManager : IListingsManager } } - return Enumerable.Empty<ProgramInfo>(); + // Reporting no programs would have the guide refresh delete the ones this channel already + // has, so a channel nothing could be fetched for has to fail instead. + if (failure is not null) + { + throw new InvalidOperationException( + $"No listings provider could supply programs for channel {channel.Number}-{channel.Name}", + failure); + } + + return []; } /// <inheritdoc /> diff --git a/src/Jellyfin.LiveTv/Listings/SchedulesDirect.cs b/src/Jellyfin.LiveTv/Listings/SchedulesDirect.cs index d2076767b8..801c4f163b 100644 --- a/src/Jellyfin.LiveTv/Listings/SchedulesDirect.cs +++ b/src/Jellyfin.LiveTv/Listings/SchedulesDirect.cs @@ -25,6 +25,8 @@ using Jellyfin.LiveTv.Listings.SchedulesDirectDtos; using MediaBrowser.Common.Configuration; using MediaBrowser.Common.Net; using MediaBrowser.Controller.Authentication; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Library; using MediaBrowser.Controller.LiveTv; using MediaBrowser.Model.Dto; using MediaBrowser.Model.Entities; @@ -38,31 +40,76 @@ namespace Jellyfin.LiveTv.Listings private const string ApiUrl = "https://json.schedulesdirect.org/20141201"; private const int CountryCacheDays = 7; + // Schedules Direct disables an account that keeps requesting images after the daily + // download limit, so stop after a short streak of rejections instead of retrying forever. + private const int MaxConsecutiveImageFailures = 10; + + // Hard server-side cap on elements in a single programs request. + private const int MaxProgramsPerRequest = 5000; + + // Memory ceiling on the rejected image uris (~2MB of 64-char keys). The daily invalid-uri + // limit stops the account long before this many can accumulate. + private const int MaxRejectedImageUris = 10000; + + // Stations that stopped being queried keep their cached schedules for this long. + private const int ScheduleCacheMaxAgeDays = 30; + + // Long enough to outlive any restart, short enough that the list cannot fill up for good. + private const int RejectedImageUriMaxAgeDays = 30; + + // The spec requires a client to stay away for an hour once the service reports Offline. + private const int OfflineBackoffMinutes = 60; + + private const int LoginBackoffMinutes = 30; + private const int MaxLoginsBackoffHours = 24; + + private const int StatusCacheMinutes = 10; + private readonly ILogger<SchedulesDirect> _logger; private readonly IHttpClientFactory _httpClientFactory; private readonly IApplicationPaths _appPaths; + private readonly ILibraryManager _libraryManager; private readonly AsyncNonKeyedLocker _tokenLock = new(1); + private readonly AsyncNonKeyedLocker _statusLock = new(1); + private readonly SchedulesDirectScheduleCache _scheduleCache; + private readonly SchedulesDirectLineupCache _lineupCache; + + // Image uris Schedules Direct answered with IMAGE_NOT_FOUND. Requesting one of these + // again is what trips MAX_IMAGE_INVALID_URI_ERRORS and gets the account blocked. + private readonly ConcurrentDictionary<string, byte> _rejectedImageUris = new(StringComparer.Ordinal); + private readonly Lock _rejectedImageUrisFileLock = new(); + + // Station/date pairs the server has queued for generation, with the retryTime it gave. + private readonly ConcurrentDictionary<string, DateTime> _scheduleRetryTimes = new(StringComparer.Ordinal); private readonly ConcurrentDictionary<string, NameValuePair> _tokens = new(); private readonly JsonSerializerOptions _jsonOptions = JsonDefaults.Options; - private long _lastErrorResponseTicks; + private long _loginBackoffUntilTicks; private volatile bool _accountError; - private bool _disposed = false; + private bool _disposed; private byte[] _countriesCache; private DateOnly? _imageLimitHitDate; private DateOnly? _metadataLimitHitDate; + private int _consecutiveImageFailures; + private long _offlineUntilTicks; + private StatusCacheEntry _status; public SchedulesDirect( ILogger<SchedulesDirect> logger, IHttpClientFactory httpClientFactory, - IApplicationPaths appPaths) + IApplicationPaths appPaths, + ILibraryManager libraryManager) { _logger = logger; _httpClientFactory = httpClientFactory; _appPaths = appPaths; + _libraryManager = libraryManager; + _scheduleCache = new SchedulesDirectScheduleCache(logger, appPaths); + _lineupCache = new SchedulesDirectLineupCache(logger, appPaths); _imageLimitHitDate = LoadDailyLimitDate(ImageLimitFilePath); _metadataLimitHitDate = LoadDailyLimitDate(MetadataLimitFilePath); + LoadRejectedImageUris(); } /// <inheritdoc /> @@ -72,6 +119,8 @@ namespace Jellyfin.LiveTv.Listings private string MetadataLimitFilePath => Path.Combine(_appPaths.CachePath, "sd-metadata-limit.txt"); + private string RejectedImageUrisFilePath => Path.Combine(_appPaths.CachePath, "sd-rejected-images.txt"); + /// <inheritdoc /> public string Type => nameof(SchedulesDirect); @@ -93,9 +142,11 @@ namespace Jellyfin.LiveTv.Listings public async Task<IEnumerable<ProgramInfo>> GetProgramsAsync(ListingsProviderInfo info, string channelId, DateTime startDateUtc, DateTime endDateUtc, CancellationToken cancellationToken) { + // A failure must not come back as "this channel has no programs": GuideManager only + // skips CleanDatabase when a channel throws, so an empty list deletes the guide. if (IsMetadataLimitActive()) { - return []; + throw new InvalidOperationException("The Schedules Direct daily request limit has been reached."); } ArgumentException.ThrowIfNullOrEmpty(channelId); @@ -107,108 +158,461 @@ namespace Jellyfin.LiveTv.Listings if (string.IsNullOrEmpty(token)) { - _logger.LogWarning("SchedulesDirect token is empty, returning empty program list"); + throw new AuthenticationException("Could not authenticate with Schedules Direct"); + } - return []; + if (!await IsSystemOnline(info, cancellationToken).ConfigureAwait(false)) + { + throw new InvalidOperationException("Schedules Direct reports that the service is offline."); } var dates = GetScheduleRequestDates(startDateUtc, endDateUtc); - _logger.LogInformation("Channel Station ID is: {ChannelID}", channelId); - var requestList = new List<RequestScheduleForChannelDto>() + + // Ask what changed before downloading anything; a day whose md5 still matches is + // served from the cache and costs no schedule or program request. + var (md5s, unavailableDates) = await GetScheduleMd5s(info, channelId, dates, cancellationToken).ConfigureAwait(false); + + var cachedDays = new List<SchedulesDirectCachedDay>(); + var datesToFetch = new List<string>(); + foreach (var date in dates) + { + if (unavailableDates.Contains(date)) { - new() - { - StationId = channelId, - Date = dates - } - }; + // The hash check already said the server has no schedule for this day, so + // requesting it would only earn the same error again. + continue; + } - _logger.LogDebug("Request string for schedules is: {@RequestString}", requestList); + md5s.TryGetValue(date, out var md5); + var cached = await _scheduleCache.GetAsync(channelId, date, md5, cancellationToken).ConfigureAwait(false); + if (cached?.Schedule is not null) + { + cachedDays.Add(cached); + } + else if (!IsScheduleRetryPending(channelId, date)) + { + datesToFetch.Add(date); + } + } - using var options = new HttpRequestMessage(HttpMethod.Post, ApiUrl + "/schedules"); - options.Content = JsonContent.Create(requestList, options: _jsonOptions); - options.Headers.TryAddWithoutValidation("token", token); - var dailySchedules = await Request<IReadOnlyList<DayDto>>(options, true, info, cancellationToken).ConfigureAwait(false); - if (dailySchedules is null) + _logger.LogDebug( + "Channel {ChannelID}: {CachedCount} of {TotalCount} days served from cache", + channelId, + cachedDays.Count, + dates.Count); + + if (datesToFetch.Count > 0) { - return []; + // The hash check may have stopped us (offline, a login limit) or replaced the token, + // and its failure is swallowed, so the schedule request must not go out regardless. + token = await GetToken(info, cancellationToken).ConfigureAwait(false); + if (string.IsNullOrEmpty(token) || IsMetadataLimitActive()) + { + throw new InvalidOperationException("Schedules Direct stopped accepting requests while checking for schedule changes."); + } } - _logger.LogDebug("Found {ScheduleCount} programs on {ChannelID} ScheduleDirect", dailySchedules.Count, channelId); + var freshDays = await DownloadSchedules(info, channelId, datesToFetch, token, cancellationToken).ConfigureAwait(false); - using var programRequestOptions = new HttpRequestMessage(HttpMethod.Post, ApiUrl + "/programs"); - programRequestOptions.Headers.TryAddWithoutValidation("token", token); + // A day that is neither cached nor downloaded (queued, a per-day error, a pending retry) + // must not go missing: the guide refresh would delete its programs. The previous copy + // stands in for it, and without one the whole channel fails. + var coveredDates = cachedDays.Concat(freshDays) + .Select(d => d.Schedule?.Metadata?.StartDate?.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)) + .ToHashSet(StringComparer.Ordinal); + foreach (var date in dates) + { + if (unavailableDates.Contains(date) || coveredDates.Contains(date)) + { + continue; + } + + var stale = await _scheduleCache.GetLatestAsync(channelId, date, cancellationToken).ConfigureAwait(false); + if (stale?.Schedule is null) + { + throw new InvalidOperationException( + string.Format( + CultureInfo.InvariantCulture, + "Could not get the schedule for station {0} on {1}", + channelId, + date)); + } + + _logger.LogDebug("Serving the previous schedule for {StationId} {Date} until a new one is available", channelId, date); + cachedDays.Add(stale); + } - var programIds = dailySchedules.SelectMany(d => d.Programs.Select(s => s.ProgramId)).Distinct(); - programRequestOptions.Content = JsonContent.Create(programIds, options: _jsonOptions); + _scheduleCache.Prune(channelId, dates); - var programDetails = await Request<IReadOnlyList<ProgramDetailsDto>>(programRequestOptions, true, info, cancellationToken).ConfigureAwait(false); - if (programDetails is null) + // Only reachable when the server reported every day as unavailable. + var allDays = cachedDays.Select(c => c.Schedule!).Concat(freshDays.Select(f => f.Schedule!)).ToList(); + if (allDays.Count == 0) { return []; } - var programDict = programDetails.ToDictionary(p => p.ProgramId, y => y); - - var programIdsWithImages = programDetails - .Where(p => p.HasImageArtwork) - .Select(p => p.ProgramId) + var programDict = cachedDays.Concat(freshDays) + .SelectMany(d => d.Programs) + .Where(p => !string.IsNullOrEmpty(p.ProgramId)) + .DistinctBy(p => p.ProgramId) + .ToDictionary(p => p.ProgramId, y => y); + + // Artwork uris are ephemeral and must not be cached, and artwork changes are not + // reflected in the schedule md5 either, so the index is requested fresh - but only for + // the programs whose images are actually going to be pre-cached. + var imageProgramIds = allDays.SelectMany(d => d.Programs) + .Where(s => WillBeImageCached(s) + && !string.IsNullOrEmpty(s.ProgramId) + && programDict.TryGetValue(s.ProgramId, out var detail) + && detail.HasImageArtwork) + .Select(s => s.ProgramId) + .Distinct() .ToList(); - var images = await GetImageForPrograms(info, programIdsWithImages, cancellationToken).ConfigureAwait(false); + var images = await GetImageForPrograms(info, imageProgramIds, cancellationToken).ConfigureAwait(false); var programsInfo = new List<ProgramInfo>(); - foreach (ProgramDto schedule in dailySchedules.SelectMany(d => d.Programs)) + foreach (ProgramDto schedule in allDays.SelectMany(d => d.Programs)) { - if (string.IsNullOrEmpty(schedule.ProgramId)) + if (string.IsNullOrEmpty(schedule.ProgramId) + || !programDict.TryGetValue(schedule.ProgramId, out var programDetail)) { continue; } // Only add images which will be pre-cached until we can implement dynamic token fetching - var endDate = schedule.AirDateTime?.AddSeconds(schedule.Duration); - var willBeCached = endDate.HasValue && endDate.Value < DateTime.UtcNow.AddDays(GuideManager.MaxCacheDays); - if (willBeCached && images is not null) + if (WillBeImageCached(schedule)) { - var imageIndex = images.FindIndex(i => - i.ProgramId is not null && schedule.ProgramId.StartsWith(i.ProgramId, StringComparison.Ordinal)); - if (imageIndex > -1) - { - var programEntry = programDict[schedule.ProgramId]; + SetProgramImages(schedule.ProgramId, programDetail, images, token); + } - var allImages = images[imageIndex].Data; - var imagesWithText = allImages.Where(i => string.Equals(i.Text, "yes", StringComparison.OrdinalIgnoreCase)).ToList(); - var imagesWithoutText = allImages.Where(i => string.Equals(i.Text, "no", StringComparison.OrdinalIgnoreCase)).ToList(); + programsInfo.Add(GetProgram(channelId, schedule, programDetail)); + } - const double DesiredAspect = 2.0 / 3; + return programsInfo; + } - programEntry.PrimaryImage = GetProgramImage(ApiUrl, imagesWithText, DesiredAspect, token) ?? - GetProgramImage(ApiUrl, allImages, DesiredAspect, token); + private async Task<(Dictionary<string, string> Md5s, HashSet<string> UnavailableDates)> GetScheduleMd5s( + ListingsProviderInfo info, + string stationId, + IReadOnlyList<string> dates, + CancellationToken cancellationToken) + { + var result = new Dictionary<string, string>(StringComparer.Ordinal); + var unavailable = new HashSet<string>(StringComparer.Ordinal); + + var token = await GetToken(info, cancellationToken).ConfigureAwait(false); + if (string.IsNullOrEmpty(token)) + { + return (result, unavailable); + } - const double WideAspect = 16.0 / 9; + var requestList = new List<RequestScheduleForChannelDto> + { + new() { StationId = stationId, Date = dates } + }; - programEntry.ThumbImage = GetProgramImage(ApiUrl, imagesWithText, WideAspect, token); + using var message = new HttpRequestMessage(HttpMethod.Post, ApiUrl + "/schedules/md5"); + message.Content = JsonContent.Create(requestList, options: _jsonOptions); + message.Headers.TryAddWithoutValidation("token", token); - // Don't supply the same image twice - if (string.Equals(programEntry.PrimaryImage, programEntry.ThumbImage, StringComparison.Ordinal)) + try + { + var response = await Request<Dictionary<string, Dictionary<string, ScheduleMd5Dto>>>(message, true, info, cancellationToken) + .ConfigureAwait(false); + if (response is null || !response.TryGetValue(stationId, out var days)) + { + return (result, unavailable); + } + + foreach (var (date, entry) in days) + { + if (entry.Code != 0) + { + // Only a definitive "there is no such day" is worth skipping; a transient + // error still gets the normal request, which may well succeed. + if (IsDefinitiveScheduleError(entry.Code)) { - programEntry.ThumbImage = null; + unavailable.Add(date); } - programEntry.BackdropImage = GetProgramImage(ApiUrl, imagesWithoutText, WideAspect, token); + LogEntryErrors([(entry.Code, date, entry.Message)], "schedule"); + } + else if (!string.IsNullOrEmpty(entry.Md5)) + { + result[date] = entry.Md5; + } + } + } + catch (Exception ex) when (ex is HttpRequestException or JsonException) + { + // Without md5s every requested day is treated as changed, which is the old + // behaviour. Nothing is marked unavailable, so no day is skipped on a failure. + _logger.LogWarning(ex, "Unable to get schedule hashes for station {StationId}", stationId); + } + + return (result, unavailable); + } + + // Whether the code means the day will never be available, as opposed to a transient + // failure that is worth another request. + private static bool IsDefinitiveScheduleError(int code) + => code is (int)SdErrorCode.ScheduleRangeExceeded + or (int)SdErrorCode.ScheduleNotInLineup + or (int)SdErrorCode.StationIdNotFound + or (int)SdErrorCode.StationIdDeleted; + + private bool IsScheduleRetryPending(string stationId, string date) + { + var key = stationId + "|" + date; + if (!_scheduleRetryTimes.TryGetValue(key, out var retryTime)) + { + return false; + } + + if (DateTime.UtcNow < retryTime) + { + _logger.LogDebug( + "Schedules Direct has queued {StationId} {Date}; not requesting it before {RetryTime}", + stationId, + date, + retryTime); + return true; + } + + _scheduleRetryTimes.TryRemove(key, out _); + return false; + } + + private static bool WillBeImageCached(ProgramDto schedule) + { + var endDate = schedule.AirDateTime?.AddSeconds(schedule.Duration); + return endDate.HasValue && endDate.Value < DateTime.UtcNow.AddDays(GuideManager.MaxCacheDays); + } + + private async Task<IReadOnlyList<SchedulesDirectCachedDay>> DownloadSchedules( + ListingsProviderInfo info, + string stationId, + IReadOnlyList<string> dates, + string token, + CancellationToken cancellationToken) + { + if (dates.Count == 0) + { + return []; + } + + var requestList = new List<RequestScheduleForChannelDto> + { + new() { StationId = stationId, Date = dates } + }; + + _logger.LogDebug("Request string for schedules is: {@RequestString}", requestList); + + using var options = new HttpRequestMessage(HttpMethod.Post, ApiUrl + "/schedules"); + options.Content = JsonContent.Create(requestList, options: _jsonOptions); + options.Headers.TryAddWithoutValidation("token", token); + + // A station-wide failure such as SCHEDULE_QUEUED arrives as a bare object rather than + // the usual array, so the shape has to be inspected before it is deserialized. + var payload = await Request<JsonElement>(options, true, info, cancellationToken, bodyCodeIsError: false).ConfigureAwait(false); + var dailySchedules = payload.ValueKind switch + { + JsonValueKind.Array => payload.Deserialize<IReadOnlyList<DayDto>>(_jsonOptions), + JsonValueKind.Object => [payload.Deserialize<DayDto>(_jsonOptions)], + _ => null + }; + + if (dailySchedules is null) + { + return []; + } + + var errorDays = dailySchedules.Where(d => d.Code.HasValue).ToList(); + RecordQueuedSchedules(stationId, dates, errorDays); + LogEntryErrors(errorDays.Select(d => (d.Code!.Value, d.StationId ?? stationId, d.Message ?? d.Response)), "schedule"); + var days = dailySchedules.Where(d => !d.Code.HasValue).ToList(); + + _logger.LogDebug("Found {ScheduleCount} days on {ChannelID} ScheduleDirect", days.Count, stationId); - // programEntry.bannerImage = GetProgramImage(ApiUrl, data, "Banner", false) ?? - // GetProgramImage(ApiUrl, data, "Banner-L1", false) ?? - // GetProgramImage(ApiUrl, data, "Banner-LO", false) ?? - // GetProgramImage(ApiUrl, data, "Banner-LOT", false); + var programIds = days.SelectMany(d => d.Programs.Select(s => s.ProgramId)) + .Where(id => !string.IsNullOrEmpty(id)) + .Distinct() + .ToList(); + + var programDetails = await GetProgramDetails(info, programIds, token, cancellationToken).ConfigureAwait(false); + var detailsById = programDetails.ToDictionary(p => p.ProgramId, StringComparer.Ordinal); + + var result = new List<SchedulesDirectCachedDay>(days.Count); + foreach (var day in days) + { + var dayPrograms = day.Programs + .Select(p => p.ProgramId) + .Where(id => !string.IsNullOrEmpty(id) && detailsById.ContainsKey(id)) + .Distinct() + .Select(id => detailsById[id]) + .ToList(); + + var cachedDay = new SchedulesDirectCachedDay + { + Md5 = day.Metadata?.Md5, + Schedule = day, + Programs = dayPrograms + }; + + var date = day.Metadata?.StartDate?.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture); + var missingIds = day.Programs + .Select(p => p.ProgramId) + .Where(id => !string.IsNullOrEmpty(id) && !detailsById.ContainsKey(id)) + .Distinct(StringComparer.Ordinal) + .ToList(); + + if (missingIds.Count == 0) + { + if (date is not null) + { + // Must stay ahead of SetProgramImages: it stamps ephemeral uris and a token + // that expires daily onto these programs, and neither may reach the cache. + await _scheduleCache.SetAsync(stationId, date, cachedDay, cancellationToken).ConfigureAwait(false); } + + result.Add(cachedDay); + continue; } - programsInfo.Add(GetProgram(channelId, schedule, programDict[schedule.ProgramId])); + // A queued program (6001) is not cached: finishing it need not change the day's md5. + // Dropping its airings would delete them, so the previous copy's details stand in; + // without those the day stays uncovered and GetProgramsAsync falls back or fails. + var stale = date is null + ? null + : await _scheduleCache.GetLatestAsync(stationId, date, cancellationToken).ConfigureAwait(false); + var staleById = stale?.Programs + .Where(p => !string.IsNullOrEmpty(p.ProgramId)) + .DistinctBy(p => p.ProgramId, StringComparer.Ordinal) + .ToDictionary(p => p.ProgramId, StringComparer.Ordinal); + if (staleById is not null && missingIds.TrueForAll(staleById.ContainsKey)) + { + cachedDay.Programs = [.. dayPrograms, .. missingIds.Select(id => staleById[id])]; + result.Add(cachedDay); + } } - return programsInfo; + return result; + } + + private void RecordQueuedSchedules(string stationId, IReadOnlyList<string> dates, IReadOnlyList<DayDto> errorDays) + { + // Dates roll out of the guide window without being asked for again, so entries that + // are no longer holding anything back have to be swept here. + var now = DateTime.UtcNow; + foreach (var (key, value) in _scheduleRetryTimes) + { + if (value <= now) + { + _scheduleRetryTimes.TryRemove(key, out _); + } + } + + foreach (var day in errorDays) + { + if (day.Code != (int)SdErrorCode.ScheduleQueued || day.RetryTime is not { } retryTime) + { + continue; + } + + // The response is station-wide and names no date, so it covers everything asked for. + _logger.LogInformation( + "Schedules Direct has queued the schedule for station {StationId}; retrying after {RetryTime}", + day.StationId ?? stationId, + retryTime); + + foreach (var date in dates) + { + _scheduleRetryTimes[(day.StationId ?? stationId) + "|" + date] = retryTime.ToUniversalTime(); + } + } + } + + private async Task<List<ProgramDetailsDto>> GetProgramDetails( + ListingsProviderInfo info, + IReadOnlyList<string> programIds, + string token, + CancellationToken cancellationToken) + { + var results = new List<ProgramDetailsDto>(programIds.Count); + + for (var i = 0; i < programIds.Count; i += MaxProgramsPerRequest) + { + var batch = programIds.Skip(i).Take(MaxProgramsPerRequest); + + using var message = new HttpRequestMessage(HttpMethod.Post, ApiUrl + "/programs"); + message.Headers.TryAddWithoutValidation("token", token); + message.Content = JsonContent.Create(batch, options: _jsonOptions); + + var batchResult = await Request<IReadOnlyList<ProgramDetailsDto>>(message, true, info, cancellationToken).ConfigureAwait(false); + if (batchResult is null) + { + continue; + } + + LogEntryErrors(batchResult.Where(p => p.Code.HasValue).Select(p => (p.Code!.Value, p.ProgramId, p.Message)), "program"); + + // Error entries carry no program data. A queued program (6001) is a soft failure, + // so it is left out, and its day is not cached so the next refresh asks again. + results.AddRange(batchResult.Where(p => !p.Code.HasValue && !string.IsNullOrEmpty(p.ProgramId))); + } + + return results.DistinctBy(p => p.ProgramId, StringComparer.Ordinal).ToList(); + } + + private void SetProgramImages(string programId, ProgramDetailsDto programEntry, IReadOnlyList<ShowImagesDto> images, string token) + { + var match = images.FirstOrDefault(i => + i.ProgramId is not null && programId.StartsWith(i.ProgramId, StringComparison.Ordinal)); + if (match is null) + { + return; + } + + var allImages = match.Data; + var imagesWithText = allImages.Where(i => string.Equals(i.Text, "yes", StringComparison.OrdinalIgnoreCase)).ToList(); + var imagesWithoutText = allImages.Where(i => string.Equals(i.Text, "no", StringComparison.OrdinalIgnoreCase)).ToList(); + + const double DesiredAspect = 2.0 / 3; + const double WideAspect = 16.0 / 9; + + programEntry.PrimaryImage = GetProgramImage(ApiUrl, imagesWithText, DesiredAspect, token) ?? + GetProgramImage(ApiUrl, allImages, DesiredAspect, token); + + programEntry.ThumbImage = GetProgramImage(ApiUrl, imagesWithText, WideAspect, token); + + // Don't supply the same image twice + if (string.Equals(programEntry.PrimaryImage, programEntry.ThumbImage, StringComparison.Ordinal)) + { + programEntry.ThumbImage = null; + } + + programEntry.BackdropImage = GetProgramImage(ApiUrl, imagesWithoutText, WideAspect, token); + } + + private void LogEntryErrors(IEnumerable<(int Code, string Id, string Message)> errors, string kind) + { + foreach (var (code, id, message) in errors) + { + _logger.LogWarning( + "Schedules Direct returned an error for {Kind} {Id}: code={Code}, message={Message}", + kind, + id, + code, + message); + + var sdCode = ToKnownErrorCode(code); + if (sdCode.HasValue) + { + ApplyErrorCode(sdCode.Value); + } + } } private static int GetSizeOrder(ImageDataDto image) @@ -299,7 +703,6 @@ namespace Jellyfin.LiveTv.Listings CommunityRating = null, EpisodeTitle = episodeTitle, Audio = audioType, - // IsNew = programInfo.@new ?? false, IsRepeat = programInfo.New is null, IsSeries = string.Equals(details.EntityType, "episode", StringComparison.OrdinalIgnoreCase), ImageUrl = details.PrimaryImage, @@ -417,9 +820,12 @@ namespace Jellyfin.LiveTv.Listings return info; } - private static string GetProgramImage(string apiUrl, IEnumerable<ImageDataDto> images, double desiredAspect, string token) + private string GetProgramImage(string apiUrl, IEnumerable<ImageDataDto> images, double desiredAspect, string token) { + // A uri Schedules Direct has already rejected must never be offered again; repeating + // the request is what trips MAX_IMAGE_INVALID_URI_ERRORS and blocks the account. var match = images + .Where(i => !string.IsNullOrWhiteSpace(i.Uri) && !_rejectedImageUris.ContainsKey(GetImageKey(i.Uri))) .OrderBy(i => Math.Abs(desiredAspect - GetAspectRatio(i))) .ThenByDescending(i => GetSizeOrder(i)) .FirstOrDefault(); @@ -431,11 +837,6 @@ namespace Jellyfin.LiveTv.Listings var uri = match.Uri; - if (string.IsNullOrWhiteSpace(uri)) - { - return null; - } - if (uri.Contains("http", StringComparison.OrdinalIgnoreCase)) { return uri; @@ -444,6 +845,20 @@ namespace Jellyfin.LiveTv.Listings return apiUrl + "/image/" + uri + "?token=" + token; } + private static string GetImageKey(string uriOrUrl) + { + var value = uriOrUrl; + + var imageSegment = value.IndexOf("/image/", StringComparison.OrdinalIgnoreCase); + if (imageSegment >= 0) + { + value = value[(imageSegment + 7)..]; + } + + var query = value.IndexOf('?', StringComparison.Ordinal); + return query < 0 ? value : value[..query]; + } + private static double GetAspectRatio(ImageDataDto i) { int width = 0; @@ -554,6 +969,13 @@ namespace Jellyfin.LiveTv.Listings throw new AuthenticationException("Could not authenticate with Schedules Direct"); } + // The spec puts the status check ahead of all processing, and an empty lineup list is + // indistinguishable from "nothing for this postal code", so this has to surface. + if (!await IsSystemOnline(info, cancellationToken).ConfigureAwait(false)) + { + throw new InvalidOperationException("Schedules Direct reports that the service is offline. Try again later."); + } + var lineups = new List<NameIdPair>(); using var options = new HttpRequestMessage(HttpMethod.Get, ApiUrl + "/headends?country=" + country + "&postalcode=" + location); @@ -586,7 +1008,9 @@ namespace Jellyfin.LiveTv.Listings private void ResetErrorState(ListingsProviderInfo info) { _accountError = false; - Interlocked.Exchange(ref _lastErrorResponseTicks, 0); + Interlocked.Exchange(ref _loginBackoffUntilTicks, 0); + Interlocked.Exchange(ref _offlineUntilTicks, 0); + _status = null; // Only the account being saved is retried, the tokens of the other accounts stay valid. if (!string.IsNullOrWhiteSpace(info.Username)) @@ -619,10 +1043,21 @@ namespace Jellyfin.LiveTv.Listings return null; } - // Avoid hammering SD after transient login failures (e.g. max attempts / temporary lockout) - if ((DateTime.UtcNow - new DateTime(Interlocked.Read(ref _lastErrorResponseTicks), DateTimeKind.Utc)).TotalMinutes < 30) + // Avoid hammering SD after login failures (temporary lockout, too many logins). + var backoffUntil = Interlocked.Read(ref _loginBackoffUntilTicks); + if (DateTime.UtcNow.Ticks < backoffUntil) + { + _logger.LogWarning( + "Skipping Schedules Direct request because of a recent login failure. Retrying no earlier than {RetryAt}.", + new DateTime(backoffUntil, DateTimeKind.Utc)); + + return null; + } + + // A login during the offline hour counts towards MAX_LOGIN_ATTEMPTS and cannot succeed. + if (IsOfflineBackoffActive()) { - _logger.LogWarning("Skipping Schedules Direct request because of a recent login failure. Retrying no earlier than 30 minutes after it."); + _logger.LogWarning("Skipping Schedules Direct request, the service reported itself offline"); return null; } @@ -663,7 +1098,7 @@ namespace Jellyfin.LiveTv.Listings && (int)ex.StatusCode.Value < 500) { _tokens.Clear(); - Interlocked.Exchange(ref _lastErrorResponseTicks, DateTime.UtcNow.Ticks); + BackOffLogins(TimeSpan.FromMinutes(LoginBackoffMinutes)); } throw; @@ -671,86 +1106,202 @@ namespace Jellyfin.LiveTv.Listings } } + private async Task<bool> IsSystemOnline(ListingsProviderInfo info, CancellationToken cancellationToken) + { + if (IsOfflineBackoffActive()) + { + _logger.LogWarning("Skipping Schedules Direct request, the service reported itself offline"); + return false; + } + + var status = await GetStatus(info, cancellationToken).ConfigureAwait(false); + if (status is null) + { + // A failed status check is not a reason to skip the refresh on its own, unless it + // failed with SERVICE_OFFLINE. + return !IsOfflineBackoffActive(); + } + + // The spec does not define the order of the entries, so any of them can be the one + // that takes the service down. + var offline = status.SystemStatus.FirstOrDefault( + i => string.Equals(i.Status, "Offline", StringComparison.OrdinalIgnoreCase)); + if (offline is not null) + { + _logger.LogWarning( + "Schedules Direct reports the service is offline ({Message}). Not retrying for {Minutes} minutes.", + offline.Message, + OfflineBackoffMinutes); + Interlocked.Exchange(ref _offlineUntilTicks, DateTime.UtcNow.AddMinutes(OfflineBackoffMinutes).Ticks); + return false; + } + + if (status.Account?.Expires is { } expires && expires < DateTime.UtcNow.AddDays(7)) + { + _logger.LogWarning("The Schedules Direct subscription expires on {Expires}", expires); + } + + return true; + } + + private async Task<StatusDto> GetStatus(ListingsProviderInfo info, CancellationToken cancellationToken) + { + if (TryGetCachedStatus(out var cached)) + { + return cached; + } + + using (await _statusLock.LockAsync(cancellationToken).ConfigureAwait(false)) + { + // Whoever held the lock may have just refreshed it, and a guide refresh asking + // once per channel must not turn into one request per channel. + if (TryGetCachedStatus(out cached)) + { + return cached; + } + + StatusDto status = null; + try + { + var token = await GetToken(info, cancellationToken).ConfigureAwait(false); + + using var message = new HttpRequestMessage(HttpMethod.Get, ApiUrl + "/status"); + message.Headers.TryAddWithoutValidation("token", token); + + status = await Request<StatusDto>(message, true, info, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is HttpRequestException or JsonException or AuthenticationException) + { + _logger.LogWarning(ex, "Unable to get the Schedules Direct service status"); + } + + // A failed check still counts as checked, or every channel retries it. + _status = new StatusCacheEntry(DateTime.UtcNow, status); + return status; + } + } + + private bool TryGetCachedStatus(out StatusDto status) + { + // One reference, so the time of the check and what it found can never disagree. + var entry = _status; + status = entry?.Status; + return entry is not null && (DateTime.UtcNow - entry.CheckedAt).TotalMinutes < StatusCacheMinutes; + } + + private bool ApplyErrorCode(SdErrorCode sdCode) + { + switch (sdCode) + { + case SdErrorCode.AccountExpired: + case SdErrorCode.InvalidHash: + case SdErrorCode.InvalidUser: + case SdErrorCode.AccountLocked: + case SdErrorCode.AppLocked: + case SdErrorCode.AccountInactive: + // Permanent account errors — disable SD for this server lifetime. + _logger.LogError("Schedules Direct account error (code {SdCode}). Disabling SD until server restart.", sdCode); + _tokens.Clear(); + _accountError = true; + return true; + + case SdErrorCode.ServiceOffline: + // The spec requires staying away for an hour once the service reports offline. + _logger.LogError("Schedules Direct is offline (code {SdCode}). Not retrying for {Minutes} minutes.", sdCode, OfflineBackoffMinutes); + _tokens.Clear(); + Interlocked.Exchange(ref _offlineUntilTicks, DateTime.UtcNow.AddMinutes(OfflineBackoffMinutes).Ticks); + return true; + + case SdErrorCode.ServiceBusy: + case SdErrorCode.AccountTempLock: + // Transient login errors — back off for 30 minutes, then allow retry. + _logger.LogError("Schedules Direct transient error (code {SdCode}). Backing off for {Minutes} minutes.", sdCode, LoginBackoffMinutes); + _tokens.Clear(); + BackOffLogins(TimeSpan.FromMinutes(LoginBackoffMinutes)); + return true; + + case SdErrorCode.MaxLoginAttempts: + // Counts logins, so asking for a token again is what earned it. SD support confirmed it + // clears 24 hours after the first login; waiting 24 hours from now is always past that. + _logger.LogError( + "Schedules Direct login limit reached (code {SdCode}). Not logging in again for {Hours} hours.", + sdCode, + MaxLoginsBackoffHours); + _tokens.Clear(); + BackOffLogins(TimeSpan.FromHours(MaxLoginsBackoffHours)); + SetImageLimitHit(); + SetMetadataLimitHit(); + return true; + + case SdErrorCode.MaxIPAttempts: + // Unconfirmed whether this one self-heals, so it stays disabled like an account error. + _logger.LogError( + "Schedules Direct unique IP limit reached (code {SdCode}). Disabling SD until server restart; the user has to contact Schedules Direct support.", + sdCode); + _tokens.Clear(); + _accountError = true; + SetImageLimitHit(); + SetMetadataLimitHit(); + return true; + + case SdErrorCode.MaxImageDownloads: + case SdErrorCode.MaxImageDownloadsTrial: + case SdErrorCode.MaxInvalidImages: + // Stop image requests until SD resets at 00:00 UTC. + _logger.LogError("Schedules Direct image download limit hit (code {SdCode}). Disabling image acquisition until SD reset.", sdCode); + SetImageLimitHit(); + return true; + + case SdErrorCode.MaxLineupChanges: + // Only blocks further lineup edits; guide data is unaffected. + _logger.LogError("Schedules Direct lineup change limit reached (code {SdCode}). Lineup edits are rejected until SD reset.", sdCode); + return true; + + default: + return false; + } + } + private async Task<T> Request<T>( HttpRequestMessage message, bool enableRetry, ListingsProviderInfo providerInfo, CancellationToken cancellationToken, - HttpCompletionOption completionOption = HttpCompletionOption.ResponseContentRead) + bool bodyCodeIsError = true) { using var response = await _httpClientFactory.CreateClient(NamedClient.Default) - .SendAsync(message, completionOption, cancellationToken) + .SendAsync(message, HttpCompletionOption.ResponseContentRead, cancellationToken) .ConfigureAwait(false); - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync<T>(_jsonOptions, cancellationToken).ConfigureAwait(false); - } - var responseBody = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); - // Try to extract the Schedules Direct error code from the response body. - SdErrorCode? sdCode = null; - try + // SD can report a failure as HTTP 200 with a non-zero code, which would otherwise + // deserialize into an empty object and read as success. + var errorCode = TryGetErrorCode(responseBody); + if (response.IsSuccessStatusCode && (!bodyCodeIsError || errorCode is null or 0)) { - using var doc = JsonDocument.Parse(responseBody); - if (doc.RootElement.TryGetProperty("code", out var codeProp) - && codeProp.TryGetInt32(out var parsedCode) - && Enum.IsDefined((SdErrorCode)parsedCode)) - { - sdCode = (SdErrorCode)parsedCode; - } - } - catch (JsonException) - { - // Response body is not valid JSON; sdCode stays null. + return JsonSerializer.Deserialize<T>(responseBody, _jsonOptions); } + var sdCode = ToKnownErrorCode(errorCode); + _logger.LogError( "Request to {Url} failed with HTTP {StatusCode}, SD code {SdCode}: {Response}", message.RequestUri, (int)response.StatusCode, - sdCode?.ToString() ?? "N/A", + sdCode?.ToString() ?? errorCode?.ToString(CultureInfo.InvariantCulture) ?? "N/A", responseBody); - if (sdCode is SdErrorCode.AccountExpired or SdErrorCode.InvalidHash or SdErrorCode.InvalidUser or SdErrorCode.AccountLocked or SdErrorCode.AppLocked or SdErrorCode.AccountInactive) - { - // Permanent account errors — disable SD for this server lifetime. - _logger.LogError("Schedules Direct account error (code {SdCode}). Disabling SD until server restart.", sdCode); - _tokens.Clear(); - _accountError = true; - } - else if (sdCode is SdErrorCode.ServiceOffline or SdErrorCode.ServiceBusy or SdErrorCode.AccountTempLock) - { - // Transient login errors — back off for 30 minutes, then allow retry. - _logger.LogError("Schedules Direct transient error (code {SdCode}). Backing off for 30 minutes.", sdCode); - _tokens.Clear(); - Interlocked.Exchange(ref _lastErrorResponseTicks, DateTime.UtcNow.Ticks); - } - else if (sdCode is SdErrorCode.MaxLoginAttempts or SdErrorCode.MaxIPAttempts) - { - // 24 hour bans - stop image and metadata requests until SD reset at 00:00 UTC. - _logger.LogError("Schedules Direct service limit error (code {SdCode}). Disabling until SD reset.", sdCode); - SetImageLimitHit(); - SetMetadataLimitHit(); - } - else if (sdCode is SdErrorCode.MaxImageDownloads or SdErrorCode.MaxImageDownloadsTrial) + if (sdCode.HasValue && ApplyErrorCode(sdCode.Value)) { - // Max image downloads — stop image requests until SD resets at 00:00 UTC. - _logger.LogError("Schedules Direct image download limit hit (code {SdCode}). Disabling image acquisition until SD reset.", sdCode); - SetImageLimitHit(); - } - else if (sdCode is SdErrorCode.MaxScheduleRequests) - { - // Max schedule/metadata requests — stop metadata requests until SD resets at 00:00 UTC. - _logger.LogError("Schedules Direct metadata download limit hit (code {SdCode}). Disabling metadata acquisition until SD reset.", sdCode); - SetMetadataLimitHit(); + // Handled by the error code. } else if (enableRetry && (int)response.StatusCode < 500 - && (sdCode == SdErrorCode.TokenExpired || (response.StatusCode == HttpStatusCode.Forbidden && sdCode is null))) + && (sdCode is SdErrorCode.TokenExpired or SdErrorCode.TokenInvalid or SdErrorCode.TokenMissing + || (response.StatusCode == HttpStatusCode.Forbidden && errorCode is null))) { - // Token expired — clear tokens and retry with a fresh token. - // Also retry on 403 with no parseable SD code (legacy/unexpected auth failure). + // Token rejected — clear tokens and retry with a fresh token. Also retry on 403 + // carrying no code at all; a code we simply do not know is not an auth failure. _tokens.Clear(); using var retryMessage = new HttpRequestMessage(message.Method, message.RequestUri); retryMessage.Content = message.Content; @@ -807,9 +1358,26 @@ namespace Jellyfin.LiveTv.Listings if (!response.IsSuccessStatusCode) { + var responseBody = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + var errorCode = TryGetErrorCode(responseBody); + var sdCode = ToKnownErrorCode(errorCode); + if (sdCode.HasValue) + { + ApplyErrorCode(sdCode.Value); + } + _logger.LogError( - "Error adding lineup to account: {Response}", - await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false)); + "Error adding lineup {Id} to account, SD code {SdCode}: {Response}", + info.ListingsId, + sdCode?.ToString() ?? errorCode?.ToString(CultureInfo.InvariantCulture) ?? "N/A", + responseBody); + + // Reporting success here leaves the user with a listings provider that silently + // has no lineup, so the failure has to surface. + throw new HttpRequestException( + string.Format(CultureInfo.InvariantCulture, "Could not add lineup {0} to the Schedules Direct account", info.ListingsId), + null, + response.StatusCode); } } @@ -913,23 +1481,225 @@ namespace Jellyfin.LiveTv.Listings } /// <inheritdoc /> - public bool IsServiceAvailable() + public bool CanDownloadImage(string imageUrl) + => !IsSchedulesDirectUrl(imageUrl) + || (!IsImageDailyLimitActive() && !_rejectedImageUris.ContainsKey(GetImageKey(imageUrl))); + + /// <inheritdoc /> + public void ReportImageDownloadSuccess(string imageUrl) { - if (_accountError) + if (IsSchedulesDirectUrl(imageUrl)) { - return false; + Interlocked.Exchange(ref _consecutiveImageFailures, 0); } + } - if ((DateTime.UtcNow - new DateTime(Interlocked.Read(ref _lastErrorResponseTicks), DateTimeKind.Utc)).TotalMinutes < 30) + /// <inheritdoc /> + public ImageDownloadFailureAction ReportImageDownloadFailure(string imageUrl, HttpStatusCode? statusCode, string responseBody) + { + if (!IsSchedulesDirectUrl(imageUrl)) { - return false; + return ImageDownloadFailureAction.None; } - return true; + var errorCode = TryGetErrorCode(responseBody); + if (errorCode is null) + { + // No readable code, so stop after a short streak of unexplained rejections rather + // than keep retrying an account that may already be refusing us. + if (Interlocked.Increment(ref _consecutiveImageFailures) == MaxConsecutiveImageFailures) + { + _logger.LogError( + "Schedules Direct image downloads keep failing without a readable error code (last status {StatusCode}). Disabling image acquisition until SD reset.", + statusCode); + SetImageLimitHit(); + } + + return ImageDownloadFailureAction.None; + } + + Interlocked.Exchange(ref _consecutiveImageFailures, 0); + + var sdCode = ToKnownErrorCode(errorCode); + if (sdCode is null) + { + return ImageDownloadFailureAction.None; + } + + // SD reports an exhausted image quota with an HTTP 200 body, so the code decides. + ApplyErrorCode(sdCode.Value); + + switch (sdCode.Value) + { + // The uri is gone. Remembering it is what keeps the next refresh from asking for + // it again and walking the account into MAX_IMAGE_INVALID_URI_ERRORS. + case SdErrorCode.ImageNotFound: + RejectImageUri(imageUrl); + return ImageDownloadFailureAction.RemoveImage; + + // The account is already being warned about invalid uris; this url is not + // necessarily one of them, so drop it for now but do not blacklist it. + case SdErrorCode.MaxInvalidImages: + return ImageDownloadFailureAction.RemoveImage; + + // The token baked into the url is stale; the next refresh mints a fresh one. + case SdErrorCode.TokenMissing: + case SdErrorCode.TokenInvalid: + case SdErrorCode.TokenExpired: + _tokens.Clear(); + return ImageDownloadFailureAction.RemoveImage; + + default: + return ImageDownloadFailureAction.None; + } } /// <inheritdoc /> - public bool IsImageDailyLimitActive() + public async Task<ItemImageInfo> ConvertImageToLocal(BaseItem item, ItemImageInfo image, int imageIndex) + { + ArgumentNullException.ThrowIfNull(image); + + var remotePath = image.Path; + if (!CanDownloadImage(remotePath)) + { + return null; + } + + try + { + var localImage = await _libraryManager.ConvertImageToLocal(item, image, imageIndex).ConfigureAwait(false); + ReportImageDownloadSuccess(remotePath); + return localImage; + } + catch (Exception ex) when ((ex as HttpRequestException ?? ex.InnerException as HttpRequestException) is { } httpException) + { + ReportImageDownloadFailure(remotePath, httpException.StatusCode, (httpException as HttpResponseBodyException)?.ResponseBody); + throw; + } + } + + private void RejectImageUri(string imageUrl) + { + var key = GetImageKey(imageUrl); + if (string.IsNullOrEmpty(key)) + { + return; + } + + // Stop learning rather than forgetting: clearing here would re-offer every known-bad + // uri at once, which is the burst that trips MAX_IMAGE_INVALID_URI_ERRORS. + if (_rejectedImageUris.Count < MaxRejectedImageUris && _rejectedImageUris.TryAdd(key, 0)) + { + // Persisted, or a restart would offer every rejected uri again in one refresh. + var line = DateTime.UtcNow.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture) + "\t" + key + Environment.NewLine; + try + { + lock (_rejectedImageUrisFileLock) + { + Directory.CreateDirectory(Path.GetDirectoryName(RejectedImageUrisFilePath)!); + File.AppendAllText(RejectedImageUrisFilePath, line); + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + _logger.LogWarning(ex, "Failed to persist rejected SD image uri to {Path}", RejectedImageUrisFilePath); + } + } + } + + private void LoadRejectedImageUris() + { + var path = RejectedImageUrisFilePath; + if (!File.Exists(path)) + { + return; + } + + try + { + var cutoff = DateOnly.FromDateTime(DateTime.UtcNow.AddDays(-RejectedImageUriMaxAgeDays)); + var kept = new List<string>(); + var lines = File.ReadAllLines(path); + foreach (var line in lines) + { + var separator = line.IndexOf('\t', StringComparison.Ordinal); + if (separator <= 0 + || !DateOnly.TryParseExact(line.AsSpan(0, separator), "yyyy-MM-dd", CultureInfo.InvariantCulture, DateTimeStyles.None, out var date) + || date < cutoff + || _rejectedImageUris.Count >= MaxRejectedImageUris) + { + continue; + } + + if (_rejectedImageUris.TryAdd(line[(separator + 1)..], 0)) + { + kept.Add(line); + } + } + + if (kept.Count != lines.Length) + { + File.WriteAllLines(path, kept); + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + _logger.LogWarning(ex, "Failed to load rejected SD image uris from {Path}", path); + } + } + + private static bool IsSchedulesDirectUrl(string url) + => url is not null && url.Contains("schedulesdirect", StringComparison.OrdinalIgnoreCase); + + private static int? TryGetErrorCode(string responseBody) + { + // Only an object can carry a code; this also spares parsing large program arrays twice. + if (string.IsNullOrWhiteSpace(responseBody) || !responseBody.AsSpan().TrimStart().StartsWith('{')) + { + return null; + } + + try + { + using var doc = JsonDocument.Parse(responseBody); + if (doc.RootElement.ValueKind == JsonValueKind.Object + && doc.RootElement.TryGetProperty("code", out var codeProp) + && codeProp.TryGetInt32(out var parsedCode)) + { + return parsedCode; + } + } + catch (JsonException) + { + // Not an SD error payload. + } + + return null; + } + + private bool IsOfflineBackoffActive() + => DateTime.UtcNow.Ticks < Interlocked.Read(ref _offlineUntilTicks); + + private void BackOffLogins(TimeSpan duration) + { + // Keep the later deadline, so a generic 30 minute back-off cannot shorten the 24 hours of MAX_LOGIN_ATTEMPTS. + var until = DateTime.UtcNow.Add(duration).Ticks; + long current; + do + { + current = Interlocked.Read(ref _loginBackoffUntilTicks); + if (current >= until) + { + return; + } + } + while (Interlocked.CompareExchange(ref _loginBackoffUntilTicks, until, current) != current); + } + + private static SdErrorCode? ToKnownErrorCode(int? code) + => code.HasValue && Enum.IsDefined((SdErrorCode)code.Value) ? (SdErrorCode)code.Value : null; + + private bool IsImageDailyLimitActive() { if (!_imageLimitHitDate.HasValue) { @@ -939,6 +1709,7 @@ namespace Jellyfin.LiveTv.Listings if (_imageLimitHitDate.Value < DateOnly.FromDateTime(DateTime.UtcNow)) { _imageLimitHitDate = null; + Interlocked.Exchange(ref _consecutiveImageFailures, 0); TryDeleteFile(ImageLimitFilePath); return false; } @@ -1016,6 +1787,11 @@ namespace Jellyfin.LiveTv.Listings } } + if (!await IsSystemOnline(info, CancellationToken.None).ConfigureAwait(false)) + { + throw new InvalidOperationException("Schedules Direct reports that the service is offline. Try again later."); + } + if (validateListings) { ArgumentException.ThrowIfNullOrEmpty(info.ListingsId); @@ -1042,20 +1818,27 @@ namespace Jellyfin.LiveTv.Listings return []; } + // An empty channel list maps no channel, which the guide refresh reads as "no programs" + // and deletes the stored ones, so a failure has to surface here as well. var token = await GetToken(info, cancellationToken).ConfigureAwait(false); if (string.IsNullOrEmpty(token)) { - return []; + throw new AuthenticationException("Could not authenticate with Schedules Direct"); } - using var options = new HttpRequestMessage(HttpMethod.Get, ApiUrl + "/lineups/" + listingsId); - options.Headers.TryAddWithoutValidation("token", token); + if (!await IsSystemOnline(info, cancellationToken).ConfigureAwait(false)) + { + throw new InvalidOperationException("Schedules Direct reports that the service is offline."); + } - var root = await Request<ChannelDto>(options, true, info, cancellationToken).ConfigureAwait(false); + // Stations that have left every lineup would otherwise keep their schedules forever. + _scheduleCache.PruneStale(TimeSpan.FromDays(ScheduleCacheMaxAgeDays)); + + var root = await GetLineup(info, listingsId, cancellationToken).ConfigureAwait(false); if (root is null) { - return new List<ChannelInfo>(); + return []; } _logger.LogInformation("Found {ChannelCount} channels on the lineup on ScheduleDirect", root.Map.Count); @@ -1093,6 +1876,44 @@ namespace Jellyfin.LiveTv.Listings return list; } + private async Task<ChannelDto> GetLineup(ListingsProviderInfo info, string listingsId, CancellationToken cancellationToken) + { + var status = await GetStatus(info, cancellationToken).ConfigureAwait(false); + var lineup = status?.Lineups.FirstOrDefault( + i => string.Equals(i.Lineup ?? i.Id, listingsId, StringComparison.OrdinalIgnoreCase)); + + if (lineup?.IsDeleted == true) + { + _logger.LogWarning( + "The Schedules Direct lineup {ListingsId} has been deleted at the headend. Pick a new lineup for this listings provider.", + listingsId); + } + + var modified = lineup?.Modified; + if (modified.HasValue) + { + var cached = await _lineupCache.GetAsync(listingsId, modified.Value, cancellationToken).ConfigureAwait(false); + if (cached is not null) + { + _logger.LogDebug("Serving the Schedules Direct lineup {ListingsId} from cache", listingsId); + return cached; + } + } + + var token = await GetToken(info, cancellationToken).ConfigureAwait(false); + + using var options = new HttpRequestMessage(HttpMethod.Get, ApiUrl + "/lineups/" + listingsId); + options.Headers.TryAddWithoutValidation("token", token); + + var root = await Request<ChannelDto>(options, true, info, cancellationToken).ConfigureAwait(false); + if (root is not null && modified.HasValue) + { + await _lineupCache.SetAsync(listingsId, modified.Value, root, cancellationToken).ConfigureAwait(false); + } + + return root; + } + /// <inheritdoc /> public void Dispose() { @@ -1114,9 +1935,12 @@ namespace Jellyfin.LiveTv.Listings if (disposing) { _tokenLock?.Dispose(); + _statusLock?.Dispose(); } _disposed = true; } + + private sealed record StatusCacheEntry(DateTime CheckedAt, StatusDto Status); } } diff --git a/src/Jellyfin.LiveTv/Listings/SchedulesDirectCachedDay.cs b/src/Jellyfin.LiveTv/Listings/SchedulesDirectCachedDay.cs new file mode 100644 index 0000000000..c5dae1682e --- /dev/null +++ b/src/Jellyfin.LiveTv/Listings/SchedulesDirectCachedDay.cs @@ -0,0 +1,29 @@ +using System.Collections.Generic; +using Jellyfin.LiveTv.Listings.SchedulesDirectDtos; + +namespace Jellyfin.LiveTv.Listings; + +/// <summary> +/// One station's schedule for one day, together with the details of its programs. +/// </summary> +/// <remarks> +/// Artwork is deliberately not cached: the spec states the image uri is ephemeral and must not be +/// stored, and artwork changes are not reflected in the schedule md5 this entry is keyed by. +/// </remarks> +public class SchedulesDirectCachedDay +{ + /// <summary> + /// Gets or sets the md5 Schedules Direct reported for this day's schedule. + /// </summary> + public string? Md5 { get; set; } + + /// <summary> + /// Gets or sets the schedule. + /// </summary> + public DayDto? Schedule { get; set; } + + /// <summary> + /// Gets or sets the details of the programs in the schedule. + /// </summary> + public IReadOnlyList<ProgramDetailsDto> Programs { get; set; } = []; +} diff --git a/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/DayDto.cs b/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/DayDto.cs index 094f9a3194..4959e12675 100644 --- a/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/DayDto.cs +++ b/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/DayDto.cs @@ -22,6 +22,35 @@ namespace Jellyfin.LiveTv.Listings.SchedulesDirectDtos public IReadOnlyList<ProgramDto> Programs { get; set; } = Array.Empty<ProgramDto>(); /// <summary> + /// Gets or sets the error code. Set when this entry is a per-station error + /// (for example SCHEDULE_RANGE_EXCEEDED) instead of a schedule. + /// </summary> + [JsonPropertyName("code")] + public int? Code { get; set; } + + /// <summary> + /// Gets or sets the error message that goes with <see cref="Code"/>. + /// </summary> + /// <remarks> + /// Not every error entry carries one; STATIONID_DELETED only sets <see cref="Response"/>. + /// </remarks> + [JsonPropertyName("message")] + public string? Message { get; set; } + + /// <summary> + /// Gets or sets the symbolic name of <see cref="Code"/>, for example STATIONID_DELETED. + /// </summary> + [JsonPropertyName("response")] + public string? Response { get; set; } + + /// <summary> + /// Gets or sets the earliest time a queued schedule (SCHEDULE_QUEUED) may be requested + /// again. The spec requires the client to wait at least this long. + /// </summary> + [JsonPropertyName("retryTime")] + public DateTime? RetryTime { get; set; } + + /// <summary> /// Gets or sets the metadata schedule. /// </summary> [JsonPropertyName("metadata")] diff --git a/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/ProgramDetailsDto.cs b/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/ProgramDetailsDto.cs index 7bfc4bc8be..0b883e44aa 100644 --- a/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/ProgramDetailsDto.cs +++ b/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/ProgramDetailsDto.cs @@ -22,6 +22,19 @@ namespace Jellyfin.LiveTv.Listings.SchedulesDirectDtos public string? ProgramId { get; set; } /// <summary> + /// Gets or sets the error code. Set when this entry is a per-program error + /// (for example INVALID_PROGRAMID) instead of a program. + /// </summary> + [JsonPropertyName("code")] + public int? Code { get; set; } + + /// <summary> + /// Gets or sets the error message that goes with <see cref="Code"/>. + /// </summary> + [JsonPropertyName("message")] + public string? Message { get; set; } + + /// <summary> /// Gets or sets the list of titles. /// </summary> [JsonPropertyName("titles")] diff --git a/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/ScheduleMd5Dto.cs b/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/ScheduleMd5Dto.cs new file mode 100644 index 0000000000..6acd2172a9 --- /dev/null +++ b/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/ScheduleMd5Dto.cs @@ -0,0 +1,34 @@ +using System; +using System.Text.Json.Serialization; + +namespace Jellyfin.LiveTv.Listings.SchedulesDirectDtos; + +/// <summary> +/// The md5 and last modified date of one station's schedule for one day. +/// </summary> +public class ScheduleMd5Dto +{ + /// <summary> + /// Gets or sets the error code. Non-zero when the day is unavailable. + /// </summary> + [JsonPropertyName("code")] + public int Code { get; set; } + + /// <summary> + /// Gets or sets the message that goes with <see cref="Code"/>. + /// </summary> + [JsonPropertyName("message")] + public string? Message { get; set; } + + /// <summary> + /// Gets or sets the time the schedule was last modified. + /// </summary> + [JsonPropertyName("lastModified")] + public DateTime? LastModified { get; set; } + + /// <summary> + /// Gets or sets the md5 of the schedule. A change means the schedule must be downloaded again. + /// </summary> + [JsonPropertyName("md5")] + public string? Md5 { get; set; } +} diff --git a/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/SdErrorCode.cs b/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/SdErrorCode.cs index fffbfb9a58..b44ddc2bb4 100644 --- a/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/SdErrorCode.cs +++ b/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/SdErrorCode.cs @@ -1,19 +1,161 @@ -#pragma warning disable CA1008 // Enums should have zero value - namespace Jellyfin.LiveTv.Listings.SchedulesDirectDtos; /// <summary> -/// Schedules Direct API error codes. See https://github.com/SchedulesDirect/JSON-Service/wiki/API-20141201#error-response for details. +/// Schedules Direct API error codes. See https://github.com/SchedulesDirect/JSON-Service/wiki/API-20141201#error-codes for details. /// </summary> +/// <remarks> +/// A complete mirror of the documented table. Codes the client does not act on are still listed, +/// because membership is what lets a response be logged by name instead of by number. +/// </remarks> public enum SdErrorCode { /// <summary> - /// Schedules Direct unavailable/out of service. + /// No error. + /// </summary> + Ok = 0, + + /// <summary> + /// Unable to decode JSON. + /// </summary> + InvalidJson = 1001, + + /// <summary> + /// Did not receive a User-Agent in the request header. + /// </summary> + UserAgentRequired = 1003, + + /// <summary> + /// Token required but not provided in the request header. + /// </summary> + TokenMissing = 1004, + + /// <summary> + /// Did not recognize the submitted client. + /// </summary> + UnknownClient = 1005, + + /// <summary> + /// The maximum number of elements in a single request is 5000. + /// </summary> + MaxChunkExceeded = 1006, + + /// <summary> + /// The request sent to the server was empty. + /// </summary> + EmptyRequest = 1007, + + /// <summary> + /// The request is improperly formatted. + /// </summary> + IncorrectRequest = 1008, + + /// <summary> + /// The maximum number of elements in a single request is 500. + /// </summary> + MaxChunkExceededMetadata = 1009, + + /// <summary> + /// Token is not 32 characters. + /// </summary> + TokenInvalid = 1010, + + /// <summary> + /// The Content-Type header must be application/json. + /// </summary> + IncorrectContentType = 1011, + + /// <summary> + /// Did not receive request. + /// </summary> + RequiredRequestMissing = 2002, + + /// <summary> + /// Searching for lineups requires a 3-letter country parameter. + /// </summary> + CountryRequired = 2004, + + /// <summary> + /// Searching for lineups requires a postal code parameter. + /// </summary> + PostalCodeRequired = 2005, + + /// <summary> + /// The request is missing the personID / nameID in the path. + /// </summary> + PersonIdRequired = 2020, + + /// <summary> + /// The country parameter must be ISO-3166-1 alpha 3. + /// </summary> + InvalidCountryParameter = 2050, + + /// <summary> + /// Unknown request; available data may be requested for COUNTRIES. + /// </summary> + UnknownRequest = 2054, + + /// <summary> + /// Unexpected debug connection from client. + /// </summary> + InvalidDebugParameter = 2055, + + /// <summary> + /// Lineup already in account. + /// </summary> + DuplicateLineup = 2100, + + /// <summary> + /// Lineup not in account; add it before requesting its mapping. + /// </summary> + LineupNotFound = 2101, + + /// <summary> + /// Invalid lineup requested; check the country/postal code combination. + /// </summary> + UnknownLineup = 2102, + + /// <summary> + /// Delete of a lineup that is not in the account. + /// </summary> + InvalidLineupDelete = 2103, + + /// <summary> + /// Lineup must be formatted COUNTRY-LINEUP-DEVICE or COUNTRY-OTA-POSTALCODE. + /// </summary> + LineupWrongFormat = 2104, + + /// <summary> + /// The requested lineup has been deleted from the server. + /// </summary> + LineupDeleted = 2106, + + /// <summary> + /// The requested country is mis-typed or has no valid data. + /// </summary> + InvalidCountry = 2108, + + /// <summary> + /// The requested personID does not exist. + /// </summary> + InvalidPersonId = 2109, + + /// <summary> + /// The requested stationID is not in any of the configured lineups. + /// </summary> + StationIdNotFound = 2200, + + /// <summary> + /// The requested stationID has been marked as deleted and will be purged. + /// </summary> + StationIdDeleted = 2201, + + /// <summary> + /// Server offline for maintenance. /// </summary> ServiceOffline = 3000, /// <summary> - /// Schedules Direct busy. + /// Server is busy processing other requests. Retry. /// </summary> ServiceBusy = 3001, @@ -23,72 +165,125 @@ public enum SdErrorCode AccountExpired = 4001, /// <summary> - /// Invalid password hash. + /// Password hash must be a lowercase 40 character sha1_hex of the password. /// </summary> InvalidHash = 4002, /// <summary> - /// Invalid user or password. + /// Invalid username or password. /// </summary> InvalidUser = 4003, /// <summary> - /// Account temporarily locked due to login failures. + /// Too many login failures. Locked for 15 minutes. /// </summary> AccountTempLock = 4004, /// <summary> - /// Account permanently locked due to abuse. + /// Access to the account via JSON has been disabled; the user must contact SD support. /// </summary> AccountLocked = 4005, /// <summary> - /// Token has expired. Request a new one. + /// Token has expired. Request a new token. /// </summary> TokenExpired = 4006, /// <summary> - /// Application locked out. + /// Application not authorized to use the data service. /// </summary> AppLocked = 4007, /// <summary> - /// Account not active. + /// Account inactive. /// </summary> AccountInactive = 4008, /// <summary> - /// Maximum login attempts exceeded. + /// Exceeded the maximum number of logins in 24 hours; clears 24 hours after the first login. /// </summary> MaxLoginAttempts = 4009, /// <summary> - /// Maximum unique IP attempts reached. + /// Exceeded the maximum number of unique IP addresses in 24 hours; the user must contact SD support. /// </summary> MaxIPAttempts = 4010, /// <summary> - /// Lineup change maximum reached. + /// Exceeded the maximum number of lineup changes for today. /// </summary> - MaxScheduleRequests = 4100, + MaxLineupChanges = 4100, /// <summary> - /// Requested image not found. + /// Exceeded the number of lineups this account may hold. + /// </summary> + MaxLineups = 4101, + + /// <summary> + /// No lineups have been added to this account. + /// </summary> + NoLineups = 4102, + + /// <summary> + /// Could not find the requested image. /// </summary> ImageNotFound = 5000, /// <summary> - /// Maximum image downloads reached for the day. + /// Maximum image downloads reached. The counter resets at 00:00Z. /// </summary> MaxImageDownloads = 5002, /// <summary> - /// Trial specific maximum image downloads reached for the day. + /// Maximum image downloads for a trial user reached. The counter resets at 00:00Z. /// </summary> MaxImageDownloadsTrial = 5003, /// <summary> - /// Maximum schedule/metadata requests reached for the day. + /// Exceeded the maximum number of invalid image URIs in 24 hours. The application is + /// requesting URIs which do not exist and the account is about to be blocked. + /// </summary> + MaxInvalidImages = 5004, + + /// <summary> + /// The requested programID does not exist. Permanent failure, do not ask again. + /// </summary> + InvalidProgramId = 6000, + + /// <summary> + /// The requested programID is queued for generation at the server. Soft failure, retry later. + /// </summary> + ProgramQueued = 6001, + + /// <summary> + /// The requested schedule should be available but was not found. + /// </summary> + ScheduleNotFound = 7000, + + /// <summary> + /// The server cannot determine whether the requested schedule is valid; the user has to open + /// a support ticket. + /// </summary> + InvalidScheduleRequest = 7010, + + /// <summary> + /// The requested date is outside the range the server holds for that station. + /// </summary> + ScheduleRangeExceeded = 7020, + + /// <summary> + /// The requested schedule is not in any of the configured lineups. + /// </summary> + ScheduleNotInLineup = 7030, + + /// <summary> + /// The schedule is queued for generation and is not ready yet. The client must wait until the + /// retryTime carried by the response before asking for it again. + /// </summary> + ScheduleQueued = 7100, + + /// <summary> + /// Unknown error; the user has to open a support ticket. /// </summary> - MaxInvalidImages = 5004 + UnknownError = 9999 } diff --git a/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/StatusAccountDto.cs b/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/StatusAccountDto.cs new file mode 100644 index 0000000000..fca28e0511 --- /dev/null +++ b/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/StatusAccountDto.cs @@ -0,0 +1,22 @@ +using System; +using System.Text.Json.Serialization; + +namespace Jellyfin.LiveTv.Listings.SchedulesDirectDtos; + +/// <summary> +/// The account block of the /status response. +/// </summary> +public class StatusAccountDto +{ + /// <summary> + /// Gets or sets the time the subscription expires. + /// </summary> + [JsonPropertyName("expires")] + public DateTime? Expires { get; set; } + + /// <summary> + /// Gets or sets the maximum number of lineups the account may hold. + /// </summary> + [JsonPropertyName("maxLineups")] + public int? MaxLineups { get; set; } +} diff --git a/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/StatusDto.cs b/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/StatusDto.cs new file mode 100644 index 0000000000..ce6b2ccc22 --- /dev/null +++ b/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/StatusDto.cs @@ -0,0 +1,35 @@ +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace Jellyfin.LiveTv.Listings.SchedulesDirectDtos; + +/// <summary> +/// The /status response. +/// </summary> +public class StatusDto +{ + /// <summary> + /// Gets or sets the account details. + /// </summary> + [JsonPropertyName("account")] + public StatusAccountDto? Account { get; set; } + + /// <summary> + /// Gets or sets the lineups on the account, with the time each was last modified. + /// </summary> + [JsonPropertyName("lineups")] + public IReadOnlyList<StatusLineupDto> Lineups { get; set; } = []; + + /// <summary> + /// Gets or sets the system status entries. The spec does not define their order, so every + /// entry has to be inspected. + /// </summary> + [JsonPropertyName("systemStatus")] + public IReadOnlyList<SystemStatusDto> SystemStatus { get; set; } = []; + + /// <summary> + /// Gets or sets the response code. + /// </summary> + [JsonPropertyName("code")] + public int Code { get; set; } +} diff --git a/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/StatusLineupDto.cs b/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/StatusLineupDto.cs new file mode 100644 index 0000000000..9a3559e406 --- /dev/null +++ b/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/StatusLineupDto.cs @@ -0,0 +1,35 @@ +using System; +using System.Text.Json.Serialization; + +namespace Jellyfin.LiveTv.Listings.SchedulesDirectDtos; + +/// <summary> +/// One lineup entry of the /status response. +/// </summary> +public class StatusLineupDto +{ + /// <summary> + /// Gets or sets the lineup id. + /// </summary> + [JsonPropertyName("lineup")] + public string? Lineup { get; set; } + + /// <summary> + /// Gets or sets the lineup id as reported for deleted lineups, which use a different name. + /// </summary> + [JsonPropertyName("ID")] + public string? Id { get; set; } + + /// <summary> + /// Gets or sets the time the lineup was last modified. A client only has to download the + /// lineup again when this is newer than the copy it holds. + /// </summary> + [JsonPropertyName("modified")] + public DateTime? Modified { get; set; } + + /// <summary> + /// Gets or sets a value indicating whether the lineup has been deleted at the headend. + /// </summary> + [JsonPropertyName("isDeleted")] + public bool IsDeleted { get; set; } +} diff --git a/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/SystemStatusDto.cs b/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/SystemStatusDto.cs new file mode 100644 index 0000000000..00909218f3 --- /dev/null +++ b/src/Jellyfin.LiveTv/Listings/SchedulesDirectDtos/SystemStatusDto.cs @@ -0,0 +1,28 @@ +using System; +using System.Text.Json.Serialization; + +namespace Jellyfin.LiveTv.Listings.SchedulesDirectDtos; + +/// <summary> +/// One entry of the system status. +/// </summary> +public class SystemStatusDto +{ + /// <summary> + /// Gets or sets the time the status was posted. + /// </summary> + [JsonPropertyName("date")] + public DateTime? Date { get; set; } + + /// <summary> + /// Gets or sets the status. "Offline" means the client must disconnect for an hour. + /// </summary> + [JsonPropertyName("status")] + public string? Status { get; set; } + + /// <summary> + /// Gets or sets the message describing the status. + /// </summary> + [JsonPropertyName("message")] + public string? Message { get; set; } +} diff --git a/src/Jellyfin.LiveTv/Listings/SchedulesDirectLineupCache.cs b/src/Jellyfin.LiveTv/Listings/SchedulesDirectLineupCache.cs new file mode 100644 index 0000000000..a24961f0aa --- /dev/null +++ b/src/Jellyfin.LiveTv/Listings/SchedulesDirectLineupCache.cs @@ -0,0 +1,117 @@ +using System; +using System.Globalization; +using System.IO; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Jellyfin.Extensions.Json; +using Jellyfin.LiveTv.Listings.SchedulesDirectDtos; +using MediaBrowser.Common.Configuration; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.LiveTv.Listings; + +/// <summary> +/// Caches a lineup's channel map, keyed by the "modified" date Schedules Direct reports for it. +/// </summary> +/// <remarks> +/// The spec's steady-state flow is to compare the lineup's modified date in /status against the +/// copy the client holds and only download the lineup again when the server's is newer. +/// </remarks> +public class SchedulesDirectLineupCache +{ + private readonly ILogger _logger; + private readonly string _cacheRoot; + private readonly JsonSerializerOptions _jsonOptions = JsonDefaults.Options; + + /// <summary> + /// Initializes a new instance of the <see cref="SchedulesDirectLineupCache"/> class. + /// </summary> + /// <param name="logger">The logger.</param> + /// <param name="appPaths">The application paths.</param> + public SchedulesDirectLineupCache(ILogger logger, IApplicationPaths appPaths) + { + _logger = logger; + _cacheRoot = Path.Combine(appPaths.CachePath, "schedulesdirect", "lineups"); + } + + /// <summary> + /// Reads the cached lineup if it was stored under the same modified date. + /// </summary> + /// <param name="lineupId">The lineup id.</param> + /// <param name="modified">The modified date Schedules Direct currently reports.</param> + /// <param name="cancellationToken">The cancellation token.</param> + /// <returns>The cached lineup, or <c>null</c> when it is missing or stale.</returns> + public async Task<ChannelDto?> GetAsync(string lineupId, DateTime modified, CancellationToken cancellationToken) + { + var path = GetPath(lineupId, modified); + if (!File.Exists(path)) + { + return null; + } + + try + { + var stream = File.OpenRead(path); + await using (stream.ConfigureAwait(false)) + { + return await JsonSerializer.DeserializeAsync<ChannelDto>(stream, _jsonOptions, cancellationToken).ConfigureAwait(false); + } + } + catch (Exception ex) when (ex is IOException or JsonException or UnauthorizedAccessException) + { + _logger.LogDebug(ex, "Discarding unreadable Schedules Direct lineup cache entry {Path}", path); + return null; + } + } + + /// <summary> + /// Stores a freshly downloaded lineup and drops the copies it supersedes. + /// </summary> + /// <param name="lineupId">The lineup id.</param> + /// <param name="modified">The modified date Schedules Direct reports for it.</param> + /// <param name="lineup">The lineup to store.</param> + /// <param name="cancellationToken">The cancellation token.</param> + /// <returns>A task representing the write.</returns> + public async Task SetAsync(string lineupId, DateTime modified, ChannelDto lineup, CancellationToken cancellationToken) + { + var path = GetPath(lineupId, modified); + + try + { + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + var stream = File.Create(path); + await using (stream.ConfigureAwait(false)) + { + await JsonSerializer.SerializeAsync(stream, lineup, _jsonOptions, cancellationToken).ConfigureAwait(false); + } + + foreach (var stale in Directory.EnumerateFiles(Path.GetDirectoryName(path)!, "*.json")) + { + if (!string.Equals(stale, path, StringComparison.Ordinal)) + { + File.Delete(stale); + } + } + } + catch (Exception ex) when (ex is IOException or JsonException or UnauthorizedAccessException) + { + _logger.LogWarning(ex, "Failed to cache Schedules Direct lineup {Path}", path); + } + } + + private static string GetSafeName(string value) + => string.Create(value.Length, value, (span, source) => + { + for (var i = 0; i < source.Length; i++) + { + span[i] = char.IsAsciiLetterOrDigit(source[i]) ? source[i] : '_'; + } + }); + + private string GetPath(string lineupId, DateTime modified) + => Path.Combine( + _cacheRoot, + GetSafeName(lineupId), + modified.ToUniversalTime().ToString("yyyyMMddHHmmss", CultureInfo.InvariantCulture) + ".json"); +} diff --git a/src/Jellyfin.LiveTv/Listings/SchedulesDirectScheduleCache.cs b/src/Jellyfin.LiveTv/Listings/SchedulesDirectScheduleCache.cs new file mode 100644 index 0000000000..e1a9e56cc4 --- /dev/null +++ b/src/Jellyfin.LiveTv/Listings/SchedulesDirectScheduleCache.cs @@ -0,0 +1,205 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Jellyfin.Extensions.Json; +using MediaBrowser.Common.Configuration; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.LiveTv.Listings; + +/// <summary> +/// Caches one station's schedule for one day, keyed by the md5 Schedules Direct reports for it. +/// </summary> +/// <remarks> +/// The API requires clients to ask /schedules/md5 what changed and re-download only that. A day +/// whose md5 still matches needs no /schedules, /programs or /metadata/programs request at all, +/// which is what keeps a guide refresh inside the account's daily quotas. +/// </remarks> +public class SchedulesDirectScheduleCache +{ + private readonly ILogger _logger; + private readonly string _cacheRoot; + private readonly JsonSerializerOptions _jsonOptions = JsonDefaults.Options; + + /// <summary> + /// Initializes a new instance of the <see cref="SchedulesDirectScheduleCache"/> class. + /// </summary> + /// <param name="logger">The logger.</param> + /// <param name="appPaths">The application paths.</param> + public SchedulesDirectScheduleCache(ILogger logger, IApplicationPaths appPaths) + { + _logger = logger; + _cacheRoot = Path.Combine(appPaths.CachePath, "schedulesdirect", "schedules"); + } + + /// <summary> + /// Reads the cached day if it was stored under the same md5. + /// </summary> + /// <param name="stationId">The station id.</param> + /// <param name="date">The date, formatted yyyy-MM-dd.</param> + /// <param name="md5">The md5 Schedules Direct currently reports, or null if unknown.</param> + /// <param name="cancellationToken">The cancellation token.</param> + /// <returns>The cached entry, or <c>null</c> when it is missing or stale.</returns> + public async Task<SchedulesDirectCachedDay?> GetAsync(string stationId, string date, string? md5, CancellationToken cancellationToken) + { + if (string.IsNullOrEmpty(md5)) + { + return null; + } + + var cached = await ReadAsync(stationId, date, cancellationToken).ConfigureAwait(false); + return string.Equals(cached?.Md5, md5, StringComparison.Ordinal) ? cached : null; + } + + /// <summary> + /// Reads the cached day whatever md5 it was stored under, to stand in for a day that could not + /// be downloaded. + /// </summary> + /// <param name="stationId">The station id.</param> + /// <param name="date">The date, formatted yyyy-MM-dd.</param> + /// <param name="cancellationToken">The cancellation token.</param> + /// <returns>The cached entry, or <c>null</c> when there is none.</returns> + public Task<SchedulesDirectCachedDay?> GetLatestAsync(string stationId, string date, CancellationToken cancellationToken) + => ReadAsync(stationId, date, cancellationToken); + + /// <summary> + /// Stores a freshly downloaded day. + /// </summary> + /// <param name="stationId">The station id.</param> + /// <param name="date">The date, formatted yyyy-MM-dd.</param> + /// <param name="day">The day to store.</param> + /// <param name="cancellationToken">The cancellation token.</param> + /// <returns>A task representing the write.</returns> + public async Task SetAsync(string stationId, string date, SchedulesDirectCachedDay day, CancellationToken cancellationToken) + { + if (string.IsNullOrEmpty(day.Md5)) + { + // Without an md5 there is nothing to validate the entry against later. + return; + } + + var path = GetPath(stationId, date); + + try + { + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + var stream = File.Create(path); + await using (stream.ConfigureAwait(false)) + { + await JsonSerializer.SerializeAsync(stream, day, _jsonOptions, cancellationToken).ConfigureAwait(false); + } + } + catch (Exception ex) when (ex is IOException or JsonException or UnauthorizedAccessException) + { + _logger.LogWarning(ex, "Failed to cache Schedules Direct schedule {Path}", path); + } + } + + /// <summary> + /// Deletes cached days that are no longer requested, so the cache cannot grow without bound. + /// </summary> + /// <param name="stationId">The station id.</param> + /// <param name="keepDates">The dates to keep.</param> + public void Prune(string stationId, IEnumerable<string> keepDates) + { + var directory = Path.Combine(_cacheRoot, GetSafeName(stationId)); + if (!Directory.Exists(directory)) + { + return; + } + + var keep = new HashSet<string>(keepDates.Select(GetSafeName), StringComparer.Ordinal); + + try + { + foreach (var file in Directory.EnumerateFiles(directory, "*.json")) + { + if (!keep.Contains(Path.GetFileNameWithoutExtension(file))) + { + File.Delete(file); + } + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + _logger.LogDebug(ex, "Failed to prune Schedules Direct cache in {Directory}", directory); + } + } + + /// <summary> + /// Deletes entries that have not been touched for a while, so that stations which have left + /// the lineup do not keep their cache forever. + /// </summary> + /// <param name="maxAge">The age past which an untouched entry is deleted.</param> + public void PruneStale(TimeSpan maxAge) + { + if (!Directory.Exists(_cacheRoot)) + { + return; + } + + var cutoff = DateTime.UtcNow - maxAge; + + try + { + foreach (var directory in Directory.EnumerateDirectories(_cacheRoot)) + { + foreach (var file in Directory.EnumerateFiles(directory, "*.json")) + { + if (File.GetLastWriteTimeUtc(file) < cutoff) + { + File.Delete(file); + } + } + + if (!Directory.EnumerateFileSystemEntries(directory).Any()) + { + Directory.Delete(directory); + } + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + _logger.LogDebug(ex, "Failed to prune stale Schedules Direct cache entries in {Directory}", _cacheRoot); + } + } + + private async Task<SchedulesDirectCachedDay?> ReadAsync(string stationId, string date, CancellationToken cancellationToken) + { + var path = GetPath(stationId, date); + if (!File.Exists(path)) + { + return null; + } + + try + { + var stream = File.OpenRead(path); + await using (stream.ConfigureAwait(false)) + { + return await JsonSerializer.DeserializeAsync<SchedulesDirectCachedDay>(stream, _jsonOptions, cancellationToken).ConfigureAwait(false); + } + } + catch (Exception ex) when (ex is IOException or JsonException or UnauthorizedAccessException) + { + _logger.LogDebug(ex, "Discarding unreadable Schedules Direct cache entry {Path}", path); + return null; + } + } + + private static string GetSafeName(string value) + => string.Create(value.Length, value, (span, source) => + { + for (var i = 0; i < source.Length; i++) + { + span[i] = char.IsAsciiLetterOrDigit(source[i]) ? source[i] : '_'; + } + }); + + private string GetPath(string stationId, string date) + => Path.Combine(_cacheRoot, GetSafeName(stationId), GetSafeName(date) + ".json"); +} diff --git a/src/Jellyfin.LiveTv/Recordings/RecordingsMetadataManager.cs b/src/Jellyfin.LiveTv/Recordings/RecordingsMetadataManager.cs index e0e5a00fb9..d9e9112fbe 100644 --- a/src/Jellyfin.LiveTv/Recordings/RecordingsMetadataManager.cs +++ b/src/Jellyfin.LiveTv/Recordings/RecordingsMetadataManager.cs @@ -30,6 +30,7 @@ public class RecordingsMetadataManager private readonly ILogger<RecordingsMetadataManager> _logger; private readonly IConfigurationManager _config; private readonly ILibraryManager _libraryManager; + private readonly ISchedulesDirectService _schedulesDirectService; /// <summary> /// Initializes a new instance of the <see cref="RecordingsMetadataManager"/> class. @@ -37,14 +38,17 @@ public class RecordingsMetadataManager /// <param name="logger">The <see cref="ILogger"/>.</param> /// <param name="config">The <see cref="IConfigurationManager"/>.</param> /// <param name="libraryManager">The <see cref="ILibraryManager"/>.</param> + /// <param name="schedulesDirectService">The <see cref="ISchedulesDirectService"/>.</param> public RecordingsMetadataManager( ILogger<RecordingsMetadataManager> logger, IConfigurationManager config, - ILibraryManager libraryManager) + ILibraryManager libraryManager, + ISchedulesDirectService schedulesDirectService) { _logger = logger; _config = config; _libraryManager = libraryManager; + _schedulesDirectService = schedulesDirectService; } /// <summary> @@ -469,7 +473,14 @@ public class RecordingsMetadataManager { if (!image.IsLocalFile) { - image = await _libraryManager.ConvertImageToLocal(program, image, 0).ConfigureAwait(false); + var localImage = await _schedulesDirectService.ConvertImageToLocal(program, image, 0).ConfigureAwait(false); + if (localImage is null) + { + _logger.LogDebug("Not saving recording image {Url}: Schedules Direct has blocked it", image.Path); + return; + } + + image = localImage; } var imageSaveFilenameWithoutExtension = image.Type switch |
