add channel downloading settings

2 files changed, 529 insertions, 12 deletions

diff --git a/MediaBrowser.Server.Implementations/Channels/ChannelDownloadScheduledTask.cs b/MediaBrowser.Server.Implementations/Channels/ChannelDownloadScheduledTask.cs
new file mode 100644
index 000000000..21b5ba6da
--- /dev/null
+++ b/MediaBrowser.Server.Implementations/Channels/ChannelDownloadScheduledTask.cs
@@ -0,0 +1,277 @@
+using MediaBrowser.Common.Extensions;
+using MediaBrowser.Common.IO;
+using MediaBrowser.Common.Net;
+using MediaBrowser.Common.ScheduledTasks;
+using MediaBrowser.Controller.Channels;
+using MediaBrowser.Controller.Configuration;
+using MediaBrowser.Controller.Library;
+using MediaBrowser.Model.Channels;
+using MediaBrowser.Model.Dto;
+using MediaBrowser.Model.Entities;
+using MediaBrowser.Model.Logging;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace MediaBrowser.Server.Implementations.Channels
+{
+    public class ChannelDownloadScheduledTask : IScheduledTask, IConfigurableScheduledTask
+    {
+        private readonly IChannelManager _manager;
+        private readonly IServerConfigurationManager _config;
+        private readonly ILogger _logger;
+        private readonly IHttpClient _httpClient;
+        private readonly IFileSystem _fileSystem;
+        private readonly ILibraryManager _libraryManager;
+
+        public ChannelDownloadScheduledTask(IChannelManager manager, IServerConfigurationManager config, ILogger logger, IHttpClient httpClient, IFileSystem fileSystem, ILibraryManager libraryManager)
+        {
+            _manager = manager;
+            _config = config;
+            _logger = logger;
+            _httpClient = httpClient;
+            _fileSystem = fileSystem;
+            _libraryManager = libraryManager;
+        }
+
+        public string Name
+        {
+            get { return "Download channel content"; }
+        }
+
+        public string Description
+        {
+            get { return "Downloads channel content based on configuration."; }
+        }
+
+        public string Category
+        {
+            get { return "Channels"; }
+        }
+
+        public async Task Execute(CancellationToken cancellationToken, IProgress<double> progress)
+        {
+            CleanChannelContent(cancellationToken);
+            progress.Report(5);
+
+            await DownloadChannelContent(cancellationToken, progress).ConfigureAwait(false);
+            progress.Report(100);
+        }
+
+        private void CleanChannelContent(CancellationToken cancellationToken)
+        {
+            if (!_config.Configuration.ChannelOptions.MaxDownloadAge.HasValue)
+            {
+                return;
+            }
+
+            var minDateModified = DateTime.UtcNow.AddDays(0 - _config.Configuration.ChannelOptions.MaxDownloadAge.Value);
+
+            var path = _manager.ChannelDownloadPath;
+
+            try
+            {
+                DeleteCacheFilesFromDirectory(cancellationToken, path, minDateModified, new Progress<double>());
+            }
+            catch (DirectoryNotFoundException)
+            {
+                // No biggie here. Nothing to delete
+            }
+        }
+
+        private async Task DownloadChannelContent(CancellationToken cancellationToken, IProgress<double> progress)
+        {
+            if (_config.Configuration.ChannelOptions.DownloadingChannels.Length == 0)
+            {
+                return;
+            }
+
+            var result = await _manager.GetAllMedia(new AllChannelMediaQuery
+            {
+                ChannelIds = _config.Configuration.ChannelOptions.DownloadingChannels
+
+            }, cancellationToken).ConfigureAwait(false);
+
+            var path = _manager.ChannelDownloadPath;
+
+            var numComplete = 0;
+
+            foreach (var item in result.Items)
+            {
+                try
+                {
+                    await DownloadChannelItem(item, cancellationToken, path);
+                }
+                catch (OperationCanceledException)
+                {
+                    break;
+                }
+                catch (Exception ex)
+                {
+                    _logger.ErrorException("Error downloading channel content for {0}", ex, item.Name);
+                }
+
+                numComplete++;
+                double percent = numComplete;
+                percent /= result.Items.Length;
+                progress.Report(percent * 95 + 5);
+            }
+        }
+
+        private async Task DownloadChannelItem(BaseItemDto item,
+            CancellationToken cancellationToken,
+            string path)
+        {
+            var sources = await _manager.GetChannelItemMediaSources(item.Id, cancellationToken)
+                .ConfigureAwait(false);
+
+            var list = sources.ToList();
+
+            var cachedVersions = list.Where(i => i.LocationType == LocationType.FileSystem).ToList();
+
+            if (cachedVersions.Count > 0)
+            {
+                await RefreshMediaSourceItems(cachedVersions, item.IsVideo, cancellationToken).ConfigureAwait(false);
+                return;
+            }
+
+            var source = list.First();
+
+            var options = new HttpRequestOptions
+            {
+                CancellationToken = cancellationToken,
+                Url = source.Path,
+                Progress = new Progress<double>()
+            };
+
+            foreach (var header in source.RequiredHttpHeaders)
+            {
+                options.RequestHeaders[header.Key] = header.Value;
+            }
+
+            var destination = Path.Combine(path, item.ChannelId, source.Path.GetMD5().ToString("N"));
+            Directory.CreateDirectory(Path.GetDirectoryName(destination));
+
+            // Determine output extension
+            var response = await _httpClient.GetTempFileResponse(options).ConfigureAwait(false);
+
+            if (item.IsVideo && response.ContentType.StartsWith("video/", StringComparison.OrdinalIgnoreCase))
+            {
+                var extension = response.ContentType.Split('/')
+                        .Last();
+
+                destination += "." + extension;
+            }
+            else if (item.IsAudio && response.ContentType.StartsWith("audio/", StringComparison.OrdinalIgnoreCase))
+            {
+                var extension = response.ContentType.Replace("audio/mpeg", "audio/mp3", StringComparison.OrdinalIgnoreCase)
+                        .Split('/')
+                        .Last();
+
+                destination += "." + extension;
+            }
+            else
+            {
+                throw new ApplicationException("Unexpected response type encountered: " + response.ContentType);
+            }
+
+            File.Move(response.TempFilePath, destination);
+
+            await RefreshMediaSourceItem(destination, item.IsVideo, cancellationToken).ConfigureAwait(false);
+        }
+
+        private async Task RefreshMediaSourceItems(IEnumerable<MediaSourceInfo> items, bool isVideo, CancellationToken cancellationToken)
+        {
+            foreach (var item in items)
+            {
+                await RefreshMediaSourceItem(item.Path, isVideo, cancellationToken).ConfigureAwait(false);
+            }
+        }
+
+        private async Task RefreshMediaSourceItem(string path, bool isVideo, CancellationToken cancellationToken)
+        {
+            var item = _libraryManager.ResolvePath(new FileInfo(path));
+
+            if (item != null)
+            {
+                await item.RefreshMetadata(cancellationToken).ConfigureAwait(false);
+            }
+        }
+
+        public IEnumerable<ITaskTrigger> GetDefaultTriggers()
+        {
+            return new ITaskTrigger[]
+                {
+                    new DailyTrigger { TimeOfDay = TimeSpan.FromHours(3) },
+                };
+        }
+
+        /// <summary>
+        /// Deletes the cache files from directory with a last write time less than a given date
+        /// </summary>
+        /// <param name="cancellationToken">The task cancellation token.</param>
+        /// <param name="directory">The directory.</param>
+        /// <param name="minDateModified">The min date modified.</param>
+        /// <param name="progress">The progress.</param>
+        private void DeleteCacheFilesFromDirectory(CancellationToken cancellationToken, string directory, DateTime minDateModified, IProgress<double> progress)
+        {
+            var filesToDelete = new DirectoryInfo(directory).EnumerateFiles("*", SearchOption.AllDirectories)
+                .Where(f => _fileSystem.GetLastWriteTimeUtc(f) < minDateModified)
+                .ToList();
+
+            var index = 0;
+
+            foreach (var file in filesToDelete)
+            {
+                double percent = index;
+                percent /= filesToDelete.Count;
+
+                progress.Report(100 * percent);
+
+                cancellationToken.ThrowIfCancellationRequested();
+
+                DeleteFile(file.FullName);
+
+                index++;
+            }
+
+            progress.Report(100);
+        }
+
+        /// <summary>
+        /// Deletes the file.
+        /// </summary>
+        /// <param name="path">The path.</param>
+        private void DeleteFile(string path)
+        {
+            try
+            {
+                File.Delete(path);
+            }
+            catch (IOException ex)
+            {
+                _logger.ErrorException("Error deleting file {0}", ex, path);
+            }
+        }
+
+        public bool IsHidden
+        {
+            get
+            {
+                return !_manager.GetAllChannelFeatures()
+                    .Any(i => i.CanDownloadAllMedia && _config.Configuration.ChannelOptions.DownloadingChannels.Contains(i.Id));
+            }
+        }
+
+        public bool IsEnabled
+        {
+            get
+            {
+                return true;
+            }
+        }
+    }
+}
diff --git a/MediaBrowser.Server.Implementations/Channels/ChannelManager.cs b/MediaBrowser.Server.Implementations/Channels/ChannelManager.cs
index b9b9b8327..b9e4e73b0 100644
--- a/MediaBrowser.Server.Implementations/Channels/ChannelManager.cs
+++ b/MediaBrowser.Server.Implementations/Channels/ChannelManager.cs
@@ -54,6 +54,19 @@ namespace MediaBrowser.Server.Implementations.Channels
             _factories = factories.ToArray();
         }
 
+        public string ChannelDownloadPath
+        {
+            get
+            {
+                if (!string.IsNullOrWhiteSpace(_config.Configuration.ChannelOptions.DownloadPath))
+                {
+                    return _config.Configuration.ChannelOptions.DownloadPath;
+                }
+
+                return Path.Combine(_config.ApplicationPaths.ProgramDataPath, "channels");
+            }
+        }
+
         private IEnumerable<IChannel> GetAllChannels()
         {
             return _factories
@@ -156,7 +169,7 @@ namespace MediaBrowser.Server.Implementations.Channels
             progress.Report(100);
         }
 
-        public Task<IEnumerable<ChannelMediaInfo>> GetChannelItemMediaSources(string id, CancellationToken cancellationToken)
+        public async Task<IEnumerable<MediaSourceInfo>> GetChannelItemMediaSources(string id, CancellationToken cancellationToken)
         {
             var item = (IChannelMediaItem)_libraryManager.GetItemById(id);
 
@@ -166,12 +179,149 @@ namespace MediaBrowser.Server.Implementations.Channels
 
             var requiresCallback = channelPlugin as IRequiresMediaInfoCallback;
 
+            IEnumerable<ChannelMediaInfo> results;
+
             if (requiresCallback != null)
             {
-                return requiresCallback.GetChannelItemMediaInfo(item.ExternalId, cancellationToken);
+                results = await requiresCallback.GetChannelItemMediaInfo(item.ExternalId, cancellationToken)
+                   .ConfigureAwait(false);
+            }
+            else
+            {
+                results = item.ChannelMediaSources;
             }
 
-            return Task.FromResult<IEnumerable<ChannelMediaInfo>>(item.ChannelMediaSources);
+            var sources = SortMediaInfoResults(results).Select(i => GetMediaSource(item, i))
+                .ToList();
+
+            var channelIdString = channel.Id.ToString("N");
+            var isVideo = string.Equals(item.MediaType, MediaType.Video, StringComparison.OrdinalIgnoreCase);
+
+            var cachedVersionTasks = sources
+                .Select(i => GetCachedVersion(channelIdString, i, isVideo, cancellationToken));
+
+            var cachedVersions = await Task.WhenAll(cachedVersionTasks).ConfigureAwait(false);
+
+            sources.InsertRange(0, cachedVersions.Where(i => i != null));
+
+            return sources;
+        }
+
+        private MediaSourceInfo GetMediaSource(IChannelMediaItem item, ChannelMediaInfo info)
+        {
+            var id = info.Path.GetMD5().ToString("N");
+
+            var source = new MediaSourceInfo
+            {
+                MediaStreams = GetMediaStreams(info).ToList(),
+
+                Container = info.Container,
+                LocationType = info.IsRemote ? LocationType.Remote : LocationType.FileSystem,
+                Path = info.Path,
+                RequiredHttpHeaders = info.RequiredHttpHeaders,
+                RunTimeTicks = item.RunTimeTicks,
+                Name = id,
+                Id = id
+            };
+
+            return source;
+        }
+
+        private async Task<MediaSourceInfo> GetCachedVersion(string channelId, 
+            MediaSourceInfo info, 
+            bool isVideo,
+            CancellationToken cancellationToken)
+        {
+            var filename = info.Path.GetMD5().ToString("N");
+
+            var path = Path.Combine(ChannelDownloadPath, channelId, filename);
+
+            try
+            {
+                var file = Directory.EnumerateFiles(Path.GetDirectoryName(path), "*", SearchOption.TopDirectoryOnly)
+                        .FirstOrDefault(i => (Path.GetFileName(i) ?? string.Empty).StartsWith(filename, StringComparison.OrdinalIgnoreCase));
+
+                if (!string.IsNullOrWhiteSpace(file))
+                {
+                    var source = new MediaSourceInfo
+                    {
+                        Path = file,
+                        LocationType = LocationType.FileSystem,
+                        Name = "Cached " + info.Name,
+                        Id = file.GetMD5().ToString("N")
+                    };
+
+                    if (isVideo)
+                    {
+                        source.VideoType = VideoType.VideoFile;
+                    }
+
+                    return source;
+                }
+            }
+            catch (DirectoryNotFoundException)
+            {
+                return null;
+            }
+
+            return null;
+        }
+
+        private IEnumerable<MediaStream> GetMediaStreams(ChannelMediaInfo info)
+        {
+            var list = new List<MediaStream>();
+
+            if (!string.IsNullOrWhiteSpace(info.VideoCodec) &&
+                !string.IsNullOrWhiteSpace(info.AudioCodec))
+            {
+                list.Add(new MediaStream
+                {
+                    Type = MediaStreamType.Video,
+                    Width = info.Width,
+                    RealFrameRate = info.Framerate,
+                    Profile = info.VideoProfile,
+                    Level = info.VideoLevel,
+                    Index = -1,
+                    Height = info.Height,
+                    Codec = info.VideoCodec,
+                    BitRate = info.VideoBitrate,
+                    AverageFrameRate = info.Framerate
+                });
+
+                list.Add(new MediaStream
+                {
+                    Type = MediaStreamType.Audio,
+                    Index = -1,
+                    Codec = info.AudioCodec,
+                    BitRate = info.AudioBitrate,
+                    Channels = info.AudioChannels,
+                    SampleRate = info.AudioSampleRate
+                });
+            }
+
+            return list;
+        }
+
+        private IEnumerable<ChannelMediaInfo> SortMediaInfoResults(IEnumerable<ChannelMediaInfo> channelMediaSources)
+        {
+            var list = channelMediaSources.ToList();
+
+            var width = _config.Configuration.ChannelOptions.PreferredStreamingWidth;
+
+            if (width.HasValue)
+            {
+                var val = width.Value;
+
+                return list
+                    .OrderBy(i => i.Width.HasValue && i.Width.Value <= val)
+                    .ThenBy(i => Math.Abs(i.Width ?? 0 - val))
+                    .ThenByDescending(i => i.Width ?? 0)
+                    .ThenBy(list.IndexOf);
+            }
+
+            return list
+                .OrderByDescending(i => i.Width ?? 0)
+                .ThenBy(list.IndexOf);
         }
 
         private async Task<Channel> GetChannel(IChannel channelInfo, CancellationToken cancellationToken)
@@ -237,26 +387,37 @@ namespace MediaBrowser.Server.Implementations.Channels
             return (Channel)_libraryManager.GetItemById(new Guid(id));
         }
 
+        public IEnumerable<ChannelFeatures> GetAllChannelFeatures()
+        {
+            return _channelEntities
+                .OrderBy(i => i.SortName)
+                .Select(i => GetChannelFeatures(i.Id.ToString("N")));
+        }
+
         public ChannelFeatures GetChannelFeatures(string id)
         {
             var channel = GetChannel(id);
 
             var channelProvider = GetChannelProvider(channel);
 
-            return GetChannelFeaturesDto(channelProvider.GetChannelFeatures());
+            return GetChannelFeaturesDto(channel, channelProvider.GetChannelFeatures());
         }
 
-        public ChannelFeatures GetChannelFeaturesDto(InternalChannelFeatures features)
+        public ChannelFeatures GetChannelFeaturesDto(Channel channel, InternalChannelFeatures features)
         {
             return new ChannelFeatures
             {
                 CanFilter = !features.MaxPageSize.HasValue,
+                CanGetAllMedia = features.CanGetAllMedia,
                 CanSearch = features.CanSearch,
                 ContentTypes = features.ContentTypes,
                 DefaultSortFields = features.DefaultSortFields,
                 MaxPageSize = features.MaxPageSize,
                 MediaTypes = features.MediaTypes,
-                SupportsSortOrderToggle = features.SupportsSortOrderToggle
+                SupportsSortOrderToggle = features.SupportsSortOrderToggle,
+                Name = channel.Name,
+                Id = channel.Id.ToString("N"),
+                CanDownloadAllMedia = features.CanGetAllMedia
             };
         }
 
@@ -270,6 +431,85 @@ namespace MediaBrowser.Server.Implementations.Channels
             return ("Channel " + name).GetMBId(typeof(Channel));
         }
 
+        public async Task<QueryResult<BaseItemDto>> GetAllMedia(AllChannelMediaQuery query, CancellationToken cancellationToken)
+        {
+            var user = string.IsNullOrWhiteSpace(query.UserId)
+                ? null
+                : _userManager.GetUserById(new Guid(query.UserId));
+
+            var channels = _channels;
+
+            if (query.ChannelIds.Length > 0)
+            {
+                channels = channels
+                    .Where(i => query.ChannelIds.Contains(GetInternalChannelId(i.Name).ToString("N")))
+                    .ToArray();
+            }
+
+            var tasks = channels
+                .Where(i => i.GetChannelFeatures().CanGetAllMedia)
+                .Select(async i =>
+                {
+                    try
+                    {
+                        var result = await i.GetAllMedia(new InternalAllChannelMediaQuery
+                        {
+                            User = user
+
+                        }, cancellationToken).ConfigureAwait(false);
+
+                        return new Tuple<IChannel, ChannelItemResult>(i, result);
+                    }
+                    catch (Exception ex)
+                    {
+                        _logger.ErrorException("Error getting all media from {0}", ex, i.Name);
+                        return new Tuple<IChannel, ChannelItemResult>(i, new ChannelItemResult { });
+                    }
+                });
+
+            var results = await Task.WhenAll(tasks).ConfigureAwait(false);
+
+            var totalCount = results.Length;
+
+            IEnumerable<Tuple<IChannel, ChannelItemInfo>> items = results
+                .SelectMany(i => i.Item2.Items.Select(m => new Tuple<IChannel, ChannelItemInfo>(i.Item1, m)))
+                .OrderBy(i => i.Item2.Name);
+
+            if (query.StartIndex.HasValue)
+            {
+                items = items.Skip(query.StartIndex.Value);
+            }
+            if (query.Limit.HasValue)
+            {
+                items = items.Take(query.Limit.Value);
+            }
+
+            // Avoid implicitly captured closure
+            var token = cancellationToken;
+            var itemTasks = items.Select(i =>
+            {
+                var channelProvider = i.Item1;
+                var channel = GetChannel(GetInternalChannelId(channelProvider.Name).ToString("N"));
+                return GetChannelItemEntity(i.Item2, channelProvider, channel, token);
+            });
+
+            var internalItems = await Task.WhenAll(itemTasks).ConfigureAwait(false);
+
+            // Get everything
+            var fields = Enum.GetNames(typeof(ItemFields))
+                    .Select(i => (ItemFields)Enum.Parse(typeof(ItemFields), i, true))
+                    .ToList();
+
+            var returnItemArray = internalItems.Select(i => _dtoService.GetBaseItemDto(i, fields, user))
+                .ToArray();
+
+            return new QueryResult<BaseItemDto>
+            {
+                TotalRecordCount = totalCount,
+                Items = returnItemArray
+            };
+        }
+
         public async Task<QueryResult<BaseItemDto>> GetChannelItems(ChannelItemQuery query, CancellationToken cancellationToken)
         {
             var queryChannelId = query.ChannelId;
@@ -301,7 +541,7 @@ namespace MediaBrowser.Server.Implementations.Channels
 
             ChannelItemSortField? sortField = null;
             ChannelItemSortField parsedField;
-            if (query.SortBy.Length == 1 && 
+            if (query.SortBy.Length == 1 &&
                 Enum.TryParse(query.SortBy[0], true, out parsedField))
             {
                 sortField = parsedField;
@@ -309,11 +549,11 @@ namespace MediaBrowser.Server.Implementations.Channels
 
             var sortDescending = query.SortOrder.HasValue && query.SortOrder.Value == SortOrder.Descending;
 
-            var itemsResult = await GetChannelItems(channelProvider, 
-                user, 
-                query.FolderId, 
-                providerStartIndex, 
-                providerLimit, 
+            var itemsResult = await GetChannelItems(channelProvider,
+                user,
+                query.FolderId,
+                providerStartIndex,
+                providerLimit,
                 sortField,
                 sortDescending,
                 cancellationToken)