avoid buffering http responses

7 files changed, 138 insertions, 95 deletions

diff --git a/MediaBrowser.Server.Implementations/LiveTv/Listings/SchedulesDirect.cs b/MediaBrowser.Server.Implementations/LiveTv/Listings/SchedulesDirect.cs
index b98919282..7574eb485 100644
--- a/MediaBrowser.Server.Implementations/LiveTv/Listings/SchedulesDirect.cs
+++ b/MediaBrowser.Server.Implementations/LiveTv/Listings/SchedulesDirect.cs
@@ -846,7 +846,8 @@ namespace MediaBrowser.Server.Implementations.LiveTv.Listings
                 Url = ApiUrl + "/lineups/" + info.ListingsId,
                 UserAgent = UserAgent,
                 CancellationToken = cancellationToken,
-                LogErrorResponseBody = true
+                LogErrorResponseBody = true,
+                BufferContent = false
             };
 
             httpOptions.RequestHeaders["token"] = token;
diff --git a/MediaBrowser.Server.Implementations/LiveTv/LiveTvManager.cs b/MediaBrowser.Server.Implementations/LiveTv/LiveTvManager.cs
index 546525fd1..72287f32d 100644
--- a/MediaBrowser.Server.Implementations/LiveTv/LiveTvManager.cs
+++ b/MediaBrowser.Server.Implementations/LiveTv/LiveTvManager.cs
@@ -558,7 +558,7 @@ namespace MediaBrowser.Server.Implementations.LiveTv
             return item;
         }
 
-        private async Task<LiveTvProgram> GetProgram(ProgramInfo info, Dictionary<Guid, LiveTvProgram> allExistingPrograms, LiveTvChannel channel, ChannelType channelType, string serviceName, CancellationToken cancellationToken)
+        private Tuple<LiveTvProgram, bool, bool> GetProgram(ProgramInfo info, Dictionary<Guid, LiveTvProgram> allExistingPrograms, LiveTvChannel channel, ChannelType channelType, string serviceName, CancellationToken cancellationToken)
         {
             var id = _tvDtoService.GetInternalProgramId(serviceName, info.Id);
 
@@ -671,13 +671,13 @@ namespace MediaBrowser.Server.Implementations.LiveTv
                 }
             }
 
+            var isUpdated = false;
             if (isNew)
             {
-                await _libraryManager.CreateItem(item, cancellationToken).ConfigureAwait(false);
             }
             else if (forceUpdate || string.IsNullOrWhiteSpace(info.Etag))
             {
-                await _libraryManager.UpdateItem(item, ItemUpdateType.MetadataImport, cancellationToken).ConfigureAwait(false);
+                isUpdated = true;
             }
             else
             {
@@ -687,13 +687,11 @@ namespace MediaBrowser.Server.Implementations.LiveTv
                 if (!string.Equals(etag, item.ExternalEtag, StringComparison.OrdinalIgnoreCase))
                 {
                     item.ExternalEtag = etag;
-                    await _libraryManager.UpdateItem(item, ItemUpdateType.MetadataImport, cancellationToken).ConfigureAwait(false);
+                    isUpdated = true;
                 }
             }
 
-            _providerManager.QueueRefresh(item.Id, new MetadataRefreshOptions(_fileSystem));
-
-            return item;
+            return new Tuple<LiveTvProgram, bool, bool>(item, isNew, isUpdated);
         }
 
         private async Task<Guid> CreateRecordingRecord(RecordingInfo info, string serviceName, Guid parentFolderId, CancellationToken cancellationToken)
@@ -1289,9 +1287,22 @@ namespace MediaBrowser.Server.Implementations.LiveTv
 
                     }).Cast<LiveTvProgram>().ToDictionary(i => i.Id);
 
+                    var newPrograms = new List<LiveTvProgram>();
+                    var updatedPrograms = new List<LiveTvProgram>();
+
                     foreach (var program in channelPrograms)
                     {
-                        var programItem = await GetProgram(program, existingPrograms, currentChannel, currentChannel.ChannelType, service.Name, cancellationToken).ConfigureAwait(false);
+                        var programTuple = GetProgram(program, existingPrograms, currentChannel, currentChannel.ChannelType, service.Name, cancellationToken);
+                        var programItem = programTuple.Item1;
+
+                        if (programTuple.Item2)
+                        {
+                            newPrograms.Add(programItem);
+                        }
+                        else if (programTuple.Item3)
+                        {
+                            updatedPrograms.Add(programItem);
+                        }
 
                         programs.Add(programItem.Id);
 
@@ -1321,6 +1332,26 @@ namespace MediaBrowser.Server.Implementations.LiveTv
                         }
                     }
 
+                    if (newPrograms.Count > 0)
+                    {
+                        await _libraryManager.CreateItems(newPrograms, cancellationToken).ConfigureAwait(false);
+                    }
+
+                    // TODO: Do this in bulk
+                    foreach (var program in updatedPrograms)
+                    {
+                        await _libraryManager.UpdateItem(program, ItemUpdateType.MetadataImport, cancellationToken).ConfigureAwait(false);
+                    }
+
+                    foreach (var program in newPrograms)
+                    {
+                        _providerManager.QueueRefresh(program.Id, new MetadataRefreshOptions(_fileSystem));
+                    }
+                    foreach (var program in updatedPrograms)
+                    {
+                        _providerManager.QueueRefresh(program.Id, new MetadataRefreshOptions(_fileSystem));
+                    }
+
                     currentChannel.IsMovie = isMovie;
                     currentChannel.IsNews = isNews;
                     currentChannel.IsSports = isSports;
diff --git a/MediaBrowser.Server.Implementations/LiveTv/LiveTvMediaSourceProvider.cs b/MediaBrowser.Server.Implementations/LiveTv/LiveTvMediaSourceProvider.cs
index 1861b79eb..1ca664ee2 100644
--- a/MediaBrowser.Server.Implementations/LiveTv/LiveTvMediaSourceProvider.cs
+++ b/MediaBrowser.Server.Implementations/LiveTv/LiveTvMediaSourceProvider.cs
@@ -12,6 +12,7 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+using MediaBrowser.Model.Dlna;
 
 namespace MediaBrowser.Server.Implementations.LiveTv
 {
@@ -138,7 +139,7 @@ namespace MediaBrowser.Server.Implementations.LiveTv
 
             try
             {
-                await AddMediaInfo(stream, isAudio, cancellationToken).ConfigureAwait(false);
+                await AddMediaInfoInternal(stream, isAudio, cancellationToken).ConfigureAwait(false);
             }
             catch (Exception ex)
             {
@@ -207,6 +208,85 @@ namespace MediaBrowser.Server.Implementations.LiveTv
             }
         }
 
+        private async Task AddMediaInfoInternal(MediaSourceInfo mediaSource, bool isAudio, CancellationToken cancellationToken)
+        {
+            var originalRuntime = mediaSource.RunTimeTicks;
+
+            var info = await _mediaEncoder.GetMediaInfo(new MediaInfoRequest
+            {
+                InputPath = mediaSource.Path,
+                Protocol = mediaSource.Protocol,
+                MediaType = isAudio ? DlnaProfileType.Audio : DlnaProfileType.Video,
+                ExtractChapters = false
+
+            }, cancellationToken).ConfigureAwait(false);
+
+            mediaSource.Bitrate = info.Bitrate;
+            mediaSource.Container = info.Container;
+            mediaSource.Formats = info.Formats;
+            mediaSource.MediaStreams = info.MediaStreams;
+            mediaSource.RunTimeTicks = info.RunTimeTicks;
+            mediaSource.Size = info.Size;
+            mediaSource.Timestamp = info.Timestamp;
+            mediaSource.Video3DFormat = info.Video3DFormat;
+            mediaSource.VideoType = info.VideoType;
+
+            mediaSource.DefaultSubtitleStreamIndex = null;
+
+            // Null this out so that it will be treated like a live stream
+            if (!originalRuntime.HasValue)
+            {
+                mediaSource.RunTimeTicks = null;
+            }
+
+            var audioStream = mediaSource.MediaStreams.FirstOrDefault(i => i.Type == Model.Entities.MediaStreamType.Audio);
+
+            if (audioStream == null || audioStream.Index == -1)
+            {
+                mediaSource.DefaultAudioStreamIndex = null;
+            }
+            else
+            {
+                mediaSource.DefaultAudioStreamIndex = audioStream.Index;
+            }
+
+            var videoStream = mediaSource.MediaStreams.FirstOrDefault(i => i.Type == Model.Entities.MediaStreamType.Video);
+            if (videoStream != null)
+            {
+                if (!videoStream.BitRate.HasValue)
+                {
+                    var width = videoStream.Width ?? 1920;
+
+                    if (width >= 1900)
+                    {
+                        videoStream.BitRate = 8000000;
+                    }
+
+                    else if (width >= 1260)
+                    {
+                        videoStream.BitRate = 3000000;
+                    }
+
+                    else if (width >= 700)
+                    {
+                        videoStream.BitRate = 1000000;
+                    }
+                }
+            }
+
+            // Try to estimate this
+            if (!mediaSource.Bitrate.HasValue)
+            {
+                var total = mediaSource.MediaStreams.Select(i => i.BitRate ?? 0).Sum();
+
+                if (total > 0)
+                {
+                    mediaSource.Bitrate = total;
+                }
+            }
+        }
+
+
         public Task CloseMediaSource(string liveStreamId)
         {
             return _liveTvManager.CloseLiveStream(liveStreamId);
diff --git a/MediaBrowser.Server.Implementations/LiveTv/TunerHosts/BaseTunerHost.cs b/MediaBrowser.Server.Implementations/LiveTv/TunerHosts/BaseTunerHost.cs
index a4236763f..84ba15e49 100644
--- a/MediaBrowser.Server.Implementations/LiveTv/TunerHosts/BaseTunerHost.cs
+++ b/MediaBrowser.Server.Implementations/LiveTv/TunerHosts/BaseTunerHost.cs
@@ -252,84 +252,6 @@ namespace MediaBrowser.Server.Implementations.LiveTv.TunerHosts
             //}
         }
 
-        private async Task AddMediaInfoInternal(MediaSourceInfo mediaSource, bool isAudio, CancellationToken cancellationToken)
-        {
-            var originalRuntime = mediaSource.RunTimeTicks;
-
-            var info = await MediaEncoder.GetMediaInfo(new MediaInfoRequest
-            {
-                InputPath = mediaSource.Path,
-                Protocol = mediaSource.Protocol,
-                MediaType = isAudio ? DlnaProfileType.Audio : DlnaProfileType.Video,
-                ExtractChapters = false
-
-            }, cancellationToken).ConfigureAwait(false);
-
-            mediaSource.Bitrate = info.Bitrate;
-            mediaSource.Container = info.Container;
-            mediaSource.Formats = info.Formats;
-            mediaSource.MediaStreams = info.MediaStreams;
-            mediaSource.RunTimeTicks = info.RunTimeTicks;
-            mediaSource.Size = info.Size;
-            mediaSource.Timestamp = info.Timestamp;
-            mediaSource.Video3DFormat = info.Video3DFormat;
-            mediaSource.VideoType = info.VideoType;
-
-            mediaSource.DefaultSubtitleStreamIndex = null;
-
-            // Null this out so that it will be treated like a live stream
-            if (!originalRuntime.HasValue)
-            {
-                mediaSource.RunTimeTicks = null;
-            }
-
-            var audioStream = mediaSource.MediaStreams.FirstOrDefault(i => i.Type == Model.Entities.MediaStreamType.Audio);
-
-            if (audioStream == null || audioStream.Index == -1)
-            {
-                mediaSource.DefaultAudioStreamIndex = null;
-            }
-            else
-            {
-                mediaSource.DefaultAudioStreamIndex = audioStream.Index;
-            }
-
-            var videoStream = mediaSource.MediaStreams.FirstOrDefault(i => i.Type == Model.Entities.MediaStreamType.Video);
-            if (videoStream != null)
-            {
-                if (!videoStream.BitRate.HasValue)
-                {
-                    var width = videoStream.Width ?? 1920;
-
-                    if (width >= 1900)
-                    {
-                        videoStream.BitRate = 8000000;
-                    }
-
-                    else if (width >= 1260)
-                    {
-                        videoStream.BitRate = 3000000;
-                    }
-
-                    else if (width >= 700)
-                    {
-                        videoStream.BitRate = 1000000;
-                    }
-                }
-            }
-
-            // Try to estimate this
-            if (!mediaSource.Bitrate.HasValue)
-            {
-                var total = mediaSource.MediaStreams.Select(i => i.BitRate ?? 0).Sum();
-
-                if (total > 0)
-                {
-                    mediaSource.Bitrate = total;
-                }
-            }
-        }
-
         protected abstract bool IsValidChannelId(string channelId);
 
         protected LiveTvOptions GetConfiguration()
diff --git a/MediaBrowser.Server.Implementations/LiveTv/TunerHosts/HdHomerun/HdHomerunDiscovery.cs b/MediaBrowser.Server.Implementations/LiveTv/TunerHosts/HdHomerun/HdHomerunDiscovery.cs
index ef37e3b35..cd168ba58 100644
--- a/MediaBrowser.Server.Implementations/LiveTv/TunerHosts/HdHomerun/HdHomerunDiscovery.cs
+++ b/MediaBrowser.Server.Implementations/LiveTv/TunerHosts/HdHomerun/HdHomerunDiscovery.cs
@@ -88,7 +88,8 @@ namespace MediaBrowser.Server.Implementations.LiveTv.TunerHosts.HdHomerun
                 using (var stream = await _httpClient.Get(new HttpRequestOptions
                 {
                     Url = string.Format("{0}/discover.json", url),
-                    CancellationToken = CancellationToken.None
+                    CancellationToken = CancellationToken.None,
+                    BufferContent = false
                 }))
                 {
                     var response = _json.DeserializeFromStream<HdHomerunHost.DiscoverResponse>(stream);
diff --git a/MediaBrowser.Server.Implementations/LiveTv/TunerHosts/HdHomerun/HdHomerunHost.cs b/MediaBrowser.Server.Implementations/LiveTv/TunerHosts/HdHomerun/HdHomerunHost.cs
index 14ae45228..101b9ba84 100644
--- a/MediaBrowser.Server.Implementations/LiveTv/TunerHosts/HdHomerun/HdHomerunHost.cs
+++ b/MediaBrowser.Server.Implementations/LiveTv/TunerHosts/HdHomerun/HdHomerunHost.cs
@@ -72,7 +72,8 @@ namespace MediaBrowser.Server.Implementations.LiveTv.TunerHosts.HdHomerun
             var options = new HttpRequestOptions
             {
                 Url = string.Format("{0}/lineup.json", GetApiUrl(info, false)),
-                CancellationToken = cancellationToken
+                CancellationToken = cancellationToken,
+                BufferContent = false
             };
             using (var stream = await _httpClient.Get(options))
             {
@@ -124,7 +125,8 @@ namespace MediaBrowser.Server.Implementations.LiveTv.TunerHosts.HdHomerun
                     CancellationToken = cancellationToken,
                     CacheLength = TimeSpan.FromDays(1),
                     CacheMode = CacheMode.Unconditional,
-                    TimeoutMs = Convert.ToInt32(TimeSpan.FromSeconds(5).TotalMilliseconds)
+                    TimeoutMs = Convert.ToInt32(TimeSpan.FromSeconds(5).TotalMilliseconds),
+                    BufferContent = false
                 }))
                 {
                     var response = JsonSerializer.DeserializeFromStream<DiscoverResponse>(stream);
@@ -165,7 +167,8 @@ namespace MediaBrowser.Server.Implementations.LiveTv.TunerHosts.HdHomerun
             {
                 Url = string.Format("{0}/tuners.html", GetApiUrl(info, false)),
                 CancellationToken = cancellationToken,
-                TimeoutMs = Convert.ToInt32(TimeSpan.FromSeconds(5).TotalMilliseconds)
+                TimeoutMs = Convert.ToInt32(TimeSpan.FromSeconds(5).TotalMilliseconds),
+                BufferContent = false
             }))
             {
                 var tuners = new List<LiveTvTunerInfo>();
@@ -538,7 +541,8 @@ namespace MediaBrowser.Server.Implementations.LiveTv.TunerHosts.HdHomerun
                 using (var stream = await _httpClient.Get(new HttpRequestOptions
                 {
                     Url = string.Format("{0}/discover.json", GetApiUrl(info, false)),
-                    CancellationToken = CancellationToken.None
+                    CancellationToken = CancellationToken.None,
+                    BufferContent = false
                 }))
                 {
                     var response = JsonSerializer.DeserializeFromStream<DiscoverResponse>(stream);
diff --git a/MediaBrowser.Server.Implementations/LiveTv/TunerHosts/HdHomerun/HdHomerunLiveStream.cs b/MediaBrowser.Server.Implementations/LiveTv/TunerHosts/HdHomerun/HdHomerunLiveStream.cs
index dd635bd55..dd7726be0 100644
--- a/MediaBrowser.Server.Implementations/LiveTv/TunerHosts/HdHomerun/HdHomerunLiveStream.cs
+++ b/MediaBrowser.Server.Implementations/LiveTv/TunerHosts/HdHomerun/HdHomerunLiveStream.cs
@@ -12,6 +12,7 @@ using MediaBrowser.Model.Logging;
 using MediaBrowser.Model.MediaInfo;
 using MediaBrowser.Server.Implementations.LiveTv.EmbyTV;
 using System.Collections.Generic;
+using System.Linq;
 
 namespace MediaBrowser.Server.Implementations.LiveTv.TunerHosts.HdHomerun
 {
@@ -139,7 +140,7 @@ namespace MediaBrowser.Server.Implementations.LiveTv.TunerHosts.HdHomerun
             }).ConfigureAwait(false);
         }
 
-        private List<Tuple<Stream, CancellationToken, TaskCompletionSource<bool>>> _additionalStreams = new List<Tuple<Stream, CancellationToken, TaskCompletionSource<bool>>>();
+        private readonly List<Tuple<Stream, CancellationToken, TaskCompletionSource<bool>>> _additionalStreams = new List<Tuple<Stream, CancellationToken, TaskCompletionSource<bool>>>();
 
         public Task CopyToAsync(Stream stream, CancellationToken cancellationToken)
         {
@@ -186,7 +187,8 @@ namespace MediaBrowser.Server.Implementations.LiveTv.TunerHosts.HdHomerun
             {
                 await destination.WriteAsync(buffer, 0, bytesRead, cancellationToken).ConfigureAwait(false);
 
-                foreach (var additionalStream in _additionalStreams)
+                var additionalStreams = _additionalStreams.ToList();
+                foreach (var additionalStream in additionalStreams)
                 {
                     cancellationToken.ThrowIfCancellationRequested();
 
@@ -196,6 +198,8 @@ namespace MediaBrowser.Server.Implementations.LiveTv.TunerHosts.HdHomerun
                     }
                     catch (Exception ex)
                     {
+                        _logger.ErrorException("Error writing HDHR data to stream", ex);
+
                         PopAdditionalStream(additionalStream, ex);
                     }
                 }