From 63c9104e624d0d22e6c5baf79db2d7bb9deb74d0 Mon Sep 17 00:00:00 2001 From: Luke Pulverenti Date: Fri, 6 Dec 2013 15:07:34 -0500 Subject: Pull ProviderData out of memory --- .../Library/UserManager.cs | 5 + .../LiveTv/ChannelImageProvider.cs | 6 +- .../MediaBrowser.Server.Implementations.csproj | 1 + .../Persistence/SqliteItemRepository.cs | 29 ++- .../Persistence/SqliteProviderInfoRepository.cs | 265 +++++++++++++++++++++ .../Persistence/SqliteUserDataRepository.cs | 2 +- .../Providers/ProviderManager.cs | 39 ++- 7 files changed, 331 insertions(+), 16 deletions(-) create mode 100644 MediaBrowser.Server.Implementations/Persistence/SqliteProviderInfoRepository.cs (limited to 'MediaBrowser.Server.Implementations') diff --git a/MediaBrowser.Server.Implementations/Library/UserManager.cs b/MediaBrowser.Server.Implementations/Library/UserManager.cs index 04d0f6ab2..4243aecfe 100644 --- a/MediaBrowser.Server.Implementations/Library/UserManager.cs +++ b/MediaBrowser.Server.Implementations/Library/UserManager.cs @@ -196,6 +196,8 @@ namespace MediaBrowser.Server.Implementations.Library var user = InstantiateNewUser(name); + user.DateLastSaved = DateTime.UtcNow; + var task = UserRepository.SaveUser(user, CancellationToken.None); // Hate having to block threads @@ -274,6 +276,7 @@ namespace MediaBrowser.Server.Implementations.Library } user.DateModified = DateTime.UtcNow; + user.DateLastSaved = DateTime.UtcNow; await UserRepository.SaveUser(user, CancellationToken.None).ConfigureAwait(false); @@ -307,6 +310,8 @@ namespace MediaBrowser.Server.Implementations.Library list.Add(user); Users = list; + user.DateLastSaved = DateTime.UtcNow; + await UserRepository.SaveUser(user, CancellationToken.None).ConfigureAwait(false); EventHelper.QueueEventIfNotNull(UserCreated, this, new GenericEventArgs { Argument = user }, _logger); diff --git a/MediaBrowser.Server.Implementations/LiveTv/ChannelImageProvider.cs b/MediaBrowser.Server.Implementations/LiveTv/ChannelImageProvider.cs index 322948bad..e16430e69 100644 --- a/MediaBrowser.Server.Implementations/LiveTv/ChannelImageProvider.cs +++ b/MediaBrowser.Server.Implementations/LiveTv/ChannelImageProvider.cs @@ -36,11 +36,11 @@ namespace MediaBrowser.Server.Implementations.LiveTv return !item.HasImage(ImageType.Primary); } - public override async Task FetchAsync(BaseItem item, bool force, CancellationToken cancellationToken) + public override async Task FetchAsync(BaseItem item, bool force, BaseProviderInfo providerInfo, CancellationToken cancellationToken) { if (item.HasImage(ImageType.Primary)) { - SetLastRefreshed(item, DateTime.UtcNow); + SetLastRefreshed(item, DateTime.UtcNow, providerInfo); return true; } @@ -58,7 +58,7 @@ namespace MediaBrowser.Server.Implementations.LiveTv } - SetLastRefreshed(item, DateTime.UtcNow); + SetLastRefreshed(item, DateTime.UtcNow, providerInfo); return true; } diff --git a/MediaBrowser.Server.Implementations/MediaBrowser.Server.Implementations.csproj b/MediaBrowser.Server.Implementations/MediaBrowser.Server.Implementations.csproj index a4eec43f7..4b30672d0 100644 --- a/MediaBrowser.Server.Implementations/MediaBrowser.Server.Implementations.csproj +++ b/MediaBrowser.Server.Implementations/MediaBrowser.Server.Implementations.csproj @@ -170,6 +170,7 @@ + diff --git a/MediaBrowser.Server.Implementations/Persistence/SqliteItemRepository.cs b/MediaBrowser.Server.Implementations/Persistence/SqliteItemRepository.cs index 799b74fe2..893d6ea62 100644 --- a/MediaBrowser.Server.Implementations/Persistence/SqliteItemRepository.cs +++ b/MediaBrowser.Server.Implementations/Persistence/SqliteItemRepository.cs @@ -1,6 +1,7 @@ using MediaBrowser.Common.Configuration; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Persistence; +using MediaBrowser.Controller.Providers; using MediaBrowser.Model.Entities; using MediaBrowser.Model.Logging; using MediaBrowser.Model.Serialization; @@ -57,6 +58,7 @@ namespace MediaBrowser.Server.Implementations.Persistence private SqliteChapterRepository _chapterRepository; private SqliteMediaStreamsRepository _mediaStreamsRepository; + private SqliteProviderInfoRepository _providerInfoRepository; private IDbCommand _deleteChildrenCommand; private IDbCommand _saveChildrenCommand; @@ -91,16 +93,16 @@ namespace MediaBrowser.Server.Implementations.Persistence _logger = logManager.GetLogger(GetType().Name); var chapterDbFile = Path.Combine(_appPaths.DataPath, "chapters.db"); - var chapterConnection = SqliteExtensions.ConnectToDb(chapterDbFile, _logger).Result; - _chapterRepository = new SqliteChapterRepository(chapterConnection, logManager); var mediaStreamsDbFile = Path.Combine(_appPaths.DataPath, "mediainfo.db"); - var mediaStreamsConnection = SqliteExtensions.ConnectToDb(mediaStreamsDbFile, _logger).Result; - _mediaStreamsRepository = new SqliteMediaStreamsRepository(mediaStreamsConnection, logManager); + + var providerInfosDbFile = Path.Combine(_appPaths.DataPath, "providerinfo.db"); + var providerInfoConnection = SqliteExtensions.ConnectToDb(providerInfosDbFile, _logger).Result; + _providerInfoRepository = new SqliteProviderInfoRepository(providerInfoConnection, logManager); } /// @@ -128,8 +130,9 @@ namespace MediaBrowser.Server.Implementations.Persistence _connection.RunQueries(queries, _logger); PrepareStatements(); - + _mediaStreamsRepository.Initialize(); + _providerInfoRepository.Initialize(); _chapterRepository.Initialize(); } @@ -427,6 +430,12 @@ namespace MediaBrowser.Server.Implementations.Persistence _mediaStreamsRepository.Dispose(); _mediaStreamsRepository = null; } + + if (_providerInfoRepository != null) + { + _providerInfoRepository.Dispose(); + _providerInfoRepository = null; + } } } @@ -535,5 +544,15 @@ namespace MediaBrowser.Server.Implementations.Persistence { return _mediaStreamsRepository.SaveMediaStreams(id, streams, cancellationToken); } + + public IEnumerable GetProviderHistory(Guid itemId) + { + return _providerInfoRepository.GetBaseProviderInfos(itemId); + } + + public Task SaveProviderHistory(Guid id, IEnumerable history, CancellationToken cancellationToken) + { + return _providerInfoRepository.SaveProviderInfos(id, history, cancellationToken); + } } } \ No newline at end of file diff --git a/MediaBrowser.Server.Implementations/Persistence/SqliteProviderInfoRepository.cs b/MediaBrowser.Server.Implementations/Persistence/SqliteProviderInfoRepository.cs new file mode 100644 index 000000000..c704808f3 --- /dev/null +++ b/MediaBrowser.Server.Implementations/Persistence/SqliteProviderInfoRepository.cs @@ -0,0 +1,265 @@ +using MediaBrowser.Controller.Providers; +using MediaBrowser.Model.Logging; +using System; +using System.Collections.Generic; +using System.Data; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace MediaBrowser.Server.Implementations.Persistence +{ + class SqliteProviderInfoRepository + { + private IDbConnection _connection; + + private readonly ILogger _logger; + + private IDbCommand _deleteInfosCommand; + private IDbCommand _saveInfoCommand; + + public SqliteProviderInfoRepository(IDbConnection connection, ILogManager logManager) + { + _connection = connection; + + _logger = logManager.GetLogger(GetType().Name); + } + + /// + /// Opens the connection to the database + /// + /// Task. + public void Initialize() + { + var createTableCommand + = "create table if not exists providerinfos "; + + createTableCommand += "(ItemId GUID, ProviderId GUID, ProviderVersion TEXT, FileStamp GUID, LastRefreshStatus TEXT, LastRefreshed datetime, PRIMARY KEY (ItemId, ProviderId))"; + + string[] queries = { + + createTableCommand, + "create index if not exists idx_providerinfos on providerinfos(ItemId, ProviderId)", + + //pragmas + "pragma temp_store = memory" + }; + + _connection.RunQueries(queries, _logger); + + PrepareStatements(); + } + + private static readonly string[] SaveColumns = + { + "ItemId", + "ProviderId", + "ProviderVersion", + "FileStamp", + "LastRefreshStatus", + "LastRefreshed" + }; + + private readonly string[] _selectColumns = SaveColumns.Skip(1).ToArray(); + + /// + /// The _write lock + /// + private readonly SemaphoreSlim _writeLock = new SemaphoreSlim(1, 1); + + /// + /// Prepares the statements. + /// + private void PrepareStatements() + { + _deleteInfosCommand = _connection.CreateCommand(); + _deleteInfosCommand.CommandText = "delete from providerinfos where ItemId=@ItemId"; + _deleteInfosCommand.Parameters.Add(_deleteInfosCommand, "@ItemId"); + + _saveInfoCommand = _connection.CreateCommand(); + + _saveInfoCommand.CommandText = string.Format("replace into providerinfos ({0}) values ({1})", + string.Join(",", SaveColumns), + string.Join(",", SaveColumns.Select(i => "@" + i).ToArray())); + + foreach (var col in SaveColumns) + { + _saveInfoCommand.Parameters.Add(_saveInfoCommand, "@" + col); + } + } + + public IEnumerable GetBaseProviderInfos(Guid itemId) + { + if (itemId == Guid.Empty) + { + throw new ArgumentNullException("itemId"); + } + + using (var cmd = _connection.CreateCommand()) + { + var cmdText = "select " + string.Join(",", _selectColumns) + " from providerinfos where"; + + cmdText += " ItemId=@ItemId"; + cmd.Parameters.Add(cmd, "@ItemId", DbType.Guid).Value = itemId; + + cmd.CommandText = cmdText; + + using (var reader = cmd.ExecuteReader(CommandBehavior.SequentialAccess | CommandBehavior.SingleResult)) + { + while (reader.Read()) + { + yield return GetBaseProviderInfo(reader); + } + } + } + } + + /// + /// Gets the chapter. + /// + /// The reader. + /// ChapterInfo. + private BaseProviderInfo GetBaseProviderInfo(IDataReader reader) + { + var item = new BaseProviderInfo + { + ProviderId = reader.GetGuid(0) + }; + + if (!reader.IsDBNull(1)) + { + item.ProviderVersion = reader.GetString(1); + } + + item.FileStamp = reader.GetGuid(2); + item.LastRefreshStatus = (ProviderRefreshStatus)Enum.Parse(typeof(ProviderRefreshStatus), reader.GetString(3), true); + item.LastRefreshed = reader.GetDateTime(4).ToUniversalTime(); + + return item; + } + + public async Task SaveProviderInfos(Guid id, IEnumerable infos, CancellationToken cancellationToken) + { + if (id == Guid.Empty) + { + throw new ArgumentNullException("id"); + } + + if (infos == null) + { + throw new ArgumentNullException("infos"); + } + + cancellationToken.ThrowIfCancellationRequested(); + + await _writeLock.WaitAsync(cancellationToken).ConfigureAwait(false); + + IDbTransaction transaction = null; + + try + { + transaction = _connection.BeginTransaction(); + + // First delete chapters + _deleteInfosCommand.GetParameter(0).Value = id; + + _deleteInfosCommand.Transaction = transaction; + + _deleteInfosCommand.ExecuteNonQuery(); + + foreach (var stream in infos) + { + if (stream.LastRefreshed == DateTime.MinValue) + { + throw new Exception("LastRefreshed still has DateTime.MinValue"); + + } + cancellationToken.ThrowIfCancellationRequested(); + + _saveInfoCommand.GetParameter(0).Value = id; + _saveInfoCommand.GetParameter(1).Value = stream.ProviderId; + _saveInfoCommand.GetParameter(2).Value = stream.ProviderVersion; + _saveInfoCommand.GetParameter(3).Value = stream.FileStamp; + _saveInfoCommand.GetParameter(4).Value = stream.LastRefreshStatus.ToString(); + _saveInfoCommand.GetParameter(5).Value = stream.LastRefreshed; + + _saveInfoCommand.Transaction = transaction; + _saveInfoCommand.ExecuteNonQuery(); + } + + transaction.Commit(); + } + catch (OperationCanceledException) + { + if (transaction != null) + { + transaction.Rollback(); + } + + throw; + } + catch (Exception e) + { + _logger.ErrorException("Failed to save provider info:", e); + + if (transaction != null) + { + transaction.Rollback(); + } + + throw; + } + finally + { + if (transaction != null) + { + transaction.Dispose(); + } + + _writeLock.Release(); + } + } + + /// + /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. + /// + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + private readonly object _disposeLock = new object(); + + /// + /// Releases unmanaged and - optionally - managed resources. + /// + /// true to release both managed and unmanaged resources; false to release only unmanaged resources. + protected virtual void Dispose(bool dispose) + { + if (dispose) + { + try + { + lock (_disposeLock) + { + if (_connection != null) + { + if (_connection.IsOpen()) + { + _connection.Close(); + } + + _connection.Dispose(); + _connection = null; + } + } + } + catch (Exception ex) + { + _logger.ErrorException("Error disposing database", ex); + } + } + } + } +} diff --git a/MediaBrowser.Server.Implementations/Persistence/SqliteUserDataRepository.cs b/MediaBrowser.Server.Implementations/Persistence/SqliteUserDataRepository.cs index c174eb08e..fa195859b 100644 --- a/MediaBrowser.Server.Implementations/Persistence/SqliteUserDataRepository.cs +++ b/MediaBrowser.Server.Implementations/Persistence/SqliteUserDataRepository.cs @@ -233,7 +233,7 @@ namespace MediaBrowser.Server.Implementations.Persistence if (!reader.IsDBNull(5)) { - userData.LastPlayedDate = reader.GetDateTime(5); + userData.LastPlayedDate = reader.GetDateTime(5).ToUniversalTime(); } } } diff --git a/MediaBrowser.Server.Implementations/Providers/ProviderManager.cs b/MediaBrowser.Server.Implementations/Providers/ProviderManager.cs index fa0620082..511092759 100644 --- a/MediaBrowser.Server.Implementations/Providers/ProviderManager.cs +++ b/MediaBrowser.Server.Implementations/Providers/ProviderManager.cs @@ -4,6 +4,7 @@ using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.IO; using MediaBrowser.Controller.Library; +using MediaBrowser.Controller.Persistence; using MediaBrowser.Controller.Providers; using MediaBrowser.Model.Entities; using MediaBrowser.Model.Logging; @@ -52,6 +53,8 @@ namespace MediaBrowser.Server.Implementations.Providers private IImageProvider[] ImageProviders { get; set; } private readonly IFileSystem _fileSystem; + private readonly IItemRepository _itemRepo; + /// /// Initializes a new instance of the class. /// @@ -59,13 +62,14 @@ namespace MediaBrowser.Server.Implementations.Providers /// The configuration manager. /// The directory watchers. /// The log manager. - public ProviderManager(IHttpClient httpClient, IServerConfigurationManager configurationManager, IDirectoryWatchers directoryWatchers, ILogManager logManager, IFileSystem fileSystem) + public ProviderManager(IHttpClient httpClient, IServerConfigurationManager configurationManager, IDirectoryWatchers directoryWatchers, ILogManager logManager, IFileSystem fileSystem, IItemRepository itemRepo) { _logger = logManager.GetLogger("ProviderManager"); _httpClient = httpClient; ConfigurationManager = configurationManager; _directoryWatchers = directoryWatchers; _fileSystem = fileSystem; + _itemRepo = itemRepo; } /// @@ -102,6 +106,10 @@ namespace MediaBrowser.Server.Implementations.Providers var enableInternetProviders = ConfigurationManager.Configuration.EnableInternetProviders; var excludeTypes = ConfigurationManager.Configuration.InternetProviderExcludeTypes; + var providerHistories = item.DateLastSaved == DateTime.MinValue ? + new List() : + _itemRepo.GetProviderHistory(item.Id).ToList(); + // Run the normal providers sequentially in order of priority foreach (var provider in MetadataProviders) { @@ -140,9 +148,20 @@ namespace MediaBrowser.Server.Implementations.Providers continue; } + var providerInfo = providerHistories.FirstOrDefault(i => i.ProviderId == provider.Id); + + if (providerInfo == null) + { + providerInfo = new BaseProviderInfo + { + ProviderId = provider.Id + }; + providerHistories.Add(providerInfo); + } + try { - if (!force && !provider.NeedsRefresh(item)) + if (!force && !provider.NeedsRefresh(item, providerInfo)) { continue; } @@ -152,7 +171,7 @@ namespace MediaBrowser.Server.Implementations.Providers _logger.Error("Error determining NeedsRefresh for {0}", ex, item.Path); } - var updateType = await FetchAsync(provider, item, force, cancellationToken).ConfigureAwait(false); + var updateType = await FetchAsync(provider, item, providerInfo, force, cancellationToken).ConfigureAwait(false); if (updateType.HasValue) { @@ -167,6 +186,11 @@ namespace MediaBrowser.Server.Implementations.Providers } } + if (result.HasValue || force) + { + await _itemRepo.SaveProviderHistory(item.Id, providerHistories, cancellationToken); + } + return result; } @@ -194,11 +218,12 @@ namespace MediaBrowser.Server.Implementations.Providers /// /// The provider. /// The item. + /// The provider information. /// if set to true [force]. /// The cancellation token. /// Task{System.Boolean}. - /// - private async Task FetchAsync(BaseMetadataProvider provider, BaseItem item, bool force, CancellationToken cancellationToken) + /// item + private async Task FetchAsync(BaseMetadataProvider provider, BaseItem item, BaseProviderInfo providerInfo, bool force, CancellationToken cancellationToken) { if (item == null) { @@ -215,7 +240,7 @@ namespace MediaBrowser.Server.Implementations.Providers try { - var changed = await provider.FetchAsync(item, force, cancellationToken).ConfigureAwait(false); + var changed = await provider.FetchAsync(item, force, providerInfo, cancellationToken).ConfigureAwait(false); if (changed) { @@ -240,7 +265,7 @@ namespace MediaBrowser.Server.Implementations.Providers { _logger.ErrorException("{0} failed refreshing {1} {2}", ex, provider.GetType().Name, item.Name, item.Path ?? string.Empty); - provider.SetLastRefreshed(item, DateTime.UtcNow, ProviderRefreshStatus.Failure); + provider.SetLastRefreshed(item, DateTime.UtcNow, providerInfo, ProviderRefreshStatus.Failure); return ItemUpdateType.Unspecified; } -- cgit v1.2.3