From e677a57bf1cedc55214b0e457778311b8f1ea5ac Mon Sep 17 00:00:00 2001 From: Luke Pulverenti Date: Mon, 17 Jun 2013 16:35:43 -0400 Subject: switch to flat file storage --- .../JsonDisplayPreferencesRepository.cs | 164 ++++++++++++++ .../Persistence/JsonItemRepository.cs | 235 ++++++++++++++++++++ .../Persistence/JsonUserDataRepository.cs | 246 +++++++++++++++++++++ .../Persistence/JsonUserRepository.cs | 189 ++++++++++++++++ 4 files changed, 834 insertions(+) create mode 100644 MediaBrowser.Server.Implementations/Persistence/JsonDisplayPreferencesRepository.cs create mode 100644 MediaBrowser.Server.Implementations/Persistence/JsonItemRepository.cs create mode 100644 MediaBrowser.Server.Implementations/Persistence/JsonUserDataRepository.cs create mode 100644 MediaBrowser.Server.Implementations/Persistence/JsonUserRepository.cs (limited to 'MediaBrowser.Server.Implementations/Persistence') diff --git a/MediaBrowser.Server.Implementations/Persistence/JsonDisplayPreferencesRepository.cs b/MediaBrowser.Server.Implementations/Persistence/JsonDisplayPreferencesRepository.cs new file mode 100644 index 0000000000..6ac2ff07a6 --- /dev/null +++ b/MediaBrowser.Server.Implementations/Persistence/JsonDisplayPreferencesRepository.cs @@ -0,0 +1,164 @@ +using MediaBrowser.Common.Configuration; +using MediaBrowser.Controller.Persistence; +using MediaBrowser.Model.Entities; +using MediaBrowser.Model.Logging; +using MediaBrowser.Model.Serialization; +using System; +using System.Collections.Concurrent; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace MediaBrowser.Server.Implementations.Persistence +{ + public class JsonDisplayPreferencesRepository : IDisplayPreferencesRepository + { + private readonly ConcurrentDictionary _fileLocks = new ConcurrentDictionary(); + + private SemaphoreSlim GetLock(string filename) + { + return _fileLocks.GetOrAdd(filename, key => new SemaphoreSlim(1, 1)); + } + + /// + /// Gets the name of the repository + /// + /// The name. + public string Name + { + get + { + return "Json"; + } + } + + /// + /// The _json serializer + /// + private readonly IJsonSerializer _jsonSerializer; + + private readonly string _dataPath; + + /// + /// Initializes a new instance of the class. + /// + /// The app paths. + /// The json serializer. + /// The log manager. + /// + /// jsonSerializer + /// or + /// appPaths + /// + public JsonDisplayPreferencesRepository(IApplicationPaths appPaths, IJsonSerializer jsonSerializer, ILogManager logManager) + { + if (jsonSerializer == null) + { + throw new ArgumentNullException("jsonSerializer"); + } + if (appPaths == null) + { + throw new ArgumentNullException("appPaths"); + } + + _jsonSerializer = jsonSerializer; + _dataPath = Path.Combine(appPaths.DataPath, "display-preferences"); + } + + /// + /// Opens the connection to the database + /// + /// Task. + public Task Initialize() + { + return Task.FromResult(true); + } + + /// + /// Save the display preferences associated with an item in the repo + /// + /// The display preferences. + /// The cancellation token. + /// Task. + /// item + public async Task SaveDisplayPreferences(DisplayPreferences displayPreferences, CancellationToken cancellationToken) + { + if (displayPreferences == null) + { + throw new ArgumentNullException("displayPreferences"); + } + if (displayPreferences.Id == Guid.Empty) + { + throw new ArgumentNullException("displayPreferences.Id"); + } + if (cancellationToken == null) + { + throw new ArgumentNullException("cancellationToken"); + } + + cancellationToken.ThrowIfCancellationRequested(); + + if (!Directory.Exists(_dataPath)) + { + Directory.CreateDirectory(_dataPath); + } + + var path = Path.Combine(_dataPath, displayPreferences.Id + ".json"); + + var semaphore = GetLock(path); + + await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); + + try + { + _jsonSerializer.SerializeToFile(displayPreferences, path); + } + finally + { + semaphore.Release(); + } + } + + /// + /// Gets the display preferences. + /// + /// The display preferences id. + /// Task{DisplayPreferences}. + /// item + public Task GetDisplayPreferences(Guid displayPreferencesId) + { + if (displayPreferencesId == Guid.Empty) + { + throw new ArgumentNullException("displayPreferencesId"); + } + + return Task.Run(() => + { + var path = Path.Combine(_dataPath, displayPreferencesId + ".json"); + + try + { + return _jsonSerializer.DeserializeFromFile(path); + } + catch (IOException) + { + // File doesn't exist or is currently bring written to + return null; + } + }); + } + + public void Dispose() + { + // Wait up to two seconds for any existing writes to finish + var locks = _fileLocks.Values.ToList() + .Where(i => i.CurrentCount == 1) + .Select(i => i.WaitAsync(2000)); + + var task = Task.WhenAll(locks); + + Task.WaitAll(task); + } + } +} diff --git a/MediaBrowser.Server.Implementations/Persistence/JsonItemRepository.cs b/MediaBrowser.Server.Implementations/Persistence/JsonItemRepository.cs new file mode 100644 index 0000000000..d0333e334e --- /dev/null +++ b/MediaBrowser.Server.Implementations/Persistence/JsonItemRepository.cs @@ -0,0 +1,235 @@ +using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.IO; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Persistence; +using MediaBrowser.Model.Entities; +using MediaBrowser.Model.Logging; +using MediaBrowser.Model.Serialization; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace MediaBrowser.Server.Implementations.Persistence +{ + public class JsonItemRepository : IItemRepository + { + private readonly ConcurrentDictionary _fileLocks = new ConcurrentDictionary(); + + private SemaphoreSlim GetLock(string filename) + { + return _fileLocks.GetOrAdd(filename, key => new SemaphoreSlim(1, 1)); + } + + /// + /// Gets the name of the repository + /// + /// The name. + public string Name + { + get + { + return "Json"; + } + } + + /// + /// Gets the json serializer. + /// + /// The json serializer. + private readonly IJsonSerializer _jsonSerializer; + + private readonly string _criticReviewsPath; + + private readonly FileSystemRepository _itemRepo; + + /// + /// Initializes a new instance of the class. + /// + /// The app paths. + /// The json serializer. + /// The log manager. + /// appPaths + public JsonItemRepository(IApplicationPaths appPaths, IJsonSerializer jsonSerializer, ILogManager logManager) + { + if (appPaths == null) + { + throw new ArgumentNullException("appPaths"); + } + if (jsonSerializer == null) + { + throw new ArgumentNullException("jsonSerializer"); + } + + _jsonSerializer = jsonSerializer; + + _criticReviewsPath = Path.Combine(appPaths.DataPath, "critic-reviews"); + + _itemRepo = new FileSystemRepository(Path.Combine(appPaths.DataPath, "library")); + } + + /// + /// Opens the connection to the database + /// + /// Task. + public Task Initialize() + { + return Task.FromResult(true); + } + + /// + /// Save a standard item in the repo + /// + /// The item. + /// The cancellation token. + /// Task. + /// item + public async Task SaveItem(BaseItem item, CancellationToken cancellationToken) + { + if (item == null) + { + throw new ArgumentNullException("item"); + } + + if (!Directory.Exists(_criticReviewsPath)) + { + Directory.CreateDirectory(_criticReviewsPath); + } + + var path = _itemRepo.GetResourcePath(item.Id + ".json"); + + var parentPath = Path.GetDirectoryName(path); + if (!Directory.Exists(parentPath)) + { + Directory.CreateDirectory(parentPath); + } + + var semaphore = GetLock(path); + + await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); + + try + { + _jsonSerializer.SerializeToFile(item, path); + } + finally + { + semaphore.Release(); + } + } + + /// + /// Saves the items. + /// + /// The items. + /// The cancellation token. + /// Task. + /// + /// items + /// or + /// cancellationToken + /// + public Task SaveItems(IEnumerable items, CancellationToken cancellationToken) + { + if (items == null) + { + throw new ArgumentNullException("items"); + } + + if (cancellationToken == null) + { + throw new ArgumentNullException("cancellationToken"); + } + + var tasks = items.Select(i => SaveItem(i, cancellationToken)); + + return Task.WhenAll(tasks); + } + + /// + /// Retrieves the item. + /// + /// The id. + /// The type. + /// BaseItem. + /// id + public BaseItem RetrieveItem(Guid id, Type type) + { + if (id == Guid.Empty) + { + throw new ArgumentNullException("id"); + } + + var path = _itemRepo.GetResourcePath(id + ".json"); + + try + { + return (BaseItem)_jsonSerializer.DeserializeFromFile(type, path); + } + catch (IOException) + { + // File doesn't exist or is currently bring written to + return null; + } + } + + /// + /// Gets the critic reviews. + /// + /// The item id. + /// Task{IEnumerable{ItemReview}}. + public Task> GetCriticReviews(Guid itemId) + { + return Task.Run>(() => + { + var path = Path.Combine(_criticReviewsPath, itemId + ".json"); + + try + { + return _jsonSerializer.DeserializeFromFile>(path); + } + catch (IOException) + { + // File doesn't exist or is currently bring written to + return new List(); + } + }); + } + + /// + /// Saves the critic reviews. + /// + /// The item id. + /// The critic reviews. + /// Task. + public Task SaveCriticReviews(Guid itemId, IEnumerable criticReviews) + { + return Task.Run(() => + { + if (!Directory.Exists(_criticReviewsPath)) + { + Directory.CreateDirectory(_criticReviewsPath); + } + + var path = Path.Combine(_criticReviewsPath, itemId + ".json"); + + _jsonSerializer.SerializeToFile(criticReviews.ToList(), path); + }); + } + + public void Dispose() + { + // Wait up to two seconds for any existing writes to finish + var locks = _fileLocks.Values.ToList() + .Where(i => i.CurrentCount == 1) + .Select(i => i.WaitAsync(2000)); + + var task = Task.WhenAll(locks); + + Task.WaitAll(task); + } + } +} diff --git a/MediaBrowser.Server.Implementations/Persistence/JsonUserDataRepository.cs b/MediaBrowser.Server.Implementations/Persistence/JsonUserDataRepository.cs new file mode 100644 index 0000000000..2f1129bebc --- /dev/null +++ b/MediaBrowser.Server.Implementations/Persistence/JsonUserDataRepository.cs @@ -0,0 +1,246 @@ +using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.Extensions; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Persistence; +using MediaBrowser.Model.Logging; +using MediaBrowser.Model.Serialization; +using System; +using System.Collections.Concurrent; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace MediaBrowser.Server.Implementations.Persistence +{ + public class JsonUserDataRepository : IUserDataRepository + { + private readonly ConcurrentDictionary _fileLocks = new ConcurrentDictionary(); + + private SemaphoreSlim GetLock(string filename) + { + return _fileLocks.GetOrAdd(filename, key => new SemaphoreSlim(1, 1)); + } + + private readonly ConcurrentDictionary _userData = new ConcurrentDictionary(); + + /// + /// Gets the name of the repository + /// + /// The name. + public string Name + { + get + { + return "Json"; + } + } + + private readonly IJsonSerializer _jsonSerializer; + + private readonly string _dataPath; + + private readonly ILogger _logger; + + /// + /// Initializes a new instance of the class. + /// + /// The app paths. + /// The json serializer. + /// The log manager. + /// + /// jsonSerializer + /// or + /// appPaths + /// + public JsonUserDataRepository(IApplicationPaths appPaths, IJsonSerializer jsonSerializer, ILogManager logManager) + { + if (jsonSerializer == null) + { + throw new ArgumentNullException("jsonSerializer"); + } + if (appPaths == null) + { + throw new ArgumentNullException("appPaths"); + } + + _logger = logManager.GetLogger(GetType().Name); + _jsonSerializer = jsonSerializer; + _dataPath = Path.Combine(appPaths.DataPath, "userdata"); + } + + /// + /// Opens the connection to the database + /// + /// Task. + public Task Initialize() + { + return Task.FromResult(true); + } + + /// + /// Saves the user data. + /// + /// The user id. + /// The key. + /// The user data. + /// The cancellation token. + /// Task. + /// userData + /// or + /// cancellationToken + /// or + /// userId + /// or + /// userDataId + public async Task SaveUserData(Guid userId, string key, UserItemData userData, CancellationToken cancellationToken) + { + if (userData == null) + { + throw new ArgumentNullException("userData"); + } + if (cancellationToken == null) + { + throw new ArgumentNullException("cancellationToken"); + } + if (userId == Guid.Empty) + { + throw new ArgumentNullException("userId"); + } + if (string.IsNullOrEmpty(key)) + { + throw new ArgumentNullException("key"); + } + + cancellationToken.ThrowIfCancellationRequested(); + + try + { + await PersistUserData(userId, key, userData, cancellationToken).ConfigureAwait(false); + + // Once it succeeds, put it into the dictionary to make it available to everyone else + _userData.AddOrUpdate(GetInternalKey(userId, key), userData, delegate { return userData; }); + } + catch (Exception ex) + { + _logger.ErrorException("Error saving user data", ex); + + throw; + } + } + + /// + /// Gets the internal key. + /// + /// The user id. + /// The key. + /// System.String. + private string GetInternalKey(Guid userId, string key) + { + return userId + key; + } + + /// + /// Persists the user data. + /// + /// The user id. + /// The key. + /// The user data. + /// The cancellation token. + /// Task. + public async Task PersistUserData(Guid userId, string key, UserItemData userData, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + var path = GetUserDataPath(userId, key); + + var parentPath = Path.GetDirectoryName(path); + if (!Directory.Exists(parentPath)) + { + Directory.CreateDirectory(parentPath); + } + + var semaphore = GetLock(path); + + await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); + + try + { + _jsonSerializer.SerializeToFile(userData, path); + } + finally + { + semaphore.Release(); + } + } + + /// + /// Gets the user data. + /// + /// The user id. + /// The key. + /// Task{UserItemData}. + /// + /// userId + /// or + /// key + /// + public UserItemData GetUserData(Guid userId, string key) + { + if (userId == Guid.Empty) + { + throw new ArgumentNullException("userId"); + } + if (string.IsNullOrEmpty(key)) + { + throw new ArgumentNullException("key"); + } + + return _userData.GetOrAdd(GetInternalKey(userId, key), keyName => RetrieveUserData(userId, key)); + } + + /// + /// Retrieves the user data. + /// + /// The user id. + /// The key. + /// Task{UserItemData}. + private UserItemData RetrieveUserData(Guid userId, string key) + { + var path = GetUserDataPath(userId, key); + + try + { + return _jsonSerializer.DeserializeFromFile(path); + } + catch (IOException) + { + // File doesn't exist or is currently bring written to + return new UserItemData { UserId = userId }; + } + } + + private string GetUserDataPath(Guid userId, string key) + { + var userFolder = Path.Combine(_dataPath, userId.ToString()); + + var keyHash = key.GetMD5().ToString(); + + var prefix = keyHash.Substring(0, 1); + + return Path.Combine(userFolder, prefix, keyHash + ".json"); + } + + public void Dispose() + { + // Wait up to two seconds for any existing writes to finish + var locks = _fileLocks.Values.ToList() + .Where(i => i.CurrentCount == 1) + .Select(i => i.WaitAsync(2000)); + + var task = Task.WhenAll(locks); + + Task.WaitAll(task); + } + } +} \ No newline at end of file diff --git a/MediaBrowser.Server.Implementations/Persistence/JsonUserRepository.cs b/MediaBrowser.Server.Implementations/Persistence/JsonUserRepository.cs new file mode 100644 index 0000000000..0573c6e2ef --- /dev/null +++ b/MediaBrowser.Server.Implementations/Persistence/JsonUserRepository.cs @@ -0,0 +1,189 @@ +using MediaBrowser.Common.Configuration; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Persistence; +using MediaBrowser.Model.Logging; +using MediaBrowser.Model.Serialization; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace MediaBrowser.Server.Implementations.Persistence +{ + public class JsonUserRepository : IUserRepository + { + private readonly ConcurrentDictionary _fileLocks = new ConcurrentDictionary(); + + private SemaphoreSlim GetLock(string filename) + { + return _fileLocks.GetOrAdd(filename, key => new SemaphoreSlim(1, 1)); + } + + /// + /// Gets the name of the repository + /// + /// The name. + public string Name + { + get + { + return "Json"; + } + } + + /// + /// Gets the json serializer. + /// + /// The json serializer. + private readonly IJsonSerializer _jsonSerializer; + + private readonly string _dataPath; + + /// + /// Initializes a new instance of the class. + /// + /// The app paths. + /// The json serializer. + /// The log manager. + /// + /// appPaths + /// or + /// jsonSerializer + /// + public JsonUserRepository(IApplicationPaths appPaths, IJsonSerializer jsonSerializer, ILogManager logManager) + { + if (appPaths == null) + { + throw new ArgumentNullException("appPaths"); + } + if (jsonSerializer == null) + { + throw new ArgumentNullException("jsonSerializer"); + } + + _jsonSerializer = jsonSerializer; + + _dataPath = Path.Combine(appPaths.DataPath, "users"); + } + + /// + /// Opens the connection to the database + /// + /// Task. + public Task Initialize() + { + return Task.FromResult(true); + } + + /// + /// Save a user in the repo + /// + /// The user. + /// The cancellation token. + /// Task. + /// user + public async Task SaveUser(User user, CancellationToken cancellationToken) + { + if (user == null) + { + throw new ArgumentNullException("user"); + } + + if (cancellationToken == null) + { + throw new ArgumentNullException("cancellationToken"); + } + + cancellationToken.ThrowIfCancellationRequested(); + + if (!Directory.Exists(_dataPath)) + { + Directory.CreateDirectory(_dataPath); + } + + var path = Path.Combine(_dataPath, user.Id + ".json"); + + var semaphore = GetLock(path); + + await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); + + try + { + _jsonSerializer.SerializeToFile(user, path); + } + finally + { + semaphore.Release(); + } + } + + /// + /// Retrieve all users from the database + /// + /// IEnumerable{User}. + public IEnumerable RetrieveAllUsers() + { + try + { + return Directory.EnumerateFiles(_dataPath, "*.json", SearchOption.TopDirectoryOnly) + .Select(i => _jsonSerializer.DeserializeFromFile(i)); + } + catch (IOException) + { + return new List(); + } + } + + /// + /// Deletes the user. + /// + /// The user. + /// The cancellation token. + /// Task. + /// user + public async Task DeleteUser(User user, CancellationToken cancellationToken) + { + if (user == null) + { + throw new ArgumentNullException("user"); + } + + if (cancellationToken == null) + { + throw new ArgumentNullException("cancellationToken"); + } + + cancellationToken.ThrowIfCancellationRequested(); + + var path = Path.Combine(_dataPath, user.Id + ".json"); + + var semaphore = GetLock(path); + + await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); + + try + { + File.Delete(path); + } + finally + { + semaphore.Release(); + } + } + + public void Dispose() + { + // Wait up to two seconds for any existing writes to finish + var locks = _fileLocks.Values.ToList() + .Where(i => i.CurrentCount == 1) + .Select(i => i.WaitAsync(2000)); + + var task = Task.WhenAll(locks); + + Task.WaitAll(task); + } + } +} -- cgit v1.2.3