From 04d480297dc9b4f9a1a9e35fe33f424ecc7662e1 Mon Sep 17 00:00:00 2001 From: Luke Pulverenti Date: Wed, 3 Aug 2016 02:38:19 -0400 Subject: reduce sync db access --- .../Dto/DtoService.cs | 113 +++++++++------------ .../Sync/SyncManager.cs | 2 +- .../Sync/SyncRepository.cs | 99 ++++++++++++++---- 3 files changed, 130 insertions(+), 84 deletions(-) (limited to 'MediaBrowser.Server.Implementations') diff --git a/MediaBrowser.Server.Implementations/Dto/DtoService.cs b/MediaBrowser.Server.Implementations/Dto/DtoService.cs index 90111ad8e..779afdcf2 100644 --- a/MediaBrowser.Server.Implementations/Dto/DtoService.cs +++ b/MediaBrowser.Server.Implementations/Dto/DtoService.cs @@ -97,8 +97,7 @@ namespace MediaBrowser.Server.Implementations.Dto throw new ArgumentNullException("options"); } - var syncJobItems = GetSyncedItemProgress(options); - var syncDictionary = GetSyncedItemProgressDictionary(syncJobItems); + var syncDictionary = GetSyncedItemProgress(options); var list = new List(); var programTuples = new List>(); @@ -106,7 +105,7 @@ namespace MediaBrowser.Server.Implementations.Dto foreach (var item in items) { - var dto = await GetBaseItemDtoInternal(item, options, syncDictionary, user, owner).ConfigureAwait(false); + var dto = await GetBaseItemDtoInternal(item, options, user, owner).ConfigureAwait(false); var tvChannel = item as LiveTvChannel; if (tvChannel != null) @@ -133,7 +132,7 @@ namespace MediaBrowser.Server.Implementations.Dto } } - FillSyncInfo(dto, item, options, user, syncJobItems); + FillSyncInfo(dto, item, options, user, syncDictionary); list.Add(dto); } @@ -151,23 +150,11 @@ namespace MediaBrowser.Server.Implementations.Dto return list; } - private Dictionary GetSyncedItemProgressDictionary(IEnumerable items) - { - var dict = new Dictionary(); - - foreach (var item in items) - { - dict[item.ItemId] = item; - } - - return dict; - } - public BaseItemDto GetBaseItemDto(BaseItem item, DtoOptions options, User user = null, BaseItem owner = null) { - var syncProgress = GetSyncedItemProgress(options); + var syncDictionary = GetSyncedItemProgress(options); - var dto = GetBaseItemDtoInternal(item, options, GetSyncedItemProgressDictionary(syncProgress), user, owner).Result; + var dto = GetBaseItemDtoInternal(item, options, user, owner).Result; var tvChannel = item as LiveTvChannel; if (tvChannel != null) { @@ -190,11 +177,11 @@ namespace MediaBrowser.Server.Implementations.Dto SetItemByNameInfo(item, dto, GetTaggedItems(byName, user), user); } - FillSyncInfo(dto, item, options, user, syncProgress); + FillSyncInfo(dto, item, options, user, syncDictionary); return dto; } - FillSyncInfo(dto, item, options, user, syncProgress); + FillSyncInfo(dto, item, options, user, syncDictionary); return dto; } @@ -210,23 +197,24 @@ namespace MediaBrowser.Server.Implementations.Dto return items; } - private SyncedItemProgress[] GetSyncedItemProgress(DtoOptions options) + public Dictionary GetSyncedItemProgress(DtoOptions options) { - if (!options.Fields.Contains(ItemFields.SyncInfo)) + if (!options.Fields.Contains(ItemFields.BasicSyncInfo) && + !options.Fields.Contains(ItemFields.SyncInfo)) { - return new SyncedItemProgress[] { }; + return new Dictionary(); } var deviceId = options.DeviceId; if (string.IsNullOrWhiteSpace(deviceId)) { - return new SyncedItemProgress[] { }; + return new Dictionary(); } var caps = _deviceManager().GetCapabilities(deviceId); if (caps == null || !caps.SupportsSync) { - return new SyncedItemProgress[] { }; + return new Dictionary(); } return _syncManager.GetSyncedItemProgresses(new SyncJobItemQuery @@ -240,12 +228,13 @@ namespace MediaBrowser.Server.Implementations.Dto SyncJobItemStatus.ReadyToTransfer, SyncJobItemStatus.Synced } - }).Items; + }); } public void FillSyncInfo(IEnumerable> tuples, DtoOptions options, User user) { - if (options.Fields.Contains(ItemFields.SyncInfo)) + if (options.Fields.Contains(ItemFields.BasicSyncInfo) || + options.Fields.Contains(ItemFields.SyncInfo)) { var syncProgress = GetSyncedItemProgress(options); @@ -258,37 +247,40 @@ namespace MediaBrowser.Server.Implementations.Dto } } - private void FillSyncInfo(IHasSyncInfo dto, BaseItem item, DtoOptions options, User user, SyncedItemProgress[] syncProgress) + private void FillSyncInfo(IHasSyncInfo dto, BaseItem item, DtoOptions options, User user, Dictionary syncProgress) { var hasFullSyncInfo = options.Fields.Contains(ItemFields.SyncInfo); - if (hasFullSyncInfo || options.Fields.Contains(ItemFields.BasicSyncInfo)) + if (!options.Fields.Contains(ItemFields.BasicSyncInfo) && + !hasFullSyncInfo) { - var userCanSync = user != null && user.Policy.EnableSync; - dto.SupportsSync = userCanSync && _syncManager.SupportsSync(item); + return; } if (dto.SupportsSync ?? false) { - dto.HasSyncJob = syncProgress.Any(i => i.Status != SyncJobItemStatus.Synced && string.Equals(i.ItemId, dto.Id, StringComparison.OrdinalIgnoreCase)); - dto.IsSynced = syncProgress.Any(i => i.Status == SyncJobItemStatus.Synced && string.Equals(i.ItemId, dto.Id, StringComparison.OrdinalIgnoreCase)); - - if (hasFullSyncInfo) + SyncedItemProgress syncStatus; + if (syncProgress.TryGetValue(dto.Id, out syncStatus)) { - if (dto.IsSynced.Value) + if (syncStatus.Status == SyncJobItemStatus.Synced) { - dto.SyncStatus = SyncJobItemStatus.Synced; + dto.SyncPercent = 100; + } + else + { + dto.SyncPercent = syncStatus.Progress; } - else if (dto.HasSyncJob.Value) + if (hasFullSyncInfo) { - dto.SyncStatus = syncProgress.Where(i => string.Equals(i.ItemId, dto.Id, StringComparison.OrdinalIgnoreCase)).Select(i => i.Status).Max(); + dto.HasSyncJob = true; + dto.SyncStatus = syncStatus.Status; } } } } - private async Task GetBaseItemDtoInternal(BaseItem item, DtoOptions options, Dictionary syncProgress, User user = null, BaseItem owner = null) + private async Task GetBaseItemDtoInternal(BaseItem item, DtoOptions options, User user = null, BaseItem owner = null) { var fields = options.Fields; @@ -337,7 +329,7 @@ namespace MediaBrowser.Server.Implementations.Dto if (user != null) { - await AttachUserSpecificInfo(dto, item, user, fields, syncProgress).ConfigureAwait(false); + await AttachUserSpecificInfo(dto, item, user, fields).ConfigureAwait(false); } var hasMediaSources = item as IHasMediaSources; @@ -400,11 +392,9 @@ namespace MediaBrowser.Server.Implementations.Dto return dto; } - public BaseItemDto GetItemByNameDto(BaseItem item, DtoOptions options, List taggedItems, User user = null) + public BaseItemDto GetItemByNameDto(BaseItem item, DtoOptions options, List taggedItems, Dictionary syncProgress, User user = null) { - var syncProgress = GetSyncedItemProgress(options); - - var dto = GetBaseItemDtoInternal(item, options, GetSyncedItemProgressDictionary(syncProgress), user).Result; + var dto = GetBaseItemDtoInternal(item, options, user).Result; if (taggedItems != null && options.Fields.Contains(ItemFields.ItemCounts)) { @@ -452,28 +442,13 @@ namespace MediaBrowser.Server.Implementations.Dto /// The item. /// The user. /// The fields. - /// The synchronize progress. - private async Task AttachUserSpecificInfo(BaseItemDto dto, BaseItem item, User user, List fields, Dictionary syncProgress) + private async Task AttachUserSpecificInfo(BaseItemDto dto, BaseItem item, User user, List fields) { if (item.IsFolder) { var folder = (Folder)item; - if (item.SourceType == SourceType.Library && folder.SupportsUserDataFromChildren && fields.Contains(ItemFields.SyncInfo)) - { - // Skip the user data manager because we've already looped through the recursive tree and don't want to do it twice - // TODO: Improve in future - dto.UserData = GetUserItemDataDto(_userDataRepository.GetUserData(user, item)); - - await SetSpecialCounts(folder, user, dto, fields, syncProgress).ConfigureAwait(false); - - dto.UserData.Played = dto.UserData.PlayedPercentage.HasValue && - dto.UserData.PlayedPercentage.Value >= 100; - } - else - { - dto.UserData = await _userDataRepository.GetUserDataDto(item, dto, user).ConfigureAwait(false); - } + dto.UserData = await _userDataRepository.GetUserDataDto(item, dto, user).ConfigureAwait(false); if (item.SourceType == SourceType.Library) { @@ -498,6 +473,12 @@ namespace MediaBrowser.Server.Implementations.Dto dto.PlayAccess = item.GetPlayAccess(user); + if (fields.Contains(ItemFields.BasicSyncInfo) || fields.Contains(ItemFields.SyncInfo)) + { + var userCanSync = user != null && user.Policy.EnableSync; + dto.SupportsSync = userCanSync && _syncManager.SupportsSync(item); + } + if (fields.Contains(ItemFields.SeasonUserData)) { var episode = item as Episode; @@ -1424,7 +1405,7 @@ namespace MediaBrowser.Server.Implementations.Dto BaseItem parent = null; var isFirst = true; - while (((!dto.HasLogo && logoLimit > 0) || (!dto.HasArtImage && artLimit > 0) || (!dto.HasThumb && thumbLimit > 0) || parent is Series) && + while (((!dto.HasLogo && logoLimit > 0) || (!dto.HasArtImage && artLimit > 0) || (!dto.HasThumb && thumbLimit > 0) || parent is Series) && (parent = parent ?? (isFirst ? item.GetParent() ?? owner : parent)) != null) { if (parent == null) @@ -1531,7 +1512,7 @@ namespace MediaBrowser.Server.Implementations.Dto /// The fields. /// The synchronize progress. /// Task. - private async Task SetSpecialCounts(Folder folder, User user, BaseItemDto dto, List fields, Dictionary syncProgress) + private async Task SetSpecialCounts(Folder folder, User user, BaseItemDto dto, List fields, Dictionary syncProgress) { var recursiveItemCount = 0; var unplayed = 0; @@ -1580,10 +1561,10 @@ namespace MediaBrowser.Server.Implementations.Dto } double percent = 0; - SyncedItemProgress syncItemProgress; + SyncJobItemStatus syncItemProgress; if (syncProgress.TryGetValue(child.Id.ToString("N"), out syncItemProgress)) { - switch (syncItemProgress.Status) + switch (syncItemProgress) { case SyncJobItemStatus.Synced: percent = 100; diff --git a/MediaBrowser.Server.Implementations/Sync/SyncManager.cs b/MediaBrowser.Server.Implementations/Sync/SyncManager.cs index 15855f553..ffba60af8 100644 --- a/MediaBrowser.Server.Implementations/Sync/SyncManager.cs +++ b/MediaBrowser.Server.Implementations/Sync/SyncManager.cs @@ -1124,7 +1124,7 @@ namespace MediaBrowser.Server.Implementations.Sync await processor.UpdateJobStatus(jobItem.JobId).ConfigureAwait(false); } - public QueryResult GetSyncedItemProgresses(SyncJobItemQuery query) + public Dictionary GetSyncedItemProgresses(SyncJobItemQuery query) { return _repo.GetSyncedItemProgresses(query); } diff --git a/MediaBrowser.Server.Implementations/Sync/SyncRepository.cs b/MediaBrowser.Server.Implementations/Sync/SyncRepository.cs index a1ed66a99..df4149fa7 100644 --- a/MediaBrowser.Server.Implementations/Sync/SyncRepository.cs +++ b/MediaBrowser.Server.Implementations/Sync/SyncRepository.cs @@ -29,6 +29,17 @@ namespace MediaBrowser.Server.Implementations.Sync DbFilePath = Path.Combine(appPaths.DataPath, "sync14.db"); } + private class SyncSummary + { + public Dictionary Items { get; set; } + + public SyncSummary() + { + Items = new Dictionary(); + } + } + + public async Task Initialize() { using (var connection = await CreateConnection().ConfigureAwait(false)) @@ -41,7 +52,10 @@ namespace MediaBrowser.Server.Implementations.Sync "create table if not exists SyncJobItems (Id GUID PRIMARY KEY, ItemId TEXT, ItemName TEXT, MediaSourceId TEXT, JobId TEXT, TemporaryPath TEXT, OutputPath TEXT, Status TEXT, TargetId TEXT, DateCreated DateTime, Progress FLOAT, AdditionalFiles TEXT, MediaSource TEXT, IsMarkedForRemoval BIT, JobItemIndex INT, ItemDateModifiedTicks BIGINT)", "create index if not exists idx_SyncJobItems1 on SyncJobItems(Id)", - "create index if not exists idx_SyncJobItems2 on SyncJobItems(TargetId)", + + "drop index if exists idx_SyncJobItems2", + "drop index if exists idx_SyncJobItems3", + "create index if not exists idx_SyncJobItems4 on SyncJobItems(TargetId,ItemId,Status,Progress,DateCreated)", "pragma shrink_memory" }; @@ -583,9 +597,74 @@ namespace MediaBrowser.Server.Implementations.Sync } } - public QueryResult GetSyncedItemProgresses(SyncJobItemQuery query) + public Dictionary GetSyncedItemProgresses(SyncJobItemQuery query) { - return GetJobItemReader(query, "select ItemId,Status from SyncJobItems", GetSyncedItemProgress); + var result = new Dictionary(); + + using (var connection = CreateConnection(true).Result) + { + using (var cmd = connection.CreateCommand()) + { + cmd.CommandText = "select ItemId,Progress,Status from SyncJobItems"; + + var whereClauses = new List(); + + if (!string.IsNullOrWhiteSpace(query.TargetId)) + { + whereClauses.Add("TargetId=@TargetId"); + cmd.Parameters.Add(cmd, "@TargetId", DbType.String).Value = query.TargetId; + } + + if (query.Statuses.Length > 0) + { + var statuses = string.Join(",", query.Statuses.Select(i => "'" + i.ToString() + "'").ToArray()); + + whereClauses.Add(string.Format("Status in ({0})", statuses)); + } + + if (whereClauses.Count > 0) + { + cmd.CommandText += " where " + string.Join(" AND ", whereClauses.ToArray()); + } + + using (var reader = cmd.ExecuteReader(CommandBehavior.SequentialAccess | CommandBehavior.SingleResult)) + { + while (reader.Read()) + { + var itemId = reader.GetString(0); + + if (!reader.IsDBNull(1)) + { + var status = (SyncJobItemStatus)Enum.Parse(typeof(SyncJobItemStatus), reader.GetString(1), true); + + if (status == SyncJobItemStatus.Synced) + { + result[itemId] = new SyncedItemProgress + { + Status = SyncJobItemStatus.Synced + }; + } + else + { + SyncedItemProgress currentStatus; + double progress = reader.IsDBNull(2) ? 0.0 : reader.GetDouble(2); + + if (!result.TryGetValue(itemId, out currentStatus) || (currentStatus.Status != SyncJobItemStatus.Synced && progress >= currentStatus.Progress)) + { + result[itemId] = new SyncedItemProgress + { + Status = status, + Progress = progress + }; + } + } + } + } + } + } + } + + return result; } public QueryResult GetJobItems(SyncJobItemQuery query) @@ -795,19 +874,5 @@ namespace MediaBrowser.Server.Implementations.Sync return info; } - - private SyncedItemProgress GetSyncedItemProgress(IDataReader reader) - { - var item = new SyncedItemProgress(); - - item.ItemId = reader.GetString(0); - - if (!reader.IsDBNull(1)) - { - item.Status = (SyncJobItemStatus)Enum.Parse(typeof(SyncJobItemStatus), reader.GetString(1), true); - } - - return item; - } } } -- cgit v1.2.3