From fe2596dc0e389c0496a384cc1893fddd4742ed37 Mon Sep 17 00:00:00 2001 From: JPVenson Date: Mon, 19 May 2025 03:39:04 +0300 Subject: Add Full system backup feature (#13945) --- .../FullSystemBackup/BackupService.cs | 463 +++++++++++++++++++++ 1 file changed, 463 insertions(+) create mode 100644 Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs (limited to 'Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs') diff --git a/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs b/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs new file mode 100644 index 0000000000..c3f5b01035 --- /dev/null +++ b/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs @@ -0,0 +1,463 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Jellyfin.Database.Implementations; +using Jellyfin.Server.Implementations.StorageHelpers; +using Jellyfin.Server.Implementations.SystemBackupService; +using MediaBrowser.Controller; +using MediaBrowser.Controller.SystemBackupService; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Server.Implementations.FullSystemBackup; + +/// +/// Contains methods for creating and restoring backups. +/// +public class BackupService : IBackupService +{ + private const string ManifestEntryName = "manifest.json"; + private readonly ILogger _logger; + private readonly IDbContextFactory _dbProvider; + private readonly IServerApplicationHost _applicationHost; + private readonly IServerApplicationPaths _applicationPaths; + private readonly IJellyfinDatabaseProvider _jellyfinDatabaseProvider; + private static readonly JsonSerializerOptions _serializerSettings = new JsonSerializerOptions(JsonSerializerDefaults.General) + { + AllowTrailingCommas = true, + ReferenceHandler = ReferenceHandler.IgnoreCycles, + }; + + private readonly Version _backupEngineVersion = Version.Parse("0.1.0"); + + /// + /// Initializes a new instance of the class. + /// + /// A logger. + /// A Database Factory. + /// The Application host. + /// The application paths. + /// The Jellyfin database Provider in use. + public BackupService( + ILogger logger, + IDbContextFactory dbProvider, + IServerApplicationHost applicationHost, + IServerApplicationPaths applicationPaths, + IJellyfinDatabaseProvider jellyfinDatabaseProvider) + { + _logger = logger; + _dbProvider = dbProvider; + _applicationHost = applicationHost; + _applicationPaths = applicationPaths; + _jellyfinDatabaseProvider = jellyfinDatabaseProvider; + } + + /// + public void ScheduleRestoreAndRestartServer(string archivePath) + { + _applicationHost.RestoreBackupPath = archivePath; + _applicationHost.ShouldRestart = true; + _applicationHost.NotifyPendingRestart(); + } + + /// + public async Task RestoreBackupAsync(string archivePath) + { + _logger.LogWarning("Begin restoring system to {BackupArchive}", archivePath); // Info isn't cutting it + if (!File.Exists(archivePath)) + { + throw new FileNotFoundException($"Requested backup file '{archivePath}' does not exist."); + } + + StorageHelper.TestCommonPathsForStorageCapacity(_applicationPaths, _logger); + + var fileStream = File.OpenRead(archivePath); + await using (fileStream.ConfigureAwait(false)) + { + using var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Read, false); + var zipArchiveEntry = zipArchive.GetEntry(ManifestEntryName); + + if (zipArchiveEntry is null) + { + throw new NotSupportedException($"The loaded archive '{archivePath}' does not appear to be a Jellyfin backup as its missing the '{ManifestEntryName}'."); + } + + BackupManifest? manifest; + var manifestStream = zipArchiveEntry.Open(); + await using (manifestStream.ConfigureAwait(false)) + { + manifest = await JsonSerializer.DeserializeAsync(manifestStream, _serializerSettings).ConfigureAwait(false); + } + + if (manifest!.ServerVersion > _applicationHost.ApplicationVersion) // newer versions of Jellyfin should be able to load older versions as we have migrations. + { + throw new NotSupportedException($"The loaded archive '{archivePath}' is made for a newer version of Jellyfin ({manifest.ServerVersion}) and cannot be loaded in this version."); + } + + if (!TestBackupVersionCompatibility(manifest.BackupEngineVersion)) + { + throw new NotSupportedException($"The loaded archive '{archivePath}' is made for a newer version of Jellyfin ({manifest.ServerVersion}) and cannot be loaded in this version."); + } + + void CopyDirectory(string source, string target) + { + source = Path.GetFullPath(source); + Directory.CreateDirectory(source); + + foreach (var item in zipArchive.Entries) + { + var sanitizedSourcePath = Path.GetFullPath(item.FullName.TrimEnd(Path.DirectorySeparatorChar) + Path.DirectorySeparatorChar); + if (!sanitizedSourcePath.StartsWith(target, StringComparison.Ordinal)) + { + continue; + } + + var targetPath = Path.Combine(source, sanitizedSourcePath[target.Length..].Trim('/')); + _logger.LogInformation("Restore and override {File}", targetPath); + item.ExtractToFile(targetPath); + } + } + + CopyDirectory(_applicationPaths.ConfigurationDirectoryPath, "Config/"); + CopyDirectory(_applicationPaths.DataPath, "Data/"); + CopyDirectory(_applicationPaths.RootFolderPath, "Root/"); + + _logger.LogInformation("Begin restoring Database"); + var dbContext = await _dbProvider.CreateDbContextAsync().ConfigureAwait(false); + await using (dbContext.ConfigureAwait(false)) + { + dbContext.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking; + var entityTypes = typeof(JellyfinDbContext).GetProperties(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance) + .Where(e => e.PropertyType.IsAssignableTo(typeof(IQueryable))) + .Select(e => (Type: e, Set: e.GetValue(dbContext) as IQueryable)) + .ToArray(); + + var tableNames = entityTypes.Select(f => dbContext.Model.FindEntityType(f.Type.PropertyType.GetGenericArguments()[0])!.GetSchemaQualifiedTableName()!); + _logger.LogInformation("Begin purging database"); + await _jellyfinDatabaseProvider.PurgeDatabase(dbContext, tableNames).ConfigureAwait(false); + _logger.LogInformation("Database Purged"); + + foreach (var entityType in entityTypes) + { + _logger.LogInformation("Read backup of {Table}", entityType.Type.Name); + + var zipEntry = zipArchive.GetEntry($"Database\\{entityType.Type.Name}.json"); + if (zipEntry is null) + { + _logger.LogInformation("No backup of expected table {Table} is present in backup. Continue anyway.", entityType.Type.Name); + continue; + } + + var zipEntryStream = zipEntry.Open(); + await using (zipEntryStream.ConfigureAwait(false)) + { + _logger.LogInformation("Restore backup of {Table}", entityType.Type.Name); + var records = 0; + await foreach (var item in JsonSerializer.DeserializeAsyncEnumerable(zipEntryStream, _serializerSettings).ConfigureAwait(false)!) + { + var entity = item.Deserialize(entityType.Type.PropertyType.GetGenericArguments()[0]); + if (entity is null) + { + throw new InvalidOperationException($"Cannot deserialize entity '{item}'"); + } + + try + { + records++; + dbContext.Add(entity); + } + catch (Exception ex) + { + _logger.LogError(ex, "Could not store entity {Entity} continue anyway.", item); + } + } + + _logger.LogInformation("Prepared to restore {Number} entries for {Table}", records, entityType.Type.Name); + } + } + + _logger.LogInformation("Try restore Database"); + await dbContext.SaveChangesAsync().ConfigureAwait(false); + _logger.LogInformation("Restored database."); + } + + _logger.LogInformation("Restored Jellyfin system from {Date}.", manifest.DateCreated); + } + } + + private bool TestBackupVersionCompatibility(Version backupEngineVersion) + { + if (backupEngineVersion == _backupEngineVersion) + { + return true; + } + + return false; + } + + /// + public async Task CreateBackupAsync(BackupOptionsDto backupOptions) + { + var manifest = new BackupManifest() + { + DateCreated = DateTime.UtcNow, + ServerVersion = _applicationHost.ApplicationVersion, + DatabaseTables = null!, + BackupEngineVersion = _backupEngineVersion, + Options = Map(backupOptions) + }; + + await _jellyfinDatabaseProvider.RunScheduledOptimisation(CancellationToken.None).ConfigureAwait(false); + + var backupFolder = Path.Combine(_applicationPaths.BackupPath); + + if (!Directory.Exists(backupFolder)) + { + Directory.CreateDirectory(backupFolder); + } + + var backupStorageSpace = StorageHelper.GetFreeSpaceOf(_applicationPaths.BackupPath); + + const long FiveGigabyte = 5_368_709_115; + if (backupStorageSpace.FreeSpace < FiveGigabyte) + { + throw new InvalidOperationException($"The backup directory '{backupStorageSpace.Path}' does not have at least '{StorageHelper.HumanizeStorageSize(FiveGigabyte)}' free space. Cannot create backup."); + } + + var backupPath = Path.Combine(backupFolder, $"jellyfin-backup-{manifest.DateCreated.ToLocalTime():yyyyMMddHHmmss}.zip"); + _logger.LogInformation("Attempt to create a new backup at {BackupPath}", backupPath); + var fileStream = File.OpenWrite(backupPath); + await using (fileStream.ConfigureAwait(false)) + using (var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Create, false)) + { + _logger.LogInformation("Start backup process."); + var dbContext = await _dbProvider.CreateDbContextAsync().ConfigureAwait(false); + await using (dbContext.ConfigureAwait(false)) + { + dbContext.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking; + var entityTypes = typeof(JellyfinDbContext).GetProperties(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance) + .Where(e => e.PropertyType.IsAssignableTo(typeof(IQueryable))) + .Select(e => (Type: e, Set: e.GetValue(dbContext) as IQueryable)) + .ToArray(); + manifest.DatabaseTables = entityTypes.Select(e => e.Type.Name).ToArray(); + var transaction = await dbContext.Database.BeginTransactionAsync().ConfigureAwait(false); + + await using (transaction.ConfigureAwait(false)) + { + _logger.LogInformation("Begin Database backup"); + static IAsyncEnumerable GetValues(IQueryable dbSet, Type type) + { + var method = dbSet.GetType().GetMethod(nameof(DbSet.AsAsyncEnumerable))!; + var enumerable = method.Invoke(dbSet, null)!; + return (IAsyncEnumerable)enumerable; + } + + foreach (var entityType in entityTypes) + { + _logger.LogInformation("Begin backup of entity {Table}", entityType.Type.Name); + var zipEntry = zipArchive.CreateEntry($"Database\\{entityType.Type.Name}.json"); + var entities = 0; + var zipEntryStream = zipEntry.Open(); + await using (zipEntryStream.ConfigureAwait(false)) + { + var jsonSerializer = new Utf8JsonWriter(zipEntryStream); + await using (jsonSerializer.ConfigureAwait(false)) + { + jsonSerializer.WriteStartArray(); + + var set = GetValues(entityType.Set!, entityType.Type.PropertyType).ConfigureAwait(false); + await foreach (var item in set.ConfigureAwait(false)) + { + entities++; + try + { + JsonSerializer.SerializeToDocument(item, _serializerSettings).WriteTo(jsonSerializer); + } + catch (Exception ex) + { + _logger.LogError(ex, "Could not load entity {Entity}", item); + throw; + } + } + + jsonSerializer.WriteEndArray(); + } + } + + _logger.LogInformation("backup of entity {Table} with {Number} created", entityType.Type.Name, entities); + } + } + } + + _logger.LogInformation("Backup of folder {Table}", _applicationPaths.ConfigurationDirectoryPath); + foreach (var item in Directory.EnumerateFiles(_applicationPaths.ConfigurationDirectoryPath, "*.xml", SearchOption.TopDirectoryOnly) + .Union(Directory.EnumerateFiles(_applicationPaths.ConfigurationDirectoryPath, "*.json", SearchOption.TopDirectoryOnly))) + { + zipArchive.CreateEntryFromFile(item, Path.Combine("Config", Path.GetFileName(item))); + } + + void CopyDirectory(string source, string target, string filter = "*") + { + if (!Directory.Exists(source)) + { + return; + } + + _logger.LogInformation("Backup of folder {Table}", source); + + foreach (var item in Directory.EnumerateFiles(source, filter, SearchOption.AllDirectories)) + { + zipArchive.CreateEntryFromFile(item, Path.Combine(target, item[..source.Length].Trim('\\'))); + } + } + + CopyDirectory(Path.Combine(_applicationPaths.ConfigurationDirectoryPath, "users"), Path.Combine("Config", "users")); + CopyDirectory(Path.Combine(_applicationPaths.ConfigurationDirectoryPath, "ScheduledTasks"), Path.Combine("Config", "ScheduledTasks")); + CopyDirectory(Path.Combine(_applicationPaths.RootFolderPath), "Root"); + CopyDirectory(Path.Combine(_applicationPaths.DataPath, "collections"), Path.Combine("Data", "collections")); + CopyDirectory(Path.Combine(_applicationPaths.DataPath, "playlists"), Path.Combine("Data", "playlists")); + CopyDirectory(Path.Combine(_applicationPaths.DataPath, "ScheduledTasks"), Path.Combine("Data", "ScheduledTasks")); + if (backupOptions.Subtitles) + { + CopyDirectory(Path.Combine(_applicationPaths.DataPath, "subtitles"), Path.Combine("Data", "subtitles")); + } + + if (backupOptions.Trickplay) + { + CopyDirectory(Path.Combine(_applicationPaths.DataPath, "trickplay"), Path.Combine("Data", "trickplay")); + } + + if (backupOptions.Metadata) + { + CopyDirectory(Path.Combine(_applicationPaths.InternalMetadataPath), Path.Combine("Data", "metadata")); + } + + var manifestStream = zipArchive.CreateEntry(ManifestEntryName).Open(); + await using (manifestStream.ConfigureAwait(false)) + { + await JsonSerializer.SerializeAsync(manifestStream, manifest).ConfigureAwait(false); + } + } + + _logger.LogInformation("Backup created"); + return Map(manifest, backupPath); + } + + /// + public async Task GetBackupManifest(string archivePath) + { + if (!File.Exists(archivePath)) + { + return null; + } + + BackupManifest? manifest; + try + { + manifest = await GetManifest(archivePath).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Tried to load archive from {Path} but failed.", archivePath); + return null; + } + + if (manifest is null) + { + return null; + } + + return Map(manifest, archivePath); + } + + /// + public async Task EnumerateBackups() + { + if (!Directory.Exists(_applicationPaths.BackupPath)) + { + return []; + } + + var archives = Directory.EnumerateFiles(_applicationPaths.BackupPath, "*.zip"); + var manifests = new List(); + foreach (var item in archives) + { + try + { + var manifest = await GetManifest(item).ConfigureAwait(false); + + if (manifest is null) + { + continue; + } + + manifests.Add(Map(manifest, item)); + } + catch (Exception ex) + { + _logger.LogError(ex, "Could not load {BackupArchive} path.", item); + } + } + + return manifests.ToArray(); + } + + private static async ValueTask GetManifest(string archivePath) + { + var archiveStream = File.OpenRead(archivePath); + await using (archiveStream.ConfigureAwait(false)) + { + using var zipStream = new ZipArchive(archiveStream, ZipArchiveMode.Read); + var manifestEntry = zipStream.GetEntry(ManifestEntryName); + if (manifestEntry is null) + { + return null; + } + + var manifestStream = manifestEntry.Open(); + await using (manifestStream.ConfigureAwait(false)) + { + return await JsonSerializer.DeserializeAsync(manifestStream, _serializerSettings).ConfigureAwait(false); + } + } + } + + private static BackupManifestDto Map(BackupManifest manifest, string path) + { + return new BackupManifestDto() + { + BackupEngineVersion = manifest.BackupEngineVersion, + DateCreated = manifest.DateCreated, + ServerVersion = manifest.ServerVersion, + Path = path, + Options = Map(manifest.Options) + }; + } + + private static BackupOptionsDto Map(BackupOptions options) + { + return new BackupOptionsDto() + { + Metadata = options.Metadata, + Subtitles = options.Subtitles, + Trickplay = options.Trickplay + }; + } + + private static BackupOptions Map(BackupOptionsDto options) + { + return new BackupOptions() + { + Metadata = options.Metadata, + Subtitles = options.Subtitles, + Trickplay = options.Trickplay + }; + } +} -- cgit v1.2.3 From 697bb6a4804dcbaeff380c2c70b84f2489c73c86 Mon Sep 17 00:00:00 2001 From: JPVenson Date: Wed, 4 Jun 2025 00:15:46 +0300 Subject: Backup MigrationHistory as well (#14136) --- .../FullSystemBackup/BackupService.cs | 54 ++++++++++++++++++---- 1 file changed, 44 insertions(+), 10 deletions(-) (limited to 'Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs') diff --git a/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs b/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs index c3f5b01035..1a153b5658 100644 --- a/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs +++ b/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs @@ -14,6 +14,8 @@ using Jellyfin.Server.Implementations.SystemBackupService; using MediaBrowser.Controller; using MediaBrowser.Controller.SystemBackupService; using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.Extensions.Logging; namespace Jellyfin.Server.Implementations.FullSystemBackup; @@ -133,6 +135,30 @@ public class BackupService : IBackupService var dbContext = await _dbProvider.CreateDbContextAsync().ConfigureAwait(false); await using (dbContext.ConfigureAwait(false)) { + // restore migration history manually + var historyEntry = zipArchive.GetEntry($"Database\\{nameof(HistoryRow)}.json"); + if (historyEntry is null) + { + _logger.LogInformation("No backup of the history table in archive. This is required for Jellyfin operation"); + throw new InvalidOperationException("Cannot restore backup that has no History data."); + } + + HistoryRow[] historyEntries; + var historyArchive = historyEntry.Open(); + await using (historyArchive.ConfigureAwait(false)) + { + historyEntries = await JsonSerializer.DeserializeAsync(historyArchive).ConfigureAwait(false) ?? + throw new InvalidOperationException("Cannot restore backup that has no History data."); + } + + var historyRepository = dbContext.GetService(); + await historyRepository.CreateIfNotExistsAsync().ConfigureAwait(false); + foreach (var item in historyEntries) + { + var insertScript = historyRepository.GetInsertScript(item); + await dbContext.Database.ExecuteSqlRawAsync(insertScript).ConfigureAwait(false); + } + dbContext.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking; var entityTypes = typeof(JellyfinDbContext).GetProperties(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance) .Where(e => e.PropertyType.IsAssignableTo(typeof(IQueryable))) @@ -242,22 +268,30 @@ public class BackupService : IBackupService await using (dbContext.ConfigureAwait(false)) { dbContext.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking; - var entityTypes = typeof(JellyfinDbContext).GetProperties(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance) + static IAsyncEnumerable GetValues(IQueryable dbSet, Type type) + { + var method = dbSet.GetType().GetMethod(nameof(DbSet.AsAsyncEnumerable))!; + var enumerable = method.Invoke(dbSet, null)!; + return (IAsyncEnumerable)enumerable; + } + + // include the migration history as well + var historyRepository = dbContext.GetService(); + var migrations = await historyRepository.GetAppliedMigrationsAsync().ConfigureAwait(false); + + ICollection<(Type Type, Func> ValueFactory)> entityTypes = [ + .. typeof(JellyfinDbContext) + .GetProperties(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance) .Where(e => e.PropertyType.IsAssignableTo(typeof(IQueryable))) - .Select(e => (Type: e, Set: e.GetValue(dbContext) as IQueryable)) - .ToArray(); + .Select(e => (Type: e.PropertyType, ValueFactory: new Func>(() => GetValues((IQueryable)e.GetValue(dbContext)!, e.PropertyType)))), + (Type: typeof(HistoryRow), ValueFactory: new Func>(() => migrations.ToAsyncEnumerable())) + ]; manifest.DatabaseTables = entityTypes.Select(e => e.Type.Name).ToArray(); var transaction = await dbContext.Database.BeginTransactionAsync().ConfigureAwait(false); await using (transaction.ConfigureAwait(false)) { _logger.LogInformation("Begin Database backup"); - static IAsyncEnumerable GetValues(IQueryable dbSet, Type type) - { - var method = dbSet.GetType().GetMethod(nameof(DbSet.AsAsyncEnumerable))!; - var enumerable = method.Invoke(dbSet, null)!; - return (IAsyncEnumerable)enumerable; - } foreach (var entityType in entityTypes) { @@ -272,7 +306,7 @@ public class BackupService : IBackupService { jsonSerializer.WriteStartArray(); - var set = GetValues(entityType.Set!, entityType.Type.PropertyType).ConfigureAwait(false); + var set = entityType.ValueFactory().ConfigureAwait(false); await foreach (var item in set.ConfigureAwait(false)) { entities++; -- cgit v1.2.3 From 48825f468e9a36c2a196ad34f45c2b054b952720 Mon Sep 17 00:00:00 2001 From: JPVenson Date: Wed, 4 Jun 2025 00:16:35 +0300 Subject: Fix server not auto restarting (#14215) --- Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs') diff --git a/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs b/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs index 1a153b5658..0e647fd241 100644 --- a/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs +++ b/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs @@ -31,6 +31,7 @@ public class BackupService : IBackupService private readonly IServerApplicationHost _applicationHost; private readonly IServerApplicationPaths _applicationPaths; private readonly IJellyfinDatabaseProvider _jellyfinDatabaseProvider; + private readonly ISystemManager _systemManager; private static readonly JsonSerializerOptions _serializerSettings = new JsonSerializerOptions(JsonSerializerDefaults.General) { AllowTrailingCommas = true, @@ -47,18 +48,21 @@ public class BackupService : IBackupService /// The Application host. /// The application paths. /// The Jellyfin database Provider in use. + /// The SystemManager. public BackupService( ILogger logger, IDbContextFactory dbProvider, IServerApplicationHost applicationHost, IServerApplicationPaths applicationPaths, - IJellyfinDatabaseProvider jellyfinDatabaseProvider) + IJellyfinDatabaseProvider jellyfinDatabaseProvider, + ISystemManager systemManager) { _logger = logger; _dbProvider = dbProvider; _applicationHost = applicationHost; _applicationPaths = applicationPaths; _jellyfinDatabaseProvider = jellyfinDatabaseProvider; + _systemManager = systemManager; } /// @@ -67,6 +71,7 @@ public class BackupService : IBackupService _applicationHost.RestoreBackupPath = archivePath; _applicationHost.ShouldRestart = true; _applicationHost.NotifyPendingRestart(); + _systemManager.Restart(); } /// -- cgit v1.2.3 From d5672ce407dda5e6e2422a7ce7ea6ad561759001 Mon Sep 17 00:00:00 2001 From: JPVenson Date: Wed, 4 Jun 2025 01:49:41 +0300 Subject: Add declarative backups for migrations (#14135) --- .../FullSystemBackup/BackupOptions.cs | 2 + .../FullSystemBackup/BackupService.cs | 152 ++++++++------- .../Migrations/JellyfinMigrationAttribute.cs | 4 +- .../Migrations/JellyfinMigrationBackupAttribute.cs | 35 ++++ .../Migrations/JellyfinMigrationService.cs | 214 ++++++++++++++++++++- .../Migrations/Routines/FixAudioData.cs | 25 +-- .../Migrations/Routines/MigrateLibraryDb.cs | 1 + .../Migrations/Routines/MigrateRatingLevels.cs | 1 + Jellyfin.Server/Migrations/Stages/CodeMigration.cs | 4 +- .../Stages/JellyfinMigrationStageTypes.cs | 2 +- Jellyfin.Server/Program.cs | 10 +- .../SystemBackupService/BackupOptionsDto.cs | 5 + .../IJellyfinDatabaseProvider.cs | 7 + .../SqliteDatabaseProvider.cs | 15 ++ .../JellyfinApplicationFactory.cs | 2 +- 15 files changed, 370 insertions(+), 109 deletions(-) create mode 100644 Jellyfin.Server/Migrations/JellyfinMigrationBackupAttribute.cs (limited to 'Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs') diff --git a/Jellyfin.Server.Implementations/FullSystemBackup/BackupOptions.cs b/Jellyfin.Server.Implementations/FullSystemBackup/BackupOptions.cs index 706f009ac2..8bd108c443 100644 --- a/Jellyfin.Server.Implementations/FullSystemBackup/BackupOptions.cs +++ b/Jellyfin.Server.Implementations/FullSystemBackup/BackupOptions.cs @@ -10,4 +10,6 @@ internal class BackupOptions public bool Trickplay { get; set; } public bool Subtitles { get; set; } + + public bool Database { get; set; } } diff --git a/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs b/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs index 0e647fd241..e266d5a3bc 100644 --- a/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs +++ b/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs @@ -16,6 +16,7 @@ using MediaBrowser.Controller.SystemBackupService; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; namespace Jellyfin.Server.Implementations.FullSystemBackup; @@ -31,7 +32,7 @@ public class BackupService : IBackupService private readonly IServerApplicationHost _applicationHost; private readonly IServerApplicationPaths _applicationPaths; private readonly IJellyfinDatabaseProvider _jellyfinDatabaseProvider; - private readonly ISystemManager _systemManager; + private readonly IHostApplicationLifetime _hostApplicationLifetime; private static readonly JsonSerializerOptions _serializerSettings = new JsonSerializerOptions(JsonSerializerDefaults.General) { AllowTrailingCommas = true, @@ -48,21 +49,21 @@ public class BackupService : IBackupService /// The Application host. /// The application paths. /// The Jellyfin database Provider in use. - /// The SystemManager. + /// The SystemManager. public BackupService( ILogger logger, IDbContextFactory dbProvider, IServerApplicationHost applicationHost, IServerApplicationPaths applicationPaths, IJellyfinDatabaseProvider jellyfinDatabaseProvider, - ISystemManager systemManager) + IHostApplicationLifetime applicationLifetime) { _logger = logger; _dbProvider = dbProvider; _applicationHost = applicationHost; _applicationPaths = applicationPaths; _jellyfinDatabaseProvider = jellyfinDatabaseProvider; - _systemManager = systemManager; + _hostApplicationLifetime = applicationLifetime; } /// @@ -71,7 +72,11 @@ public class BackupService : IBackupService _applicationHost.RestoreBackupPath = archivePath; _applicationHost.ShouldRestart = true; _applicationHost.NotifyPendingRestart(); - _systemManager.Restart(); + _ = Task.Run(async () => + { + await Task.Delay(500).ConfigureAwait(false); + _hostApplicationLifetime.StopApplication(); + }); } /// @@ -136,87 +141,90 @@ public class BackupService : IBackupService CopyDirectory(_applicationPaths.DataPath, "Data/"); CopyDirectory(_applicationPaths.RootFolderPath, "Root/"); - _logger.LogInformation("Begin restoring Database"); - var dbContext = await _dbProvider.CreateDbContextAsync().ConfigureAwait(false); - await using (dbContext.ConfigureAwait(false)) + if (manifest.Options.Database) { - // restore migration history manually - var historyEntry = zipArchive.GetEntry($"Database\\{nameof(HistoryRow)}.json"); - if (historyEntry is null) - { - _logger.LogInformation("No backup of the history table in archive. This is required for Jellyfin operation"); - throw new InvalidOperationException("Cannot restore backup that has no History data."); - } - - HistoryRow[] historyEntries; - var historyArchive = historyEntry.Open(); - await using (historyArchive.ConfigureAwait(false)) + _logger.LogInformation("Begin restoring Database"); + var dbContext = await _dbProvider.CreateDbContextAsync().ConfigureAwait(false); + await using (dbContext.ConfigureAwait(false)) { - historyEntries = await JsonSerializer.DeserializeAsync(historyArchive).ConfigureAwait(false) ?? + // restore migration history manually + var historyEntry = zipArchive.GetEntry($"Database\\{nameof(HistoryRow)}.json"); + if (historyEntry is null) + { + _logger.LogInformation("No backup of the history table in archive. This is required for Jellyfin operation"); throw new InvalidOperationException("Cannot restore backup that has no History data."); - } + } - var historyRepository = dbContext.GetService(); - await historyRepository.CreateIfNotExistsAsync().ConfigureAwait(false); - foreach (var item in historyEntries) - { - var insertScript = historyRepository.GetInsertScript(item); - await dbContext.Database.ExecuteSqlRawAsync(insertScript).ConfigureAwait(false); - } + HistoryRow[] historyEntries; + var historyArchive = historyEntry.Open(); + await using (historyArchive.ConfigureAwait(false)) + { + historyEntries = await JsonSerializer.DeserializeAsync(historyArchive).ConfigureAwait(false) ?? + throw new InvalidOperationException("Cannot restore backup that has no History data."); + } - dbContext.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking; - var entityTypes = typeof(JellyfinDbContext).GetProperties(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance) - .Where(e => e.PropertyType.IsAssignableTo(typeof(IQueryable))) - .Select(e => (Type: e, Set: e.GetValue(dbContext) as IQueryable)) - .ToArray(); + var historyRepository = dbContext.GetService(); + await historyRepository.CreateIfNotExistsAsync().ConfigureAwait(false); + foreach (var item in historyEntries) + { + var insertScript = historyRepository.GetInsertScript(item); + await dbContext.Database.ExecuteSqlRawAsync(insertScript).ConfigureAwait(false); + } - var tableNames = entityTypes.Select(f => dbContext.Model.FindEntityType(f.Type.PropertyType.GetGenericArguments()[0])!.GetSchemaQualifiedTableName()!); - _logger.LogInformation("Begin purging database"); - await _jellyfinDatabaseProvider.PurgeDatabase(dbContext, tableNames).ConfigureAwait(false); - _logger.LogInformation("Database Purged"); + dbContext.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking; + var entityTypes = typeof(JellyfinDbContext).GetProperties(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance) + .Where(e => e.PropertyType.IsAssignableTo(typeof(IQueryable))) + .Select(e => (Type: e, Set: e.GetValue(dbContext) as IQueryable)) + .ToArray(); - foreach (var entityType in entityTypes) - { - _logger.LogInformation("Read backup of {Table}", entityType.Type.Name); + var tableNames = entityTypes.Select(f => dbContext.Model.FindEntityType(f.Type.PropertyType.GetGenericArguments()[0])!.GetSchemaQualifiedTableName()!); + _logger.LogInformation("Begin purging database"); + await _jellyfinDatabaseProvider.PurgeDatabase(dbContext, tableNames).ConfigureAwait(false); + _logger.LogInformation("Database Purged"); - var zipEntry = zipArchive.GetEntry($"Database\\{entityType.Type.Name}.json"); - if (zipEntry is null) + foreach (var entityType in entityTypes) { - _logger.LogInformation("No backup of expected table {Table} is present in backup. Continue anyway.", entityType.Type.Name); - continue; - } + _logger.LogInformation("Read backup of {Table}", entityType.Type.Name); - var zipEntryStream = zipEntry.Open(); - await using (zipEntryStream.ConfigureAwait(false)) - { - _logger.LogInformation("Restore backup of {Table}", entityType.Type.Name); - var records = 0; - await foreach (var item in JsonSerializer.DeserializeAsyncEnumerable(zipEntryStream, _serializerSettings).ConfigureAwait(false)!) + var zipEntry = zipArchive.GetEntry($"Database\\{entityType.Type.Name}.json"); + if (zipEntry is null) { - var entity = item.Deserialize(entityType.Type.PropertyType.GetGenericArguments()[0]); - if (entity is null) - { - throw new InvalidOperationException($"Cannot deserialize entity '{item}'"); - } + _logger.LogInformation("No backup of expected table {Table} is present in backup. Continue anyway.", entityType.Type.Name); + continue; + } - try - { - records++; - dbContext.Add(entity); - } - catch (Exception ex) + var zipEntryStream = zipEntry.Open(); + await using (zipEntryStream.ConfigureAwait(false)) + { + _logger.LogInformation("Restore backup of {Table}", entityType.Type.Name); + var records = 0; + await foreach (var item in JsonSerializer.DeserializeAsyncEnumerable(zipEntryStream, _serializerSettings).ConfigureAwait(false)!) { - _logger.LogError(ex, "Could not store entity {Entity} continue anyway.", item); + var entity = item.Deserialize(entityType.Type.PropertyType.GetGenericArguments()[0]); + if (entity is null) + { + throw new InvalidOperationException($"Cannot deserialize entity '{item}'"); + } + + try + { + records++; + dbContext.Add(entity); + } + catch (Exception ex) + { + _logger.LogError(ex, "Could not store entity {Entity} continue anyway.", item); + } } - } - _logger.LogInformation("Prepared to restore {Number} entries for {Table}", records, entityType.Type.Name); + _logger.LogInformation("Prepared to restore {Number} entries for {Table}", records, entityType.Type.Name); + } } - } - _logger.LogInformation("Try restore Database"); - await dbContext.SaveChangesAsync().ConfigureAwait(false); - _logger.LogInformation("Restored database."); + _logger.LogInformation("Try restore Database"); + await dbContext.SaveChangesAsync().ConfigureAwait(false); + _logger.LogInformation("Restored database."); + } } _logger.LogInformation("Restored Jellyfin system from {Date}.", manifest.DateCreated); @@ -486,7 +494,8 @@ public class BackupService : IBackupService { Metadata = options.Metadata, Subtitles = options.Subtitles, - Trickplay = options.Trickplay + Trickplay = options.Trickplay, + Database = options.Database }; } @@ -496,7 +505,8 @@ public class BackupService : IBackupService { Metadata = options.Metadata, Subtitles = options.Subtitles, - Trickplay = options.Trickplay + Trickplay = options.Trickplay, + Database = options.Database }; } } diff --git a/Jellyfin.Server/Migrations/JellyfinMigrationAttribute.cs b/Jellyfin.Server/Migrations/JellyfinMigrationAttribute.cs index 5c8322ef78..70e54125b8 100644 --- a/Jellyfin.Server/Migrations/JellyfinMigrationAttribute.cs +++ b/Jellyfin.Server/Migrations/JellyfinMigrationAttribute.cs @@ -47,9 +47,9 @@ public sealed class JellyfinMigrationAttribute : Attribute public bool RunMigrationOnSetup { get; set; } /// - /// Gets or Sets the stage the annoated migration should be executed at. Defaults to . + /// Gets or Sets the stage the annoated migration should be executed at. Defaults to . /// - public JellyfinMigrationStageTypes Stage { get; set; } = JellyfinMigrationStageTypes.CoreInitialisaition; + public JellyfinMigrationStageTypes Stage { get; set; } = JellyfinMigrationStageTypes.CoreInitialisation; /// /// Gets the ordering of the migration. diff --git a/Jellyfin.Server/Migrations/JellyfinMigrationBackupAttribute.cs b/Jellyfin.Server/Migrations/JellyfinMigrationBackupAttribute.cs new file mode 100644 index 0000000000..6c8da7e820 --- /dev/null +++ b/Jellyfin.Server/Migrations/JellyfinMigrationBackupAttribute.cs @@ -0,0 +1,35 @@ +using System; + +namespace Jellyfin.Server.Migrations; + +/// +/// Marks an migration and instructs the to perform a backup. +/// +[AttributeUsage(System.AttributeTargets.Class, Inherited = true, AllowMultiple = true)] +public sealed class JellyfinMigrationBackupAttribute : System.Attribute +{ + /// + /// Gets or Sets a value indicating whether a backup of the old library.db should be performed. + /// + public bool LegacyLibraryDb { get; set; } + + /// + /// Gets or Sets a value indicating whether a backup of the Database should be performed. + /// + public bool JellyfinDb { get; set; } + + /// + /// Gets or Sets a value indicating whether a backup of the metadata folder should be performed. + /// + public bool Metadata { get; set; } + + /// + /// Gets or Sets a value indicating whether a backup of the Trickplay folder should be performed. + /// + public bool Trickplay { get; set; } + + /// + /// Gets or Sets a value indicating whether a backup of the Subtitles folder should be performed. + /// + public bool Subtitles { get; set; } +} diff --git a/Jellyfin.Server/Migrations/JellyfinMigrationService.cs b/Jellyfin.Server/Migrations/JellyfinMigrationService.cs index 3d6ed73bc6..fc4045da02 100644 --- a/Jellyfin.Server/Migrations/JellyfinMigrationService.cs +++ b/Jellyfin.Server/Migrations/JellyfinMigrationService.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.IO; using System.Linq; using System.Reflection; @@ -25,21 +26,37 @@ namespace Jellyfin.Server.Migrations; /// internal class JellyfinMigrationService { + private const string DbFilename = "library.db"; private readonly IDbContextFactory _dbContextFactory; private readonly ILoggerFactory _loggerFactory; + private readonly IBackupService? _backupService; + private readonly IJellyfinDatabaseProvider? _jellyfinDatabaseProvider; + private readonly IApplicationPaths _applicationPaths; + private (string? LibraryDb, string? JellyfinDb, BackupManifestDto? FullBackup) _backupKey; /// /// Initializes a new instance of the class. /// /// Provides access to the jellyfin database. /// The logger factory. - public JellyfinMigrationService(IDbContextFactory dbContextFactory, ILoggerFactory loggerFactory) + /// Application paths for library.db backup. + /// The jellyfin backup service. + /// The jellyfin database provider. + public JellyfinMigrationService( + IDbContextFactory dbContextFactory, + ILoggerFactory loggerFactory, + IApplicationPaths applicationPaths, + IBackupService? backupService = null, + IJellyfinDatabaseProvider? jellyfinDatabaseProvider = null) { _dbContextFactory = dbContextFactory; _loggerFactory = loggerFactory; + _backupService = backupService; + _jellyfinDatabaseProvider = jellyfinDatabaseProvider; + _applicationPaths = applicationPaths; #pragma warning disable CS0618 // Type or member is obsolete Migrations = [.. typeof(IMigrationRoutine).Assembly.GetTypes().Where(e => typeof(IMigrationRoutine).IsAssignableFrom(e) || typeof(IAsyncMigrationRoutine).IsAssignableFrom(e)) - .Select(e => (Type: e, Metadata: e.GetCustomAttribute())) + .Select(e => (Type: e, Metadata: e.GetCustomAttribute(), Backup: e.GetCustomAttributes())) .Where(e => e.Metadata != null) .GroupBy(e => e.Metadata!.Stage) .Select(f => @@ -47,7 +64,13 @@ internal class JellyfinMigrationService var stage = new MigrationStage(f.Key); foreach (var item in f) { - stage.Add(new(item.Type, item.Metadata!)); + JellyfinMigrationBackupAttribute? backupMetadata = null; + if (item.Backup?.Any() == true) + { + backupMetadata = item.Backup.Aggregate(MergeBackupAttributes); + } + + stage.Add(new(item.Type, item.Metadata!, backupMetadata)); } return stage; @@ -155,7 +178,7 @@ internal class JellyfinMigrationService .ToArray(); (string Key, InternalDatabaseMigration Migration)[] pendingDatabaseMigrations = []; - if (stage is JellyfinMigrationStageTypes.CoreInitialisaition) + if (stage is JellyfinMigrationStageTypes.CoreInitialisation) { pendingDatabaseMigrations = migrationsAssembly.Migrations.Where(f => appliedMigrations.All(e => e.MigrationId != f.Key)) .Select(e => (Key: e.Key, Migration: new InternalDatabaseMigration(e, dbContext))) @@ -176,7 +199,51 @@ internal class JellyfinMigrationService } catch (Exception ex) { - logger.LogCritical(ex, "Migration {Name} failed", item.Key); + logger.LogCritical(ex, "Migration {Name} failed, migration service will attempt to roll back.", item.Key); + + if (_backupKey != default && _backupService is not null && _jellyfinDatabaseProvider is not null) + { + if (_backupKey.LibraryDb is not null) + { + logger.LogInformation("Attempt to rollback librarydb."); + try + { + var libraryDbPath = Path.Combine(_applicationPaths.DataPath, DbFilename); + File.Move(_backupKey.LibraryDb, libraryDbPath, true); + } + catch (Exception inner) + { + logger.LogCritical(inner, "Could not rollback {LibraryPath}. Manual intervention might be required to restore a operational state.", _backupKey.LibraryDb); + } + } + + if (_backupKey.JellyfinDb is not null) + { + logger.LogInformation("Attempt to rollback JellyfinDb."); + try + { + await _jellyfinDatabaseProvider.RestoreBackupFast(_backupKey.JellyfinDb, CancellationToken.None).ConfigureAwait(false); + } + catch (Exception inner) + { + logger.LogCritical(inner, "Could not rollback {LibraryPath}. Manual intervention might be required to restore a operational state.", _backupKey.JellyfinDb); + } + } + + if (_backupKey.FullBackup is not null) + { + logger.LogInformation("Attempt to rollback from backup."); + try + { + await _backupService.RestoreBackupAsync(_backupKey.FullBackup.Path).ConfigureAwait(false); + } + catch (Exception inner) + { + logger.LogCritical(inner, "Could not rollback from backup {Backup}. Manual intervention might be required to restore a operational state.", _backupKey.FullBackup.Path); + } + } + } + throw; } } @@ -188,6 +255,143 @@ internal class JellyfinMigrationService return Assembly.GetEntryAssembly()!.GetName().Version!.ToString(); } + public async Task CleanupSystemAfterMigration(ILogger logger) + { + if (_backupKey != default) + { + if (_backupKey.LibraryDb is not null) + { + logger.LogInformation("Attempt to cleanup librarydb backup."); + try + { + File.Delete(_backupKey.LibraryDb); + } + catch (Exception inner) + { + logger.LogCritical(inner, "Could not cleanup {LibraryPath}.", _backupKey.LibraryDb); + } + } + + if (_backupKey.JellyfinDb is not null && _jellyfinDatabaseProvider is not null) + { + logger.LogInformation("Attempt to cleanup JellyfinDb backup."); + try + { + await _jellyfinDatabaseProvider.DeleteBackup(_backupKey.JellyfinDb).ConfigureAwait(false); + } + catch (Exception inner) + { + logger.LogCritical(inner, "Could not cleanup {LibraryPath}.", _backupKey.JellyfinDb); + } + } + + if (_backupKey.FullBackup is not null) + { + logger.LogInformation("Attempt to cleanup from migration backup."); + try + { + File.Delete(_backupKey.FullBackup.Path); + } + catch (Exception inner) + { + logger.LogCritical(inner, "Could not cleanup backup {Backup}.", _backupKey.FullBackup.Path); + } + } + } + } + + public async Task PrepareSystemForMigration(ILogger logger) + { + logger.LogInformation("Prepare system for possible migrations"); + JellyfinMigrationBackupAttribute backupInstruction; + IReadOnlyList appliedMigrations; + var dbContext = await _dbContextFactory.CreateDbContextAsync().ConfigureAwait(false); + await using (dbContext.ConfigureAwait(false)) + { + var historyRepository = dbContext.GetService(); + var migrationsAssembly = dbContext.GetService(); + appliedMigrations = await historyRepository.GetAppliedMigrationsAsync().ConfigureAwait(false); + backupInstruction = new JellyfinMigrationBackupAttribute() + { + JellyfinDb = migrationsAssembly.Migrations.Any(f => appliedMigrations.All(e => e.MigrationId != f.Key)) + }; + } + + backupInstruction = Migrations.SelectMany(e => e) + .Where(e => appliedMigrations.All(f => f.MigrationId != e.BuildCodeMigrationId())) + .Select(e => e.BackupRequirements) + .Where(e => e is not null) + .Aggregate(backupInstruction, MergeBackupAttributes!); + + if (backupInstruction.LegacyLibraryDb) + { + logger.LogInformation("A migration will attempt to modify the library.db, will attempt to backup the file now."); + // for legacy migrations that still operates on the library.db + var libraryDbPath = Path.Combine(_applicationPaths.DataPath, DbFilename); + if (File.Exists(libraryDbPath)) + { + for (int i = 1; ; i++) + { + var bakPath = string.Format(CultureInfo.InvariantCulture, "{0}.bak{1}", libraryDbPath, i); + if (!File.Exists(bakPath)) + { + try + { + logger.LogInformation("Backing up {Library} to {BackupPath}", DbFilename, bakPath); + File.Copy(libraryDbPath, bakPath); + _backupKey = (bakPath, _backupKey.JellyfinDb, _backupKey.FullBackup); + logger.LogInformation("{Library} backed up to {BackupPath}", DbFilename, bakPath); + break; + } + catch (Exception ex) + { + logger.LogError(ex, "Cannot make a backup of {Library} at path {BackupPath}", DbFilename, bakPath); + throw; + } + } + } + + logger.LogInformation("{Library} has been backed up as {BackupPath}", DbFilename, _backupKey.LibraryDb); + } + else + { + logger.LogError("Cannot make a backup of {Library} at path {BackupPath} because file could not be found at {LibraryPath}", DbFilename, libraryDbPath, _applicationPaths.DataPath); + } + } + + if (backupInstruction.JellyfinDb && _jellyfinDatabaseProvider != null) + { + logger.LogInformation("A migration will attempt to modify the jellyfin.db, will attempt to backup the file now."); + _backupKey = (_backupKey.LibraryDb, await _jellyfinDatabaseProvider.MigrationBackupFast(CancellationToken.None).ConfigureAwait(false), _backupKey.FullBackup); + logger.LogInformation("Jellyfin database has been backed up as {BackupPath}", _backupKey.JellyfinDb); + } + + if (_backupService is not null && (backupInstruction.Metadata || backupInstruction.Subtitles || backupInstruction.Trickplay)) + { + logger.LogInformation("A migration will attempt to modify system resources. Will attempt to create backup now."); + _backupKey = (_backupKey.LibraryDb, _backupKey.JellyfinDb, await _backupService.CreateBackupAsync(new BackupOptionsDto() + { + Metadata = backupInstruction.Metadata, + Subtitles = backupInstruction.Subtitles, + Trickplay = backupInstruction.Trickplay, + Database = false // database backups are explicitly handled by the provider itself as the backup service requires parity with the current model + }).ConfigureAwait(false)); + logger.LogInformation("Pre-Migration backup successfully created as {BackupKey}", _backupKey.FullBackup.Path); + } + } + + private static JellyfinMigrationBackupAttribute MergeBackupAttributes(JellyfinMigrationBackupAttribute left, JellyfinMigrationBackupAttribute right) + { + return new JellyfinMigrationBackupAttribute() + { + JellyfinDb = left!.JellyfinDb || right!.JellyfinDb, + LegacyLibraryDb = left.LegacyLibraryDb || right!.LegacyLibraryDb, + Metadata = left.Metadata || right!.Metadata, + Subtitles = left.Subtitles || right!.Subtitles, + Trickplay = left.Trickplay || right!.Trickplay + }; + } + private class InternalCodeMigration : IInternalMigration { private readonly CodeMigration _codeMigration; diff --git a/Jellyfin.Server/Migrations/Routines/FixAudioData.cs b/Jellyfin.Server/Migrations/Routines/FixAudioData.cs index af8787b955..05ded06ba8 100644 --- a/Jellyfin.Server/Migrations/Routines/FixAudioData.cs +++ b/Jellyfin.Server/Migrations/Routines/FixAudioData.cs @@ -18,10 +18,10 @@ namespace Jellyfin.Server.Migrations.Routines /// #pragma warning disable CS0618 // Type or member is obsolete [JellyfinMigration("2025-04-20T18:00:00", nameof(FixAudioData), "CF6FABC2-9FBE-4933-84A5-FFE52EF22A58")] + [JellyfinMigrationBackup(LegacyLibraryDb = true)] internal class FixAudioData : IMigrationRoutine #pragma warning restore CS0618 // Type or member is obsolete { - private const string DbFilename = "library.db"; private readonly ILogger _logger; private readonly IServerApplicationPaths _applicationPaths; private readonly IItemRepository _itemRepository; @@ -39,29 +39,6 @@ namespace Jellyfin.Server.Migrations.Routines /// public void Perform() { - var dbPath = Path.Combine(_applicationPaths.DataPath, DbFilename); - - // Back up the database before modifying any entries - for (int i = 1; ; i++) - { - var bakPath = string.Format(CultureInfo.InvariantCulture, "{0}.bak{1}", dbPath, i); - if (!File.Exists(bakPath)) - { - try - { - _logger.LogInformation("Backing up {Library} to {BackupPath}", DbFilename, bakPath); - File.Copy(dbPath, bakPath); - _logger.LogInformation("{Library} backed up to {BackupPath}", DbFilename, bakPath); - break; - } - catch (Exception ex) - { - _logger.LogError(ex, "Cannot make a backup of {Library} at path {BackupPath}", DbFilename, bakPath); - throw; - } - } - } - _logger.LogInformation("Backfilling audio lyrics data to database."); var startIndex = 0; var records = _itemRepository.GetCount(new InternalItemsQuery diff --git a/Jellyfin.Server/Migrations/Routines/MigrateLibraryDb.cs b/Jellyfin.Server/Migrations/Routines/MigrateLibraryDb.cs index c9d2899407..309858ca70 100644 --- a/Jellyfin.Server/Migrations/Routines/MigrateLibraryDb.cs +++ b/Jellyfin.Server/Migrations/Routines/MigrateLibraryDb.cs @@ -29,6 +29,7 @@ namespace Jellyfin.Server.Migrations.Routines; /// The migration routine for migrating the userdata database to EF Core. /// [JellyfinMigration("2025-04-20T20:00:00", nameof(MigrateLibraryDb))] +[JellyfinMigrationBackup(JellyfinDb = true, LegacyLibraryDb = true)] internal class MigrateLibraryDb : IDatabaseMigrationRoutine { private const string DbFilename = "library.db"; diff --git a/Jellyfin.Server/Migrations/Routines/MigrateRatingLevels.cs b/Jellyfin.Server/Migrations/Routines/MigrateRatingLevels.cs index 234965c0a5..9aed449882 100644 --- a/Jellyfin.Server/Migrations/Routines/MigrateRatingLevels.cs +++ b/Jellyfin.Server/Migrations/Routines/MigrateRatingLevels.cs @@ -12,6 +12,7 @@ namespace Jellyfin.Server.Migrations.Routines; /// #pragma warning disable CS0618 // Type or member is obsolete [JellyfinMigration("2025-04-20T22:00:00", nameof(MigrateRatingLevels))] +[JellyfinMigrationBackup(JellyfinDb = true)] #pragma warning restore CS0618 // Type or member is obsolete internal class MigrateRatingLevels : IDatabaseMigrationRoutine { diff --git a/Jellyfin.Server/Migrations/Stages/CodeMigration.cs b/Jellyfin.Server/Migrations/Stages/CodeMigration.cs index 1e4dfb237c..addbb69bfd 100644 --- a/Jellyfin.Server/Migrations/Stages/CodeMigration.cs +++ b/Jellyfin.Server/Migrations/Stages/CodeMigration.cs @@ -6,12 +6,14 @@ using Microsoft.Extensions.DependencyInjection; namespace Jellyfin.Server.Migrations.Stages; -internal class CodeMigration(Type migrationType, JellyfinMigrationAttribute metadata) +internal class CodeMigration(Type migrationType, JellyfinMigrationAttribute metadata, JellyfinMigrationBackupAttribute? migrationBackupAttribute) { public Type MigrationType { get; } = migrationType; public JellyfinMigrationAttribute Metadata { get; } = metadata; + public JellyfinMigrationBackupAttribute? BackupRequirements { get; set; } = migrationBackupAttribute; + public string BuildCodeMigrationId() { return Metadata.Order.ToString("yyyyMMddHHmmsss", CultureInfo.InvariantCulture) + "_" + MigrationType.Name!; diff --git a/Jellyfin.Server/Migrations/Stages/JellyfinMigrationStageTypes.cs b/Jellyfin.Server/Migrations/Stages/JellyfinMigrationStageTypes.cs index d90ad3d9be..3d5ec233b3 100644 --- a/Jellyfin.Server/Migrations/Stages/JellyfinMigrationStageTypes.cs +++ b/Jellyfin.Server/Migrations/Stages/JellyfinMigrationStageTypes.cs @@ -17,7 +17,7 @@ public enum JellyfinMigrationStageTypes /// Runs after the host has been configured and includes the database migrations. /// Allows the mix order of migrations that contain application code and database changes. /// - CoreInitialisaition = 2, + CoreInitialisation = 2, /// /// Runs after services has been registered and initialised. Last step before running the server. diff --git a/Jellyfin.Server/Program.cs b/Jellyfin.Server/Program.cs index 4584b25bdf..9f2c71ce25 100644 --- a/Jellyfin.Server/Program.cs +++ b/Jellyfin.Server/Program.cs @@ -16,10 +16,10 @@ using Jellyfin.Server.Extensions; using Jellyfin.Server.Helpers; using Jellyfin.Server.Implementations.DatabaseConfiguration; using Jellyfin.Server.Implementations.Extensions; -using Jellyfin.Server.Implementations.FullSystemBackup; using Jellyfin.Server.Implementations.StorageHelpers; using Jellyfin.Server.Implementations.SystemBackupService; using Jellyfin.Server.Migrations; +using Jellyfin.Server.Migrations.Stages; using Jellyfin.Server.ServerSetupApp; using MediaBrowser.Common.Configuration; using MediaBrowser.Common.Net; @@ -190,12 +190,14 @@ namespace Jellyfin.Server return; } - await ApplyCoreMigrationsAsync(appHost.ServiceProvider, Migrations.Stages.JellyfinMigrationStageTypes.CoreInitialisaition).ConfigureAwait(false); + var jellyfinMigrationService = ActivatorUtilities.CreateInstance(appHost.ServiceProvider); + await jellyfinMigrationService.PrepareSystemForMigration(_logger).ConfigureAwait(false); + await jellyfinMigrationService.MigrateStepAsync(JellyfinMigrationStageTypes.CoreInitialisation, appHost.ServiceProvider).ConfigureAwait(false); await appHost.InitializeServices(startupConfig).ConfigureAwait(false); - await ApplyCoreMigrationsAsync(appHost.ServiceProvider, Migrations.Stages.JellyfinMigrationStageTypes.AppInitialisation).ConfigureAwait(false); - + await jellyfinMigrationService.MigrateStepAsync(JellyfinMigrationStageTypes.AppInitialisation, appHost.ServiceProvider).ConfigureAwait(false); + await jellyfinMigrationService.CleanupSystemAfterMigration(_logger).ConfigureAwait(false); try { await _setupServer!.StopAsync().ConfigureAwait(false); diff --git a/MediaBrowser.Controller/SystemBackupService/BackupOptionsDto.cs b/MediaBrowser.Controller/SystemBackupService/BackupOptionsDto.cs index 228839a1d8..fc5a109f1a 100644 --- a/MediaBrowser.Controller/SystemBackupService/BackupOptionsDto.cs +++ b/MediaBrowser.Controller/SystemBackupService/BackupOptionsDto.cs @@ -21,4 +21,9 @@ public class BackupOptionsDto /// Gets or sets a value indicating whether the archive contains the Subtitle contents. /// public bool Subtitles { get; set; } + + /// + /// Gets or sets a value indicating whether the archive contains the Database contents. + /// + public bool Database { get; set; } = true; } diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/IJellyfinDatabaseProvider.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/IJellyfinDatabaseProvider.cs index b0dc984699..6b35810b22 100644 --- a/src/Jellyfin.Database/Jellyfin.Database.Implementations/IJellyfinDatabaseProvider.cs +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/IJellyfinDatabaseProvider.cs @@ -64,6 +64,13 @@ public interface IJellyfinDatabaseProvider /// A representing the result of the asynchronous operation. Task RestoreBackupFast(string key, CancellationToken cancellationToken); + /// + /// Deletes a backup that has been previously created by . + /// + /// The key to the backup which should be cleaned up. + /// A representing the result of the asynchronous operation. + Task DeleteBackup(string key); + /// /// Removes all contents from the database. /// diff --git a/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/SqliteDatabaseProvider.cs b/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/SqliteDatabaseProvider.cs index 519584003c..dda1ca0758 100644 --- a/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/SqliteDatabaseProvider.cs +++ b/src/Jellyfin.Database/Jellyfin.Database.Providers.Sqlite/SqliteDatabaseProvider.cs @@ -129,6 +129,21 @@ public sealed class SqliteDatabaseProvider : IJellyfinDatabaseProvider return Task.CompletedTask; } + /// + public Task DeleteBackup(string key) + { + var backupFile = Path.Combine(_applicationPaths.DataPath, BackupFolderName, $"{key}_jellyfin.db"); + + if (!File.Exists(backupFile)) + { + _logger.LogCritical("Tried to delete a backup that does not exist: {Key}", key); + return Task.CompletedTask; + } + + File.Delete(backupFile); + return Task.CompletedTask; + } + /// public async Task PurgeDatabase(JellyfinDbContext dbContext, IEnumerable? tableNames) { diff --git a/tests/Jellyfin.Server.Integration.Tests/JellyfinApplicationFactory.cs b/tests/Jellyfin.Server.Integration.Tests/JellyfinApplicationFactory.cs index c09bce52da..b2cde2aabc 100644 --- a/tests/Jellyfin.Server.Integration.Tests/JellyfinApplicationFactory.cs +++ b/tests/Jellyfin.Server.Integration.Tests/JellyfinApplicationFactory.cs @@ -106,7 +106,7 @@ namespace Jellyfin.Server.Integration.Tests appHost.ServiceProvider = host.Services; var applicationPaths = appHost.ServiceProvider.GetRequiredService(); Program.ApplyStartupMigrationAsync((ServerApplicationPaths)applicationPaths, appHost.ServiceProvider.GetRequiredService()).GetAwaiter().GetResult(); - Program.ApplyCoreMigrationsAsync(appHost.ServiceProvider, Migrations.Stages.JellyfinMigrationStageTypes.CoreInitialisaition).GetAwaiter().GetResult(); + Program.ApplyCoreMigrationsAsync(appHost.ServiceProvider, Migrations.Stages.JellyfinMigrationStageTypes.CoreInitialisation).GetAwaiter().GetResult(); appHost.InitializeServices(Mock.Of()).GetAwaiter().GetResult(); Program.ApplyCoreMigrationsAsync(appHost.ServiceProvider, Migrations.Stages.JellyfinMigrationStageTypes.AppInitialisation).GetAwaiter().GetResult(); host.Start(); -- cgit v1.2.3 From 91da1c035d7b8ee8c7456d62f52dd9e63b5fc59e Mon Sep 17 00:00:00 2001 From: JPVenson Date: Tue, 10 Jun 2025 14:31:01 +0000 Subject: Fix schema name on backup --- Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs') diff --git a/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs b/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs index e266d5a3bc..ad8f5f3370 100644 --- a/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs +++ b/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs @@ -297,7 +297,7 @@ public class BackupService : IBackupService .GetProperties(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance) .Where(e => e.PropertyType.IsAssignableTo(typeof(IQueryable))) .Select(e => (Type: e.PropertyType, ValueFactory: new Func>(() => GetValues((IQueryable)e.GetValue(dbContext)!, e.PropertyType)))), - (Type: typeof(HistoryRow), ValueFactory: new Func>(() => migrations.ToAsyncEnumerable())) + (Type: typeof(IQueryable), ValueFactory: new Func>(() => migrations.ToAsyncEnumerable())) ]; manifest.DatabaseTables = entityTypes.Select(e => e.Type.Name).ToArray(); var transaction = await dbContext.Database.BeginTransactionAsync().ConfigureAwait(false); @@ -308,6 +308,7 @@ public class BackupService : IBackupService foreach (var entityType in entityTypes) { + var schemaName = dbContext.Model.FindEntityType(entityType.Type.GetGenericArguments()[0])!.GetSchemaQualifiedTableName()!; _logger.LogInformation("Begin backup of entity {Table}", entityType.Type.Name); var zipEntry = zipArchive.CreateEntry($"Database\\{entityType.Type.Name}.json"); var entities = 0; -- cgit v1.2.3 From 4a0a45a0459a9921cee88456f2f5bac0dc294f1f Mon Sep 17 00:00:00 2001 From: JPVenson Date: Tue, 10 Jun 2025 14:33:41 +0000 Subject: Use explicit naming --- .../FullSystemBackup/BackupService.cs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs') diff --git a/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs b/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs index ad8f5f3370..d439fcb18b 100644 --- a/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs +++ b/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs @@ -292,12 +292,12 @@ public class BackupService : IBackupService var historyRepository = dbContext.GetService(); var migrations = await historyRepository.GetAppliedMigrationsAsync().ConfigureAwait(false); - ICollection<(Type Type, Func> ValueFactory)> entityTypes = [ + ICollection<(Type Type, string SourceName, Func> ValueFactory)> entityTypes = [ .. typeof(JellyfinDbContext) .GetProperties(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance) .Where(e => e.PropertyType.IsAssignableTo(typeof(IQueryable))) - .Select(e => (Type: e.PropertyType, ValueFactory: new Func>(() => GetValues((IQueryable)e.GetValue(dbContext)!, e.PropertyType)))), - (Type: typeof(IQueryable), ValueFactory: new Func>(() => migrations.ToAsyncEnumerable())) + .Select(e => (Type: e.PropertyType, dbContext.Model.FindEntityType(e.PropertyType.GetGenericArguments()[0])!.GetSchemaQualifiedTableName()!, ValueFactory: new Func>(() => GetValues((IQueryable)e.GetValue(dbContext)!, e.PropertyType)))), + (Type: typeof(HistoryRow), SourceName: nameof(HistoryRow), ValueFactory: new Func>(() => migrations.ToAsyncEnumerable())) ]; manifest.DatabaseTables = entityTypes.Select(e => e.Type.Name).ToArray(); var transaction = await dbContext.Database.BeginTransactionAsync().ConfigureAwait(false); @@ -308,9 +308,8 @@ public class BackupService : IBackupService foreach (var entityType in entityTypes) { - var schemaName = dbContext.Model.FindEntityType(entityType.Type.GetGenericArguments()[0])!.GetSchemaQualifiedTableName()!; - _logger.LogInformation("Begin backup of entity {Table}", entityType.Type.Name); - var zipEntry = zipArchive.CreateEntry($"Database\\{entityType.Type.Name}.json"); + _logger.LogInformation("Begin backup of entity {Table}", entityType.SourceName); + var zipEntry = zipArchive.CreateEntry($"Database\\{entityType.SourceName}.json"); var entities = 0; var zipEntryStream = zipEntry.Open(); await using (zipEntryStream.ConfigureAwait(false)) -- cgit v1.2.3 From 21a6d6f0d61c4a24e2a899c57ad351f4632fb4f9 Mon Sep 17 00:00:00 2001 From: JPVenson Date: Fri, 18 Jul 2025 02:19:41 +0300 Subject: Delete old migrations on restore (#14486) --- Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs') diff --git a/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs b/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs index d439fcb18b..6e8936d536 100644 --- a/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs +++ b/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs @@ -165,6 +165,13 @@ public class BackupService : IBackupService var historyRepository = dbContext.GetService(); await historyRepository.CreateIfNotExistsAsync().ConfigureAwait(false); + + foreach (var item in await historyRepository.GetAppliedMigrationsAsync(CancellationToken.None).ConfigureAwait(false)) + { + var insertScript = historyRepository.GetDeleteScript(item.MigrationId); + await dbContext.Database.ExecuteSqlRawAsync(insertScript).ConfigureAwait(false); + } + foreach (var item in historyEntries) { var insertScript = historyRepository.GetInsertScript(item); -- cgit v1.2.3 From 36c90ce2ce361dfdc165a1deda3434e66d28ef52 Mon Sep 17 00:00:00 2001 From: Cody Robibero Date: Sun, 20 Jul 2025 20:01:13 -0600 Subject: Clean up and fix backup/restore (#14489) --- .../FullSystemBackup/BackupService.cs | 53 ++++++++++++++-------- 1 file changed, 33 insertions(+), 20 deletions(-) (limited to 'Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs') diff --git a/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs b/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs index 6e8936d536..74d99455df 100644 --- a/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs +++ b/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs @@ -39,7 +39,7 @@ public class BackupService : IBackupService ReferenceHandler = ReferenceHandler.IgnoreCycles, }; - private readonly Version _backupEngineVersion = Version.Parse("0.1.0"); + private readonly Version _backupEngineVersion = Version.Parse("0.2.0"); /// /// Initializes a new instance of the class. @@ -120,26 +120,29 @@ public class BackupService : IBackupService void CopyDirectory(string source, string target) { - source = Path.GetFullPath(source); - Directory.CreateDirectory(source); - + var fullSourcePath = NormalizePathSeparator(Path.GetFullPath(source) + Path.DirectorySeparatorChar); + var fullTargetRoot = Path.GetFullPath(target) + Path.DirectorySeparatorChar; foreach (var item in zipArchive.Entries) { - var sanitizedSourcePath = Path.GetFullPath(item.FullName.TrimEnd(Path.DirectorySeparatorChar) + Path.DirectorySeparatorChar); - if (!sanitizedSourcePath.StartsWith(target, StringComparison.Ordinal)) + var sourcePath = NormalizePathSeparator(Path.GetFullPath(item.FullName)); + var targetPath = Path.GetFullPath(Path.Combine(target, Path.GetRelativePath(source, item.FullName))); + + if (!sourcePath.StartsWith(fullSourcePath, StringComparison.Ordinal) + || !targetPath.StartsWith(fullTargetRoot, StringComparison.Ordinal)) { continue; } - var targetPath = Path.Combine(source, sanitizedSourcePath[target.Length..].Trim('/')); _logger.LogInformation("Restore and override {File}", targetPath); - item.ExtractToFile(targetPath); + + Directory.CreateDirectory(Path.GetDirectoryName(targetPath)!); + item.ExtractToFile(targetPath, overwrite: true); } } - CopyDirectory(_applicationPaths.ConfigurationDirectoryPath, "Config/"); - CopyDirectory(_applicationPaths.DataPath, "Data/"); - CopyDirectory(_applicationPaths.RootFolderPath, "Root/"); + CopyDirectory("Config", _applicationPaths.ConfigurationDirectoryPath); + CopyDirectory("Data", _applicationPaths.DataPath); + CopyDirectory("Root", _applicationPaths.RootFolderPath); if (manifest.Options.Database) { @@ -148,7 +151,7 @@ public class BackupService : IBackupService await using (dbContext.ConfigureAwait(false)) { // restore migration history manually - var historyEntry = zipArchive.GetEntry($"Database\\{nameof(HistoryRow)}.json"); + var historyEntry = zipArchive.GetEntry(NormalizePathSeparator(Path.Combine("Database", $"{nameof(HistoryRow)}.json"))); if (historyEntry is null) { _logger.LogInformation("No backup of the history table in archive. This is required for Jellyfin operation"); @@ -193,7 +196,7 @@ public class BackupService : IBackupService { _logger.LogInformation("Read backup of {Table}", entityType.Type.Name); - var zipEntry = zipArchive.GetEntry($"Database\\{entityType.Type.Name}.json"); + var zipEntry = zipArchive.GetEntry(NormalizePathSeparator(Path.Combine("Database", $"{entityType.Type.Name}.json"))); if (zipEntry is null) { _logger.LogInformation("No backup of expected table {Table} is present in backup. Continue anyway.", entityType.Type.Name); @@ -205,7 +208,7 @@ public class BackupService : IBackupService { _logger.LogInformation("Restore backup of {Table}", entityType.Type.Name); var records = 0; - await foreach (var item in JsonSerializer.DeserializeAsyncEnumerable(zipEntryStream, _serializerSettings).ConfigureAwait(false)!) + await foreach (var item in JsonSerializer.DeserializeAsyncEnumerable(zipEntryStream, _serializerSettings).ConfigureAwait(false)) { var entity = item.Deserialize(entityType.Type.PropertyType.GetGenericArguments()[0]); if (entity is null) @@ -288,7 +291,7 @@ public class BackupService : IBackupService await using (dbContext.ConfigureAwait(false)) { dbContext.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking; - static IAsyncEnumerable GetValues(IQueryable dbSet, Type type) + static IAsyncEnumerable GetValues(IQueryable dbSet) { var method = dbSet.GetType().GetMethod(nameof(DbSet.AsAsyncEnumerable))!; var enumerable = method.Invoke(dbSet, null)!; @@ -303,8 +306,8 @@ public class BackupService : IBackupService .. typeof(JellyfinDbContext) .GetProperties(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance) .Where(e => e.PropertyType.IsAssignableTo(typeof(IQueryable))) - .Select(e => (Type: e.PropertyType, dbContext.Model.FindEntityType(e.PropertyType.GetGenericArguments()[0])!.GetSchemaQualifiedTableName()!, ValueFactory: new Func>(() => GetValues((IQueryable)e.GetValue(dbContext)!, e.PropertyType)))), - (Type: typeof(HistoryRow), SourceName: nameof(HistoryRow), ValueFactory: new Func>(() => migrations.ToAsyncEnumerable())) + .Select(e => (Type: e.PropertyType, dbContext.Model.FindEntityType(e.PropertyType.GetGenericArguments()[0])!.GetSchemaQualifiedTableName()!, ValueFactory: new Func>(() => GetValues((IQueryable)e.GetValue(dbContext)!)))), + (Type: typeof(HistoryRow), SourceName: nameof(HistoryRow), ValueFactory: () => migrations.ToAsyncEnumerable()) ]; manifest.DatabaseTables = entityTypes.Select(e => e.Type.Name).ToArray(); var transaction = await dbContext.Database.BeginTransactionAsync().ConfigureAwait(false); @@ -316,7 +319,7 @@ public class BackupService : IBackupService foreach (var entityType in entityTypes) { _logger.LogInformation("Begin backup of entity {Table}", entityType.SourceName); - var zipEntry = zipArchive.CreateEntry($"Database\\{entityType.SourceName}.json"); + var zipEntry = zipArchive.CreateEntry(NormalizePathSeparator(Path.Combine("Database", $"{entityType.SourceName}.json"))); var entities = 0; var zipEntryStream = zipEntry.Open(); await using (zipEntryStream.ConfigureAwait(false)) @@ -354,7 +357,7 @@ public class BackupService : IBackupService foreach (var item in Directory.EnumerateFiles(_applicationPaths.ConfigurationDirectoryPath, "*.xml", SearchOption.TopDirectoryOnly) .Union(Directory.EnumerateFiles(_applicationPaths.ConfigurationDirectoryPath, "*.json", SearchOption.TopDirectoryOnly))) { - zipArchive.CreateEntryFromFile(item, Path.Combine("Config", Path.GetFileName(item))); + zipArchive.CreateEntryFromFile(item, NormalizePathSeparator(Path.Combine("Config", Path.GetFileName(item)))); } void CopyDirectory(string source, string target, string filter = "*") @@ -368,7 +371,7 @@ public class BackupService : IBackupService foreach (var item in Directory.EnumerateFiles(source, filter, SearchOption.AllDirectories)) { - zipArchive.CreateEntryFromFile(item, Path.Combine(target, item[..source.Length].Trim('\\'))); + zipArchive.CreateEntryFromFile(item, NormalizePathSeparator(Path.Combine(target, Path.GetRelativePath(source, item)))); } } @@ -516,4 +519,14 @@ public class BackupService : IBackupService Database = options.Database }; } + + /// + /// Windows is able to handle '/' as a path seperator in zip files + /// but linux isn't able to handle '\' as a path seperator in zip files, + /// So normalize to '/'. + /// + /// The path to normalize. + /// The normalized path. + private static string NormalizePathSeparator(string path) + => path.Replace('\\', '/'); } -- cgit v1.2.3