Add support for VobSub subtitle streams (#16552)

* Add support for VobSub subtitle streams

* update logic to determine separate extraction for VobSub subtitles

* simplify VobSub extraction logic and fix ffmpeg command

* Match `ExtractAllExtractableSubtitlesMKS` with `ExtractAllExtractableSubtitlesInternal` Matroska's VobSub option

* Add a comments clarify why MKS was used, and remove the redundant VobSub extension branch

* remove redundant VobSub format check

* fix type errors

1 files changed, 25 insertions, 11 deletions

diff --git a/MediaBrowser.MediaEncoding/Subtitles/SubtitleEncoder.cs b/MediaBrowser.MediaEncoding/Subtitles/SubtitleEncoder.cs
index e0c5f3ad39..8d237473a3 100644
--- a/MediaBrowser.MediaEncoding/Subtitles/SubtitleEncoder.cs
+++ b/MediaBrowser.MediaEncoding/Subtitles/SubtitleEncoder.cs
@@ -220,12 +220,11 @@ namespace MediaBrowser.MediaEncoding.Subtitles
                     Path = outputPath,
                     Protocol = MediaProtocol.File,
                     Format = outputFormat,
-                    IsExternal = false
+                    IsExternal = MediaStream.IsVobSubFormat(outputFormat)
                 };
             }
 
-            var currentFormat = subtitleStream.Codec ?? Path.GetExtension(subtitleStream.Path)
-                .TrimStart('.');
+            var currentFormat = subtitleStream.Codec ?? Path.GetExtension(subtitleStream.Path).TrimStart('.');
 
             // Handle PGS subtitles as raw streams for the client to render
             if (MediaStream.IsPgsFormat(currentFormat))
@@ -475,6 +474,10 @@ namespace MediaBrowser.MediaEncoding.Subtitles
             {
                 return subtitleStream.Codec;
             }
+            else if (MediaStream.IsVobSubFormat(subtitleStream.Codec))
+            {
+                return "mks";
+            }
             else
             {
                 return "srt";
@@ -488,6 +491,11 @@ namespace MediaBrowser.MediaEncoding.Subtitles
             {
                 return "sup";
             }
+            else if (MediaStream.IsVobSubFormat(subtitleStream.Codec))
+            {
+                // FFmpeg cannot mux VobSub subtitle streams back into the .idx/.sub pair, so we use .mks container instead.
+                return "mks";
+            }
             else
             {
                 return GetExtractableSubtitleFormat(subtitleStream);
@@ -500,7 +508,8 @@ namespace MediaBrowser.MediaEncoding.Subtitles
                 || string.Equals(codec, "ssa", StringComparison.OrdinalIgnoreCase)
                 || string.Equals(codec, "srt", StringComparison.OrdinalIgnoreCase)
                 || string.Equals(codec, "subrip", StringComparison.OrdinalIgnoreCase)
-                || string.Equals(codec, "pgssub", StringComparison.OrdinalIgnoreCase);
+                || string.Equals(codec, "pgssub", StringComparison.OrdinalIgnoreCase)
+                || MediaStream.IsVobSubFormat(codec);
         }
 
         /// <inheritdoc />
@@ -516,7 +525,8 @@ namespace MediaBrowser.MediaEncoding.Subtitles
 
                 foreach (var subtitleStream in subtitleStreams)
                 {
-                    if (subtitleStream.IsExternal && !subtitleStream.Path.EndsWith(".mks", StringComparison.OrdinalIgnoreCase))
+                    if (subtitleStream.IsExternal
+                        && !subtitleStream.Path.EndsWith(".mks", StringComparison.OrdinalIgnoreCase))
                     {
                         continue;
                     }
@@ -603,6 +613,8 @@ namespace MediaBrowser.MediaEncoding.Subtitles
                     }
 
                     var outputCodec = IsCodecCopyable(subtitleStream.Codec) ? "copy" : "srt";
+                    // FFmpeg does not provide an .idx/.sub muxer, so VobSub streams must be written as MKS files.
+                    var outputFormatOption = MediaStream.IsVobSubFormat(subtitleStream.Codec) ? " -f matroska" : string.Empty;
                     var streamIndex = EncodingHelper.FindIndex(mediaSource.MediaStreams, subtitleStream);
 
                     if (streamIndex == -1)
@@ -616,9 +628,10 @@ namespace MediaBrowser.MediaEncoding.Subtitles
                     outputPaths.Add(outputPath);
                     args += string.Format(
                         CultureInfo.InvariantCulture,
-                        " -map 0:{0} -an -vn -c:s {1} -flush_packets 1 \"{2}\"",
+                        " -map 0:{0} -an -vn -c:s {1}{2} -flush_packets 1 \"{3}\"",
                         streamIndex,
                         outputCodec,
+                        outputFormatOption,
                         outputPath);
                 }
 
@@ -653,6 +666,8 @@ namespace MediaBrowser.MediaEncoding.Subtitles
                 }
 
                 var outputCodec = IsCodecCopyable(subtitleStream.Codec) ? "copy" : "srt";
+                // FFmpeg does not provide an .idx/.sub muxer, so VobSub streams must be written as MKS files.
+                var outputFormatOption = MediaStream.IsVobSubFormat(subtitleStream.Codec) ? " -f matroska" : string.Empty;
                 var streamIndex = EncodingHelper.FindIndex(mediaSource.MediaStreams, subtitleStream);
 
                 if (streamIndex == -1)
@@ -666,18 +681,17 @@ namespace MediaBrowser.MediaEncoding.Subtitles
                 outputPaths.Add(outputPath);
                 args += string.Format(
                     CultureInfo.InvariantCulture,
-                    " -map 0:{0} -an -vn -c:s {1} -flush_packets 1 \"{2}\"",
+                    " -map 0:{0} -an -vn -c:s {1}{2} -flush_packets 1 \"{3}\"",
                     streamIndex,
                     outputCodec,
+                    outputFormatOption,
                     outputPath);
             }
 
-            if (outputPaths.Count == 0)
+            if (outputPaths.Count > 0)
             {
-                return;
+                await ExtractSubtitlesForFile(inputPath, args, outputPaths, cancellationToken).ConfigureAwait(false);
             }
-
-            await ExtractSubtitlesForFile(inputPath, args, outputPaths, cancellationToken).ConfigureAwait(false);
         }
 
         private async Task ExtractSubtitlesForFile(