Subtitles!

Need to add support for dumping subtitles as a file...

server/src/main/java/xyz/e3ndr/athena/Athena.java

@@ -27,6 +27,7 @@
 import xyz.e3ndr.athena.transcoding.Transcoder;
 import xyz.e3ndr.athena.types.AudioCodec;
 import xyz.e3ndr.athena.types.ContainerFormat;
+import xyz.e3ndr.athena.types.SubtitleCodec;
 import xyz.e3ndr.athena.types.VideoCodec;
 import xyz.e3ndr.athena.types.VideoQuality;
 import xyz.e3ndr.athena.types.media.Media;
@@ -243,13 +244,13 @@ public static List<String> listIngestables() {
     /* Media                */
     /* -------------------- */
 
-    public static @Nullable MediaSession startStream(Media media, VideoQuality desiredQuality, VideoCodec desiredVCodec, AudioCodec desiredACodec, ContainerFormat desiredContainer, int... streamIds) throws IOException {
+    public static @Nullable MediaSession startStream(Media media, VideoQuality desiredQuality, VideoCodec desiredVCodec, AudioCodec desiredACodec, SubtitleCodec desiredSCodec, ContainerFormat desiredContainer, int... streamIds) throws IOException {
         if (desiredVCodec == VideoCodec.SOURCE) {
             desiredQuality = VideoQuality.UHD; // Doesn't really matter what we pick here. We just want to reduce duplicate
                                                // transcodes, and this is part of the id.
         }
 
-        final File cacheFile = Transcoder.getFile(media, desiredQuality, desiredVCodec, desiredACodec, desiredContainer, streamIds);
+        final File cacheFile = Transcoder.getFile(media, desiredQuality, desiredVCodec, desiredACodec, desiredSCodec, desiredContainer, streamIds);
         TranscodeSession transcodeSession = null;
 
         if (cacheFile.exists()) {
@@ -261,11 +262,11 @@ public static List<String> listIngestables() {
                 }
             }
         } else {
-            transcodeSession = Transcoder.start(cacheFile, media, desiredQuality, desiredVCodec, desiredACodec, desiredContainer, streamIds);
+            transcodeSession = Transcoder.start(cacheFile, media, desiredQuality, desiredVCodec, desiredACodec, desiredSCodec, desiredContainer, streamIds);
             if (transcodeSession == null) return null; // Couldn't start transcode, check the logs.
         }
 
-        return new MediaSession(cacheFile, transcodeSession, media.getId(), desiredQuality, desiredVCodec, desiredACodec, desiredContainer, streamIds);
+        return new MediaSession(cacheFile, transcodeSession, media.getId(), desiredQuality, desiredVCodec, desiredACodec, desiredSCodec, desiredContainer, streamIds);
     }
 
     public static @Nullable Media getMedia(String mediaId) {

server/src/main/java/xyz/e3ndr/athena/MediaSession.java

@@ -19,6 +19,7 @@
 import xyz.e3ndr.athena.transcoding.TranscodeSession;
 import xyz.e3ndr.athena.types.AudioCodec;
 import xyz.e3ndr.athena.types.ContainerFormat;
+import xyz.e3ndr.athena.types.SubtitleCodec;
 import xyz.e3ndr.athena.types.VideoCodec;
 import xyz.e3ndr.athena.types.VideoQuality;
 import xyz.e3ndr.fastloggingframework.logging.FastLogger;
@@ -38,6 +39,7 @@ public class MediaSession {
     private VideoQuality videoQuality;
     private VideoCodec videoCodec;
     private AudioCodec audioCodec;
+    private SubtitleCodec subtitleCodec;
     private ContainerFormat containerFormat;
     private int[] streamIds;
 
@@ -50,14 +52,15 @@ public class MediaSession {
     @Getter(AccessLevel.NONE)
     public final FastLogger logger = new FastLogger("Media Session: ".concat(this.id));
 
-    public MediaSession(File file, @Nullable TranscodeSession transcodeSession, String mediaId, VideoQuality desiredQuality, VideoCodec desiredVCodec, AudioCodec desiredACodec, ContainerFormat desiredContainer, int... streamIds) throws IOException {
+    public MediaSession(File file, @Nullable TranscodeSession transcodeSession, String mediaId, VideoQuality desiredQuality, VideoCodec desiredVCodec, AudioCodec desiredACodec, SubtitleCodec desiredSCodec, ContainerFormat desiredContainer, int... streamIds) throws IOException {
         this.file = file;
         this.transcodeSession = transcodeSession;
         this.isCached = this.transcodeSession == null;
         this.mediaId = mediaId;
         this.videoQuality = desiredQuality;
         this.videoCodec = desiredVCodec;
         this.audioCodec = desiredACodec;
+        this.subtitleCodec = desiredSCodec;
         this.containerFormat = desiredContainer;
         this.streamIds = streamIds;
     }

server/src/main/java/xyz/e3ndr/athena/service/MediaStreamRoutes.java

@@ -17,6 +17,7 @@
 import xyz.e3ndr.athena.MediaSession;
 import xyz.e3ndr.athena.types.AudioCodec;
 import xyz.e3ndr.athena.types.ContainerFormat;
+import xyz.e3ndr.athena.types.SubtitleCodec;
 import xyz.e3ndr.athena.types.VideoCodec;
 import xyz.e3ndr.athena.types.VideoQuality;
 import xyz.e3ndr.athena.types.media.Media;
@@ -28,6 +29,7 @@ private MediaSession startSession(Media media, Map<String, String> query) throws
         VideoQuality videoQuality = VideoQuality.valueOf(query.getOrDefault("quality", VideoQuality.UHD.name()).toUpperCase());
         VideoCodec videoCodec = VideoCodec.valueOf(query.getOrDefault("videoCodec", VideoCodec.SOURCE.name()).toUpperCase());
         AudioCodec audioCodec = AudioCodec.valueOf(query.getOrDefault("audioCodec", AudioCodec.SOURCE.name()).toUpperCase());
+        SubtitleCodec subtitleCodec = SubtitleCodec.valueOf(query.getOrDefault("subtitleCodec", SubtitleCodec.SOURCE.name()).toUpperCase());
         ContainerFormat containerFormat = ContainerFormat.valueOf(query.getOrDefault("format", ContainerFormat.MKV.name()).toUpperCase());
 
         // Parse out the streamIds.
@@ -48,7 +50,7 @@ private MediaSession startSession(Media media, Map<String, String> query) throws
         return Athena.startStream(
             media,
             videoQuality,
-            videoCodec, audioCodec,
+            videoCodec, audioCodec, subtitleCodec,
             containerFormat,
             streamIds
         );

server/src/main/java/xyz/e3ndr/athena/service/ftp/FtpClient.java

@@ -18,6 +18,7 @@
 import xyz.e3ndr.athena.MediaSession;
 import xyz.e3ndr.athena.types.AudioCodec;
 import xyz.e3ndr.athena.types.ContainerFormat;
+import xyz.e3ndr.athena.types.SubtitleCodec;
 import xyz.e3ndr.athena.types.VideoCodec;
 import xyz.e3ndr.athena.types.VideoQuality;
 import xyz.e3ndr.athena.types.media.Media;
@@ -625,7 +626,7 @@ private void command_RETR(String file) {
                 MediaSession session = Athena.startStream(
                     media,
                     this.videoQuality,
-                    this.videoCodec, this.audioCodec,
+                    this.videoCodec, this.audioCodec, SubtitleCodec.SOURCE,
                     this.containerFormat,
                     streamIds
                 );

server/src/main/java/xyz/e3ndr/athena/service/http/MediaRoutes.java

@@ -42,8 +42,8 @@ public HttpResponse onGetMediaById(SoraHttpSession session) {
             Rson.DEFAULT.toJson(media),
             Map.of(
                 "list", "GET /api/media?start&limit",
-                "stream_raw", "GET /api/media/:mediaId/stream/raw?quality&videoCodec&audioCodec&format&skipTo",
-                "stream_hls", "GET /api/media/:mediaId/stream/hls?quality&videoCodec&audioCodec&format&skipTo"
+                "stream_raw", "GET /api/media/:mediaId/stream/raw?quality&videoCodec&audioCodec&subtitleCodec&format&skipTo",
+                "stream_hls", "GET /api/media/:mediaId/stream/hls?quality&videoCodec&audioCodec&subtitleCodec&format&skipTo"
             )
         )
             .putHeader("Access-Control-Allow-Origin", session.getHeaders().getOrDefault("Origin", Arrays.asList("*")).get(0));

server/src/main/java/xyz/e3ndr/athena/service/simple_ui/UIRoutes.java

@@ -28,11 +28,13 @@
 import xyz.e3ndr.athena.service.HTMLBuilder;
 import xyz.e3ndr.athena.types.AudioCodec;
 import xyz.e3ndr.athena.types.ContainerFormat;
+import xyz.e3ndr.athena.types.SubtitleCodec;
 import xyz.e3ndr.athena.types.VideoCodec;
 import xyz.e3ndr.athena.types.VideoQuality;
 import xyz.e3ndr.athena.types.media.Media;
 import xyz.e3ndr.athena.types.media.MediaFiles.Streams;
 import xyz.e3ndr.athena.types.media.MediaFiles.Streams.AudioStream;
+import xyz.e3ndr.athena.types.media.MediaFiles.Streams.SubtitleStream;
 import xyz.e3ndr.athena.types.media.MediaFiles.Streams.VideoStream;
 import xyz.e3ndr.fastloggingframework.logging.FastLogger;
 import xyz.e3ndr.fastloggingframework.logging.LogLevel;
@@ -246,6 +248,35 @@ public HttpResponse onViewIngestMapStreams(SoraHttpSession session) {
                     break;
                 }
 
+                case "subtitle": {
+                    String codecName = codec.getString("codec_name");
+
+                    try {
+                        SubtitleCodec.valueOf(codecName.toUpperCase());
+                    } catch (Exception ignored) {
+                        continue;
+                    }
+
+                    String language = "Unknown";
+                    String title = "Subtitle (" + codecName + ")";
+                    if (codec.containsKey("tags")) {
+                        JsonObject tags = codec.getObject("tags");
+                        if (tags.containsKey("language")) {
+                            language = tags.getString("language");
+                        }
+                        if (tags.containsKey("title")) {
+                            title = tags.getString("title");
+                        }
+                    }
+
+                    html
+                        .f("Name: <input type=\"input\" name=\"stream/%d/name\" value=\"%s\" />", codecIdx, title)
+                        .f("Language: <input type=\"input\" name=\"stream/%d/language\" value=\"%s\" />", codecIdx, language)
+                        .f("<input type=\"input\" name=\"stream/%d/codec\" value=\"%s\" style=\"display: none;\" />", codecIdx, codecName)
+                        .f("<input type=\"input\" name=\"stream/%d/type\" value=\"subtitle\" style=\"display: none;\" />", codecIdx);
+                    break;
+                }
+
                 // TODO others.
             }
             codecIdx++;
@@ -272,6 +303,7 @@ public HttpResponse onViewIngestFinalize(SoraHttpSession session) {
         media.getFiles().setStreams(new Streams());
         media.getFiles().getStreams().setVideo(new LinkedList<>());
         media.getFiles().getStreams().setAudio(new LinkedList<>());
+        media.getFiles().getStreams().setSubtitles(new LinkedList<>());
 
         // defaultStream
         JsonArray defaultStreams = new JsonArray();
@@ -337,6 +369,16 @@ public HttpResponse onViewIngestFinalize(SoraHttpSession session) {
                             Rson.DEFAULT.fromJson(json, AudioStream.class)
                         );
                     break;
+
+                case "subtitle":
+                    media
+                        .getFiles()
+                        .getStreams()
+                        .getSubtitles()
+                        .add(
+                            Rson.DEFAULT.fromJson(json, SubtitleStream.class)
+                        );
+                    break;
             }
         }
 
@@ -514,6 +556,7 @@ public HttpResponse onViewSpecificMedia(SoraHttpSession session) {
             .f("          <select name=\"container\">" + containerOptions + "</select>")
             .f("          <select name=\"vCodec\">" + vCodecOptions + "</select>")
             .f("          <select name=\"aCodec\">" + aCodecOptions + "</select>")
+            .f("          <select name=\"sCodec\"><option selected>WEBVTT</option></select>")
             .f("          <select name=\"quality\">" + qualityOptions + "</select>")
             .f("          <button type=\"submit\">Watch</button>");
 
@@ -541,16 +584,17 @@ public HttpResponse onWatchSpecificMedia(SoraHttpSession session) {
         ContainerFormat container = ContainerFormat.valueOf(session.getQueryParameters().get("container"));
         VideoCodec vCodec = VideoCodec.valueOf(session.getQueryParameters().get("vCodec"));
         AudioCodec aCodec = AudioCodec.valueOf(session.getQueryParameters().get("aCodec"));
+        SubtitleCodec sCodec = SubtitleCodec.valueOf(session.getQueryParameters().get("sCodec"));
         VideoQuality quality = VideoQuality.valueOf(session.getQueryParameters().get("quality"));
 
         String videoUrl = container == ContainerFormat.HLS ? //
             String.format(
-                "/_internal/media/%s/stream/hls/media.m3u8?format=%s&videoCodec=%s&audioCodec=%s&quality=%s",
-                media.getId(), container, vCodec, aCodec, quality
+                "/_internal/media/%s/stream/hls/media.m3u8?format=%s&videoCodec=%s&audioCodec=%s&subtitleCodec=%s&quality=%s",
+                media.getId(), container, vCodec, aCodec, sCodec, quality
             )
             : String.format(
-                "/_internal/media/%s/stream?format=%s&videoCodec=%s&audioCodec=%s&quality=%s",
-                media.getId(), container, vCodec, aCodec, quality
+                "/_internal/media/%s/stream?format=%s&videoCodec=%s&audioCodec=%s&subtitleCodec=%s&quality=%s",
+                media.getId(), container, vCodec, aCodec, sCodec, quality
             );
 
         return new HTMLBuilder()
@@ -573,16 +617,17 @@ public HttpResponse onWatchSpecificMediaInVLCDeepLink(SoraHttpSession session) {
         ContainerFormat container = ContainerFormat.MKV;// ContainerFormat.valueOf(session.getQueryParameters().get("container"));
         VideoCodec vCodec = VideoCodec.valueOf(session.getQueryParameters().get("vCodec"));
         AudioCodec aCodec = AudioCodec.valueOf(session.getQueryParameters().get("aCodec"));
+        SubtitleCodec sCodec = SubtitleCodec.valueOf(session.getQueryParameters().get("sCodec"));
         VideoQuality quality = VideoQuality.valueOf(session.getQueryParameters().get("quality"));
 
         String videoUrl = container == ContainerFormat.HLS ? //
             String.format(
-                "/_internal/media/%s/stream/hls/media.m3u8?format=%s&videoCodec=%s&audioCodec=%s&quality=%s",
-                media.getId(), container, vCodec, aCodec, quality
+                "/_internal/media/%s/stream/hls/media.m3u8?format=%s&videoCodec=%s&audioCodec=%s&subtitleCodec=%s&quality=%s",
+                media.getId(), container, vCodec, aCodec, sCodec, quality
             )
             : String.format(
-                "/_internal/media/%s/stream?format=%s&videoCodec=%s&audioCodec=%s&quality=%s",
-                media.getId(), container, vCodec, aCodec, quality
+                "/_internal/media/%s/stream?format=%s&videoCodec=%s&audioCodec=%s&subtitleCodec=%s&quality=%s",
+                media.getId(), container, vCodec, aCodec, sCodec, quality
             );
 
         videoUrl = session.getHeader("Referer").substring(0, session.getHeader("Referer").indexOf("/media")) + videoUrl;

server/src/main/java/xyz/e3ndr/athena/transcoding/FFMpegArgs.java

@@ -8,6 +8,7 @@
 import xyz.e3ndr.athena.Athena;
 import xyz.e3ndr.athena.transcoding.accelerator.TranscodeAcceleration;
 import xyz.e3ndr.athena.types.AudioCodec;
+import xyz.e3ndr.athena.types.SubtitleCodec;
 import xyz.e3ndr.athena.types.VideoCodec;
 import xyz.e3ndr.athena.types.VideoQuality;
 
@@ -59,4 +60,10 @@ public class FFMpegArgs {
         return args;
     }
 
+    public static @NonNull List<String> s_getFF(SubtitleCodec desiredSCodec) {
+        return Arrays.asList(
+            "-c:s", desiredSCodec.ff
+        );
+    }
+
 }

server/src/main/java/xyz/e3ndr/athena/transcoding/Transcoder.java

@@ -27,9 +27,11 @@
 import xyz.e3ndr.athena.Athena;
 import xyz.e3ndr.athena.types.AudioCodec;
 import xyz.e3ndr.athena.types.ContainerFormat;
+import xyz.e3ndr.athena.types.SubtitleCodec;
 import xyz.e3ndr.athena.types.VideoCodec;
 import xyz.e3ndr.athena.types.VideoQuality;
 import xyz.e3ndr.athena.types.media.Media;
+import xyz.e3ndr.athena.types.media.MediaFiles.Streams.SubtitleStream;
 import xyz.e3ndr.fastloggingframework.logging.FastLogger;
 
 public class Transcoder {
@@ -71,7 +73,7 @@ public class Transcoder {
     }
 
     @SneakyThrows
-    public static @Nullable TranscodeSession start(File targetFile, Media media, VideoQuality desiredQuality, VideoCodec desiredVCodec, AudioCodec desiredACodec, ContainerFormat desiredContainer, int... streamIds) {
+    public static @Nullable TranscodeSession start(File targetFile, Media media, VideoQuality desiredQuality, VideoCodec desiredVCodec, AudioCodec desiredACodec, SubtitleCodec desiredSCodec, ContainerFormat desiredContainer, int... streamIds) {
         if (!Athena.config.transcoding.enable && (desiredACodec != AudioCodec.SOURCE || desiredVCodec != VideoCodec.SOURCE)) {
             logger.severe("Transcoding is disabled, but a session was requested.");
             return null;
@@ -92,13 +94,21 @@ public class Transcoder {
             command.add("-map", String.format("0:%d", streamId));
         }
 
+        // Include all subtitles that we support.
+        for (SubtitleStream subtitle : media.getFiles().getStreams().getSubtitles()) {
+            command.add("-map", String.format("0:%d", subtitle.getId()));
+        }
+
         /* ---- Audio ---- */
         command.add(FFMpegArgs.a_getFF(desiredACodec));
 
         if (desiredACodec != AudioCodec.SOURCE) {
             command.add("-ar", "48000");
         }
 
+        /* ---- Subtitles ---- */
+        command.add(FFMpegArgs.s_getFF(desiredSCodec));
+
         /* ---- Video ---- */
         command.add(FFMpegArgs.v_getFF(desiredVCodec, desiredQuality));
 
@@ -228,7 +238,7 @@ public class Transcoder {
     }
 
     @SneakyThrows
-    public static File getFile(Media media, VideoQuality desiredQuality, VideoCodec desiredVCodec, AudioCodec desiredACodec, ContainerFormat desiredContainer, int... streamIds) {
+    public static File getFile(Media media, VideoQuality desiredQuality, VideoCodec desiredVCodec, AudioCodec desiredACodec, SubtitleCodec desiredSCodec, ContainerFormat desiredContainer, int... streamIds) {
         List<String> str_streamIds = new ArrayList<>(streamIds.length);
         for (int streamId : streamIds) {
             str_streamIds.add(String.valueOf(streamId));
@@ -237,6 +247,7 @@ public static File getFile(Media media, VideoQuality desiredQuality, VideoCodec
         List<String> codecs = new ArrayList<>();
         codecs.add(desiredVCodec.name().toLowerCase());
         codecs.add(desiredACodec.name().toLowerCase());
+        codecs.add(desiredSCodec.name().toLowerCase());
 
         File mediaFile = new File(
             Athena.cacheDirectory,

server/src/main/java/xyz/e3ndr/athena/types/SubtitleCodec.java

@@ -0,0 +1,18 @@
+package xyz.e3ndr.athena.types;
+
+import lombok.AllArgsConstructor;
+
+@AllArgsConstructor
+public enum SubtitleCodec {
+    SOURCE("copy"),
+
+    // @formatter:off
+    WEBVTT ("webvtt"),
+    STR    ("srt"),
+    ASS    ("ass"),
+    // @formatter:on
+    ;
+
+    public final String ff;
+
+}

server/src/main/java/xyz/e3ndr/athena/types/media/Media.java

@@ -1,15 +1,8 @@
 package xyz.e3ndr.athena.types.media;
 
-import java.io.File;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-
-import co.casterlabs.commons.functional.tuples.Pair;
 import co.casterlabs.rakurai.json.annotating.JsonClass;
 import lombok.Getter;
 import lombok.NonNull;
-import xyz.e3ndr.athena.Athena;
 
 @Getter
 @NonNull
@@ -29,33 +22,4 @@ public String toString() {
         }
     }
 
-    /**
-     * a: The main subtitle file. (nullable)<br />
-     * b: Any forced subtitles.
-     */
-    public @NonNull Pair<File, List<File>> getSubtitle(String language) {
-        for (MediaFiles.Subtitle subtitle : this.files.getSubtitles()) {
-            if (!subtitle.getLanguage().equals(language)) continue;
-
-            File mainSubtitle = new File(
-                Athena.indexDirectory,
-                String.format("%s/subtitles/%s", this.id, subtitle.getFile())
-            );
-
-            List<File> forcedSubtitles = new LinkedList<>();
-            for (String forced : subtitle.getForced()) {
-                forcedSubtitles.add(
-                    new File(
-                        Athena.indexDirectory,
-                        String.format("%s/subtitles/%s", this.id, forced)
-                    )
-                );
-            }
-
-            return new Pair<>(mainSubtitle, forcedSubtitles);
-        }
-
-        return new Pair<>(null, Collections.emptyList());
-    }
-
 }