From 659f6e520123ae0aa8afbf88a2a883f5d8644c39 Mon Sep 17 00:00:00 2001 From: carlobortolan Date: Mon, 4 Nov 2024 17:25:37 +0100 Subject: [PATCH 1/7] Add transcript synchronization functions Signed-off-by: carlobortolan --- web/ts/data-store/data-store.ts | 2 + web/ts/data-store/transcript.ts | 8 ++ web/ts/entry/video.ts | 1 + web/ts/transcript.ts | 165 ++++++++++++++++++++++++++++++++ web/ts/watch.ts | 2 +- 5 files changed, 177 insertions(+), 1 deletion(-) create mode 100644 web/ts/data-store/transcript.ts create mode 100644 web/ts/transcript.ts diff --git a/web/ts/data-store/data-store.ts b/web/ts/data-store/data-store.ts index 39890a666..743ea67d1 100644 --- a/web/ts/data-store/data-store.ts +++ b/web/ts/data-store/data-store.ts @@ -1,10 +1,12 @@ import { VideoSectionProvider } from "./video-sections"; import { BookmarksProvider } from "./bookmarks"; +import { TranscriptProvider } from "./transcript"; import { StreamPlaylistProvider } from "./stream-playlist"; import { AdminLectureListProvider } from "./admin-lecture-list"; export abstract class DataStore { static bookmarks: BookmarksProvider = new BookmarksProvider(); + static transcript: TranscriptProvider = new TranscriptProvider(); static videoSections: VideoSectionProvider = new VideoSectionProvider(); static streamPlaylist: StreamPlaylistProvider = new StreamPlaylistProvider(); diff --git a/web/ts/data-store/transcript.ts b/web/ts/data-store/transcript.ts new file mode 100644 index 000000000..45a686081 --- /dev/null +++ b/web/ts/data-store/transcript.ts @@ -0,0 +1,8 @@ +import { StreamableMapProvider } from "./provider"; + +export class TranscriptProvider extends StreamableMapProvider { + protected async fetcher(): Promise { + return []; + } +} + diff --git a/web/ts/entry/video.ts b/web/ts/entry/video.ts index 094fdf963..7356c475b 100644 --- a/web/ts/entry/video.ts +++ b/web/ts/entry/video.ts @@ -4,6 +4,7 @@ export * from "../TUMLiveVjs"; export * from "../watch"; export * from "../splitview"; export * from "../bookmarks"; +export * from "../transcript"; export * from "../subtitle-search"; export * from "../components/video-sections"; // Lecture Units are currently not used, so we don't include them in the bundle at the moment diff --git a/web/ts/transcript.ts b/web/ts/transcript.ts new file mode 100644 index 000000000..b22e59613 --- /dev/null +++ b/web/ts/transcript.ts @@ -0,0 +1,165 @@ +import { getPlayers } from "./TUMLiveVjs"; +import { VideoJsPlayer } from "video.js"; + +export class TranscriptController { + static initiatedInstances: Map> = new Map< + string, + Promise + >(); + + private list: VTTCue[]; + private elem: HTMLElement; + private lastSyncTime: number; + private player: VideoJsPlayer; + private selectedTrackLabel: string; + + reset(): void { + this.player = getPlayers()[0]; + } + + constructor() { + this.lastSyncTime = 0; + this.selectedTrackLabel = "English"; + } + + async init(key: string, element: HTMLElement) { + if (TranscriptController.initiatedInstances[key]) { + (await TranscriptController.initiatedInstances[key]).unsub(); + } + TranscriptController.initiatedInstances[key] = new Promise(() => { + this.elem = element; + }); + + this.player = getPlayers()[0]; + window.setInterval(() => this.syncTranscript(), 1000); + } + + async syncTranscript() { + const transcriptDesktop = document.getElementById('transcript-desktop'); + if (!this.elem || !transcriptDesktop || transcriptDesktop.offsetParent === null) { + return; + } + + const now = Date.now(); + if (now - this.lastSyncTime < 1000) { + return; + } + this.lastSyncTime = now; + + if (this.player.paused()) { + return; + } + + console.debug("Syncing transcript..."); + const currentTime = this.player.currentTime(); + const transcript = await this.fetchTranscript(this.player); + this.updateTranscript(transcript); + this.highlightActiveCue(currentTime); + } + + async fetchTranscript(player: VideoJsPlayer): Promise { + const textTracks = player.textTracks(); + let transcript: VTTCue[] = []; + + // Try to find the selected track first + for (let i = 0; i < textTracks.length; i++) { + const track = textTracks[i]; + if ((track.kind === "captions" || track.kind === "subtitles") && track.label === this.selectedTrackLabel) { + for (let j = 0; j < track.cues.length; j++) { + const cue = track.cues[j] as VTTCue; + transcript.push(cue); + } + if (transcript.length > 0) { + return transcript; + } + } + } + + // If no selected track is found, use other available subtitles + for (let i = 0; i < textTracks.length; i++) { + const track = textTracks[i]; + if (track.kind === "captions" || track.kind === "subtitles") { + for (let j = 0; j < track.cues.length; j++) { + const cue = track.cues[j] as VTTCue; + transcript.push(cue); + } + } + } + + return transcript; + } + + updateTranscript(transcript: VTTCue[]) { + this.list = transcript; + const event = new CustomEvent('update', { detail: transcript }); + this.elem.dispatchEvent(event); + } + + highlightActiveCue(currentTime: number) { + const activeCue = this.list.find(cue => cue.startTime <= currentTime && cue.endTime >= currentTime); + const cueElements = this.elem.querySelectorAll('[data-cue-start]'); + cueElements.forEach((cueElement: HTMLElement) => { + cueElement.classList.remove('bg-blue-100', 'dark:bg-blue-700'); + }); + + if (activeCue) { + const cueElement = this.elem.querySelector(`[data-cue-start="${activeCue.startTime}"]`); + if (cueElement) { + cueElement.classList.add('bg-blue-100', 'dark:bg-blue-700'); + cueElement.scrollIntoView({ behavior: 'smooth', block: 'center' }); + } + } + } + + onUpdate(data: any) { + // Process the data and update the transcript list + this.updateTranscript(data); + } + + length(): number { + return this.list !== undefined ? this.list.length : 0; + } + + async downloadTranscript() { + const player = getPlayers()[0]; + const textTracks = player.textTracks(); + let transcript = ""; + + // Iterate over the text tracks to find the selected track + for (let i = 0; i < textTracks.length; i++) { + const track = textTracks[i]; + if ((track.kind === "captions" || track.kind === "subtitles") && track.label === this.selectedTrackLabel) { + // Iterate over the cues to extract the transcript text + for (let j = 0; j < track.cues.length; j++) { + const cue = track.cues[j]; + transcript += `${cue.text}\n\n`; + } + } + } + + // If no selected track is found, use other available subtitles + if (transcript === "") { + for (let i = 0; i < textTracks.length; i++) { + const track = textTracks[i]; + if (track.kind === "captions" || track.kind === "subtitles") { + // Iterate over the cues to extract the transcript text + for (let j = 0; j < track.cues.length; j++) { + const cue = track.cues[j]; + transcript += `${cue.text}\n\n`; + } + } + } + } + + // Create a Blob from the transcript text + const blob = new Blob([transcript], { type: "text/plain" }); + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = "transcript"; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + URL.revokeObjectURL(url); + } +} \ No newline at end of file diff --git a/web/ts/watch.ts b/web/ts/watch.ts index beafa3f93..24bffe2ff 100644 --- a/web/ts/watch.ts +++ b/web/ts/watch.ts @@ -1,12 +1,12 @@ import { getPlayers } from "./TUMLiveVjs"; import { copyToClipboard, Time } from "./global"; -import { seekbarOverlay } from "./seekbar-overlay"; export enum SidebarState { Hidden = "hidden", Chat = "chat", Bookmarks = "bookmarks", Streams = "streams", + Transcript = "transcript", } /* From c64116501d868d7ad25e26722a4c9dd9f7e2b5ea Mon Sep 17 00:00:00 2001 From: carlobortolan Date: Mon, 4 Nov 2024 17:26:46 +0100 Subject: [PATCH 2/7] Add transcript component to watch.gohtml Signed-off-by: carlobortolan --- web/template/partial/stream/transcript.gohtml | 48 +++++++++++++++++++ web/template/watch.gohtml | 13 +++++ 2 files changed, 61 insertions(+) create mode 100644 web/template/partial/stream/transcript.gohtml diff --git a/web/template/partial/stream/transcript.gohtml b/web/template/partial/stream/transcript.gohtml new file mode 100644 index 000000000..afa981dfe --- /dev/null +++ b/web/template/partial/stream/transcript.gohtml @@ -0,0 +1,48 @@ +{{define "transcript-modal"}} +
+
+
+
+

Transcript

+ beta +
+ +
+ {{template "transcript-list" .}} +
+ +
+
+
+{{end}} + +{{define "transcript-list"}} +
+
+ +
+
+{{end}} \ No newline at end of file diff --git a/web/template/watch.gohtml b/web/template/watch.gohtml index 07ed1960c..8928de106 100644 --- a/web/template/watch.gohtml +++ b/web/template/watch.gohtml @@ -424,6 +424,13 @@ {{end}} + + + {{if .IndexData.TUMLiveContext.User}}
{{end}} + +
- {{template "transcript-list" .}} -
+
+ {{template "transcript-list" .}} +
+
{{end}} @@ -30,8 +32,8 @@ x-data="{ transcriptController: new watch.TranscriptController(), transcript: [] }" x-init="() => { transcriptController.init('transcript-list', $el);}" @update="(e) => (transcript = e.detail)" - class="h-[85%] w-full max-w-3xl mx-auto transcript-container"> -
+ class="h-full w-full max-w-3xl mx-auto transcript-container overflow-hidden"> +