From c979a1a7211c377895a9587ac961cf27fd6105c0 Mon Sep 17 00:00:00 2001 From: Rafael Araujo Lehmkuhl Date: Tue, 19 Dec 2023 19:27:29 -0300 Subject: [PATCH 01/10] sensors-logging: Remove leftover console logs --- src/libs/sensors-logging.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/libs/sensors-logging.ts b/src/libs/sensors-logging.ts index 16a20b964..79959f13d 100644 --- a/src/libs/sensors-logging.ts +++ b/src/libs/sensors-logging.ts @@ -231,9 +231,6 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text` } }) - console.log(videoWidth, videoHeight) - console.log(0.1*videoWidth, 0.05*videoHeight) - assFile = assFile.concat(`\nDialogue: 0,0:0:${secondsStart}.00,0:0:${secondsFinish}.00,Default,,${0.1*videoWidth},0,${0.05*videoHeight},,${subtitleDataString1}`) assFile = assFile.concat(`\nDialogue: 0,0:0:${secondsStart}.00,0:0:${secondsFinish}.00,Default,,${0.4*videoWidth},0,${0.05*videoHeight},,${subtitleDataString2}`) assFile = assFile.concat(`\nDialogue: 0,0:0:${secondsStart}.00,0:0:${secondsFinish}.00,Default,,${0.7*videoWidth},0,${0.05*videoHeight},,${subtitleDataString3}`) From 5f52be7a69724e4b117e48b7c876357c8365efee Mon Sep 17 00:00:00 2001 From: Rafael Araujo Lehmkuhl Date: Tue, 19 Dec 2023 19:33:36 -0300 Subject: [PATCH 02/10] types: Add video-related types --- src/types/video.ts | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 src/types/video.ts diff --git a/src/types/video.ts b/src/types/video.ts new file mode 100644 index 000000000..ea472fb64 --- /dev/null +++ b/src/types/video.ts @@ -0,0 +1,22 @@ +import type { Ref } from 'vue' + +import { WebRTCManager } from '@/composables/webRTC' +import type { Stream } from '@/libs/webrtc/signalling_protocol' + +/** + * Everything needed for every stream + */ +export interface StreamData { + /** + * The actual WebRTC stream + */ + stream: Ref + /** + * The responsible for its management + */ + webRtcManager: WebRTCManager + /** + * MediaStream object, if WebRTC stream is chosen + */ + mediaStream: Ref +} From 43bb79ce8a463e810e88450830fecd3245b66e43 Mon Sep 17 00:00:00 2001 From: Rafael Araujo Lehmkuhl Date: Thu, 21 Dec 2023 15:46:40 -0300 Subject: [PATCH 03/10] WebRTC: Make list of availableICEIPs public --- src/composables/webRTC.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/composables/webRTC.ts b/src/composables/webRTC.ts index 9c21d04d1..70530ae12 100644 --- a/src/composables/webRTC.ts +++ b/src/composables/webRTC.ts @@ -38,7 +38,7 @@ interface startStreamReturn { */ export class WebRTCManager { private availableStreams: Ref> = ref(new Array()) - private availableICEIPs: Ref> = ref(new Array()) + public availableICEIPs: Ref> = ref(new Array()) private mediaStream: Ref = ref() private signallerStatus: Ref = ref('waiting...') private streamStatus: Ref = ref('waiting...') From 67857e8f60bd17e609aa07f0f3d5ddf5d7ba150e Mon Sep 17 00:00:00 2001 From: Rafael Araujo Lehmkuhl Date: Thu, 21 Dec 2023 15:47:15 -0300 Subject: [PATCH 04/10] webrtc-signaller: Catch errors on websocket instantiation --- src/libs/webrtc/signaller.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/libs/webrtc/signaller.ts b/src/libs/webrtc/signaller.ts index 3ceb40cf0..4858b5c9a 100644 --- a/src/libs/webrtc/signaller.ts +++ b/src/libs/webrtc/signaller.ts @@ -42,7 +42,11 @@ export class Signaller { console.debug('[WebRTC] [Signaller] ' + status) this.onStatusChange?.(status) - this.ws = this.connect() + try { + this.ws = this.connect() + } catch (error) { + console.error(`Could not establish initial connection. ${error}`) + } } /** @@ -609,7 +613,11 @@ export class Signaller { oldWs.onmessage = null oldWs.onerror = null - this.ws = this.connect() + try { + this.ws = this.connect() + } catch (error) { + console.error(`[WebRTC] [Signaller] Could not reconnect. ${error}`) + } } /** From e883383f5766cc085f3cb8d2667bf987f681d751 Mon Sep 17 00:00:00 2001 From: Rafael Araujo Lehmkuhl Date: Thu, 21 Dec 2023 15:48:27 -0300 Subject: [PATCH 05/10] video-store: Refactor streams pipeline so multiple consumers use the same stream source --- src/stores/video.ts | 85 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 82 insertions(+), 3 deletions(-) diff --git a/src/stores/video.ts b/src/stores/video.ts index 5e2773846..1c8c5cdf2 100644 --- a/src/stores/video.ts +++ b/src/stores/video.ts @@ -3,11 +3,84 @@ import { saveAs } from 'file-saver' import localforage from 'localforage' import { defineStore } from 'pinia' import Swal from 'sweetalert2' -import { ref } from 'vue' +import { computed, ref, watch } from 'vue' +import adapter from 'webrtc-adapter' + +import { WebRTCManager } from '@/composables/webRTC' +import { isEqual } from '@/libs/utils' +import { useMainVehicleStore } from '@/stores/mainVehicle' +import type { StreamData } from '@/types/video' export const useVideoStore = defineStore('video', () => { - const availableIceIps = ref(undefined) + const { rtcConfiguration, webRTCSignallingURI } = useMainVehicleStore() + console.debug('[WebRTC] Using webrtc-adapter for', adapter.browserDetails) + const allowedIceIps = useStorage('cockpit-allowed-stream-ips', []) + const activeStreams = ref<{ [key in string]: StreamData | undefined }>({}) + const mainWebRTCManager = new WebRTCManager(webRTCSignallingURI.val, rtcConfiguration) + const { availableStreams } = mainWebRTCManager.startStream(ref(undefined), allowedIceIps) + const availableIceIps = ref([]) + + const namesAvailableStreams = computed(() => availableStreams.value.map((stream) => stream.name)) + + // If the allowed ICE IPs are updated, all the streams should be reconnected + watch(allowedIceIps, () => { + Object.keys(activeStreams.value).forEach((streamName) => (activeStreams.value[streamName] = undefined)) + }) + + // Streams update routine. Responsible for starting and updating the streams. + setInterval(() => { + Object.keys(activeStreams.value).forEach((streamName) => { + if (activeStreams.value[streamName] === undefined) return + // Update the list of available remote ICE Ips with those available for each stream + // @ts-ignore: availableICEIPs is not reactive here, for some yet to know reason + const newIps = activeStreams.value[streamName].webRtcManager.availableICEIPs.filter( + (ip: string) => !availableIceIps.value.includes(ip) + ) + availableIceIps.value = [...availableIceIps.value, ...newIps] + + const updatedStream = availableStreams.value.find((s) => s.name === streamName) + if (isEqual(updatedStream, activeStreams.value[streamName]!.stream)) return + + // Whenever the stream is to be updated we first reset it's variables (activateStream method), so + // consumers can be updated as well. + console.log(`New stream for '${streamName}':`) + console.log(JSON.stringify(updatedStream, null, 2)) + activateStream(streamName) + // @ts-ignore + activeStreams.value[streamName].stream = updatedStream + }) + }, 300) + + /** + * Activates a stream by starting it and storing it's variables inside a common object. + * This way multiple consumers will always access the same resource, so we don't consume unnecessary + * bandwith or stress the stream provider more than we need to. + * @param {string} streamName - Unique name for the stream, common between the multiple consumers + */ + const activateStream = (streamName: string): void => { + const stream = ref() + const webRtcManager = new WebRTCManager(webRTCSignallingURI.val, rtcConfiguration) + const { mediaStream } = webRtcManager.startStream(stream, allowedIceIps) + activeStreams.value[streamName] = { + stream: stream, + webRtcManager: webRtcManager, + mediaStream: mediaStream, + } + } + + /** + * bandwith or stress the stream provider more than we need to. + * @param {string} streamName - Name of the stream + * @returns {MediaStream | undefined} MediaStream that is running, if available + */ + const getMediaStream = (streamName: string): MediaStream | undefined => { + if (activeStreams.value[streamName] === undefined) { + activateStream(streamName) + } + // @ts-ignore + return activeStreams.value[streamName].mediaStream + } // Offer download of backuped videos const videoRecoveryDB = localforage.createInstance({ @@ -76,5 +149,11 @@ export const useVideoStore = defineStore('video', () => { } }, 5000) - return { availableIceIps, allowedIceIps, videoRecoveryDB } + return { + availableIceIps, + allowedIceIps, + namesAvailableStreams, + videoRecoveryDB, + getMediaStream, + } }) From d7be83108f87fdb8646010ac920eb0696721544d Mon Sep 17 00:00:00 2001 From: Rafael Araujo Lehmkuhl Date: Thu, 21 Dec 2023 15:49:18 -0300 Subject: [PATCH 06/10] video-configuration: Create video configuration page It initially has the configuration of the allowed ICE IPs. --- src/components/ConfigurationMenu.vue | 6 ++++ src/views/ConfigurationVideoView.vue | 46 ++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 src/views/ConfigurationVideoView.vue diff --git a/src/components/ConfigurationMenu.vue b/src/components/ConfigurationMenu.vue index ab00c883e..af89ed40c 100644 --- a/src/components/ConfigurationMenu.vue +++ b/src/components/ConfigurationMenu.vue @@ -42,6 +42,7 @@ import ConfigurationDevelopmentView from '../views/ConfigurationDevelopmentView. import ConfigurationGeneralView from '../views/ConfigurationGeneralView.vue' import ConfigurationJoystickView from '../views/ConfigurationJoystickView.vue' import ConfigurationLogsView from '../views/ConfigurationLogsView.vue' +import ConfigurationVideoView from '../views/ConfigurationVideoView.vue' const store = useMainVehicleStore() @@ -59,6 +60,11 @@ const menus = [ title: 'Joystick', component: ConfigurationJoystickView, }, + { + icon: 'mdi-video', + title: 'Video', + component: ConfigurationVideoView, + }, { icon: 'mdi-script', title: 'Logs', diff --git a/src/views/ConfigurationVideoView.vue b/src/views/ConfigurationVideoView.vue new file mode 100644 index 000000000..8130378d4 --- /dev/null +++ b/src/views/ConfigurationVideoView.vue @@ -0,0 +1,46 @@ + + + From b3011e8fdd39b08bf3599ce021037409ea64a838 Mon Sep 17 00:00:00 2001 From: Rafael Araujo Lehmkuhl Date: Thu, 21 Dec 2023 15:53:03 -0300 Subject: [PATCH 07/10] mini-video-recorder: Remove support for screen recording We don't have Secure Context right now, so it does not have value. --- .../mini-widgets/MiniVideoRecorder.vue | 48 +------------------ 1 file changed, 1 insertion(+), 47 deletions(-) diff --git a/src/components/mini-widgets/MiniVideoRecorder.vue b/src/components/mini-widgets/MiniVideoRecorder.vue index 4fa0399ce..2a8e00a16 100644 --- a/src/components/mini-widgets/MiniVideoRecorder.vue +++ b/src/components/mini-widgets/MiniVideoRecorder.vue @@ -109,7 +109,6 @@ onBeforeMount(async () => { streamName: undefined as string | undefined, } } - addScreenStream() }) const toggleRecording = async (): Promise => { @@ -120,21 +119,6 @@ const toggleRecording = async (): Promise => { // Open dialog so user can choose the stream which will be recorded isStreamSelectDialogOpen.value = true } - -const addScreenStream = (): void => { - const screenStream = { - id: 'screenStream', - name: 'Entire screen', - encode: null, - height: null, - width: null, - interval: null, - source: null, - created: null, - } - availableStreams.value.push(screenStream) -} - onBeforeUnmount(() => { webRTCManager.close('WebRTC manager removed') }) @@ -150,32 +134,6 @@ const startRecording = async (): Promise => { return Swal.fire({ text: 'No stream selected. Please choose one before continuing.', icon: 'error' }) } } - if (selectedStream.value?.id === 'screenStream') { - try { - // @ts-ignore: camera permission check is currently available in most browsers, including chromium-based ones - const displayPermission = await navigator.permissions.query({ name: 'display-capture' }) - if (displayPermission.state === 'denied') { - const noPermissionHtml = ` -

Your browser is currently blocking screen recording.

-

We are working to solve this automatically for you.

-

By the meantime, please follow the instructions.

-
- -
  • Copy Cockpit's URL (usually "http://blueos.local:49153").
    • -
  • Open the following URL: "chrome://flags/#unsafely-treat-insecure-origin-as-secure".
    • -
  • Add Cockpit's URL to the "Insecure origins treated as secure" list.
    • -
  • Select "Enabled" on the side menu.
    • -
  • Restart your browser.
    • -     - ` - return Swal.fire({ html: noPermissionHtml, icon: 'error' }) - } - // @ts-ignore: preferCurrentTab option is currently available in most browsers, including chromium-based ones - mediaStream.value = await navigator.mediaDevices.getDisplayMedia({ preferCurrentTab: true }) - } catch (err) { - return Swal.fire({ text: 'Could not get stream from user screen.', icon: 'error' }) - } - } if (mediaStream.value === undefined) { return Swal.fire({ text: 'Media stream not defined.', icon: 'error' }) } @@ -211,10 +169,6 @@ const startRecording = async (): Promise => { }) chunks = [] mediaRecorder.value = undefined - if (selectedStream.value?.id === 'screenStream' && mediaStream.value !== undefined) { - // If recording the screen stream, stop the tracks also, so the browser removes the recording warning. - mediaStream.value.getTracks().forEach((track: MediaStreamTrack) => track.stop()) - } } } @@ -233,7 +187,7 @@ const timePassedString = computed(() => { const updateCurrentStream = async (stream: Stream | undefined): Promise => { selectedStream.value = stream mediaStream.value = undefined - if (selectedStream.value !== undefined && selectedStream.value.id !== 'screenStream') { + if (selectedStream.value !== undefined) { isLoadingStream.value = true let millisPassed = 0 const timeStep = 100 From 717b95a9b19c13247b6f00d383b148cfd5e09dd5 Mon Sep 17 00:00:00 2001 From: Rafael Araujo Lehmkuhl Date: Thu, 21 Dec 2023 15:54:34 -0300 Subject: [PATCH 08/10] mini-video-recorder: Adapt widget to new video stream pipeline --- .../mini-widgets/MiniVideoRecorder.vue | 72 ++++++++----------- 1 file changed, 31 insertions(+), 41 deletions(-) diff --git a/src/components/mini-widgets/MiniVideoRecorder.vue b/src/components/mini-widgets/MiniVideoRecorder.vue index 2a8e00a16..e5f2419b7 100644 --- a/src/components/mini-widgets/MiniVideoRecorder.vue +++ b/src/components/mini-widgets/MiniVideoRecorder.vue @@ -20,9 +20,9 @@

    Choose a stream to record

    () const miniWidget = toRefs(props).miniWidget -const selectedStream = ref() -const webRTCManager = new WebRTCManager(webRTCSignallingURI.val, rtcConfiguration) -const { availableStreams: externalStreams, mediaStream } = webRTCManager.startStream(selectedStream, allowedIceIps) +const nameSelectedStream = ref() +const { namesAvailableStreams } = storeToRefs(videoStore) const mediaRecorder = ref() const recorderWidget = ref() const { isOutside } = useMouseInElement(recorderWidget) -const availableStreams = ref([]) const isStreamSelectDialogOpen = ref(false) const isLoadingStream = ref(false) const timeRecordingStart = ref(new Date()) const timeNow = useTimestamp({ interval: 100 }) +const mediaStream = ref() const isRecording = computed(() => { return mediaRecorder.value !== undefined && mediaRecorder.value.state === 'recording' @@ -109,6 +100,12 @@ onBeforeMount(async () => { streamName: undefined as string | undefined, } } + nameSelectedStream.value = miniWidget.value.options.streamName +}) + +watch(nameSelectedStream, () => { + miniWidget.value.options.streamName = nameSelectedStream.value + mediaStream.value = undefined }) const toggleRecording = async (): Promise => { @@ -119,17 +116,14 @@ const toggleRecording = async (): Promise => { // Open dialog so user can choose the stream which will be recorded isStreamSelectDialogOpen.value = true } -onBeforeUnmount(() => { - webRTCManager.close('WebRTC manager removed') -}) const startRecording = async (): Promise => { - if (availableStreams.value.isEmpty()) { + if (namesAvailableStreams.value.isEmpty()) { return Swal.fire({ text: 'No streams available.', icon: 'error' }) } - if (selectedStream.value === undefined) { - if (availableStreams.value.length === 1) { - await updateCurrentStream(availableStreams.value[0]) + if (nameSelectedStream.value === undefined) { + if (namesAvailableStreams.value.length === 1) { + await updateCurrentStream(namesAvailableStreams.value[0]) } else { return Swal.fire({ text: 'No stream selected. Please choose one before continuing.', icon: 'error' }) } @@ -184,10 +178,10 @@ const timePassedString = computed(() => { return `${durationHours}:${durationMinutes}:${durationSeconds}` }) -const updateCurrentStream = async (stream: Stream | undefined): Promise => { - selectedStream.value = stream +const updateCurrentStream = async (streamName: string | undefined): Promise => { + nameSelectedStream.value = streamName mediaStream.value = undefined - if (selectedStream.value !== undefined) { + if (nameSelectedStream.value !== undefined) { isLoadingStream.value = true let millisPassed = 0 const timeStep = 100 @@ -202,27 +196,23 @@ const updateCurrentStream = async (stream: Stream | undefined): Promise { - const savedStreamName: string | undefined = miniWidget.value.options.streamName as string - availableStreams.value = externalStreams.value - if (!availableStreams.value.find((stream) => stream.id === 'screenStream')) { - addScreenStream() - } - if (availableStreams.value.isEmpty()) { - return +const streamConnectionRoutine = setInterval(() => { + // If the video player widget is cold booted, assign the first stream to it + if (miniWidget.value.options.streamName === undefined && !namesAvailableStreams.value.isEmpty()) { + miniWidget.value.options.streamName = namesAvailableStreams.value[0] + nameSelectedStream.value = miniWidget.value.options.streamName } - // Retrieve stream from the saved stream name, otherwise choose the first available stream as a fallback - const savedStream = savedStreamName ? availableStreams.value.find((s) => s.name === savedStreamName) : undefined - - if (savedStream !== undefined && savedStream.id !== selectedStream.value?.id && selectedStream.value === undefined) { - console.debug!('[WebRTC] trying to set stream...') - updateCurrentStream(savedStream) + const updatedMediaStream = videoStore.getMediaStream(miniWidget.value.options.streamName) + // If the widget is not connected to the MediaStream, try to connect it + if (!isEqual(updatedMediaStream, mediaStream.value)) { + mediaStream.value = updatedMediaStream } -}) +}, 1000) +onBeforeUnmount(() => clearInterval(streamConnectionRoutine)) // Try to prevent user from closing Cockpit when a stream is being recorded watch(isRecording, () => { From 5a8ff7060073b634dd9297f158f08e96515bda03 Mon Sep 17 00:00:00 2001 From: Rafael Araujo Lehmkuhl Date: Thu, 21 Dec 2023 15:54:42 -0300 Subject: [PATCH 09/10] video-player: Adapt widget to new video stream pipeline --- src/components/widgets/VideoPlayer.vue | 115 +++++++------------------ 1 file changed, 32 insertions(+), 83 deletions(-) diff --git a/src/components/widgets/VideoPlayer.vue b/src/components/widgets/VideoPlayer.vue index e7861be58..9ff928db4 100644 --- a/src/components/widgets/VideoPlayer.vue +++ b/src/components/widgets/VideoPlayer.vue @@ -1,6 +1,6 @@