google-home: support streaming to the android app!

koush · Feb 29, 2024 · 625c1d4 · 625c1d4
1 parent 88604bc
commit 625c1d4
Show file tree

Hide file tree

Showing 6 changed files with 144 additions and 22 deletions.
diff --git a/plugins/google-home/package-lock.json b/plugins/google-home/package-lock.json
diff --git a/plugins/google-home/package.json b/plugins/google-home/package.json
@@ -49,5 +49,5 @@
       "@types/lodash": "^4.14.168",
       "@types/url-parse": "^1.4.3"
    },
-   "version": "0.0.57"
+   "version": "0.0.58"
 }
diff --git a/plugins/google-home/src/commands/camerastream.ts b/plugins/google-home/src/commands/camerastream.ts
@@ -1,9 +1,10 @@
-import { RTCSignalingChannel, ScryptedDevice, ScryptedMimeTypes, VideoCamera } from "@scrypted/sdk";
+import { RTCAVSignalingSetup, RTCSignalingChannel, RTCSignalingOptions, RTCSignalingSendIceCandidate, RTCSignalingSession, ScryptedDevice, ScryptedMimeTypes, VideoCamera } from "@scrypted/sdk";
 import { executeResponse } from "../common";
 import { commandHandlers } from "../handlers";
+import { Deferred } from '@scrypted/common/src/deferred';
 
 import sdk from "@scrypted/sdk";
-const {mediaManager, endpointManager, systemManager } = sdk;
+const { mediaManager, endpointManager, systemManager } = sdk;
 
 const tokens: { [token: string]: string } = {};
 
@@ -12,21 +13,100 @@ export function canAccess(token: string) {
     return systemManager.getDeviceById(id) as ScryptedDevice & RTCSignalingChannel;
 }
 
-commandHandlers['action.devices.commands.GetCameraStream'] = async (device: ScryptedDevice, execution) => {
+commandHandlers['action.devices.commands.GetCameraStream'] = async (device: ScryptedDevice & RTCSignalingChannel, execution) => {
     const ret = executeResponse(device);
 
-    const engineio = await endpointManager.getPublicLocalEndpoint() + 'engine.io/';
-    const mo = await mediaManager.createMediaObject(Buffer.from(engineio), ScryptedMimeTypes.LocalUrl);
-    const cameraStreamAccessUrl = await mediaManager.convertMediaObjectToUrl(mo, ScryptedMimeTypes.LocalUrl);
-
     const cameraStreamAuthToken = `tok-${Math.round(Math.random() * 10000).toString(16)}`;
     tokens[cameraStreamAuthToken] = device.id;
 
-    ret.states = {
-        cameraStreamAccessUrl,
-        // cameraStreamReceiverAppId: "9E3714BD",
-        cameraStreamReceiverAppId: "00F7C5DD",
-        cameraStreamAuthToken,
+    if (execution.params.SupportedStreamProtocols.length === 1 && execution.params.SupportedStreamProtocols.includes('webrtc')) {
+        const endpoint = await endpointManager.getPublicLocalEndpoint() + `signaling/`;
+        const mo = await mediaManager.createMediaObject(Buffer.from(endpoint), ScryptedMimeTypes.LocalUrl);
+        const cameraStreamSignalingUrl = await mediaManager.convertMediaObjectToUrl(mo, ScryptedMimeTypes.LocalUrl);
+
+        tokens[cameraStreamAuthToken] = device.id;
+
+        ret.states = {
+            // cameraStreamOffer,
+            cameraStreamSignalingUrl,
+            cameraStreamAuthToken,
+            cameraStreamProtocol: 'webrtc',
+            // cameraStreamIceServers: JSON.stringify(cameraStreamIceServers),
+        }
     }
+    else {
+        const engineio = await endpointManager.getPublicLocalEndpoint() + 'engine.io/';
+        const mo = await mediaManager.createMediaObject(Buffer.from(engineio), ScryptedMimeTypes.LocalUrl);
+        const cameraStreamAccessUrl = await mediaManager.convertMediaObjectToUrl(mo, ScryptedMimeTypes.LocalUrl);
+
+        ret.states = {
+            cameraStreamAccessUrl,
+            // cameraStreamReceiverAppId: "9E3714BD",
+            cameraStreamReceiverAppId: "00F7C5DD",
+            cameraStreamAuthToken,
+        }
+    }
+
     return ret;
 }
+
+class Session implements RTCSignalingSession {
+    __proxy_props: { options: RTCSignalingOptions; };
+    options: RTCSignalingOptions;
+    deferred: Deferred<{
+        description: RTCSessionDescriptionInit, setup: RTCAVSignalingSetup,
+    }>;
+    offer: RTCSessionDescriptionInit;
+
+    async createLocalDescription(type: "offer" | "answer", setup: RTCAVSignalingSetup, sendIceCandidate: RTCSignalingSendIceCandidate): Promise<RTCSessionDescriptionInit> {
+        return this.offer;
+    }
+
+    async setRemoteDescription(description: RTCSessionDescriptionInit, setup: RTCAVSignalingSetup): Promise<void> {
+        this.deferred.resolve({description, setup});
+    }
+
+    addIceCandidate(candidate: RTCIceCandidateInit): Promise<void> {
+        throw new Error("Method not implemented.");
+    }
+
+    async getOptions(): Promise<RTCSignalingOptions> {
+        return this.options;
+    }
+};
+
+export async function signalCamera(device: ScryptedDevice & RTCSignalingChannel, body: any) {
+    if (body.action === 'offer') {
+        const offer: RTCSessionDescriptionInit = {
+            sdp: body.sdp,
+            type: 'offer',
+        };
+
+        const options: RTCSignalingOptions = {
+            requiresOffer: true,
+            disableTrickle: true,
+        }
+
+        const session = new Session();
+        session.__proxy_props = { options };
+        session.options = options;
+        session.deferred = new Deferred<{
+            description: RTCSessionDescriptionInit, setup: RTCAVSignalingSetup,
+        }>();
+        session.offer = offer;
+
+        device.startRTCSignalingSession(session);
+
+        const answer = await session.deferred.promise;
+
+        const sdp = answer.description.sdp.replace('sendrecv', 'sendonly');
+
+        return {
+            action: 'answer',
+            sdp,
+        }
+    }
+    else {
+        return {};
+    }
+}
diff --git a/plugins/google-home/src/main.ts b/plugins/google-home/src/main.ts
@@ -10,7 +10,7 @@ import { supportedTypes } from './common';
 import './types';
 import { StorageSettings } from '@scrypted/sdk/storage-settings';
 
-import { canAccess } from './commands/camerastream';
+import { canAccess, signalCamera } from './commands/camerastream';
 import { commandHandlers } from './handlers';
 
 import { homegraph } from '@googleapis/homegraph';
@@ -94,6 +94,12 @@ class GoogleHome extends ScryptedDeviceBase implements HttpRequestHandler, Engin
     constructor() {
         super();
 
+        endpointManager.setAccessControlAllowOrigin({
+            origins: [
+                'https://www.gstatic.com',
+            ],
+        });
+
         if (this.jwt) {
             this.googleAuthClient = googleAuth.fromJSON(this.jwt);
         }
@@ -532,6 +538,38 @@ class GoogleHome extends ScryptedDeviceBase implements HttpRequestHandler, Engin
             });
             return;
         }
+        this.console.log(request.body);
+
+        if (request.url.startsWith('/endpoint/@scrypted/google-home/public/signaling/')) {
+            if (request.method === 'OPTIONS') {
+                response.send('', {
+                    headers: {
+                        'Access-Control-Allow-Origin': request.headers.origin,
+                        'Access-Control-Allow-Methods': 'POST, OPTIONS, GET',
+                        'Access-Control-Allow-Headers': request.headers['access-control-request-headers'],
+                    },
+                    code: 200,
+                });
+                return;
+            }
+
+            const token = request.headers['authorization'].split('Bearer ')[1];
+            const camera = canAccess(token);
+            if (!camera) {
+                this.console.error(`request failed due to invalid authorization`);
+                response.send('Invalid Token', {
+                    code: 500,
+                });
+            }
+
+            const answer = await signalCamera(camera, JSON.parse(request.body));
+            response.send(JSON.stringify(answer), {
+                headers: {
+                    'Content-Type': 'application/json',
+                }
+            });
+            return;
+        }
 
         const { authorization } = request.headers;
         if (authorization !== this.localAuthorization) {
@@ -565,7 +603,6 @@ class GoogleHome extends ScryptedDeviceBase implements HttpRequestHandler, Engin
             }
         }
 
-        this.console.log(request.body);
         const body = JSON.parse(request.body);
         try {
             let result: any;

diff --git a/plugins/google-home/src/types/camera.ts b/plugins/google-home/src/types/camera.ts
@@ -12,11 +12,11 @@ addSupportedType({
         ret.attributes = {
             cameraStreamSupportedProtocols: [
                 // this may be supported on gen 2 hub?
-                // "webrtc",
                 "progressive_mp4",
-                "hls",
-                "dash",
-                "smooth_stream",
+                // "hls",
+                // "dash",
+                // "smooth_stream",
+                "webrtc",
             ],
             cameraStreamNeedAuthToken: true,
             cameraStreamNeedDrmEncryption: false

diff --git a/plugins/google-home/src/types/doorbell.ts b/plugins/google-home/src/types/doorbell.ts
@@ -14,7 +14,12 @@ addSupportedType({
         ret.traits.push('action.devices.traits.ObjectDetection');
         ret.attributes = {
             cameraStreamSupportedProtocols: [
-                "progressive_mp4", "hls", "dash", "smooth_stream"
+                // this may be supported on gen 2 hub?
+                "progressive_mp4",
+                // "hls",
+                // "dash",
+                // "smooth_stream",
+                "webrtc",
             ],
             cameraStreamNeedAuthToken: true,
             cameraStreamNeedDrmEncryption: false