From c22fbd17b95c6cade848565dab16a1a7793c8b5c Mon Sep 17 00:00:00 2001 From: Christopher Lentocha Date: Sat, 28 Dec 2024 12:58:28 -0500 Subject: [PATCH] Initial Support for Spacebar WebRTC Signed-off-by: Christopher Lentocha --- package-lock.json | 172 ++++++++++++++++++++++++ package.json | 5 +- src/bundle/Server.ts | 10 +- src/gateway/Server.ts | 1 + src/gateway/opcodes/VoiceStateUpdate.ts | 8 +- src/gateway/util/Constants.ts | 4 +- src/gateway/util/WebSocket.ts | 4 +- src/util/entities/VoiceState.ts | 2 +- src/webrtc/Server.ts | 20 +-- src/webrtc/events/Message.ts | 5 +- src/webrtc/opcodes/BackendVersion.ts | 4 +- src/webrtc/opcodes/Identify.ts | 22 ++- src/webrtc/opcodes/SelectProtocol.ts | 54 +++++--- src/webrtc/opcodes/Speaking.ts | 9 +- src/webrtc/opcodes/Video.ts | 152 ++++++++++++--------- src/webrtc/opcodes/index.ts | 4 +- src/webrtc/util/MediaServer.ts | 4 +- tsconfig.json | 4 +- 18 files changed, 358 insertions(+), 126 deletions(-) diff --git a/package-lock.json b/package-lock.json index 563a9a4c4..b60542057 100644 --- a/package-lock.json +++ b/package-lock.json @@ -87,11 +87,13 @@ "optionalDependencies": { "@yukikaze-bot/erlpack": "^1.0.1", "jimp": "^1.6.0", + "medooze-media-server": "0.129.9", "mysql": "^2.18.1", "nodemailer-mailgun-transport": "^2.1.5", "nodemailer-mailjet-transport": "github:n0script22/nodemailer-mailjet-transport", "nodemailer-sendgrid-transport": "github:Maria-Golomb/nodemailer-sendgrid-transport", "pg": "^8.13.1", + "semantic-sdp": "3.26.0", "sqlite3": "^5.1.7" } }, @@ -5110,6 +5112,21 @@ "node": ">=12" } }, + "node_modules/clone-deep": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz", + "integrity": "sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "is-plain-object": "^2.0.4", + "kind-of": "^6.0.2", + "shallow-clone": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -7280,6 +7297,19 @@ "node": ">=0.12.0" } }, + "node_modules/is-plain-object": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", + "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "license": "MIT", + "optional": true, + "dependencies": { + "isobject": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/is-stream": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", @@ -7305,6 +7335,16 @@ "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", "license": "ISC" }, + "node_modules/isobject": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", + "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/jackspeak": { "version": "3.4.3", "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", @@ -7521,6 +7561,16 @@ "json-buffer": "3.0.1" } }, + "node_modules/kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/lambert-server": { "version": "1.2.12", "resolved": "https://registry.npmjs.org/lambert-server/-/lambert-server-1.2.12.tgz", @@ -7549,6 +7599,13 @@ "node": ">= 0.8.0" } }, + "node_modules/lfsr": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/lfsr/-/lfsr-0.0.3.tgz", + "integrity": "sha512-ULgtgP6beEo825H4BExHQgNeD3YuKK/roMnWKdDbWA/g1PzFPkAb2tF0yPdMCF4T4OJ6LhzhG3TRQu56usjFfA==", + "license": "MIT", + "optional": true + }, "node_modules/libbase64": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/libbase64/-/libbase64-1.3.0.tgz", @@ -7784,6 +7841,35 @@ "node": ">= 0.6" } }, + "node_modules/medooze-media-server": { + "version": "0.129.9", + "resolved": "https://registry.npmjs.org/medooze-media-server/-/medooze-media-server-0.129.9.tgz", + "integrity": "sha512-SDTkljvKthyiWaF0ThvgNo7euXIqHbvHDRLW+O6+J14KzAKUv1P5gbv8r6bFuC1OQljWpR73R3Ro1X2moEXWyg==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "dependencies": { + "clone-deep": "^4.0.1", + "lfsr": "0.0.3", + "nan": "^2.11.1", + "semantic-sdp": "^3", + "uuid": "^3.3.2" + }, + "optionalDependencies": { + "netlink": "0.2.4" + } + }, + "node_modules/medooze-media-server/node_modules/uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", + "deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.", + "license": "MIT", + "optional": true, + "bin": { + "uuid": "bin/uuid" + } + }, "node_modules/merge-descriptors": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", @@ -8198,6 +8284,13 @@ "thenify-all": "^1.0.0" } }, + "node_modules/nan": { + "version": "2.22.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.22.0.tgz", + "integrity": "sha512-nbajikzWTMwsW+eSsNm3QwlOs7het9gGJU5dDZzRTQGk03vyBOauxgI4VakDzE0PtsGTmXPsXTbbjVhRwR5mpw==", + "license": "MIT", + "optional": true + }, "node_modules/napi-build-utils": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", @@ -8247,6 +8340,29 @@ "node": ">= 0.6" } }, + "node_modules/netlink": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/netlink/-/netlink-0.2.4.tgz", + "integrity": "sha512-eiuW0xuXbrtN+ao8Pjqd8te77bHf9I5FuIQ1X2nWT0JXvy7LySTskvlTOW45YDxL3MIw1pP297ETlPFaGAmIsg==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@types/node": "^12.0.0", + "node-addon-api": "*", + "node-gyp-build": "^4.2.3" + }, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/netlink/node_modules/@types/node": { + "version": "12.20.55", + "resolved": "https://registry.npmjs.org/@types/node/-/node-12.20.55.tgz", + "integrity": "sha512-J8xLz7q2OFulZ2cyGTLE1TbbZcjpno7FaN6zdJNrgAdrJ+DZzh/uFR6YrTb4C+nXakvud8Q4+rbhoIWlYQbUFQ==", + "license": "MIT", + "optional": true + }, "node_modules/netmask": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz", @@ -8369,6 +8485,18 @@ "node": ">= 10.12.0" } }, + "node_modules/node-gyp-build": { + "version": "4.8.4", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.4.tgz", + "integrity": "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==", + "license": "MIT", + "optional": true, + "bin": { + "node-gyp-build": "bin.js", + "node-gyp-build-optional": "optional.js", + "node-gyp-build-test": "build-test.js" + } + }, "node_modules/node-gyp-build-optional-packages": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.1.1.tgz", @@ -9519,6 +9647,16 @@ ], "license": "MIT" }, + "node_modules/randombytes": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", + "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "safe-buffer": "^5.1.0" + } + }, "node_modules/range-parser": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", @@ -9796,6 +9934,27 @@ "integrity": "sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg==", "license": "ISC" }, + "node_modules/sdp-transform": { + "version": "2.15.0", + "resolved": "https://registry.npmjs.org/sdp-transform/-/sdp-transform-2.15.0.tgz", + "integrity": "sha512-KrOH82c/W+GYQ0LHqtr3caRpM3ITglq3ljGUIb8LTki7ByacJZ9z+piSGiwZDsRyhQbYBOBJgr2k6X4BZXi3Kw==", + "license": "MIT", + "optional": true, + "bin": { + "sdp-verify": "checker.js" + } + }, + "node_modules/semantic-sdp": { + "version": "3.26.0", + "resolved": "https://registry.npmjs.org/semantic-sdp/-/semantic-sdp-3.26.0.tgz", + "integrity": "sha512-DdCNZbJBjsqPQA810kQ6HO6mTgco+caj10H8vtzgXcsx8dh0a2J1DVuCHPsAq0n8Ivg2lh2L4VqJ0TOTWQIJvw==", + "license": "MIT", + "optional": true, + "dependencies": { + "randombytes": "^2.0.3", + "sdp-transform": "^2" + } + }, "node_modules/semver": { "version": "7.6.3", "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", @@ -9913,6 +10072,19 @@ "sha.js": "bin.js" } }, + "node_modules/shallow-clone": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-3.0.1.tgz", + "integrity": "sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==", + "license": "MIT", + "optional": true, + "dependencies": { + "kind-of": "^6.0.2" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", diff --git a/package.json b/package.json index c2f3536dd..502e07f29 100644 --- a/package.json +++ b/package.json @@ -116,16 +116,19 @@ "@spacebar/api": "dist/api", "@spacebar/cdn": "dist/cdn", "@spacebar/gateway": "dist/gateway", - "@spacebar/util": "dist/util" + "@spacebar/util": "dist/util", + "@spacebar/webrtc": "dist/webrtc" }, "optionalDependencies": { "@yukikaze-bot/erlpack": "^1.0.1", "jimp": "^1.6.0", "mysql": "^2.18.1", + "medooze-media-server": "0.129.9", "nodemailer-mailgun-transport": "^2.1.5", "nodemailer-mailjet-transport": "github:n0script22/nodemailer-mailjet-transport", "nodemailer-sendgrid-transport": "github:Maria-Golomb/nodemailer-sendgrid-transport", "pg": "^8.13.1", + "semantic-sdp": "3.26.0", "sqlite3": "^5.1.7" } } diff --git a/src/bundle/Server.ts b/src/bundle/Server.ts index d281120d1..14a16d2ad 100644 --- a/src/bundle/Server.ts +++ b/src/bundle/Server.ts @@ -23,6 +23,7 @@ import http from "http"; import * as Api from "@spacebar/api"; import * as Gateway from "@spacebar/gateway"; import { CDNServer } from "@spacebar/cdn"; +import * as Webrtc from "@spacebar/webrtc"; import express from "express"; import { green, bold } from "picocolors"; import { Config, initDatabase, Sentry } from "@spacebar/util"; @@ -36,12 +37,14 @@ server.on("request", app); const api = new Api.SpacebarServer({ server, port, production, app }); const cdn = new CDNServer({ server, port, production, app }); const gateway = new Gateway.Server({ server, port, production }); +const webrtc = new Webrtc.Server({ server, port, production }); process.on("SIGTERM", async () => { console.log("Shutting down due to SIGTERM"); await gateway.stop(); await cdn.stop(); await api.stop(); + await webrtc.stop(); server.close(); Sentry.close(); }); @@ -54,7 +57,12 @@ async function main() { await new Promise((resolve) => server.listen({ port }, () => resolve(undefined)), ); - await Promise.all([api.start(), cdn.start(), gateway.start()]); + await Promise.all([ + api.start(), + cdn.start(), + gateway.start(), + webrtc.start(), + ]); Sentry.errorHandler(app); diff --git a/src/gateway/Server.ts b/src/gateway/Server.ts index 9fba2d4c5..da3313dd8 100644 --- a/src/gateway/Server.ts +++ b/src/gateway/Server.ts @@ -56,6 +56,7 @@ export class Server { } this.server.on("upgrade", (request, socket, head) => { + if (request.url?.includes("voice")) return; this.ws.handleUpgrade(request, socket, head, (socket) => { this.ws.emit("connection", socket, request); }); diff --git a/src/gateway/opcodes/VoiceStateUpdate.ts b/src/gateway/opcodes/VoiceStateUpdate.ts index b45c82031..661ef981f 100644 --- a/src/gateway/opcodes/VoiceStateUpdate.ts +++ b/src/gateway/opcodes/VoiceStateUpdate.ts @@ -39,6 +39,8 @@ import { export async function onVoiceStateUpdate(this: WebSocket, data: Payload) { check.call(this, VoiceStateUpdateSchema, data.d); const body = data.d as VoiceStateUpdateSchema; + const isNew = body.channel_id === null && body.guild_id === null; + let isChanged = false; let voiceState: VoiceState; try { @@ -54,6 +56,8 @@ export async function onVoiceStateUpdate(this: WebSocket, data: Payload) { return; } + if (voiceState.channel_id !== body.channel_id) isChanged = true; + //If a user change voice channel between guild we should send a left event first if ( voiceState.guild_id !== body.guild_id && @@ -111,7 +115,7 @@ export async function onVoiceStateUpdate(this: WebSocket, data: Payload) { ]); //If it's null it means that we are leaving the channel and this event is not needed - if (voiceState.channel_id !== null) { + if ((isNew || isChanged) && voiceState.channel_id !== null) { const guild = await Guild.findOne({ where: { id: voiceState.guild_id }, }); @@ -134,7 +138,7 @@ export async function onVoiceStateUpdate(this: WebSocket, data: Payload) { guild_id: voiceState.guild_id, endpoint: guildRegion.endpoint, }, - guild_id: voiceState.guild_id, + user_id: this.user_id, } as VoiceServerUpdateEvent); } } diff --git a/src/gateway/util/Constants.ts b/src/gateway/util/Constants.ts index 5c0f134a6..0fc9545bf 100644 --- a/src/gateway/util/Constants.ts +++ b/src/gateway/util/Constants.ts @@ -16,7 +16,7 @@ along with this program. If not, see . */ -// import { VoiceOPCodes } from "@spacebar/webrtc"; +import { VoiceOPCodes } from "@spacebar/webrtc"; export enum OPCODES { Dispatch = 0, @@ -63,7 +63,7 @@ export enum CLOSECODES { } export interface Payload { - op: OPCODES /* | VoiceOPCodes */; + op: OPCODES | VoiceOPCodes; // eslint-disable-next-line @typescript-eslint/no-explicit-any d?: any; s?: number; diff --git a/src/gateway/util/WebSocket.ts b/src/gateway/util/WebSocket.ts index 8cfc5e083..2af89e339 100644 --- a/src/gateway/util/WebSocket.ts +++ b/src/gateway/util/WebSocket.ts @@ -20,7 +20,7 @@ import { Intents, ListenEventOpts, Permissions } from "@spacebar/util"; import WS from "ws"; import { Deflate, Inflate } from "fast-zlib"; import { Capabilities } from "./Capabilities"; -// import { Client } from "@spacebar/webrtc"; +import { Client } from "@spacebar/webrtc"; export interface WebSocket extends WS { version: number; @@ -42,6 +42,6 @@ export interface WebSocket extends WS { member_events: Record unknown>; listen_options: ListenEventOpts; capabilities?: Capabilities; - // client?: Client; + webrtcClient?: Client; large_threshold: number; } diff --git a/src/util/entities/VoiceState.ts b/src/util/entities/VoiceState.ts index 83a0af63b..601e2915d 100644 --- a/src/util/entities/VoiceState.ts +++ b/src/util/entities/VoiceState.ts @@ -88,7 +88,7 @@ export class VoiceState extends BaseClass { @Column({ nullable: true }) self_stream?: boolean; - @Column() + @Column({ nullable: true }) self_video: boolean; @Column() diff --git a/src/webrtc/Server.ts b/src/webrtc/Server.ts index 0ba2e41b2..fa98eeac2 100644 --- a/src/webrtc/Server.ts +++ b/src/webrtc/Server.ts @@ -19,6 +19,7 @@ import { closeDatabase, Config, initDatabase, initEvent } from "@spacebar/util"; import dotenv from "dotenv"; import http from "http"; +import MediaServer from "medooze-media-server"; import ws from "ws"; import { Connection } from "./events/Connection"; dotenv.config(); @@ -48,18 +49,18 @@ export class Server { }); } - // this.server.on("upgrade", (request, socket, head) => { - // if (!request.url?.includes("voice")) return; - // this.ws.handleUpgrade(request, socket, head, (socket) => { - // // @ts-ignore - // socket.server = this; - // this.ws.emit("connection", socket, request); - // }); - // }); + this.server.on("upgrade", (request, socket, head) => { + if (!request.url?.includes("voice")) return; + this.ws.handleUpgrade(request, socket, head, (socket) => { + // @ts-ignore + socket.server = this; + this.ws.emit("connection", socket, request); + }); + }); this.ws = new ws.Server({ maxPayload: 1024 * 1024 * 100, - server: this.server, + noServer: true, }); this.ws.on("connection", Connection); this.ws.on("error", console.error); @@ -77,6 +78,7 @@ export class Server { async stop() { closeDatabase(); + MediaServer.terminate(); this.server.close(); } } diff --git a/src/webrtc/events/Message.ts b/src/webrtc/events/Message.ts index 22189e95b..6c8058052 100644 --- a/src/webrtc/events/Message.ts +++ b/src/webrtc/events/Message.ts @@ -30,14 +30,12 @@ const PayloadSchema = { export async function onMessage(this: WebSocket, buffer: Buffer) { try { - var data: Payload = JSON.parse(buffer.toString()); + const data: Payload = JSON.parse(buffer.toString()); if (data.op !== VoiceOPCodes.IDENTIFY && !this.user_id) return this.close(CLOSECODES.Not_authenticated); - // @ts-ignore const OPCodeHandler = OPCodeHandlers[data.op]; if (!OPCodeHandler) { - // @ts-ignore console.error("[WebRTC] Unkown opcode " + VoiceOPCodes[data.op]); // TODO: if all opcodes are implemented comment this out: // this.close(CloseCodes.Unknown_opcode); @@ -49,7 +47,6 @@ export async function onMessage(this: WebSocket, buffer: Buffer) { data.op as VoiceOPCodes, ) ) { - // @ts-ignore console.log("[WebRTC] Opcode " + VoiceOPCodes[data.op]); } diff --git a/src/webrtc/opcodes/BackendVersion.ts b/src/webrtc/opcodes/BackendVersion.ts index 60de3e58d..f8b6797bb 100644 --- a/src/webrtc/opcodes/BackendVersion.ts +++ b/src/webrtc/opcodes/BackendVersion.ts @@ -16,10 +16,10 @@ along with this program. If not, see . */ -import { Payload, Send, WebSocket } from "@spacebar/gateway"; +import { Send, WebSocket } from "@spacebar/gateway"; import { VoiceOPCodes } from "../util"; -export async function onBackendVersion(this: WebSocket, data: Payload) { +export async function onBackendVersion(this: WebSocket) { await Send(this, { op: VoiceOPCodes.VOICE_BACKEND_VERSION, d: { voice: "0.8.43", rtc_worker: "0.3.26" }, diff --git a/src/webrtc/opcodes/Identify.ts b/src/webrtc/opcodes/Identify.ts index 3f65127e1..f65495bcd 100644 --- a/src/webrtc/opcodes/Identify.ts +++ b/src/webrtc/opcodes/Identify.ts @@ -24,7 +24,7 @@ import { } from "@spacebar/util"; import { endpoint, getClients, VoiceOPCodes, PublicIP } from "@spacebar/webrtc"; import SemanticSDP from "semantic-sdp"; -const defaultSDP = require("./sdp.json"); +import defaultSDP from "./sdp.json"; export async function onIdentify(this: WebSocket, data: Payload) { clearTimeout(this.readyTimeout); @@ -47,7 +47,7 @@ export async function onIdentify(this: WebSocket, data: Payload) { }), ); - this.client = { + this.webrtcClient = { websocket: this, out: { tracks: new Map(), @@ -61,24 +61,32 @@ export async function onIdentify(this: WebSocket, data: Payload) { channel_id: voiceState.channel_id, }; - const clients = getClients(voiceState.channel_id)!; - clients.add(this.client); + const clients = getClients(voiceState.channel_id); + clients.add(this.webrtcClient); this.on("close", () => { - clients.delete(this.client!); + if (this.webrtcClient) clients.delete(this.webrtcClient); }); await Send(this, { op: VoiceOPCodes.READY, d: { streams: [ - // { type: "video", ssrc: this.ssrc + 1, rtx_ssrc: this.ssrc + 2, rid: "100", quality: 100, active: false } + // { + // type: "video", + // ssrc: this.webrtcClient.in.video_ssrc, + // rtx_ssrc: this.webrtcClient.in.rtx_ssrc, + // rid: "100", + // quality: 100, + // active: false, + // }, ], - ssrc: -1, + ssrc: 1, port: endpoint.getLocalPort(), modes: [ "aead_aes256_gcm_rtpsize", "aead_aes256_gcm", + "aead_xchacha20_poly1305_rtpsize", "xsalsa20_poly1305_lite_rtpsize", "xsalsa20_poly1305_lite", "xsalsa20_poly1305_suffix", diff --git a/src/webrtc/opcodes/SelectProtocol.ts b/src/webrtc/opcodes/SelectProtocol.ts index 0a06e7221..ad01d54d8 100644 --- a/src/webrtc/opcodes/SelectProtocol.ts +++ b/src/webrtc/opcodes/SelectProtocol.ts @@ -19,24 +19,28 @@ import { Payload, Send, WebSocket } from "@spacebar/gateway"; import { SelectProtocolSchema, validateSchema } from "@spacebar/util"; import { PublicIP, VoiceOPCodes, endpoint } from "@spacebar/webrtc"; -import SemanticSDP, { MediaInfo, SDPInfo } from "semantic-sdp"; +import MediaServer from "medooze-media-server"; +import SemanticSDP, { MediaInfo } from "semantic-sdp"; +import DefaultSDP from "./sdp.json"; export async function onSelectProtocol(this: WebSocket, payload: Payload) { - if (!this.client) return; + if (!this.webrtcClient) return; const data = validateSchema( "SelectProtocolSchema", payload.d, ) as SelectProtocolSchema; - const offer = SemanticSDP.SDPInfo.parse("m=audio\n" + data.sdp!); - this.client.sdp!.setICE(offer.getICE()); - this.client.sdp!.setDTLS(offer.getDTLS()); + const offer = SemanticSDP.SDPInfo.parse("m=audio\n" + data.sdp); + //@ts-ignore + offer.getMedias()[0].type = "audio"; + this.webrtcClient.sdp.setICE(offer.getICE()); + this.webrtcClient.sdp.setDTLS(offer.getDTLS()); - const transport = endpoint.createTransport(this.client.sdp!); - this.client.transport = transport; - transport.setRemoteProperties(this.client.sdp!); - transport.setLocalProperties(this.client.sdp!); + const transport = endpoint.createTransport(this.webrtcClient.sdp); + this.webrtcClient.transport = transport; + transport.setRemoteProperties(this.webrtcClient.sdp); + transport.setLocalProperties(this.webrtcClient.sdp); const dtls = transport.getLocalDTLSInfo(); const ice = transport.getLocalICEInfo(); @@ -45,21 +49,33 @@ export async function onSelectProtocol(this: WebSocket, payload: Payload) { const candidates = transport.getLocalCandidates(); const candidate = candidates[0]; + // discord answer + /* + m=audio 50026 ICE/SDP\n + a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87\n + c=IN IP4 66.22.206.174\n + a=rtcp:50026\n + a=ice-ufrag:XxnE\n + a=ice-pwd:GLQatPT3Q9dCZVVgVf3J1F\n + a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87\n + a=candidate:1 1 UDP 4261412862 66.22.206.174 50026 typ host\n + */ + const answer = - `m=audio ${port} ICE/SDP` + - `a=fingerprint:${fingerprint}` + - `c=IN IP4 ${PublicIP}` + - `a=rtcp:${port}` + - `a=ice-ufrag:${ice.getUfrag()}` + - `a=ice-pwd:${ice.getPwd()}` + - `a=fingerprint:${fingerprint}` + - `a=candidate:1 1 ${candidate.getTransport()} ${candidate.getFoundation()} ${candidate.getAddress()} ${candidate.getPort()} typ host`; + `m=audio ${port} ICE/SDP\n` + + `a=fingerprint:${fingerprint}\n` + + `c=IN IP4 ${PublicIP}\n` + + `a=rtcp:${port}\n` + + `a=ice-ufrag:${ice.getUfrag()}\n` + + `a=ice-pwd:${ice.getPwd()}\n` + + `a=fingerprint:${fingerprint}\n` + + `a=candidate:1 1 ${candidate.getTransport()} ${candidate.getFoundation()} ${candidate.getAddress()} ${candidate.getPort()} typ host\n`; await Send(this, { op: VoiceOPCodes.SESSION_DESCRIPTION, d: { - video_codec: "H264", - sdp: answer, + video_codec: "VP8", + sdp: answer.toString(), media_session_id: this.session_id, audio_codec: "opus", }, diff --git a/src/webrtc/opcodes/Speaking.ts b/src/webrtc/opcodes/Speaking.ts index 97055e0a2..bf9216070 100644 --- a/src/webrtc/opcodes/Speaking.ts +++ b/src/webrtc/opcodes/Speaking.ts @@ -22,11 +22,12 @@ import { getClients, VoiceOPCodes } from "../util"; // {"speaking":1,"delay":5,"ssrc":2805246727} export async function onSpeaking(this: WebSocket, data: Payload) { - if (!this.client) return; + if (!this.webrtcClient) return; - getClients(this.client.channel_id).forEach((client) => { - if (client === this.client) return; - const ssrc = this.client!.out.tracks.get(client.websocket.user_id); + getClients(this.webrtcClient.channel_id).forEach((client) => { + if (client === this.webrtcClient) return; + if (!this.webrtcClient) return; + const ssrc = this.webrtcClient.out.tracks.get(client.websocket.user_id); Send(client.websocket, { op: VoiceOPCodes.SPEAKING, diff --git a/src/webrtc/opcodes/Video.ts b/src/webrtc/opcodes/Video.ts index 3228d4eed..2d646d4b9 100644 --- a/src/webrtc/opcodes/Video.ts +++ b/src/webrtc/opcodes/Video.ts @@ -19,67 +19,78 @@ import { Payload, Send, WebSocket } from "@spacebar/gateway"; import { validateSchema, VoiceVideoSchema } from "@spacebar/util"; import { channels, getClients, VoiceOPCodes } from "@spacebar/webrtc"; -import { IncomingStreamTrack, SSRCs } from "medooze-media-server"; +import { + IncomingStream, + IncomingStreamTrack, + SSRCs, + Transport, +} from "medooze-media-server"; import SemanticSDP from "semantic-sdp"; -export async function onVideo(this: WebSocket, payload: Payload) { - if (!this.client) return; - const { transport, channel_id } = this.client; - if (!transport) return; - const d = validateSchema("VoiceVideoSchema", payload.d) as VoiceVideoSchema; - - await Send(this, { op: VoiceOPCodes.MEDIA_SINK_WANTS, d: { any: 100 } }); +function createStream( + this: WebSocket, + transport: Transport, + channel_id: string, +) { + if (!this.webrtcClient) return; + if (!this.webrtcClient.transport) return; const id = "stream" + this.user_id; - var stream = this.client.in.stream!; - if (!stream) { - stream = this.client.transport!.createIncomingStream( - // @ts-ignore - SemanticSDP.StreamInfo.expand({ - id, - // @ts-ignore - tracks: [], - }), - ); - this.client.in.stream = stream; - - const interval = setInterval(() => { - for (const track of stream.getTracks()) { - for (const layer of Object.values(track.getStats())) { - console.log(track.getId(), layer.total); - } + const stream = this.webrtcClient.transport.createIncomingStream( + SemanticSDP.StreamInfo.expand({ + id, + tracks: [], + }), + ); + this.webrtcClient.in.stream = stream; + + const interval = setInterval(() => { + for (const track of stream.getTracks()) { + for (const layer of Object.values(track.getStats())) { + console.log(track.getId(), layer.total); } - }, 5000); + } + }, 5000); - stream.on("stopped", () => { - console.log("stream stopped"); - clearInterval(interval); - }); - this.on("close", () => { - transport!.stop(); - }); - const out = transport.createOutgoingStream( - // @ts-ignore - SemanticSDP.StreamInfo.expand({ - id: "out" + this.user_id, - // @ts-ignore - tracks: [], - }), - ); - this.client.out.stream = out; - - const clients = channels.get(channel_id)!; + stream.on("stopped", () => { + console.log("stream stopped"); + clearInterval(interval); + }); + this.on("close", () => { + transport.stop(); + }); + const out = transport.createOutgoingStream( + SemanticSDP.StreamInfo.expand({ + id: "out" + this.user_id, + tracks: [], + }), + ); + this.webrtcClient.out.stream = out; - clients.forEach((client) => { - if (client.websocket.user_id === this.user_id) return; - if (!client.in.stream) return; + const clients = channels.get(channel_id); + if (!clients) return; - client.in.stream?.getTracks().forEach((track) => { - attachTrack.call(this, track, client.websocket.user_id); - }); + clients.forEach((client) => { + if (client.websocket.user_id === this.user_id) return; + if (!client.in.stream) return; + + client.in.stream?.getTracks().forEach((track) => { + attachTrack.call(this, track, client.websocket.user_id); }); - } + }); +} + +export async function onVideo(this: WebSocket, payload: Payload) { + if (!this.webrtcClient) return; + const { transport, channel_id } = this.webrtcClient; + if (!transport) return; + const d = validateSchema("VoiceVideoSchema", payload.d) as VoiceVideoSchema; + + await Send(this, { op: VoiceOPCodes.MEDIA_SINK_WANTS, d: { any: 100 } }); + + if (!this.webrtcClient.in.stream) + createStream.call(this, transport, channel_id); if (d.audio_ssrc) { handleSSRC.call(this, "audio", { @@ -100,23 +111,32 @@ function attachTrack( track: IncomingStreamTrack, user_id: string, ) { - if (!this.client) return; - const outTrack = this.client.transport!.createOutgoingStreamTrack( + if ( + !this.webrtcClient || + !this.webrtcClient.transport || + !this.webrtcClient.out.stream + ) + return; + + const outTrack = this.webrtcClient.transport.createOutgoingStreamTrack( track.getMedia(), ); outTrack.attachTo(track); - this.client.out.stream!.addTrack(outTrack); - var ssrcs = this.client.out.tracks.get(user_id)!; + + this.webrtcClient.out.stream.addTrack(outTrack); + let ssrcs = this.webrtcClient.out.tracks.get(user_id); if (!ssrcs) - ssrcs = this.client.out.tracks + ssrcs = this.webrtcClient.out.tracks .set(user_id, { audio_ssrc: 0, rtx_ssrc: 0, video_ssrc: 0 }) - .get(user_id)!; + .get(user_id); + + if (!ssrcs) return; // hmm if (track.getMedia() === "audio") { - ssrcs.audio_ssrc = outTrack.getSSRCs().media!; + ssrcs.audio_ssrc = outTrack.getSSRCs().media || 0; } else if (track.getMedia() === "video") { - ssrcs.video_ssrc = outTrack.getSSRCs().media!; - ssrcs.rtx_ssrc = outTrack.getSSRCs().rtx!; + ssrcs.video_ssrc = outTrack.getSSRCs().media || 0; + ssrcs.rtx_ssrc = outTrack.getSSRCs().rtx || 0; } Send(this, { @@ -129,23 +149,23 @@ function attachTrack( } function handleSSRC(this: WebSocket, type: "audio" | "video", ssrcs: SSRCs) { - if (!this.client) return; - const stream = this.client.in.stream!; - const transport = this.client.transport!; + if (!this.webrtcClient) return; + const stream = this.webrtcClient.in.stream as IncomingStream; + const transport = this.webrtcClient.transport as Transport; const id = type + ssrcs.media; - var track = stream.getTrack(id); + let track = stream.getTrack(id); if (!track) { console.log("createIncomingStreamTrack", id); track = transport.createIncomingStreamTrack(type, { id, ssrcs }); stream.addTrack(track); - const clients = getClients(this.client.channel_id)!; + const clients = getClients(this.webrtcClient.channel_id); clients.forEach((client) => { if (client.websocket.user_id === this.user_id) return; if (!client.out.stream) return; - attachTrack.call(this, track, client.websocket.user_id); + attachTrack.call(client.websocket, track, this.user_id); }); } } diff --git a/src/webrtc/opcodes/index.ts b/src/webrtc/opcodes/index.ts index 346810556..bbfc59c5a 100644 --- a/src/webrtc/opcodes/index.ts +++ b/src/webrtc/opcodes/index.ts @@ -25,7 +25,7 @@ import { onSelectProtocol } from "./SelectProtocol"; import { onSpeaking } from "./Speaking"; import { onVideo } from "./Video"; -export type OPCodeHandler = (this: WebSocket, data: Payload) => any; +export type OPCodeHandler = (this: WebSocket, data: Payload) => unknown; export default { [VoiceOPCodes.HEARTBEAT]: onHeartbeat, @@ -34,4 +34,4 @@ export default { [VoiceOPCodes.VIDEO]: onVideo, [VoiceOPCodes.SPEAKING]: onSpeaking, [VoiceOPCodes.SELECT_PROTOCOL]: onSelectProtocol, -}; +} as { [key: number]: OPCodeHandler }; diff --git a/src/webrtc/util/MediaServer.ts b/src/webrtc/util/MediaServer.ts index 0c12876c6..5f4d44b98 100644 --- a/src/webrtc/util/MediaServer.ts +++ b/src/webrtc/util/MediaServer.ts @@ -28,7 +28,7 @@ MediaServer.enableLog(true); export const PublicIP = process.env.PUBLIC_IP || "127.0.0.1"; try { - const range = process.env.WEBRTC_PORT_RANGE || "4000"; + const range = process.env.WEBRTC_PORT_RANGE || "3690-3960"; var ports = range.split("-"); const min = Number(ports[0]); const max = Number(ports[1]); @@ -73,5 +73,5 @@ export interface Client { export function getClients(channel_id: string) { if (!channels.has(channel_id)) channels.set(channel_id, new Set()); - return channels.get(channel_id)!; + return channels.get(channel_id) as Set; } diff --git a/tsconfig.json b/tsconfig.json index 63b5e96cb..23259bb83 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,5 +1,4 @@ { - "exclude": ["./src/webrtc"], "include": ["./src"], "compilerOptions": { /* Visit https://aka.ms/tsconfig to read more about this file */ @@ -37,7 +36,8 @@ "@spacebar/api*": ["./api"], "@spacebar/gateway*": ["./gateway"], "@spacebar/cdn*": ["./cdn"], - "@spacebar/util*": ["./util"] + "@spacebar/util*": ["./util"], + "@spacebar/webrtc*":["./webrtc"] } /* Specify a set of entries that re-map imports to additional lookup locations. */, // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ // "typeRoots": [], /* Specify multiple folders that act like './node_modules/@types'. */