feat: add support for keyboard shortcuts, xkeys, streamdeck and space…

…mouse

packages/apps/client/package.json

@@ -34,11 +34,14 @@
         "vite": "^4.5.0"
     },
     "dependencies": {
+        "@elgato-stream-deck/webhid": "^6.0.0",
         "@popperjs/core": "^2.11.8",
         "@react-hook/resize-observer": "^1.2.6",
+        "@sofie-automation/sorensen": "^1.4.2",
         "@sofie-prompter-editor/shared-lib": "0.0.0",
         "@sofie-prompter-editor/shared-model": "0.0.0",
         "bootstrap": "^5.3.2",
+        "buffer": "^6.0.3",
         "eventemitter3": "^5.0.1",
         "mobx": "^6.10.2",
         "mobx-react-lite": "^4.0.5",
@@ -56,7 +59,9 @@
         "react-helmet-async": "^1.3.0",
         "react-router-dom": "^6.18.0",
         "socket.io-client": "^4.7.2",
-        "uuid": "^9.0.1"
+        "spacemouse-webhid": "^0.0.2",
+        "uuid": "^9.0.1",
+        "xkeys-webhid": "^3.1.0"
     },
     "lint-staged": {
         "*.{js,css,json,md,scss}": [

packages/apps/client/src/components/ScriptEditor/plugins/updateModel.ts

@@ -1,5 +1,5 @@
 import { Plugin } from 'prosemirror-state'
-import { UILineId } from '../../model/UILine'
+import { UILineId } from '../../../model/UILine'
 import { Node } from 'prosemirror-model'
 import { AddMarkStep, RemoveMarkStep, StepMap } from 'prosemirror-transform'
 import { schema } from '../scriptSchema'

packages/apps/client/src/components/SystemStatusAlertBars/SystemStatusAlertBars.tsx

@@ -2,14 +2,42 @@ import { observer } from 'mobx-react-lite'
 import React from 'react'
 import { RootAppStore } from 'src/stores/RootAppStore'
 import { AlertBar } from 'src/components/AlertBar/AlertBar'
+import { Button } from 'react-bootstrap'
 
 export const SystemStatusAlertBars = observer(function SystemStatusAlertBars(): React.JSX.Element {
 	const isAPIConnected = RootAppStore.connected
 	const isSofieConnected = RootAppStore.sofieConnected
+
+	const hidAccessRequest = Array.from(RootAppStore.triggerStore.hidDeviceAccessRequests.values())[0]
+
+	let requestAccess: {
+		name: string
+		allow: () => void
+		deny: () => void
+	} | null = null
+	if (hidAccessRequest) {
+		requestAccess = {
+			name: hidAccessRequest.deviceName,
+			allow: () => hidAccessRequest.callback(true),
+			deny: () => hidAccessRequest.callback(false),
+		}
+	}
+
 	return (
 		<>
 			{!isAPIConnected ? <AlertBar variant="danger">Prompter is having network troubles</AlertBar> : null}
 			{!isSofieConnected ? <AlertBar variant="danger">Prompter is having trouble connecting to Sofie</AlertBar> : null}
+			{requestAccess ? (
+				<AlertBar variant="info">
+					Please allow access to {requestAccess.name} to setup shortcuts:
+					<Button variant="primary" onClick={requestAccess.allow}>
+						Allow
+					</Button>
+					<Button variant="secondary" onClick={requestAccess.deny}>
+						Deny
+					</Button>
+				</AlertBar>
+			) : null}
 		</>
 	)
 })

packages/apps/client/src/hooks/useControllerMessages.ts

@@ -9,7 +9,7 @@ export function useControllerMessages(ref: React.RefObject<HTMLElement>, heightP
 
 	useEffect(() => {
 		const onMessage = (message: { speed: number }) => {
-			console.log('received message', message)
+			// console.log('received message', message)
 
 			speed.current = message.speed
 		}
@@ -21,7 +21,7 @@ export function useControllerMessages(ref: React.RefObject<HTMLElement>, heightP
 			const frameTime = lastFrameTime.current === null ? 16 : now - lastFrameTime.current
 			const scrollBy = ((speed.current * fontSizePx) / 300) * frameTime
 			position.current = Math.max(0, position.current + scrollBy)
-			console.log(position.current)
+			// console.log(position.current)
 
 			ref.current?.scrollTo(0, position.current)
 

packages/apps/client/src/lib/triggerActions/TriggerActionHandler.ts

@@ -0,0 +1,64 @@
+import { assertNever } from '@sofie-prompter-editor/shared-lib'
+import { RootAppStore } from '../../stores/RootAppStore.ts'
+import { AnyTriggerAction } from './triggerActions.ts'
+
+/**
+ * The TriggerActionHandler is responsible for listening to some action events and executing the actions accordingly.
+ */
+export class TriggerActionHandler {
+	private prompterSpeed = 0
+
+	constructor(private store: typeof RootAppStore) {
+		this.onAction = this.onAction.bind(this)
+
+		this.store.triggerStore.on('action', this.onAction)
+	}
+
+	private onAction(action: AnyTriggerAction) {
+		console.log('action', JSON.stringify(action))
+
+		if (action.type === 'prompterMove') {
+			this.prompterSpeed = action.payload.speed
+			this.sendPrompterSpeed()
+		} else if (action.type === 'prompterAccelerate') {
+			this.prompterSpeed += action.payload.accelerate
+			this.sendPrompterSpeed()
+		} else if (action.type === 'prompterJump') {
+			// TODO
+			// this.store.connection.controller.sendMessage({
+			// 	speed: 0,
+			// 	offset: action.payload.offset,
+			// })
+		} else if (action.type === 'movePrompterToHere') {
+			// Not handled here
+		} else {
+			assertNever(action)
+		}
+	}
+
+	private sendPrompterSpeed() {
+		// Send message with speed:
+
+		// Modify the value so that
+		const speed = this.attackCurve(this.prompterSpeed, 10, 0.7)
+
+		this.store.connection.controller.sendMessage({
+			speed: speed,
+			offset: null,
+		})
+	}
+	destroy(): void {}
+
+	/**
+	 * Scales a value which follows a curve which at low values (0-normalValue) is pretty much linear,
+	 * but at higher values (normalValue+) grows much faster
+	 * @param x The value to scale
+	 * @param power The power of the curve
+	 * @param normalValue The value up to which at which the curve is mostly linear
+	 */
+	private attackCurve(x: number, power = 1, normalValue = 1): number {
+		const attack = Math.sign(x) * Math.abs(Math.pow(x, power) * Math.pow(1 / normalValue, power))
+		const linear = x
+		return linear + attack
+	}
+}

packages/apps/client/src/lib/triggerActions/triggerActions.ts

@@ -0,0 +1,36 @@
+/* eslint-disable no-mixed-spaces-and-tabs */
+
+import { ControllerMessage } from '@sofie-prompter-editor/shared-model'
+
+export type AnyTriggerAction =
+	| TriggerAction<
+			'prompterMove',
+			{
+				/** The speed to move the prompter */
+				speed: number
+			}
+	  >
+	| TriggerAction<
+			'prompterAccelerate',
+			{
+				/** The acceleration move the prompter */
+				accelerate: number
+			}
+	  >
+	| TriggerAction<
+			'prompterJump',
+			{
+				offset: ControllerMessage['offset']
+			}
+	  >
+	| TriggerAction<
+			'movePrompterToHere',
+			{
+				// nothing
+			}
+	  >
+
+type TriggerAction<Type extends string, Payload extends Record<string, unknown>> = {
+	type: Type
+	payload: Payload
+}

packages/apps/client/src/lib/triggers/triggerConfig.ts

@@ -0,0 +1,102 @@
+import { BindOptions } from '@sofie-automation/sorensen'
+import { AnyTriggerAction } from '../triggerActions/triggerActions.ts'
+import { DeviceModelId } from '@elgato-stream-deck/webhid'
+
+export type TriggerConfig =
+	| TriggerConfigKeyboard
+	| TriggerConfigXkeys
+	| TriggerConfigStreamdeck
+	| TriggerConfigSpacemouse
+
+export enum TriggerConfigType {
+	KEYBOARD = 'keyboard',
+	XKEYS = 'xkeys',
+	STREAMDECK = 'streamdeck',
+	SPACEMOUSE = 'spacemouse',
+}
+export interface TriggerConfigBase {
+	type: TriggerConfigType
+}
+export interface TriggerConfigKeyboard extends TriggerConfigBase {
+	type: TriggerConfigType.KEYBOARD
+
+	action: AnyTriggerAction
+	/**
+	 * a "+" and space concatenated list of KeyboardEvent.key key values (see: https://developer.mozilla.org/en-US/docs/Web/API/KeyboardEvent/key/Key_Values),
+	 * in order (order not significant for modifier keys), f.g. "Control+Shift+KeyA", "Control+Shift+KeyB KeyU".
+	 * "Control" means "either ControlLeft or ControlRight", same for "Shift" and "Alt"
+	 * Spaces indicate chord sequences.
+	 */
+	keys: string
+
+	/**
+	 * If enabled, actions will happen on keyUp, as opposed to keyDown
+	 *
+	 * @type {boolean}
+	 * @memberof IBlueprintHotkeyTrigger
+	 */
+	up?: boolean
+
+	global?: BindOptions['global']
+}
+export interface TriggerConfigXkeys extends TriggerConfigBase {
+	type: TriggerConfigType.XKEYS
+
+	/** userId of the xkeys panel, or null to match any */
+	productId: number | null
+	/** userId of the xkeys, or null to match any */
+	unitId: number | null
+
+	eventType: 'down' | 'up' | 'jog' | 'joystick' | 'rotary' | 'shuttle' | 'tbar' | 'trackball'
+	/** Index of the key, joystick, etc */
+	index: number
+
+	/** If action.payload is not set, use value from the xkeys */
+	action:
+		| AnyTriggerAction
+		| {
+				type: AnyTriggerAction['type']
+				// no payload, use value from the xkeys
+		  }
+}
+export interface TriggerConfigStreamdeck extends TriggerConfigBase {
+	type: TriggerConfigType.STREAMDECK
+
+	/** userId of the Streamdeck, or null to match any */
+	modelId: DeviceModelId | null
+	/** userId of the Streamdeck, or null to match any */
+	serialNumber: string | null
+
+	eventType: 'down' | 'up' | 'rotate' | 'encoderDown' | 'encoderUp'
+	/** Index of the key, knob, etc */
+	index: number
+
+	/** If action.payload is not set, use value from the xkeys */
+	action:
+		| AnyTriggerAction
+		| {
+				type: AnyTriggerAction['type']
+				// no payload, use value from the streamdeck
+		  }
+}
+
+export interface TriggerConfigSpacemouse extends TriggerConfigBase {
+	type: TriggerConfigType.SPACEMOUSE
+
+	/** userId of the xkeys panel, or null to match any */
+	productId: number | null
+	/** userId of the xkeys, or null to match any */
+	unitId: number | null
+
+	eventType: 'down' | 'up' | 'rotate' | 'translate'
+	/** Index of the key, if needed, 0 otherwise */
+	index: number
+
+	/** If action.payload is not set, use value from the xkeys */
+	action:
+		| AnyTriggerAction
+		| {
+				type: AnyTriggerAction['type']
+				// no payload, use value from the xkeys
+		  }
+}

packages/apps/client/src/lib/triggers/triggerHandlers/TriggerHandler.ts

@@ -0,0 +1,16 @@
+import { EventEmitter } from 'eventemitter3'
+import { AnyTriggerAction } from '../../triggerActions/triggerActions.ts'
+import { TriggerConfig } from '../triggerConfig.ts'
+
+export interface TriggerHandlerEvents {
+	action: [action: AnyTriggerAction]
+
+	/** A message indicating that we want to ask the user if we should request access to the HIDDevice */
+	requestHIDDeviceAccess: [deviceName: string, callback: (access: boolean) => void]
+}
+
+export abstract class TriggerHandler extends EventEmitter<TriggerHandlerEvents> {
+	protected triggers: TriggerConfig[] = []
+	abstract initialize(triggers?: TriggerConfig[]): Promise<void>
+	abstract destroy(): Promise<void>
+}

packages/apps/client/src/lib/triggers/triggerHandlers/TriggerHandlerKeyboard.ts

@@ -0,0 +1,43 @@
+import Sorensen from '@sofie-automation/sorensen'
+import { TriggerHandler } from './TriggerHandler'
+import { TriggerConfig, TriggerConfigType } from '../triggerConfig'
+
+export class TriggerHandlerKeyboard extends TriggerHandler {
+	async initialize(triggers?: TriggerConfig[]): Promise<void> {
+		if (triggers) this.triggers = triggers
+
+		// eslint-disable-next-line @typescript-eslint/no-explicit-any
+		const theWindow = window as any
+		// hot-module-reload fix:
+		if (!theWindow.sorensenInitialized) {
+			theWindow.sorensenInitialized = true
+			await Sorensen.init()
+		} else {
+			await Sorensen.destroy()
+			await Sorensen.init()
+		}
+
+		for (const trigger of this.triggers) {
+			if (trigger.type !== TriggerConfigType.KEYBOARD) continue
+
+			Sorensen.bind(
+				trigger.keys,
+				() => {
+					this.emit('action', trigger.action)
+				},
+				{
+					up: trigger.up,
+					global: trigger.global,
+
+					exclusive: true,
+					ordered: 'modifiersFirst',
+					preventDefaultPartials: false,
+					preventDefaultDown: true,
+				}
+			)
+		}
+	}
+	async destroy(): Promise<void> {
+		Sorensen.destroy().catch((e: Error) => console.error('Sorensen.destroy', e))
+	}
+}